1 | /* |
2 | * Copyright 2018-present Facebook, Inc. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | #include <folly/synchronization/AtomicUtil.h> |
18 | |
19 | #include <folly/Benchmark.h> |
20 | #include <folly/Portability.h> |
21 | #include <folly/Utility.h> |
22 | #include <folly/portability/GTest.h> |
23 | |
24 | namespace folly { |
25 | |
26 | namespace { |
27 | auto default_fetch_set = [](auto&&... args) { |
28 | return atomic_fetch_set(args...); |
29 | }; |
30 | auto default_fetch_reset = [](auto&&... args) { |
31 | return atomic_fetch_reset(args...); |
32 | }; |
33 | |
34 | template <typename Integer, typename FetchSet = decltype(default_fetch_set)> |
35 | void atomic_fetch_set_basic(FetchSet fetch_set = default_fetch_set) { |
36 | { |
37 | auto&& atomic = std::atomic<Integer>{0}; |
38 | EXPECT_EQ(fetch_set(atomic, 0), false); |
39 | EXPECT_EQ(fetch_set(atomic, 1), false); |
40 | EXPECT_EQ(atomic.load(), 0b11); |
41 | EXPECT_EQ(fetch_set(atomic, 2), false); |
42 | EXPECT_EQ(atomic.load(), 0b111); |
43 | } |
44 | |
45 | { |
46 | auto&& atomic = std::atomic<Integer>{0b1}; |
47 | EXPECT_EQ(fetch_set(atomic, 0), true); |
48 | EXPECT_EQ(fetch_set(atomic, 0), true); |
49 | EXPECT_EQ(fetch_set(atomic, 1), false); |
50 | EXPECT_EQ(atomic.load(), 0b11); |
51 | EXPECT_EQ(fetch_set(atomic, 2), false); |
52 | EXPECT_EQ(atomic.load(), 0b111); |
53 | } |
54 | |
55 | { |
56 | for (auto i = 0; i < 100000; ++i) { |
57 | // call makeUnpredictable() to ensure that the bit integer does not get |
58 | // optimized away. This is testing the feasability of this code in |
59 | // situations where bit is not known at compile time and will likely force |
60 | // a register load |
61 | auto&& atomic = std::atomic<Integer>{0}; |
62 | auto&& bit = 0; |
63 | folly::makeUnpredictable(bit); |
64 | |
65 | EXPECT_EQ(fetch_set(atomic, bit), false); |
66 | EXPECT_EQ(fetch_set(atomic, bit + 1), false); |
67 | EXPECT_EQ(atomic.load(), 0b11); |
68 | EXPECT_EQ(fetch_set(atomic, bit + 2), false); |
69 | EXPECT_EQ(atomic.load(), 0b111); |
70 | } |
71 | } |
72 | } |
73 | |
74 | template <typename Integer, typename FetchReset = decltype(default_fetch_reset)> |
75 | void atomic_fetch_reset_basic(FetchReset fetch_reset = default_fetch_reset) { |
76 | { |
77 | auto&& atomic = std::atomic<Integer>{0}; |
78 | EXPECT_EQ(fetch_reset(atomic, 0), false); |
79 | EXPECT_EQ(fetch_reset(atomic, 1), false); |
80 | atomic.store(0b11); |
81 | EXPECT_EQ(fetch_reset(atomic, 0), true); |
82 | EXPECT_EQ(fetch_reset(atomic, 1), true); |
83 | EXPECT_EQ(atomic.load(), 0); |
84 | } |
85 | |
86 | { |
87 | auto&& atomic = std::atomic<Integer>{0}; |
88 | EXPECT_EQ(fetch_reset(atomic, 0), false); |
89 | EXPECT_EQ(fetch_reset(atomic, 1), false); |
90 | atomic.store(0b11); |
91 | EXPECT_EQ(fetch_reset(atomic, 1), true); |
92 | EXPECT_EQ(fetch_reset(atomic, 0), true); |
93 | EXPECT_EQ(atomic.load(), 0); |
94 | } |
95 | } |
96 | |
97 | template <typename Integer> |
98 | class Atomic { |
99 | public: |
100 | Atomic(std::function<void()> onFetchOr, std::function<void()> onFetchAnd) |
101 | : onFetchOr_{std::move(onFetchOr)}, onFetchAnd_{std::move(onFetchAnd)} {} |
102 | |
103 | Integer fetch_or( |
104 | Integer value, |
105 | std::memory_order = std::memory_order_seq_cst) { |
106 | onFetchOr_(); |
107 | return exchange(integer_, integer_ | value); |
108 | } |
109 | Integer fetch_and( |
110 | Integer value, |
111 | std::memory_order = std::memory_order_seq_cst) { |
112 | onFetchAnd_(); |
113 | return exchange(integer_, integer_ & value); |
114 | } |
115 | |
116 | Integer load(std::memory_order = std::memory_order_seq_cst) { |
117 | return integer_; |
118 | } |
119 | |
120 | std::function<void()> onFetchOr_; |
121 | std::function<void()> onFetchAnd_; |
122 | Integer integer_{0}; |
123 | }; |
124 | |
125 | template <typename Integer, typename FetchSet = decltype(default_fetch_set)> |
126 | void atomic_fetch_set_non_std_atomic(FetchSet fetch_set = default_fetch_set) { |
127 | auto sets = 0; |
128 | auto resets = 0; |
129 | auto atomic = Atomic<Integer>{[&] { ++sets; }, [&] { ++resets; }}; |
130 | |
131 | fetch_set(atomic, 0); |
132 | EXPECT_EQ(sets, 1); |
133 | EXPECT_EQ(resets, 0); |
134 | EXPECT_EQ(atomic.integer_, 0b1); |
135 | |
136 | fetch_set(atomic, 2); |
137 | EXPECT_EQ(sets, 2); |
138 | EXPECT_EQ(resets, 0); |
139 | EXPECT_EQ(atomic.integer_, 0b101); |
140 | } |
141 | |
142 | template <typename Integer, typename F = decltype(default_fetch_reset)> |
143 | void atomic_fetch_reset_non_std_atomic(F fetch_reset = default_fetch_reset) { |
144 | auto sets = 0; |
145 | auto resets = 0; |
146 | auto atomic = Atomic<Integer>{[&] { ++sets; }, [&] { ++resets; }}; |
147 | atomic.integer_ = 0b111; |
148 | |
149 | fetch_reset(atomic, 0); |
150 | EXPECT_EQ(sets, 0); |
151 | EXPECT_EQ(resets, 1); |
152 | EXPECT_EQ(atomic.integer_, 0b110); |
153 | |
154 | fetch_reset(atomic, 2); |
155 | EXPECT_EQ(sets, 0); |
156 | EXPECT_EQ(resets, 2); |
157 | EXPECT_EQ(atomic.integer_, 0b010); |
158 | } |
159 | } // namespace |
160 | |
161 | class AtomicFetchSetTest : public ::testing::Test {}; |
162 | class AtomicFetchResetTest : public ::testing::Test {}; |
163 | |
164 | TEST_F(AtomicFetchSetTest, Basic) { |
165 | atomic_fetch_set_basic<std::uint16_t>(); |
166 | atomic_fetch_set_basic<std::uint32_t>(); |
167 | atomic_fetch_set_basic<std::uint64_t>(); |
168 | atomic_fetch_set_basic<std::uint8_t>(); |
169 | } |
170 | |
171 | TEST_F(AtomicFetchResetTest, Basic) { |
172 | atomic_fetch_reset_basic<std::uint16_t>(); |
173 | atomic_fetch_reset_basic<std::uint32_t>(); |
174 | atomic_fetch_reset_basic<std::uint64_t>(); |
175 | atomic_fetch_reset_basic<std::uint8_t>(); |
176 | } |
177 | |
178 | TEST_F(AtomicFetchSetTest, EnsureFetchOrUsed) { |
179 | atomic_fetch_set_non_std_atomic<std::uint8_t>(); |
180 | atomic_fetch_set_non_std_atomic<std::uint16_t>(); |
181 | atomic_fetch_set_non_std_atomic<std::uint32_t>(); |
182 | atomic_fetch_set_non_std_atomic<std::uint64_t>(); |
183 | } |
184 | |
185 | TEST_F(AtomicFetchResetTest, EnsureFetchAndUsed) { |
186 | atomic_fetch_reset_non_std_atomic<std::uint8_t>(); |
187 | atomic_fetch_reset_non_std_atomic<std::uint16_t>(); |
188 | atomic_fetch_reset_non_std_atomic<std::uint32_t>(); |
189 | atomic_fetch_reset_non_std_atomic<std::uint64_t>(); |
190 | } |
191 | |
192 | TEST_F(AtomicFetchSetTest, FetchSetDefault) { |
193 | auto fetch_set = [](auto&&... args) { |
194 | return detail::atomic_fetch_set_default(args..., std::memory_order_seq_cst); |
195 | }; |
196 | |
197 | atomic_fetch_set_basic<std::uint16_t>(fetch_set); |
198 | atomic_fetch_set_basic<std::uint32_t>(fetch_set); |
199 | atomic_fetch_set_basic<std::uint64_t>(fetch_set); |
200 | atomic_fetch_set_basic<std::uint8_t>(fetch_set); |
201 | |
202 | atomic_fetch_set_non_std_atomic<std::uint8_t>(fetch_set); |
203 | atomic_fetch_set_non_std_atomic<std::uint16_t>(fetch_set); |
204 | atomic_fetch_set_non_std_atomic<std::uint32_t>(fetch_set); |
205 | atomic_fetch_set_non_std_atomic<std::uint64_t>(fetch_set); |
206 | } |
207 | |
208 | TEST_F(AtomicFetchSetTest, FetchResetDefault) { |
209 | auto fetch_reset = [](auto&&... args) { |
210 | return detail::atomic_fetch_reset_default( |
211 | args..., std::memory_order_seq_cst); |
212 | }; |
213 | |
214 | atomic_fetch_reset_basic<std::uint16_t>(fetch_reset); |
215 | atomic_fetch_reset_basic<std::uint32_t>(fetch_reset); |
216 | atomic_fetch_reset_basic<std::uint64_t>(fetch_reset); |
217 | atomic_fetch_reset_basic<std::uint8_t>(fetch_reset); |
218 | |
219 | atomic_fetch_reset_non_std_atomic<std::uint8_t>(fetch_reset); |
220 | atomic_fetch_reset_non_std_atomic<std::uint16_t>(fetch_reset); |
221 | atomic_fetch_reset_non_std_atomic<std::uint32_t>(fetch_reset); |
222 | atomic_fetch_reset_non_std_atomic<std::uint64_t>(fetch_reset); |
223 | } |
224 | |
225 | TEST_F(AtomicFetchSetTest, FetchSetX86) { |
226 | if (folly::kIsArchAmd64) { |
227 | auto fetch_set = [](auto&&... args) { |
228 | return detail::atomic_fetch_set_x86(args..., std::memory_order_seq_cst); |
229 | }; |
230 | |
231 | atomic_fetch_set_basic<std::uint16_t>(fetch_set); |
232 | atomic_fetch_set_basic<std::uint32_t>(fetch_set); |
233 | atomic_fetch_set_basic<std::uint64_t>(fetch_set); |
234 | atomic_fetch_set_basic<std::uint8_t>(fetch_set); |
235 | |
236 | atomic_fetch_set_non_std_atomic<std::uint8_t>(fetch_set); |
237 | atomic_fetch_set_non_std_atomic<std::uint16_t>(fetch_set); |
238 | atomic_fetch_set_non_std_atomic<std::uint32_t>(fetch_set); |
239 | atomic_fetch_set_non_std_atomic<std::uint64_t>(fetch_set); |
240 | } |
241 | } |
242 | |
243 | TEST_F(AtomicFetchResetTest, FetchResetX86) { |
244 | if (folly::kIsArchAmd64) { |
245 | auto fetch_reset = [](auto&&... args) { |
246 | return detail::atomic_fetch_reset_x86(args..., std::memory_order_seq_cst); |
247 | }; |
248 | |
249 | atomic_fetch_reset_basic<std::uint16_t>(fetch_reset); |
250 | atomic_fetch_reset_basic<std::uint32_t>(fetch_reset); |
251 | atomic_fetch_reset_basic<std::uint64_t>(fetch_reset); |
252 | atomic_fetch_reset_basic<std::uint8_t>(fetch_reset); |
253 | |
254 | atomic_fetch_reset_non_std_atomic<std::uint8_t>(fetch_reset); |
255 | atomic_fetch_reset_non_std_atomic<std::uint16_t>(fetch_reset); |
256 | atomic_fetch_reset_non_std_atomic<std::uint32_t>(fetch_reset); |
257 | atomic_fetch_reset_non_std_atomic<std::uint64_t>(fetch_reset); |
258 | } |
259 | } |
260 | |
261 | } // namespace folly |
262 | |