1 | /* |
2 | * Copyright 2004-present Facebook, Inc. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | #pragma once |
18 | |
19 | #include <folly/Portability.h> |
20 | #include <folly/Traits.h> |
21 | |
22 | #include <atomic> |
23 | #include <cassert> |
24 | #include <cstdint> |
25 | #include <tuple> |
26 | #include <type_traits> |
27 | |
28 | #if _WIN32 |
29 | #include <intrin.h> |
30 | #endif |
31 | |
32 | namespace folly { |
33 | namespace detail { |
34 | |
35 | // TODO: Remove the non-default implementations when both gcc and clang |
36 | // can recognize single bit set/reset patterns and compile them down to locked |
37 | // bts and btr instructions. |
38 | // |
39 | // Currently, at the time of writing it seems like gcc7 and greater can make |
40 | // this optimization and clang cannot - https://gcc.godbolt.org/z/Q83rxX |
41 | |
42 | template <typename Atomic> |
43 | bool atomic_fetch_set_default( |
44 | Atomic& atomic, |
45 | std::size_t bit, |
46 | std::memory_order order) { |
47 | using Integer = decltype(atomic.load()); |
48 | auto mask = Integer{0b1} << static_cast<Integer>(bit); |
49 | return (atomic.fetch_or(mask, order) & mask); |
50 | } |
51 | |
52 | template <typename Atomic> |
53 | bool atomic_fetch_reset_default( |
54 | Atomic& atomic, |
55 | std::size_t bit, |
56 | std::memory_order order) { |
57 | using Integer = decltype(atomic.load()); |
58 | auto mask = Integer{0b1} << static_cast<Integer>(bit); |
59 | return (atomic.fetch_and(~mask, order) & mask); |
60 | } |
61 | |
62 | /** |
63 | * A simple trait to determine if the given type is an instantiation of |
64 | * std::atomic |
65 | */ |
66 | template <typename T> |
67 | constexpr auto is_atomic = false; |
68 | template <typename Integer> |
69 | constexpr auto is_atomic<std::atomic<Integer>> = true; |
70 | |
71 | #if FOLLY_X64 |
72 | |
73 | #if _MSC_VER |
74 | |
75 | template <typename Integer> |
76 | inline bool atomic_fetch_set_x86( |
77 | std::atomic<Integer>& atomic, |
78 | std::size_t bit, |
79 | std::memory_order order) { |
80 | static_assert(alignof(std::atomic<Integer>) == alignof(Integer), "" ); |
81 | static_assert(sizeof(std::atomic<Integer>) == sizeof(Integer), "" ); |
82 | assert(atomic.is_lock_free()); |
83 | |
84 | if /* constexpr */ (sizeof(Integer) == 4) { |
85 | return _interlockedbittestandset( |
86 | reinterpret_cast<volatile long*>(&atomic), static_cast<long>(bit)); |
87 | } else if /* constexpr */ (sizeof(Integer) == 8) { |
88 | return _interlockedbittestandset64( |
89 | reinterpret_cast<volatile long long*>(&atomic), |
90 | static_cast<long long>(bit)); |
91 | } else { |
92 | assert(sizeof(Integer) != 4 && sizeof(Integer) != 8); |
93 | return atomic_fetch_set_default(atomic, bit, order); |
94 | } |
95 | } |
96 | |
97 | template <typename Atomic> |
98 | inline bool |
99 | atomic_fetch_set_x86(Atomic& atomic, std::size_t bit, std::memory_order order) { |
100 | static_assert(!std::is_same<Atomic, std::atomic<std::uint32_t>>{}, "" ); |
101 | static_assert(!std::is_same<Atomic, std::atomic<std::uint64_t>>{}, "" ); |
102 | return atomic_fetch_set_default(atomic, bit, order); |
103 | } |
104 | |
105 | template <typename Integer> |
106 | inline bool atomic_fetch_reset_x86( |
107 | std::atomic<Integer>& atomic, |
108 | std::size_t bit, |
109 | std::memory_order order) { |
110 | static_assert(alignof(std::atomic<Integer>) == alignof(Integer), "" ); |
111 | static_assert(sizeof(std::atomic<Integer>) == sizeof(Integer), "" ); |
112 | assert(atomic.is_lock_free()); |
113 | |
114 | if /* constexpr */ (sizeof(Integer) == 4) { |
115 | return _interlockedbittestandreset( |
116 | reinterpret_cast<volatile long*>(&atomic), static_cast<long>(bit)); |
117 | } else if /* constexpr */ (sizeof(Integer) == 8) { |
118 | return _interlockedbittestandreset64( |
119 | reinterpret_cast<volatile long long*>(&atomic), |
120 | static_cast<long long>(bit)); |
121 | } else { |
122 | assert(sizeof(Integer) != 4 && sizeof(Integer) != 8); |
123 | return atomic_fetch_reset_default(atomic, bit, order); |
124 | } |
125 | } |
126 | |
127 | template <typename Atomic> |
128 | inline bool |
129 | atomic_fetch_reset_x86(Atomic& atomic, std::size_t bit, std::memory_order mo) { |
130 | static_assert(!std::is_same<Atomic, std::atomic<std::uint32_t>>{}, "" ); |
131 | static_assert(!std::is_same<Atomic, std::atomic<std::uint64_t>>{}, "" ); |
132 | return atomic_fetch_reset_default(atomic, bit, mo); |
133 | } |
134 | |
135 | #else |
136 | |
137 | template <typename Integer> |
138 | inline bool atomic_fetch_set_x86( |
139 | std::atomic<Integer>& atomic, |
140 | std::size_t bit, |
141 | std::memory_order order) { |
142 | auto previous = false; |
143 | |
144 | if /* constexpr */ (sizeof(Integer) == 2) { |
145 | auto pointer = reinterpret_cast<std::uint16_t*>(&atomic); |
146 | asm volatile("lock; btsw %1, (%2); setc %0" |
147 | : "=r" (previous) |
148 | : "ri" (static_cast<std::uint16_t>(bit)), "r" (pointer) |
149 | : "memory" , "flags" ); |
150 | } else if /* constexpr */ (sizeof(Integer) == 4) { |
151 | auto pointer = reinterpret_cast<std::uint32_t*>(&atomic); |
152 | asm volatile("lock; btsl %1, (%2); setc %0" |
153 | : "=r" (previous) |
154 | : "ri" (static_cast<std::uint32_t>(bit)), "r" (pointer) |
155 | : "memory" , "flags" ); |
156 | } else if /* constexpr */ (sizeof(Integer) == 8) { |
157 | auto pointer = reinterpret_cast<std::uint64_t*>(&atomic); |
158 | asm volatile("lock; btsq %1, (%2); setc %0" |
159 | : "=r" (previous) |
160 | : "ri" (static_cast<std::uint64_t>(bit)), "r" (pointer) |
161 | : "memory" , "flags" ); |
162 | } else { |
163 | assert(sizeof(Integer) == 1); |
164 | return atomic_fetch_set_default(atomic, bit, order); |
165 | } |
166 | |
167 | return previous; |
168 | } |
169 | |
170 | template <typename Atomic> |
171 | inline bool |
172 | atomic_fetch_set_x86(Atomic& atomic, std::size_t bit, std::memory_order order) { |
173 | static_assert(!is_atomic<Atomic>, "" ); |
174 | return atomic_fetch_set_default(atomic, bit, order); |
175 | } |
176 | |
177 | template <typename Integer> |
178 | inline bool atomic_fetch_reset_x86( |
179 | std::atomic<Integer>& atomic, |
180 | std::size_t bit, |
181 | std::memory_order order) { |
182 | auto previous = false; |
183 | |
184 | if /* constexpr */ (sizeof(Integer) == 2) { |
185 | auto pointer = reinterpret_cast<std::uint16_t*>(&atomic); |
186 | asm volatile("lock; btrw %1, (%2); setc %0" |
187 | : "=r" (previous) |
188 | : "ri" (static_cast<std::uint16_t>(bit)), "r" (pointer) |
189 | : "memory" , "flags" ); |
190 | } else if /* constexpr */ (sizeof(Integer) == 4) { |
191 | auto pointer = reinterpret_cast<std::uint32_t*>(&atomic); |
192 | asm volatile("lock; btrl %1, (%2); setc %0" |
193 | : "=r" (previous) |
194 | : "ri" (static_cast<std::uint32_t>(bit)), "r" (pointer) |
195 | : "memory" , "flags" ); |
196 | } else if /* constexpr */ (sizeof(Integer) == 8) { |
197 | auto pointer = reinterpret_cast<std::uint64_t*>(&atomic); |
198 | asm volatile("lock; btrq %1, (%2); setc %0" |
199 | : "=r" (previous) |
200 | : "ri" (static_cast<std::uint64_t>(bit)), "r" (pointer) |
201 | : "memory" , "flags" ); |
202 | } else { |
203 | assert(sizeof(Integer) == 1); |
204 | return atomic_fetch_reset_default(atomic, bit, order); |
205 | } |
206 | |
207 | return previous; |
208 | } |
209 | |
210 | template <typename Atomic> |
211 | bool atomic_fetch_reset_x86( |
212 | Atomic& atomic, |
213 | std::size_t bit, |
214 | std::memory_order order) { |
215 | static_assert(!is_atomic<Atomic>, "" ); |
216 | return atomic_fetch_reset_default(atomic, bit, order); |
217 | } |
218 | |
219 | #endif |
220 | |
221 | #else |
222 | |
223 | template <typename Atomic> |
224 | bool atomic_fetch_set_x86(Atomic&, std::size_t, std::memory_order) noexcept { |
225 | throw std::logic_error{"Incorrect function called" }; |
226 | } |
227 | template <typename Atomic> |
228 | bool atomic_fetch_reset_x86(Atomic&, std::size_t, std::memory_order) noexcept { |
229 | throw std::logic_error{"Incorrect function called" }; |
230 | } |
231 | |
232 | #endif |
233 | |
234 | } // namespace detail |
235 | |
236 | template <typename Atomic> |
237 | bool atomic_fetch_set(Atomic& atomic, std::size_t bit, std::memory_order mo) { |
238 | using Integer = decltype(atomic.load()); |
239 | static_assert(std::is_unsigned<Integer>{}, "" ); |
240 | static_assert(!std::is_const<Atomic>{}, "" ); |
241 | assert(bit < (sizeof(Integer) * 8)); |
242 | |
243 | if (folly::kIsArchAmd64) { |
244 | // do the optimized thing on x86 builds |
245 | return detail::atomic_fetch_set_x86(atomic, bit, mo); |
246 | } else { |
247 | // otherwise default to the default implementation using fetch_or() |
248 | return detail::atomic_fetch_set_default(atomic, bit, mo); |
249 | } |
250 | } |
251 | |
252 | template <typename Atomic> |
253 | bool atomic_fetch_reset(Atomic& atomic, std::size_t bit, std::memory_order mo) { |
254 | using Integer = decltype(atomic.load()); |
255 | static_assert(std::is_unsigned<Integer>{}, "" ); |
256 | static_assert(!std::is_const<Atomic>{}, "" ); |
257 | assert(bit < (sizeof(Integer) * 8)); |
258 | |
259 | if (folly::kIsArchAmd64) { |
260 | // do the optimized thing on x86 builds |
261 | return detail::atomic_fetch_reset_x86(atomic, bit, mo); |
262 | } else { |
263 | // otherwise default to the default implementation using fetch_and() |
264 | return detail::atomic_fetch_reset_default(atomic, bit, mo); |
265 | } |
266 | } |
267 | |
268 | } // namespace folly |
269 | |