1/*
2 * Copyright 2004-present Facebook, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#pragma once
18
19#include <folly/Portability.h>
20#include <folly/Traits.h>
21
22#include <atomic>
23#include <cassert>
24#include <cstdint>
25#include <tuple>
26#include <type_traits>
27
28#if _WIN32
29#include <intrin.h>
30#endif
31
32namespace folly {
33namespace detail {
34
35// TODO: Remove the non-default implementations when both gcc and clang
36// can recognize single bit set/reset patterns and compile them down to locked
37// bts and btr instructions.
38//
39// Currently, at the time of writing it seems like gcc7 and greater can make
40// this optimization and clang cannot - https://gcc.godbolt.org/z/Q83rxX
41
42template <typename Atomic>
43bool atomic_fetch_set_default(
44 Atomic& atomic,
45 std::size_t bit,
46 std::memory_order order) {
47 using Integer = decltype(atomic.load());
48 auto mask = Integer{0b1} << static_cast<Integer>(bit);
49 return (atomic.fetch_or(mask, order) & mask);
50}
51
52template <typename Atomic>
53bool atomic_fetch_reset_default(
54 Atomic& atomic,
55 std::size_t bit,
56 std::memory_order order) {
57 using Integer = decltype(atomic.load());
58 auto mask = Integer{0b1} << static_cast<Integer>(bit);
59 return (atomic.fetch_and(~mask, order) & mask);
60}
61
62/**
63 * A simple trait to determine if the given type is an instantiation of
64 * std::atomic
65 */
66template <typename T>
67constexpr auto is_atomic = false;
68template <typename Integer>
69constexpr auto is_atomic<std::atomic<Integer>> = true;
70
71#if FOLLY_X64
72
73#if _MSC_VER
74
75template <typename Integer>
76inline bool atomic_fetch_set_x86(
77 std::atomic<Integer>& atomic,
78 std::size_t bit,
79 std::memory_order order) {
80 static_assert(alignof(std::atomic<Integer>) == alignof(Integer), "");
81 static_assert(sizeof(std::atomic<Integer>) == sizeof(Integer), "");
82 assert(atomic.is_lock_free());
83
84 if /* constexpr */ (sizeof(Integer) == 4) {
85 return _interlockedbittestandset(
86 reinterpret_cast<volatile long*>(&atomic), static_cast<long>(bit));
87 } else if /* constexpr */ (sizeof(Integer) == 8) {
88 return _interlockedbittestandset64(
89 reinterpret_cast<volatile long long*>(&atomic),
90 static_cast<long long>(bit));
91 } else {
92 assert(sizeof(Integer) != 4 && sizeof(Integer) != 8);
93 return atomic_fetch_set_default(atomic, bit, order);
94 }
95}
96
97template <typename Atomic>
98inline bool
99atomic_fetch_set_x86(Atomic& atomic, std::size_t bit, std::memory_order order) {
100 static_assert(!std::is_same<Atomic, std::atomic<std::uint32_t>>{}, "");
101 static_assert(!std::is_same<Atomic, std::atomic<std::uint64_t>>{}, "");
102 return atomic_fetch_set_default(atomic, bit, order);
103}
104
105template <typename Integer>
106inline bool atomic_fetch_reset_x86(
107 std::atomic<Integer>& atomic,
108 std::size_t bit,
109 std::memory_order order) {
110 static_assert(alignof(std::atomic<Integer>) == alignof(Integer), "");
111 static_assert(sizeof(std::atomic<Integer>) == sizeof(Integer), "");
112 assert(atomic.is_lock_free());
113
114 if /* constexpr */ (sizeof(Integer) == 4) {
115 return _interlockedbittestandreset(
116 reinterpret_cast<volatile long*>(&atomic), static_cast<long>(bit));
117 } else if /* constexpr */ (sizeof(Integer) == 8) {
118 return _interlockedbittestandreset64(
119 reinterpret_cast<volatile long long*>(&atomic),
120 static_cast<long long>(bit));
121 } else {
122 assert(sizeof(Integer) != 4 && sizeof(Integer) != 8);
123 return atomic_fetch_reset_default(atomic, bit, order);
124 }
125}
126
127template <typename Atomic>
128inline bool
129atomic_fetch_reset_x86(Atomic& atomic, std::size_t bit, std::memory_order mo) {
130 static_assert(!std::is_same<Atomic, std::atomic<std::uint32_t>>{}, "");
131 static_assert(!std::is_same<Atomic, std::atomic<std::uint64_t>>{}, "");
132 return atomic_fetch_reset_default(atomic, bit, mo);
133}
134
135#else
136
137template <typename Integer>
138inline bool atomic_fetch_set_x86(
139 std::atomic<Integer>& atomic,
140 std::size_t bit,
141 std::memory_order order) {
142 auto previous = false;
143
144 if /* constexpr */ (sizeof(Integer) == 2) {
145 auto pointer = reinterpret_cast<std::uint16_t*>(&atomic);
146 asm volatile("lock; btsw %1, (%2); setc %0"
147 : "=r"(previous)
148 : "ri"(static_cast<std::uint16_t>(bit)), "r"(pointer)
149 : "memory", "flags");
150 } else if /* constexpr */ (sizeof(Integer) == 4) {
151 auto pointer = reinterpret_cast<std::uint32_t*>(&atomic);
152 asm volatile("lock; btsl %1, (%2); setc %0"
153 : "=r"(previous)
154 : "ri"(static_cast<std::uint32_t>(bit)), "r"(pointer)
155 : "memory", "flags");
156 } else if /* constexpr */ (sizeof(Integer) == 8) {
157 auto pointer = reinterpret_cast<std::uint64_t*>(&atomic);
158 asm volatile("lock; btsq %1, (%2); setc %0"
159 : "=r"(previous)
160 : "ri"(static_cast<std::uint64_t>(bit)), "r"(pointer)
161 : "memory", "flags");
162 } else {
163 assert(sizeof(Integer) == 1);
164 return atomic_fetch_set_default(atomic, bit, order);
165 }
166
167 return previous;
168}
169
170template <typename Atomic>
171inline bool
172atomic_fetch_set_x86(Atomic& atomic, std::size_t bit, std::memory_order order) {
173 static_assert(!is_atomic<Atomic>, "");
174 return atomic_fetch_set_default(atomic, bit, order);
175}
176
177template <typename Integer>
178inline bool atomic_fetch_reset_x86(
179 std::atomic<Integer>& atomic,
180 std::size_t bit,
181 std::memory_order order) {
182 auto previous = false;
183
184 if /* constexpr */ (sizeof(Integer) == 2) {
185 auto pointer = reinterpret_cast<std::uint16_t*>(&atomic);
186 asm volatile("lock; btrw %1, (%2); setc %0"
187 : "=r"(previous)
188 : "ri"(static_cast<std::uint16_t>(bit)), "r"(pointer)
189 : "memory", "flags");
190 } else if /* constexpr */ (sizeof(Integer) == 4) {
191 auto pointer = reinterpret_cast<std::uint32_t*>(&atomic);
192 asm volatile("lock; btrl %1, (%2); setc %0"
193 : "=r"(previous)
194 : "ri"(static_cast<std::uint32_t>(bit)), "r"(pointer)
195 : "memory", "flags");
196 } else if /* constexpr */ (sizeof(Integer) == 8) {
197 auto pointer = reinterpret_cast<std::uint64_t*>(&atomic);
198 asm volatile("lock; btrq %1, (%2); setc %0"
199 : "=r"(previous)
200 : "ri"(static_cast<std::uint64_t>(bit)), "r"(pointer)
201 : "memory", "flags");
202 } else {
203 assert(sizeof(Integer) == 1);
204 return atomic_fetch_reset_default(atomic, bit, order);
205 }
206
207 return previous;
208}
209
210template <typename Atomic>
211bool atomic_fetch_reset_x86(
212 Atomic& atomic,
213 std::size_t bit,
214 std::memory_order order) {
215 static_assert(!is_atomic<Atomic>, "");
216 return atomic_fetch_reset_default(atomic, bit, order);
217}
218
219#endif
220
221#else
222
223template <typename Atomic>
224bool atomic_fetch_set_x86(Atomic&, std::size_t, std::memory_order) noexcept {
225 throw std::logic_error{"Incorrect function called"};
226}
227template <typename Atomic>
228bool atomic_fetch_reset_x86(Atomic&, std::size_t, std::memory_order) noexcept {
229 throw std::logic_error{"Incorrect function called"};
230}
231
232#endif
233
234} // namespace detail
235
236template <typename Atomic>
237bool atomic_fetch_set(Atomic& atomic, std::size_t bit, std::memory_order mo) {
238 using Integer = decltype(atomic.load());
239 static_assert(std::is_unsigned<Integer>{}, "");
240 static_assert(!std::is_const<Atomic>{}, "");
241 assert(bit < (sizeof(Integer) * 8));
242
243 if (folly::kIsArchAmd64) {
244 // do the optimized thing on x86 builds
245 return detail::atomic_fetch_set_x86(atomic, bit, mo);
246 } else {
247 // otherwise default to the default implementation using fetch_or()
248 return detail::atomic_fetch_set_default(atomic, bit, mo);
249 }
250}
251
252template <typename Atomic>
253bool atomic_fetch_reset(Atomic& atomic, std::size_t bit, std::memory_order mo) {
254 using Integer = decltype(atomic.load());
255 static_assert(std::is_unsigned<Integer>{}, "");
256 static_assert(!std::is_const<Atomic>{}, "");
257 assert(bit < (sizeof(Integer) * 8));
258
259 if (folly::kIsArchAmd64) {
260 // do the optimized thing on x86 builds
261 return detail::atomic_fetch_reset_x86(atomic, bit, mo);
262 } else {
263 // otherwise default to the default implementation using fetch_and()
264 return detail::atomic_fetch_reset_default(atomic, bit, mo);
265 }
266}
267
268} // namespace folly
269