1// Licensed to the .NET Foundation under one or more agreements.
2// The .NET Foundation licenses this file to you under the MIT license.
3// See the LICENSE file in the project root for more information.
4//
5// Volatile.h
6//
7
8//
9// Defines the Volatile<T> type, which provides uniform volatile-ness on
10// Visual C++ and GNU C++.
11//
12// Visual C++ treats accesses to volatile variables as follows: no read or write
13// can be removed by the compiler, no global memory access can be moved backwards past
14// a volatile read, and no global memory access can be moved forward past a volatile
15// write.
16//
17// The GCC volatile semantic is straight out of the C standard: the compiler is not
18// allowed to remove accesses to volatile variables, and it is not allowed to reorder
19// volatile accesses relative to other volatile accesses. It is allowed to freely
20// reorder non-volatile accesses relative to volatile accesses.
21//
22// We have lots of code that assumes that ordering of non-volatile accesses will be
23// constrained relative to volatile accesses. For example, this pattern appears all
24// over the place:
25//
26// static volatile int lock = 0;
27//
28// while (InterlockedCompareExchange(&lock, 0, 1))
29// {
30// //spin
31// }
32//
33// //read and write variables protected by the lock
34//
35// lock = 0;
36//
37// This depends on the reads and writes in the critical section not moving past the
38// final statement, which releases the lock. If this should happen, then you have an
39// unintended race.
40//
41// The solution is to ban the use of the "volatile" keyword, and instead define our
42// own type Volatile<T>, which acts like a variable of type T except that accesses to
43// the variable are always given VC++'s volatile semantics.
44//
45// (NOTE: The code above is not intended to be an example of how a spinlock should be
46// implemented; it has many flaws, and should not be used. This code is intended only
47// to illustrate where we might get into trouble with GCC's volatile semantics.)
48//
49// @TODO: many of the variables marked volatile in the CLR do not actually need to be
50// volatile. For example, if a variable is just always passed to Interlocked functions
51// (such as a refcount variable), there is no need for it to be volatile. A future
52// cleanup task should be to examine each volatile variable and make them non-volatile
53// if possible.
54//
55// @TODO: link to a "Memory Models for CLR Devs" doc here (this doc does not yet exist).
56//
57
58#ifndef _VOLATILE_H_
59#define _VOLATILE_H_
60
61//
62// This code is extremely compiler- and CPU-specific, and will need to be altered to
63// support new compilers and/or CPUs. Here we enforce that we can only compile using
64// VC++, or GCC on x86 or AMD64.
65//
66#if !defined(_MSC_VER) && !defined(__GNUC__)
67#error The Volatile type is currently only defined for Visual C++ and GNU C++
68#endif
69
70#if defined(__GNUC__) && !defined(_X86_) && !defined(_AMD64_) && !defined(_ARM_) && !defined(_ARM64_)
71#error The Volatile type is currently only defined for GCC when targeting x86, AMD64, ARM or ARM64 CPUs
72#endif
73
74#if defined(__GNUC__)
75#if defined(_ARM_) || defined(_ARM64_)
76// This is functionally equivalent to the MemoryBarrier() macro used on ARM on Windows.
77#define VOLATILE_MEMORY_BARRIER() asm volatile ("dmb ish" : : : "memory")
78#else
79//
80// For GCC, we prevent reordering by the compiler by inserting the following after a volatile
81// load (to prevent subsequent operations from moving before the read), and before a volatile
82// write (to prevent prior operations from moving past the write). We don't need to do anything
83// special to prevent CPU reorderings, because the x86 and AMD64 architectures are already
84// sufficiently constrained for our purposes. If we ever need to run on weaker CPU architectures
85// (such as PowerPC), then we will need to do more work.
86//
87// Please do not use this macro outside of this file. It is subject to change or removal without
88// notice.
89//
90#define VOLATILE_MEMORY_BARRIER() asm volatile ("" : : : "memory")
91#endif // _ARM_ || _ARM64_
92#elif (defined(_ARM_) || defined(_ARM64_)) && _ISO_VOLATILE
93// ARM & ARM64 have a very weak memory model and very few tools to control that model. We're forced to perform a full
94// memory barrier to preserve the volatile semantics. Technically this is only necessary on MP systems but we
95// currently don't have a cheap way to determine the number of CPUs from this header file. Revisit this if it
96// turns out to be a performance issue for the uni-proc case.
97#define VOLATILE_MEMORY_BARRIER() MemoryBarrier()
98#else
99//
100// On VC++, reorderings at the compiler and machine level are prevented by the use of the
101// "volatile" keyword in VolatileLoad and VolatileStore. This should work on any CPU architecture
102// targeted by VC++ with /iso_volatile-.
103//
104#define VOLATILE_MEMORY_BARRIER()
105#endif // __GNUC__
106
107template<typename T>
108struct RemoveVolatile
109{
110 typedef T type;
111};
112
113template<typename T>
114struct RemoveVolatile<volatile T>
115{
116 typedef T type;
117};
118
119
120//
121// VolatileLoad loads a T from a pointer to T. It is guaranteed that this load will not be optimized
122// away by the compiler, and that any operation that occurs after this load, in program order, will
123// not be moved before this load. In general it is not guaranteed that the load will be atomic, though
124// this is the case for most aligned scalar data types. If you need atomic loads or stores, you need
125// to consult the compiler and CPU manuals to find which circumstances allow atomicity.
126//
127// Starting at version 3.8, clang errors out on initializing of type int * to volatile int *. To fix this, we add two templates to cast away volatility
128// Helper structures for casting away volatileness
129
130
131template<typename T>
132inline
133T VolatileLoad(T const * pt)
134{
135#ifndef DACCESS_COMPILE
136#if defined(_ARM64_) && defined(__GNUC__)
137 T val;
138 static const unsigned lockFreeAtomicSizeMask = (1 << 1) | (1 << 2) | (1 << 4) | (1 << 8);
139 if((1 << sizeof(T)) & lockFreeAtomicSizeMask)
140 {
141 __atomic_load((T const *)pt, const_cast<typename RemoveVolatile<T>::type *>(&val), __ATOMIC_ACQUIRE);
142 }
143 else
144 {
145 val = *(T volatile const *)pt;
146 asm volatile ("dmb ishld" : : : "memory");
147 }
148#else
149 T val = *(T volatile const *)pt;
150 VOLATILE_MEMORY_BARRIER();
151#endif
152#else
153 T val = *pt;
154#endif
155 return val;
156}
157
158template<typename T>
159inline
160T VolatileLoadWithoutBarrier(T const * pt)
161{
162#ifndef DACCESS_COMPILE
163 T val = *(T volatile const *)pt;
164#else
165 T val = *pt;
166#endif
167 return val;
168}
169
170template <typename T> class Volatile;
171
172template<typename T>
173inline
174T VolatileLoad(Volatile<T> const * pt)
175{
176 return pt->Load();
177}
178
179//
180// VolatileStore stores a T into the target of a pointer to T. Is is guaranteed that this store will
181// not be optimized away by the compiler, and that any operation that occurs before this store, in program
182// order, will not be moved after this store. In general, it is not guaranteed that the store will be
183// atomic, though this is the case for most aligned scalar data types. If you need atomic loads or stores,
184// you need to consult the compiler and CPU manuals to find which circumstances allow atomicity.
185//
186template<typename T>
187inline
188void VolatileStore(T* pt, T val)
189{
190#ifndef DACCESS_COMPILE
191#if defined(_ARM64_) && defined(__GNUC__)
192 static const unsigned lockFreeAtomicSizeMask = (1 << 1) | (1 << 2) | (1 << 4) | (1 << 8);
193 if((1 << sizeof(T)) & lockFreeAtomicSizeMask)
194 {
195 __atomic_store((T volatile *)pt, &val, __ATOMIC_RELEASE);
196 }
197 else
198 {
199 VOLATILE_MEMORY_BARRIER();
200 *(T volatile *)pt = val;
201 }
202#else
203 VOLATILE_MEMORY_BARRIER();
204 *(T volatile *)pt = val;
205#endif
206#else
207 *pt = val;
208#endif
209}
210
211template<typename T>
212inline
213void VolatileStoreWithoutBarrier(T* pt, T val)
214{
215#ifndef DACCESS_COMPILE
216 *(T volatile *)pt = val;
217#else
218 *pt = val;
219#endif
220}
221
222//
223// Volatile<T> implements accesses with our volatile semantics over a variable of type T.
224// Wherever you would have used a "volatile Foo" or, equivalently, "Foo volatile", use Volatile<Foo>
225// instead. If Foo is a pointer type, use VolatilePtr.
226//
227// Note that there are still some things that don't work with a Volatile<T>,
228// that would have worked with a "volatile T". For example, you can't cast a Volatile<int> to a float.
229// You must instead cast to an int, then to a float. Or you can call Load on the Volatile<int>, and
230// cast the result to a float. In general, calling Load or Store explicitly will work around
231// any problems that can't be solved by operator overloading.
232//
233// @TODO: it's not clear that we actually *want* any operator overloading here. It's in here primarily
234// to ease the task of converting all of the old uses of the volatile keyword, but in the long
235// run it's probably better if users of this class are forced to call Load() and Store() explicitly.
236// This would make it much more clear where the memory barriers are, and which operations are actually
237// being performed, but it will have to wait for another cleanup effort.
238//
239template <typename T>
240class Volatile
241{
242private:
243 //
244 // The data which we are treating as volatile
245 //
246 T m_val;
247
248public:
249 //
250 // Default constructor. Results in an unitialized value!
251 //
252 inline Volatile()
253 {
254 }
255
256 //
257 // Allow initialization of Volatile<T> from a T
258 //
259 inline Volatile(const T& val)
260 {
261 ((volatile T &)m_val) = val;
262 }
263
264 //
265 // Copy constructor
266 //
267 inline Volatile(const Volatile<T>& other)
268 {
269 ((volatile T &)m_val) = other.Load();
270 }
271
272 //
273 // Loads the value of the volatile variable. See code:VolatileLoad for the semantics of this operation.
274 //
275 inline T Load() const
276 {
277 return VolatileLoad(&m_val);
278 }
279
280 //
281 // Loads the value of the volatile variable atomically without erecting the memory barrier.
282 //
283 inline T LoadWithoutBarrier() const
284 {
285 return ((volatile T &)m_val);
286 }
287
288 //
289 // Stores a new value to the volatile variable. See code:VolatileStore for the semantics of this
290 // operation.
291 //
292 inline void Store(const T& val)
293 {
294 VolatileStore(&m_val, val);
295 }
296
297
298 //
299 // Stores a new value to the volatile variable atomically without erecting the memory barrier.
300 //
301 inline void StoreWithoutBarrier(const T& val) const
302 {
303 ((volatile T &)m_val) = val;
304 }
305
306
307 //
308 // Gets a pointer to the volatile variable. This is dangerous, as it permits the variable to be
309 // accessed without using Load and Store, but it is necessary for passing Volatile<T> to APIs like
310 // InterlockedIncrement.
311 //
312 inline volatile T* GetPointer() { return (volatile T*)&m_val; }
313
314
315 //
316 // Gets the raw value of the variable. This is dangerous, as it permits the variable to be
317 // accessed without using Load and Store
318 //
319 inline T& RawValue() { return m_val; }
320
321 //
322 // Allow casts from Volatile<T> to T. Note that this allows implicit casts, so you can
323 // pass a Volatile<T> directly to a method that expects a T.
324 //
325 inline operator T() const
326 {
327 return this->Load();
328 }
329
330 //
331 // Assignment from T
332 //
333 inline Volatile<T>& operator=(T val) {Store(val); return *this;}
334
335 //
336 // Get the address of the volatile variable. This is dangerous, as it allows the value of the
337 // volatile variable to be accessed directly, without going through Load and Store, but it is
338 // necessary for passing Volatile<T> to APIs like InterlockedIncrement. Note that we are returning
339 // a pointer to a volatile T here, so we cannot accidentally pass this pointer to an API that
340 // expects a normal pointer.
341 //
342 inline T volatile * operator&() {return this->GetPointer();}
343 inline T volatile const * operator&() const {return this->GetPointer();}
344
345 //
346 // Comparison operators
347 //
348 template<typename TOther>
349 inline bool operator==(const TOther& other) const {return this->Load() == other;}
350
351 template<typename TOther>
352 inline bool operator!=(const TOther& other) const {return this->Load() != other;}
353
354 //
355 // Miscellaneous operators. Add more as necessary.
356 //
357 inline Volatile<T>& operator+=(T val) {Store(this->Load() + val); return *this;}
358 inline Volatile<T>& operator-=(T val) {Store(this->Load() - val); return *this;}
359 inline Volatile<T>& operator|=(T val) {Store(this->Load() | val); return *this;}
360 inline Volatile<T>& operator&=(T val) {Store(this->Load() & val); return *this;}
361 inline bool operator!() const { return !this->Load();}
362
363 //
364 // Prefix increment
365 //
366 inline Volatile& operator++() {this->Store(this->Load()+1); return *this;}
367
368 //
369 // Postfix increment
370 //
371 inline T operator++(int) {T val = this->Load(); this->Store(val+1); return val;}
372
373 //
374 // Prefix decrement
375 //
376 inline Volatile& operator--() {this->Store(this->Load()-1); return *this;}
377
378 //
379 // Postfix decrement
380 //
381 inline T operator--(int) {T val = this->Load(); this->Store(val-1); return val;}
382};
383
384//
385// A VolatilePtr builds on Volatile<T> by adding operators appropriate to pointers.
386// Wherever you would have used "Foo * volatile", use "VolatilePtr<Foo>" instead.
387//
388// VolatilePtr also allows the substution of other types for the underlying pointer. This
389// allows you to wrap a VolatilePtr around a custom type that looks like a pointer. For example,
390// if what you want is a "volatile DPTR<Foo>", use "VolatilePtr<Foo, DPTR<Foo>>".
391//
392template <typename T, typename P = T*>
393class VolatilePtr : public Volatile<P>
394{
395public:
396 //
397 // Default constructor. Results in an uninitialized pointer!
398 //
399 inline VolatilePtr()
400 {
401 }
402
403 //
404 // Allow assignment from the pointer type.
405 //
406 inline VolatilePtr(P val) : Volatile<P>(val)
407 {
408 }
409
410 //
411 // Copy constructor
412 //
413 inline VolatilePtr(const VolatilePtr& other) : Volatile<P>(other)
414 {
415 }
416
417 //
418 // Cast to the pointer type
419 //
420 inline operator P() const
421 {
422 return (P)this->Load();
423 }
424
425 //
426 // Member access
427 //
428 inline P operator->() const
429 {
430 return (P)this->Load();
431 }
432
433 //
434 // Dereference the pointer
435 //
436 inline T& operator*() const
437 {
438 return *(P)this->Load();
439 }
440
441 //
442 // Access the pointer as an array
443 //
444 template <typename TIndex>
445 inline T& operator[](TIndex index)
446 {
447 return ((P)this->Load())[index];
448 }
449};
450
451#define VOLATILE(T) Volatile<T>
452
453#endif //_VOLATILE_H_
454