1/*
2 Copyright (c) 2005-2019 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15*/
16
17#include "harness_defs.h"
18
19#if __TBB_TEST_SKIP_PIC_MODE || (__TBB_TEST_SKIP_GCC_BUILTINS_MODE && __TBB_TEST_SKIP_ICC_BUILTINS_MODE)
20#include "harness.h"
21int TestMain() {
22 REPORT("Known issue: %s\n",
23 __TBB_TEST_SKIP_PIC_MODE? "PIC mode is not supported" : "Compiler builtins for atomic operations aren't available");
24 return Harness::Skipped;
25}
26#else
27
28// Put tbb/atomic.h first, so if it is missing a prerequisite header, we find out about it.
29// The tests here do *not* test for atomicity, just serial correctness. */
30
31#include "tbb/atomic.h"
32#include "harness_assert.h"
33#include <cstring> // memcmp
34#include "tbb/aligned_space.h"
35#include <new> //for placement new
36
37using std::memcmp;
38
39#if _MSC_VER && !defined(__INTEL_COMPILER)
40 // Unary minus operator applied to unsigned type, result still unsigned
41 // Constant conditional expression
42 #pragma warning( disable: 4127 4310 )
43#endif
44
45#if __TBB_GCC_STRICT_ALIASING_BROKEN
46 #pragma GCC diagnostic ignored "-Wstrict-aliasing"
47#endif
48
49enum LoadStoreExpression {
50 UseOperators,
51 UseImplicitAcqRel,
52 UseExplicitFullyFenced,
53 UseExplicitAcqRel,
54 UseExplicitRelaxed,
55 UseGlobalHelperFullyFenced,
56 UseGlobalHelperAcqRel,
57 UseGlobalHelperRelaxed
58};
59
60//! Structure that holds an atomic<T> and some guard bytes around it.
61template<typename T, LoadStoreExpression E = UseOperators>
62struct TestStruct {
63 typedef unsigned char byte_type;
64 T prefix;
65 tbb::atomic<T> counter;
66 T suffix;
67 TestStruct( T i ) {
68 ASSERT( sizeof(*this)==3*sizeof(T), NULL );
69 for (size_t j = 0; j < sizeof(T); ++j) {
70 reinterpret_cast<byte_type*>(&prefix)[j] = byte_type(0x11*(j+1));
71 reinterpret_cast<byte_type*>(&suffix)[sizeof(T)-j-1] = byte_type(0x11*(j+1));
72 }
73 if ( E == UseOperators )
74 counter = i;
75 else if ( E == UseExplicitRelaxed )
76 counter.template store<tbb::relaxed>(i);
77 else
78 tbb::store<tbb::full_fence>( counter, i );
79 }
80 ~TestStruct() {
81 // Check for writes outside the counter.
82 for (size_t j = 0; j < sizeof(T); ++j) {
83 ASSERT( reinterpret_cast<byte_type*>(&prefix)[j] == byte_type(0x11*(j+1)), NULL );
84 ASSERT( reinterpret_cast<byte_type*>(&suffix)[sizeof(T)-j-1] == byte_type(0x11*(j+1)), NULL );
85 }
86 }
87 static tbb::atomic<T> gCounter;
88};
89
90// A global variable of type tbb::atomic<>
91template<typename T, LoadStoreExpression E> tbb::atomic<T> TestStruct<T, E>::gCounter;
92
93//! Test compare_and_swap template members of class atomic<T> for memory_semantics=M
94template<typename T,tbb::memory_semantics M>
95void TestCompareAndSwapWithExplicitOrdering( T i, T j, T k ) {
96 ASSERT( i!=k && i!=j, "values must be distinct" );
97 // Test compare_and_swap that should fail
98 TestStruct<T> x(i);
99 T old = x.counter.template compare_and_swap<M>( j, k );
100 ASSERT( old==i, NULL );
101 ASSERT( x.counter==i, "old value not retained" );
102 // Test compare and swap that should succeed
103 old = x.counter.template compare_and_swap<M>( j, i );
104 ASSERT( old==i, NULL );
105 ASSERT( x.counter==j, "value not updated?" );
106}
107
108//! i, j, k must be different values
109template<typename T>
110void TestCompareAndSwap( T i, T j, T k ) {
111 ASSERT( i!=k && i!=j, "values must be distinct" );
112 // Test compare_and_swap that should fail
113 TestStruct<T> x(i);
114 T old = x.counter.compare_and_swap( j, k );
115 ASSERT( old==i, NULL );
116 ASSERT( x.counter==i, "old value not retained" );
117 // Test compare and swap that should succeed
118 old = x.counter.compare_and_swap( j, i );
119 ASSERT( old==i, NULL );
120 if( x.counter==i ) {
121 ASSERT( x.counter==j, "value not updated?" );
122 } else {
123 ASSERT( x.counter==j, "value trashed" );
124 }
125 // Check that atomic global variables work
126 TestStruct<T>::gCounter = i;
127 old = TestStruct<T>::gCounter.compare_and_swap( j, i );
128 ASSERT( old==i, NULL );
129 ASSERT( TestStruct<T>::gCounter==j, "value not updated?" );
130 TestCompareAndSwapWithExplicitOrdering<T,tbb::full_fence>(i,j,k);
131 TestCompareAndSwapWithExplicitOrdering<T,tbb::acquire>(i,j,k);
132 TestCompareAndSwapWithExplicitOrdering<T,tbb::release>(i,j,k);
133 TestCompareAndSwapWithExplicitOrdering<T,tbb::relaxed>(i,j,k);
134}
135
136//! memory_semantics variation on TestFetchAndStore
137template<typename T, tbb::memory_semantics M>
138void TestFetchAndStoreWithExplicitOrdering( T i, T j ) {
139 ASSERT( i!=j, "values must be distinct" );
140 TestStruct<T> x(i);
141 T old = x.counter.template fetch_and_store<M>( j );
142 ASSERT( old==i, NULL );
143 ASSERT( x.counter==j, NULL );
144}
145
146//! i and j must be different values
147template<typename T>
148void TestFetchAndStore( T i, T j ) {
149 ASSERT( i!=j, "values must be distinct" );
150 TestStruct<T> x(i);
151 T old = x.counter.fetch_and_store( j );
152 ASSERT( old==i, NULL );
153 ASSERT( x.counter==j, NULL );
154 // Check that atomic global variables work
155 TestStruct<T>::gCounter = i;
156 old = TestStruct<T>::gCounter.fetch_and_store( j );
157 ASSERT( old==i, NULL );
158 ASSERT( TestStruct<T>::gCounter==j, "value not updated?" );
159 TestFetchAndStoreWithExplicitOrdering<T,tbb::full_fence>(i,j);
160 TestFetchAndStoreWithExplicitOrdering<T,tbb::acquire>(i,j);
161 TestFetchAndStoreWithExplicitOrdering<T,tbb::release>(i,j);
162 TestFetchAndStoreWithExplicitOrdering<T,tbb::relaxed>(i,j);
163}
164
165#if _MSC_VER && !defined(__INTEL_COMPILER)
166 // conversion from <bigger integer> to <smaller integer>, possible loss of data
167 // the warning seems a complete nonsense when issued for e.g. short+=short
168 #pragma warning( disable: 4244 )
169#endif
170
171//! Test fetch_and_add members of class atomic<T> for memory_semantics=M
172template<typename T,tbb::memory_semantics M>
173void TestFetchAndAddWithExplicitOrdering( T i ) {
174 TestStruct<T> x(i);
175 T actual;
176 T expected = i;
177
178 // Test fetch_and_add member template
179 for( int j=0; j<10; ++j ) {
180 actual = x.counter.fetch_and_add(j);
181 ASSERT( actual==expected, NULL );
182 expected += j;
183 }
184 for( int j=0; j<10; ++j ) {
185 actual = x.counter.fetch_and_add(-j);
186 ASSERT( actual==expected, NULL );
187 expected -= j;
188 }
189
190 // Test fetch_and_increment member template
191 ASSERT( x.counter==i, NULL );
192 actual = x.counter.template fetch_and_increment<M>();
193 ASSERT( actual==i, NULL );
194 ASSERT( x.counter==T(i+1), NULL );
195
196 // Test fetch_and_decrement member template
197 actual = x.counter.template fetch_and_decrement<M>();
198 ASSERT( actual==T(i+1), NULL );
199 ASSERT( x.counter==i, NULL );
200}
201
202//! Test fetch_and_add and related operators
203template<typename T>
204void TestFetchAndAdd( T i ) {
205 TestStruct<T> x(i);
206 T value;
207 value = ++x.counter;
208 ASSERT( value==T(i+1), NULL );
209 value = x.counter++;
210 ASSERT( value==T(i+1), NULL );
211 value = x.counter--;
212 ASSERT( value==T(i+2), NULL );
213 value = --x.counter;
214 ASSERT( value==i, NULL );
215 T actual;
216 T expected = i;
217 for( int j=-100; j<=100; ++j ) {
218 expected += j;
219 actual = x.counter += j;
220 ASSERT( actual==expected, NULL );
221 }
222 for( int j=-100; j<=100; ++j ) {
223 expected -= j;
224 actual = x.counter -= j;
225 ASSERT( actual==expected, NULL );
226 }
227 // Test fetch_and_increment
228 ASSERT( x.counter==i, NULL );
229 actual = x.counter.fetch_and_increment();
230 ASSERT( actual==i, NULL );
231 ASSERT( x.counter==T(i+1), NULL );
232
233 // Test fetch_and_decrement
234 actual = x.counter.fetch_and_decrement();
235 ASSERT( actual==T(i+1), NULL );
236 ASSERT( x.counter==i, NULL );
237 x.counter = i;
238 ASSERT( x.counter==i, NULL );
239
240 // Check that atomic global variables work
241 TestStruct<T>::gCounter = i;
242 value = TestStruct<T>::gCounter.fetch_and_add( 42 );
243 expected = i+42;
244 ASSERT( value==i, NULL );
245 ASSERT( TestStruct<T>::gCounter==expected, "value not updated?" );
246 TestFetchAndAddWithExplicitOrdering<T,tbb::full_fence>(i);
247 TestFetchAndAddWithExplicitOrdering<T,tbb::acquire>(i);
248 TestFetchAndAddWithExplicitOrdering<T,tbb::release>(i);
249 TestFetchAndAddWithExplicitOrdering<T,tbb::relaxed>(i);
250}
251
252//! A type with unknown size.
253class IncompleteType;
254
255void TestFetchAndAdd( IncompleteType* ) {
256 // There are no fetch-and-add operations on a IncompleteType*.
257}
258void TestFetchAndAdd( void* ) {
259 // There are no fetch-and-add operations on a void*.
260}
261
262void TestFetchAndAdd( bool ) {
263 // There are no fetch-and-add operations on a bool.
264}
265
266template<typename T>
267void TestConst( T i ) {
268 // Try const
269 const TestStruct<T> x(i);
270 ASSERT( memcmp( &i, &x.counter, sizeof(T) )==0, "write to atomic<T> broken?" );
271 ASSERT( x.counter==i, "read of atomic<T> broken?" );
272 const TestStruct<T, UseExplicitRelaxed> y(i);
273 ASSERT( memcmp( &i, &y.counter, sizeof(T) )==0, "relaxed write to atomic<T> broken?" );
274 ASSERT( tbb::load<tbb::relaxed>(y.counter) == i, "relaxed read of atomic<T> broken?" );
275 const TestStruct<T, UseGlobalHelperFullyFenced> z(i);
276 ASSERT( memcmp( &i, &z.counter, sizeof(T) )==0, "sequentially consistent write to atomic<T> broken?" );
277 ASSERT( z.counter.template load<tbb::full_fence>() == i, "sequentially consistent read of atomic<T> broken?" );
278}
279
280#include "harness.h"
281
282#include <sstream>
283
284//TODO: consider moving it to separate file, and unify with one in examples command line interface
285template<typename T>
286std::string to_string(const T& a){
287 std::stringstream str; str <<a;
288 return str.str();
289}
290namespace initialization_tests {
291 template<typename T>
292 struct test_initialization_fixture{
293 typedef tbb::atomic<T> atomic_t;
294 tbb::aligned_space<atomic_t> non_zeroed_storage;
295 enum {fill_value = 0xFF };
296 test_initialization_fixture(){
297 memset(static_cast<void*>(non_zeroed_storage.begin()),fill_value,
298 sizeof(non_zeroed_storage));
299 ASSERT( char(fill_value)==*(reinterpret_cast<char*>(non_zeroed_storage.begin()))
300 ,"failed to fill the storage; memset error?");
301 }
302 //TODO: consider move it to destructor, even in a price of UB
303 void tear_down(){
304 non_zeroed_storage.begin()->~atomic_t();
305 }
306 };
307
308 template<typename T>
309 struct TestValueInitialization : test_initialization_fixture<T>{
310 void operator()(){
311 typedef typename test_initialization_fixture<T>::atomic_t atomic_type;
312 //please note that explicit braces below are needed to get zero initialization.
313 //in C++11, 8.5 Initializers [dcl.init], see paragraphs 10,7,5
314 new (this->non_zeroed_storage.begin()) atomic_type();
315 //TODO: add use of KNOWN_ISSUE macro on SunCC 5.11
316 #if !__SUNPRO_CC || __SUNPRO_CC > 0x5110
317 //TODO: add printing of typename to the assertion
318 ASSERT(char(0)==*(reinterpret_cast<char*>(this->non_zeroed_storage.begin()))
319 ,("value initialization for tbb::atomic should do zero initialization; "
320 "actual value:"+to_string(this->non_zeroed_storage.begin()->load())).c_str());
321 #endif
322 this->tear_down();
323 };
324 };
325
326 template<typename T>
327 struct TestDefaultInitialization : test_initialization_fixture<T>{
328 void operator ()(){
329 typedef typename test_initialization_fixture<T>::atomic_t atomic_type;
330 new (this->non_zeroed_storage.begin()) atomic_type;
331 ASSERT( char(this->fill_value)==*(reinterpret_cast<char*>(this->non_zeroed_storage.begin()))
332 ,"default initialization for atomic should do no initialization");
333 this->tear_down();
334 }
335 };
336# if __TBB_ATOMIC_CTORS
337 template<typename T>
338 struct TestDirectInitialization : test_initialization_fixture<T> {
339 void operator()(T i){
340 typedef typename test_initialization_fixture<T>::atomic_t atomic_type;
341 new (this->non_zeroed_storage.begin()) atomic_type(i);
342 ASSERT(i == this->non_zeroed_storage.begin()->load()
343 ,("tbb::atomic initialization failed; "
344 "value:"+to_string(this->non_zeroed_storage.begin()->load())+
345 "; expected:"+to_string(i)).c_str());
346 this->tear_down();
347 }
348 };
349# endif
350}
351template<typename T>
352void TestValueInitialization(){
353 initialization_tests::TestValueInitialization<T>()();
354}
355template<typename T>
356void TestDefaultInitialization(){
357 initialization_tests::TestDefaultInitialization<T>()();
358}
359
360#if __TBB_ATOMIC_CTORS
361template<typename T>
362void TestDirectInitialization(T i){
363 initialization_tests::TestDirectInitialization<T>()(i);
364}
365//TODO: it would be great to have constructor doing dynamic initialization of local atomic objects implicitly (with zero?),
366// but do no dynamic initializations by default for static objects
367namespace test_constexpr_initialization_helper {
368 struct white_box_ad_hoc_type {
369 int _int;
370 constexpr white_box_ad_hoc_type(int a =0) : _int(a) {};
371 constexpr operator int() const { return _int; }
372 };
373}
374//some white boxing
375namespace tbb { namespace internal {
376 template<>
377 struct atomic_impl<test_constexpr_initialization_helper::white_box_ad_hoc_type>: atomic_impl<int> {
378 atomic_impl() = default;
379 constexpr atomic_impl(test_constexpr_initialization_helper::white_box_ad_hoc_type value):atomic_impl<int>(value){}
380 constexpr operator int() const { return this->my_storage.my_value; }
381 };
382}}
383
384//TODO: make this a parameterized macro
385void TestConstExprInitializationIsTranslationTime(){
386 const char* ct_init_failed_msg = "translation time init failed?";
387 typedef tbb::atomic<int> atomic_t;
388 constexpr atomic_t a(8);
389 ASSERT(a == 8,ct_init_failed_msg);
390
391#if !__TBB_CONSTEXPR_MEMBER_FUNCTION_BROKEN
392 constexpr tbb::atomic<test_constexpr_initialization_helper::white_box_ad_hoc_type> ct_atomic(10);
393 //for some unknown reason clang does not managed to enum syntax
394#if __clang__
395 constexpr int ct_atomic_value_ten = (int)ct_atomic;
396#else
397 enum {ct_atomic_value_ten = (int)ct_atomic};
398#endif
399 __TBB_STATIC_ASSERT(ct_atomic_value_ten == 10, "translation time init failed?");
400 ASSERT(ct_atomic_value_ten == 10,ct_init_failed_msg);
401 int array[ct_atomic_value_ten];
402 ASSERT(Harness::array_length(array) == 10,ct_init_failed_msg);
403#endif //__TBB_CONSTEXPR_MEMBER_FUNCTION_BROKEN
404}
405
406#include <string>
407#include <vector>
408namespace TestConstExprInitializationOfGlobalObjectsHelper{
409 struct static_objects_dynamic_init_order_tester {
410 static int order_hash;
411 template<int N> struct nth {
412 nth(){ order_hash = (order_hash<<4)+N; }
413 };
414
415 static nth<2> second;
416 static nth<3> third;
417 };
418
419 int static_objects_dynamic_init_order_tester::order_hash=1;
420 static_objects_dynamic_init_order_tester::nth<2> static_objects_dynamic_init_order_tester::second;
421 static_objects_dynamic_init_order_tester::nth<3> static_objects_dynamic_init_order_tester::third;
422
423 void TestStaticsDynamicInitializationOrder(){
424 ASSERT(static_objects_dynamic_init_order_tester::order_hash==0x123,"Statics dynamic initialization order is broken? ");
425 }
426
427 template<typename T>
428 void TestStaticInit();
429
430 namespace auto_registered_tests_helper {
431 template<typename T>
432 struct type_name ;
433
434 #define REGISTER_TYPE_NAME(T) \
435 namespace auto_registered_tests_helper{ \
436 template<> \
437 struct type_name<T> { \
438 static const char* name; \
439 }; \
440 const char* type_name<T>::name = #T; \
441 } \
442
443 typedef void (* p_test_function_type)();
444 static std::vector<p_test_function_type> const_expr_tests;
445
446 template <typename T>
447 struct registration{
448 registration(){const_expr_tests.push_back(&TestStaticInit<T>);}
449 };
450 }
451 //according to ISO C++11 [basic.start.init], static data fields of class template have unordered
452 //initialization unless it is an explicit specialization
453 template<typename T>
454 struct tester;
455
456 #define TESTER_SPECIALIZATION(T,ct_value) \
457 template<> \
458 struct tester<T> { \
459 struct static_before; \
460 static bool result; \
461 static static_before static_before_; \
462 static tbb::atomic<T> static_atomic; \
463 \
464 static auto_registered_tests_helper::registration<T> registered; \
465 }; \
466 bool tester<T>::result = false; \
467 \
468 struct tester<T>::static_before { \
469 static_before(){ result = (static_atomic==ct_value); } \
470 } ; \
471 \
472 tester<T>::static_before tester<T>::static_before_; \
473 tbb::atomic<T> tester<T>::static_atomic(ct_value); \
474 \
475 auto_registered_tests_helper::registration<T> tester<T>::registered; \
476 REGISTER_TYPE_NAME(T) \
477
478 template<typename T>
479 void TestStaticInit(){
480 //TODO: add printing of values to the assertion
481 std::string type_name = auto_registered_tests_helper::type_name<T>::name;
482 ASSERT(tester<T>::result,("Static initialization failed for atomic " + type_name).c_str());
483 }
484
485 void CallExprInitTests(){
486# if __TBB_STATIC_CONSTEXPR_INIT_BROKEN
487 REPORT("Known issue: Compile-time initialization fails for static tbb::atomic variables\n");
488# else
489 using namespace auto_registered_tests_helper;
490 for (size_t i =0; i<const_expr_tests.size(); ++i){
491 (*const_expr_tests[i])();
492 }
493 REMARK("ran %d constexpr static init test \n",const_expr_tests.size());
494# endif
495 }
496
497 //TODO: unify somehow list of tested types with one in TestMain
498 //TODO: add specializations for:
499 //T,T(-T(1)
500 //T,1
501# if __TBB_64BIT_ATOMICS
502 TESTER_SPECIALIZATION(long long,8LL)
503 TESTER_SPECIALIZATION(unsigned long long,8ULL)
504# endif
505 TESTER_SPECIALIZATION(unsigned long,8UL)
506 TESTER_SPECIALIZATION(long,8L)
507 TESTER_SPECIALIZATION(unsigned int,8U)
508 TESTER_SPECIALIZATION(int,8)
509 TESTER_SPECIALIZATION(unsigned short,8)
510 TESTER_SPECIALIZATION(short,8)
511 TESTER_SPECIALIZATION(unsigned char,8)
512 TESTER_SPECIALIZATION(signed char,8)
513 TESTER_SPECIALIZATION(char,8)
514 TESTER_SPECIALIZATION(wchar_t,8)
515
516 int dummy;
517 TESTER_SPECIALIZATION(void*,&dummy);
518 TESTER_SPECIALIZATION(bool,false);
519 //TODO: add test for constexpt initialization of floating types
520 //for some unknown reasons 0.1 becomes 0.10000001 and equality comparison fails
521 enum written_number_enum{one=2,two};
522 TESTER_SPECIALIZATION(written_number_enum,one);
523 //TODO: add test for ArrayElement<> as in TestMain
524}
525
526void TestConstExprInitializationOfGlobalObjects(){
527 //first assert that assumption the test based on are correct
528 TestConstExprInitializationOfGlobalObjectsHelper::TestStaticsDynamicInitializationOrder();
529 TestConstExprInitializationOfGlobalObjectsHelper::CallExprInitTests();
530}
531#endif //__TBB_ATOMIC_CTORS
532template<typename T>
533void TestOperations( T i, T j, T k ) {
534 TestValueInitialization<T>();
535 TestDefaultInitialization<T>();
536# if __TBB_ATOMIC_CTORS
537 TestConstExprInitializationIsTranslationTime();
538 TestDirectInitialization<T>(i);
539 TestDirectInitialization<T>(j);
540 TestDirectInitialization<T>(k);
541# endif
542 TestConst(i);
543 TestCompareAndSwap(i,j,k);
544 TestFetchAndStore(i,k); // Pass i,k instead of i,j, because callee requires two distinct values.
545}
546
547template<typename T>
548void TestParallel( const char* name );
549
550bool ParallelError;
551
552template<typename T>
553struct AlignmentChecker {
554 char c;
555 tbb::atomic<T> i;
556};
557
558//TODO: candidate for test_compiler?
559template<typename T>
560void TestAlignment( const char* name ) {
561 AlignmentChecker<T> ac;
562 tbb::atomic<T> x;
563 x = T(0);
564 bool is_stack_variable_aligned = tbb::internal::is_aligned(&x,sizeof(T));
565 bool is_member_variable_aligned = tbb::internal::is_aligned(&ac.i,sizeof(T));
566 bool is_struct_size_correct = (sizeof(AlignmentChecker<T>)==2*sizeof(tbb::atomic<T>));
567 bool known_issue_condition = __TBB_FORCE_64BIT_ALIGNMENT_BROKEN && ( sizeof(T)==8);
568 //TODO: replace these ifs with KNOWN_ISSUE macro when it available
569 if (!is_stack_variable_aligned){
570 std::string msg = "Compiler failed to properly align local atomic variable?; size:"+to_string(sizeof(T)) + " type: "
571 +to_string(name) + " location:" + to_string(&x) +"\n";
572 if (known_issue_condition) {
573 REPORT(("Known issue: "+ msg).c_str());
574 }else{
575 ASSERT(false,msg.c_str());
576 }
577 }
578 if (!is_member_variable_aligned){
579 std::string msg = "Compiler failed to properly align atomic member variable?; size:"+to_string(sizeof(T)) + " type: "
580 +to_string(name) + " location:" + to_string(&ac.i) +"\n";
581 if (known_issue_condition) {
582 REPORT(("Known issue: "+ msg).c_str());
583 }else{
584 ASSERT(false,msg.c_str());
585 }
586 }
587 if (!is_struct_size_correct){
588 std::string msg = "Compiler failed to properly add padding to structure with atomic member variable?; Structure size:"+to_string(sizeof(AlignmentChecker<T>))
589 + " atomic size:"+to_string(sizeof(tbb::atomic<T>)) + " type: " + to_string(name) +"\n";
590 if (known_issue_condition) {
591 REPORT(("Known issue: "+ msg).c_str());
592 }else{
593 ASSERT(false,msg.c_str());
594 }
595 }
596
597 AlignmentChecker<T> array[5];
598 for( int k=0; k<5; ++k ) {
599 bool is_member_variable_in_array_aligned = tbb::internal::is_aligned(&array[k].i,sizeof(T));
600 if (!is_member_variable_in_array_aligned) {
601 std::string msg = "Compiler failed to properly align atomic member variable inside an array?; size:"+to_string(sizeof(T)) + " type:"+to_string(name)
602 + " location:" + to_string(&array[k].i) + "\n";
603 if (known_issue_condition){
604 REPORT(("Known issue: "+ msg).c_str());
605 }else{
606 ASSERT(false,msg.c_str());
607 }
608 }
609 }
610}
611
612#if _MSC_VER && !defined(__INTEL_COMPILER)
613 #pragma warning( disable: 4146 ) // unary minus operator applied to unsigned type, result still unsigned
614 #pragma warning( disable: 4334 ) // result of 32-bit shift implicitly converted to 64 bits
615#endif
616
617/** T is an integral type. */
618template<typename T>
619void TestAtomicInteger( const char* name ) {
620 REMARK("testing atomic<%s> (size=%d)\n",name,sizeof(tbb::atomic<T>));
621 TestAlignment<T>(name);
622 TestOperations<T>(0L, T(-T(1)), T(1));
623 for( int k=0; k<int(sizeof(long))*8-1; ++k ) {
624 const long p = 1L<<k;
625 TestOperations<T>(T(p), T(~(p)), T(1-(p)));
626 TestOperations<T>(T(-(p)), T(~(-(p))), T(1-(-(p))));
627 TestFetchAndAdd<T>(T(-(p)));
628 }
629 TestParallel<T>( name );
630}
631
632namespace test_indirection_helpers {
633 template<typename T>
634 struct Foo {
635 T x, y, z;
636 };
637}
638
639template<typename T>
640void TestIndirection() {
641 using test_indirection_helpers::Foo;
642 Foo<T> item;
643 tbb::atomic<Foo<T>*> pointer;
644 pointer = &item;
645 for( int k=-10; k<=10; ++k ) {
646 // Test various syntaxes for indirection to fields with non-zero offset.
647 T value1=T(), value2=T();
648 for( size_t j=0; j<sizeof(T); ++j ) {
649 ((char*)&value1)[j] = char(k^j);
650 ((char*)&value2)[j] = char(k^j*j);
651 }
652 pointer->y = value1;
653 (*pointer).z = value2;
654 T result1 = (*pointer).y;
655 T result2 = pointer->z;
656 ASSERT( memcmp(&value1,&result1,sizeof(T))==0, NULL );
657 ASSERT( memcmp(&value2,&result2,sizeof(T))==0, NULL );
658 }
659 #if __TBB_ICC_BUILTIN_ATOMICS_POINTER_ALIASING_BROKEN
660 //prevent ICC compiler from assuming 'item' is unused and reusing it's storage
661 item.x = item.y=item.z;
662 #endif
663}
664
665//! Test atomic<T*>
666template<typename T>
667void TestAtomicPointer() {
668 REMARK("testing atomic pointer (%d)\n",int(sizeof(T)));
669 T array[1000];
670 TestOperations<T*>(&array[500],&array[250],&array[750]);
671 TestFetchAndAdd<T*>(&array[500]);
672 TestIndirection<T>();
673 TestParallel<T*>( "pointer" );
674
675}
676
677//! Test atomic<Ptr> where Ptr is a pointer to a type of unknown size
678template<typename Ptr>
679void TestAtomicPointerToTypeOfUnknownSize( const char* name ) {
680 REMARK("testing atomic<%s>\n",name);
681 char array[1000];
682 TestOperations<Ptr>((Ptr)(void*)&array[500],(Ptr)(void*)&array[250],(Ptr)(void*)&array[750]);
683 TestParallel<Ptr>( name );
684}
685
686void TestAtomicBool() {
687 REMARK("testing atomic<bool>\n");
688 TestOperations<bool>(false,true,true);
689 TestOperations<bool>(true,false,false);
690 TestParallel<bool>( "bool" );
691}
692
693template<typename EnumType>
694struct HasImplicitConversionToInt {
695 typedef bool yes;
696 typedef int no;
697 __TBB_STATIC_ASSERT( sizeof(yes) != sizeof(no), "The helper needs two types of different sizes to work." );
698
699 static yes detect( int );
700 static no detect( ... );
701
702 enum { value = (sizeof(yes) == sizeof(detect( EnumType() ))) };
703};
704
705enum Color {Red=0,Green=1,Blue=-1};
706
707void TestAtomicEnum() {
708 REMARK("testing atomic<Color>\n");
709 TestOperations<Color>(Red,Green,Blue);
710 TestParallel<Color>( "Color" );
711 __TBB_STATIC_ASSERT( HasImplicitConversionToInt< tbb::atomic<Color> >::value, "The implicit conversion is expected." );
712}
713
714#if __TBB_SCOPED_ENUM_PRESENT
715enum class ScopedColor1 {ScopedRed,ScopedGreen,ScopedBlue=-1};
716// TODO: extend the test to cover 2 byte scoped enum as well
717#if __TBB_ICC_SCOPED_ENUM_WITH_UNDERLYING_TYPE_NEGATIVE_VALUE_BROKEN
718enum class ScopedColor2 : signed char {ScopedZero, ScopedOne,ScopedRed=42,ScopedGreen=-1,ScopedBlue=127};
719#else
720enum class ScopedColor2 : signed char {ScopedZero, ScopedOne,ScopedRed=-128,ScopedGreen=-1,ScopedBlue=127};
721#endif
722
723// TODO: replace the hack of getting symbolic enum name with a better implementation
724std::string enum_strings[] = {"ScopedZero","ScopedOne","ScopedRed","ScopedGreen","ScopedBlue"};
725template<>
726std::string to_string<ScopedColor1>(const ScopedColor1& a){
727 return enum_strings[a==ScopedColor1::ScopedBlue? 4 : (int)a+2];
728}
729template<>
730std::string to_string<ScopedColor2>(const ScopedColor2& a){
731 return enum_strings[a==ScopedColor2::ScopedRed? 2 :
732 a==ScopedColor2::ScopedGreen? 3 : a==ScopedColor2::ScopedBlue? 4 : (int)a ];
733}
734
735void TestAtomicScopedEnum() {
736 REMARK("testing atomic<ScopedColor>\n");
737 TestOperations<ScopedColor1>(ScopedColor1::ScopedRed,ScopedColor1::ScopedGreen,ScopedColor1::ScopedBlue);
738 TestParallel<ScopedColor1>( "ScopedColor1" );
739#if __TBB_ICC_SCOPED_ENUM_WITH_UNDERLYING_TYPE_ATOMIC_LOAD_BROKEN
740 REPORT("Known issue: the operation tests for a scoped enum with a specified underlying type are skipped.\n");
741#else
742 TestOperations<ScopedColor2>(ScopedColor2::ScopedRed,ScopedColor2::ScopedGreen,ScopedColor2::ScopedBlue);
743 TestParallel<ScopedColor2>( "ScopedColor2" );
744#endif
745 __TBB_STATIC_ASSERT( !HasImplicitConversionToInt< tbb::atomic<ScopedColor1> >::value, "The implicit conversion is not expected." );
746 __TBB_STATIC_ASSERT( !HasImplicitConversionToInt< tbb::atomic<ScopedColor1> >::value, "The implicit conversion is not expected." );
747 __TBB_STATIC_ASSERT( sizeof(tbb::atomic<ScopedColor1>) == sizeof(ScopedColor1), "tbb::atomic instantiated with scoped enum should have the same size as scoped enum." );
748 __TBB_STATIC_ASSERT( sizeof(tbb::atomic<ScopedColor2>) == sizeof(ScopedColor2), "tbb::atomic instantiated with scoped enum should have the same size as scoped enum." );
749}
750#endif /* __TBB_SCOPED_ENUM_PRESENT */
751
752template<typename T>
753void TestAtomicFloat( const char* name ) {
754 REMARK("testing atomic<%s>\n", name );
755 TestAlignment<T>(name);
756 TestOperations<T>(0.5,3.25,10.75);
757 TestParallel<T>( name );
758}
759
760#define __TBB_TEST_GENERIC_PART_WORD_CAS (__TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED)
761#if __TBB_TEST_GENERIC_PART_WORD_CAS
762void TestEndianness() {
763 // Test for pure endianness (assumed by simpler probe in __TBB_MaskedCompareAndSwap()).
764 bool is_big_endian = true, is_little_endian = true;
765 const tbb::internal::uint32_t probe = 0x03020100;
766 ASSERT (tbb::internal::is_aligned(&probe,4), NULL);
767 for( const char *pc_begin = reinterpret_cast<const char*>(&probe)
768 , *pc = pc_begin, *pc_end = pc_begin + sizeof(probe)
769 ; pc != pc_end; ++pc) {
770 if (*pc != pc_end-1-pc) is_big_endian = false;
771 if (*pc != pc-pc_begin) is_little_endian = false;
772 }
773 ASSERT (!is_big_endian || !is_little_endian, NULL);
774 #if __TBB_ENDIANNESS==__TBB_ENDIAN_DETECT
775 ASSERT (is_big_endian || is_little_endian, "__TBB_ENDIANNESS should be set to __TBB_ENDIAN_UNSUPPORTED");
776 #elif __TBB_ENDIANNESS==__TBB_ENDIAN_BIG
777 ASSERT (is_big_endian, "__TBB_ENDIANNESS should NOT be set to __TBB_ENDIAN_BIG");
778 #elif __TBB_ENDIANNESS==__TBB_ENDIAN_LITTLE
779 ASSERT (is_little_endian, "__TBB_ENDIANNESS should NOT be set to __TBB_ENDIAN_LITTLE");
780 #elif __TBB_ENDIANNESS==__TBB_ENDIAN_UNSUPPORTED
781 #error Generic implementation of part-word CAS may not be used: unsupported endianness
782 #else
783 #error Unexpected value of __TBB_ENDIANNESS
784 #endif
785}
786
787namespace masked_cas_helpers {
788 const int numMaskedOperations = 100000;
789 const int testSpaceSize = 8;
790 int prime[testSpaceSize] = {3,5,7,11,13,17,19,23};
791
792 template<typename T>
793 class TestMaskedCAS_Body: NoAssign {
794 T* test_space_uncontended;
795 T* test_space_contended;
796 public:
797 TestMaskedCAS_Body( T* _space1, T* _space2 ) : test_space_uncontended(_space1), test_space_contended(_space2) {}
798 void operator()( int my_idx ) const {
799 using tbb::internal::__TBB_MaskedCompareAndSwap;
800 const volatile T my_prime = T(prime[my_idx]); // 'volatile' prevents erroneous optimizations by SunCC
801 T* const my_ptr = test_space_uncontended+my_idx;
802 T old_value=0;
803 for( int i=0; i<numMaskedOperations; ++i, old_value+=my_prime ){
804 T result;
805 // Test uncontended case
806 T new_value = old_value + my_prime;
807 // The following CAS should always fail
808 result = __TBB_MaskedCompareAndSwap<T>(my_ptr,new_value,old_value-1);
809 ASSERT(result!=old_value-1, "masked CAS succeeded while it should fail");
810 ASSERT(result==*my_ptr, "masked CAS result mismatch with real value");
811 // The following one should succeed
812 result = __TBB_MaskedCompareAndSwap<T>(my_ptr,new_value,old_value);
813 ASSERT(result==old_value && *my_ptr==new_value, "masked CAS failed while it should succeed");
814 // The following one should fail again
815 result = __TBB_MaskedCompareAndSwap<T>(my_ptr,new_value,old_value);
816 ASSERT(result!=old_value, "masked CAS succeeded while it should fail");
817 ASSERT(result==*my_ptr, "masked CAS result mismatch with real value");
818 // Test contended case
819 for( int j=0; j<testSpaceSize; ++j ){
820 // try adding my_prime until success
821 T value;
822 do {
823 value = test_space_contended[j];
824 result = __TBB_MaskedCompareAndSwap<T>(test_space_contended+j,value+my_prime,value);
825 } while( result!=value );
826 }
827 }
828 }
829 };
830
831 template<typename T>
832 struct intptr_as_array_of
833 {
834 static const int how_many_Ts = sizeof(intptr_t)/sizeof(T);
835 union {
836 intptr_t result;
837 T space[ how_many_Ts ];
838 };
839 };
840
841 template<typename T>
842 intptr_t getCorrectUncontendedValue(int slot_idx) {
843 intptr_as_array_of<T> slot;
844 slot.result = 0;
845 for( int i=0; i<slot.how_many_Ts; ++i ) {
846 const T my_prime = T(prime[slot_idx*slot.how_many_Ts + i]);
847 for( int j=0; j<numMaskedOperations; ++j )
848 slot.space[i] += my_prime;
849 }
850 return slot.result;
851 }
852
853 template<typename T>
854 intptr_t getCorrectContendedValue() {
855 intptr_as_array_of<T> slot;
856 slot.result = 0;
857 for( int i=0; i<slot.how_many_Ts; ++i )
858 for( int primes=0; primes<testSpaceSize; ++primes )
859 for( int j=0; j<numMaskedOperations; ++j )
860 slot.space[i] += prime[primes];
861 return slot.result;
862 }
863} // namespace masked_cas_helpers
864
865template<typename T>
866void TestMaskedCAS() {
867 using namespace masked_cas_helpers;
868 REMARK("testing masked CAS<%d>\n",int(sizeof(T)));
869
870 const int num_slots = sizeof(T)*testSpaceSize/sizeof(intptr_t);
871 intptr_t arr1[num_slots+2]; // two more "canary" slots at boundaries
872 intptr_t arr2[num_slots+2];
873 for(int i=0; i<num_slots+2; ++i)
874 arr2[i] = arr1[i] = 0;
875 T* test_space_uncontended = (T*)(arr1+1);
876 T* test_space_contended = (T*)(arr2+1);
877
878 NativeParallelFor( testSpaceSize, TestMaskedCAS_Body<T>(test_space_uncontended, test_space_contended) );
879
880 ASSERT( arr1[0]==0 && arr1[num_slots+1]==0 && arr2[0]==0 && arr2[num_slots+1]==0 , "adjacent memory was overwritten" );
881 const intptr_t correctContendedValue = getCorrectContendedValue<T>();
882 for(int i=0; i<num_slots; ++i) {
883 ASSERT( arr1[i+1]==getCorrectUncontendedValue<T>(i), "unexpected value in an uncontended slot" );
884 ASSERT( arr2[i+1]==correctContendedValue, "unexpected value in a contended slot" );
885 }
886}
887#endif // __TBB_TEST_GENERIC_PART_WORD_CAS
888
889template <typename T>
890class TestRelaxedLoadStorePlainBody {
891 static T s_turn,
892 s_ready;
893
894public:
895 static unsigned s_count1,
896 s_count2;
897
898 void operator() ( int id ) const {
899 using tbb::internal::__TBB_load_relaxed;
900 using tbb::internal::__TBB_store_relaxed;
901
902 if ( id == 0 ) {
903 while ( !__TBB_load_relaxed(s_turn) ) {
904 ++s_count1;
905 __TBB_store_relaxed(s_ready, 1);
906 }
907 }
908 else {
909 while ( !__TBB_load_relaxed(s_ready) ) {
910 ++s_count2;
911 continue;
912 }
913 __TBB_store_relaxed(s_turn, 1);
914 }
915 }
916}; // class TestRelaxedLoadStorePlainBody<T>
917
918template <typename T> T TestRelaxedLoadStorePlainBody<T>::s_turn = 0;
919template <typename T> T TestRelaxedLoadStorePlainBody<T>::s_ready = 0;
920template <typename T> unsigned TestRelaxedLoadStorePlainBody<T>::s_count1 = 0;
921template <typename T> unsigned TestRelaxedLoadStorePlainBody<T>::s_count2 = 0;
922
923template <typename T>
924class TestRelaxedLoadStoreAtomicBody {
925 static tbb::atomic<T> s_turn,
926 s_ready;
927
928public:
929 static unsigned s_count1,
930 s_count2;
931
932 void operator() ( int id ) const {
933 if ( id == 0 ) {
934 while ( s_turn.template load<tbb::relaxed>() == 0 ) {
935 ++s_count1;
936 s_ready.template store<tbb::relaxed>(1);
937 }
938 }
939 else {
940 while ( s_ready.template load<tbb::relaxed>() == 0 ) {
941 ++s_count2;
942 continue;
943 }
944 s_turn.template store<tbb::relaxed>(1);
945 }
946 }
947}; // class TestRelaxedLoadStoreAtomicBody<T>
948
949template <typename T> tbb::atomic<T> TestRelaxedLoadStoreAtomicBody<T>::s_turn;
950template <typename T> tbb::atomic<T> TestRelaxedLoadStoreAtomicBody<T>::s_ready;
951template <typename T> unsigned TestRelaxedLoadStoreAtomicBody<T>::s_count1 = 0;
952template <typename T> unsigned TestRelaxedLoadStoreAtomicBody<T>::s_count2 = 0;
953
954template <typename T>
955void TestRegisterPromotionSuppression () {
956 REMARK("testing register promotion suppression (size=%d)\n", (int)sizeof(T));
957 NativeParallelFor( 2, TestRelaxedLoadStorePlainBody<T>() );
958 NativeParallelFor( 2, TestRelaxedLoadStoreAtomicBody<T>() );
959}
960
961template<unsigned N>
962class ArrayElement {
963 char item[N];
964};
965
966#include "harness_barrier.h"
967namespace bit_operation_test_suite{
968 struct fixture : NoAssign{
969 static const uintptr_t zero = 0;
970 const uintptr_t random_value ;
971 const uintptr_t inverted_random_value ;
972 fixture():
973 random_value (tbb::internal::select_size_t_constant<0x9E3779B9,0x9E3779B97F4A7C15ULL>::value),
974 inverted_random_value ( ~random_value)
975 {}
976 };
977
978 struct TestAtomicORSerially : fixture {
979 void operator()(){
980 //these additional variable are needed to get more meaningful expression in the assert
981 uintptr_t initial_value = zero;
982 uintptr_t atomic_or_result = initial_value;
983 uintptr_t atomic_or_operand = random_value;
984
985 __TBB_AtomicOR(&atomic_or_result,atomic_or_operand);
986
987 ASSERT(atomic_or_result == (initial_value | atomic_or_operand),"AtomicOR should do the OR operation");
988 }
989 };
990 struct TestAtomicANDSerially : fixture {
991 void operator()(){
992 //these additional variable are needed to get more meaningful expression in the assert
993 uintptr_t initial_value = inverted_random_value;
994 uintptr_t atomic_and_result = initial_value;
995 uintptr_t atomic_and_operand = random_value;
996
997 __TBB_AtomicAND(&atomic_and_result,atomic_and_operand);
998
999 ASSERT(atomic_and_result == (initial_value & atomic_and_operand),"AtomicAND should do the AND operation");
1000 }
1001 };
1002
1003 struct TestAtomicORandANDConcurrently : fixture {
1004 static const uintptr_t bit_per_word = sizeof(uintptr_t) * 8;
1005 static const uintptr_t threads_number = bit_per_word;
1006 Harness::SpinBarrier m_barrier;
1007 uintptr_t bitmap;
1008 TestAtomicORandANDConcurrently():bitmap(zero) {}
1009
1010 struct thread_body{
1011 TestAtomicORandANDConcurrently* test;
1012 thread_body(TestAtomicORandANDConcurrently* the_test) : test(the_test) {}
1013 void operator()(int thread_index)const{
1014 const uintptr_t single_bit_mask = ((uintptr_t)1u) << (thread_index % bit_per_word);
1015 test->m_barrier.wait();
1016 static const char* error_msg = "AtomicOR and AtomicAND should be atomic";
1017 for (uintptr_t attempts=0; attempts<1000; attempts++ ){
1018 //Set and clear designated bits in a word.
1019 __TBB_AtomicOR(&test->bitmap,single_bit_mask);
1020 __TBB_Yield();
1021 bool the_bit_is_set_after_set_via_atomic_or = ((__TBB_load_with_acquire(test->bitmap) & single_bit_mask )== single_bit_mask);
1022 ASSERT(the_bit_is_set_after_set_via_atomic_or,error_msg);
1023
1024 __TBB_AtomicAND(&test->bitmap,~single_bit_mask);
1025 __TBB_Yield();
1026 bool the_bit_is_clear_after_clear_via_atomic_and = ((__TBB_load_with_acquire(test->bitmap) & single_bit_mask )== zero);
1027 ASSERT(the_bit_is_clear_after_clear_via_atomic_and,error_msg);
1028 }
1029 }
1030 };
1031 void operator()(){
1032 m_barrier.initialize(threads_number);
1033 NativeParallelFor(threads_number,thread_body(this));
1034 }
1035 };
1036}
1037void TestBitOperations(){
1038 using namespace bit_operation_test_suite;
1039 TestAtomicORSerially()();
1040 TestAtomicANDSerially()();
1041 TestAtomicORandANDConcurrently()();
1042}
1043
1044int TestMain () {
1045# if __TBB_ATOMIC_CTORS
1046 TestConstExprInitializationOfGlobalObjects();
1047# endif //__TBB_ATOMIC_CTORS
1048# if __TBB_64BIT_ATOMICS && !__TBB_CAS_8_CODEGEN_BROKEN
1049 TestAtomicInteger<unsigned long long>("unsigned long long");
1050 TestAtomicInteger<long long>("long long");
1051# elif __TBB_CAS_8_CODEGEN_BROKEN
1052 REPORT("Known issue: compiler generates incorrect code for 64-bit atomics on this configuration\n");
1053# else
1054 REPORT("Known issue: 64-bit atomics are not supported\n");
1055 ASSERT(sizeof(long long)==8, "type long long is not 64 bits");
1056# endif
1057 TestAtomicInteger<unsigned long>("unsigned long");
1058 TestAtomicInteger<long>("long");
1059 TestAtomicInteger<unsigned int>("unsigned int");
1060 TestAtomicInteger<int>("int");
1061 TestAtomicInteger<unsigned short>("unsigned short");
1062 TestAtomicInteger<short>("short");
1063 TestAtomicInteger<signed char>("signed char");
1064 TestAtomicInteger<unsigned char>("unsigned char");
1065 TestAtomicInteger<char>("char");
1066 TestAtomicInteger<wchar_t>("wchar_t");
1067 TestAtomicInteger<size_t>("size_t");
1068 TestAtomicInteger<ptrdiff_t>("ptrdiff_t");
1069 TestAtomicPointer<ArrayElement<1> >();
1070 TestAtomicPointer<ArrayElement<2> >();
1071 TestAtomicPointer<ArrayElement<3> >();
1072 TestAtomicPointer<ArrayElement<4> >();
1073 TestAtomicPointer<ArrayElement<5> >();
1074 TestAtomicPointer<ArrayElement<6> >();
1075 TestAtomicPointer<ArrayElement<7> >();
1076 TestAtomicPointer<ArrayElement<8> >();
1077 TestAtomicPointerToTypeOfUnknownSize<IncompleteType*>( "IncompleteType*" );
1078 TestAtomicPointerToTypeOfUnknownSize<void*>( "void*" );
1079 TestAtomicBool();
1080 TestAtomicEnum();
1081# if __TBB_SCOPED_ENUM_PRESENT
1082 TestAtomicScopedEnum();
1083# endif
1084 TestAtomicFloat<float>("float");
1085# if __TBB_64BIT_ATOMICS && !__TBB_CAS_8_CODEGEN_BROKEN
1086 TestAtomicFloat<double>("double");
1087# else
1088 ASSERT(sizeof(double)==8, "type double is not 64 bits");
1089# endif
1090 ASSERT( !ParallelError, NULL );
1091# if __TBB_TEST_GENERIC_PART_WORD_CAS
1092 TestEndianness();
1093 ASSERT (sizeof(short)==2, NULL);
1094 TestMaskedCAS<unsigned short>();
1095 TestMaskedCAS<short>();
1096 TestMaskedCAS<unsigned char>();
1097 TestMaskedCAS<signed char>();
1098 TestMaskedCAS<char>();
1099# elif __TBB_USE_GENERIC_PART_WORD_CAS
1100# error Generic part-word CAS is enabled, but not covered by the test
1101# else
1102 REPORT("Skipping test for generic part-word CAS\n");
1103# endif
1104# if __TBB_64BIT_ATOMICS && !__TBB_CAS_8_CODEGEN_BROKEN
1105 TestRegisterPromotionSuppression<tbb::internal::int64_t>();
1106# endif
1107 TestRegisterPromotionSuppression<tbb::internal::int32_t>();
1108 TestRegisterPromotionSuppression<tbb::internal::int16_t>();
1109 TestRegisterPromotionSuppression<tbb::internal::int8_t>();
1110 TestBitOperations();
1111
1112 return Harness::Done;
1113}
1114
1115template<typename T, bool aligned>
1116class AlignedAtomic: NoAssign {
1117 //tbb::aligned_space can not be used here, because internally it utilize align pragma/attribute,
1118 //which has bugs on 8byte alignment on ia32 on some compilers( see according ****_BROKEN macro)
1119 // Allocate space big enough to always contain sizeof(T)-byte locations that are aligned and misaligned.
1120 char raw_space[2*sizeof(T) -1];
1121public:
1122 tbb::atomic<T>& construct_atomic(){
1123 std::memset(&raw_space[0],0, sizeof(raw_space));
1124 uintptr_t delta = aligned ? 0 : sizeof(T)/2;
1125 size_t index=sizeof(T)-1;
1126 tbb::atomic<T>* y = reinterpret_cast<tbb::atomic<T>*>((reinterpret_cast<uintptr_t>(&raw_space[index+delta])&~index) - delta);
1127 // Assertion checks that y really did end up somewhere inside "raw_space".
1128 ASSERT( raw_space<=reinterpret_cast<char*>(y), "y starts before raw_space" );
1129 ASSERT( reinterpret_cast<char*>(y+1) <= raw_space+sizeof(raw_space), "y starts after raw_space" );
1130 ASSERT( !(aligned ^ tbb::internal::is_aligned(y,sizeof(T))), "y is not aligned as it required" );
1131 return *(new (y) tbb::atomic<T>());
1132 }
1133};
1134
1135template<typename T, bool aligned>
1136struct FlagAndMessage: AlignedAtomic<T,aligned> {
1137 //! 0 if message not set yet, 1 if message is set.
1138 tbb::atomic<T>& flag;
1139 /** Force flag and message to be on distinct cache lines for machines with cache line size <= 4096 bytes */
1140 char pad[4096/sizeof(T)];
1141 //! Non-zero if message is ready
1142 T message;
1143 FlagAndMessage(): flag(FlagAndMessage::construct_atomic()) {
1144 std::memset(pad,0,sizeof(pad));
1145 }
1146};
1147
1148// A special template function used for summation.
1149// Actually it is only necessary because of its specialization for void*
1150template<typename T>
1151T special_sum(intptr_t arg1, intptr_t arg2) {
1152 return (T)((T)arg1 + arg2);
1153}
1154
1155// The specialization for IncompleteType* is required
1156// because pointer arithmetic (+) is impossible with IncompleteType*
1157template<>
1158IncompleteType* special_sum<IncompleteType*>(intptr_t arg1, intptr_t arg2) {
1159 return (IncompleteType*)(arg1 + arg2);
1160}
1161
1162// The specialization for void* is required
1163// because pointer arithmetic (+) is impossible with void*
1164template<>
1165void* special_sum<void*>(intptr_t arg1, intptr_t arg2) {
1166 return (void*)(arg1 + arg2);
1167}
1168
1169// The specialization for bool is required to shut up gratuitous compiler warnings,
1170// because some compilers warn about casting int to bool.
1171template<>
1172bool special_sum<bool>(intptr_t arg1, intptr_t arg2) {
1173 return ((arg1!=0) + arg2)!=0;
1174}
1175
1176#if __TBB_SCOPED_ENUM_PRESENT
1177// The specialization for scoped enumerators is required
1178// because scoped enumerators prohibit implicit conversion to int
1179template<>
1180ScopedColor1 special_sum<ScopedColor1>(intptr_t arg1, intptr_t arg2) {
1181 return (ScopedColor1)(arg1 + arg2);
1182}
1183template<>
1184ScopedColor2 special_sum<ScopedColor2>(intptr_t arg1, intptr_t arg2) {
1185 return (ScopedColor2)(arg1 + arg2);
1186}
1187#endif
1188
1189volatile int One = 1;
1190
1191inline bool IsRelaxed ( LoadStoreExpression e ) {
1192 return e == UseExplicitRelaxed || e == UseGlobalHelperRelaxed;
1193}
1194
1195template <typename T, LoadStoreExpression E>
1196struct LoadStoreTraits;
1197
1198template <typename T>
1199struct LoadStoreTraits<T, UseOperators> {
1200 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = src; }
1201 static void store ( tbb::atomic<T>& dst, const T& src ) { dst = src; }
1202};
1203
1204template <typename T>
1205struct LoadStoreTraits<T, UseImplicitAcqRel> {
1206 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = src.load(); }
1207 static void store ( tbb::atomic<T>& dst, const T& src ) { dst.store(src); }
1208};
1209
1210template <typename T>
1211struct LoadStoreTraits<T, UseExplicitFullyFenced> {
1212 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = src.template load<tbb::full_fence>(); }
1213 static void store ( tbb::atomic<T>& dst, const T& src ) { dst.template store<tbb::full_fence>(src); }
1214};
1215
1216template <typename T>
1217struct LoadStoreTraits<T, UseExplicitAcqRel> {
1218 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = src.template load<tbb::acquire>(); }
1219 static void store ( tbb::atomic<T>& dst, const T& src ) { dst.template store<tbb::release>(src); }
1220};
1221
1222template <typename T>
1223struct LoadStoreTraits<T, UseExplicitRelaxed> {
1224 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = src.template load<tbb::relaxed>(); }
1225 static void store ( tbb::atomic<T>& dst, const T& src ) { dst.template store<tbb::relaxed>(src); }
1226};
1227
1228template <typename T>
1229struct LoadStoreTraits<T, UseGlobalHelperFullyFenced> {
1230 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = tbb::load<tbb::full_fence>(src); }
1231 static void store ( tbb::atomic<T>& dst, const T& src ) { tbb::store<tbb::full_fence>(dst, src); }
1232};
1233
1234template <typename T>
1235struct LoadStoreTraits<T, UseGlobalHelperAcqRel> {
1236 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = tbb::load<tbb::acquire>(src); }
1237 static void store ( tbb::atomic<T>& dst, const T& src ) { tbb::store<tbb::release>(dst, src); }
1238};
1239
1240template <typename T>
1241struct LoadStoreTraits<T, UseGlobalHelperRelaxed> {
1242 static void load ( T& dst, const tbb::atomic<T>& src ) { dst = tbb::load<tbb::relaxed>(src); }
1243 static void store ( tbb::atomic<T>& dst, const T& src ) { tbb::store<tbb::relaxed>(dst, src); }
1244};
1245
1246template<typename T, bool aligned, LoadStoreExpression E>
1247struct HammerLoadAndStoreFence: NoAssign {
1248 typedef FlagAndMessage<T,aligned> fam_type;
1249private:
1250 typedef LoadStoreTraits<T, E> trait;
1251 fam_type* fam;
1252 const int n;
1253 const int p;
1254 const int trial;
1255 const char* name;
1256 mutable T accum;
1257public:
1258 HammerLoadAndStoreFence( fam_type* fam_, int n_, int p_, const char* name_, int trial_ ) : fam(fam_), n(n_), p(p_), trial(trial_), name(name_) {}
1259 void operator()( int k ) const {
1260 int one = One;
1261 fam_type* s = fam+k;
1262 fam_type* s_next = fam + (k+1)%p;
1263 for( int i=0; i<n; ++i ) {
1264 // The inner for loop is a spin-wait loop, which is normally considered very bad style.
1265 // But we must use it here because we are interested in examining subtle hardware effects.
1266 for(unsigned short cnt=1; ; ++cnt) {
1267 if( !(cnt%1024) ) // to help 1-core or oversubscribed systems complete the test, yield every 2^10 iterations
1268 __TBB_Yield();
1269 // Compilers typically generate non-trivial sequence for division by a constant.
1270 // The expression here is dependent on the loop index i, so it cannot be hoisted.
1271 #define COMPLICATED_ZERO (i*(one-1)/100)
1272 // Read flag and then the message
1273 T flag, message;
1274 if( trial&1 ) {
1275 // COMPLICATED_ZERO here tempts compiler to hoist load of message above reading of flag.
1276 trait::load( flag, (s+COMPLICATED_ZERO)->flag );
1277 message = s->message;
1278 } else {
1279 trait::load( flag, s->flag );
1280 message = s->message;
1281 }
1282 if ( flag != T(0) ) {
1283 if( flag!=(T)-1 ) {
1284 REPORT("ERROR: flag!=(T)-1 k=%d i=%d trial=%x type=%s (atomicity problem?)\n", k, i, trial, name );
1285 ParallelError = true;
1286 }
1287 if( !IsRelaxed(E) && message!=(T)-1 ) {
1288 REPORT("ERROR: message!=(T)-1 k=%d i=%d trial=%x type=%s mode=%d (memory fence problem?)\n", k, i, trial, name, E );
1289 ParallelError = true;
1290 }
1291 s->message = T(0);
1292 trait::store( s->flag, T(0) );
1293 // Prevent deadlock possible in relaxed mode because of store(0)
1294 // to the first thread's flag being reordered after the last
1295 // thread's store(-1) into it.
1296 if ( IsRelaxed(E) ) {
1297 while( s_next->flag.template load<tbb::relaxed>() != T(0) )
1298 __TBB_Yield();
1299 }
1300 else
1301 ASSERT( s_next->flag == T(0), NULL );
1302 // Set message and then the flag
1303 if( trial&2 ) {
1304 // COMPLICATED_ZERO here tempts compiler to sink store below setting of flag
1305 s_next->message = special_sum<T>(-1, COMPLICATED_ZERO);
1306 trait::store( s_next->flag, (T)-1 );
1307 } else {
1308 s_next->message = (T)-1;
1309 trait::store( s_next->flag, (T)-1 );
1310 }
1311 break;
1312 } else {
1313 // Force compiler to use message anyway, so it cannot sink read of s->message below the if.
1314 accum = message;
1315 }
1316 }
1317 }
1318 }
1319};
1320
1321//! Test that atomic<T> has acquire semantics for loads and release semantics for stores.
1322/** Test performs round-robin passing of message among p processors,
1323 where p goes from MinThread to MaxThread. */
1324template<typename T, bool aligned, LoadStoreExpression E>
1325void TestLoadAndStoreFences( const char* name ) {
1326 typedef HammerLoadAndStoreFence<T, aligned, E> hammer_load_store_type;
1327 typedef typename hammer_load_store_type::fam_type fam_type;
1328 for( int p=MinThread<2 ? 2 : MinThread; p<=MaxThread; ++p ) {
1329 fam_type * fam = new fam_type[p];
1330 // Each of four trials exercise slightly different expression pattern within the test.
1331 // See occurrences of COMPLICATED_ZERO for details.
1332 for( int trial=0; trial<4; ++trial ) {
1333 fam->message = (T)-1;
1334 fam->flag = (T)-1;
1335 NativeParallelFor( p, hammer_load_store_type( fam, 100, p, name, trial ) );
1336 if ( !IsRelaxed(E) ) {
1337 for( int k=0; k<p; ++k ) {
1338 ASSERT( fam[k].message==(k==0 ? (T)-1 : T(0)), "incomplete round-robin?" );
1339 ASSERT( fam[k].flag==(k==0 ? (T)-1 : T(0)), "incomplete round-robin?" );
1340 }
1341 }
1342 }
1343 delete[] fam;
1344 }
1345}
1346
1347//! Sparse set of values of integral type T.
1348/** Set is designed so that if a value is read or written non-atomically,
1349 the resulting intermediate value is likely to not be a member of the set. */
1350template<typename T>
1351class SparseValueSet {
1352 T factor;
1353public:
1354 SparseValueSet() {
1355 // Compute factor such that:
1356 // 1. It has at least one 1 in most of its bytes.
1357 // 2. The bytes are typically different.
1358 // 3. When multiplied by any value <=127, the product does not overflow.
1359 factor = T(0);
1360 for( unsigned i=0; i<sizeof(T)*8-7; i+=7 )
1361 factor = T(factor | T(1)<<i);
1362 }
1363 //! Get ith member of set
1364 T get( int i ) const {
1365 // Create multiple of factor. The & prevents overflow of the product.
1366 return T((i&0x7F)*factor);
1367 }
1368 //! True if set contains x
1369 bool contains( T x ) const {
1370 // True if
1371 return (x%factor)==0;
1372 }
1373};
1374
1375//! Specialization for pointer types. The pointers are random and should not be dereferenced.
1376template<typename T>
1377class SparseValueSet<T*> {
1378 SparseValueSet<ptrdiff_t> my_set;
1379public:
1380 T* get( int i ) const {return reinterpret_cast<T*>(my_set.get(i));}
1381 bool contains( T* x ) const {return my_set.contains(reinterpret_cast<ptrdiff_t>(x));}
1382};
1383
1384//! Specialization for bool.
1385/** Checking bool for atomic read/write is pointless in practice, because
1386 there is no way to *not* atomically read or write a bool value. */
1387template<>
1388class SparseValueSet<bool> {
1389public:
1390 bool get( int i ) const {return i&1;}
1391 bool contains( bool ) const {return true;}
1392};
1393
1394#if _MSC_VER==1500 && !defined(__INTEL_COMPILER)
1395 // VS2008/VC9 seems to have an issue; limits pull in math.h
1396 #pragma warning( push )
1397 #pragma warning( disable: 4985 )
1398#endif
1399#include <limits> /* Need std::numeric_limits */
1400#if _MSC_VER==1500 && !defined(__INTEL_COMPILER)
1401 #pragma warning( pop )
1402#endif
1403
1404//! Commonality inherited by specializations for floating-point types.
1405template<typename T>
1406class SparseFloatSet: NoAssign {
1407 const T epsilon;
1408public:
1409 SparseFloatSet() : epsilon(std::numeric_limits<T>::epsilon()) {}
1410 T get( int i ) const {
1411 return i==0 ? T(0) : 1/T((i&0x7F)+1);
1412 }
1413 bool contains( T x ) const {
1414 if( x==T(0) ) {
1415 return true;
1416 } else {
1417 int j = int(1/x+T(0.5));
1418 if( 0<j && j<=128 ) {
1419 T error = x*T(j)-T(1);
1420 // In the calculation above, if x was indeed generated by method get, the error should be
1421 // at most epsilon, because x is off by at most 1/2 ulp from its infinitely precise value,
1422 // j is exact, and the multiplication incurs at most another 1/2 ulp of round-off error.
1423 if( -epsilon<=error && error<=epsilon ) {
1424 return true;
1425 } else {
1426 REPORT("Warning: excessive floating-point error encountered j=%d x=%.15g error=%.15g\n",j,x,error);
1427 }
1428 }
1429 return false;
1430 }
1431 };
1432};
1433
1434template<>
1435class SparseValueSet<float>: public SparseFloatSet<float> {};
1436
1437template<>
1438class SparseValueSet<double>: public SparseFloatSet<double> {};
1439
1440#if __TBB_SCOPED_ENUM_PRESENT
1441//! Commonality inherited by specializations for scoped enumerator types.
1442template<typename EnumType>
1443class SparseEnumValueSet {
1444public:
1445 EnumType get( int i ) const {return i%3==0 ? EnumType::ScopedRed : i%3==1 ? EnumType::ScopedGreen : EnumType::ScopedBlue;}
1446 bool contains( EnumType e ) const {return e==EnumType::ScopedRed || e==EnumType::ScopedGreen || e==EnumType::ScopedBlue;}
1447};
1448template<>
1449class SparseValueSet<ScopedColor1> : public SparseEnumValueSet<ScopedColor1> {};
1450template<>
1451class SparseValueSet<ScopedColor2> : public SparseEnumValueSet<ScopedColor2> {};
1452#endif
1453
1454template<typename T, bool aligned>
1455class HammerAssignment: AlignedAtomic<T,aligned> {
1456 tbb::atomic<T>& x;
1457 const char* name;
1458 SparseValueSet<T> set;
1459public:
1460 HammerAssignment(const char* name_ ) : x(HammerAssignment::construct_atomic()), name(name_) {
1461 x = set.get(0);
1462 }
1463 void operator()( int k ) const {
1464 const int n = 1000000;
1465 if( k ) {
1466 tbb::atomic<T> z;
1467 AssertSameType( z=x, z ); // Check that return type from assignment is correct
1468 for( int i=0; i<n; ++i ) {
1469 // Read x atomically into z.
1470 z = x;
1471 if( !set.contains(z) ) {
1472 REPORT("ERROR: assignment of atomic<%s> is not atomic\n", name);
1473 ParallelError = true;
1474 return;
1475 }
1476 }
1477 } else {
1478 tbb::atomic<T> y;
1479 for( int i=0; i<n; ++i ) {
1480 // Get pseudo-random value.
1481 y = set.get(i);
1482 // Write y atomically into x.
1483 x = y;
1484 }
1485 }
1486 }
1487};
1488
1489// Compile-time check that a class method has the required signature.
1490// Intended to check the assignment operator of tbb::atomic.
1491template<typename T> void TestAssignmentSignature( T& (T::*)(const T&) ) {}
1492
1493#if _MSC_VER && !defined(__INTEL_COMPILER)
1494 #pragma warning( disable: 4355 4800 )
1495#endif
1496
1497template<typename T, bool aligned>
1498void TestAssignment( const char* name ) {
1499 TestAssignmentSignature( &tbb::atomic<T>::operator= );
1500 NativeParallelFor( 2, HammerAssignment<T,aligned>(name ) );
1501}
1502
1503template <typename T, bool aligned, LoadStoreExpression E>
1504class DekkerArbitrationBody : NoAssign, Harness::NoAfterlife {
1505 typedef LoadStoreTraits<T, E> trait;
1506
1507 mutable Harness::FastRandom my_rand;
1508 static const unsigned short c_rand_ceil = 10;
1509 mutable AlignedAtomic<T,aligned> s_ready_storage[2];
1510 mutable AlignedAtomic<T,aligned> s_turn_storage;
1511 mutable tbb::atomic<T>* s_ready[2];
1512 tbb::atomic<T>& s_turn;
1513 mutable volatile bool s_inside;
1514
1515public:
1516 void operator() ( int id ) const {
1517 const int me = id;
1518 const T other = (T)(uintptr_t)(1 - id),
1519 cleared = T(0),
1520 signaled = T(1);
1521 for ( int i = 0; i < 100000; ++i ) {
1522 trait::store( *s_ready[me], signaled );
1523 trait::store( s_turn, other );
1524 T r, t;
1525 for ( int j = 0; ; ++j ) {
1526 trait::load(r, *s_ready[(uintptr_t)other]);
1527 trait::load(t, s_turn);
1528 if ( r != signaled || t != other )
1529 break;
1530 __TBB_Pause(1);
1531 if ( j == 2<<12 ) {
1532 j = 0;
1533 __TBB_Yield();
1534 }
1535 }
1536 // Entered critical section
1537 ASSERT( !s_inside, "Peterson lock is broken - some fences are missing" );
1538 s_inside = true;
1539 unsigned short spin = my_rand.get() % c_rand_ceil;
1540 for ( volatile int j = 0; j < spin; ++j )
1541 continue;
1542 s_inside = false;
1543 ASSERT( !s_inside, "Peterson lock is broken - some fences are missing" );
1544 // leaving critical section
1545 trait::store( *s_ready[me], cleared );
1546 spin = my_rand.get() % c_rand_ceil;
1547 for ( volatile int j = 0; j < spin; ++j )
1548 continue;
1549 }
1550 }
1551
1552 DekkerArbitrationBody ()
1553 : my_rand((unsigned)(uintptr_t)this)
1554 , s_turn(s_turn_storage.construct_atomic())
1555 , s_inside (false)
1556 {
1557 //atomics pointed to by s_ready and s_turn will be zeroed by the
1558 //according construct_atomic() calls
1559 s_ready[0] = &s_ready_storage[0].construct_atomic();
1560 s_ready[1] = &s_ready_storage[1].construct_atomic();
1561 }
1562};
1563
1564template <typename T, bool aligned, LoadStoreExpression E>
1565void TestDekkerArbitration () {
1566 NativeParallelFor( 2, DekkerArbitrationBody<T,aligned, E>() );
1567}
1568
1569template<typename T>
1570void TestParallel( const char* name ) {
1571 //TODO: looks like there are no tests for operations other than load/store ?
1572#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN
1573 if (sizeof(T)==8){
1574 TestLoadAndStoreFences<T, false, UseOperators>(name);
1575 TestLoadAndStoreFences<T, false, UseImplicitAcqRel>(name);
1576 TestLoadAndStoreFences<T, false, UseExplicitFullyFenced>(name);
1577 TestLoadAndStoreFences<T, false, UseExplicitAcqRel>(name);
1578 TestLoadAndStoreFences<T, false, UseExplicitRelaxed>(name);
1579 TestLoadAndStoreFences<T, false, UseGlobalHelperFullyFenced>(name);
1580 TestLoadAndStoreFences<T, false, UseGlobalHelperAcqRel>(name);
1581 TestLoadAndStoreFences<T, false, UseGlobalHelperRelaxed>(name);
1582 TestAssignment<T,false>(name);
1583 TestDekkerArbitration<T, false, UseExplicitFullyFenced>();
1584 TestDekkerArbitration<T, false, UseGlobalHelperFullyFenced>();
1585 }
1586#endif
1587
1588 TestLoadAndStoreFences<T, true, UseOperators>(name);
1589 TestLoadAndStoreFences<T, true, UseImplicitAcqRel>(name);
1590 TestLoadAndStoreFences<T, true, UseExplicitFullyFenced>(name);
1591 TestLoadAndStoreFences<T, true, UseExplicitAcqRel>(name);
1592 TestLoadAndStoreFences<T, true, UseExplicitRelaxed>(name);
1593 TestLoadAndStoreFences<T, true, UseGlobalHelperFullyFenced>(name);
1594 TestLoadAndStoreFences<T, true, UseGlobalHelperAcqRel>(name);
1595 TestLoadAndStoreFences<T, true, UseGlobalHelperRelaxed>(name);
1596 TestAssignment<T,true>(name);
1597 TestDekkerArbitration<T, true, UseExplicitFullyFenced>();
1598 TestDekkerArbitration<T, true, UseGlobalHelperFullyFenced>();
1599}
1600
1601#endif // __TBB_TEST_SKIP_PIC_MODE || __TBB_TEST_SKIP_BUILTINS_MODE
1602