1 | /* |
2 | Copyright (c) 2005-2019 Intel Corporation |
3 | |
4 | Licensed under the Apache License, Version 2.0 (the "License"); |
5 | you may not use this file except in compliance with the License. |
6 | You may obtain a copy of the License at |
7 | |
8 | http://www.apache.org/licenses/LICENSE-2.0 |
9 | |
10 | Unless required by applicable law or agreed to in writing, software |
11 | distributed under the License is distributed on an "AS IS" BASIS, |
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | See the License for the specific language governing permissions and |
14 | limitations under the License. |
15 | */ |
16 | |
17 | #ifndef __TBB_cache_aligned_allocator_H |
18 | #define __TBB_cache_aligned_allocator_H |
19 | |
20 | #include <new> |
21 | #include "tbb_stddef.h" |
22 | #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC |
23 | #include <utility> // std::forward |
24 | #endif |
25 | |
26 | #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT |
27 | #include <memory_resource> |
28 | #endif |
29 | |
30 | namespace tbb { |
31 | |
32 | //! @cond INTERNAL |
33 | namespace internal { |
34 | //! Cache/sector line size. |
35 | /** @ingroup memory_allocation */ |
36 | size_t __TBB_EXPORTED_FUNC NFS_GetLineSize(); |
37 | |
38 | //! Allocate memory on cache/sector line boundary. |
39 | /** @ingroup memory_allocation */ |
40 | void* __TBB_EXPORTED_FUNC NFS_Allocate( size_t n_element, size_t element_size, void* hint ); |
41 | |
42 | //! Free memory allocated by NFS_Allocate. |
43 | /** Freeing a NULL pointer is allowed, but has no effect. |
44 | @ingroup memory_allocation */ |
45 | void __TBB_EXPORTED_FUNC NFS_Free( void* ); |
46 | } |
47 | //! @endcond |
48 | |
49 | #if _MSC_VER && !defined(__INTEL_COMPILER) |
50 | // Workaround for erroneous "unreferenced parameter" warning in method destroy. |
51 | #pragma warning (push) |
52 | #pragma warning (disable: 4100) |
53 | #endif |
54 | |
55 | //! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 |
56 | /** The members are ordered the same way they are in section 20.4.1 |
57 | of the ISO C++ standard. |
58 | @ingroup memory_allocation */ |
59 | template<typename T> |
60 | class cache_aligned_allocator { |
61 | public: |
62 | typedef typename internal::allocator_type<T>::value_type value_type; |
63 | typedef value_type* pointer; |
64 | typedef const value_type* const_pointer; |
65 | typedef value_type& reference; |
66 | typedef const value_type& const_reference; |
67 | typedef size_t size_type; |
68 | typedef ptrdiff_t difference_type; |
69 | template<typename U> struct rebind { |
70 | typedef cache_aligned_allocator<U> other; |
71 | }; |
72 | cache_aligned_allocator() throw() {} |
73 | cache_aligned_allocator( const cache_aligned_allocator& ) throw() {} |
74 | template<typename U> cache_aligned_allocator(const cache_aligned_allocator<U>&) throw() {} |
75 | |
76 | pointer address(reference x) const {return &x;} |
77 | const_pointer address(const_reference x) const {return &x;} |
78 | |
79 | //! Allocate space for n objects, starting on a cache/sector line. |
80 | pointer allocate( size_type n, const void* hint=0 ) { |
81 | // The "hint" argument is always ignored in NFS_Allocate thus const_cast shouldn't hurt |
82 | return pointer(internal::NFS_Allocate( n, sizeof(value_type), const_cast<void*>(hint) )); |
83 | } |
84 | |
85 | //! Free block of memory that starts on a cache line |
86 | void deallocate( pointer p, size_type ) { |
87 | internal::NFS_Free(p); |
88 | } |
89 | |
90 | //! Largest value for which method allocate might succeed. |
91 | size_type max_size() const throw() { |
92 | return (~size_t(0)-internal::NFS_MaxLineSize)/sizeof(value_type); |
93 | } |
94 | |
95 | //! Copy-construct value at location pointed to by p. |
96 | #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC |
97 | template<typename U, typename... Args> |
98 | void construct(U *p, Args&&... args) |
99 | { ::new((void *)p) U(std::forward<Args>(args)...); } |
100 | #else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC |
101 | #if __TBB_CPP11_RVALUE_REF_PRESENT |
102 | void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} |
103 | #endif |
104 | void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} |
105 | #endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC |
106 | |
107 | //! Destroy value at location pointed to by p. |
108 | void destroy( pointer p ) {p->~value_type();} |
109 | }; |
110 | |
111 | #if _MSC_VER && !defined(__INTEL_COMPILER) |
112 | #pragma warning (pop) |
113 | #endif // warning 4100 is back |
114 | |
115 | //! Analogous to std::allocator<void>, as defined in ISO C++ Standard, Section 20.4.1 |
116 | /** @ingroup memory_allocation */ |
117 | template<> |
118 | class cache_aligned_allocator<void> { |
119 | public: |
120 | typedef void* pointer; |
121 | typedef const void* const_pointer; |
122 | typedef void value_type; |
123 | template<typename U> struct rebind { |
124 | typedef cache_aligned_allocator<U> other; |
125 | }; |
126 | }; |
127 | |
128 | template<typename T, typename U> |
129 | inline bool operator==( const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>& ) {return true;} |
130 | |
131 | template<typename T, typename U> |
132 | inline bool operator!=( const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>& ) {return false;} |
133 | |
134 | #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT |
135 | |
136 | //! C++17 memory resource wrapper to ensure cache line size alignment |
137 | class cache_aligned_resource : public std::pmr::memory_resource { |
138 | public: |
139 | cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {} |
140 | explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {} |
141 | |
142 | std::pmr::memory_resource* upstream_resource() const { |
143 | return m_upstream; |
144 | } |
145 | |
146 | private: |
147 | //! We don't know what memory resource set. Use padding to guarantee alignment |
148 | void* do_allocate(size_t bytes, size_t alignment) override { |
149 | size_t cache_line_alignment = correct_alignment(alignment); |
150 | uintptr_t base = (uintptr_t)m_upstream->allocate(correct_size(bytes) + cache_line_alignment); |
151 | __TBB_ASSERT(base != 0, "Upstream resource returned NULL." ); |
152 | #if _MSC_VER && !defined(__INTEL_COMPILER) |
153 | // unary minus operator applied to unsigned type, result still unsigned |
154 | #pragma warning(push) |
155 | #pragma warning(disable: 4146 4706) |
156 | #endif |
157 | // Round up to the next cache line (align the base address) |
158 | uintptr_t result = (base + cache_line_alignment) & -cache_line_alignment; |
159 | #if _MSC_VER && !defined(__INTEL_COMPILER) |
160 | #pragma warning(pop) |
161 | #endif |
162 | // Record where block actually starts. |
163 | ((uintptr_t*)result)[-1] = base; |
164 | return (void*)result; |
165 | } |
166 | |
167 | void do_deallocate(void* ptr, size_t bytes, size_t alignment) override { |
168 | if (ptr) { |
169 | // Recover where block actually starts |
170 | uintptr_t base = ((uintptr_t*)ptr)[-1]; |
171 | m_upstream->deallocate((void*)base, correct_size(bytes) + correct_alignment(alignment)); |
172 | } |
173 | } |
174 | |
175 | bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { |
176 | if (this == &other) { return true; } |
177 | #if __TBB_USE_OPTIONAL_RTTI |
178 | const cache_aligned_resource* other_res = dynamic_cast<const cache_aligned_resource*>(&other); |
179 | return other_res && (this->upstream_resource() == other_res->upstream_resource()); |
180 | #else |
181 | return false; |
182 | #endif |
183 | } |
184 | |
185 | size_t correct_alignment(size_t alignment) { |
186 | __TBB_ASSERT(tbb::internal::is_power_of_two(alignment), "Alignment is not a power of 2" ); |
187 | #if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT |
188 | size_t cache_line_size = std::hardware_destructive_interference_size; |
189 | #else |
190 | size_t cache_line_size = internal::NFS_GetLineSize(); |
191 | #endif |
192 | return alignment < cache_line_size ? cache_line_size : alignment; |
193 | } |
194 | |
195 | size_t correct_size(size_t bytes) { |
196 | // To handle the case, when small size requested. There could be not |
197 | // enough space to store the original pointer. |
198 | return bytes < sizeof(uintptr_t) ? sizeof(uintptr_t) : bytes; |
199 | } |
200 | |
201 | std::pmr::memory_resource* m_upstream; |
202 | }; |
203 | |
204 | #endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ |
205 | |
206 | } // namespace tbb |
207 | |
208 | #endif /* __TBB_cache_aligned_allocator_H */ |
209 | |
210 | |