1/*
2 Copyright (c) 2005-2019 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15*/
16
17#ifndef __TBB_cache_aligned_allocator_H
18#define __TBB_cache_aligned_allocator_H
19
20#include <new>
21#include "tbb_stddef.h"
22#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
23#include <utility> // std::forward
24#endif
25
26#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
27#include <memory_resource>
28#endif
29
30namespace tbb {
31
32//! @cond INTERNAL
33namespace internal {
34 //! Cache/sector line size.
35 /** @ingroup memory_allocation */
36 size_t __TBB_EXPORTED_FUNC NFS_GetLineSize();
37
38 //! Allocate memory on cache/sector line boundary.
39 /** @ingroup memory_allocation */
40 void* __TBB_EXPORTED_FUNC NFS_Allocate( size_t n_element, size_t element_size, void* hint );
41
42 //! Free memory allocated by NFS_Allocate.
43 /** Freeing a NULL pointer is allowed, but has no effect.
44 @ingroup memory_allocation */
45 void __TBB_EXPORTED_FUNC NFS_Free( void* );
46}
47//! @endcond
48
49#if _MSC_VER && !defined(__INTEL_COMPILER)
50 // Workaround for erroneous "unreferenced parameter" warning in method destroy.
51 #pragma warning (push)
52 #pragma warning (disable: 4100)
53#endif
54
55//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5
56/** The members are ordered the same way they are in section 20.4.1
57 of the ISO C++ standard.
58 @ingroup memory_allocation */
59template<typename T>
60class cache_aligned_allocator {
61public:
62 typedef typename internal::allocator_type<T>::value_type value_type;
63 typedef value_type* pointer;
64 typedef const value_type* const_pointer;
65 typedef value_type& reference;
66 typedef const value_type& const_reference;
67 typedef size_t size_type;
68 typedef ptrdiff_t difference_type;
69 template<typename U> struct rebind {
70 typedef cache_aligned_allocator<U> other;
71 };
72 cache_aligned_allocator() throw() {}
73 cache_aligned_allocator( const cache_aligned_allocator& ) throw() {}
74 template<typename U> cache_aligned_allocator(const cache_aligned_allocator<U>&) throw() {}
75
76 pointer address(reference x) const {return &x;}
77 const_pointer address(const_reference x) const {return &x;}
78
79 //! Allocate space for n objects, starting on a cache/sector line.
80 pointer allocate( size_type n, const void* hint=0 ) {
81 // The "hint" argument is always ignored in NFS_Allocate thus const_cast shouldn't hurt
82 return pointer(internal::NFS_Allocate( n, sizeof(value_type), const_cast<void*>(hint) ));
83 }
84
85 //! Free block of memory that starts on a cache line
86 void deallocate( pointer p, size_type ) {
87 internal::NFS_Free(p);
88 }
89
90 //! Largest value for which method allocate might succeed.
91 size_type max_size() const throw() {
92 return (~size_t(0)-internal::NFS_MaxLineSize)/sizeof(value_type);
93 }
94
95 //! Copy-construct value at location pointed to by p.
96#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
97 template<typename U, typename... Args>
98 void construct(U *p, Args&&... args)
99 { ::new((void *)p) U(std::forward<Args>(args)...); }
100#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
101#if __TBB_CPP11_RVALUE_REF_PRESENT
102 void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));}
103#endif
104 void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);}
105#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
106
107 //! Destroy value at location pointed to by p.
108 void destroy( pointer p ) {p->~value_type();}
109};
110
111#if _MSC_VER && !defined(__INTEL_COMPILER)
112 #pragma warning (pop)
113#endif // warning 4100 is back
114
115//! Analogous to std::allocator<void>, as defined in ISO C++ Standard, Section 20.4.1
116/** @ingroup memory_allocation */
117template<>
118class cache_aligned_allocator<void> {
119public:
120 typedef void* pointer;
121 typedef const void* const_pointer;
122 typedef void value_type;
123 template<typename U> struct rebind {
124 typedef cache_aligned_allocator<U> other;
125 };
126};
127
128template<typename T, typename U>
129inline bool operator==( const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>& ) {return true;}
130
131template<typename T, typename U>
132inline bool operator!=( const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>& ) {return false;}
133
134#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
135
136//! C++17 memory resource wrapper to ensure cache line size alignment
137class cache_aligned_resource : public std::pmr::memory_resource {
138public:
139 cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {}
140 explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {}
141
142 std::pmr::memory_resource* upstream_resource() const {
143 return m_upstream;
144 }
145
146private:
147 //! We don't know what memory resource set. Use padding to guarantee alignment
148 void* do_allocate(size_t bytes, size_t alignment) override {
149 size_t cache_line_alignment = correct_alignment(alignment);
150 uintptr_t base = (uintptr_t)m_upstream->allocate(correct_size(bytes) + cache_line_alignment);
151 __TBB_ASSERT(base != 0, "Upstream resource returned NULL.");
152#if _MSC_VER && !defined(__INTEL_COMPILER)
153 // unary minus operator applied to unsigned type, result still unsigned
154 #pragma warning(push)
155 #pragma warning(disable: 4146 4706)
156#endif
157 // Round up to the next cache line (align the base address)
158 uintptr_t result = (base + cache_line_alignment) & -cache_line_alignment;
159#if _MSC_VER && !defined(__INTEL_COMPILER)
160 #pragma warning(pop)
161#endif
162 // Record where block actually starts.
163 ((uintptr_t*)result)[-1] = base;
164 return (void*)result;
165 }
166
167 void do_deallocate(void* ptr, size_t bytes, size_t alignment) override {
168 if (ptr) {
169 // Recover where block actually starts
170 uintptr_t base = ((uintptr_t*)ptr)[-1];
171 m_upstream->deallocate((void*)base, correct_size(bytes) + correct_alignment(alignment));
172 }
173 }
174
175 bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
176 if (this == &other) { return true; }
177#if __TBB_USE_OPTIONAL_RTTI
178 const cache_aligned_resource* other_res = dynamic_cast<const cache_aligned_resource*>(&other);
179 return other_res && (this->upstream_resource() == other_res->upstream_resource());
180#else
181 return false;
182#endif
183 }
184
185 size_t correct_alignment(size_t alignment) {
186 __TBB_ASSERT(tbb::internal::is_power_of_two(alignment), "Alignment is not a power of 2");
187#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT
188 size_t cache_line_size = std::hardware_destructive_interference_size;
189#else
190 size_t cache_line_size = internal::NFS_GetLineSize();
191#endif
192 return alignment < cache_line_size ? cache_line_size : alignment;
193 }
194
195 size_t correct_size(size_t bytes) {
196 // To handle the case, when small size requested. There could be not
197 // enough space to store the original pointer.
198 return bytes < sizeof(uintptr_t) ? sizeof(uintptr_t) : bytes;
199 }
200
201 std::pmr::memory_resource* m_upstream;
202};
203
204#endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */
205
206} // namespace tbb
207
208#endif /* __TBB_cache_aligned_allocator_H */
209
210