1 | /* |
2 | Copyright (c) 2005-2019 Intel Corporation |
3 | |
4 | Licensed under the Apache License, Version 2.0 (the "License"); |
5 | you may not use this file except in compliance with the License. |
6 | You may obtain a copy of the License at |
7 | |
8 | http://www.apache.org/licenses/LICENSE-2.0 |
9 | |
10 | Unless required by applicable law or agreed to in writing, software |
11 | distributed under the License is distributed on an "AS IS" BASIS, |
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | See the License for the specific language governing permissions and |
14 | limitations under the License. |
15 | */ |
16 | |
17 | #ifndef __TBB_scalable_allocator_H |
18 | #define __TBB_scalable_allocator_H |
19 | /** @file */ |
20 | |
21 | #include <stddef.h> /* Need ptrdiff_t and size_t from here. */ |
22 | #if !_MSC_VER |
23 | #include <stdint.h> /* Need intptr_t from here. */ |
24 | #endif |
25 | |
26 | #if !defined(__cplusplus) && __ICC==1100 |
27 | #pragma warning (push) |
28 | #pragma warning (disable: 991) |
29 | #endif |
30 | |
31 | #ifdef __cplusplus |
32 | extern "C" { |
33 | #endif /* __cplusplus */ |
34 | |
35 | #if _MSC_VER >= 1400 |
36 | #define __TBB_EXPORTED_FUNC __cdecl |
37 | #else |
38 | #define __TBB_EXPORTED_FUNC |
39 | #endif |
40 | |
41 | /** The "malloc" analogue to allocate block of memory of size bytes. |
42 | * @ingroup memory_allocation */ |
43 | void * __TBB_EXPORTED_FUNC scalable_malloc (size_t size); |
44 | |
45 | /** The "free" analogue to discard a previously allocated piece of memory. |
46 | @ingroup memory_allocation */ |
47 | void __TBB_EXPORTED_FUNC scalable_free (void* ptr); |
48 | |
49 | /** The "realloc" analogue complementing scalable_malloc. |
50 | @ingroup memory_allocation */ |
51 | void * __TBB_EXPORTED_FUNC scalable_realloc (void* ptr, size_t size); |
52 | |
53 | /** The "calloc" analogue complementing scalable_malloc. |
54 | @ingroup memory_allocation */ |
55 | void * __TBB_EXPORTED_FUNC scalable_calloc (size_t nobj, size_t size); |
56 | |
57 | /** The "posix_memalign" analogue. |
58 | @ingroup memory_allocation */ |
59 | int __TBB_EXPORTED_FUNC scalable_posix_memalign (void** memptr, size_t alignment, size_t size); |
60 | |
61 | /** The "_aligned_malloc" analogue. |
62 | @ingroup memory_allocation */ |
63 | void * __TBB_EXPORTED_FUNC scalable_aligned_malloc (size_t size, size_t alignment); |
64 | |
65 | /** The "_aligned_realloc" analogue. |
66 | @ingroup memory_allocation */ |
67 | void * __TBB_EXPORTED_FUNC scalable_aligned_realloc (void* ptr, size_t size, size_t alignment); |
68 | |
69 | /** The "_aligned_free" analogue. |
70 | @ingroup memory_allocation */ |
71 | void __TBB_EXPORTED_FUNC scalable_aligned_free (void* ptr); |
72 | |
73 | /** The analogue of _msize/malloc_size/malloc_usable_size. |
74 | Returns the usable size of a memory block previously allocated by scalable_*, |
75 | or 0 (zero) if ptr does not point to such a block. |
76 | @ingroup memory_allocation */ |
77 | size_t __TBB_EXPORTED_FUNC scalable_msize (void* ptr); |
78 | |
79 | /* Results for scalable_allocation_* functions */ |
80 | typedef enum { |
81 | TBBMALLOC_OK, |
82 | TBBMALLOC_INVALID_PARAM, |
83 | TBBMALLOC_UNSUPPORTED, |
84 | TBBMALLOC_NO_MEMORY, |
85 | TBBMALLOC_NO_EFFECT |
86 | } ScalableAllocationResult; |
87 | |
88 | /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. |
89 | scalable_allocation_mode call has priority over environment variable. */ |
90 | typedef enum { |
91 | TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ |
92 | /* deprecated, kept for backward compatibility only */ |
93 | USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, |
94 | /* try to limit memory consumption value (Bytes), clean internal buffers |
95 | if limit is exceeded, but not prevents from requesting memory from OS */ |
96 | TBBMALLOC_SET_SOFT_HEAP_LIMIT, |
97 | /* Lower bound for the size (Bytes), that is interpreted as huge |
98 | * and not released during regular cleanup operations. */ |
99 | TBBMALLOC_SET_HUGE_SIZE_THRESHOLD |
100 | } AllocationModeParam; |
101 | |
102 | /** Set TBB allocator-specific allocation modes. |
103 | @ingroup memory_allocation */ |
104 | int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); |
105 | |
106 | typedef enum { |
107 | /* Clean internal allocator buffers for all threads. |
108 | Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, |
109 | TBBMALLOC_OK if some memory released from buffers. */ |
110 | TBBMALLOC_CLEAN_ALL_BUFFERS, |
111 | /* Clean internal allocator buffer for current thread only. |
112 | Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ |
113 | TBBMALLOC_CLEAN_THREAD_BUFFERS |
114 | } ScalableAllocationCmd; |
115 | |
116 | /** Call TBB allocator-specific commands. |
117 | @ingroup memory_allocation */ |
118 | int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); |
119 | |
120 | #ifdef __cplusplus |
121 | } /* extern "C" */ |
122 | #endif /* __cplusplus */ |
123 | |
124 | #ifdef __cplusplus |
125 | |
126 | //! The namespace rml contains components of low-level memory pool interface. |
127 | namespace rml { |
128 | class MemoryPool; |
129 | |
130 | typedef void *(*rawAllocType)(intptr_t pool_id, size_t &bytes); |
131 | // returns non-zero in case of error |
132 | typedef int (*rawFreeType)(intptr_t pool_id, void* raw_ptr, size_t raw_bytes); |
133 | |
134 | /* |
135 | MemPoolPolicy extension must be compatible with such structure fields layout |
136 | |
137 | struct MemPoolPolicy { |
138 | rawAllocType pAlloc; |
139 | rawFreeType pFree; |
140 | size_t granularity; // granularity of pAlloc allocations |
141 | }; |
142 | */ |
143 | |
144 | struct MemPoolPolicy { |
145 | enum { |
146 | TBBMALLOC_POOL_VERSION = 1 |
147 | }; |
148 | |
149 | rawAllocType pAlloc; |
150 | rawFreeType pFree; |
151 | // granularity of pAlloc allocations. 0 means default used. |
152 | size_t granularity; |
153 | int version; |
154 | // all memory consumed at 1st pAlloc call and never returned, |
155 | // no more pAlloc calls after 1st |
156 | unsigned fixedPool : 1, |
157 | // memory consumed but returned only at pool termination |
158 | keepAllMemory : 1, |
159 | reserved : 30; |
160 | |
161 | MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, |
162 | size_t granularity_ = 0, bool fixedPool_ = false, |
163 | bool keepAllMemory_ = false) : |
164 | pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), |
165 | fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), |
166 | reserved(0) {} |
167 | }; |
168 | |
169 | // enums have same values as appropriate enums from ScalableAllocationResult |
170 | // TODO: use ScalableAllocationResult in pool_create directly |
171 | enum MemPoolError { |
172 | // pool created successfully |
173 | POOL_OK = TBBMALLOC_OK, |
174 | // invalid policy parameters found |
175 | INVALID_POLICY = TBBMALLOC_INVALID_PARAM, |
176 | // requested pool policy is not supported by allocator library |
177 | UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, |
178 | // lack of memory during pool creation |
179 | NO_MEMORY = TBBMALLOC_NO_MEMORY, |
180 | // action takes no effect |
181 | NO_EFFECT = TBBMALLOC_NO_EFFECT |
182 | }; |
183 | |
184 | MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy, |
185 | rml::MemoryPool **pool); |
186 | |
187 | bool pool_destroy(MemoryPool* memPool); |
188 | void *pool_malloc(MemoryPool* memPool, size_t size); |
189 | void *pool_realloc(MemoryPool* memPool, void *object, size_t size); |
190 | void *pool_aligned_malloc(MemoryPool* mPool, size_t size, size_t alignment); |
191 | void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t alignment); |
192 | bool pool_reset(MemoryPool* memPool); |
193 | bool pool_free(MemoryPool *memPool, void *object); |
194 | MemoryPool *pool_identify(void *object); |
195 | size_t pool_msize(MemoryPool *memPool, void *object); |
196 | |
197 | } // namespace rml |
198 | |
199 | #include <new> /* To use new with the placement argument */ |
200 | |
201 | /* Ensure that including this header does not cause implicit linkage with TBB */ |
202 | #ifndef __TBB_NO_IMPLICIT_LINKAGE |
203 | #define __TBB_NO_IMPLICIT_LINKAGE 1 |
204 | #include "tbb_stddef.h" |
205 | #undef __TBB_NO_IMPLICIT_LINKAGE |
206 | #else |
207 | #include "tbb_stddef.h" |
208 | #endif |
209 | |
210 | #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC |
211 | #include <utility> // std::forward |
212 | #endif |
213 | |
214 | #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT |
215 | #include <memory_resource> |
216 | #endif |
217 | |
218 | namespace tbb { |
219 | |
220 | #if _MSC_VER && !defined(__INTEL_COMPILER) |
221 | // Workaround for erroneous "unreferenced parameter" warning in method destroy. |
222 | #pragma warning (push) |
223 | #pragma warning (disable: 4100) |
224 | #endif |
225 | |
226 | //! @cond INTERNAL |
227 | namespace internal { |
228 | |
229 | #if TBB_USE_EXCEPTIONS |
230 | // forward declaration is for inlining prevention |
231 | template<typename E> __TBB_NOINLINE( void throw_exception(const E &e) ); |
232 | #endif |
233 | |
234 | // keep throw in a separate function to prevent code bloat |
235 | template<typename E> |
236 | void throw_exception(const E &e) { |
237 | __TBB_THROW(e); |
238 | } |
239 | |
240 | } // namespace internal |
241 | //! @endcond |
242 | |
243 | //! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 |
244 | /** The members are ordered the same way they are in section 20.4.1 |
245 | of the ISO C++ standard. |
246 | @ingroup memory_allocation */ |
247 | template<typename T> |
248 | class scalable_allocator { |
249 | public: |
250 | typedef typename internal::allocator_type<T>::value_type value_type; |
251 | typedef value_type* pointer; |
252 | typedef const value_type* const_pointer; |
253 | typedef value_type& reference; |
254 | typedef const value_type& const_reference; |
255 | typedef size_t size_type; |
256 | typedef ptrdiff_t difference_type; |
257 | template<class U> struct rebind { |
258 | typedef scalable_allocator<U> other; |
259 | }; |
260 | |
261 | scalable_allocator() throw() {} |
262 | scalable_allocator( const scalable_allocator& ) throw() {} |
263 | template<typename U> scalable_allocator(const scalable_allocator<U>&) throw() {} |
264 | |
265 | pointer address(reference x) const {return &x;} |
266 | const_pointer address(const_reference x) const {return &x;} |
267 | |
268 | //! Allocate space for n objects. |
269 | pointer allocate( size_type n, const void* /*hint*/ =0 ) { |
270 | pointer p = static_cast<pointer>( scalable_malloc( n * sizeof(value_type) ) ); |
271 | if (!p) |
272 | internal::throw_exception(std::bad_alloc()); |
273 | return p; |
274 | } |
275 | |
276 | //! Free previously allocated block of memory |
277 | void deallocate( pointer p, size_type ) { |
278 | scalable_free( p ); |
279 | } |
280 | |
281 | //! Largest value for which method allocate might succeed. |
282 | size_type max_size() const throw() { |
283 | size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type); |
284 | return (absolutemax > 0 ? absolutemax : 1); |
285 | } |
286 | #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC |
287 | template<typename U, typename... Args> |
288 | void construct(U *p, Args&&... args) |
289 | { ::new((void *)p) U(std::forward<Args>(args)...); } |
290 | #else /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ |
291 | #if __TBB_CPP11_RVALUE_REF_PRESENT |
292 | void construct( pointer p, value_type&& value ) { ::new((void*)(p)) value_type( std::move( value ) ); } |
293 | #endif |
294 | void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} |
295 | #endif /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ |
296 | void destroy( pointer p ) {p->~value_type();} |
297 | }; |
298 | |
299 | #if _MSC_VER && !defined(__INTEL_COMPILER) |
300 | #pragma warning (pop) |
301 | #endif /* warning 4100 is back */ |
302 | |
303 | //! Analogous to std::allocator<void>, as defined in ISO C++ Standard, Section 20.4.1 |
304 | /** @ingroup memory_allocation */ |
305 | template<> |
306 | class scalable_allocator<void> { |
307 | public: |
308 | typedef void* pointer; |
309 | typedef const void* const_pointer; |
310 | typedef void value_type; |
311 | template<class U> struct rebind { |
312 | typedef scalable_allocator<U> other; |
313 | }; |
314 | }; |
315 | |
316 | template<typename T, typename U> |
317 | inline bool operator==( const scalable_allocator<T>&, const scalable_allocator<U>& ) {return true;} |
318 | |
319 | template<typename T, typename U> |
320 | inline bool operator!=( const scalable_allocator<T>&, const scalable_allocator<U>& ) {return false;} |
321 | |
322 | #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT |
323 | |
324 | namespace internal { |
325 | |
326 | //! C++17 memory resource implementation for scalable allocator |
327 | //! ISO C++ Section 23.12.2 |
328 | class scalable_resource_impl : public std::pmr::memory_resource { |
329 | private: |
330 | void* do_allocate(size_t bytes, size_t alignment) override { |
331 | void* ptr = scalable_aligned_malloc( bytes, alignment ); |
332 | if (!ptr) { |
333 | throw_exception(std::bad_alloc()); |
334 | } |
335 | return ptr; |
336 | } |
337 | |
338 | void do_deallocate(void* ptr, size_t /*bytes*/, size_t /*alignment*/) override { |
339 | scalable_free(ptr); |
340 | } |
341 | |
342 | //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any |
343 | //! other instance of this class |
344 | bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { |
345 | return this == &other || |
346 | #if __TBB_USE_OPTIONAL_RTTI |
347 | dynamic_cast<const scalable_resource_impl*>(&other) != NULL; |
348 | #else |
349 | false; |
350 | #endif |
351 | } |
352 | }; |
353 | |
354 | } // namespace internal |
355 | |
356 | //! Global scalable allocator memory resource provider |
357 | inline std::pmr::memory_resource* scalable_memory_resource() noexcept { |
358 | static tbb::internal::scalable_resource_impl scalable_res; |
359 | return &scalable_res; |
360 | } |
361 | |
362 | #endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ |
363 | |
364 | } // namespace tbb |
365 | |
366 | #if _MSC_VER |
367 | #if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBBMALLOC_NO_IMPLICIT_LINKAGE) |
368 | #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 |
369 | #endif |
370 | |
371 | #if !__TBBMALLOC_NO_IMPLICIT_LINKAGE |
372 | #ifdef _DEBUG |
373 | #pragma comment(lib, "tbbmalloc_debug.lib") |
374 | #else |
375 | #pragma comment(lib, "tbbmalloc.lib") |
376 | #endif |
377 | #endif |
378 | |
379 | |
380 | #endif |
381 | |
382 | #endif /* __cplusplus */ |
383 | |
384 | #if !defined(__cplusplus) && __ICC==1100 |
385 | #pragma warning (pop) |
386 | #endif /* ICC 11.0 warning 991 is back */ |
387 | |
388 | #endif /* __TBB_scalable_allocator_H */ |
389 | |