1#include "duckdb/common/allocator.hpp"
2
3#include "duckdb/common/assert.hpp"
4#include "duckdb/common/atomic.hpp"
5#include "duckdb/common/exception.hpp"
6
7#include <cstdint>
8
9#ifdef DUCKDB_DEBUG_ALLOCATION
10#include "duckdb/common/mutex.hpp"
11#include "duckdb/common/pair.hpp"
12#include "duckdb/common/unordered_map.hpp"
13
14#include <execinfo.h>
15#endif
16
17#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(WIN32)
18#include "jemalloc-extension.hpp"
19#endif
20
21namespace duckdb {
22
23AllocatedData::AllocatedData() : allocator(nullptr), pointer(nullptr), allocated_size(0) {
24}
25
26AllocatedData::AllocatedData(Allocator &allocator, data_ptr_t pointer, idx_t allocated_size)
27 : allocator(&allocator), pointer(pointer), allocated_size(allocated_size) {
28 if (!pointer) {
29 throw InternalException("AllocatedData object constructed with nullptr");
30 }
31}
32AllocatedData::~AllocatedData() {
33 Reset();
34}
35
36AllocatedData::AllocatedData(AllocatedData &&other) noexcept
37 : allocator(other.allocator), pointer(nullptr), allocated_size(0) {
38 std::swap(a&: pointer, b&: other.pointer);
39 std::swap(a&: allocated_size, b&: other.allocated_size);
40}
41
42AllocatedData &AllocatedData::operator=(AllocatedData &&other) noexcept {
43 std::swap(a&: allocator, b&: other.allocator);
44 std::swap(a&: pointer, b&: other.pointer);
45 std::swap(a&: allocated_size, b&: other.allocated_size);
46 return *this;
47}
48
49void AllocatedData::Reset() {
50 if (!pointer) {
51 return;
52 }
53 D_ASSERT(allocator);
54 allocator->FreeData(pointer, size: allocated_size);
55 allocated_size = 0;
56 pointer = nullptr;
57}
58
59//===--------------------------------------------------------------------===//
60// Debug Info
61//===--------------------------------------------------------------------===//
62struct AllocatorDebugInfo {
63#ifdef DEBUG
64 AllocatorDebugInfo();
65 ~AllocatorDebugInfo();
66
67 void AllocateData(data_ptr_t pointer, idx_t size);
68 void FreeData(data_ptr_t pointer, idx_t size);
69 void ReallocateData(data_ptr_t pointer, data_ptr_t new_pointer, idx_t old_size, idx_t new_size);
70
71private:
72 //! The number of bytes that are outstanding (i.e. that have been allocated - but not freed)
73 //! Used for debug purposes
74 atomic<idx_t> allocation_count;
75#ifdef DUCKDB_DEBUG_ALLOCATION
76 mutex pointer_lock;
77 //! Set of active outstanding pointers together with stack traces
78 unordered_map<data_ptr_t, pair<idx_t, string>> pointers;
79#endif
80#endif
81};
82
83PrivateAllocatorData::PrivateAllocatorData() {
84}
85
86PrivateAllocatorData::~PrivateAllocatorData() {
87}
88
89//===--------------------------------------------------------------------===//
90// Allocator
91//===--------------------------------------------------------------------===//
92#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(WIN32)
93Allocator::Allocator()
94 : Allocator(JEMallocExtension::Allocate, JEMallocExtension::Free, JEMallocExtension::Reallocate, nullptr) {
95}
96#else
97Allocator::Allocator()
98 : Allocator(Allocator::DefaultAllocate, Allocator::DefaultFree, Allocator::DefaultReallocate, nullptr) {
99}
100#endif
101
102Allocator::Allocator(allocate_function_ptr_t allocate_function_p, free_function_ptr_t free_function_p,
103 reallocate_function_ptr_t reallocate_function_p, unique_ptr<PrivateAllocatorData> private_data_p)
104 : allocate_function(allocate_function_p), free_function(free_function_p),
105 reallocate_function(reallocate_function_p), private_data(std::move(private_data_p)) {
106 D_ASSERT(allocate_function);
107 D_ASSERT(free_function);
108 D_ASSERT(reallocate_function);
109#ifdef DEBUG
110 if (!private_data) {
111 private_data = make_uniq<PrivateAllocatorData>();
112 }
113 private_data->debug_info = make_uniq<AllocatorDebugInfo>();
114#endif
115}
116
117Allocator::~Allocator() {
118}
119
120data_ptr_t Allocator::AllocateData(idx_t size) {
121 D_ASSERT(size > 0);
122 if (size >= MAXIMUM_ALLOC_SIZE) {
123 D_ASSERT(false);
124 throw InternalException("Requested allocation size of %llu is out of range - maximum allocation size is %llu",
125 size, MAXIMUM_ALLOC_SIZE);
126 }
127 auto result = allocate_function(private_data.get(), size);
128#ifdef DEBUG
129 D_ASSERT(private_data);
130 private_data->debug_info->AllocateData(result, size);
131#endif
132 if (!result) {
133 throw OutOfMemoryException("Failed to allocate block of %llu bytes", size);
134 }
135 return result;
136}
137
138void Allocator::FreeData(data_ptr_t pointer, idx_t size) {
139 if (!pointer) {
140 return;
141 }
142 D_ASSERT(size > 0);
143#ifdef DEBUG
144 D_ASSERT(private_data);
145 private_data->debug_info->FreeData(pointer, size);
146#endif
147 free_function(private_data.get(), pointer, size);
148}
149
150data_ptr_t Allocator::ReallocateData(data_ptr_t pointer, idx_t old_size, idx_t size) {
151 if (!pointer) {
152 return nullptr;
153 }
154 if (size >= MAXIMUM_ALLOC_SIZE) {
155 D_ASSERT(false);
156 throw InternalException(
157 "Requested re-allocation size of %llu is out of range - maximum allocation size is %llu", size,
158 MAXIMUM_ALLOC_SIZE);
159 }
160 auto new_pointer = reallocate_function(private_data.get(), pointer, old_size, size);
161#ifdef DEBUG
162 D_ASSERT(private_data);
163 private_data->debug_info->ReallocateData(pointer, new_pointer, old_size, size);
164#endif
165 if (!new_pointer) {
166 throw OutOfMemoryException("Failed to re-allocate block of %llu bytes", size);
167 }
168 return new_pointer;
169}
170
171shared_ptr<Allocator> &Allocator::DefaultAllocatorReference() {
172 static shared_ptr<Allocator> DEFAULT_ALLOCATOR = make_shared<Allocator>();
173 return DEFAULT_ALLOCATOR;
174}
175
176Allocator &Allocator::DefaultAllocator() {
177 return *DefaultAllocatorReference();
178}
179
180//===--------------------------------------------------------------------===//
181// Debug Info (extended)
182//===--------------------------------------------------------------------===//
183#ifdef DEBUG
184AllocatorDebugInfo::AllocatorDebugInfo() {
185 allocation_count = 0;
186}
187AllocatorDebugInfo::~AllocatorDebugInfo() {
188#ifdef DUCKDB_DEBUG_ALLOCATION
189 if (allocation_count != 0) {
190 printf("Outstanding allocations found for Allocator\n");
191 for (auto &entry : pointers) {
192 printf("Allocation of size %llu at address %p\n", entry.second.first, (void *)entry.first);
193 printf("Stack trace:\n%s\n", entry.second.second.c_str());
194 printf("\n");
195 }
196 }
197#endif
198 //! Verify that there is no outstanding memory still associated with the batched allocator
199 //! Only works for access to the batched allocator through the batched allocator interface
200 //! If this assertion triggers, enable DUCKDB_DEBUG_ALLOCATION for more information about the allocations
201 D_ASSERT(allocation_count == 0);
202}
203
204void AllocatorDebugInfo::AllocateData(data_ptr_t pointer, idx_t size) {
205 allocation_count += size;
206#ifdef DUCKDB_DEBUG_ALLOCATION
207 lock_guard<mutex> l(pointer_lock);
208 pointers[pointer] = make_pair(size, Exception::GetStackTrace());
209#endif
210}
211
212void AllocatorDebugInfo::FreeData(data_ptr_t pointer, idx_t size) {
213 D_ASSERT(allocation_count >= size);
214 allocation_count -= size;
215#ifdef DUCKDB_DEBUG_ALLOCATION
216 lock_guard<mutex> l(pointer_lock);
217 // verify that the pointer exists
218 D_ASSERT(pointers.find(pointer) != pointers.end());
219 // verify that the stored size matches the passed in size
220 D_ASSERT(pointers[pointer].first == size);
221 // erase the pointer
222 pointers.erase(pointer);
223#endif
224}
225
226void AllocatorDebugInfo::ReallocateData(data_ptr_t pointer, data_ptr_t new_pointer, idx_t old_size, idx_t new_size) {
227 FreeData(pointer, old_size);
228 AllocateData(new_pointer, new_size);
229}
230
231#endif
232
233} // namespace duckdb
234