1 | /* |
2 | * Copyright 2011-present Facebook, Inc. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | // Functions to provide smarter use of jemalloc, if jemalloc is being used. |
18 | // http://www.canonware.com/download/jemalloc/jemalloc-latest/doc/jemalloc.html |
19 | |
20 | #pragma once |
21 | |
22 | #include <folly/CPortability.h> |
23 | #include <folly/portability/Config.h> |
24 | |
25 | /** |
26 | * Define various MALLOCX_* macros normally provided by jemalloc. We define |
27 | * them so that we don't have to include jemalloc.h, in case the program is |
28 | * built without jemalloc support. |
29 | */ |
30 | #if (defined(USE_JEMALLOC) || defined(FOLLY_USE_JEMALLOC)) && !FOLLY_SANITIZE |
31 | // We have JEMalloc, so use it. |
32 | #include <jemalloc/jemalloc.h> |
33 | #else |
34 | #ifndef MALLOCX_LG_ALIGN |
35 | #define MALLOCX_LG_ALIGN(la) (la) |
36 | #endif |
37 | #ifndef MALLOCX_ZERO |
38 | #define MALLOCX_ZERO (static_cast<int>(0x40)) |
39 | #endif |
40 | #endif |
41 | |
42 | // If using fbstring from libstdc++ (see comment in FBString.h), then |
43 | // just define stub code here to typedef the fbstring type into the |
44 | // folly namespace. |
45 | // This provides backwards compatibility for code that explicitly |
46 | // includes and uses fbstring. |
47 | #if defined(_GLIBCXX_USE_FB) && !defined(_LIBSTDCXX_FBSTRING) |
48 | |
49 | #include <folly/lang/Exception.h> |
50 | #include <folly/memory/detail/MallocImpl.h> |
51 | |
52 | #include <string> |
53 | |
54 | namespace folly { |
55 | using std::checkedCalloc; |
56 | using std::checkedMalloc; |
57 | using std::checkedRealloc; |
58 | using std::goodMallocSize; |
59 | using std::jemallocMinInPlaceExpandable; |
60 | using std::smartRealloc; |
61 | using std::usingJEMalloc; |
62 | } // namespace folly |
63 | |
64 | #else // !defined(_GLIBCXX_USE_FB) || defined(_LIBSTDCXX_FBSTRING) |
65 | |
66 | #ifdef _LIBSTDCXX_FBSTRING |
67 | #pragma GCC system_header |
68 | |
69 | /** |
70 | * Declare *allocx() and mallctl*() as weak symbols. These will be provided by |
71 | * jemalloc if we are using jemalloc, or will be nullptr if we are using another |
72 | * malloc implementation. |
73 | */ |
74 | extern "C" void* mallocx(size_t, int) __attribute__((__weak__)); |
75 | extern "C" void* rallocx(void*, size_t, int) __attribute__((__weak__)); |
76 | extern "C" size_t xallocx(void*, size_t, size_t, int) __attribute__((__weak__)); |
77 | extern "C" size_t sallocx(const void*, int) __attribute__((__weak__)); |
78 | extern "C" void dallocx(void*, int) __attribute__((__weak__)); |
79 | extern "C" void sdallocx(void*, size_t, int) __attribute__((__weak__)); |
80 | extern "C" size_t nallocx(size_t, int) __attribute__((__weak__)); |
81 | extern "C" int mallctl(const char*, void*, size_t*, void*, size_t) |
82 | __attribute__((__weak__)); |
83 | extern "C" int mallctlnametomib(const char*, size_t*, size_t*) |
84 | __attribute__((__weak__)); |
85 | extern "C" int |
86 | mallctlbymib(const size_t*, size_t, void*, size_t*, void*, size_t) |
87 | __attribute__((__weak__)); |
88 | |
89 | #else // !defined(_LIBSTDCXX_FBSTRING) |
90 | |
91 | #include <folly/lang/Exception.h> /* nolint */ |
92 | #include <folly/memory/detail/MallocImpl.h> /* nolint */ |
93 | |
94 | #endif |
95 | |
96 | // for malloc_usable_size |
97 | // NOTE: FreeBSD 9 doesn't have malloc.h. Its definitions |
98 | // are found in stdlib.h. |
99 | #if __has_include(<malloc.h>) |
100 | #include <malloc.h> |
101 | #else |
102 | #include <stdlib.h> |
103 | #endif |
104 | |
105 | #include <cassert> |
106 | #include <cstddef> |
107 | #include <cstdint> |
108 | #include <cstdlib> |
109 | #include <cstring> |
110 | |
111 | #include <atomic> |
112 | #include <new> |
113 | |
114 | // clang-format off |
115 | |
116 | #ifdef _LIBSTDCXX_FBSTRING |
117 | namespace std _GLIBCXX_VISIBILITY(default) { |
118 | _GLIBCXX_BEGIN_NAMESPACE_VERSION |
119 | #else |
120 | namespace folly { |
121 | #endif |
122 | |
123 | // Cannot depend on Portability.h when _LIBSTDCXX_FBSTRING. |
124 | #if defined(__GNUC__) |
125 | #define FOLLY_MALLOC_NOINLINE __attribute__((__noinline__)) |
126 | #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL) >= 40900 |
127 | // This is for checked malloc-like functions (returns non-null pointer |
128 | // which cannot alias any outstanding pointer). |
129 | #define FOLLY_MALLOC_CHECKED_MALLOC \ |
130 | __attribute__((__returns_nonnull__, __malloc__)) |
131 | #else |
132 | #define FOLLY_MALLOC_CHECKED_MALLOC __attribute__((__malloc__)) |
133 | #endif |
134 | #else |
135 | #define FOLLY_MALLOC_NOINLINE |
136 | #define FOLLY_MALLOC_CHECKED_MALLOC |
137 | #endif |
138 | |
139 | /** |
140 | * Determine if we are using jemalloc or not. |
141 | */ |
142 | #if defined(USE_JEMALLOC) && !FOLLY_SANITIZE |
143 | inline bool usingJEMalloc() noexcept { |
144 | return true; |
145 | } |
146 | #else |
147 | FOLLY_MALLOC_NOINLINE inline bool usingJEMalloc() noexcept { |
148 | // Checking for rallocx != nullptr is not sufficient; we may be in a |
149 | // dlopen()ed module that depends on libjemalloc, so rallocx is resolved, but |
150 | // the main program might be using a different memory allocator. |
151 | // How do we determine that we're using jemalloc? In the hackiest |
152 | // way possible. We allocate memory using malloc() and see if the |
153 | // per-thread counter of allocated memory increases. This makes me |
154 | // feel dirty inside. Also note that this requires jemalloc to have |
155 | // been compiled with --enable-stats. |
156 | static const bool result = []() noexcept { |
157 | // Some platforms (*cough* OSX *cough*) require weak symbol checks to be |
158 | // in the form if (mallctl != nullptr). Not if (mallctl) or if (!mallctl) |
159 | // (!!). http://goo.gl/xpmctm |
160 | if (mallocx == nullptr || rallocx == nullptr || xallocx == nullptr || |
161 | sallocx == nullptr || dallocx == nullptr || sdallocx == nullptr || |
162 | nallocx == nullptr || mallctl == nullptr || |
163 | mallctlnametomib == nullptr || mallctlbymib == nullptr) { |
164 | return false; |
165 | } |
166 | |
167 | // "volatile" because gcc optimizes out the reads from *counter, because |
168 | // it "knows" malloc doesn't modify global state... |
169 | /* nolint */ volatile uint64_t* counter; |
170 | size_t counterLen = sizeof(uint64_t*); |
171 | |
172 | if (mallctl( |
173 | "thread.allocatedp" , |
174 | static_cast<void*>(&counter), |
175 | &counterLen, |
176 | nullptr, |
177 | 0) != 0) { |
178 | return false; |
179 | } |
180 | |
181 | if (counterLen != sizeof(uint64_t*)) { |
182 | return false; |
183 | } |
184 | |
185 | uint64_t origAllocated = *counter; |
186 | |
187 | static const void* volatile ptr = malloc(1); |
188 | if (!ptr) { |
189 | // wtf, failing to allocate 1 byte |
190 | return false; |
191 | } |
192 | |
193 | return (origAllocated != *counter); |
194 | } |
195 | (); |
196 | |
197 | return result; |
198 | } |
199 | #endif |
200 | |
201 | inline size_t goodMallocSize(size_t minSize) noexcept { |
202 | if (minSize == 0) { |
203 | return 0; |
204 | } |
205 | |
206 | if (!usingJEMalloc()) { |
207 | // Not using jemalloc - no smarts |
208 | return minSize; |
209 | } |
210 | |
211 | // nallocx returns 0 if minSize can't succeed, but 0 is not actually |
212 | // a goodMallocSize if you want minSize |
213 | auto rv = nallocx(minSize, 0); |
214 | return rv ? rv : minSize; |
215 | } |
216 | |
217 | // We always request "good" sizes for allocation, so jemalloc can |
218 | // never grow in place small blocks; they're already occupied to the |
219 | // brim. Blocks larger than or equal to 4096 bytes can in fact be |
220 | // expanded in place, and this constant reflects that. |
221 | static const size_t jemallocMinInPlaceExpandable = 4096; |
222 | |
223 | /** |
224 | * Trivial wrappers around malloc, calloc, realloc that check for allocation |
225 | * failure and throw std::bad_alloc in that case. |
226 | */ |
227 | inline void* checkedMalloc(size_t size) { |
228 | void* p = malloc(size); |
229 | if (!p) { |
230 | throw_exception<std::bad_alloc>(); |
231 | } |
232 | return p; |
233 | } |
234 | |
235 | inline void* checkedCalloc(size_t n, size_t size) { |
236 | void* p = calloc(n, size); |
237 | if (!p) { |
238 | throw_exception<std::bad_alloc>(); |
239 | } |
240 | return p; |
241 | } |
242 | |
243 | inline void* checkedRealloc(void* ptr, size_t size) { |
244 | void* p = realloc(ptr, size); |
245 | if (!p) { |
246 | throw_exception<std::bad_alloc>(); |
247 | } |
248 | return p; |
249 | } |
250 | |
251 | /** |
252 | * This function tries to reallocate a buffer of which only the first |
253 | * currentSize bytes are used. The problem with using realloc is that |
254 | * if currentSize is relatively small _and_ if realloc decides it |
255 | * needs to move the memory chunk to a new buffer, then realloc ends |
256 | * up copying data that is not used. It's generally not a win to try |
257 | * to hook in to realloc() behavior to avoid copies - at least in |
258 | * jemalloc, realloc() almost always ends up doing a copy, because |
259 | * there is little fragmentation / slack space to take advantage of. |
260 | */ |
261 | FOLLY_MALLOC_CHECKED_MALLOC FOLLY_MALLOC_NOINLINE inline void* smartRealloc( |
262 | void* p, |
263 | const size_t currentSize, |
264 | const size_t currentCapacity, |
265 | const size_t newCapacity) { |
266 | assert(p); |
267 | assert(currentSize <= currentCapacity && |
268 | currentCapacity < newCapacity); |
269 | |
270 | auto const slack = currentCapacity - currentSize; |
271 | if (slack * 2 > currentSize) { |
272 | // Too much slack, malloc-copy-free cycle: |
273 | auto const result = checkedMalloc(newCapacity); |
274 | std::memcpy(result, p, currentSize); |
275 | free(p); |
276 | return result; |
277 | } |
278 | // If there's not too much slack, we realloc in hope of coalescing |
279 | return checkedRealloc(p, newCapacity); |
280 | } |
281 | |
282 | #ifdef _LIBSTDCXX_FBSTRING |
283 | _GLIBCXX_END_NAMESPACE_VERSION |
284 | #endif |
285 | |
286 | } // namespace folly |
287 | |
288 | // clang-format on |
289 | |
290 | #endif // !defined(_GLIBCXX_USE_FB) || defined(_LIBSTDCXX_FBSTRING) |
291 | |