1/*
2 * Copyright 2011-present Facebook, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17// Functions to provide smarter use of jemalloc, if jemalloc is being used.
18// http://www.canonware.com/download/jemalloc/jemalloc-latest/doc/jemalloc.html
19
20#pragma once
21
22#include <folly/CPortability.h>
23#include <folly/portability/Config.h>
24
25/**
26 * Define various MALLOCX_* macros normally provided by jemalloc. We define
27 * them so that we don't have to include jemalloc.h, in case the program is
28 * built without jemalloc support.
29 */
30#if (defined(USE_JEMALLOC) || defined(FOLLY_USE_JEMALLOC)) && !FOLLY_SANITIZE
31// We have JEMalloc, so use it.
32#include <jemalloc/jemalloc.h>
33#else
34#ifndef MALLOCX_LG_ALIGN
35#define MALLOCX_LG_ALIGN(la) (la)
36#endif
37#ifndef MALLOCX_ZERO
38#define MALLOCX_ZERO (static_cast<int>(0x40))
39#endif
40#endif
41
42// If using fbstring from libstdc++ (see comment in FBString.h), then
43// just define stub code here to typedef the fbstring type into the
44// folly namespace.
45// This provides backwards compatibility for code that explicitly
46// includes and uses fbstring.
47#if defined(_GLIBCXX_USE_FB) && !defined(_LIBSTDCXX_FBSTRING)
48
49#include <folly/lang/Exception.h>
50#include <folly/memory/detail/MallocImpl.h>
51
52#include <string>
53
54namespace folly {
55using std::checkedCalloc;
56using std::checkedMalloc;
57using std::checkedRealloc;
58using std::goodMallocSize;
59using std::jemallocMinInPlaceExpandable;
60using std::smartRealloc;
61using std::usingJEMalloc;
62} // namespace folly
63
64#else // !defined(_GLIBCXX_USE_FB) || defined(_LIBSTDCXX_FBSTRING)
65
66#ifdef _LIBSTDCXX_FBSTRING
67#pragma GCC system_header
68
69/**
70 * Declare *allocx() and mallctl*() as weak symbols. These will be provided by
71 * jemalloc if we are using jemalloc, or will be nullptr if we are using another
72 * malloc implementation.
73 */
74extern "C" void* mallocx(size_t, int) __attribute__((__weak__));
75extern "C" void* rallocx(void*, size_t, int) __attribute__((__weak__));
76extern "C" size_t xallocx(void*, size_t, size_t, int) __attribute__((__weak__));
77extern "C" size_t sallocx(const void*, int) __attribute__((__weak__));
78extern "C" void dallocx(void*, int) __attribute__((__weak__));
79extern "C" void sdallocx(void*, size_t, int) __attribute__((__weak__));
80extern "C" size_t nallocx(size_t, int) __attribute__((__weak__));
81extern "C" int mallctl(const char*, void*, size_t*, void*, size_t)
82 __attribute__((__weak__));
83extern "C" int mallctlnametomib(const char*, size_t*, size_t*)
84 __attribute__((__weak__));
85extern "C" int
86mallctlbymib(const size_t*, size_t, void*, size_t*, void*, size_t)
87 __attribute__((__weak__));
88
89#else // !defined(_LIBSTDCXX_FBSTRING)
90
91#include <folly/lang/Exception.h> /* nolint */
92#include <folly/memory/detail/MallocImpl.h> /* nolint */
93
94#endif
95
96// for malloc_usable_size
97// NOTE: FreeBSD 9 doesn't have malloc.h. Its definitions
98// are found in stdlib.h.
99#if __has_include(<malloc.h>)
100#include <malloc.h>
101#else
102#include <stdlib.h>
103#endif
104
105#include <cassert>
106#include <cstddef>
107#include <cstdint>
108#include <cstdlib>
109#include <cstring>
110
111#include <atomic>
112#include <new>
113
114// clang-format off
115
116#ifdef _LIBSTDCXX_FBSTRING
117namespace std _GLIBCXX_VISIBILITY(default) {
118 _GLIBCXX_BEGIN_NAMESPACE_VERSION
119#else
120namespace folly {
121#endif
122
123// Cannot depend on Portability.h when _LIBSTDCXX_FBSTRING.
124#if defined(__GNUC__)
125#define FOLLY_MALLOC_NOINLINE __attribute__((__noinline__))
126#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL) >= 40900
127// This is for checked malloc-like functions (returns non-null pointer
128// which cannot alias any outstanding pointer).
129#define FOLLY_MALLOC_CHECKED_MALLOC \
130 __attribute__((__returns_nonnull__, __malloc__))
131#else
132#define FOLLY_MALLOC_CHECKED_MALLOC __attribute__((__malloc__))
133#endif
134#else
135#define FOLLY_MALLOC_NOINLINE
136#define FOLLY_MALLOC_CHECKED_MALLOC
137#endif
138
139/**
140 * Determine if we are using jemalloc or not.
141 */
142#if defined(USE_JEMALLOC) && !FOLLY_SANITIZE
143 inline bool usingJEMalloc() noexcept {
144 return true;
145 }
146#else
147FOLLY_MALLOC_NOINLINE inline bool usingJEMalloc() noexcept {
148 // Checking for rallocx != nullptr is not sufficient; we may be in a
149 // dlopen()ed module that depends on libjemalloc, so rallocx is resolved, but
150 // the main program might be using a different memory allocator.
151 // How do we determine that we're using jemalloc? In the hackiest
152 // way possible. We allocate memory using malloc() and see if the
153 // per-thread counter of allocated memory increases. This makes me
154 // feel dirty inside. Also note that this requires jemalloc to have
155 // been compiled with --enable-stats.
156 static const bool result = []() noexcept {
157 // Some platforms (*cough* OSX *cough*) require weak symbol checks to be
158 // in the form if (mallctl != nullptr). Not if (mallctl) or if (!mallctl)
159 // (!!). http://goo.gl/xpmctm
160 if (mallocx == nullptr || rallocx == nullptr || xallocx == nullptr ||
161 sallocx == nullptr || dallocx == nullptr || sdallocx == nullptr ||
162 nallocx == nullptr || mallctl == nullptr ||
163 mallctlnametomib == nullptr || mallctlbymib == nullptr) {
164 return false;
165 }
166
167 // "volatile" because gcc optimizes out the reads from *counter, because
168 // it "knows" malloc doesn't modify global state...
169 /* nolint */ volatile uint64_t* counter;
170 size_t counterLen = sizeof(uint64_t*);
171
172 if (mallctl(
173 "thread.allocatedp",
174 static_cast<void*>(&counter),
175 &counterLen,
176 nullptr,
177 0) != 0) {
178 return false;
179 }
180
181 if (counterLen != sizeof(uint64_t*)) {
182 return false;
183 }
184
185 uint64_t origAllocated = *counter;
186
187 static const void* volatile ptr = malloc(1);
188 if (!ptr) {
189 // wtf, failing to allocate 1 byte
190 return false;
191 }
192
193 return (origAllocated != *counter);
194 }
195 ();
196
197 return result;
198}
199#endif
200
201inline size_t goodMallocSize(size_t minSize) noexcept {
202 if (minSize == 0) {
203 return 0;
204 }
205
206 if (!usingJEMalloc()) {
207 // Not using jemalloc - no smarts
208 return minSize;
209 }
210
211 // nallocx returns 0 if minSize can't succeed, but 0 is not actually
212 // a goodMallocSize if you want minSize
213 auto rv = nallocx(minSize, 0);
214 return rv ? rv : minSize;
215}
216
217// We always request "good" sizes for allocation, so jemalloc can
218// never grow in place small blocks; they're already occupied to the
219// brim. Blocks larger than or equal to 4096 bytes can in fact be
220// expanded in place, and this constant reflects that.
221static const size_t jemallocMinInPlaceExpandable = 4096;
222
223/**
224 * Trivial wrappers around malloc, calloc, realloc that check for allocation
225 * failure and throw std::bad_alloc in that case.
226 */
227inline void* checkedMalloc(size_t size) {
228 void* p = malloc(size);
229 if (!p) {
230 throw_exception<std::bad_alloc>();
231 }
232 return p;
233}
234
235inline void* checkedCalloc(size_t n, size_t size) {
236 void* p = calloc(n, size);
237 if (!p) {
238 throw_exception<std::bad_alloc>();
239 }
240 return p;
241}
242
243inline void* checkedRealloc(void* ptr, size_t size) {
244 void* p = realloc(ptr, size);
245 if (!p) {
246 throw_exception<std::bad_alloc>();
247 }
248 return p;
249}
250
251/**
252 * This function tries to reallocate a buffer of which only the first
253 * currentSize bytes are used. The problem with using realloc is that
254 * if currentSize is relatively small _and_ if realloc decides it
255 * needs to move the memory chunk to a new buffer, then realloc ends
256 * up copying data that is not used. It's generally not a win to try
257 * to hook in to realloc() behavior to avoid copies - at least in
258 * jemalloc, realloc() almost always ends up doing a copy, because
259 * there is little fragmentation / slack space to take advantage of.
260 */
261FOLLY_MALLOC_CHECKED_MALLOC FOLLY_MALLOC_NOINLINE inline void* smartRealloc(
262 void* p,
263 const size_t currentSize,
264 const size_t currentCapacity,
265 const size_t newCapacity) {
266 assert(p);
267 assert(currentSize <= currentCapacity &&
268 currentCapacity < newCapacity);
269
270 auto const slack = currentCapacity - currentSize;
271 if (slack * 2 > currentSize) {
272 // Too much slack, malloc-copy-free cycle:
273 auto const result = checkedMalloc(newCapacity);
274 std::memcpy(result, p, currentSize);
275 free(p);
276 return result;
277 }
278 // If there's not too much slack, we realloc in hope of coalescing
279 return checkedRealloc(p, newCapacity);
280}
281
282#ifdef _LIBSTDCXX_FBSTRING
283 _GLIBCXX_END_NAMESPACE_VERSION
284#endif
285
286} // namespace folly
287
288// clang-format on
289
290#endif // !defined(_GLIBCXX_USE_FB) || defined(_LIBSTDCXX_FBSTRING)
291