1/*
2 * Copyright 2014-present Facebook, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#pragma once
18
19#include <algorithm>
20#include <atomic>
21#include <chrono>
22
23#include <folly/detail/Futex.h>
24#include <folly/hash/Hash.h>
25#include <folly/synchronization/AtomicStruct.h>
26#include <folly/system/ThreadId.h>
27
28namespace folly {
29namespace detail {
30
31/// MemoryIdler provides helper routines that allow routines to return
32/// some assigned memory resources back to the system. The intended
33/// use is that when a thread is waiting for a long time (perhaps it
34/// is in a LIFO thread pool and hasn't been needed for a long time)
35/// it should release its thread-local malloc caches (both jemalloc and
36/// tcmalloc use these for better performance) and unmap the stack pages
37/// that contain no useful data.
38struct MemoryIdler {
39 /// Returns memory from thread-local allocation pools to the global
40 /// pool, if we know how to for the current malloc implementation.
41 /// jemalloc is supported.
42 static void flushLocalMallocCaches();
43
44 enum {
45 /// This value is a tradeoff between reclaiming memory and triggering
46 /// a page fault immediately on wakeup. Note that the actual unit
47 /// of idling for the stack is pages, so the actual stack that
48 /// will be available on wakeup without a page fault is between
49 /// kDefaultStackToRetain and kDefaultStackToRetain + PageSize -
50 /// 1 bytes.
51 kDefaultStackToRetain = 1024,
52 };
53
54 /// Uses madvise to discard the portion of the thread's stack that
55 /// currently doesn't hold any data, trying to ensure that no page
56 /// faults will occur during the next retain bytes of stack allocation
57 static void unmapUnusedStack(size_t retain = kDefaultStackToRetain);
58
59 /// The system-wide default for the amount of time a blocking
60 /// thread should wait before reclaiming idle memory. Set this to
61 /// Duration::max() to never wait. The default value is 5 seconds.
62 /// Endpoints using this idle timeout might randomly wait longer to
63 /// avoid synchronizing their flushes.
64 static AtomicStruct<std::chrono::steady_clock::duration> defaultIdleTimeout;
65
66 /// Selects a timeout pseudo-randomly chosen to be between
67 /// idleTimeout and idleTimeout * (1 + timeoutVariationFraction), to
68 /// smooth out the behavior in a bursty system
69 template <typename IdleTime = std::chrono::steady_clock::duration>
70 static IdleTime getVariationTimeout(
71 IdleTime const& idleTimeout =
72 defaultIdleTimeout.load(std::memory_order_acquire),
73 float timeoutVariationFrac = 0.5) {
74 if (idleTimeout <= IdleTime::zero() || timeoutVariationFrac <= 0) {
75 return idleTimeout;
76 }
77
78 // hash the pthread_t and the time to get the adjustment
79 // Standard hash func isn't very good, so bit mix the result
80 uint64_t h = folly::hash::twang_mix64(folly::hash::hash_combine(
81 getCurrentThreadID(),
82 std::chrono::system_clock::now().time_since_epoch().count()));
83
84 // multiplying the duration by a floating point doesn't work, grr
85 auto extraFrac = timeoutVariationFrac /
86 static_cast<float>(std::numeric_limits<uint64_t>::max()) * h;
87 auto tics = uint64_t(idleTimeout.count() * (1 + extraFrac));
88 return IdleTime(tics);
89 }
90
91 /// Equivalent to fut.futexWait(expected, waitMask), but calls
92 /// flushLocalMallocCaches() and unmapUnusedStack(stackToRetain)
93 /// after idleTimeout has passed (if it has passed). Internally uses
94 /// fut.futexWait and fut.futexWaitUntil. The actual timeout will be
95 /// pseudo-randomly chosen to be between idleTimeout and idleTimeout *
96 /// (1 + timeoutVariationFraction), to smooth out the behavior in a
97 /// system with bursty requests. The default is to wait up to 50%
98 /// extra, so on average 25% extra.
99 template <
100 typename Futex,
101 typename IdleTime = std::chrono::steady_clock::duration>
102 static FutexResult futexWait(
103 Futex& fut,
104 uint32_t expected,
105 uint32_t waitMask = -1,
106 IdleTime const& idleTimeout =
107 defaultIdleTimeout.load(std::memory_order_acquire),
108 size_t stackToRetain = kDefaultStackToRetain,
109 float timeoutVariationFrac = 0.5) {
110 FutexResult pre;
111 if (futexWaitPreIdle(
112 pre,
113 fut,
114 expected,
115 std::chrono::steady_clock::time_point::max(),
116 waitMask,
117 idleTimeout,
118 stackToRetain,
119 timeoutVariationFrac)) {
120 return pre;
121 }
122
123 using folly::detail::futexWait;
124 return futexWait(&fut, expected, waitMask);
125 }
126
127 /// Equivalent to fut.futexWaitUntil(expected, deadline, waitMask), but
128 /// calls flushLocalMallocCaches() and unmapUnusedStack(stackToRetain)
129 /// after idleTimeout has passed (if it has passed). Internally uses
130 /// fut.futexWaitUntil. The actual timeout will be pseudo-randomly
131 /// chosen to be between idleTimeout and idleTimeout *
132 /// (1 + timeoutVariationFraction), to smooth out the behavior in a
133 /// system with bursty requests. The default is to wait up to 50%
134 /// extra, so on average 25% extra.
135 template <
136 typename Futex,
137 typename Deadline,
138 typename IdleTime = std::chrono::steady_clock::duration>
139 static FutexResult futexWaitUntil(
140 Futex& fut,
141 uint32_t expected,
142 Deadline const& deadline,
143 uint32_t waitMask = -1,
144 IdleTime const& idleTimeout =
145 defaultIdleTimeout.load(std::memory_order_acquire),
146 size_t stackToRetain = kDefaultStackToRetain,
147 float timeoutVariationFrac = 0.5) {
148 FutexResult pre;
149 if (futexWaitPreIdle(
150 pre,
151 fut,
152 expected,
153 deadline,
154 waitMask,
155 idleTimeout,
156 stackToRetain,
157 timeoutVariationFrac)) {
158 return pre;
159 }
160
161 using folly::detail::futexWaitUntil;
162 return futexWaitUntil(&fut, expected, deadline, waitMask);
163 }
164
165 private:
166 template <typename Futex, typename Deadline, typename IdleTime>
167 static bool futexWaitPreIdle(
168 FutexResult& _ret,
169 Futex& fut,
170 uint32_t expected,
171 Deadline const& deadline,
172 uint32_t waitMask,
173 IdleTime idleTimeout,
174 size_t stackToRetain,
175 float timeoutVariationFrac) {
176 // idleTimeout < 0 means no flush behavior
177 if (idleTimeout < IdleTime::zero()) {
178 return false;
179 }
180
181 // idleTimeout == 0 means flush immediately, without variation
182 // idleTimeout > 0 means flush after delay, with variation
183 if (idleTimeout > IdleTime::zero()) {
184 idleTimeout = std::max(
185 IdleTime::zero(),
186 getVariationTimeout(idleTimeout, timeoutVariationFrac));
187 }
188 if (idleTimeout > IdleTime::zero()) {
189 auto idleDeadline = Deadline::clock::now() + idleTimeout;
190 if (idleDeadline < deadline) {
191 using folly::detail::futexWaitUntil;
192 auto rv = futexWaitUntil(&fut, expected, idleDeadline, waitMask);
193 if (rv != FutexResult::TIMEDOUT) {
194 // finished before timeout hit, no flush
195 _ret = rv;
196 return true;
197 }
198 }
199 }
200
201 // flush, then wait
202 flushLocalMallocCaches();
203 unmapUnusedStack(stackToRetain);
204 return false;
205 }
206};
207
208} // namespace detail
209} // namespace folly
210