1// Protocol Buffers - Google's data interchange format
2// Copyright 2008 Google Inc. All rights reserved.
3// https://developers.google.com/protocol-buffers/
4//
5// Redistribution and use in source and binary forms, with or without
6// modification, are permitted provided that the following conditions are
7// met:
8//
9// * Redistributions of source code must retain the above copyright
10// notice, this list of conditions and the following disclaimer.
11// * Redistributions in binary form must reproduce the above
12// copyright notice, this list of conditions and the following disclaimer
13// in the documentation and/or other materials provided with the
14// distribution.
15// * Neither the name of Google Inc. nor the names of its
16// contributors may be used to endorse or promote products derived from
17// this software without specific prior written permission.
18//
19// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31#include <google/protobuf/arenaz_sampler.h>
32
33#include <atomic>
34#include <cstdint>
35#include <limits>
36
37
38// Must be included last.
39#include <google/protobuf/port_def.inc>
40
41namespace google {
42namespace protobuf {
43namespace internal {
44
45ThreadSafeArenazSampler& GlobalThreadSafeArenazSampler() {
46 static auto* sampler = new ThreadSafeArenazSampler();
47 return *sampler;
48}
49
50void UnsampleSlow(ThreadSafeArenaStats* info) {
51 GlobalThreadSafeArenazSampler().Unregister(info);
52}
53
54#if defined(PROTOBUF_ARENAZ_SAMPLE)
55namespace {
56
57PROTOBUF_CONSTINIT std::atomic<bool> g_arenaz_enabled{true};
58PROTOBUF_CONSTINIT std::atomic<int32_t> g_arenaz_sample_parameter{1 << 10};
59PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased
60 g_exponential_biased_generator;
61
62} // namespace
63
64PROTOBUF_THREAD_LOCAL int64_t global_next_sample = 1LL << 10;
65
66ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(); }
67ThreadSafeArenaStats::~ThreadSafeArenaStats() = default;
68
69void ThreadSafeArenaStats::PrepareForSampling() {
70 num_allocations.store(0, std::memory_order_relaxed);
71 num_resets.store(0, std::memory_order_relaxed);
72 bytes_requested.store(0, std::memory_order_relaxed);
73 bytes_allocated.store(0, std::memory_order_relaxed);
74 bytes_wasted.store(0, std::memory_order_relaxed);
75 max_bytes_allocated.store(0, std::memory_order_relaxed);
76 thread_ids.store(0, std::memory_order_relaxed);
77 // The inliner makes hardcoded skip_count difficult (especially when combined
78 // with LTO). We use the ability to exclude stacks by regex when encoding
79 // instead.
80 depth = absl::GetStackTrace(stack, kMaxStackDepth, /* skip_count= */ 0);
81}
82
83void RecordResetSlow(ThreadSafeArenaStats* info) {
84 const size_t max_bytes =
85 info->max_bytes_allocated.load(std::memory_order_relaxed);
86 const size_t allocated_bytes =
87 info->bytes_allocated.load(std::memory_order_relaxed);
88 if (max_bytes < allocated_bytes) {
89 info->max_bytes_allocated.store(allocated_bytes);
90 }
91 info->bytes_requested.store(0, std::memory_order_relaxed);
92 info->bytes_allocated.store(0, std::memory_order_relaxed);
93 info->bytes_wasted.fetch_add(0, std::memory_order_relaxed);
94 info->num_allocations.fetch_add(0, std::memory_order_relaxed);
95 info->num_resets.fetch_add(1, std::memory_order_relaxed);
96}
97
98void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested,
99 size_t allocated, size_t wasted) {
100 info->bytes_requested.fetch_add(requested, std::memory_order_relaxed);
101 info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed);
102 info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed);
103 info->num_allocations.fetch_add(1, std::memory_order_relaxed);
104 const uint64_t tid = (1ULL << (GetCachedTID() % 63));
105 const uint64_t thread_ids = info->thread_ids.load(std::memory_order_relaxed);
106 if (!(thread_ids & tid)) {
107 info->thread_ids.store(thread_ids | tid, std::memory_order_relaxed);
108 }
109}
110
111ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {
112 bool first = *next_sample < 0;
113 *next_sample = g_exponential_biased_generator.GetStride(
114 g_arenaz_sample_parameter.load(std::memory_order_relaxed));
115 // Small values of interval are equivalent to just sampling next time.
116 ABSL_ASSERT(*next_sample >= 1);
117
118 // g_arenaz_enabled can be dynamically flipped, we need to set a threshold low
119 // enough that we will start sampling in a reasonable time, so we just use the
120 // default sampling rate.
121 if (!g_arenaz_enabled.load(std::memory_order_relaxed)) return nullptr;
122 // We will only be negative on our first count, so we should just retry in
123 // that case.
124 if (first) {
125 if (PROTOBUF_PREDICT_TRUE(--*next_sample > 0)) return nullptr;
126 return SampleSlow(next_sample);
127 }
128
129 return GlobalThreadSafeArenazSampler().Register();
130}
131
132void SetThreadSafeArenazEnabled(bool enabled) {
133 g_arenaz_enabled.store(enabled, std::memory_order_release);
134}
135
136void SetThreadSafeArenazSampleParameter(int32_t rate) {
137 if (rate > 0) {
138 g_arenaz_sample_parameter.store(rate, std::memory_order_release);
139 } else {
140 ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz sample rate: %lld",
141 static_cast<long long>(rate)); // NOLINT(runtime/int)
142 }
143}
144
145void SetThreadSafeArenazMaxSamples(int32_t max) {
146 if (max > 0) {
147 GlobalThreadSafeArenazSampler().SetMaxSamples(max);
148 } else {
149 ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz max samples: %lld",
150 static_cast<long long>(max)); // NOLINT(runtime/int)
151 }
152}
153
154void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {
155 if (next_sample >= 0) {
156 global_next_sample = next_sample;
157 } else {
158 ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz next sample: %lld",
159 static_cast<long long>(next_sample)); // NOLINT(runtime/int)
160 }
161}
162
163#else
164ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {
165 *next_sample = std::numeric_limits<int64_t>::max();
166 return nullptr;
167}
168
169void SetThreadSafeArenazEnabled(bool enabled) {}
170void SetThreadSafeArenazSampleParameter(int32_t rate) {}
171void SetThreadSafeArenazMaxSamples(int32_t max) {}
172void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {}
173#endif // defined(PROTOBUF_ARENAZ_SAMPLE)
174
175} // namespace internal
176} // namespace protobuf
177} // namespace google
178