1#include "duckdb/main/config.hpp"
2
3#include "duckdb/common/operator/cast_operators.hpp"
4#include "duckdb/common/string_util.hpp"
5#include "duckdb/main/settings.hpp"
6#include "duckdb/storage/storage_extension.hpp"
7
8#ifndef DUCKDB_NO_THREADS
9#include "duckdb/common/thread.hpp"
10#endif
11
12#include <cstdio>
13#include <cinttypes>
14
15namespace duckdb {
16
17#ifdef DEBUG
18bool DBConfigOptions::debug_print_bindings = false;
19#endif
20
21#define DUCKDB_GLOBAL(_PARAM) \
22 { \
23 _PARAM::Name, _PARAM::Description, _PARAM::InputType, _PARAM::SetGlobal, nullptr, _PARAM::ResetGlobal, \
24 nullptr, _PARAM::GetSetting \
25 }
26#define DUCKDB_GLOBAL_ALIAS(_ALIAS, _PARAM) \
27 { \
28 _ALIAS, _PARAM::Description, _PARAM::InputType, _PARAM::SetGlobal, nullptr, _PARAM::ResetGlobal, nullptr, \
29 _PARAM::GetSetting \
30 }
31
32#define DUCKDB_LOCAL(_PARAM) \
33 { \
34 _PARAM::Name, _PARAM::Description, _PARAM::InputType, nullptr, _PARAM::SetLocal, nullptr, _PARAM::ResetLocal, \
35 _PARAM::GetSetting \
36 }
37#define DUCKDB_LOCAL_ALIAS(_ALIAS, _PARAM) \
38 { \
39 _ALIAS, _PARAM::Description, _PARAM::InputType, nullptr, _PARAM::SetLocal, nullptr, _PARAM::ResetLocal, \
40 _PARAM::GetSetting \
41 }
42
43#define DUCKDB_GLOBAL_LOCAL(_PARAM) \
44 { \
45 _PARAM::Name, _PARAM::Description, _PARAM::InputType, _PARAM::SetGlobal, _PARAM::SetLocal, \
46 _PARAM::ResetGlobal, _PARAM::ResetLocal, _PARAM::GetSetting \
47 }
48#define DUCKDB_GLOBAL_LOCAL_ALIAS(_ALIAS, _PARAM) \
49 { \
50 _ALIAS, _PARAM::Description, _PARAM::InputType, _PARAM::SetGlobal, _PARAM::SetLocal, _PARAM::ResetGlobal, \
51 _PARAM::ResetLocal, _PARAM::GetSetting \
52 }
53#define FINAL_SETTING \
54 { nullptr, nullptr, LogicalTypeId::INVALID, nullptr, nullptr, nullptr, nullptr, nullptr }
55
56static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting),
57 DUCKDB_GLOBAL(CheckpointThresholdSetting),
58 DUCKDB_GLOBAL(DebugCheckpointAbort),
59 DUCKDB_LOCAL(DebugForceExternal),
60 DUCKDB_LOCAL(DebugForceNoCrossProduct),
61 DUCKDB_LOCAL(DebugAsOfIEJoin),
62 DUCKDB_GLOBAL(DebugWindowMode),
63 DUCKDB_GLOBAL_LOCAL(DefaultCollationSetting),
64 DUCKDB_GLOBAL(DefaultOrderSetting),
65 DUCKDB_GLOBAL(DefaultNullOrderSetting),
66 DUCKDB_GLOBAL(DisabledOptimizersSetting),
67 DUCKDB_GLOBAL(EnableExternalAccessSetting),
68 DUCKDB_GLOBAL(EnableFSSTVectors),
69 DUCKDB_GLOBAL(AllowUnsignedExtensionsSetting),
70 DUCKDB_LOCAL(CustomExtensionRepository),
71 DUCKDB_GLOBAL(EnableObjectCacheSetting),
72 DUCKDB_GLOBAL(EnableHTTPMetadataCacheSetting),
73 DUCKDB_LOCAL(EnableProfilingSetting),
74 DUCKDB_LOCAL(EnableProgressBarSetting),
75 DUCKDB_LOCAL(EnableProgressBarPrintSetting),
76 DUCKDB_GLOBAL(ExperimentalParallelCSVSetting),
77 DUCKDB_LOCAL(ExplainOutputSetting),
78 DUCKDB_GLOBAL(ExtensionDirectorySetting),
79 DUCKDB_GLOBAL(ExternalThreadsSetting),
80 DUCKDB_LOCAL(FileSearchPathSetting),
81 DUCKDB_GLOBAL(ForceCompressionSetting),
82 DUCKDB_GLOBAL(ForceBitpackingModeSetting),
83 DUCKDB_LOCAL(HomeDirectorySetting),
84 DUCKDB_LOCAL(LogQueryPathSetting),
85 DUCKDB_GLOBAL(LockConfigurationSetting),
86 DUCKDB_GLOBAL(ImmediateTransactionModeSetting),
87 DUCKDB_LOCAL(IntegerDivisionSetting),
88 DUCKDB_LOCAL(MaximumExpressionDepthSetting),
89 DUCKDB_GLOBAL(MaximumMemorySetting),
90 DUCKDB_GLOBAL_ALIAS("memory_limit", MaximumMemorySetting),
91 DUCKDB_GLOBAL_ALIAS("null_order", DefaultNullOrderSetting),
92 DUCKDB_LOCAL(OrderedAggregateThreshold),
93 DUCKDB_GLOBAL(PasswordSetting),
94 DUCKDB_LOCAL(PerfectHashThresholdSetting),
95 DUCKDB_LOCAL(PivotLimitSetting),
96 DUCKDB_LOCAL(PreserveIdentifierCase),
97 DUCKDB_GLOBAL(PreserveInsertionOrder),
98 DUCKDB_LOCAL(ProfilerHistorySize),
99 DUCKDB_LOCAL(ProfileOutputSetting),
100 DUCKDB_LOCAL(ProfilingModeSetting),
101 DUCKDB_LOCAL_ALIAS("profiling_output", ProfileOutputSetting),
102 DUCKDB_LOCAL(ProgressBarTimeSetting),
103 DUCKDB_LOCAL(SchemaSetting),
104 DUCKDB_LOCAL(SearchPathSetting),
105 DUCKDB_GLOBAL(TempDirectorySetting),
106 DUCKDB_GLOBAL(ThreadsSetting),
107 DUCKDB_GLOBAL(UsernameSetting),
108 DUCKDB_GLOBAL(ExportLargeBufferArrow),
109 DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
110 DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
111 DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),
112 FINAL_SETTING};
113
114vector<ConfigurationOption> DBConfig::GetOptions() {
115 vector<ConfigurationOption> options;
116 for (idx_t index = 0; internal_options[index].name; index++) {
117 options.push_back(x: internal_options[index]);
118 }
119 return options;
120}
121
122idx_t DBConfig::GetOptionCount() {
123 idx_t count = 0;
124 for (idx_t index = 0; internal_options[index].name; index++) {
125 count++;
126 }
127 return count;
128}
129
130vector<std::string> DBConfig::GetOptionNames() {
131 vector<string> names;
132 for (idx_t i = 0, option_count = DBConfig::GetOptionCount(); i < option_count; i++) {
133 names.emplace_back(args&: DBConfig::GetOptionByIndex(index: i)->name);
134 }
135 return names;
136}
137
138ConfigurationOption *DBConfig::GetOptionByIndex(idx_t target_index) {
139 for (idx_t index = 0; internal_options[index].name; index++) {
140 if (index == target_index) {
141 return internal_options + index;
142 }
143 }
144 return nullptr;
145}
146
147ConfigurationOption *DBConfig::GetOptionByName(const string &name) {
148 auto lname = StringUtil::Lower(str: name);
149 for (idx_t index = 0; internal_options[index].name; index++) {
150 D_ASSERT(StringUtil::Lower(internal_options[index].name) == string(internal_options[index].name));
151 if (internal_options[index].name == lname) {
152 return internal_options + index;
153 }
154 }
155 return nullptr;
156}
157
158void DBConfig::SetOption(const ConfigurationOption &option, const Value &value) {
159 SetOption(db: nullptr, option, value);
160}
161
162void DBConfig::SetOptionByName(const string &name, const Value &value) {
163 auto option = DBConfig::GetOptionByName(name);
164 if (option) {
165 SetOption(option: *option, value);
166 } else {
167 options.unrecognized_options[name] = value;
168 }
169}
170
171void DBConfig::SetOption(DatabaseInstance *db, const ConfigurationOption &option, const Value &value) {
172 lock_guard<mutex> l(config_lock);
173 if (!option.set_global) {
174 throw InternalException("Could not set option \"%s\" as a global option", option.name);
175 }
176 D_ASSERT(option.reset_global);
177 Value input = value.DefaultCastAs(target_type: option.parameter_type);
178 option.set_global(db, *this, input);
179}
180
181void DBConfig::ResetOption(DatabaseInstance *db, const ConfigurationOption &option) {
182 lock_guard<mutex> l(config_lock);
183 if (!option.reset_global) {
184 throw InternalException("Could not reset option \"%s\" as a global option", option.name);
185 }
186 D_ASSERT(option.set_global);
187 option.reset_global(db, *this);
188}
189
190void DBConfig::SetOption(const string &name, Value value) {
191 lock_guard<mutex> l(config_lock);
192 options.set_variables[name] = std::move(value);
193}
194
195void DBConfig::ResetOption(const string &name) {
196 lock_guard<mutex> l(config_lock);
197 auto extension_option = extension_parameters.find(x: name);
198 D_ASSERT(extension_option != extension_parameters.end());
199 auto &default_value = extension_option->second.default_value;
200 if (!default_value.IsNull()) {
201 // Default is not NULL, override the setting
202 options.set_variables[name] = default_value;
203 } else {
204 // Otherwise just remove it from the 'set_variables' map
205 options.set_variables.erase(x: name);
206 }
207}
208
209void DBConfig::AddExtensionOption(const string &name, string description, LogicalType parameter,
210 const Value &default_value, set_option_callback_t function) {
211 extension_parameters.insert(
212 x: make_pair(x: name, y: ExtensionOption(std::move(description), std::move(parameter), function, default_value)));
213 if (!default_value.IsNull()) {
214 // Default value is set, insert it into the 'set_variables' list
215 options.set_variables[name] = default_value;
216 }
217}
218
219CastFunctionSet &DBConfig::GetCastFunctions() {
220 return *cast_functions;
221}
222
223void DBConfig::SetDefaultMaxMemory() {
224 auto memory = FileSystem::GetAvailableMemory();
225 if (memory != DConstants::INVALID_INDEX) {
226 options.maximum_memory = memory * 8 / 10;
227 }
228}
229
230idx_t CGroupBandwidthQuota(idx_t physical_cores, FileSystem &fs) {
231 static constexpr const char *CPU_MAX = "/sys/fs/cgroup/cpu.max";
232 static constexpr const char *CFS_QUOTA = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
233 static constexpr const char *CFS_PERIOD = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
234
235 int64_t quota, period;
236 char byte_buffer[1000];
237 unique_ptr<FileHandle> handle;
238 int64_t read_bytes;
239
240 if (fs.FileExists(filename: CPU_MAX)) {
241 // cgroup v2
242 // https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
243 handle =
244 fs.OpenFile(path: CPU_MAX, flags: FileFlags::FILE_FLAGS_READ, lock: FileSystem::DEFAULT_LOCK, compression: FileSystem::DEFAULT_COMPRESSION);
245 read_bytes = fs.Read(handle&: *handle, buffer: (void *)byte_buffer, nr_bytes: 999);
246 byte_buffer[read_bytes] = '\0';
247 if (std::sscanf(s: byte_buffer, format: "%" SCNd64 " %" SCNd64 "", &quota, &period) != 2) {
248 return physical_cores;
249 }
250 } else if (fs.FileExists(filename: CFS_QUOTA) && fs.FileExists(filename: CFS_PERIOD)) {
251 // cgroup v1
252 // https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management
253
254 // Read the quota, this indicates how many microseconds the CPU can be utilized by this cgroup per period
255 handle = fs.OpenFile(path: CFS_QUOTA, flags: FileFlags::FILE_FLAGS_READ, lock: FileSystem::DEFAULT_LOCK,
256 compression: FileSystem::DEFAULT_COMPRESSION);
257 read_bytes = fs.Read(handle&: *handle, buffer: (void *)byte_buffer, nr_bytes: 999);
258 byte_buffer[read_bytes] = '\0';
259 if (std::sscanf(s: byte_buffer, format: "%" SCNd64 "", &quota) != 1) {
260 return physical_cores;
261 }
262
263 // Read the time period, a cgroup can utilize the CPU up to quota microseconds every period
264 handle = fs.OpenFile(path: CFS_PERIOD, flags: FileFlags::FILE_FLAGS_READ, lock: FileSystem::DEFAULT_LOCK,
265 compression: FileSystem::DEFAULT_COMPRESSION);
266 read_bytes = fs.Read(handle&: *handle, buffer: (void *)byte_buffer, nr_bytes: 999);
267 byte_buffer[read_bytes] = '\0';
268 if (std::sscanf(s: byte_buffer, format: "%" SCNd64 "", &period) != 1) {
269 return physical_cores;
270 }
271 } else {
272 // No cgroup quota
273 return physical_cores;
274 }
275 if (quota > 0 && period > 0) {
276 return idx_t(std::ceil(x: (double)quota / (double)period));
277 } else {
278 return physical_cores;
279 }
280}
281
282idx_t GetSystemMaxThreadsInternal(FileSystem &fs) {
283#ifndef DUCKDB_NO_THREADS
284 idx_t physical_cores = std::thread::hardware_concurrency();
285#ifdef __linux__
286 auto cores_available_per_period = CGroupBandwidthQuota(physical_cores, fs);
287 return MaxValue<idx_t>(a: cores_available_per_period, b: 1);
288#else
289 return physical_cores;
290#endif
291#else
292 return 1;
293#endif
294}
295
296void DBConfig::SetDefaultMaxThreads() {
297#ifndef DUCKDB_NO_THREADS
298 options.maximum_threads = GetSystemMaxThreadsInternal(fs&: *file_system);
299#else
300 options.maximum_threads = 1;
301#endif
302}
303
304idx_t DBConfig::ParseMemoryLimit(const string &arg) {
305 if (arg[0] == '-' || arg == "null" || arg == "none") {
306 return DConstants::INVALID_INDEX;
307 }
308 // split based on the number/non-number
309 idx_t idx = 0;
310 while (StringUtil::CharacterIsSpace(c: arg[idx])) {
311 idx++;
312 }
313 idx_t num_start = idx;
314 while ((arg[idx] >= '0' && arg[idx] <= '9') || arg[idx] == '.' || arg[idx] == 'e' || arg[idx] == 'E' ||
315 arg[idx] == '-') {
316 idx++;
317 }
318 if (idx == num_start) {
319 throw ParserException("Memory limit must have a number (e.g. SET memory_limit=1GB");
320 }
321 string number = arg.substr(pos: num_start, n: idx - num_start);
322
323 // try to parse the number
324 double limit = Cast::Operation<string_t, double>(input: string_t(number));
325
326 // now parse the memory limit unit (e.g. bytes, gb, etc)
327 while (StringUtil::CharacterIsSpace(c: arg[idx])) {
328 idx++;
329 }
330 idx_t start = idx;
331 while (idx < arg.size() && !StringUtil::CharacterIsSpace(c: arg[idx])) {
332 idx++;
333 }
334 if (limit < 0) {
335 // limit < 0, set limit to infinite
336 return (idx_t)-1;
337 }
338 string unit = StringUtil::Lower(str: arg.substr(pos: start, n: idx - start));
339 idx_t multiplier;
340 if (unit == "byte" || unit == "bytes" || unit == "b") {
341 multiplier = 1;
342 } else if (unit == "kilobyte" || unit == "kilobytes" || unit == "kb" || unit == "k") {
343 multiplier = 1000LL;
344 } else if (unit == "megabyte" || unit == "megabytes" || unit == "mb" || unit == "m") {
345 multiplier = 1000LL * 1000LL;
346 } else if (unit == "gigabyte" || unit == "gigabytes" || unit == "gb" || unit == "g") {
347 multiplier = 1000LL * 1000LL * 1000LL;
348 } else if (unit == "terabyte" || unit == "terabytes" || unit == "tb" || unit == "t") {
349 multiplier = 1000LL * 1000LL * 1000LL * 1000LL;
350 } else {
351 throw ParserException("Unknown unit for memory_limit: %s (expected: b, mb, gb or tb)", unit);
352 }
353 return (idx_t)multiplier * limit;
354}
355
356// Right now we only really care about access mode when comparing DBConfigs
357bool DBConfigOptions::operator==(const DBConfigOptions &other) const {
358 return other.access_mode == access_mode;
359}
360
361bool DBConfig::operator==(const DBConfig &other) {
362 return other.options == options;
363}
364
365bool DBConfig::operator!=(const DBConfig &other) {
366 return !(other.options == options);
367}
368
369OrderType DBConfig::ResolveOrder(OrderType order_type) const {
370 if (order_type != OrderType::ORDER_DEFAULT) {
371 return order_type;
372 }
373 return options.default_order_type;
374}
375
376OrderByNullType DBConfig::ResolveNullOrder(OrderType order_type, OrderByNullType null_type) const {
377 if (null_type != OrderByNullType::ORDER_DEFAULT) {
378 return null_type;
379 }
380 switch (options.default_null_order) {
381 case DefaultOrderByNullType::NULLS_FIRST:
382 return OrderByNullType::NULLS_FIRST;
383 case DefaultOrderByNullType::NULLS_LAST:
384 return OrderByNullType::NULLS_LAST;
385 case DefaultOrderByNullType::NULLS_FIRST_ON_ASC_LAST_ON_DESC:
386 return order_type == OrderType::ASCENDING ? OrderByNullType::NULLS_FIRST : OrderByNullType::NULLS_LAST;
387 case DefaultOrderByNullType::NULLS_LAST_ON_ASC_FIRST_ON_DESC:
388 return order_type == OrderType::ASCENDING ? OrderByNullType::NULLS_LAST : OrderByNullType::NULLS_FIRST;
389 default:
390 throw InternalException("Unknown null order setting");
391 }
392}
393
394} // namespace duckdb
395