1 | #include "Server.h" |
2 | |
3 | #include <memory> |
4 | #include <sys/resource.h> |
5 | #include <sys/stat.h> |
6 | #include <sys/types.h> |
7 | #include <errno.h> |
8 | #include <pwd.h> |
9 | #include <unistd.h> |
10 | #include <Poco/Version.h> |
11 | #include <Poco/DirectoryIterator.h> |
12 | #include <Poco/Net/HTTPServer.h> |
13 | #include <Poco/Net/NetException.h> |
14 | #include <Poco/Util/HelpFormatter.h> |
15 | #include <ext/scope_guard.h> |
16 | #include <common/logger_useful.h> |
17 | #include <common/phdr_cache.h> |
18 | #include <common/config_common.h> |
19 | #include <common/ErrorHandlers.h> |
20 | #include <common/getMemoryAmount.h> |
21 | #include <common/coverage.h> |
22 | #include <Common/ClickHouseRevision.h> |
23 | #include <Common/DNSResolver.h> |
24 | #include <Common/CurrentMetrics.h> |
25 | #include <Common/Macros.h> |
26 | #include <Common/StringUtils/StringUtils.h> |
27 | #include <Common/ZooKeeper/ZooKeeper.h> |
28 | #include <Common/ZooKeeper/ZooKeeperNodeCache.h> |
29 | #include "config_core.h" |
30 | #include <Common/getFQDNOrHostName.h> |
31 | #include <Common/getMultipleKeysFromConfig.h> |
32 | #include <Common/getNumberOfPhysicalCPUCores.h> |
33 | #include <Common/getExecutablePath.h> |
34 | #include <Common/TaskStatsInfoGetter.h> |
35 | #include <Common/ThreadStatus.h> |
36 | #include <IO/HTTPCommon.h> |
37 | #include <IO/UseSSL.h> |
38 | #include <Interpreters/AsynchronousMetrics.h> |
39 | #include <Interpreters/DDLWorker.h> |
40 | #include <Interpreters/ExternalDictionariesLoader.h> |
41 | #include <Interpreters/ExternalModelsLoader.h> |
42 | #include <Interpreters/ProcessList.h> |
43 | #include <Interpreters/loadMetadata.h> |
44 | #include <Interpreters/DNSCacheUpdater.h> |
45 | #include <Interpreters/SystemLog.cpp> |
46 | #include <Interpreters/ExternalLoaderXMLConfigRepository.h> |
47 | #include <Storages/StorageReplicatedMergeTree.h> |
48 | #include <Storages/System/attachSystemTables.h> |
49 | #include <AggregateFunctions/registerAggregateFunctions.h> |
50 | #include <Functions/registerFunctions.h> |
51 | #include <TableFunctions/registerTableFunctions.h> |
52 | #include <Storages/registerStorages.h> |
53 | #include <Dictionaries/registerDictionaries.h> |
54 | #include <Disks/registerDisks.h> |
55 | #include <Common/Config/ConfigReloader.h> |
56 | #include "HTTPHandlerFactory.h" |
57 | #include "MetricsTransmitter.h" |
58 | #include <Common/StatusFile.h> |
59 | #include "TCPHandlerFactory.h" |
60 | #include "Common/config_version.h" |
61 | #include <Common/SensitiveDataMasker.h> |
62 | #include "MySQLHandlerFactory.h" |
63 | |
64 | #if defined(OS_LINUX) |
65 | #include <Common/hasLinuxCapability.h> |
66 | #include <sys/mman.h> |
67 | #endif |
68 | |
69 | #if USE_POCO_NETSSL |
70 | #include <Poco/Net/Context.h> |
71 | #include <Poco/Net/SecureServerSocket.h> |
72 | #endif |
73 | |
74 | namespace CurrentMetrics |
75 | { |
76 | extern const Metric Revision; |
77 | extern const Metric VersionInteger; |
78 | } |
79 | |
80 | namespace DB |
81 | { |
82 | |
83 | namespace ErrorCodes |
84 | { |
85 | extern const int NO_ELEMENTS_IN_CONFIG; |
86 | extern const int SUPPORT_IS_DISABLED; |
87 | extern const int ARGUMENT_OUT_OF_BOUND; |
88 | extern const int EXCESSIVE_ELEMENT_IN_CONFIG; |
89 | extern const int INVALID_CONFIG_PARAMETER; |
90 | extern const int SYSTEM_ERROR; |
91 | extern const int FAILED_TO_GETPWUID; |
92 | extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; |
93 | extern const int NETWORK_ERROR; |
94 | extern const int PATH_ACCESS_DENIED; |
95 | } |
96 | |
97 | |
98 | static std::string getCanonicalPath(std::string && path) |
99 | { |
100 | Poco::trimInPlace(path); |
101 | if (path.empty()) |
102 | throw Exception("path configuration parameter is empty" , ErrorCodes::INVALID_CONFIG_PARAMETER); |
103 | if (path.back() != '/') |
104 | path += '/'; |
105 | return std::move(path); |
106 | } |
107 | |
108 | static std::string getUserName(uid_t user_id) |
109 | { |
110 | /// Try to convert user id into user name. |
111 | auto buffer_size = sysconf(_SC_GETPW_R_SIZE_MAX); |
112 | if (buffer_size <= 0) |
113 | buffer_size = 1024; |
114 | std::string buffer; |
115 | buffer.reserve(buffer_size); |
116 | |
117 | struct passwd passwd_entry; |
118 | struct passwd * result = nullptr; |
119 | const auto error = getpwuid_r(user_id, &passwd_entry, buffer.data(), buffer_size, &result); |
120 | |
121 | if (error) |
122 | throwFromErrno("Failed to find user name for " + toString(user_id), ErrorCodes::FAILED_TO_GETPWUID, error); |
123 | else if (result) |
124 | return result->pw_name; |
125 | return toString(user_id); |
126 | } |
127 | |
128 | void Server::uninitialize() |
129 | { |
130 | logger().information("shutting down" ); |
131 | BaseDaemon::uninitialize(); |
132 | } |
133 | |
134 | int Server::run() |
135 | { |
136 | if (config().hasOption("help" )) |
137 | { |
138 | Poco::Util::HelpFormatter helpFormatter(Server::options()); |
139 | std::stringstream ; |
140 | header << commandName() << " [OPTION] [-- [ARG]...]\n" ; |
141 | header << "positional arguments can be used to rewrite config.xml properties, for example, --http_port=8010" ; |
142 | helpFormatter.setHeader(header.str()); |
143 | helpFormatter.format(std::cout); |
144 | return 0; |
145 | } |
146 | if (config().hasOption("version" )) |
147 | { |
148 | std::cout << DBMS_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl; |
149 | return 0; |
150 | } |
151 | return Application::run(); |
152 | } |
153 | |
154 | void Server::initialize(Poco::Util::Application & self) |
155 | { |
156 | BaseDaemon::initialize(self); |
157 | logger().information("starting up" ); |
158 | } |
159 | |
160 | std::string Server::getDefaultCorePath() const |
161 | { |
162 | return getCanonicalPath(config().getString("path" , DBMS_DEFAULT_PATH)) + "cores" ; |
163 | } |
164 | |
165 | void Server::defineOptions(Poco::Util::OptionSet & options) |
166 | { |
167 | options.addOption( |
168 | Poco::Util::Option("help" , "h" , "show help and exit" ) |
169 | .required(false) |
170 | .repeatable(false) |
171 | .binding("help" )); |
172 | options.addOption( |
173 | Poco::Util::Option("version" , "V" , "show version and exit" ) |
174 | .required(false) |
175 | .repeatable(false) |
176 | .binding("version" )); |
177 | BaseDaemon::defineOptions(options); |
178 | } |
179 | |
180 | int Server::main(const std::vector<std::string> & /*args*/) |
181 | { |
182 | Logger * log = &logger(); |
183 | UseSSL use_ssl; |
184 | |
185 | ThreadStatus thread_status; |
186 | |
187 | registerFunctions(); |
188 | registerAggregateFunctions(); |
189 | registerTableFunctions(); |
190 | registerStorages(); |
191 | registerDictionaries(); |
192 | registerDisks(); |
193 | |
194 | CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); |
195 | CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); |
196 | |
197 | /** Context contains all that query execution is dependent: |
198 | * settings, available functions, data types, aggregate functions, databases... |
199 | */ |
200 | global_context = std::make_unique<Context>(Context::createGlobal()); |
201 | global_context->makeGlobalContext(); |
202 | global_context->setApplicationType(Context::ApplicationType::SERVER); |
203 | |
204 | bool has_zookeeper = config().has("zookeeper" ); |
205 | |
206 | zkutil::ZooKeeperNodeCache main_config_zk_node_cache([&] { return global_context->getZooKeeper(); }); |
207 | zkutil::EventPtr main_config_zk_changed_event = std::make_shared<Poco::Event>(); |
208 | if (loaded_config.has_zk_includes) |
209 | { |
210 | auto old_configuration = loaded_config.configuration; |
211 | ConfigProcessor config_processor(config_path); |
212 | loaded_config = config_processor.loadConfigWithZooKeeperIncludes( |
213 | main_config_zk_node_cache, main_config_zk_changed_event, /* fallback_to_preprocessed = */ true); |
214 | config_processor.savePreprocessedConfig(loaded_config, config().getString("path" , DBMS_DEFAULT_PATH)); |
215 | config().removeConfiguration(old_configuration.get()); |
216 | config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); |
217 | } |
218 | |
219 | const auto memory_amount = getMemoryAmount(); |
220 | |
221 | #if defined(__linux__) |
222 | std::string executable_path = getExecutablePath(); |
223 | if (executable_path.empty()) |
224 | executable_path = "/usr/bin/clickhouse" ; /// It is used for information messages. |
225 | |
226 | /// After full config loaded |
227 | { |
228 | if (config().getBool("mlock_executable" , false)) |
229 | { |
230 | if (hasLinuxCapability(CAP_IPC_LOCK)) |
231 | { |
232 | LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds." ); |
233 | if (0 != mlockall(MCL_CURRENT)) |
234 | LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR)); |
235 | else |
236 | LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed" ); |
237 | } |
238 | else |
239 | { |
240 | LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." |
241 | " It could happen due to incorrect ClickHouse package installation." |
242 | " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep " << executable_path << "'." |
243 | " Note that it will not work on 'nosuid' mounted filesystems." ); |
244 | } |
245 | } |
246 | } |
247 | #endif |
248 | |
249 | global_context->setRemoteHostFilter(config()); |
250 | |
251 | std::string path = getCanonicalPath(config().getString("path" , DBMS_DEFAULT_PATH)); |
252 | std::string default_database = config().getString("default_database" , "default" ); |
253 | |
254 | /// Check that the process' user id matches the owner of the data. |
255 | const auto effective_user_id = geteuid(); |
256 | struct stat statbuf; |
257 | if (stat(path.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid) |
258 | { |
259 | const auto effective_user = getUserName(effective_user_id); |
260 | const auto data_owner = getUserName(statbuf.st_uid); |
261 | std::string message = "Effective user of the process (" + effective_user + |
262 | ") does not match the owner of the data (" + data_owner + ")." ; |
263 | if (effective_user_id == 0) |
264 | { |
265 | message += " Run under 'sudo -u " + data_owner + "'." ; |
266 | throw Exception(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA); |
267 | } |
268 | else |
269 | { |
270 | LOG_WARNING(log, message); |
271 | } |
272 | } |
273 | |
274 | global_context->setPath(path); |
275 | |
276 | /// Create directories for 'path' and for default database, if not exist. |
277 | Poco::File(path + "data/" + default_database).createDirectories(); |
278 | Poco::File(path + "metadata/" + default_database).createDirectories(); |
279 | |
280 | /// Check that we have read and write access to all data paths |
281 | auto disk_selector = global_context->getDiskSelector(); |
282 | for (const auto & [name, disk] : disk_selector.getDisksMap()) |
283 | { |
284 | Poco::File disk_path(disk->getPath()); |
285 | if (!disk_path.canRead() || !disk_path.canWrite()) |
286 | throw Exception("There is no RW access to disk " + name + " (" + disk->getPath() + ")" , ErrorCodes::PATH_ACCESS_DENIED); |
287 | } |
288 | |
289 | StatusFile status{path + "status" }; |
290 | |
291 | SCOPE_EXIT({ |
292 | /** Ask to cancel background jobs all table engines, |
293 | * and also query_log. |
294 | * It is important to do early, not in destructor of Context, because |
295 | * table engines could use Context on destroy. |
296 | */ |
297 | LOG_INFO(log, "Shutting down storages." ); |
298 | |
299 | global_context->shutdown(); |
300 | |
301 | LOG_DEBUG(log, "Shut down storages." ); |
302 | |
303 | /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. |
304 | * At this moment, no one could own shared part of Context. |
305 | */ |
306 | global_context.reset(); |
307 | LOG_DEBUG(log, "Destroyed global context." ); |
308 | }); |
309 | |
310 | /// Try to increase limit on number of open files. |
311 | { |
312 | rlimit rlim; |
313 | if (getrlimit(RLIMIT_NOFILE, &rlim)) |
314 | throw Poco::Exception("Cannot getrlimit" ); |
315 | |
316 | if (rlim.rlim_cur == rlim.rlim_max) |
317 | { |
318 | LOG_DEBUG(log, "rlimit on number of file descriptors is " << rlim.rlim_cur); |
319 | } |
320 | else |
321 | { |
322 | rlim_t old = rlim.rlim_cur; |
323 | rlim.rlim_cur = config().getUInt("max_open_files" , rlim.rlim_max); |
324 | int rc = setrlimit(RLIMIT_NOFILE, &rlim); |
325 | if (rc != 0) |
326 | LOG_WARNING(log, |
327 | "Cannot set max number of file descriptors to " << rlim.rlim_cur |
328 | << ". Try to specify max_open_files according to your system limits. error: " |
329 | << strerror(errno)); |
330 | else |
331 | LOG_DEBUG(log, "Set max number of file descriptors to " << rlim.rlim_cur << " (was " << old << ")." ); |
332 | } |
333 | } |
334 | |
335 | static ServerErrorHandler error_handler; |
336 | Poco::ErrorHandler::set(&error_handler); |
337 | |
338 | /// Initialize DateLUT early, to not interfere with running time of first query. |
339 | LOG_DEBUG(log, "Initializing DateLUT." ); |
340 | DateLUT::instance(); |
341 | LOG_TRACE(log, "Initialized DateLUT with time zone '" << DateLUT::instance().getTimeZone() << "'." ); |
342 | |
343 | /// Directory with temporary data for processing of heavy queries. |
344 | { |
345 | std::string tmp_path = config().getString("tmp_path" , path + "tmp/" ); |
346 | global_context->setTemporaryPath(tmp_path); |
347 | Poco::File(tmp_path).createDirectories(); |
348 | |
349 | /// Clearing old temporary files. |
350 | Poco::DirectoryIterator dir_end; |
351 | for (Poco::DirectoryIterator it(tmp_path); it != dir_end; ++it) |
352 | { |
353 | if (it->isFile() && startsWith(it.name(), "tmp" )) |
354 | { |
355 | LOG_DEBUG(log, "Removing old temporary file " << it->path()); |
356 | it->remove(); |
357 | } |
358 | } |
359 | } |
360 | |
361 | /** Directory with 'flags': files indicating temporary settings for the server set by system administrator. |
362 | * Flags may be cleared automatically after being applied by the server. |
363 | * Examples: do repair of local data; clone all replicated tables from replica. |
364 | */ |
365 | { |
366 | Poco::File(path + "flags/" ).createDirectories(); |
367 | global_context->setFlagsPath(path + "flags/" ); |
368 | } |
369 | |
370 | /** Directory with user provided files that are usable by 'file' table function. |
371 | */ |
372 | { |
373 | |
374 | std::string user_files_path = config().getString("user_files_path" , path + "user_files/" ); |
375 | global_context->setUserFilesPath(user_files_path); |
376 | Poco::File(user_files_path).createDirectories(); |
377 | } |
378 | |
379 | { |
380 | std::string dictionaries_lib_path = config().getString("dictionaries_lib_path" , path + "dictionaries_lib/" ); |
381 | global_context->setDictionariesLibPath(dictionaries_lib_path); |
382 | Poco::File(dictionaries_lib_path).createDirectories(); |
383 | } |
384 | |
385 | if (config().has("interserver_http_port" ) && config().has("interserver_https_port" )) |
386 | throw Exception("Both http and https interserver ports are specified" , ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); |
387 | |
388 | static const auto interserver_tags = |
389 | { |
390 | std::make_tuple("interserver_http_host" , "interserver_http_port" , "http" ), |
391 | std::make_tuple("interserver_https_host" , "interserver_https_port" , "https" ) |
392 | }; |
393 | |
394 | for (auto [host_tag, port_tag, scheme] : interserver_tags) |
395 | { |
396 | if (config().has(port_tag)) |
397 | { |
398 | String this_host = config().getString(host_tag, "" ); |
399 | |
400 | if (this_host.empty()) |
401 | { |
402 | this_host = getFQDNOrHostName(); |
403 | LOG_DEBUG(log, |
404 | "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host |
405 | + "' as replica host." ); |
406 | } |
407 | |
408 | String port_str = config().getString(port_tag); |
409 | int port = parse<int>(port_str); |
410 | |
411 | if (port < 0 || port > 0xFFFF) |
412 | throw Exception("Out of range '" + String(port_tag) + "': " + toString(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND); |
413 | |
414 | global_context->setInterserverIOAddress(this_host, port); |
415 | global_context->setInterserverScheme(scheme); |
416 | } |
417 | } |
418 | |
419 | if (config().has("interserver_http_credentials" )) |
420 | { |
421 | String user = config().getString("interserver_http_credentials.user" , "" ); |
422 | String password = config().getString("interserver_http_credentials.password" , "" ); |
423 | |
424 | if (user.empty()) |
425 | throw Exception("Configuration parameter interserver_http_credentials user can't be empty" , ErrorCodes::NO_ELEMENTS_IN_CONFIG); |
426 | |
427 | global_context->setInterserverCredentials(user, password); |
428 | } |
429 | |
430 | if (config().has("macros" )) |
431 | global_context->setMacros(std::make_unique<Macros>(config(), "macros" )); |
432 | |
433 | /// Initialize main config reloader. |
434 | std::string include_from_path = config().getString("include_from" , "/etc/metrika.xml" ); |
435 | |
436 | if (config().has("query_masking_rules" )) |
437 | { |
438 | SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules" )); |
439 | } |
440 | |
441 | auto main_config_reloader = std::make_unique<ConfigReloader>(config_path, |
442 | include_from_path, |
443 | config().getString("path" , "" ), |
444 | std::move(main_config_zk_node_cache), |
445 | main_config_zk_changed_event, |
446 | [&](ConfigurationPtr config) |
447 | { |
448 | setTextLog(global_context->getTextLog()); |
449 | buildLoggers(*config, logger()); |
450 | global_context->setClustersConfig(config); |
451 | global_context->setMacros(std::make_unique<Macros>(*config, "macros" )); |
452 | |
453 | /// Setup protection to avoid accidental DROP for big tables (that are greater than 50 GB by default) |
454 | if (config->has("max_table_size_to_drop" )) |
455 | global_context->setMaxTableSizeToDrop(config->getUInt64("max_table_size_to_drop" )); |
456 | |
457 | if (config->has("max_partition_size_to_drop" )) |
458 | global_context->setMaxPartitionSizeToDrop(config->getUInt64("max_partition_size_to_drop" )); |
459 | }, |
460 | /* already_loaded = */ true); |
461 | |
462 | /// Initialize users config reloader. |
463 | std::string users_config_path = config().getString("users_config" , config_path); |
464 | /// If path to users' config isn't absolute, try guess its root (current) dir. |
465 | /// At first, try to find it in dir of main config, after will use current dir. |
466 | if (users_config_path.empty() || users_config_path[0] != '/') |
467 | { |
468 | std::string config_dir = Poco::Path(config_path).parent().toString(); |
469 | if (Poco::File(config_dir + users_config_path).exists()) |
470 | users_config_path = config_dir + users_config_path; |
471 | } |
472 | auto users_config_reloader = std::make_unique<ConfigReloader>(users_config_path, |
473 | include_from_path, |
474 | config().getString("path" , "" ), |
475 | zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }), |
476 | std::make_shared<Poco::Event>(), |
477 | [&](ConfigurationPtr config) { global_context->setUsersConfig(config); }, |
478 | /* already_loaded = */ false); |
479 | |
480 | /// Reload config in SYSTEM RELOAD CONFIG query. |
481 | global_context->setConfigReloadCallback([&]() |
482 | { |
483 | main_config_reloader->reload(); |
484 | users_config_reloader->reload(); |
485 | }); |
486 | |
487 | /// Limit on total number of concurrently executed queries. |
488 | global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries" , 0)); |
489 | |
490 | /// Set up caches. |
491 | |
492 | /// Lower cache size on low-memory systems. |
493 | double cache_size_to_ram_max_ratio = config().getDouble("cache_size_to_ram_max_ratio" , 0.5); |
494 | size_t max_cache_size = memory_amount * cache_size_to_ram_max_ratio; |
495 | |
496 | /// Size of cache for uncompressed blocks. Zero means disabled. |
497 | size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size" , 0); |
498 | if (uncompressed_cache_size > max_cache_size) |
499 | { |
500 | uncompressed_cache_size = max_cache_size; |
501 | LOG_INFO(log, "Uncompressed cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) |
502 | << " because the system has low amount of memory" ); |
503 | } |
504 | global_context->setUncompressedCache(uncompressed_cache_size); |
505 | |
506 | /// Load global settings from default_profile and system_profile. |
507 | global_context->setDefaultProfiles(config()); |
508 | Settings & settings = global_context->getSettingsRef(); |
509 | |
510 | /// Size of cache for marks (index of MergeTree family of tables). It is mandatory. |
511 | size_t mark_cache_size = config().getUInt64("mark_cache_size" ); |
512 | if (!mark_cache_size) |
513 | LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation." ); |
514 | if (mark_cache_size > max_cache_size) |
515 | { |
516 | mark_cache_size = max_cache_size; |
517 | LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size) |
518 | << " because the system has low amount of memory" ); |
519 | } |
520 | global_context->setMarkCache(mark_cache_size); |
521 | |
522 | #if USE_EMBEDDED_COMPILER |
523 | size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size" , 500); |
524 | if (compiled_expression_cache_size) |
525 | global_context->setCompiledExpressionCache(compiled_expression_cache_size); |
526 | #endif |
527 | |
528 | /// Set path for format schema files |
529 | auto format_schema_path = Poco::File(config().getString("format_schema_path" , path + "format_schemas/" )); |
530 | global_context->setFormatSchemaPath(format_schema_path.path()); |
531 | format_schema_path.createDirectories(); |
532 | |
533 | LOG_INFO(log, "Loading metadata from " + path); |
534 | |
535 | try |
536 | { |
537 | loadMetadataSystem(*global_context); |
538 | /// After attaching system databases we can initialize system log. |
539 | global_context->initializeSystemLogs(); |
540 | /// After the system database is created, attach virtual system tables (in addition to query_log and part_log) |
541 | attachSystemTablesServer(*global_context->getDatabase("system" ), has_zookeeper); |
542 | /// Then, load remaining databases |
543 | loadMetadata(*global_context); |
544 | } |
545 | catch (...) |
546 | { |
547 | tryLogCurrentException(log, "Caught exception while loading metadata" ); |
548 | throw; |
549 | } |
550 | LOG_DEBUG(log, "Loaded metadata." ); |
551 | |
552 | /// Init trace collector only after trace_log system table was created |
553 | /// Disable it if we collect test coverage information, because it will work extremely slow. |
554 | /// |
555 | /// It also cannot work with sanitizers. |
556 | /// Sanitizers are using quick "frame walking" stack unwinding (this implies -fno-omit-frame-pointer) |
557 | /// And they do unwiding frequently (on every malloc/free, thread/mutex operations, etc). |
558 | /// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwiding |
559 | /// and query profiler decide to unwind stack with libunwind at this moment. |
560 | /// |
561 | /// Symptoms: you'll get silent Segmentation Fault - without sanitizer message and without usual ClickHouse diagnostics. |
562 | /// |
563 | /// Look at compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h |
564 | /// |
565 | #if USE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER) |
566 | /// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache. |
567 | if (hasPHDRCache()) |
568 | global_context->initializeTraceCollector(); |
569 | #endif |
570 | |
571 | global_context->setCurrentDatabase(default_database); |
572 | |
573 | if (has_zookeeper && config().has("distributed_ddl" )) |
574 | { |
575 | /// DDL worker should be started after all tables were loaded |
576 | String ddl_zookeeper_path = config().getString("distributed_ddl.path" , "/clickhouse/task_queue/ddl/" ); |
577 | global_context->setDDLWorker(std::make_unique<DDLWorker>(ddl_zookeeper_path, *global_context, &config(), "distributed_ddl" )); |
578 | } |
579 | |
580 | std::unique_ptr<DNSCacheUpdater> dns_cache_updater; |
581 | if (config().has("disable_internal_dns_cache" ) && config().getInt("disable_internal_dns_cache" )) |
582 | { |
583 | /// Disable DNS caching at all |
584 | DNSResolver::instance().setDisableCacheFlag(); |
585 | } |
586 | else |
587 | { |
588 | /// Initialize a watcher periodically updating DNS cache |
589 | dns_cache_updater = std::make_unique<DNSCacheUpdater>(*global_context, config().getInt("dns_cache_update_period" , 15)); |
590 | } |
591 | |
592 | #if defined(__linux__) |
593 | if (!TaskStatsInfoGetter::checkPermissions()) |
594 | { |
595 | LOG_INFO(log, "It looks like the process has no CAP_NET_ADMIN capability, 'taskstats' performance statistics will be disabled." |
596 | " It could happen due to incorrect ClickHouse package installation." |
597 | " You could resolve the problem manually with 'sudo setcap cap_net_admin=+ep " << executable_path << "'." |
598 | " Note that it will not work on 'nosuid' mounted filesystems." |
599 | " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers." ); |
600 | } |
601 | |
602 | if (!hasLinuxCapability(CAP_SYS_NICE)) |
603 | { |
604 | LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect." |
605 | " It could happen due to incorrect ClickHouse package installation." |
606 | " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'." |
607 | " Note that it will not work on 'nosuid' mounted filesystems." ); |
608 | } |
609 | #else |
610 | LOG_INFO(log, "TaskStats is not implemented for this OS. IO accounting will be disabled." ); |
611 | #endif |
612 | |
613 | { |
614 | Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout" , 10), 0); |
615 | |
616 | Poco::ThreadPool server_pool(3, config().getUInt("max_connections" , 1024)); |
617 | Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; |
618 | http_params->setTimeout(settings.http_receive_timeout); |
619 | http_params->setKeepAliveTimeout(keep_alive_timeout); |
620 | |
621 | std::vector<std::unique_ptr<Poco::Net::TCPServer>> servers; |
622 | |
623 | std::vector<std::string> listen_hosts = DB::getMultipleValuesFromConfig(config(), "" , "listen_host" ); |
624 | |
625 | bool listen_try = config().getBool("listen_try" , false); |
626 | if (listen_hosts.empty()) |
627 | { |
628 | listen_hosts.emplace_back("::1" ); |
629 | listen_hosts.emplace_back("127.0.0.1" ); |
630 | listen_try = true; |
631 | } |
632 | |
633 | auto make_socket_address = [&](const std::string & host, UInt16 port) |
634 | { |
635 | Poco::Net::SocketAddress socket_address; |
636 | try |
637 | { |
638 | socket_address = Poco::Net::SocketAddress(host, port); |
639 | } |
640 | catch (const Poco::Net::DNSException & e) |
641 | { |
642 | const auto code = e.code(); |
643 | if (code == EAI_FAMILY |
644 | #if defined(EAI_ADDRFAMILY) |
645 | || code == EAI_ADDRFAMILY |
646 | #endif |
647 | ) |
648 | { |
649 | LOG_ERROR(log, |
650 | "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". " |
651 | "If it is an IPv6 address and your host has disabled IPv6, then consider to " |
652 | "specify IPv4 address to listen in <listen_host> element of configuration " |
653 | "file. Example: <listen_host>0.0.0.0</listen_host>" ); |
654 | } |
655 | |
656 | throw; |
657 | } |
658 | return socket_address; |
659 | }; |
660 | |
661 | auto socket_bind_listen = [&](auto & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = 0) |
662 | { |
663 | auto address = make_socket_address(host, port); |
664 | #if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100 |
665 | if (secure) |
666 | /// Bug in old (<1.9.1) poco, listen() after bind() with reusePort param will fail because have no implementation in SecureServerSocketImpl |
667 | /// https://github.com/pocoproject/poco/pull/2257 |
668 | socket.bind(address, /* reuseAddress = */ true); |
669 | else |
670 | #endif |
671 | #if POCO_VERSION < 0x01080000 |
672 | socket.bind(address, /* reuseAddress = */ true); |
673 | #else |
674 | socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port" , false)); |
675 | #endif |
676 | |
677 | socket.listen(/* backlog = */ config().getUInt("listen_backlog" , 64)); |
678 | |
679 | return address; |
680 | }; |
681 | |
682 | /// This object will periodically calculate some metrics. |
683 | AsynchronousMetrics async_metrics(*global_context); |
684 | attachSystemTablesAsync(*global_context->getDatabase("system" ), async_metrics); |
685 | |
686 | for (const auto & listen_host : listen_hosts) |
687 | { |
688 | auto create_server = [&](const char * port_name, auto && func) |
689 | { |
690 | /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file. |
691 | if (!config().has(port_name)) |
692 | return; |
693 | |
694 | auto port = config().getInt(port_name); |
695 | try |
696 | { |
697 | func(port); |
698 | } |
699 | catch (const Poco::Exception &) |
700 | { |
701 | std::string message = "Listen [" + listen_host + "]:" + std::to_string(port) + " failed: " + getCurrentExceptionMessage(false); |
702 | |
703 | if (listen_try) |
704 | { |
705 | LOG_ERROR(log, message |
706 | << ". If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " |
707 | "specify not disabled IPv4 or IPv6 address to listen in <listen_host> element of configuration " |
708 | "file. Example for disabled IPv6: <listen_host>0.0.0.0</listen_host> ." |
709 | " Example for disabled IPv4: <listen_host>::</listen_host>" ); |
710 | } |
711 | else |
712 | { |
713 | throw Exception{message, ErrorCodes::NETWORK_ERROR}; |
714 | } |
715 | } |
716 | }; |
717 | |
718 | /// HTTP |
719 | create_server("http_port" , [&](UInt16 port) |
720 | { |
721 | Poco::Net::ServerSocket socket; |
722 | auto address = socket_bind_listen(socket, listen_host, port); |
723 | socket.setReceiveTimeout(settings.http_receive_timeout); |
724 | socket.setSendTimeout(settings.http_send_timeout); |
725 | auto handler_factory = createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPHandler-factory" ); |
726 | if (config().has("prometheus" ) && config().getInt("prometheus.port" , 0) == 0) |
727 | handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics); |
728 | |
729 | servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>( |
730 | handler_factory, |
731 | server_pool, |
732 | socket, |
733 | http_params)); |
734 | |
735 | LOG_INFO(log, "Listening for http://" + address.toString()); |
736 | }); |
737 | |
738 | /// HTTPS |
739 | create_server("https_port" , [&](UInt16 port) |
740 | { |
741 | #if USE_POCO_NETSSL |
742 | Poco::Net::SecureServerSocket socket; |
743 | auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); |
744 | socket.setReceiveTimeout(settings.http_receive_timeout); |
745 | socket.setSendTimeout(settings.http_send_timeout); |
746 | servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>( |
747 | createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPSHandler-factory" ), |
748 | server_pool, |
749 | socket, |
750 | http_params)); |
751 | |
752 | LOG_INFO(log, "Listening for https://" + address.toString()); |
753 | #else |
754 | UNUSED(port); |
755 | throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support." , |
756 | ErrorCodes::SUPPORT_IS_DISABLED}; |
757 | #endif |
758 | }); |
759 | |
760 | /// TCP |
761 | create_server("tcp_port" , [&](UInt16 port) |
762 | { |
763 | Poco::Net::ServerSocket socket; |
764 | auto address = socket_bind_listen(socket, listen_host, port); |
765 | socket.setReceiveTimeout(settings.receive_timeout); |
766 | socket.setSendTimeout(settings.send_timeout); |
767 | servers.emplace_back(std::make_unique<Poco::Net::TCPServer>( |
768 | new TCPHandlerFactory(*this), |
769 | server_pool, |
770 | socket, |
771 | new Poco::Net::TCPServerParams)); |
772 | |
773 | LOG_INFO(log, "Listening for connections with native protocol (tcp): " + address.toString()); |
774 | }); |
775 | |
776 | /// TCP with SSL |
777 | create_server("tcp_port_secure" , [&](UInt16 port) |
778 | { |
779 | #if USE_POCO_NETSSL |
780 | Poco::Net::SecureServerSocket socket; |
781 | auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); |
782 | socket.setReceiveTimeout(settings.receive_timeout); |
783 | socket.setSendTimeout(settings.send_timeout); |
784 | servers.emplace_back(std::make_unique<Poco::Net::TCPServer>( |
785 | new TCPHandlerFactory(*this, /* secure= */ true), |
786 | server_pool, |
787 | socket, |
788 | new Poco::Net::TCPServerParams)); |
789 | LOG_INFO(log, "Listening for connections with secure native protocol (tcp_secure): " + address.toString()); |
790 | #else |
791 | UNUSED(port); |
792 | throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support." , |
793 | ErrorCodes::SUPPORT_IS_DISABLED}; |
794 | #endif |
795 | }); |
796 | |
797 | /// Interserver IO HTTP |
798 | create_server("interserver_http_port" , [&](UInt16 port) |
799 | { |
800 | Poco::Net::ServerSocket socket; |
801 | auto address = socket_bind_listen(socket, listen_host, port); |
802 | socket.setReceiveTimeout(settings.http_receive_timeout); |
803 | socket.setSendTimeout(settings.http_send_timeout); |
804 | servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>( |
805 | createDefaultHandlerFatory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory" ), |
806 | server_pool, |
807 | socket, |
808 | http_params)); |
809 | |
810 | LOG_INFO(log, "Listening for replica communication (interserver): http://" + address.toString()); |
811 | }); |
812 | |
813 | create_server("interserver_https_port" , [&](UInt16 port) |
814 | { |
815 | #if USE_POCO_NETSSL |
816 | Poco::Net::SecureServerSocket socket; |
817 | auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); |
818 | socket.setReceiveTimeout(settings.http_receive_timeout); |
819 | socket.setSendTimeout(settings.http_send_timeout); |
820 | servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>( |
821 | createDefaultHandlerFatory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory" ), |
822 | server_pool, |
823 | socket, |
824 | http_params)); |
825 | |
826 | LOG_INFO(log, "Listening for secure replica communication (interserver): https://" + address.toString()); |
827 | #else |
828 | UNUSED(port); |
829 | throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support." , |
830 | ErrorCodes::SUPPORT_IS_DISABLED}; |
831 | #endif |
832 | }); |
833 | |
834 | create_server("mysql_port" , [&](UInt16 port) |
835 | { |
836 | Poco::Net::ServerSocket socket; |
837 | auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true); |
838 | socket.setReceiveTimeout(Poco::Timespan()); |
839 | socket.setSendTimeout(settings.send_timeout); |
840 | servers.emplace_back(std::make_unique<Poco::Net::TCPServer>( |
841 | new MySQLHandlerFactory(*this), |
842 | server_pool, |
843 | socket, |
844 | new Poco::Net::TCPServerParams)); |
845 | |
846 | LOG_INFO(log, "Listening for MySQL compatibility protocol: " + address.toString()); |
847 | }); |
848 | |
849 | /// Prometheus (if defined and not setup yet with http_port) |
850 | create_server("prometheus.port" , [&](UInt16 port) |
851 | { |
852 | Poco::Net::ServerSocket socket; |
853 | auto address = socket_bind_listen(socket, listen_host, port); |
854 | socket.setReceiveTimeout(settings.http_receive_timeout); |
855 | socket.setSendTimeout(settings.http_send_timeout); |
856 | auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory" ); |
857 | handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics); |
858 | servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>( |
859 | handler_factory, |
860 | server_pool, |
861 | socket, |
862 | http_params)); |
863 | |
864 | LOG_INFO(log, "Listening for Prometheus: http://" + address.toString()); |
865 | }); |
866 | } |
867 | |
868 | if (servers.empty()) |
869 | throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)" , ErrorCodes::NO_ELEMENTS_IN_CONFIG); |
870 | |
871 | for (auto & server : servers) |
872 | server->start(); |
873 | |
874 | main_config_reloader->start(); |
875 | users_config_reloader->start(); |
876 | if (dns_cache_updater) |
877 | dns_cache_updater->start(); |
878 | |
879 | { |
880 | std::stringstream message; |
881 | message << "Available RAM: " << formatReadableSizeWithBinarySuffix(memory_amount) << ";" |
882 | << " physical cores: " << getNumberOfPhysicalCPUCores() << ";" |
883 | // on ARM processors it can show only enabled at current moment cores |
884 | << " logical cores: " << std::thread::hardware_concurrency() << "." ; |
885 | LOG_INFO(log, message.str()); |
886 | } |
887 | |
888 | LOG_INFO(log, "Ready for connections." ); |
889 | |
890 | SCOPE_EXIT({ |
891 | LOG_DEBUG(log, "Received termination signal." ); |
892 | LOG_DEBUG(log, "Waiting for current connections to close." ); |
893 | |
894 | is_cancelled = true; |
895 | |
896 | int current_connections = 0; |
897 | for (auto & server : servers) |
898 | { |
899 | server->stop(); |
900 | current_connections += server->currentConnections(); |
901 | } |
902 | |
903 | LOG_INFO(log, |
904 | "Closed all listening sockets." |
905 | << (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : "" )); |
906 | |
907 | /// Killing remaining queries. |
908 | global_context->getProcessList().killAllQueries(); |
909 | |
910 | if (current_connections) |
911 | { |
912 | const int sleep_max_ms = 1000 * config().getInt("shutdown_wait_unfinished" , 5); |
913 | const int sleep_one_ms = 100; |
914 | int sleep_current_ms = 0; |
915 | while (sleep_current_ms < sleep_max_ms) |
916 | { |
917 | current_connections = 0; |
918 | for (auto & server : servers) |
919 | current_connections += server->currentConnections(); |
920 | if (!current_connections) |
921 | break; |
922 | sleep_current_ms += sleep_one_ms; |
923 | std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms)); |
924 | } |
925 | } |
926 | |
927 | LOG_INFO( |
928 | log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains." |
929 | " Tip: To increase wait time add to config: <shutdown_wait_unfinished>60</shutdown_wait_unfinished>" : "" )); |
930 | |
931 | dns_cache_updater.reset(); |
932 | main_config_reloader.reset(); |
933 | users_config_reloader.reset(); |
934 | |
935 | if (current_connections) |
936 | { |
937 | /// There is no better way to force connections to close in Poco. |
938 | /// Otherwise connection handlers will continue to live |
939 | /// (they are effectively dangling objects, but they use global thread pool |
940 | /// and global thread pool destructor will wait for threads, preventing server shutdown). |
941 | |
942 | /// Dump coverage here, because std::atexit callback would not be called. |
943 | dumpCoverageReportIfPossible(); |
944 | LOG_INFO(log, "Will shutdown forcefully." ); |
945 | _exit(Application::EXIT_OK); |
946 | } |
947 | }); |
948 | |
949 | /// try to load dictionaries immediately, throw on error and die |
950 | ext::scope_guard dictionaries_xmls, models_xmls; |
951 | try |
952 | { |
953 | if (!config().getBool("dictionaries_lazy_load" , true)) |
954 | { |
955 | global_context->tryCreateEmbeddedDictionaries(); |
956 | global_context->getExternalDictionariesLoader().enableAlwaysLoadEverything(true); |
957 | } |
958 | dictionaries_xmls = global_context->getExternalDictionariesLoader().addConfigRepository( |
959 | std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "dictionaries_config" )); |
960 | models_xmls = global_context->getExternalModelsLoader().addConfigRepository( |
961 | std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "models_config" )); |
962 | } |
963 | catch (...) |
964 | { |
965 | LOG_ERROR(log, "Caught exception while loading dictionaries." ); |
966 | throw; |
967 | } |
968 | |
969 | std::vector<std::unique_ptr<MetricsTransmitter>> metrics_transmitters; |
970 | for (const auto & graphite_key : DB::getMultipleKeysFromConfig(config(), "" , "graphite" )) |
971 | { |
972 | metrics_transmitters.emplace_back(std::make_unique<MetricsTransmitter>( |
973 | global_context->getConfigRef(), graphite_key, async_metrics)); |
974 | } |
975 | |
976 | SessionCleaner session_cleaner(*global_context); |
977 | |
978 | waitForTerminationRequest(); |
979 | } |
980 | |
981 | return Application::EXIT_OK; |
982 | } |
983 | } |
984 | |
985 | #pragma GCC diagnostic ignored "-Wunused-function" |
986 | #pragma GCC diagnostic ignored "-Wmissing-declarations" |
987 | |
988 | int mainEntryClickHouseServer(int argc, char ** argv) |
989 | { |
990 | DB::Server app; |
991 | try |
992 | { |
993 | return app.run(argc, argv); |
994 | } |
995 | catch (...) |
996 | { |
997 | std::cerr << DB::getCurrentExceptionMessage(true) << "\n" ; |
998 | auto code = DB::getCurrentExceptionCode(); |
999 | return code ? code : 1; |
1000 | } |
1001 | } |
1002 | |