1#include "Server.h"
2
3#include <memory>
4#include <sys/resource.h>
5#include <sys/stat.h>
6#include <sys/types.h>
7#include <errno.h>
8#include <pwd.h>
9#include <unistd.h>
10#include <Poco/Version.h>
11#include <Poco/DirectoryIterator.h>
12#include <Poco/Net/HTTPServer.h>
13#include <Poco/Net/NetException.h>
14#include <Poco/Util/HelpFormatter.h>
15#include <ext/scope_guard.h>
16#include <common/logger_useful.h>
17#include <common/phdr_cache.h>
18#include <common/config_common.h>
19#include <common/ErrorHandlers.h>
20#include <common/getMemoryAmount.h>
21#include <common/coverage.h>
22#include <Common/ClickHouseRevision.h>
23#include <Common/DNSResolver.h>
24#include <Common/CurrentMetrics.h>
25#include <Common/Macros.h>
26#include <Common/StringUtils/StringUtils.h>
27#include <Common/ZooKeeper/ZooKeeper.h>
28#include <Common/ZooKeeper/ZooKeeperNodeCache.h>
29#include "config_core.h"
30#include <Common/getFQDNOrHostName.h>
31#include <Common/getMultipleKeysFromConfig.h>
32#include <Common/getNumberOfPhysicalCPUCores.h>
33#include <Common/getExecutablePath.h>
34#include <Common/TaskStatsInfoGetter.h>
35#include <Common/ThreadStatus.h>
36#include <IO/HTTPCommon.h>
37#include <IO/UseSSL.h>
38#include <Interpreters/AsynchronousMetrics.h>
39#include <Interpreters/DDLWorker.h>
40#include <Interpreters/ExternalDictionariesLoader.h>
41#include <Interpreters/ExternalModelsLoader.h>
42#include <Interpreters/ProcessList.h>
43#include <Interpreters/loadMetadata.h>
44#include <Interpreters/DNSCacheUpdater.h>
45#include <Interpreters/SystemLog.cpp>
46#include <Interpreters/ExternalLoaderXMLConfigRepository.h>
47#include <Storages/StorageReplicatedMergeTree.h>
48#include <Storages/System/attachSystemTables.h>
49#include <AggregateFunctions/registerAggregateFunctions.h>
50#include <Functions/registerFunctions.h>
51#include <TableFunctions/registerTableFunctions.h>
52#include <Storages/registerStorages.h>
53#include <Dictionaries/registerDictionaries.h>
54#include <Disks/registerDisks.h>
55#include <Common/Config/ConfigReloader.h>
56#include "HTTPHandlerFactory.h"
57#include "MetricsTransmitter.h"
58#include <Common/StatusFile.h>
59#include "TCPHandlerFactory.h"
60#include "Common/config_version.h"
61#include <Common/SensitiveDataMasker.h>
62#include "MySQLHandlerFactory.h"
63
64#if defined(OS_LINUX)
65#include <Common/hasLinuxCapability.h>
66#include <sys/mman.h>
67#endif
68
69#if USE_POCO_NETSSL
70#include <Poco/Net/Context.h>
71#include <Poco/Net/SecureServerSocket.h>
72#endif
73
74namespace CurrentMetrics
75{
76 extern const Metric Revision;
77 extern const Metric VersionInteger;
78}
79
80namespace DB
81{
82
83namespace ErrorCodes
84{
85 extern const int NO_ELEMENTS_IN_CONFIG;
86 extern const int SUPPORT_IS_DISABLED;
87 extern const int ARGUMENT_OUT_OF_BOUND;
88 extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
89 extern const int INVALID_CONFIG_PARAMETER;
90 extern const int SYSTEM_ERROR;
91 extern const int FAILED_TO_GETPWUID;
92 extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA;
93 extern const int NETWORK_ERROR;
94 extern const int PATH_ACCESS_DENIED;
95}
96
97
98static std::string getCanonicalPath(std::string && path)
99{
100 Poco::trimInPlace(path);
101 if (path.empty())
102 throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
103 if (path.back() != '/')
104 path += '/';
105 return std::move(path);
106}
107
108static std::string getUserName(uid_t user_id)
109{
110 /// Try to convert user id into user name.
111 auto buffer_size = sysconf(_SC_GETPW_R_SIZE_MAX);
112 if (buffer_size <= 0)
113 buffer_size = 1024;
114 std::string buffer;
115 buffer.reserve(buffer_size);
116
117 struct passwd passwd_entry;
118 struct passwd * result = nullptr;
119 const auto error = getpwuid_r(user_id, &passwd_entry, buffer.data(), buffer_size, &result);
120
121 if (error)
122 throwFromErrno("Failed to find user name for " + toString(user_id), ErrorCodes::FAILED_TO_GETPWUID, error);
123 else if (result)
124 return result->pw_name;
125 return toString(user_id);
126}
127
128void Server::uninitialize()
129{
130 logger().information("shutting down");
131 BaseDaemon::uninitialize();
132}
133
134int Server::run()
135{
136 if (config().hasOption("help"))
137 {
138 Poco::Util::HelpFormatter helpFormatter(Server::options());
139 std::stringstream header;
140 header << commandName() << " [OPTION] [-- [ARG]...]\n";
141 header << "positional arguments can be used to rewrite config.xml properties, for example, --http_port=8010";
142 helpFormatter.setHeader(header.str());
143 helpFormatter.format(std::cout);
144 return 0;
145 }
146 if (config().hasOption("version"))
147 {
148 std::cout << DBMS_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
149 return 0;
150 }
151 return Application::run();
152}
153
154void Server::initialize(Poco::Util::Application & self)
155{
156 BaseDaemon::initialize(self);
157 logger().information("starting up");
158}
159
160std::string Server::getDefaultCorePath() const
161{
162 return getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)) + "cores";
163}
164
165void Server::defineOptions(Poco::Util::OptionSet & options)
166{
167 options.addOption(
168 Poco::Util::Option("help", "h", "show help and exit")
169 .required(false)
170 .repeatable(false)
171 .binding("help"));
172 options.addOption(
173 Poco::Util::Option("version", "V", "show version and exit")
174 .required(false)
175 .repeatable(false)
176 .binding("version"));
177 BaseDaemon::defineOptions(options);
178}
179
180int Server::main(const std::vector<std::string> & /*args*/)
181{
182 Logger * log = &logger();
183 UseSSL use_ssl;
184
185 ThreadStatus thread_status;
186
187 registerFunctions();
188 registerAggregateFunctions();
189 registerTableFunctions();
190 registerStorages();
191 registerDictionaries();
192 registerDisks();
193
194 CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
195 CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
196
197 /** Context contains all that query execution is dependent:
198 * settings, available functions, data types, aggregate functions, databases...
199 */
200 global_context = std::make_unique<Context>(Context::createGlobal());
201 global_context->makeGlobalContext();
202 global_context->setApplicationType(Context::ApplicationType::SERVER);
203
204 bool has_zookeeper = config().has("zookeeper");
205
206 zkutil::ZooKeeperNodeCache main_config_zk_node_cache([&] { return global_context->getZooKeeper(); });
207 zkutil::EventPtr main_config_zk_changed_event = std::make_shared<Poco::Event>();
208 if (loaded_config.has_zk_includes)
209 {
210 auto old_configuration = loaded_config.configuration;
211 ConfigProcessor config_processor(config_path);
212 loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
213 main_config_zk_node_cache, main_config_zk_changed_event, /* fallback_to_preprocessed = */ true);
214 config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
215 config().removeConfiguration(old_configuration.get());
216 config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
217 }
218
219 const auto memory_amount = getMemoryAmount();
220
221#if defined(__linux__)
222 std::string executable_path = getExecutablePath();
223 if (executable_path.empty())
224 executable_path = "/usr/bin/clickhouse"; /// It is used for information messages.
225
226 /// After full config loaded
227 {
228 if (config().getBool("mlock_executable", false))
229 {
230 if (hasLinuxCapability(CAP_IPC_LOCK))
231 {
232 LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds.");
233 if (0 != mlockall(MCL_CURRENT))
234 LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR));
235 else
236 LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed");
237 }
238 else
239 {
240 LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled."
241 " It could happen due to incorrect ClickHouse package installation."
242 " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep " << executable_path << "'."
243 " Note that it will not work on 'nosuid' mounted filesystems.");
244 }
245 }
246 }
247#endif
248
249 global_context->setRemoteHostFilter(config());
250
251 std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH));
252 std::string default_database = config().getString("default_database", "default");
253
254 /// Check that the process' user id matches the owner of the data.
255 const auto effective_user_id = geteuid();
256 struct stat statbuf;
257 if (stat(path.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid)
258 {
259 const auto effective_user = getUserName(effective_user_id);
260 const auto data_owner = getUserName(statbuf.st_uid);
261 std::string message = "Effective user of the process (" + effective_user +
262 ") does not match the owner of the data (" + data_owner + ").";
263 if (effective_user_id == 0)
264 {
265 message += " Run under 'sudo -u " + data_owner + "'.";
266 throw Exception(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA);
267 }
268 else
269 {
270 LOG_WARNING(log, message);
271 }
272 }
273
274 global_context->setPath(path);
275
276 /// Create directories for 'path' and for default database, if not exist.
277 Poco::File(path + "data/" + default_database).createDirectories();
278 Poco::File(path + "metadata/" + default_database).createDirectories();
279
280 /// Check that we have read and write access to all data paths
281 auto disk_selector = global_context->getDiskSelector();
282 for (const auto & [name, disk] : disk_selector.getDisksMap())
283 {
284 Poco::File disk_path(disk->getPath());
285 if (!disk_path.canRead() || !disk_path.canWrite())
286 throw Exception("There is no RW access to disk " + name + " (" + disk->getPath() + ")", ErrorCodes::PATH_ACCESS_DENIED);
287 }
288
289 StatusFile status{path + "status"};
290
291 SCOPE_EXIT({
292 /** Ask to cancel background jobs all table engines,
293 * and also query_log.
294 * It is important to do early, not in destructor of Context, because
295 * table engines could use Context on destroy.
296 */
297 LOG_INFO(log, "Shutting down storages.");
298
299 global_context->shutdown();
300
301 LOG_DEBUG(log, "Shut down storages.");
302
303 /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available.
304 * At this moment, no one could own shared part of Context.
305 */
306 global_context.reset();
307 LOG_DEBUG(log, "Destroyed global context.");
308 });
309
310 /// Try to increase limit on number of open files.
311 {
312 rlimit rlim;
313 if (getrlimit(RLIMIT_NOFILE, &rlim))
314 throw Poco::Exception("Cannot getrlimit");
315
316 if (rlim.rlim_cur == rlim.rlim_max)
317 {
318 LOG_DEBUG(log, "rlimit on number of file descriptors is " << rlim.rlim_cur);
319 }
320 else
321 {
322 rlim_t old = rlim.rlim_cur;
323 rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max);
324 int rc = setrlimit(RLIMIT_NOFILE, &rlim);
325 if (rc != 0)
326 LOG_WARNING(log,
327 "Cannot set max number of file descriptors to " << rlim.rlim_cur
328 << ". Try to specify max_open_files according to your system limits. error: "
329 << strerror(errno));
330 else
331 LOG_DEBUG(log, "Set max number of file descriptors to " << rlim.rlim_cur << " (was " << old << ").");
332 }
333 }
334
335 static ServerErrorHandler error_handler;
336 Poco::ErrorHandler::set(&error_handler);
337
338 /// Initialize DateLUT early, to not interfere with running time of first query.
339 LOG_DEBUG(log, "Initializing DateLUT.");
340 DateLUT::instance();
341 LOG_TRACE(log, "Initialized DateLUT with time zone '" << DateLUT::instance().getTimeZone() << "'.");
342
343 /// Directory with temporary data for processing of heavy queries.
344 {
345 std::string tmp_path = config().getString("tmp_path", path + "tmp/");
346 global_context->setTemporaryPath(tmp_path);
347 Poco::File(tmp_path).createDirectories();
348
349 /// Clearing old temporary files.
350 Poco::DirectoryIterator dir_end;
351 for (Poco::DirectoryIterator it(tmp_path); it != dir_end; ++it)
352 {
353 if (it->isFile() && startsWith(it.name(), "tmp"))
354 {
355 LOG_DEBUG(log, "Removing old temporary file " << it->path());
356 it->remove();
357 }
358 }
359 }
360
361 /** Directory with 'flags': files indicating temporary settings for the server set by system administrator.
362 * Flags may be cleared automatically after being applied by the server.
363 * Examples: do repair of local data; clone all replicated tables from replica.
364 */
365 {
366 Poco::File(path + "flags/").createDirectories();
367 global_context->setFlagsPath(path + "flags/");
368 }
369
370 /** Directory with user provided files that are usable by 'file' table function.
371 */
372 {
373
374 std::string user_files_path = config().getString("user_files_path", path + "user_files/");
375 global_context->setUserFilesPath(user_files_path);
376 Poco::File(user_files_path).createDirectories();
377 }
378
379 {
380 std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path + "dictionaries_lib/");
381 global_context->setDictionariesLibPath(dictionaries_lib_path);
382 Poco::File(dictionaries_lib_path).createDirectories();
383 }
384
385 if (config().has("interserver_http_port") && config().has("interserver_https_port"))
386 throw Exception("Both http and https interserver ports are specified", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
387
388 static const auto interserver_tags =
389 {
390 std::make_tuple("interserver_http_host", "interserver_http_port", "http"),
391 std::make_tuple("interserver_https_host", "interserver_https_port", "https")
392 };
393
394 for (auto [host_tag, port_tag, scheme] : interserver_tags)
395 {
396 if (config().has(port_tag))
397 {
398 String this_host = config().getString(host_tag, "");
399
400 if (this_host.empty())
401 {
402 this_host = getFQDNOrHostName();
403 LOG_DEBUG(log,
404 "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host
405 + "' as replica host.");
406 }
407
408 String port_str = config().getString(port_tag);
409 int port = parse<int>(port_str);
410
411 if (port < 0 || port > 0xFFFF)
412 throw Exception("Out of range '" + String(port_tag) + "': " + toString(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
413
414 global_context->setInterserverIOAddress(this_host, port);
415 global_context->setInterserverScheme(scheme);
416 }
417 }
418
419 if (config().has("interserver_http_credentials"))
420 {
421 String user = config().getString("interserver_http_credentials.user", "");
422 String password = config().getString("interserver_http_credentials.password", "");
423
424 if (user.empty())
425 throw Exception("Configuration parameter interserver_http_credentials user can't be empty", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
426
427 global_context->setInterserverCredentials(user, password);
428 }
429
430 if (config().has("macros"))
431 global_context->setMacros(std::make_unique<Macros>(config(), "macros"));
432
433 /// Initialize main config reloader.
434 std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
435
436 if (config().has("query_masking_rules"))
437 {
438 SensitiveDataMasker::setInstance(std::make_unique<SensitiveDataMasker>(config(), "query_masking_rules"));
439 }
440
441 auto main_config_reloader = std::make_unique<ConfigReloader>(config_path,
442 include_from_path,
443 config().getString("path", ""),
444 std::move(main_config_zk_node_cache),
445 main_config_zk_changed_event,
446 [&](ConfigurationPtr config)
447 {
448 setTextLog(global_context->getTextLog());
449 buildLoggers(*config, logger());
450 global_context->setClustersConfig(config);
451 global_context->setMacros(std::make_unique<Macros>(*config, "macros"));
452
453 /// Setup protection to avoid accidental DROP for big tables (that are greater than 50 GB by default)
454 if (config->has("max_table_size_to_drop"))
455 global_context->setMaxTableSizeToDrop(config->getUInt64("max_table_size_to_drop"));
456
457 if (config->has("max_partition_size_to_drop"))
458 global_context->setMaxPartitionSizeToDrop(config->getUInt64("max_partition_size_to_drop"));
459 },
460 /* already_loaded = */ true);
461
462 /// Initialize users config reloader.
463 std::string users_config_path = config().getString("users_config", config_path);
464 /// If path to users' config isn't absolute, try guess its root (current) dir.
465 /// At first, try to find it in dir of main config, after will use current dir.
466 if (users_config_path.empty() || users_config_path[0] != '/')
467 {
468 std::string config_dir = Poco::Path(config_path).parent().toString();
469 if (Poco::File(config_dir + users_config_path).exists())
470 users_config_path = config_dir + users_config_path;
471 }
472 auto users_config_reloader = std::make_unique<ConfigReloader>(users_config_path,
473 include_from_path,
474 config().getString("path", ""),
475 zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }),
476 std::make_shared<Poco::Event>(),
477 [&](ConfigurationPtr config) { global_context->setUsersConfig(config); },
478 /* already_loaded = */ false);
479
480 /// Reload config in SYSTEM RELOAD CONFIG query.
481 global_context->setConfigReloadCallback([&]()
482 {
483 main_config_reloader->reload();
484 users_config_reloader->reload();
485 });
486
487 /// Limit on total number of concurrently executed queries.
488 global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0));
489
490 /// Set up caches.
491
492 /// Lower cache size on low-memory systems.
493 double cache_size_to_ram_max_ratio = config().getDouble("cache_size_to_ram_max_ratio", 0.5);
494 size_t max_cache_size = memory_amount * cache_size_to_ram_max_ratio;
495
496 /// Size of cache for uncompressed blocks. Zero means disabled.
497 size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0);
498 if (uncompressed_cache_size > max_cache_size)
499 {
500 uncompressed_cache_size = max_cache_size;
501 LOG_INFO(log, "Uncompressed cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size)
502 << " because the system has low amount of memory");
503 }
504 global_context->setUncompressedCache(uncompressed_cache_size);
505
506 /// Load global settings from default_profile and system_profile.
507 global_context->setDefaultProfiles(config());
508 Settings & settings = global_context->getSettingsRef();
509
510 /// Size of cache for marks (index of MergeTree family of tables). It is mandatory.
511 size_t mark_cache_size = config().getUInt64("mark_cache_size");
512 if (!mark_cache_size)
513 LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
514 if (mark_cache_size > max_cache_size)
515 {
516 mark_cache_size = max_cache_size;
517 LOG_INFO(log, "Mark cache size was lowered to " << formatReadableSizeWithBinarySuffix(uncompressed_cache_size)
518 << " because the system has low amount of memory");
519 }
520 global_context->setMarkCache(mark_cache_size);
521
522#if USE_EMBEDDED_COMPILER
523 size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", 500);
524 if (compiled_expression_cache_size)
525 global_context->setCompiledExpressionCache(compiled_expression_cache_size);
526#endif
527
528 /// Set path for format schema files
529 auto format_schema_path = Poco::File(config().getString("format_schema_path", path + "format_schemas/"));
530 global_context->setFormatSchemaPath(format_schema_path.path());
531 format_schema_path.createDirectories();
532
533 LOG_INFO(log, "Loading metadata from " + path);
534
535 try
536 {
537 loadMetadataSystem(*global_context);
538 /// After attaching system databases we can initialize system log.
539 global_context->initializeSystemLogs();
540 /// After the system database is created, attach virtual system tables (in addition to query_log and part_log)
541 attachSystemTablesServer(*global_context->getDatabase("system"), has_zookeeper);
542 /// Then, load remaining databases
543 loadMetadata(*global_context);
544 }
545 catch (...)
546 {
547 tryLogCurrentException(log, "Caught exception while loading metadata");
548 throw;
549 }
550 LOG_DEBUG(log, "Loaded metadata.");
551
552 /// Init trace collector only after trace_log system table was created
553 /// Disable it if we collect test coverage information, because it will work extremely slow.
554 ///
555 /// It also cannot work with sanitizers.
556 /// Sanitizers are using quick "frame walking" stack unwinding (this implies -fno-omit-frame-pointer)
557 /// And they do unwiding frequently (on every malloc/free, thread/mutex operations, etc).
558 /// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwiding
559 /// and query profiler decide to unwind stack with libunwind at this moment.
560 ///
561 /// Symptoms: you'll get silent Segmentation Fault - without sanitizer message and without usual ClickHouse diagnostics.
562 ///
563 /// Look at compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h
564 ///
565#if USE_UNWIND && !WITH_COVERAGE && !defined(SANITIZER)
566 /// QueryProfiler cannot work reliably with any other libunwind or without PHDR cache.
567 if (hasPHDRCache())
568 global_context->initializeTraceCollector();
569#endif
570
571 global_context->setCurrentDatabase(default_database);
572
573 if (has_zookeeper && config().has("distributed_ddl"))
574 {
575 /// DDL worker should be started after all tables were loaded
576 String ddl_zookeeper_path = config().getString("distributed_ddl.path", "/clickhouse/task_queue/ddl/");
577 global_context->setDDLWorker(std::make_unique<DDLWorker>(ddl_zookeeper_path, *global_context, &config(), "distributed_ddl"));
578 }
579
580 std::unique_ptr<DNSCacheUpdater> dns_cache_updater;
581 if (config().has("disable_internal_dns_cache") && config().getInt("disable_internal_dns_cache"))
582 {
583 /// Disable DNS caching at all
584 DNSResolver::instance().setDisableCacheFlag();
585 }
586 else
587 {
588 /// Initialize a watcher periodically updating DNS cache
589 dns_cache_updater = std::make_unique<DNSCacheUpdater>(*global_context, config().getInt("dns_cache_update_period", 15));
590 }
591
592#if defined(__linux__)
593 if (!TaskStatsInfoGetter::checkPermissions())
594 {
595 LOG_INFO(log, "It looks like the process has no CAP_NET_ADMIN capability, 'taskstats' performance statistics will be disabled."
596 " It could happen due to incorrect ClickHouse package installation."
597 " You could resolve the problem manually with 'sudo setcap cap_net_admin=+ep " << executable_path << "'."
598 " Note that it will not work on 'nosuid' mounted filesystems."
599 " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers.");
600 }
601
602 if (!hasLinuxCapability(CAP_SYS_NICE))
603 {
604 LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect."
605 " It could happen due to incorrect ClickHouse package installation."
606 " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep " << executable_path << "'."
607 " Note that it will not work on 'nosuid' mounted filesystems.");
608 }
609#else
610 LOG_INFO(log, "TaskStats is not implemented for this OS. IO accounting will be disabled.");
611#endif
612
613 {
614 Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);
615
616 Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024));
617 Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
618 http_params->setTimeout(settings.http_receive_timeout);
619 http_params->setKeepAliveTimeout(keep_alive_timeout);
620
621 std::vector<std::unique_ptr<Poco::Net::TCPServer>> servers;
622
623 std::vector<std::string> listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host");
624
625 bool listen_try = config().getBool("listen_try", false);
626 if (listen_hosts.empty())
627 {
628 listen_hosts.emplace_back("::1");
629 listen_hosts.emplace_back("127.0.0.1");
630 listen_try = true;
631 }
632
633 auto make_socket_address = [&](const std::string & host, UInt16 port)
634 {
635 Poco::Net::SocketAddress socket_address;
636 try
637 {
638 socket_address = Poco::Net::SocketAddress(host, port);
639 }
640 catch (const Poco::Net::DNSException & e)
641 {
642 const auto code = e.code();
643 if (code == EAI_FAMILY
644#if defined(EAI_ADDRFAMILY)
645 || code == EAI_ADDRFAMILY
646#endif
647 )
648 {
649 LOG_ERROR(log,
650 "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() << ". "
651 "If it is an IPv6 address and your host has disabled IPv6, then consider to "
652 "specify IPv4 address to listen in <listen_host> element of configuration "
653 "file. Example: <listen_host>0.0.0.0</listen_host>");
654 }
655
656 throw;
657 }
658 return socket_address;
659 };
660
661 auto socket_bind_listen = [&](auto & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = 0)
662 {
663 auto address = make_socket_address(host, port);
664#if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100
665 if (secure)
666 /// Bug in old (<1.9.1) poco, listen() after bind() with reusePort param will fail because have no implementation in SecureServerSocketImpl
667 /// https://github.com/pocoproject/poco/pull/2257
668 socket.bind(address, /* reuseAddress = */ true);
669 else
670#endif
671#if POCO_VERSION < 0x01080000
672 socket.bind(address, /* reuseAddress = */ true);
673#else
674 socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false));
675#endif
676
677 socket.listen(/* backlog = */ config().getUInt("listen_backlog", 64));
678
679 return address;
680 };
681
682 /// This object will periodically calculate some metrics.
683 AsynchronousMetrics async_metrics(*global_context);
684 attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics);
685
686 for (const auto & listen_host : listen_hosts)
687 {
688 auto create_server = [&](const char * port_name, auto && func)
689 {
690 /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file.
691 if (!config().has(port_name))
692 return;
693
694 auto port = config().getInt(port_name);
695 try
696 {
697 func(port);
698 }
699 catch (const Poco::Exception &)
700 {
701 std::string message = "Listen [" + listen_host + "]:" + std::to_string(port) + " failed: " + getCurrentExceptionMessage(false);
702
703 if (listen_try)
704 {
705 LOG_ERROR(log, message
706 << ". If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to "
707 "specify not disabled IPv4 or IPv6 address to listen in <listen_host> element of configuration "
708 "file. Example for disabled IPv6: <listen_host>0.0.0.0</listen_host> ."
709 " Example for disabled IPv4: <listen_host>::</listen_host>");
710 }
711 else
712 {
713 throw Exception{message, ErrorCodes::NETWORK_ERROR};
714 }
715 }
716 };
717
718 /// HTTP
719 create_server("http_port", [&](UInt16 port)
720 {
721 Poco::Net::ServerSocket socket;
722 auto address = socket_bind_listen(socket, listen_host, port);
723 socket.setReceiveTimeout(settings.http_receive_timeout);
724 socket.setSendTimeout(settings.http_send_timeout);
725 auto handler_factory = createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPHandler-factory");
726 if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0)
727 handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics);
728
729 servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
730 handler_factory,
731 server_pool,
732 socket,
733 http_params));
734
735 LOG_INFO(log, "Listening for http://" + address.toString());
736 });
737
738 /// HTTPS
739 create_server("https_port", [&](UInt16 port)
740 {
741#if USE_POCO_NETSSL
742 Poco::Net::SecureServerSocket socket;
743 auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true);
744 socket.setReceiveTimeout(settings.http_receive_timeout);
745 socket.setSendTimeout(settings.http_send_timeout);
746 servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
747 createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPSHandler-factory"),
748 server_pool,
749 socket,
750 http_params));
751
752 LOG_INFO(log, "Listening for https://" + address.toString());
753#else
754 UNUSED(port);
755 throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support.",
756 ErrorCodes::SUPPORT_IS_DISABLED};
757#endif
758 });
759
760 /// TCP
761 create_server("tcp_port", [&](UInt16 port)
762 {
763 Poco::Net::ServerSocket socket;
764 auto address = socket_bind_listen(socket, listen_host, port);
765 socket.setReceiveTimeout(settings.receive_timeout);
766 socket.setSendTimeout(settings.send_timeout);
767 servers.emplace_back(std::make_unique<Poco::Net::TCPServer>(
768 new TCPHandlerFactory(*this),
769 server_pool,
770 socket,
771 new Poco::Net::TCPServerParams));
772
773 LOG_INFO(log, "Listening for connections with native protocol (tcp): " + address.toString());
774 });
775
776 /// TCP with SSL
777 create_server("tcp_port_secure", [&](UInt16 port)
778 {
779#if USE_POCO_NETSSL
780 Poco::Net::SecureServerSocket socket;
781 auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true);
782 socket.setReceiveTimeout(settings.receive_timeout);
783 socket.setSendTimeout(settings.send_timeout);
784 servers.emplace_back(std::make_unique<Poco::Net::TCPServer>(
785 new TCPHandlerFactory(*this, /* secure= */ true),
786 server_pool,
787 socket,
788 new Poco::Net::TCPServerParams));
789 LOG_INFO(log, "Listening for connections with secure native protocol (tcp_secure): " + address.toString());
790#else
791 UNUSED(port);
792 throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
793 ErrorCodes::SUPPORT_IS_DISABLED};
794#endif
795 });
796
797 /// Interserver IO HTTP
798 create_server("interserver_http_port", [&](UInt16 port)
799 {
800 Poco::Net::ServerSocket socket;
801 auto address = socket_bind_listen(socket, listen_host, port);
802 socket.setReceiveTimeout(settings.http_receive_timeout);
803 socket.setSendTimeout(settings.http_send_timeout);
804 servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
805 createDefaultHandlerFatory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory"),
806 server_pool,
807 socket,
808 http_params));
809
810 LOG_INFO(log, "Listening for replica communication (interserver): http://" + address.toString());
811 });
812
813 create_server("interserver_https_port", [&](UInt16 port)
814 {
815#if USE_POCO_NETSSL
816 Poco::Net::SecureServerSocket socket;
817 auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true);
818 socket.setReceiveTimeout(settings.http_receive_timeout);
819 socket.setSendTimeout(settings.http_send_timeout);
820 servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
821 createDefaultHandlerFatory<InterserverIOHTTPHandler>(*this, "InterserverIOHTTPHandler-factory"),
822 server_pool,
823 socket,
824 http_params));
825
826 LOG_INFO(log, "Listening for secure replica communication (interserver): https://" + address.toString());
827#else
828 UNUSED(port);
829 throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
830 ErrorCodes::SUPPORT_IS_DISABLED};
831#endif
832 });
833
834 create_server("mysql_port", [&](UInt16 port)
835 {
836 Poco::Net::ServerSocket socket;
837 auto address = socket_bind_listen(socket, listen_host, port, /* secure = */ true);
838 socket.setReceiveTimeout(Poco::Timespan());
839 socket.setSendTimeout(settings.send_timeout);
840 servers.emplace_back(std::make_unique<Poco::Net::TCPServer>(
841 new MySQLHandlerFactory(*this),
842 server_pool,
843 socket,
844 new Poco::Net::TCPServerParams));
845
846 LOG_INFO(log, "Listening for MySQL compatibility protocol: " + address.toString());
847 });
848
849 /// Prometheus (if defined and not setup yet with http_port)
850 create_server("prometheus.port", [&](UInt16 port)
851 {
852 Poco::Net::ServerSocket socket;
853 auto address = socket_bind_listen(socket, listen_host, port);
854 socket.setReceiveTimeout(settings.http_receive_timeout);
855 socket.setSendTimeout(settings.http_send_timeout);
856 auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory");
857 handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics);
858 servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
859 handler_factory,
860 server_pool,
861 socket,
862 http_params));
863
864 LOG_INFO(log, "Listening for Prometheus: http://" + address.toString());
865 });
866 }
867
868 if (servers.empty())
869 throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG);
870
871 for (auto & server : servers)
872 server->start();
873
874 main_config_reloader->start();
875 users_config_reloader->start();
876 if (dns_cache_updater)
877 dns_cache_updater->start();
878
879 {
880 std::stringstream message;
881 message << "Available RAM: " << formatReadableSizeWithBinarySuffix(memory_amount) << ";"
882 << " physical cores: " << getNumberOfPhysicalCPUCores() << ";"
883 // on ARM processors it can show only enabled at current moment cores
884 << " logical cores: " << std::thread::hardware_concurrency() << ".";
885 LOG_INFO(log, message.str());
886 }
887
888 LOG_INFO(log, "Ready for connections.");
889
890 SCOPE_EXIT({
891 LOG_DEBUG(log, "Received termination signal.");
892 LOG_DEBUG(log, "Waiting for current connections to close.");
893
894 is_cancelled = true;
895
896 int current_connections = 0;
897 for (auto & server : servers)
898 {
899 server->stop();
900 current_connections += server->currentConnections();
901 }
902
903 LOG_INFO(log,
904 "Closed all listening sockets."
905 << (current_connections ? " Waiting for " + toString(current_connections) + " outstanding connections." : ""));
906
907 /// Killing remaining queries.
908 global_context->getProcessList().killAllQueries();
909
910 if (current_connections)
911 {
912 const int sleep_max_ms = 1000 * config().getInt("shutdown_wait_unfinished", 5);
913 const int sleep_one_ms = 100;
914 int sleep_current_ms = 0;
915 while (sleep_current_ms < sleep_max_ms)
916 {
917 current_connections = 0;
918 for (auto & server : servers)
919 current_connections += server->currentConnections();
920 if (!current_connections)
921 break;
922 sleep_current_ms += sleep_one_ms;
923 std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms));
924 }
925 }
926
927 LOG_INFO(
928 log, "Closed connections." << (current_connections ? " But " + toString(current_connections) + " remains."
929 " Tip: To increase wait time add to config: <shutdown_wait_unfinished>60</shutdown_wait_unfinished>" : ""));
930
931 dns_cache_updater.reset();
932 main_config_reloader.reset();
933 users_config_reloader.reset();
934
935 if (current_connections)
936 {
937 /// There is no better way to force connections to close in Poco.
938 /// Otherwise connection handlers will continue to live
939 /// (they are effectively dangling objects, but they use global thread pool
940 /// and global thread pool destructor will wait for threads, preventing server shutdown).
941
942 /// Dump coverage here, because std::atexit callback would not be called.
943 dumpCoverageReportIfPossible();
944 LOG_INFO(log, "Will shutdown forcefully.");
945 _exit(Application::EXIT_OK);
946 }
947 });
948
949 /// try to load dictionaries immediately, throw on error and die
950 ext::scope_guard dictionaries_xmls, models_xmls;
951 try
952 {
953 if (!config().getBool("dictionaries_lazy_load", true))
954 {
955 global_context->tryCreateEmbeddedDictionaries();
956 global_context->getExternalDictionariesLoader().enableAlwaysLoadEverything(true);
957 }
958 dictionaries_xmls = global_context->getExternalDictionariesLoader().addConfigRepository(
959 std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "dictionaries_config"));
960 models_xmls = global_context->getExternalModelsLoader().addConfigRepository(
961 std::make_unique<ExternalLoaderXMLConfigRepository>(config(), "models_config"));
962 }
963 catch (...)
964 {
965 LOG_ERROR(log, "Caught exception while loading dictionaries.");
966 throw;
967 }
968
969 std::vector<std::unique_ptr<MetricsTransmitter>> metrics_transmitters;
970 for (const auto & graphite_key : DB::getMultipleKeysFromConfig(config(), "", "graphite"))
971 {
972 metrics_transmitters.emplace_back(std::make_unique<MetricsTransmitter>(
973 global_context->getConfigRef(), graphite_key, async_metrics));
974 }
975
976 SessionCleaner session_cleaner(*global_context);
977
978 waitForTerminationRequest();
979 }
980
981 return Application::EXIT_OK;
982}
983}
984
985#pragma GCC diagnostic ignored "-Wunused-function"
986#pragma GCC diagnostic ignored "-Wmissing-declarations"
987
988int mainEntryClickHouseServer(int argc, char ** argv)
989{
990 DB::Server app;
991 try
992 {
993 return app.run(argc, argv);
994 }
995 catch (...)
996 {
997 std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
998 auto code = DB::getCurrentExceptionCode();
999 return code ? code : 1;
1000 }
1001}
1002