1 | #include "LocalServer.h" |
2 | |
3 | #include <Poco/Util/XMLConfiguration.h> |
4 | #include <Poco/Util/HelpFormatter.h> |
5 | #include <Poco/Util/OptionCallback.h> |
6 | #include <Poco/String.h> |
7 | #include <Poco/Logger.h> |
8 | #include <Poco/NullChannel.h> |
9 | #include <Databases/DatabaseMemory.h> |
10 | #include <Storages/System/attachSystemTables.h> |
11 | #include <Interpreters/Context.h> |
12 | #include <Interpreters/ProcessList.h> |
13 | #include <Interpreters/executeQuery.h> |
14 | #include <Interpreters/loadMetadata.h> |
15 | #include <Common/Exception.h> |
16 | #include <Common/Macros.h> |
17 | #include <Common/Config/ConfigProcessor.h> |
18 | #include <Common/escapeForFileName.h> |
19 | #include <Common/ClickHouseRevision.h> |
20 | #include <Common/ThreadStatus.h> |
21 | #include <Common/config_version.h> |
22 | #include <Common/quoteString.h> |
23 | #include <IO/ReadBufferFromString.h> |
24 | #include <IO/WriteBufferFromFileDescriptor.h> |
25 | #include <IO/UseSSL.h> |
26 | #include <Parsers/parseQuery.h> |
27 | #include <Parsers/IAST.h> |
28 | #include <common/ErrorHandlers.h> |
29 | #include <Common/StatusFile.h> |
30 | #include <Functions/registerFunctions.h> |
31 | #include <AggregateFunctions/registerAggregateFunctions.h> |
32 | #include <TableFunctions/registerTableFunctions.h> |
33 | #include <Storages/registerStorages.h> |
34 | #include <Dictionaries/registerDictionaries.h> |
35 | #include <Disks/registerDisks.h> |
36 | #include <boost/program_options/options_description.hpp> |
37 | #include <boost/program_options.hpp> |
38 | #include <common/argsToConfig.h> |
39 | #include <Common/TerminalSize.h> |
40 | |
41 | |
42 | namespace DB |
43 | { |
44 | |
45 | namespace ErrorCodes |
46 | { |
47 | extern const int SYNTAX_ERROR; |
48 | extern const int CANNOT_LOAD_CONFIG; |
49 | } |
50 | |
51 | |
52 | LocalServer::LocalServer() = default; |
53 | |
54 | LocalServer::~LocalServer() |
55 | { |
56 | if (context) |
57 | context->shutdown(); /// required for properly exception handling |
58 | } |
59 | |
60 | |
61 | void LocalServer::initialize(Poco::Util::Application & self) |
62 | { |
63 | Poco::Util::Application::initialize(self); |
64 | |
65 | /// Load config files if exists |
66 | if (config().has("config-file" ) || Poco::File("config.xml" ).exists()) |
67 | { |
68 | const auto config_path = config().getString("config-file" , "config.xml" ); |
69 | ConfigProcessor config_processor(config_path, false, true); |
70 | config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); |
71 | auto loaded_config = config_processor.loadConfig(); |
72 | config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path" , "." )); |
73 | config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); |
74 | } |
75 | |
76 | if (config().has("logger" ) || config().has("logger.level" ) || config().has("logger.log" )) |
77 | { |
78 | // sensitive data rules are not used here |
79 | buildLoggers(config(), logger(), self.commandName()); |
80 | } |
81 | else |
82 | { |
83 | // Turn off server logging to stderr |
84 | if (!config().has("verbose" )) |
85 | { |
86 | Poco::Logger::root().setLevel("none" ); |
87 | Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel())); |
88 | } |
89 | } |
90 | } |
91 | |
92 | void LocalServer::applyCmdSettings() |
93 | { |
94 | context->getSettingsRef().copyChangesFrom(cmd_settings); |
95 | } |
96 | |
97 | /// If path is specified and not empty, will try to setup server environment and load existing metadata |
98 | void LocalServer::tryInitPath() |
99 | { |
100 | std::string path = config().getString("path" , "" ); |
101 | Poco::trimInPlace(path); |
102 | |
103 | if (!path.empty()) |
104 | { |
105 | if (path.back() != '/') |
106 | path += '/'; |
107 | |
108 | context->setPath(path); |
109 | return; |
110 | } |
111 | |
112 | /// In case of empty path set paths to helpful directories |
113 | std::string cd = Poco::Path::current(); |
114 | context->setTemporaryPath(cd + "tmp" ); |
115 | context->setFlagsPath(cd + "flags" ); |
116 | context->setUserFilesPath("" ); // user's files are everywhere |
117 | } |
118 | |
119 | |
120 | int LocalServer::main(const std::vector<std::string> & /*args*/) |
121 | try |
122 | { |
123 | Logger * log = &logger(); |
124 | ThreadStatus thread_status; |
125 | UseSSL use_ssl; |
126 | |
127 | if (!config().has("query" ) && !config().has("table-structure" )) /// Nothing to process |
128 | { |
129 | if (config().hasOption("verbose" )) |
130 | std::cerr << "There are no queries to process." << '\n'; |
131 | |
132 | return Application::EXIT_OK; |
133 | } |
134 | |
135 | |
136 | context = std::make_unique<Context>(Context::createGlobal()); |
137 | context->makeGlobalContext(); |
138 | context->setApplicationType(Context::ApplicationType::LOCAL); |
139 | tryInitPath(); |
140 | |
141 | std::optional<StatusFile> status; |
142 | |
143 | /// Skip temp path installation |
144 | |
145 | /// We will terminate process on error |
146 | static KillingErrorHandler error_handler; |
147 | Poco::ErrorHandler::set(&error_handler); |
148 | |
149 | /// Don't initialize DateLUT |
150 | |
151 | registerFunctions(); |
152 | registerAggregateFunctions(); |
153 | registerTableFunctions(); |
154 | registerStorages(); |
155 | registerDictionaries(); |
156 | registerDisks(); |
157 | |
158 | /// Maybe useless |
159 | if (config().has("macros" )) |
160 | context->setMacros(std::make_unique<Macros>(config(), "macros" )); |
161 | |
162 | /// Skip networking |
163 | |
164 | setupUsers(); |
165 | |
166 | /// Limit on total number of concurrently executing queries. |
167 | /// Threre are no need for concurrent threads, override max_concurrent_queries. |
168 | context->getProcessList().setMaxSize(0); |
169 | |
170 | /// Size of cache for uncompressed blocks. Zero means disabled. |
171 | size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size" , 0); |
172 | if (uncompressed_cache_size) |
173 | context->setUncompressedCache(uncompressed_cache_size); |
174 | |
175 | /// Size of cache for marks (index of MergeTree family of tables). It is necessary. |
176 | /// Specify default value for mark_cache_size explicitly! |
177 | size_t mark_cache_size = config().getUInt64("mark_cache_size" , 5368709120); |
178 | if (mark_cache_size) |
179 | context->setMarkCache(mark_cache_size); |
180 | |
181 | /// Load global settings from default_profile and system_profile. |
182 | context->setDefaultProfiles(config()); |
183 | |
184 | /** Init dummy default DB |
185 | * NOTE: We force using isolated default database to avoid conflicts with default database from server enviroment |
186 | * Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory; |
187 | * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. |
188 | */ |
189 | std::string default_database = config().getString("default_database" , "_local" ); |
190 | context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database)); |
191 | context->setCurrentDatabase(default_database); |
192 | applyCmdOptions(); |
193 | |
194 | if (!context->getPath().empty()) |
195 | { |
196 | /// Lock path directory before read |
197 | status.emplace(context->getPath() + "status" ); |
198 | |
199 | LOG_DEBUG(log, "Loading metadata from " << context->getPath()); |
200 | loadMetadataSystem(*context); |
201 | attachSystemTables(); |
202 | loadMetadata(*context); |
203 | LOG_DEBUG(log, "Loaded metadata." ); |
204 | } |
205 | else |
206 | { |
207 | attachSystemTables(); |
208 | } |
209 | |
210 | processQueries(); |
211 | |
212 | context->shutdown(); |
213 | context.reset(); |
214 | |
215 | return Application::EXIT_OK; |
216 | } |
217 | catch (const Exception & e) |
218 | { |
219 | std::cerr << getCurrentExceptionMessage(config().hasOption("stacktrace" )) << '\n'; |
220 | |
221 | /// If exception code isn't zero, we should return non-zero return code anyway. |
222 | return e.code() ? e.code() : -1; |
223 | } |
224 | |
225 | |
226 | std::string LocalServer::getInitialCreateTableQuery() |
227 | { |
228 | if (!config().has("table-structure" )) |
229 | return {}; |
230 | |
231 | auto table_name = backQuoteIfNeed(config().getString("table-name" , "table" )); |
232 | auto table_structure = config().getString("table-structure" ); |
233 | auto data_format = backQuoteIfNeed(config().getString("table-data-format" , "TSV" )); |
234 | String table_file; |
235 | if (!config().has("table-file" ) || config().getString("table-file" ) == "-" ) /// Use Unix tools stdin naming convention |
236 | table_file = "stdin" ; |
237 | else /// Use regular file |
238 | table_file = quoteString(config().getString("table-file" )); |
239 | |
240 | return |
241 | "CREATE TABLE " + table_name + |
242 | " (" + table_structure + ") " + |
243 | "ENGINE = " |
244 | "File(" + data_format + ", " + table_file + ")" |
245 | "; " ; |
246 | } |
247 | |
248 | |
249 | void LocalServer::attachSystemTables() |
250 | { |
251 | DatabasePtr system_database = context->tryGetDatabase("system" ); |
252 | if (!system_database) |
253 | { |
254 | /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading |
255 | system_database = std::make_shared<DatabaseMemory>("system" ); |
256 | context->addDatabase("system" , system_database); |
257 | } |
258 | |
259 | attachSystemTablesLocal(*system_database); |
260 | } |
261 | |
262 | |
263 | void LocalServer::processQueries() |
264 | { |
265 | String initial_create_query = getInitialCreateTableQuery(); |
266 | String queries_str = initial_create_query + config().getRawString("query" ); |
267 | |
268 | std::vector<String> queries; |
269 | auto parse_res = splitMultipartQuery(queries_str, queries); |
270 | |
271 | if (!parse_res.second) |
272 | throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); |
273 | |
274 | context->makeSessionContext(); |
275 | context->makeQueryContext(); |
276 | |
277 | context->setUser("default" , "" , Poco::Net::SocketAddress{}, "" ); |
278 | context->setCurrentQueryId("" ); |
279 | applyCmdSettings(); |
280 | |
281 | /// Use the same query_id (and thread group) for all queries |
282 | CurrentThread::QueryScope query_scope_holder(*context); |
283 | |
284 | bool echo_queries = config().hasOption("echo" ) || config().hasOption("verbose" ); |
285 | std::exception_ptr exception; |
286 | |
287 | for (const auto & query : queries) |
288 | { |
289 | ReadBufferFromString read_buf(query); |
290 | WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO); |
291 | |
292 | if (echo_queries) |
293 | { |
294 | writeString(query, write_buf); |
295 | writeChar('\n', write_buf); |
296 | write_buf.next(); |
297 | } |
298 | |
299 | try |
300 | { |
301 | executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {}, {}); |
302 | } |
303 | catch (...) |
304 | { |
305 | if (!config().hasOption("ignore-error" )) |
306 | throw; |
307 | |
308 | if (!exception) |
309 | exception = std::current_exception(); |
310 | |
311 | std::cerr << getCurrentExceptionMessage(config().hasOption("stacktrace" )) << '\n'; |
312 | } |
313 | } |
314 | |
315 | if (exception) |
316 | std::rethrow_exception(exception); |
317 | } |
318 | |
319 | static const char * minimal_default_user_xml = |
320 | "<yandex>" |
321 | " <profiles>" |
322 | " <default></default>" |
323 | " </profiles>" |
324 | " <users>" |
325 | " <default>" |
326 | " <password></password>" |
327 | " <networks>" |
328 | " <ip>::/0</ip>" |
329 | " </networks>" |
330 | " <profile>default</profile>" |
331 | " <quota>default</quota>" |
332 | " </default>" |
333 | " </users>" |
334 | " <quotas>" |
335 | " <default></default>" |
336 | " </quotas>" |
337 | "</yandex>" ; |
338 | |
339 | |
340 | static ConfigurationPtr getConfigurationFromXMLString(const char * xml_data) |
341 | { |
342 | std::stringstream ss{std::string{xml_data}}; |
343 | Poco::XML::InputSource input_source{ss}; |
344 | return {new Poco::Util::XMLConfiguration{&input_source}}; |
345 | } |
346 | |
347 | |
348 | void LocalServer::setupUsers() |
349 | { |
350 | ConfigurationPtr users_config; |
351 | |
352 | if (config().has("users_config" ) || config().has("config-file" ) || Poco::File("config.xml" ).exists()) |
353 | { |
354 | const auto users_config_path = config().getString("users_config" , config().getString("config-file" , "config.xml" )); |
355 | ConfigProcessor config_processor(users_config_path); |
356 | const auto loaded_config = config_processor.loadConfig(); |
357 | config_processor.savePreprocessedConfig(loaded_config, config().getString("path" , DBMS_DEFAULT_PATH)); |
358 | users_config = loaded_config.configuration; |
359 | } |
360 | else |
361 | { |
362 | users_config = getConfigurationFromXMLString(minimal_default_user_xml); |
363 | } |
364 | |
365 | if (users_config) |
366 | context->setUsersConfig(users_config); |
367 | else |
368 | throw Exception("Can't load config for users" , ErrorCodes::CANNOT_LOAD_CONFIG); |
369 | } |
370 | |
371 | static void showClientVersion() |
372 | { |
373 | std::cout << DBMS_NAME << " client version " << VERSION_STRING << VERSION_OFFICIAL << "." << '\n'; |
374 | } |
375 | |
376 | std::string LocalServer::() const |
377 | { |
378 | return |
379 | "usage: clickhouse-local [initial table definition] [--query <query>]\n" |
380 | |
381 | "clickhouse-local allows to execute SQL queries on your data files via single command line call." |
382 | " To do so, initially you need to define your data source and its format." |
383 | " After you can execute your SQL queries in usual manner.\n" |
384 | |
385 | "There are two ways to define initial table keeping your data." |
386 | " Either just in first query like this:\n" |
387 | " CREATE TABLE <table> (<structure>) ENGINE = File(<input-format>, <file>);\n" |
388 | "Either through corresponding command line parameters --table --structure --input-format and --file." ; |
389 | } |
390 | |
391 | std::string LocalServer::() const |
392 | { |
393 | return |
394 | "Example printing memory used by each Unix user:\n" |
395 | "ps aux | tail -n +2 | awk '{ printf(\"%s\\t%s\\n\", $1, $4) }' | " |
396 | "clickhouse-local -S \"user String, mem Float64\" -q" |
397 | " \"SELECT user, round(sum(mem), 2) as mem_total FROM table GROUP BY user ORDER" |
398 | " BY mem_total DESC FORMAT PrettyCompact\"" ; |
399 | } |
400 | |
401 | void LocalServer::init(int argc, char ** argv) |
402 | { |
403 | namespace po = boost::program_options; |
404 | |
405 | /// Don't parse options with Poco library, we prefer neat boost::program_options |
406 | stopOptionsProcessing(); |
407 | |
408 | po::options_description description = createOptionsDescription("Main options" , getTerminalWidth()); |
409 | description.add_options() |
410 | ("help" , "produce help message" ) |
411 | ("config-file,c" , po::value<std::string>(), "config-file path" ) |
412 | ("query,q" , po::value<std::string>(), "query" ) |
413 | ("database,d" , po::value<std::string>(), "database" ) |
414 | |
415 | ("table,N" , po::value<std::string>(), "name of the initial table" ) |
416 | /// If structure argument is omitted then initial query is not generated |
417 | ("structure,S" , po::value<std::string>(), "structure of the initial table (list of column and type names)" ) |
418 | ("file,f" , po::value<std::string>(), "path to file with data of the initial table (stdin if not specified)" ) |
419 | ("input-format" , po::value<std::string>(), "input format of the initial table data" ) |
420 | ("format,f" , po::value<std::string>(), "default output format (clickhouse-client compatibility)" ) |
421 | ("output-format" , po::value<std::string>(), "default output format" ) |
422 | |
423 | ("stacktrace" , "print stack traces of exceptions" ) |
424 | ("echo" , "print query before execution" ) |
425 | ("verbose" , "print query and other debugging info" ) |
426 | ("logger.log" , po::value<std::string>(), "Log file name" ) |
427 | ("logger.level" , po::value<std::string>(), "Log level" ) |
428 | ("ignore-error" , "do not stop processing if a query failed" ) |
429 | ("version,V" , "print version information and exit" ) |
430 | ; |
431 | |
432 | cmd_settings.addProgramOptions(description); |
433 | |
434 | /// Parse main commandline options. |
435 | po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run(); |
436 | po::variables_map options; |
437 | po::store(parsed, options); |
438 | po::notify(options); |
439 | |
440 | if (options.count("version" ) || options.count("V" )) |
441 | { |
442 | showClientVersion(); |
443 | exit(0); |
444 | } |
445 | |
446 | if (options.empty() || options.count("help" )) |
447 | { |
448 | std::cout << getHelpHeader() << "\n" ; |
449 | std::cout << description << "\n" ; |
450 | std::cout << getHelpFooter() << "\n" ; |
451 | exit(0); |
452 | } |
453 | |
454 | /// Save received data into the internal config. |
455 | if (options.count("config-file" )) |
456 | config().setString("config-file" , options["config-file" ].as<std::string>()); |
457 | if (options.count("query" )) |
458 | config().setString("query" , options["query" ].as<std::string>()); |
459 | if (options.count("database" )) |
460 | config().setString("default_database" , options["database" ].as<std::string>()); |
461 | |
462 | if (options.count("table" )) |
463 | config().setString("table-name" , options["table" ].as<std::string>()); |
464 | if (options.count("file" )) |
465 | config().setString("table-file" , options["file" ].as<std::string>()); |
466 | if (options.count("structure" )) |
467 | config().setString("table-structure" , options["structure" ].as<std::string>()); |
468 | if (options.count("input-format" )) |
469 | config().setString("table-data-format" , options["input-format" ].as<std::string>()); |
470 | if (options.count("format" )) |
471 | config().setString("format" , options["format" ].as<std::string>()); |
472 | if (options.count("output-format" )) |
473 | config().setString("output-format" , options["output-format" ].as<std::string>()); |
474 | |
475 | if (options.count("stacktrace" )) |
476 | config().setBool("stacktrace" , true); |
477 | if (options.count("echo" )) |
478 | config().setBool("echo" , true); |
479 | if (options.count("verbose" )) |
480 | config().setBool("verbose" , true); |
481 | if (options.count("logger.log" )) |
482 | config().setString("logger.log" , options["logger.log" ].as<std::string>()); |
483 | if (options.count("logger.level" )) |
484 | config().setString("logger.level" , options["logger.level" ].as<std::string>()); |
485 | if (options.count("ignore-error" )) |
486 | config().setBool("ignore-error" , true); |
487 | |
488 | std::vector<std::string> arguments; |
489 | for (int arg_num = 1; arg_num < argc; ++arg_num) |
490 | arguments.emplace_back(argv[arg_num]); |
491 | argsToConfig(arguments, config(), 100); |
492 | } |
493 | |
494 | void LocalServer::applyCmdOptions() |
495 | { |
496 | context->setDefaultFormat(config().getString("output-format" , config().getString("format" , "TSV" ))); |
497 | applyCmdSettings(); |
498 | } |
499 | |
500 | } |
501 | |
502 | #pragma GCC diagnostic ignored "-Wunused-function" |
503 | #pragma GCC diagnostic ignored "-Wmissing-declarations" |
504 | |
505 | int mainEntryClickHouseLocal(int argc, char ** argv) |
506 | { |
507 | DB::LocalServer app; |
508 | try |
509 | { |
510 | app.init(argc, argv); |
511 | return app.run(); |
512 | } |
513 | catch (...) |
514 | { |
515 | std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; |
516 | auto code = DB::getCurrentExceptionCode(); |
517 | return code ? code : 1; |
518 | } |
519 | } |
520 | |