1#include "LocalServer.h"
2
3#include <Poco/Util/XMLConfiguration.h>
4#include <Poco/Util/HelpFormatter.h>
5#include <Poco/Util/OptionCallback.h>
6#include <Poco/String.h>
7#include <Poco/Logger.h>
8#include <Poco/NullChannel.h>
9#include <Databases/DatabaseMemory.h>
10#include <Storages/System/attachSystemTables.h>
11#include <Interpreters/Context.h>
12#include <Interpreters/ProcessList.h>
13#include <Interpreters/executeQuery.h>
14#include <Interpreters/loadMetadata.h>
15#include <Common/Exception.h>
16#include <Common/Macros.h>
17#include <Common/Config/ConfigProcessor.h>
18#include <Common/escapeForFileName.h>
19#include <Common/ClickHouseRevision.h>
20#include <Common/ThreadStatus.h>
21#include <Common/config_version.h>
22#include <Common/quoteString.h>
23#include <IO/ReadBufferFromString.h>
24#include <IO/WriteBufferFromFileDescriptor.h>
25#include <IO/UseSSL.h>
26#include <Parsers/parseQuery.h>
27#include <Parsers/IAST.h>
28#include <common/ErrorHandlers.h>
29#include <Common/StatusFile.h>
30#include <Functions/registerFunctions.h>
31#include <AggregateFunctions/registerAggregateFunctions.h>
32#include <TableFunctions/registerTableFunctions.h>
33#include <Storages/registerStorages.h>
34#include <Dictionaries/registerDictionaries.h>
35#include <Disks/registerDisks.h>
36#include <boost/program_options/options_description.hpp>
37#include <boost/program_options.hpp>
38#include <common/argsToConfig.h>
39#include <Common/TerminalSize.h>
40
41
42namespace DB
43{
44
45namespace ErrorCodes
46{
47 extern const int SYNTAX_ERROR;
48 extern const int CANNOT_LOAD_CONFIG;
49}
50
51
52LocalServer::LocalServer() = default;
53
54LocalServer::~LocalServer()
55{
56 if (context)
57 context->shutdown(); /// required for properly exception handling
58}
59
60
61void LocalServer::initialize(Poco::Util::Application & self)
62{
63 Poco::Util::Application::initialize(self);
64
65 /// Load config files if exists
66 if (config().has("config-file") || Poco::File("config.xml").exists())
67 {
68 const auto config_path = config().getString("config-file", "config.xml");
69 ConfigProcessor config_processor(config_path, false, true);
70 config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString());
71 auto loaded_config = config_processor.loadConfig();
72 config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", "."));
73 config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
74 }
75
76 if (config().has("logger") || config().has("logger.level") || config().has("logger.log"))
77 {
78 // sensitive data rules are not used here
79 buildLoggers(config(), logger(), self.commandName());
80 }
81 else
82 {
83 // Turn off server logging to stderr
84 if (!config().has("verbose"))
85 {
86 Poco::Logger::root().setLevel("none");
87 Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel()));
88 }
89 }
90}
91
92void LocalServer::applyCmdSettings()
93{
94 context->getSettingsRef().copyChangesFrom(cmd_settings);
95}
96
97/// If path is specified and not empty, will try to setup server environment and load existing metadata
98void LocalServer::tryInitPath()
99{
100 std::string path = config().getString("path", "");
101 Poco::trimInPlace(path);
102
103 if (!path.empty())
104 {
105 if (path.back() != '/')
106 path += '/';
107
108 context->setPath(path);
109 return;
110 }
111
112 /// In case of empty path set paths to helpful directories
113 std::string cd = Poco::Path::current();
114 context->setTemporaryPath(cd + "tmp");
115 context->setFlagsPath(cd + "flags");
116 context->setUserFilesPath(""); // user's files are everywhere
117}
118
119
120int LocalServer::main(const std::vector<std::string> & /*args*/)
121try
122{
123 Logger * log = &logger();
124 ThreadStatus thread_status;
125 UseSSL use_ssl;
126
127 if (!config().has("query") && !config().has("table-structure")) /// Nothing to process
128 {
129 if (config().hasOption("verbose"))
130 std::cerr << "There are no queries to process." << '\n';
131
132 return Application::EXIT_OK;
133 }
134
135
136 context = std::make_unique<Context>(Context::createGlobal());
137 context->makeGlobalContext();
138 context->setApplicationType(Context::ApplicationType::LOCAL);
139 tryInitPath();
140
141 std::optional<StatusFile> status;
142
143 /// Skip temp path installation
144
145 /// We will terminate process on error
146 static KillingErrorHandler error_handler;
147 Poco::ErrorHandler::set(&error_handler);
148
149 /// Don't initialize DateLUT
150
151 registerFunctions();
152 registerAggregateFunctions();
153 registerTableFunctions();
154 registerStorages();
155 registerDictionaries();
156 registerDisks();
157
158 /// Maybe useless
159 if (config().has("macros"))
160 context->setMacros(std::make_unique<Macros>(config(), "macros"));
161
162 /// Skip networking
163
164 setupUsers();
165
166 /// Limit on total number of concurrently executing queries.
167 /// Threre are no need for concurrent threads, override max_concurrent_queries.
168 context->getProcessList().setMaxSize(0);
169
170 /// Size of cache for uncompressed blocks. Zero means disabled.
171 size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0);
172 if (uncompressed_cache_size)
173 context->setUncompressedCache(uncompressed_cache_size);
174
175 /// Size of cache for marks (index of MergeTree family of tables). It is necessary.
176 /// Specify default value for mark_cache_size explicitly!
177 size_t mark_cache_size = config().getUInt64("mark_cache_size", 5368709120);
178 if (mark_cache_size)
179 context->setMarkCache(mark_cache_size);
180
181 /// Load global settings from default_profile and system_profile.
182 context->setDefaultProfiles(config());
183
184 /** Init dummy default DB
185 * NOTE: We force using isolated default database to avoid conflicts with default database from server enviroment
186 * Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory;
187 * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
188 */
189 std::string default_database = config().getString("default_database", "_local");
190 context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
191 context->setCurrentDatabase(default_database);
192 applyCmdOptions();
193
194 if (!context->getPath().empty())
195 {
196 /// Lock path directory before read
197 status.emplace(context->getPath() + "status");
198
199 LOG_DEBUG(log, "Loading metadata from " << context->getPath());
200 loadMetadataSystem(*context);
201 attachSystemTables();
202 loadMetadata(*context);
203 LOG_DEBUG(log, "Loaded metadata.");
204 }
205 else
206 {
207 attachSystemTables();
208 }
209
210 processQueries();
211
212 context->shutdown();
213 context.reset();
214
215 return Application::EXIT_OK;
216}
217catch (const Exception & e)
218{
219 std::cerr << getCurrentExceptionMessage(config().hasOption("stacktrace")) << '\n';
220
221 /// If exception code isn't zero, we should return non-zero return code anyway.
222 return e.code() ? e.code() : -1;
223}
224
225
226std::string LocalServer::getInitialCreateTableQuery()
227{
228 if (!config().has("table-structure"))
229 return {};
230
231 auto table_name = backQuoteIfNeed(config().getString("table-name", "table"));
232 auto table_structure = config().getString("table-structure");
233 auto data_format = backQuoteIfNeed(config().getString("table-data-format", "TSV"));
234 String table_file;
235 if (!config().has("table-file") || config().getString("table-file") == "-") /// Use Unix tools stdin naming convention
236 table_file = "stdin";
237 else /// Use regular file
238 table_file = quoteString(config().getString("table-file"));
239
240 return
241 "CREATE TABLE " + table_name +
242 " (" + table_structure + ") " +
243 "ENGINE = "
244 "File(" + data_format + ", " + table_file + ")"
245 "; ";
246}
247
248
249void LocalServer::attachSystemTables()
250{
251 DatabasePtr system_database = context->tryGetDatabase("system");
252 if (!system_database)
253 {
254 /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading
255 system_database = std::make_shared<DatabaseMemory>("system");
256 context->addDatabase("system", system_database);
257 }
258
259 attachSystemTablesLocal(*system_database);
260}
261
262
263void LocalServer::processQueries()
264{
265 String initial_create_query = getInitialCreateTableQuery();
266 String queries_str = initial_create_query + config().getRawString("query");
267
268 std::vector<String> queries;
269 auto parse_res = splitMultipartQuery(queries_str, queries);
270
271 if (!parse_res.second)
272 throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR);
273
274 context->makeSessionContext();
275 context->makeQueryContext();
276
277 context->setUser("default", "", Poco::Net::SocketAddress{}, "");
278 context->setCurrentQueryId("");
279 applyCmdSettings();
280
281 /// Use the same query_id (and thread group) for all queries
282 CurrentThread::QueryScope query_scope_holder(*context);
283
284 bool echo_queries = config().hasOption("echo") || config().hasOption("verbose");
285 std::exception_ptr exception;
286
287 for (const auto & query : queries)
288 {
289 ReadBufferFromString read_buf(query);
290 WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO);
291
292 if (echo_queries)
293 {
294 writeString(query, write_buf);
295 writeChar('\n', write_buf);
296 write_buf.next();
297 }
298
299 try
300 {
301 executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {}, {});
302 }
303 catch (...)
304 {
305 if (!config().hasOption("ignore-error"))
306 throw;
307
308 if (!exception)
309 exception = std::current_exception();
310
311 std::cerr << getCurrentExceptionMessage(config().hasOption("stacktrace")) << '\n';
312 }
313 }
314
315 if (exception)
316 std::rethrow_exception(exception);
317}
318
319static const char * minimal_default_user_xml =
320"<yandex>"
321" <profiles>"
322" <default></default>"
323" </profiles>"
324" <users>"
325" <default>"
326" <password></password>"
327" <networks>"
328" <ip>::/0</ip>"
329" </networks>"
330" <profile>default</profile>"
331" <quota>default</quota>"
332" </default>"
333" </users>"
334" <quotas>"
335" <default></default>"
336" </quotas>"
337"</yandex>";
338
339
340static ConfigurationPtr getConfigurationFromXMLString(const char * xml_data)
341{
342 std::stringstream ss{std::string{xml_data}};
343 Poco::XML::InputSource input_source{ss};
344 return {new Poco::Util::XMLConfiguration{&input_source}};
345}
346
347
348void LocalServer::setupUsers()
349{
350 ConfigurationPtr users_config;
351
352 if (config().has("users_config") || config().has("config-file") || Poco::File("config.xml").exists())
353 {
354 const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
355 ConfigProcessor config_processor(users_config_path);
356 const auto loaded_config = config_processor.loadConfig();
357 config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH));
358 users_config = loaded_config.configuration;
359 }
360 else
361 {
362 users_config = getConfigurationFromXMLString(minimal_default_user_xml);
363 }
364
365 if (users_config)
366 context->setUsersConfig(users_config);
367 else
368 throw Exception("Can't load config for users", ErrorCodes::CANNOT_LOAD_CONFIG);
369}
370
371static void showClientVersion()
372{
373 std::cout << DBMS_NAME << " client version " << VERSION_STRING << VERSION_OFFICIAL << "." << '\n';
374}
375
376std::string LocalServer::getHelpHeader() const
377{
378 return
379 "usage: clickhouse-local [initial table definition] [--query <query>]\n"
380
381 "clickhouse-local allows to execute SQL queries on your data files via single command line call."
382 " To do so, initially you need to define your data source and its format."
383 " After you can execute your SQL queries in usual manner.\n"
384
385 "There are two ways to define initial table keeping your data."
386 " Either just in first query like this:\n"
387 " CREATE TABLE <table> (<structure>) ENGINE = File(<input-format>, <file>);\n"
388 "Either through corresponding command line parameters --table --structure --input-format and --file.";
389}
390
391std::string LocalServer::getHelpFooter() const
392{
393 return
394 "Example printing memory used by each Unix user:\n"
395 "ps aux | tail -n +2 | awk '{ printf(\"%s\\t%s\\n\", $1, $4) }' | "
396 "clickhouse-local -S \"user String, mem Float64\" -q"
397 " \"SELECT user, round(sum(mem), 2) as mem_total FROM table GROUP BY user ORDER"
398 " BY mem_total DESC FORMAT PrettyCompact\"";
399}
400
401void LocalServer::init(int argc, char ** argv)
402{
403 namespace po = boost::program_options;
404
405 /// Don't parse options with Poco library, we prefer neat boost::program_options
406 stopOptionsProcessing();
407
408 po::options_description description = createOptionsDescription("Main options", getTerminalWidth());
409 description.add_options()
410 ("help", "produce help message")
411 ("config-file,c", po::value<std::string>(), "config-file path")
412 ("query,q", po::value<std::string>(), "query")
413 ("database,d", po::value<std::string>(), "database")
414
415 ("table,N", po::value<std::string>(), "name of the initial table")
416 /// If structure argument is omitted then initial query is not generated
417 ("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")
418 ("file,f", po::value<std::string>(), "path to file with data of the initial table (stdin if not specified)")
419 ("input-format", po::value<std::string>(), "input format of the initial table data")
420 ("format,f", po::value<std::string>(), "default output format (clickhouse-client compatibility)")
421 ("output-format", po::value<std::string>(), "default output format")
422
423 ("stacktrace", "print stack traces of exceptions")
424 ("echo", "print query before execution")
425 ("verbose", "print query and other debugging info")
426 ("logger.log", po::value<std::string>(), "Log file name")
427 ("logger.level", po::value<std::string>(), "Log level")
428 ("ignore-error", "do not stop processing if a query failed")
429 ("version,V", "print version information and exit")
430 ;
431
432 cmd_settings.addProgramOptions(description);
433
434 /// Parse main commandline options.
435 po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run();
436 po::variables_map options;
437 po::store(parsed, options);
438 po::notify(options);
439
440 if (options.count("version") || options.count("V"))
441 {
442 showClientVersion();
443 exit(0);
444 }
445
446 if (options.empty() || options.count("help"))
447 {
448 std::cout << getHelpHeader() << "\n";
449 std::cout << description << "\n";
450 std::cout << getHelpFooter() << "\n";
451 exit(0);
452 }
453
454 /// Save received data into the internal config.
455 if (options.count("config-file"))
456 config().setString("config-file", options["config-file"].as<std::string>());
457 if (options.count("query"))
458 config().setString("query", options["query"].as<std::string>());
459 if (options.count("database"))
460 config().setString("default_database", options["database"].as<std::string>());
461
462 if (options.count("table"))
463 config().setString("table-name", options["table"].as<std::string>());
464 if (options.count("file"))
465 config().setString("table-file", options["file"].as<std::string>());
466 if (options.count("structure"))
467 config().setString("table-structure", options["structure"].as<std::string>());
468 if (options.count("input-format"))
469 config().setString("table-data-format", options["input-format"].as<std::string>());
470 if (options.count("format"))
471 config().setString("format", options["format"].as<std::string>());
472 if (options.count("output-format"))
473 config().setString("output-format", options["output-format"].as<std::string>());
474
475 if (options.count("stacktrace"))
476 config().setBool("stacktrace", true);
477 if (options.count("echo"))
478 config().setBool("echo", true);
479 if (options.count("verbose"))
480 config().setBool("verbose", true);
481 if (options.count("logger.log"))
482 config().setString("logger.log", options["logger.log"].as<std::string>());
483 if (options.count("logger.level"))
484 config().setString("logger.level", options["logger.level"].as<std::string>());
485 if (options.count("ignore-error"))
486 config().setBool("ignore-error", true);
487
488 std::vector<std::string> arguments;
489 for (int arg_num = 1; arg_num < argc; ++arg_num)
490 arguments.emplace_back(argv[arg_num]);
491 argsToConfig(arguments, config(), 100);
492}
493
494void LocalServer::applyCmdOptions()
495{
496 context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
497 applyCmdSettings();
498}
499
500}
501
502#pragma GCC diagnostic ignored "-Wunused-function"
503#pragma GCC diagnostic ignored "-Wmissing-declarations"
504
505int mainEntryClickHouseLocal(int argc, char ** argv)
506{
507 DB::LocalServer app;
508 try
509 {
510 app.init(argc, argv);
511 return app.run();
512 }
513 catch (...)
514 {
515 std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
516 auto code = DB::getCurrentExceptionCode();
517 return code ? code : 1;
518 }
519}
520