1#include <algorithm>
2#include <iostream>
3#include <limits>
4#include <regex>
5#include <thread>
6#include <memory>
7#include <filesystem>
8
9#include <port/unistd.h>
10#include <sys/stat.h>
11
12#include <boost/program_options.hpp>
13
14#include <Poco/AutoPtr.h>
15#include <Poco/ConsoleChannel.h>
16#include <Poco/FormattingChannel.h>
17#include <Poco/Logger.h>
18#include <Poco/Path.h>
19#include <Poco/PatternFormatter.h>
20#include <Poco/Util/XMLConfiguration.h>
21
22#include <common/logger_useful.h>
23#include <Client/Connection.h>
24#include <Core/Types.h>
25#include <Interpreters/Context.h>
26#include <IO/ConnectionTimeouts.h>
27#include <IO/UseSSL.h>
28#include <Core/Settings.h>
29#include <Common/Exception.h>
30#include <Common/InterruptListener.h>
31#include <Common/TerminalSize.h>
32
33#include "TestStopConditions.h"
34#include "TestStats.h"
35#include "ConfigPreprocessor.h"
36#include "PerformanceTest.h"
37#include "ReportBuilder.h"
38
39
40namespace fs = std::filesystem;
41namespace po = boost::program_options;
42
43namespace DB
44{
45namespace ErrorCodes
46{
47 extern const int BAD_ARGUMENTS;
48 extern const int FILE_DOESNT_EXIST;
49}
50
51/** Tests launcher for ClickHouse.
52 * The tool walks through given or default folder in order to find files with
53 * tests' descriptions and launches it.
54 */
55class PerformanceTestSuite
56{
57public:
58
59 PerformanceTestSuite(const std::string & host_,
60 const UInt16 port_,
61 const bool secure_,
62 const std::string & default_database_,
63 const std::string & user_,
64 const std::string & password_,
65 const Settings & cmd_settings,
66 const bool lite_output_,
67 const std::string & profiles_file_,
68 Strings && input_files_,
69 Strings && tests_tags_,
70 Strings && skip_tags_,
71 Strings && tests_names_,
72 Strings && skip_names_,
73 Strings && tests_names_regexp_,
74 Strings && skip_names_regexp_,
75 const std::unordered_map<std::string, std::vector<size_t>> query_indexes_,
76 const ConnectionTimeouts & timeouts_)
77 : connection(host_, port_, default_database_, user_,
78 password_, "performance-test", Protocol::Compression::Enable,
79 secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable)
80 , timeouts(timeouts_)
81 , tests_tags(std::move(tests_tags_))
82 , tests_names(std::move(tests_names_))
83 , tests_names_regexp(std::move(tests_names_regexp_))
84 , skip_tags(std::move(skip_tags_))
85 , skip_names(std::move(skip_names_))
86 , skip_names_regexp(std::move(skip_names_regexp_))
87 , query_indexes(query_indexes_)
88 , lite_output(lite_output_)
89 , profiles_file(profiles_file_)
90 , input_files(input_files_)
91 , log(&Poco::Logger::get("PerformanceTestSuite"))
92 {
93 global_context.makeGlobalContext();
94 global_context.getSettingsRef().copyChangesFrom(cmd_settings);
95 if (input_files.size() < 1)
96 throw Exception("No tests were specified", ErrorCodes::BAD_ARGUMENTS);
97 }
98
99 int run()
100 {
101 std::string name;
102 UInt64 version_major;
103 UInt64 version_minor;
104 UInt64 version_patch;
105 UInt64 version_revision;
106 connection.getServerVersion(timeouts, name, version_major, version_minor, version_patch, version_revision);
107
108 std::stringstream ss;
109 ss << version_major << "." << version_minor << "." << version_patch;
110 server_version = ss.str();
111
112 report_builder = std::make_shared<ReportBuilder>(server_version);
113
114 processTestsConfigurations(input_files);
115
116 return 0;
117 }
118
119private:
120 Connection connection;
121 const ConnectionTimeouts & timeouts;
122
123 const Strings & tests_tags;
124 const Strings & tests_names;
125 const Strings & tests_names_regexp;
126 const Strings & skip_tags;
127 const Strings & skip_names;
128 const Strings & skip_names_regexp;
129 std::unordered_map<std::string, std::vector<size_t>> query_indexes;
130
131 Context global_context = Context::createGlobal();
132 std::shared_ptr<ReportBuilder> report_builder;
133
134 std::string server_version;
135
136 InterruptListener interrupt_listener;
137
138 using XMLConfiguration = Poco::Util::XMLConfiguration;
139 using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
140
141 bool lite_output;
142 std::string profiles_file;
143
144 Strings input_files;
145 std::vector<XMLConfigurationPtr> tests_configurations;
146 Poco::Logger * log;
147
148 void processTestsConfigurations(const Strings & paths)
149 {
150 LOG_INFO(log, "Preparing test configurations");
151 ConfigPreprocessor config_prep(paths);
152 tests_configurations = config_prep.processConfig(
153 tests_tags,
154 tests_names,
155 tests_names_regexp,
156 skip_tags,
157 skip_names,
158 skip_names_regexp);
159
160 LOG_INFO(log, "Test configurations prepared");
161
162 if (tests_configurations.size())
163 {
164 Strings outputs;
165
166 for (auto & test_config : tests_configurations)
167 {
168 auto [output, signal] = runTest(test_config);
169 if (!output.empty())
170 {
171 if (lite_output)
172 std::cout << output;
173 else
174 outputs.push_back(output);
175 }
176 if (signal)
177 break;
178 }
179
180 if (!lite_output && outputs.size())
181 {
182 std::cout << "[" << std::endl;
183
184 for (size_t i = 0; i != outputs.size(); ++i)
185 {
186 std::cout << outputs[i];
187 if (i != outputs.size() - 1)
188 std::cout << ",";
189
190 std::cout << std::endl;
191 }
192
193 std::cout << "]" << std::endl;
194 }
195 }
196 }
197
198 std::pair<std::string, bool> runTest(XMLConfigurationPtr & test_config)
199 {
200 PerformanceTestInfo info(test_config, profiles_file, global_context.getSettingsRef());
201 LOG_INFO(log, "Config for test '" << info.test_name << "' parsed");
202 PerformanceTest current(test_config, connection, timeouts, interrupt_listener, info, global_context, query_indexes[info.path]);
203
204 if (current.checkPreconditions())
205 {
206 LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fullfilled");
207 LOG_INFO(
208 log,
209 "Preparing for run, have " << info.create_and_fill_queries.size() << " create and fill queries");
210 current.prepare();
211 LOG_INFO(log, "Prepared");
212 LOG_INFO(log, "Running test '" << info.test_name << "'");
213 auto result = current.execute();
214 LOG_INFO(log, "Test '" << info.test_name << "' finished");
215
216 LOG_INFO(log, "Running post run queries");
217 current.finish();
218 LOG_INFO(log, "Postqueries finished");
219 if (lite_output)
220 return {report_builder->buildCompactReport(info, result, query_indexes[info.path]), current.checkSIGINT()};
221 else
222 return {report_builder->buildFullReport(info, result, query_indexes[info.path]), current.checkSIGINT()};
223 }
224 else
225 LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are not fullfilled, skip run");
226
227 return {"", current.checkSIGINT()};
228 }
229};
230
231}
232
233static void getFilesFromDir(const fs::path & dir, std::vector<std::string> & input_files, const bool recursive = false)
234{
235 Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite");
236 if (dir.extension().string() == ".xml")
237 LOG_WARNING(log, dir.string() + "' is a directory, but has .xml extension");
238
239 fs::directory_iterator end;
240 for (fs::directory_iterator it(dir); it != end; ++it)
241 {
242 const fs::path file = (*it);
243 if (recursive && fs::is_directory(file))
244 getFilesFromDir(file, input_files, recursive);
245 else if (!fs::is_directory(file) && file.extension().string() == ".xml")
246 input_files.push_back(file.string());
247 }
248}
249
250static std::vector<std::string> getInputFiles(const po::variables_map & options, Poco::Logger * log)
251{
252 std::vector<std::string> input_files;
253 bool recursive = options.count("recursive");
254
255 if (!options.count("input-files"))
256 {
257 LOG_INFO(log, "Trying to find test scenario files in the current folder...");
258 fs::path curr_dir(".");
259
260 getFilesFromDir(curr_dir, input_files, recursive);
261
262 if (input_files.empty())
263 throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
264 }
265 else
266 {
267 input_files = options["input-files"].as<std::vector<std::string>>();
268
269 std::vector<std::string> collected_files;
270 for (const std::string & filename : input_files)
271 {
272 fs::path file(filename);
273
274 if (!fs::exists(file))
275 throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
276
277 if (fs::is_directory(file))
278 {
279 getFilesFromDir(file, collected_files, recursive);
280 }
281 else
282 {
283 if (file.extension().string() != ".xml")
284 throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
285 collected_files.push_back(filename);
286 }
287 }
288
289 input_files = std::move(collected_files);
290 }
291
292 LOG_INFO(log, "Found " + std::to_string(input_files.size()) + " input files");
293 std::sort(input_files.begin(), input_files.end());
294 return input_files;
295}
296
297static std::unordered_map<std::string, std::vector<std::size_t>> getTestQueryIndexes(const po::basic_parsed_options<char> & parsed_opts)
298{
299 std::unordered_map<std::string, std::vector<std::size_t>> result;
300 const auto & options = parsed_opts.options;
301 if (options.empty())
302 return result;
303 for (size_t i = 0; i < options.size() - 1; ++i)
304 {
305 const auto & opt = options[i];
306 if (opt.string_key == "input-files")
307 {
308 if (options[i + 1].string_key == "query-indexes")
309 {
310 const std::string & test_path = Poco::Path(opt.value[0]).absolute().toString();
311 for (const auto & query_num_str : options[i + 1].value)
312 {
313 size_t query_num = std::stoul(query_num_str);
314 result[test_path].push_back(query_num);
315 }
316 }
317 }
318 }
319 return result;
320}
321
322#pragma GCC diagnostic ignored "-Wunused-function"
323#pragma GCC diagnostic ignored "-Wmissing-declarations"
324
325int mainEntryClickHousePerformanceTest(int argc, char ** argv)
326try
327{
328 using po::value;
329 using Strings = DB::Strings;
330
331 po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
332 desc.add_options()
333 ("help", "produce help message")
334 ("lite", "use lite version of output")
335 ("profiles-file", value<std::string>()->default_value(""), "Specify a file with global profiles")
336 ("host,h", value<std::string>()->default_value("localhost"), "")
337 ("port", value<UInt16>()->default_value(9000), "")
338 ("secure,s", "Use TLS connection")
339 ("database", value<std::string>()->default_value("default"), "")
340 ("user", value<std::string>()->default_value("default"), "")
341 ("password", value<std::string>()->default_value(""), "")
342 ("log-level", value<std::string>()->default_value("information"), "Set log level")
343 ("tags", value<Strings>()->multitoken(), "Run only tests with tag")
344 ("skip-tags", value<Strings>()->multitoken(), "Do not run tests with tag")
345 ("names", value<Strings>()->multitoken(), "Run tests with specific name")
346 ("skip-names", value<Strings>()->multitoken(), "Do not run tests with name")
347 ("names-regexp", value<Strings>()->multitoken(), "Run tests with names matching regexp")
348 ("skip-names-regexp", value<Strings>()->multitoken(), "Do not run tests with names matching regexp")
349 ("input-files", value<Strings>()->multitoken(), "Input .xml files")
350 ("query-indexes", value<std::vector<size_t>>()->multitoken(), "Input query indexes")
351 ("recursive,r", "Recurse in directories to find all xml's")
352 ;
353
354 DB::Settings cmd_settings;
355 cmd_settings.addProgramOptions(desc);
356
357 po::options_description cmdline_options;
358 cmdline_options.add(desc);
359
360 po::variables_map options;
361 po::basic_parsed_options<char> parsed = po::command_line_parser(argc, argv).options(cmdline_options).run();
362 auto queries_with_indexes = getTestQueryIndexes(parsed);
363 po::store(parsed, options);
364
365 po::notify(options);
366
367 Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter("%Y.%m.%d %H:%M:%S.%F <%p> %s: %t"));
368 Poco::AutoPtr<Poco::ConsoleChannel> console_chanel(new Poco::ConsoleChannel);
369 Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_chanel));
370
371 Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
372 Poco::Logger::root().setChannel(channel);
373
374 Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite");
375 if (options.count("help"))
376 {
377 std::cout << "Usage: " << argv[0] << " [options]\n";
378 std::cout << desc << "\n";
379 return 0;
380 }
381
382 Strings input_files = getInputFiles(options, log);
383
384 Strings tests_tags = options.count("tags") ? options["tags"].as<Strings>() : Strings({});
385 Strings skip_tags = options.count("skip-tags") ? options["skip-tags"].as<Strings>() : Strings({});
386 Strings tests_names = options.count("names") ? options["names"].as<Strings>() : Strings({});
387 Strings skip_names = options.count("skip-names") ? options["skip-names"].as<Strings>() : Strings({});
388 Strings tests_names_regexp = options.count("names-regexp") ? options["names-regexp"].as<Strings>() : Strings({});
389 Strings skip_names_regexp = options.count("skip-names-regexp") ? options["skip-names-regexp"].as<Strings>() : Strings({});
390
391 auto timeouts = DB::ConnectionTimeouts::getTCPTimeoutsWithoutFailover(DB::Settings());
392
393 DB::UseSSL use_ssl;
394
395 DB::PerformanceTestSuite performance_test_suite(
396 options["host"].as<std::string>(),
397 options["port"].as<UInt16>(),
398 options.count("secure"),
399 options["database"].as<std::string>(),
400 options["user"].as<std::string>(),
401 options["password"].as<std::string>(),
402 cmd_settings,
403 options.count("lite") > 0,
404 options["profiles-file"].as<std::string>(),
405 std::move(input_files),
406 std::move(tests_tags),
407 std::move(skip_tags),
408 std::move(tests_names),
409 std::move(skip_names),
410 std::move(tests_names_regexp),
411 std::move(skip_names_regexp),
412 queries_with_indexes,
413 timeouts);
414 return performance_test_suite.run();
415}
416catch (...)
417{
418 std::cout << DB::getCurrentExceptionMessage(/*with stacktrace = */ true) << std::endl;
419 int code = DB::getCurrentExceptionCode();
420 return code ? code : 1;
421}
422