1 | #include <algorithm> |
2 | #include <iostream> |
3 | #include <limits> |
4 | #include <regex> |
5 | #include <thread> |
6 | #include <memory> |
7 | #include <filesystem> |
8 | |
9 | #include <port/unistd.h> |
10 | #include <sys/stat.h> |
11 | |
12 | #include <boost/program_options.hpp> |
13 | |
14 | #include <Poco/AutoPtr.h> |
15 | #include <Poco/ConsoleChannel.h> |
16 | #include <Poco/FormattingChannel.h> |
17 | #include <Poco/Logger.h> |
18 | #include <Poco/Path.h> |
19 | #include <Poco/PatternFormatter.h> |
20 | #include <Poco/Util/XMLConfiguration.h> |
21 | |
22 | #include <common/logger_useful.h> |
23 | #include <Client/Connection.h> |
24 | #include <Core/Types.h> |
25 | #include <Interpreters/Context.h> |
26 | #include <IO/ConnectionTimeouts.h> |
27 | #include <IO/UseSSL.h> |
28 | #include <Core/Settings.h> |
29 | #include <Common/Exception.h> |
30 | #include <Common/InterruptListener.h> |
31 | #include <Common/TerminalSize.h> |
32 | |
33 | #include "TestStopConditions.h" |
34 | #include "TestStats.h" |
35 | #include "ConfigPreprocessor.h" |
36 | #include "PerformanceTest.h" |
37 | #include "ReportBuilder.h" |
38 | |
39 | |
40 | namespace fs = std::filesystem; |
41 | namespace po = boost::program_options; |
42 | |
43 | namespace DB |
44 | { |
45 | namespace ErrorCodes |
46 | { |
47 | extern const int BAD_ARGUMENTS; |
48 | extern const int FILE_DOESNT_EXIST; |
49 | } |
50 | |
51 | /** Tests launcher for ClickHouse. |
52 | * The tool walks through given or default folder in order to find files with |
53 | * tests' descriptions and launches it. |
54 | */ |
55 | class PerformanceTestSuite |
56 | { |
57 | public: |
58 | |
59 | PerformanceTestSuite(const std::string & host_, |
60 | const UInt16 port_, |
61 | const bool secure_, |
62 | const std::string & default_database_, |
63 | const std::string & user_, |
64 | const std::string & password_, |
65 | const Settings & cmd_settings, |
66 | const bool lite_output_, |
67 | const std::string & profiles_file_, |
68 | Strings && input_files_, |
69 | Strings && tests_tags_, |
70 | Strings && skip_tags_, |
71 | Strings && tests_names_, |
72 | Strings && skip_names_, |
73 | Strings && tests_names_regexp_, |
74 | Strings && skip_names_regexp_, |
75 | const std::unordered_map<std::string, std::vector<size_t>> query_indexes_, |
76 | const ConnectionTimeouts & timeouts_) |
77 | : connection(host_, port_, default_database_, user_, |
78 | password_, "performance-test" , Protocol::Compression::Enable, |
79 | secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable) |
80 | , timeouts(timeouts_) |
81 | , tests_tags(std::move(tests_tags_)) |
82 | , tests_names(std::move(tests_names_)) |
83 | , tests_names_regexp(std::move(tests_names_regexp_)) |
84 | , skip_tags(std::move(skip_tags_)) |
85 | , skip_names(std::move(skip_names_)) |
86 | , skip_names_regexp(std::move(skip_names_regexp_)) |
87 | , query_indexes(query_indexes_) |
88 | , lite_output(lite_output_) |
89 | , profiles_file(profiles_file_) |
90 | , input_files(input_files_) |
91 | , log(&Poco::Logger::get("PerformanceTestSuite" )) |
92 | { |
93 | global_context.makeGlobalContext(); |
94 | global_context.getSettingsRef().copyChangesFrom(cmd_settings); |
95 | if (input_files.size() < 1) |
96 | throw Exception("No tests were specified" , ErrorCodes::BAD_ARGUMENTS); |
97 | } |
98 | |
99 | int run() |
100 | { |
101 | std::string name; |
102 | UInt64 version_major; |
103 | UInt64 version_minor; |
104 | UInt64 version_patch; |
105 | UInt64 version_revision; |
106 | connection.getServerVersion(timeouts, name, version_major, version_minor, version_patch, version_revision); |
107 | |
108 | std::stringstream ss; |
109 | ss << version_major << "." << version_minor << "." << version_patch; |
110 | server_version = ss.str(); |
111 | |
112 | report_builder = std::make_shared<ReportBuilder>(server_version); |
113 | |
114 | processTestsConfigurations(input_files); |
115 | |
116 | return 0; |
117 | } |
118 | |
119 | private: |
120 | Connection connection; |
121 | const ConnectionTimeouts & timeouts; |
122 | |
123 | const Strings & tests_tags; |
124 | const Strings & tests_names; |
125 | const Strings & tests_names_regexp; |
126 | const Strings & skip_tags; |
127 | const Strings & skip_names; |
128 | const Strings & skip_names_regexp; |
129 | std::unordered_map<std::string, std::vector<size_t>> query_indexes; |
130 | |
131 | Context global_context = Context::createGlobal(); |
132 | std::shared_ptr<ReportBuilder> report_builder; |
133 | |
134 | std::string server_version; |
135 | |
136 | InterruptListener interrupt_listener; |
137 | |
138 | using XMLConfiguration = Poco::Util::XMLConfiguration; |
139 | using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>; |
140 | |
141 | bool lite_output; |
142 | std::string profiles_file; |
143 | |
144 | Strings input_files; |
145 | std::vector<XMLConfigurationPtr> tests_configurations; |
146 | Poco::Logger * log; |
147 | |
148 | void processTestsConfigurations(const Strings & paths) |
149 | { |
150 | LOG_INFO(log, "Preparing test configurations" ); |
151 | ConfigPreprocessor config_prep(paths); |
152 | tests_configurations = config_prep.processConfig( |
153 | tests_tags, |
154 | tests_names, |
155 | tests_names_regexp, |
156 | skip_tags, |
157 | skip_names, |
158 | skip_names_regexp); |
159 | |
160 | LOG_INFO(log, "Test configurations prepared" ); |
161 | |
162 | if (tests_configurations.size()) |
163 | { |
164 | Strings outputs; |
165 | |
166 | for (auto & test_config : tests_configurations) |
167 | { |
168 | auto [output, signal] = runTest(test_config); |
169 | if (!output.empty()) |
170 | { |
171 | if (lite_output) |
172 | std::cout << output; |
173 | else |
174 | outputs.push_back(output); |
175 | } |
176 | if (signal) |
177 | break; |
178 | } |
179 | |
180 | if (!lite_output && outputs.size()) |
181 | { |
182 | std::cout << "[" << std::endl; |
183 | |
184 | for (size_t i = 0; i != outputs.size(); ++i) |
185 | { |
186 | std::cout << outputs[i]; |
187 | if (i != outputs.size() - 1) |
188 | std::cout << "," ; |
189 | |
190 | std::cout << std::endl; |
191 | } |
192 | |
193 | std::cout << "]" << std::endl; |
194 | } |
195 | } |
196 | } |
197 | |
198 | std::pair<std::string, bool> runTest(XMLConfigurationPtr & test_config) |
199 | { |
200 | PerformanceTestInfo info(test_config, profiles_file, global_context.getSettingsRef()); |
201 | LOG_INFO(log, "Config for test '" << info.test_name << "' parsed" ); |
202 | PerformanceTest current(test_config, connection, timeouts, interrupt_listener, info, global_context, query_indexes[info.path]); |
203 | |
204 | if (current.checkPreconditions()) |
205 | { |
206 | LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fullfilled" ); |
207 | LOG_INFO( |
208 | log, |
209 | "Preparing for run, have " << info.create_and_fill_queries.size() << " create and fill queries" ); |
210 | current.prepare(); |
211 | LOG_INFO(log, "Prepared" ); |
212 | LOG_INFO(log, "Running test '" << info.test_name << "'" ); |
213 | auto result = current.execute(); |
214 | LOG_INFO(log, "Test '" << info.test_name << "' finished" ); |
215 | |
216 | LOG_INFO(log, "Running post run queries" ); |
217 | current.finish(); |
218 | LOG_INFO(log, "Postqueries finished" ); |
219 | if (lite_output) |
220 | return {report_builder->buildCompactReport(info, result, query_indexes[info.path]), current.checkSIGINT()}; |
221 | else |
222 | return {report_builder->buildFullReport(info, result, query_indexes[info.path]), current.checkSIGINT()}; |
223 | } |
224 | else |
225 | LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are not fullfilled, skip run" ); |
226 | |
227 | return {"" , current.checkSIGINT()}; |
228 | } |
229 | }; |
230 | |
231 | } |
232 | |
233 | static void getFilesFromDir(const fs::path & dir, std::vector<std::string> & input_files, const bool recursive = false) |
234 | { |
235 | Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite" ); |
236 | if (dir.extension().string() == ".xml" ) |
237 | LOG_WARNING(log, dir.string() + "' is a directory, but has .xml extension" ); |
238 | |
239 | fs::directory_iterator end; |
240 | for (fs::directory_iterator it(dir); it != end; ++it) |
241 | { |
242 | const fs::path file = (*it); |
243 | if (recursive && fs::is_directory(file)) |
244 | getFilesFromDir(file, input_files, recursive); |
245 | else if (!fs::is_directory(file) && file.extension().string() == ".xml" ) |
246 | input_files.push_back(file.string()); |
247 | } |
248 | } |
249 | |
250 | static std::vector<std::string> getInputFiles(const po::variables_map & options, Poco::Logger * log) |
251 | { |
252 | std::vector<std::string> input_files; |
253 | bool recursive = options.count("recursive" ); |
254 | |
255 | if (!options.count("input-files" )) |
256 | { |
257 | LOG_INFO(log, "Trying to find test scenario files in the current folder..." ); |
258 | fs::path curr_dir("." ); |
259 | |
260 | getFilesFromDir(curr_dir, input_files, recursive); |
261 | |
262 | if (input_files.empty()) |
263 | throw DB::Exception("Did not find any xml files" , DB::ErrorCodes::BAD_ARGUMENTS); |
264 | } |
265 | else |
266 | { |
267 | input_files = options["input-files" ].as<std::vector<std::string>>(); |
268 | |
269 | std::vector<std::string> collected_files; |
270 | for (const std::string & filename : input_files) |
271 | { |
272 | fs::path file(filename); |
273 | |
274 | if (!fs::exists(file)) |
275 | throw DB::Exception("File '" + filename + "' does not exist" , DB::ErrorCodes::FILE_DOESNT_EXIST); |
276 | |
277 | if (fs::is_directory(file)) |
278 | { |
279 | getFilesFromDir(file, collected_files, recursive); |
280 | } |
281 | else |
282 | { |
283 | if (file.extension().string() != ".xml" ) |
284 | throw DB::Exception("File '" + filename + "' does not have .xml extension" , DB::ErrorCodes::BAD_ARGUMENTS); |
285 | collected_files.push_back(filename); |
286 | } |
287 | } |
288 | |
289 | input_files = std::move(collected_files); |
290 | } |
291 | |
292 | LOG_INFO(log, "Found " + std::to_string(input_files.size()) + " input files" ); |
293 | std::sort(input_files.begin(), input_files.end()); |
294 | return input_files; |
295 | } |
296 | |
297 | static std::unordered_map<std::string, std::vector<std::size_t>> getTestQueryIndexes(const po::basic_parsed_options<char> & parsed_opts) |
298 | { |
299 | std::unordered_map<std::string, std::vector<std::size_t>> result; |
300 | const auto & options = parsed_opts.options; |
301 | if (options.empty()) |
302 | return result; |
303 | for (size_t i = 0; i < options.size() - 1; ++i) |
304 | { |
305 | const auto & opt = options[i]; |
306 | if (opt.string_key == "input-files" ) |
307 | { |
308 | if (options[i + 1].string_key == "query-indexes" ) |
309 | { |
310 | const std::string & test_path = Poco::Path(opt.value[0]).absolute().toString(); |
311 | for (const auto & query_num_str : options[i + 1].value) |
312 | { |
313 | size_t query_num = std::stoul(query_num_str); |
314 | result[test_path].push_back(query_num); |
315 | } |
316 | } |
317 | } |
318 | } |
319 | return result; |
320 | } |
321 | |
322 | #pragma GCC diagnostic ignored "-Wunused-function" |
323 | #pragma GCC diagnostic ignored "-Wmissing-declarations" |
324 | |
325 | int mainEntryClickHousePerformanceTest(int argc, char ** argv) |
326 | try |
327 | { |
328 | using po::value; |
329 | using Strings = DB::Strings; |
330 | |
331 | po::options_description desc = createOptionsDescription("Allowed options" , getTerminalWidth()); |
332 | desc.add_options() |
333 | ("help" , "produce help message" ) |
334 | ("lite" , "use lite version of output" ) |
335 | ("profiles-file" , value<std::string>()->default_value("" ), "Specify a file with global profiles" ) |
336 | ("host,h" , value<std::string>()->default_value("localhost" ), "" ) |
337 | ("port" , value<UInt16>()->default_value(9000), "" ) |
338 | ("secure,s" , "Use TLS connection" ) |
339 | ("database" , value<std::string>()->default_value("default" ), "" ) |
340 | ("user" , value<std::string>()->default_value("default" ), "" ) |
341 | ("password" , value<std::string>()->default_value("" ), "" ) |
342 | ("log-level" , value<std::string>()->default_value("information" ), "Set log level" ) |
343 | ("tags" , value<Strings>()->multitoken(), "Run only tests with tag" ) |
344 | ("skip-tags" , value<Strings>()->multitoken(), "Do not run tests with tag" ) |
345 | ("names" , value<Strings>()->multitoken(), "Run tests with specific name" ) |
346 | ("skip-names" , value<Strings>()->multitoken(), "Do not run tests with name" ) |
347 | ("names-regexp" , value<Strings>()->multitoken(), "Run tests with names matching regexp" ) |
348 | ("skip-names-regexp" , value<Strings>()->multitoken(), "Do not run tests with names matching regexp" ) |
349 | ("input-files" , value<Strings>()->multitoken(), "Input .xml files" ) |
350 | ("query-indexes" , value<std::vector<size_t>>()->multitoken(), "Input query indexes" ) |
351 | ("recursive,r" , "Recurse in directories to find all xml's" ) |
352 | ; |
353 | |
354 | DB::Settings cmd_settings; |
355 | cmd_settings.addProgramOptions(desc); |
356 | |
357 | po::options_description cmdline_options; |
358 | cmdline_options.add(desc); |
359 | |
360 | po::variables_map options; |
361 | po::basic_parsed_options<char> parsed = po::command_line_parser(argc, argv).options(cmdline_options).run(); |
362 | auto queries_with_indexes = getTestQueryIndexes(parsed); |
363 | po::store(parsed, options); |
364 | |
365 | po::notify(options); |
366 | |
367 | Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter("%Y.%m.%d %H:%M:%S.%F <%p> %s: %t" )); |
368 | Poco::AutoPtr<Poco::ConsoleChannel> console_chanel(new Poco::ConsoleChannel); |
369 | Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_chanel)); |
370 | |
371 | Poco::Logger::root().setLevel(options["log-level" ].as<std::string>()); |
372 | Poco::Logger::root().setChannel(channel); |
373 | |
374 | Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite" ); |
375 | if (options.count("help" )) |
376 | { |
377 | std::cout << "Usage: " << argv[0] << " [options]\n" ; |
378 | std::cout << desc << "\n" ; |
379 | return 0; |
380 | } |
381 | |
382 | Strings input_files = getInputFiles(options, log); |
383 | |
384 | Strings tests_tags = options.count("tags" ) ? options["tags" ].as<Strings>() : Strings({}); |
385 | Strings skip_tags = options.count("skip-tags" ) ? options["skip-tags" ].as<Strings>() : Strings({}); |
386 | Strings tests_names = options.count("names" ) ? options["names" ].as<Strings>() : Strings({}); |
387 | Strings skip_names = options.count("skip-names" ) ? options["skip-names" ].as<Strings>() : Strings({}); |
388 | Strings tests_names_regexp = options.count("names-regexp" ) ? options["names-regexp" ].as<Strings>() : Strings({}); |
389 | Strings skip_names_regexp = options.count("skip-names-regexp" ) ? options["skip-names-regexp" ].as<Strings>() : Strings({}); |
390 | |
391 | auto timeouts = DB::ConnectionTimeouts::getTCPTimeoutsWithoutFailover(DB::Settings()); |
392 | |
393 | DB::UseSSL use_ssl; |
394 | |
395 | DB::PerformanceTestSuite performance_test_suite( |
396 | options["host" ].as<std::string>(), |
397 | options["port" ].as<UInt16>(), |
398 | options.count("secure" ), |
399 | options["database" ].as<std::string>(), |
400 | options["user" ].as<std::string>(), |
401 | options["password" ].as<std::string>(), |
402 | cmd_settings, |
403 | options.count("lite" ) > 0, |
404 | options["profiles-file" ].as<std::string>(), |
405 | std::move(input_files), |
406 | std::move(tests_tags), |
407 | std::move(skip_tags), |
408 | std::move(tests_names), |
409 | std::move(skip_names), |
410 | std::move(tests_names_regexp), |
411 | std::move(skip_names_regexp), |
412 | queries_with_indexes, |
413 | timeouts); |
414 | return performance_test_suite.run(); |
415 | } |
416 | catch (...) |
417 | { |
418 | std::cout << DB::getCurrentExceptionMessage(/*with stacktrace = */ true) << std::endl; |
419 | int code = DB::getCurrentExceptionCode(); |
420 | return code ? code : 1; |
421 | } |
422 | |