1 | #include <iomanip> |
2 | #include <thread> |
3 | #include <future> |
4 | |
5 | #include <Common/ThreadPool.h> |
6 | |
7 | #include <Poco/DirectoryIterator.h> |
8 | #include <Poco/FileStream.h> |
9 | |
10 | #include <Parsers/ParserCreateQuery.h> |
11 | #include <Parsers/ASTCreateQuery.h> |
12 | #include <Parsers/parseQuery.h> |
13 | |
14 | #include <Interpreters/InterpreterCreateQuery.h> |
15 | #include <Interpreters/Context.h> |
16 | #include <Interpreters/loadMetadata.h> |
17 | |
18 | #include <Databases/DatabaseOrdinary.h> |
19 | |
20 | #include <IO/ReadBufferFromFile.h> |
21 | #include <IO/ReadHelpers.h> |
22 | #include <Common/escapeForFileName.h> |
23 | |
24 | #include <Common/Stopwatch.h> |
25 | #include <Common/typeid_cast.h> |
26 | |
27 | |
28 | namespace DB |
29 | { |
30 | |
31 | static void executeCreateQuery( |
32 | const String & query, |
33 | Context & context, |
34 | const String & database, |
35 | const String & file_name, |
36 | bool has_force_restore_data_flag) |
37 | { |
38 | ParserCreateQuery parser; |
39 | ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "in file " + file_name, 0); |
40 | |
41 | auto & ast_create_query = ast->as<ASTCreateQuery &>(); |
42 | ast_create_query.attach = true; |
43 | ast_create_query.database = database; |
44 | |
45 | InterpreterCreateQuery interpreter(ast, context); |
46 | interpreter.setInternal(true); |
47 | interpreter.setForceRestoreData(has_force_restore_data_flag); |
48 | interpreter.execute(); |
49 | } |
50 | |
51 | |
52 | static void loadDatabase( |
53 | Context & context, |
54 | const String & database, |
55 | const String & database_path, |
56 | bool force_restore_data) |
57 | { |
58 | /// There may exist .sql file with database creation statement. |
59 | /// Or, if it is absent, then database with default engine is created. |
60 | |
61 | String database_attach_query; |
62 | String database_metadata_file = database_path + ".sql" ; |
63 | |
64 | if (Poco::File(database_metadata_file).exists()) |
65 | { |
66 | ReadBufferFromFile in(database_metadata_file, 1024); |
67 | readStringUntilEOF(database_attach_query, in); |
68 | } |
69 | else |
70 | database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database); |
71 | |
72 | executeCreateQuery(database_attach_query, context, database, |
73 | database_metadata_file, force_restore_data); |
74 | } |
75 | |
76 | |
77 | #define SYSTEM_DATABASE "system" |
78 | |
79 | |
80 | void loadMetadata(Context & context) |
81 | { |
82 | String path = context.getPath() + "metadata" ; |
83 | |
84 | /** There may exist 'force_restore_data' file, that means, |
85 | * skip safety threshold on difference of data parts while initializing tables. |
86 | * This file is deleted after successful loading of tables. |
87 | * (flag is "one-shot") |
88 | */ |
89 | Poco::File force_restore_data_flag_file(context.getFlagsPath() + "force_restore_data" ); |
90 | bool has_force_restore_data_flag = force_restore_data_flag_file.exists(); |
91 | |
92 | /// Loop over databases. |
93 | std::map<String, String> databases; |
94 | Poco::DirectoryIterator dir_end; |
95 | for (Poco::DirectoryIterator it(path); it != dir_end; ++it) |
96 | { |
97 | if (!it->isDirectory()) |
98 | continue; |
99 | |
100 | /// For '.svn', '.gitignore' directory and similar. |
101 | if (it.name().at(0) == '.') |
102 | continue; |
103 | |
104 | if (it.name() == SYSTEM_DATABASE) |
105 | continue; |
106 | |
107 | databases.emplace(unescapeForFileName(it.name()), it.path().toString()); |
108 | } |
109 | |
110 | for (const auto & [name, db_path] : databases) |
111 | loadDatabase(context, name, db_path, has_force_restore_data_flag); |
112 | |
113 | if (has_force_restore_data_flag) |
114 | { |
115 | try |
116 | { |
117 | force_restore_data_flag_file.remove(); |
118 | } |
119 | catch (...) |
120 | { |
121 | tryLogCurrentException("Load metadata" , "Can't remove force restore file to enable data santity checks" ); |
122 | } |
123 | } |
124 | } |
125 | |
126 | |
127 | void loadMetadataSystem(Context & context) |
128 | { |
129 | String path = context.getPath() + "metadata/" SYSTEM_DATABASE; |
130 | if (Poco::File(path).exists()) |
131 | { |
132 | /// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted. |
133 | loadDatabase(context, SYSTEM_DATABASE, path, true); |
134 | } |
135 | else |
136 | { |
137 | /// Initialize system database manually |
138 | String global_path = context.getPath(); |
139 | Poco::File(global_path + "data/" SYSTEM_DATABASE).createDirectories(); |
140 | Poco::File(global_path + "metadata/" SYSTEM_DATABASE).createDirectories(); |
141 | |
142 | auto system_database = std::make_shared<DatabaseOrdinary>(SYSTEM_DATABASE, global_path + "metadata/" SYSTEM_DATABASE, context); |
143 | context.addDatabase(SYSTEM_DATABASE, system_database); |
144 | } |
145 | |
146 | } |
147 | |
148 | } |
149 | |