1 | #include <Databases/DatabaseOnDisk.h> |
2 | |
3 | #include <IO/ReadBufferFromFile.h> |
4 | #include <IO/ReadHelpers.h> |
5 | #include <IO/WriteBufferFromFile.h> |
6 | #include <IO/WriteHelpers.h> |
7 | #include <Interpreters/Context.h> |
8 | #include <Interpreters/InterpreterCreateQuery.h> |
9 | #include <Parsers/ASTCreateQuery.h> |
10 | #include <Parsers/ParserCreateQuery.h> |
11 | #include <Parsers/formatAST.h> |
12 | #include <Parsers/parseQuery.h> |
13 | #include <Storages/IStorage.h> |
14 | #include <Storages/StorageFactory.h> |
15 | #include <TableFunctions/TableFunctionFactory.h> |
16 | #include <Common/escapeForFileName.h> |
17 | |
18 | #include <common/logger_useful.h> |
19 | #include <Poco/DirectoryIterator.h> |
20 | |
21 | |
22 | |
23 | namespace DB |
24 | { |
25 | |
26 | static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; |
27 | |
28 | namespace ErrorCodes |
29 | { |
30 | extern const int FILE_DOESNT_EXIST; |
31 | extern const int INCORRECT_FILE_NAME; |
32 | extern const int SYNTAX_ERROR; |
33 | extern const int TABLE_ALREADY_EXISTS; |
34 | extern const int UNKNOWN_TABLE; |
35 | extern const int DICTIONARY_ALREADY_EXISTS; |
36 | extern const int EMPTY_LIST_OF_COLUMNS_PASSED; |
37 | } |
38 | |
39 | |
40 | std::pair<String, StoragePtr> createTableFromAST( |
41 | ASTCreateQuery ast_create_query, |
42 | const String & database_name, |
43 | const String & table_data_path_relative, |
44 | Context & context, |
45 | bool has_force_restore_data_flag) |
46 | { |
47 | ast_create_query.attach = true; |
48 | ast_create_query.database = database_name; |
49 | |
50 | if (ast_create_query.as_table_function) |
51 | { |
52 | const auto & table_function = ast_create_query.as_table_function->as<ASTFunction &>(); |
53 | const auto & factory = TableFunctionFactory::instance(); |
54 | StoragePtr storage = factory.get(table_function.name, context)->execute(ast_create_query.as_table_function, context, ast_create_query.table); |
55 | return {ast_create_query.table, storage}; |
56 | } |
57 | /// We do not directly use `InterpreterCreateQuery::execute`, because |
58 | /// - the database has not been loaded yet; |
59 | /// - the code is simpler, since the query is already brought to a suitable form. |
60 | if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns) |
61 | throw Exception("Missing definition of columns." , ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); |
62 | |
63 | ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context); |
64 | ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); |
65 | |
66 | return |
67 | { |
68 | ast_create_query.table, |
69 | StorageFactory::instance().get( |
70 | ast_create_query, |
71 | table_data_path_relative, ast_create_query.table, database_name, context, context.getGlobalContext(), |
72 | columns, constraints, |
73 | true, has_force_restore_data_flag) |
74 | }; |
75 | } |
76 | |
77 | |
78 | String getObjectDefinitionFromCreateQuery(const ASTPtr & query) |
79 | { |
80 | ASTPtr query_clone = query->clone(); |
81 | auto * create = query_clone->as<ASTCreateQuery>(); |
82 | |
83 | if (!create) |
84 | { |
85 | std::ostringstream query_stream; |
86 | formatAST(*query, query_stream, true); |
87 | throw Exception("Query '" + query_stream.str() + "' is not CREATE query" , ErrorCodes::LOGICAL_ERROR); |
88 | } |
89 | |
90 | if (!create->is_dictionary) |
91 | create->attach = true; |
92 | |
93 | /// We remove everything that is not needed for ATTACH from the query. |
94 | create->database.clear(); |
95 | create->as_database.clear(); |
96 | create->as_table.clear(); |
97 | create->if_not_exists = false; |
98 | create->is_populate = false; |
99 | create->replace_view = false; |
100 | |
101 | /// For views it is necessary to save the SELECT query itself, for the rest - on the contrary |
102 | if (!create->is_view && !create->is_materialized_view && !create->is_live_view) |
103 | create->select = nullptr; |
104 | |
105 | create->format = nullptr; |
106 | create->out_file = nullptr; |
107 | |
108 | std::ostringstream statement_stream; |
109 | formatAST(*create, statement_stream, false); |
110 | statement_stream << '\n'; |
111 | return statement_stream.str(); |
112 | } |
113 | |
114 | void DatabaseOnDisk::createTable( |
115 | const Context & context, |
116 | const String & table_name, |
117 | const StoragePtr & table, |
118 | const ASTPtr & query) |
119 | { |
120 | const auto & settings = context.getSettingsRef(); |
121 | |
122 | /// Create a file with metadata if necessary - if the query is not ATTACH. |
123 | /// Write the query of `ATTACH table` to it. |
124 | |
125 | /** The code is based on the assumption that all threads share the same order of operations |
126 | * - creating the .sql.tmp file; |
127 | * - adding a table to `tables`; |
128 | * - rename .sql.tmp to .sql. |
129 | */ |
130 | |
131 | /// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH. |
132 | /// But there is protection from it - see using DDLGuard in InterpreterCreateQuery. |
133 | |
134 | if (isDictionaryExist(context, table_name)) |
135 | throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists." , |
136 | ErrorCodes::DICTIONARY_ALREADY_EXISTS); |
137 | |
138 | if (isTableExist(context, table_name)) |
139 | throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists." , ErrorCodes::TABLE_ALREADY_EXISTS); |
140 | |
141 | String table_metadata_path = getObjectMetadataPath(table_name); |
142 | String table_metadata_tmp_path = table_metadata_path + ".tmp" ; |
143 | String statement; |
144 | |
145 | { |
146 | statement = getObjectDefinitionFromCreateQuery(query); |
147 | |
148 | /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. |
149 | WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); |
150 | writeString(statement, out); |
151 | out.next(); |
152 | if (settings.fsync_metadata) |
153 | out.sync(); |
154 | out.close(); |
155 | } |
156 | |
157 | try |
158 | { |
159 | /// Add a table to the map of known tables. |
160 | attachTable(table_name, table); |
161 | |
162 | /// If it was ATTACH query and file with table metadata already exist |
163 | /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. |
164 | Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); |
165 | } |
166 | catch (...) |
167 | { |
168 | Poco::File(table_metadata_tmp_path).remove(); |
169 | throw; |
170 | } |
171 | } |
172 | |
173 | void DatabaseOnDisk::removeTable(const Context & /* context */, const String & table_name) |
174 | { |
175 | StoragePtr res = detachTable(table_name); |
176 | |
177 | String table_metadata_path = getObjectMetadataPath(table_name); |
178 | |
179 | try |
180 | { |
181 | Poco::File(table_metadata_path).remove(); |
182 | } |
183 | catch (...) |
184 | { |
185 | try |
186 | { |
187 | Poco::File(table_metadata_path + ".tmp_drop" ).remove(); |
188 | return; |
189 | } |
190 | catch (...) |
191 | { |
192 | LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); |
193 | } |
194 | attachTable(table_name, res); |
195 | throw; |
196 | } |
197 | } |
198 | |
199 | void DatabaseOnDisk::renameTable( |
200 | const Context & context, |
201 | const String & table_name, |
202 | IDatabase & to_database, |
203 | const String & to_table_name, |
204 | TableStructureWriteLockHolder & lock) |
205 | { |
206 | if (typeid(*this) != typeid(to_database)) |
207 | throw Exception("Moving tables between databases of different engines is not supported" , ErrorCodes::NOT_IMPLEMENTED); |
208 | |
209 | StoragePtr table = tryGetTable(context, table_name); |
210 | |
211 | if (!table) |
212 | throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist." , ErrorCodes::UNKNOWN_TABLE); |
213 | |
214 | ASTPtr ast = parseQueryFromMetadata(getObjectMetadataPath(table_name)); |
215 | if (!ast) |
216 | throw Exception("There is no metadata file for table " + backQuote(table_name) + "." , ErrorCodes::FILE_DOESNT_EXIST); |
217 | auto & create = ast->as<ASTCreateQuery &>(); |
218 | create.table = to_table_name; |
219 | |
220 | /// Notify the table that it is renamed. If the table does not support renaming, exception is thrown. |
221 | try |
222 | { |
223 | table->rename(to_database.getTableDataPath(create), |
224 | to_database.getDatabaseName(), |
225 | to_table_name, lock); |
226 | } |
227 | catch (const Exception &) |
228 | { |
229 | throw; |
230 | } |
231 | catch (const Poco::Exception & e) |
232 | { |
233 | /// Better diagnostics. |
234 | throw Exception{Exception::CreateFromPoco, e}; |
235 | } |
236 | |
237 | /// NOTE Non-atomic. |
238 | to_database.createTable(context, to_table_name, table, ast); |
239 | removeTable(context, table_name); |
240 | } |
241 | |
242 | ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const |
243 | { |
244 | ASTPtr ast; |
245 | |
246 | auto table_metadata_path = getObjectMetadataPath(table_name); |
247 | ast = getCreateQueryFromMetadata(table_metadata_path, throw_on_error); |
248 | if (!ast && throw_on_error) |
249 | { |
250 | /// Handle system.* tables for which there are no table.sql files. |
251 | bool has_table = tryGetTable(context, table_name) != nullptr; |
252 | |
253 | auto msg = has_table |
254 | ? "There is no CREATE TABLE query for table " |
255 | : "There is no metadata file for table " ; |
256 | |
257 | throw Exception(msg + backQuote(table_name), ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); |
258 | } |
259 | |
260 | return ast; |
261 | } |
262 | |
263 | ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const |
264 | { |
265 | ASTPtr ast; |
266 | |
267 | auto metadata_dir_path = getMetadataPath(); |
268 | auto database_metadata_path = metadata_dir_path.substr(0, metadata_dir_path.size() - 1) + ".sql" ; |
269 | ast = getCreateQueryFromMetadata(database_metadata_path, true); |
270 | if (!ast) |
271 | { |
272 | /// Handle databases (such as default) for which there are no database.sql files. |
273 | /// If database.sql doesn't exist, then engine is Ordinary |
274 | String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary" ; |
275 | ParserCreateQuery parser; |
276 | ast = parseQuery(parser, query.data(), query.data() + query.size(), "" , 0); |
277 | } |
278 | |
279 | return ast; |
280 | } |
281 | |
282 | void DatabaseOnDisk::drop(const Context & context) |
283 | { |
284 | Poco::File(context.getPath() + getDataPath()).remove(false); |
285 | Poco::File(getMetadataPath()).remove(false); |
286 | } |
287 | |
288 | String DatabaseOnDisk::getObjectMetadataPath(const String & table_name) const |
289 | { |
290 | return getMetadataPath() + escapeForFileName(table_name) + ".sql" ; |
291 | } |
292 | |
293 | time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & table_name) const |
294 | { |
295 | String table_metadata_path = getObjectMetadataPath(table_name); |
296 | Poco::File meta_file(table_metadata_path); |
297 | |
298 | if (meta_file.exists()) |
299 | return meta_file.getLastModified().epochTime(); |
300 | else |
301 | return static_cast<time_t>(0); |
302 | } |
303 | |
304 | void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const |
305 | { |
306 | Poco::DirectoryIterator dir_end; |
307 | for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) |
308 | { |
309 | /// For '.svn', '.gitignore' directory and similar. |
310 | if (dir_it.name().at(0) == '.') |
311 | continue; |
312 | |
313 | /// There are .sql.bak files - skip them. |
314 | if (endsWith(dir_it.name(), ".sql.bak" )) |
315 | continue; |
316 | |
317 | // There are files that we tried to delete previously |
318 | static const char * tmp_drop_ext = ".sql.tmp_drop" ; |
319 | if (endsWith(dir_it.name(), tmp_drop_ext)) |
320 | { |
321 | const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext)); |
322 | if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists()) |
323 | { |
324 | /// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK) |
325 | Poco::File(dir_it->path()).renameTo(getMetadataPath() + object_name + ".sql" ); |
326 | LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored" ); |
327 | iterating_function(object_name + ".sql" ); |
328 | } |
329 | else |
330 | { |
331 | LOG_INFO(log, "Removing file " << dir_it->path()); |
332 | Poco::File(dir_it->path()).remove(); |
333 | } |
334 | continue; |
335 | } |
336 | |
337 | /// There are files .sql.tmp - delete |
338 | if (endsWith(dir_it.name(), ".sql.tmp" )) |
339 | { |
340 | LOG_INFO(log, "Removing file " << dir_it->path()); |
341 | Poco::File(dir_it->path()).remove(); |
342 | continue; |
343 | } |
344 | |
345 | /// The required files have names like `table_name.sql` |
346 | if (endsWith(dir_it.name(), ".sql" )) |
347 | { |
348 | iterating_function(dir_it.name()); |
349 | } |
350 | else |
351 | throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(), |
352 | ErrorCodes::INCORRECT_FILE_NAME); |
353 | } |
354 | } |
355 | |
356 | ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const |
357 | { |
358 | String query; |
359 | |
360 | try |
361 | { |
362 | ReadBufferFromFile in(metadata_file_path, METADATA_FILE_BUFFER_SIZE); |
363 | readStringUntilEOF(query, in); |
364 | } |
365 | catch (const Exception & e) |
366 | { |
367 | if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST) |
368 | return nullptr; |
369 | else |
370 | throw; |
371 | } |
372 | |
373 | /** Empty files with metadata are generated after a rough restart of the server. |
374 | * Remove these files to slightly reduce the work of the admins on startup. |
375 | */ |
376 | if (remove_empty && query.empty()) |
377 | { |
378 | LOG_ERROR(log, "File " << metadata_file_path << " is empty. Removing." ); |
379 | Poco::File(metadata_file_path).remove(); |
380 | return nullptr; |
381 | } |
382 | |
383 | ParserCreateQuery parser; |
384 | const char * pos = query.data(); |
385 | std::string error_message; |
386 | auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false, |
387 | "in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0); |
388 | |
389 | if (!ast && throw_on_error) |
390 | throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); |
391 | else if (!ast) |
392 | return nullptr; |
393 | |
394 | return ast; |
395 | } |
396 | |
397 | ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const String & database_metadata_path, bool throw_on_error) const |
398 | { |
399 | ASTPtr ast = parseQueryFromMetadata(database_metadata_path, throw_on_error); |
400 | |
401 | if (ast) |
402 | { |
403 | auto & ast_create_query = ast->as<ASTCreateQuery &>(); |
404 | ast_create_query.attach = false; |
405 | ast_create_query.database = database_name; |
406 | } |
407 | |
408 | return ast; |
409 | } |
410 | |
411 | } |
412 | |