1 | #include <Storages/IStorage.h> |
2 | |
3 | #include <Storages/AlterCommands.h> |
4 | #include <Parsers/ASTCreateQuery.h> |
5 | #include <Parsers/ASTSetQuery.h> |
6 | #include <Common/StringUtils/StringUtils.h> |
7 | #include <Common/quoteString.h> |
8 | |
9 | #include <Processors/Executors/TreeExecutorBlockInputStream.h> |
10 | |
11 | #include <sparsehash/dense_hash_map> |
12 | #include <sparsehash/dense_hash_set> |
13 | |
14 | |
15 | namespace DB |
16 | { |
17 | |
18 | namespace ErrorCodes |
19 | { |
20 | extern const int COLUMN_QUERIED_MORE_THAN_ONCE; |
21 | extern const int DUPLICATE_COLUMN; |
22 | extern const int EMPTY_LIST_OF_COLUMNS_PASSED; |
23 | extern const int EMPTY_LIST_OF_COLUMNS_QUERIED; |
24 | extern const int NO_SUCH_COLUMN_IN_TABLE; |
25 | extern const int NOT_FOUND_COLUMN_IN_BLOCK; |
26 | extern const int TYPE_MISMATCH; |
27 | extern const int SETTINGS_ARE_NOT_SUPPORTED; |
28 | extern const int UNKNOWN_SETTING; |
29 | extern const int TABLE_IS_DROPPED; |
30 | extern const int NOT_IMPLEMENTED; |
31 | } |
32 | |
33 | IStorage::IStorage(ColumnsDescription virtuals_) : virtuals(std::move(virtuals_)) |
34 | { |
35 | } |
36 | |
37 | const ColumnsDescription & IStorage::getColumns() const |
38 | { |
39 | return columns; |
40 | } |
41 | |
42 | const ColumnsDescription & IStorage::getVirtuals() const |
43 | { |
44 | return virtuals; |
45 | } |
46 | |
47 | const IndicesDescription & IStorage::getIndices() const |
48 | { |
49 | return indices; |
50 | } |
51 | |
52 | const ConstraintsDescription & IStorage::getConstraints() const |
53 | { |
54 | return constraints; |
55 | } |
56 | |
57 | NameAndTypePair IStorage::getColumn(const String & column_name) const |
58 | { |
59 | /// By default, we assume that there are no virtual columns in the storage. |
60 | return getColumns().getPhysical(column_name); |
61 | } |
62 | |
63 | bool IStorage::hasColumn(const String & column_name) const |
64 | { |
65 | /// By default, we assume that there are no virtual columns in the storage. |
66 | return getColumns().hasPhysical(column_name); |
67 | } |
68 | |
69 | Block IStorage::getSampleBlock() const |
70 | { |
71 | Block res; |
72 | |
73 | for (const auto & column : getColumns().getAllPhysical()) |
74 | res.insert({column.type->createColumn(), column.type, column.name}); |
75 | |
76 | return res; |
77 | } |
78 | |
79 | Block IStorage::getSampleBlockWithVirtuals() const |
80 | { |
81 | auto res = getSampleBlock(); |
82 | |
83 | for (const auto & column : getColumns().getVirtuals()) |
84 | res.insert({column.type->createColumn(), column.type, column.name}); |
85 | |
86 | return res; |
87 | } |
88 | |
89 | Block IStorage::getSampleBlockNonMaterialized() const |
90 | { |
91 | Block res; |
92 | |
93 | for (const auto & column : getColumns().getOrdinary()) |
94 | res.insert({column.type->createColumn(), column.type, column.name}); |
95 | |
96 | return res; |
97 | } |
98 | |
99 | Block IStorage::getSampleBlockForColumns(const Names & column_names) const |
100 | { |
101 | Block res; |
102 | |
103 | NamesAndTypesList all_columns = getColumns().getAll(); |
104 | std::unordered_map<String, DataTypePtr> columns_map; |
105 | for (const auto & elem : all_columns) |
106 | columns_map.emplace(elem.name, elem.type); |
107 | |
108 | for (const auto & name : column_names) |
109 | { |
110 | auto it = columns_map.find(name); |
111 | if (it != columns_map.end()) |
112 | { |
113 | res.insert({it->second->createColumn(), it->second, it->first}); |
114 | } |
115 | else |
116 | { |
117 | /// Virtual columns. |
118 | NameAndTypePair elem = getColumn(name); |
119 | res.insert({elem.type->createColumn(), elem.type, elem.name}); |
120 | } |
121 | } |
122 | |
123 | return res; |
124 | } |
125 | |
126 | namespace |
127 | { |
128 | using NamesAndTypesMap = ::google::dense_hash_map<StringRef, const IDataType *, StringRefHash>; |
129 | using UniqueStrings = ::google::dense_hash_set<StringRef, StringRefHash>; |
130 | |
131 | String listOfColumns(const NamesAndTypesList & available_columns) |
132 | { |
133 | std::stringstream ss; |
134 | for (auto it = available_columns.begin(); it != available_columns.end(); ++it) |
135 | { |
136 | if (it != available_columns.begin()) |
137 | ss << ", " ; |
138 | ss << it->name; |
139 | } |
140 | return ss.str(); |
141 | } |
142 | |
143 | NamesAndTypesMap getColumnsMap(const NamesAndTypesList & columns) |
144 | { |
145 | NamesAndTypesMap res; |
146 | res.set_empty_key(StringRef()); |
147 | |
148 | for (const auto & column : columns) |
149 | res.insert({column.name, column.type.get()}); |
150 | |
151 | return res; |
152 | } |
153 | |
154 | UniqueStrings initUniqueStrings() |
155 | { |
156 | UniqueStrings strings; |
157 | strings.set_empty_key(StringRef()); |
158 | return strings; |
159 | } |
160 | } |
161 | |
162 | void IStorage::check(const Names & column_names, bool include_virtuals) const |
163 | { |
164 | NamesAndTypesList available_columns = getColumns().getAllPhysical(); |
165 | if (include_virtuals) |
166 | available_columns.splice(available_columns.end(), getColumns().getVirtuals()); |
167 | |
168 | const String list_of_columns = listOfColumns(available_columns); |
169 | |
170 | if (column_names.empty()) |
171 | throw Exception("Empty list of columns queried. There are columns: " + list_of_columns, ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED); |
172 | |
173 | const auto columns_map = getColumnsMap(available_columns); |
174 | |
175 | auto unique_names = initUniqueStrings(); |
176 | for (const auto & name : column_names) |
177 | { |
178 | if (columns_map.end() == columns_map.find(name)) |
179 | throw Exception( |
180 | "There is no column with name " + backQuote(name) + " in table " + getTableName() + ". There are columns: " + list_of_columns, |
181 | ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); |
182 | |
183 | if (unique_names.end() != unique_names.find(name)) |
184 | throw Exception("Column " + name + " queried more than once" , ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE); |
185 | unique_names.insert(name); |
186 | } |
187 | } |
188 | |
189 | void IStorage::check(const NamesAndTypesList & provided_columns) const |
190 | { |
191 | const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); |
192 | const auto columns_map = getColumnsMap(available_columns); |
193 | |
194 | auto unique_names = initUniqueStrings(); |
195 | for (const NameAndTypePair & column : provided_columns) |
196 | { |
197 | auto it = columns_map.find(column.name); |
198 | if (columns_map.end() == it) |
199 | throw Exception( |
200 | "There is no column with name " + column.name + ". There are columns: " + listOfColumns(available_columns), |
201 | ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); |
202 | |
203 | if (!column.type->equals(*it->second)) |
204 | throw Exception( |
205 | "Type mismatch for column " + column.name + ". Column has type " + it->second->getName() + ", got type " |
206 | + column.type->getName(), |
207 | ErrorCodes::TYPE_MISMATCH); |
208 | |
209 | if (unique_names.end() != unique_names.find(column.name)) |
210 | throw Exception("Column " + column.name + " queried more than once" , ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE); |
211 | unique_names.insert(column.name); |
212 | } |
213 | } |
214 | |
215 | void IStorage::check(const NamesAndTypesList & provided_columns, const Names & column_names) const |
216 | { |
217 | const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); |
218 | const auto available_columns_map = getColumnsMap(available_columns); |
219 | const auto & provided_columns_map = getColumnsMap(provided_columns); |
220 | |
221 | if (column_names.empty()) |
222 | throw Exception( |
223 | "Empty list of columns queried. There are columns: " + listOfColumns(available_columns), |
224 | ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED); |
225 | |
226 | auto unique_names = initUniqueStrings(); |
227 | for (const String & name : column_names) |
228 | { |
229 | auto it = provided_columns_map.find(name); |
230 | if (provided_columns_map.end() == it) |
231 | continue; |
232 | |
233 | auto jt = available_columns_map.find(name); |
234 | if (available_columns_map.end() == jt) |
235 | throw Exception( |
236 | "There is no column with name " + name + ". There are columns: " + listOfColumns(available_columns), |
237 | ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); |
238 | |
239 | if (!it->second->equals(*jt->second)) |
240 | throw Exception( |
241 | "Type mismatch for column " + name + ". Column has type " + jt->second->getName() + ", got type " + it->second->getName(), |
242 | ErrorCodes::TYPE_MISMATCH); |
243 | |
244 | if (unique_names.end() != unique_names.find(name)) |
245 | throw Exception("Column " + name + " queried more than once" , ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE); |
246 | unique_names.insert(name); |
247 | } |
248 | } |
249 | |
250 | void IStorage::check(const Block & block, bool need_all) const |
251 | { |
252 | const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); |
253 | const auto columns_map = getColumnsMap(available_columns); |
254 | |
255 | NameSet names_in_block; |
256 | |
257 | block.checkNumberOfRows(); |
258 | |
259 | for (const auto & column : block) |
260 | { |
261 | if (names_in_block.count(column.name)) |
262 | throw Exception("Duplicate column " + column.name + " in block" , ErrorCodes::DUPLICATE_COLUMN); |
263 | |
264 | names_in_block.insert(column.name); |
265 | |
266 | auto it = columns_map.find(column.name); |
267 | if (columns_map.end() == it) |
268 | throw Exception( |
269 | "There is no column with name " + column.name + ". There are columns: " + listOfColumns(available_columns), |
270 | ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); |
271 | |
272 | if (!column.type->equals(*it->second)) |
273 | throw Exception( |
274 | "Type mismatch for column " + column.name + ". Column has type " + it->second->getName() + ", got type " |
275 | + column.type->getName(), |
276 | ErrorCodes::TYPE_MISMATCH); |
277 | } |
278 | |
279 | if (need_all && names_in_block.size() < columns_map.size()) |
280 | { |
281 | for (auto it = available_columns.begin(); it != available_columns.end(); ++it) |
282 | { |
283 | if (!names_in_block.count(it->name)) |
284 | throw Exception("Expected column " + it->name, ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); |
285 | } |
286 | } |
287 | } |
288 | |
289 | void IStorage::setColumns(ColumnsDescription columns_) |
290 | { |
291 | if (columns_.getOrdinary().empty()) |
292 | throw Exception("Empty list of columns passed" , ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); |
293 | columns = std::move(columns_); |
294 | |
295 | for (const auto & column : virtuals) |
296 | { |
297 | if (!columns.has(column.name)) |
298 | columns.add(column); |
299 | } |
300 | } |
301 | |
302 | void IStorage::setIndices(IndicesDescription indices_) |
303 | { |
304 | indices = std::move(indices_); |
305 | } |
306 | |
307 | void IStorage::setConstraints(ConstraintsDescription constraints_) |
308 | { |
309 | constraints = std::move(constraints_); |
310 | } |
311 | |
312 | bool IStorage::isVirtualColumn(const String & column_name) const |
313 | { |
314 | return getColumns().get(column_name).is_virtual; |
315 | } |
316 | |
317 | TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id) |
318 | { |
319 | TableStructureReadLockHolder result; |
320 | if (will_add_new_data) |
321 | result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Read, query_id); |
322 | result.structure_lock = structure_lock->getLock(RWLockImpl::Read, query_id); |
323 | |
324 | if (is_dropped) |
325 | throw Exception("Table is dropped" , ErrorCodes::TABLE_IS_DROPPED); |
326 | return result; |
327 | } |
328 | |
329 | TableStructureWriteLockHolder IStorage::lockAlterIntention(const String & query_id) |
330 | { |
331 | TableStructureWriteLockHolder result; |
332 | result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id); |
333 | |
334 | if (is_dropped) |
335 | throw Exception("Table is dropped" , ErrorCodes::TABLE_IS_DROPPED); |
336 | return result; |
337 | } |
338 | |
339 | void IStorage::lockNewDataStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id) |
340 | { |
341 | if (!lock_holder.alter_intention_lock) |
342 | throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug." , ErrorCodes::LOGICAL_ERROR); |
343 | |
344 | lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); |
345 | } |
346 | |
347 | void IStorage::lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id) |
348 | { |
349 | if (!lock_holder.alter_intention_lock) |
350 | throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug." , ErrorCodes::LOGICAL_ERROR); |
351 | |
352 | if (!lock_holder.new_data_structure_lock) |
353 | lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); |
354 | lock_holder.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id); |
355 | } |
356 | |
357 | TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id) |
358 | { |
359 | TableStructureWriteLockHolder result; |
360 | result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id); |
361 | |
362 | if (is_dropped) |
363 | throw Exception("Table is dropped" , ErrorCodes::TABLE_IS_DROPPED); |
364 | |
365 | result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id); |
366 | result.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id); |
367 | |
368 | return result; |
369 | } |
370 | |
371 | StorageInMemoryMetadata IStorage::getInMemoryMetadata() const |
372 | { |
373 | return |
374 | { |
375 | .columns = getColumns(), |
376 | .indices = getIndices(), |
377 | .constraints = getConstraints(), |
378 | }; |
379 | } |
380 | |
381 | void IStorage::alter( |
382 | const AlterCommands & params, |
383 | const Context & context, |
384 | TableStructureWriteLockHolder & table_lock_holder) |
385 | { |
386 | const String database_name = getDatabaseName(); |
387 | const String table_name = getTableName(); |
388 | |
389 | lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); |
390 | |
391 | StorageInMemoryMetadata metadata = getInMemoryMetadata(); |
392 | params.apply(metadata); |
393 | context.getDatabase(database_name)->alterTable(context, table_name, metadata); |
394 | setColumns(std::move(metadata.columns)); |
395 | } |
396 | |
397 | |
398 | void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) |
399 | { |
400 | for (const auto & command : commands) |
401 | { |
402 | if (!command.isCommentAlter()) |
403 | throw Exception( |
404 | "Alter of type '" + alterTypeToString(command.type) + "' is not supported by storage " + getName(), |
405 | ErrorCodes::NOT_IMPLEMENTED); |
406 | } |
407 | } |
408 | |
409 | BlockInputStreams IStorage::read( |
410 | const Names & column_names, |
411 | const SelectQueryInfo & query_info, |
412 | const Context & context, |
413 | QueryProcessingStage::Enum processed_stage, |
414 | size_t max_block_size, |
415 | unsigned num_streams) |
416 | { |
417 | auto pipes = readWithProcessors(column_names, query_info, context, processed_stage, max_block_size, num_streams); |
418 | |
419 | BlockInputStreams res; |
420 | res.reserve(pipes.size()); |
421 | |
422 | for (auto & pipe : pipes) |
423 | res.emplace_back(std::make_shared<TreeExecutorBlockInputStream>(std::move(pipe))); |
424 | |
425 | return res; |
426 | } |
427 | |
428 | DB::CompressionMethod IStorage::chooseCompressionMethod(const String & uri, const String & compression_method) |
429 | { |
430 | if (compression_method == "auto" || compression_method == "" ) |
431 | { |
432 | if (endsWith(uri, ".gz" )) |
433 | return DB::CompressionMethod::Gzip; |
434 | else |
435 | return DB::CompressionMethod::None; |
436 | } |
437 | else if (compression_method == "gzip" ) |
438 | return DB::CompressionMethod::Gzip; |
439 | else if (compression_method == "none" ) |
440 | return DB::CompressionMethod::None; |
441 | else |
442 | throw Exception("Only auto, none, gzip supported as compression method" , ErrorCodes::NOT_IMPLEMENTED); |
443 | } |
444 | |
445 | } |
446 | |