1#include <Storages/IStorage.h>
2
3#include <Storages/AlterCommands.h>
4#include <Parsers/ASTCreateQuery.h>
5#include <Parsers/ASTSetQuery.h>
6#include <Common/StringUtils/StringUtils.h>
7#include <Common/quoteString.h>
8
9#include <Processors/Executors/TreeExecutorBlockInputStream.h>
10
11#include <sparsehash/dense_hash_map>
12#include <sparsehash/dense_hash_set>
13
14
15namespace DB
16{
17
18namespace ErrorCodes
19{
20 extern const int COLUMN_QUERIED_MORE_THAN_ONCE;
21 extern const int DUPLICATE_COLUMN;
22 extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
23 extern const int EMPTY_LIST_OF_COLUMNS_QUERIED;
24 extern const int NO_SUCH_COLUMN_IN_TABLE;
25 extern const int NOT_FOUND_COLUMN_IN_BLOCK;
26 extern const int TYPE_MISMATCH;
27 extern const int SETTINGS_ARE_NOT_SUPPORTED;
28 extern const int UNKNOWN_SETTING;
29 extern const int TABLE_IS_DROPPED;
30 extern const int NOT_IMPLEMENTED;
31}
32
33IStorage::IStorage(ColumnsDescription virtuals_) : virtuals(std::move(virtuals_))
34{
35}
36
37const ColumnsDescription & IStorage::getColumns() const
38{
39 return columns;
40}
41
42const ColumnsDescription & IStorage::getVirtuals() const
43{
44 return virtuals;
45}
46
47const IndicesDescription & IStorage::getIndices() const
48{
49 return indices;
50}
51
52const ConstraintsDescription & IStorage::getConstraints() const
53{
54 return constraints;
55}
56
57NameAndTypePair IStorage::getColumn(const String & column_name) const
58{
59 /// By default, we assume that there are no virtual columns in the storage.
60 return getColumns().getPhysical(column_name);
61}
62
63bool IStorage::hasColumn(const String & column_name) const
64{
65 /// By default, we assume that there are no virtual columns in the storage.
66 return getColumns().hasPhysical(column_name);
67}
68
69Block IStorage::getSampleBlock() const
70{
71 Block res;
72
73 for (const auto & column : getColumns().getAllPhysical())
74 res.insert({column.type->createColumn(), column.type, column.name});
75
76 return res;
77}
78
79Block IStorage::getSampleBlockWithVirtuals() const
80{
81 auto res = getSampleBlock();
82
83 for (const auto & column : getColumns().getVirtuals())
84 res.insert({column.type->createColumn(), column.type, column.name});
85
86 return res;
87}
88
89Block IStorage::getSampleBlockNonMaterialized() const
90{
91 Block res;
92
93 for (const auto & column : getColumns().getOrdinary())
94 res.insert({column.type->createColumn(), column.type, column.name});
95
96 return res;
97}
98
99Block IStorage::getSampleBlockForColumns(const Names & column_names) const
100{
101 Block res;
102
103 NamesAndTypesList all_columns = getColumns().getAll();
104 std::unordered_map<String, DataTypePtr> columns_map;
105 for (const auto & elem : all_columns)
106 columns_map.emplace(elem.name, elem.type);
107
108 for (const auto & name : column_names)
109 {
110 auto it = columns_map.find(name);
111 if (it != columns_map.end())
112 {
113 res.insert({it->second->createColumn(), it->second, it->first});
114 }
115 else
116 {
117 /// Virtual columns.
118 NameAndTypePair elem = getColumn(name);
119 res.insert({elem.type->createColumn(), elem.type, elem.name});
120 }
121 }
122
123 return res;
124}
125
126namespace
127{
128 using NamesAndTypesMap = ::google::dense_hash_map<StringRef, const IDataType *, StringRefHash>;
129 using UniqueStrings = ::google::dense_hash_set<StringRef, StringRefHash>;
130
131 String listOfColumns(const NamesAndTypesList & available_columns)
132 {
133 std::stringstream ss;
134 for (auto it = available_columns.begin(); it != available_columns.end(); ++it)
135 {
136 if (it != available_columns.begin())
137 ss << ", ";
138 ss << it->name;
139 }
140 return ss.str();
141 }
142
143 NamesAndTypesMap getColumnsMap(const NamesAndTypesList & columns)
144 {
145 NamesAndTypesMap res;
146 res.set_empty_key(StringRef());
147
148 for (const auto & column : columns)
149 res.insert({column.name, column.type.get()});
150
151 return res;
152 }
153
154 UniqueStrings initUniqueStrings()
155 {
156 UniqueStrings strings;
157 strings.set_empty_key(StringRef());
158 return strings;
159 }
160}
161
162void IStorage::check(const Names & column_names, bool include_virtuals) const
163{
164 NamesAndTypesList available_columns = getColumns().getAllPhysical();
165 if (include_virtuals)
166 available_columns.splice(available_columns.end(), getColumns().getVirtuals());
167
168 const String list_of_columns = listOfColumns(available_columns);
169
170 if (column_names.empty())
171 throw Exception("Empty list of columns queried. There are columns: " + list_of_columns, ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED);
172
173 const auto columns_map = getColumnsMap(available_columns);
174
175 auto unique_names = initUniqueStrings();
176 for (const auto & name : column_names)
177 {
178 if (columns_map.end() == columns_map.find(name))
179 throw Exception(
180 "There is no column with name " + backQuote(name) + " in table " + getTableName() + ". There are columns: " + list_of_columns,
181 ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
182
183 if (unique_names.end() != unique_names.find(name))
184 throw Exception("Column " + name + " queried more than once", ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE);
185 unique_names.insert(name);
186 }
187}
188
189void IStorage::check(const NamesAndTypesList & provided_columns) const
190{
191 const NamesAndTypesList & available_columns = getColumns().getAllPhysical();
192 const auto columns_map = getColumnsMap(available_columns);
193
194 auto unique_names = initUniqueStrings();
195 for (const NameAndTypePair & column : provided_columns)
196 {
197 auto it = columns_map.find(column.name);
198 if (columns_map.end() == it)
199 throw Exception(
200 "There is no column with name " + column.name + ". There are columns: " + listOfColumns(available_columns),
201 ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
202
203 if (!column.type->equals(*it->second))
204 throw Exception(
205 "Type mismatch for column " + column.name + ". Column has type " + it->second->getName() + ", got type "
206 + column.type->getName(),
207 ErrorCodes::TYPE_MISMATCH);
208
209 if (unique_names.end() != unique_names.find(column.name))
210 throw Exception("Column " + column.name + " queried more than once", ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE);
211 unique_names.insert(column.name);
212 }
213}
214
215void IStorage::check(const NamesAndTypesList & provided_columns, const Names & column_names) const
216{
217 const NamesAndTypesList & available_columns = getColumns().getAllPhysical();
218 const auto available_columns_map = getColumnsMap(available_columns);
219 const auto & provided_columns_map = getColumnsMap(provided_columns);
220
221 if (column_names.empty())
222 throw Exception(
223 "Empty list of columns queried. There are columns: " + listOfColumns(available_columns),
224 ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED);
225
226 auto unique_names = initUniqueStrings();
227 for (const String & name : column_names)
228 {
229 auto it = provided_columns_map.find(name);
230 if (provided_columns_map.end() == it)
231 continue;
232
233 auto jt = available_columns_map.find(name);
234 if (available_columns_map.end() == jt)
235 throw Exception(
236 "There is no column with name " + name + ". There are columns: " + listOfColumns(available_columns),
237 ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
238
239 if (!it->second->equals(*jt->second))
240 throw Exception(
241 "Type mismatch for column " + name + ". Column has type " + jt->second->getName() + ", got type " + it->second->getName(),
242 ErrorCodes::TYPE_MISMATCH);
243
244 if (unique_names.end() != unique_names.find(name))
245 throw Exception("Column " + name + " queried more than once", ErrorCodes::COLUMN_QUERIED_MORE_THAN_ONCE);
246 unique_names.insert(name);
247 }
248}
249
250void IStorage::check(const Block & block, bool need_all) const
251{
252 const NamesAndTypesList & available_columns = getColumns().getAllPhysical();
253 const auto columns_map = getColumnsMap(available_columns);
254
255 NameSet names_in_block;
256
257 block.checkNumberOfRows();
258
259 for (const auto & column : block)
260 {
261 if (names_in_block.count(column.name))
262 throw Exception("Duplicate column " + column.name + " in block", ErrorCodes::DUPLICATE_COLUMN);
263
264 names_in_block.insert(column.name);
265
266 auto it = columns_map.find(column.name);
267 if (columns_map.end() == it)
268 throw Exception(
269 "There is no column with name " + column.name + ". There are columns: " + listOfColumns(available_columns),
270 ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
271
272 if (!column.type->equals(*it->second))
273 throw Exception(
274 "Type mismatch for column " + column.name + ". Column has type " + it->second->getName() + ", got type "
275 + column.type->getName(),
276 ErrorCodes::TYPE_MISMATCH);
277 }
278
279 if (need_all && names_in_block.size() < columns_map.size())
280 {
281 for (auto it = available_columns.begin(); it != available_columns.end(); ++it)
282 {
283 if (!names_in_block.count(it->name))
284 throw Exception("Expected column " + it->name, ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK);
285 }
286 }
287}
288
289void IStorage::setColumns(ColumnsDescription columns_)
290{
291 if (columns_.getOrdinary().empty())
292 throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
293 columns = std::move(columns_);
294
295 for (const auto & column : virtuals)
296 {
297 if (!columns.has(column.name))
298 columns.add(column);
299 }
300}
301
302void IStorage::setIndices(IndicesDescription indices_)
303{
304 indices = std::move(indices_);
305}
306
307void IStorage::setConstraints(ConstraintsDescription constraints_)
308{
309 constraints = std::move(constraints_);
310}
311
312bool IStorage::isVirtualColumn(const String & column_name) const
313{
314 return getColumns().get(column_name).is_virtual;
315}
316
317TableStructureReadLockHolder IStorage::lockStructureForShare(bool will_add_new_data, const String & query_id)
318{
319 TableStructureReadLockHolder result;
320 if (will_add_new_data)
321 result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Read, query_id);
322 result.structure_lock = structure_lock->getLock(RWLockImpl::Read, query_id);
323
324 if (is_dropped)
325 throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
326 return result;
327}
328
329TableStructureWriteLockHolder IStorage::lockAlterIntention(const String & query_id)
330{
331 TableStructureWriteLockHolder result;
332 result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id);
333
334 if (is_dropped)
335 throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
336 return result;
337}
338
339void IStorage::lockNewDataStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id)
340{
341 if (!lock_holder.alter_intention_lock)
342 throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug.", ErrorCodes::LOGICAL_ERROR);
343
344 lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id);
345}
346
347void IStorage::lockStructureExclusively(TableStructureWriteLockHolder & lock_holder, const String & query_id)
348{
349 if (!lock_holder.alter_intention_lock)
350 throw Exception("Alter intention lock for table " + getTableName() + " was not taken. This is a bug.", ErrorCodes::LOGICAL_ERROR);
351
352 if (!lock_holder.new_data_structure_lock)
353 lock_holder.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id);
354 lock_holder.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id);
355}
356
357TableStructureWriteLockHolder IStorage::lockExclusively(const String & query_id)
358{
359 TableStructureWriteLockHolder result;
360 result.alter_intention_lock = alter_intention_lock->getLock(RWLockImpl::Write, query_id);
361
362 if (is_dropped)
363 throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
364
365 result.new_data_structure_lock = new_data_structure_lock->getLock(RWLockImpl::Write, query_id);
366 result.structure_lock = structure_lock->getLock(RWLockImpl::Write, query_id);
367
368 return result;
369}
370
371StorageInMemoryMetadata IStorage::getInMemoryMetadata() const
372{
373 return
374 {
375 .columns = getColumns(),
376 .indices = getIndices(),
377 .constraints = getConstraints(),
378 };
379}
380
381void IStorage::alter(
382 const AlterCommands & params,
383 const Context & context,
384 TableStructureWriteLockHolder & table_lock_holder)
385{
386 const String database_name = getDatabaseName();
387 const String table_name = getTableName();
388
389 lockStructureExclusively(table_lock_holder, context.getCurrentQueryId());
390
391 StorageInMemoryMetadata metadata = getInMemoryMetadata();
392 params.apply(metadata);
393 context.getDatabase(database_name)->alterTable(context, table_name, metadata);
394 setColumns(std::move(metadata.columns));
395}
396
397
398void IStorage::checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */)
399{
400 for (const auto & command : commands)
401 {
402 if (!command.isCommentAlter())
403 throw Exception(
404 "Alter of type '" + alterTypeToString(command.type) + "' is not supported by storage " + getName(),
405 ErrorCodes::NOT_IMPLEMENTED);
406 }
407}
408
409BlockInputStreams IStorage::read(
410 const Names & column_names,
411 const SelectQueryInfo & query_info,
412 const Context & context,
413 QueryProcessingStage::Enum processed_stage,
414 size_t max_block_size,
415 unsigned num_streams)
416{
417 auto pipes = readWithProcessors(column_names, query_info, context, processed_stage, max_block_size, num_streams);
418
419 BlockInputStreams res;
420 res.reserve(pipes.size());
421
422 for (auto & pipe : pipes)
423 res.emplace_back(std::make_shared<TreeExecutorBlockInputStream>(std::move(pipe)));
424
425 return res;
426}
427
428DB::CompressionMethod IStorage::chooseCompressionMethod(const String & uri, const String & compression_method)
429{
430 if (compression_method == "auto" || compression_method == "")
431 {
432 if (endsWith(uri, ".gz"))
433 return DB::CompressionMethod::Gzip;
434 else
435 return DB::CompressionMethod::None;
436 }
437 else if (compression_method == "gzip")
438 return DB::CompressionMethod::Gzip;
439 else if (compression_method == "none")
440 return DB::CompressionMethod::None;
441 else
442 throw Exception("Only auto, none, gzip supported as compression method", ErrorCodes::NOT_IMPLEMENTED);
443}
444
445}
446