1 | #include <Dictionaries/getDictionaryConfigurationFromAST.h> |
2 | |
3 | #include <Poco/DOM/AutoPtr.h> |
4 | #include <Poco/DOM/Document.h> |
5 | #include <Poco/DOM/Element.h> |
6 | #include <Poco/DOM/Text.h> |
7 | #include <Poco/Util/AbstractConfiguration.h> |
8 | #include <Poco/Util/XMLConfiguration.h> |
9 | #include <IO/WriteHelpers.h> |
10 | #include <Parsers/queryToString.h> |
11 | #include <Parsers/ASTIdentifier.h> |
12 | #include <Parsers/ASTFunction.h> |
13 | #include <Core/Names.h> |
14 | #include <Parsers/ASTFunctionWithKeyValueArguments.h> |
15 | #include <Parsers/ASTDictionaryAttributeDeclaration.h> |
16 | #include <Dictionaries/DictionaryFactory.h> |
17 | |
18 | namespace DB |
19 | { |
20 | |
21 | namespace ErrorCodes |
22 | { |
23 | extern const int INCORRECT_DICTIONARY_DEFINITION; |
24 | } |
25 | |
26 | /// There are a lot of code, but it's very simple and straightforward |
27 | /// We just convert |
28 | namespace |
29 | { |
30 | |
31 | using NamesToTypeNames = std::unordered_map<std::string, std::string>; |
32 | /// Get value from field and convert it to string. |
33 | /// Also remove quotes from strings. |
34 | String getUnescapedFieldString(const Field & field) |
35 | { |
36 | String string = applyVisitor(FieldVisitorToString(), field); |
37 | if (!string.empty() && string.front() == '\'' && string.back() == '\'') |
38 | return string.substr(1, string.size() - 2); |
39 | return string; |
40 | } |
41 | |
42 | |
43 | using namespace Poco; |
44 | using namespace Poco::XML; |
45 | /* |
46 | * Transforms next definition |
47 | * LIFETIME(MIN 10, MAX 100) |
48 | * to the next configuration |
49 | * <lifetime> |
50 | * <min>10</min> |
51 | * <max>100</max> |
52 | * </lifetime> |
53 | */ |
54 | void buildLifetimeConfiguration( |
55 | AutoPtr<Document> doc, |
56 | AutoPtr<Element> root, |
57 | const ASTDictionaryLifetime * lifetime) |
58 | { |
59 | |
60 | AutoPtr<Element> lifetime_element(doc->createElement("lifetime" )); |
61 | AutoPtr<Element> min_element(doc->createElement("min" )); |
62 | AutoPtr<Element> max_element(doc->createElement("max" )); |
63 | AutoPtr<Text> min_sec(doc->createTextNode(toString(lifetime->min_sec))); |
64 | min_element->appendChild(min_sec); |
65 | AutoPtr<Text> max_sec(doc->createTextNode(toString(lifetime->max_sec))); |
66 | max_element->appendChild(max_sec); |
67 | lifetime_element->appendChild(min_element); |
68 | lifetime_element->appendChild(max_element); |
69 | root->appendChild(lifetime_element); |
70 | } |
71 | |
72 | /* |
73 | * Transforms next definition |
74 | * LAYOUT(FLAT()) |
75 | * to the next configuration |
76 | * <layout> |
77 | * <flat/> |
78 | * </layout> |
79 | * |
80 | * And next definition |
81 | * LAYOUT(CACHE(SIZE_IN_CELLS 1000)) |
82 | * to the next one |
83 | * <layout> |
84 | * <cache> |
85 | * <size_in_cells>1000</size_in_cells> |
86 | * </cache> |
87 | * </layout> |
88 | */ |
89 | void buildLayoutConfiguration( |
90 | AutoPtr<Document> doc, |
91 | AutoPtr<Element> root, |
92 | const ASTDictionaryLayout * layout) |
93 | { |
94 | AutoPtr<Element> layout_element(doc->createElement("layout" )); |
95 | root->appendChild(layout_element); |
96 | AutoPtr<Element> layout_type_element(doc->createElement(layout->layout_type)); |
97 | layout_element->appendChild(layout_type_element); |
98 | if (layout->parameter.has_value()) |
99 | { |
100 | const auto & param = layout->parameter; |
101 | AutoPtr<Element> layout_type_parameter_element(doc->createElement(param->first)); |
102 | const ASTLiteral & literal = param->second->as<const ASTLiteral &>(); |
103 | AutoPtr<Text> value(doc->createTextNode(toString(literal.value.get<UInt64>()))); |
104 | layout_type_parameter_element->appendChild(value); |
105 | layout_type_element->appendChild(layout_type_parameter_element); |
106 | } |
107 | } |
108 | |
109 | /* |
110 | * Transforms next definition |
111 | * RANGE(MIN StartDate, MAX EndDate) |
112 | * to the next configuration |
113 | * <range_min><name>StartDate</name></range_min> |
114 | * <range_max><name>EndDate</name></range_max> |
115 | */ |
116 | void buildRangeConfiguration(AutoPtr<Document> doc, AutoPtr<Element> root, const ASTDictionaryRange * range, const NamesToTypeNames & all_attrs) |
117 | { |
118 | // appends <key><name>value</name></key> to root |
119 | auto appendElem = [&doc, &root](const std::string & key, const std::string & name, const std::string & type) |
120 | { |
121 | AutoPtr<Element> element(doc->createElement(key)); |
122 | AutoPtr<Element> name_node(doc->createElement("name" )); |
123 | AutoPtr<Text> name_text(doc->createTextNode(name)); |
124 | name_node->appendChild(name_text); |
125 | element->appendChild(name_node); |
126 | |
127 | AutoPtr<Element> type_node(doc->createElement("type" )); |
128 | AutoPtr<Text> type_text(doc->createTextNode(type)); |
129 | type_node->appendChild(type_text); |
130 | element->appendChild(type_node); |
131 | |
132 | root->appendChild(element); |
133 | }; |
134 | |
135 | appendElem("range_min" , range->min_attr_name, all_attrs.at(range->min_attr_name)); |
136 | appendElem("range_max" , range->max_attr_name, all_attrs.at(range->max_attr_name)); |
137 | } |
138 | |
139 | |
140 | /// Get primary key columns names from AST |
141 | Names getPrimaryKeyColumns(const ASTExpressionList * primary_key) |
142 | { |
143 | Names result; |
144 | const auto & children = primary_key->children; |
145 | |
146 | for (size_t index = 0; index != children.size(); ++index) |
147 | { |
148 | const ASTIdentifier * key_part = children[index]->as<const ASTIdentifier>(); |
149 | result.push_back(key_part->name); |
150 | } |
151 | return result; |
152 | } |
153 | |
154 | /** |
155 | * Transofrms single dictionary attribute to configuration |
156 | * third_column UInt8 DEFAULT 2 EXPRESSION rand() % 100 * 77 |
157 | * to |
158 | * <attribute> |
159 | * <name>third_column</name> |
160 | * <type>UInt8</type> |
161 | * <null_value>2</null_value> |
162 | * <expression>(rand() % 100) * 77</expression> |
163 | * </attribute> |
164 | */ |
165 | void buildSingleAttribute( |
166 | AutoPtr<Document> doc, |
167 | AutoPtr<Element> root, |
168 | const ASTDictionaryAttributeDeclaration * dict_attr) |
169 | { |
170 | AutoPtr<Element> attribute_element(doc->createElement("attribute" )); |
171 | root->appendChild(attribute_element); |
172 | |
173 | AutoPtr<Element> name_element(doc->createElement("name" )); |
174 | AutoPtr<Text> name(doc->createTextNode(dict_attr->name)); |
175 | name_element->appendChild(name); |
176 | attribute_element->appendChild(name_element); |
177 | |
178 | AutoPtr<Element> type_element(doc->createElement("type" )); |
179 | AutoPtr<Text> type(doc->createTextNode(queryToString(dict_attr->type))); |
180 | type_element->appendChild(type); |
181 | attribute_element->appendChild(type_element); |
182 | |
183 | AutoPtr<Element> null_value_element(doc->createElement("null_value" )); |
184 | String null_value_str; |
185 | if (dict_attr->default_value) |
186 | null_value_str = getUnescapedFieldString(dict_attr->default_value->as<ASTLiteral>()->value); |
187 | AutoPtr<Text> null_value(doc->createTextNode(null_value_str)); |
188 | null_value_element->appendChild(null_value); |
189 | attribute_element->appendChild(null_value_element); |
190 | |
191 | if (dict_attr->expression != nullptr) |
192 | { |
193 | AutoPtr<Element> expression_element(doc->createElement("expression" )); |
194 | |
195 | /// EXPRESSION PROPERTY should be expression or string |
196 | String expression_str; |
197 | if (const auto * literal = dict_attr->expression->as<ASTLiteral>(); |
198 | literal && literal->value.getType() == Field::Types::String) |
199 | { |
200 | expression_str = getUnescapedFieldString(literal->value); |
201 | } |
202 | else |
203 | expression_str = queryToString(dict_attr->expression); |
204 | |
205 | |
206 | AutoPtr<Text> expression(doc->createTextNode(expression_str)); |
207 | expression_element->appendChild(expression); |
208 | attribute_element->appendChild(expression_element); |
209 | } |
210 | |
211 | if (dict_attr->hierarchical) |
212 | { |
213 | AutoPtr<Element> hierarchical_element(doc->createElement("hierarchical" )); |
214 | AutoPtr<Text> hierarchical(doc->createTextNode("true" )); |
215 | hierarchical_element->appendChild(hierarchical); |
216 | attribute_element->appendChild(hierarchical_element); |
217 | } |
218 | |
219 | if (dict_attr->injective) |
220 | { |
221 | AutoPtr<Element> injective_element(doc->createElement("injective" )); |
222 | AutoPtr<Text> injective(doc->createTextNode("true" )); |
223 | injective_element->appendChild(injective); |
224 | attribute_element->appendChild(injective_element); |
225 | } |
226 | |
227 | if (dict_attr->is_object_id) |
228 | { |
229 | AutoPtr<Element> is_object_id_element(doc->createElement("is_object_id" )); |
230 | AutoPtr<Text> is_object_id(doc->createTextNode("true" )); |
231 | is_object_id_element->appendChild(is_object_id); |
232 | attribute_element->appendChild(is_object_id_element); |
233 | } |
234 | } |
235 | |
236 | |
237 | /** |
238 | * Transforms |
239 | * PRIMARY KEY Attr1 ,..., AttrN |
240 | * to the next configuration |
241 | * <id><name>Attr1</name></id> |
242 | * or |
243 | * <key> |
244 | * <attribute> |
245 | * <name>Attr1</name> |
246 | * <type>UInt8</type> |
247 | * </attribute> |
248 | * ... |
249 | * <attribute> fe |
250 | * </key> |
251 | * |
252 | */ |
253 | void buildPrimaryKeyConfiguration( |
254 | AutoPtr<Document> doc, |
255 | AutoPtr<Element> root, |
256 | bool complex, |
257 | const Names & key_names, |
258 | const ASTExpressionList * dictionary_attributes) |
259 | { |
260 | if (!complex) |
261 | { |
262 | if (key_names.size() != 1) |
263 | throw Exception("Primary key for simple dictionary must contain exactly one element" , |
264 | ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
265 | |
266 | AutoPtr<Element> id_element(doc->createElement("id" )); |
267 | root->appendChild(id_element); |
268 | AutoPtr<Element> name_element(doc->createElement("name" )); |
269 | id_element->appendChild(name_element); |
270 | AutoPtr<Text> name(doc->createTextNode(*key_names.begin())); |
271 | name_element->appendChild(name); |
272 | } |
273 | else |
274 | { |
275 | const auto & children = dictionary_attributes->children; |
276 | if (children.size() < key_names.size()) |
277 | throw Exception( |
278 | "Primary key fields count is more, than dictionary attributes count." , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
279 | |
280 | AutoPtr<Element> key_element(doc->createElement("key" )); |
281 | root->appendChild(key_element); |
282 | for (const auto & key_name : key_names) |
283 | { |
284 | bool found = false; |
285 | for (const auto & attr : children) |
286 | { |
287 | const ASTDictionaryAttributeDeclaration * dict_attr = attr->as<const ASTDictionaryAttributeDeclaration>(); |
288 | if (dict_attr->name == key_name) |
289 | { |
290 | found = true; |
291 | buildSingleAttribute(doc, key_element, dict_attr); |
292 | break; |
293 | } |
294 | } |
295 | if (!found) |
296 | throw Exception( |
297 | "Primary key field '" + key_name + "' not found among attributes." , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
298 | } |
299 | } |
300 | } |
301 | |
302 | |
303 | /** |
304 | * Transforms list of ASTDictionaryAttributeDeclarations to list of dictionary attributes |
305 | */ |
306 | NamesToTypeNames buildDictionaryAttributesConfiguration( |
307 | AutoPtr<Document> doc, |
308 | AutoPtr<Element> root, |
309 | const ASTExpressionList * dictionary_attributes, |
310 | const Names & key_columns) |
311 | { |
312 | const auto & children = dictionary_attributes->children; |
313 | NamesToTypeNames attributes_names_and_types; |
314 | for (size_t i = 0; i < children.size(); ++i) |
315 | { |
316 | const ASTDictionaryAttributeDeclaration * dict_attr = children[i]->as<const ASTDictionaryAttributeDeclaration>(); |
317 | if (!dict_attr->type) |
318 | throw Exception("Dictionary attribute must has type" , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
319 | |
320 | attributes_names_and_types.emplace(dict_attr->name, queryToString(dict_attr->type)); |
321 | if (std::find(key_columns.begin(), key_columns.end(), dict_attr->name) == key_columns.end()) |
322 | buildSingleAttribute(doc, root, dict_attr); |
323 | } |
324 | return attributes_names_and_types; |
325 | } |
326 | |
327 | /** Transform function with key-value arguments to configuration |
328 | * (used for source transformation) |
329 | */ |
330 | void buildConfigurationFromFunctionWithKeyValueArguments( |
331 | AutoPtr<Document> doc, |
332 | AutoPtr<Element> root, |
333 | const ASTExpressionList * ast_expr_list) |
334 | { |
335 | const auto & children = ast_expr_list->children; |
336 | for (size_t i = 0; i != children.size(); ++i) |
337 | { |
338 | const ASTPair * pair = children[i]->as<const ASTPair>(); |
339 | AutoPtr<Element> current_xml_element(doc->createElement(pair->first)); |
340 | root->appendChild(current_xml_element); |
341 | |
342 | if (auto identifier = pair->second->as<const ASTIdentifier>(); identifier) |
343 | { |
344 | AutoPtr<Text> value(doc->createTextNode(identifier->name)); |
345 | current_xml_element->appendChild(value); |
346 | } |
347 | else if (auto literal = pair->second->as<const ASTLiteral>(); literal) |
348 | { |
349 | AutoPtr<Text> value(doc->createTextNode(getUnescapedFieldString(literal->value))); |
350 | current_xml_element->appendChild(value); |
351 | } |
352 | else if (auto list = pair->second->as<const ASTExpressionList>(); list) |
353 | { |
354 | buildConfigurationFromFunctionWithKeyValueArguments(doc, current_xml_element, list); |
355 | } |
356 | else |
357 | { |
358 | throw Exception( |
359 | "Incorrect ASTPair contains wrong value, should be literal, identifier or list" , |
360 | ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
361 | } |
362 | } |
363 | } |
364 | |
365 | /** Build source definition from ast. |
366 | * SOURCE(MYSQL(HOST 'localhost' PORT 9000 USER 'default' REPLICA(HOST '127.0.0.1' PRIORITY 1) PASSWORD '')) |
367 | * to |
368 | * <source> |
369 | * <mysql> |
370 | * <host>localhost</host> |
371 | * ... |
372 | * <replica> |
373 | * <host>127.0.0.1</host> |
374 | * ... |
375 | * </replica> |
376 | * </mysql> |
377 | * </source> |
378 | */ |
379 | void buildSourceConfiguration(AutoPtr<Document> doc, AutoPtr<Element> root, const ASTFunctionWithKeyValueArguments * source) |
380 | { |
381 | AutoPtr<Element> outer_element(doc->createElement("source" )); |
382 | root->appendChild(outer_element); |
383 | AutoPtr<Element> source_element(doc->createElement(source->name)); |
384 | outer_element->appendChild(source_element); |
385 | buildConfigurationFromFunctionWithKeyValueArguments(doc, source_element, source->elements->as<const ASTExpressionList>()); |
386 | } |
387 | |
388 | /** Check all AST fields are filled, throws exception |
389 | * in other case |
390 | */ |
391 | void checkAST(const ASTCreateQuery & query) |
392 | { |
393 | if (!query.is_dictionary || query.dictionary == nullptr) |
394 | throw Exception("Cannot convert dictionary to configuration from non-dictionary AST." , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
395 | |
396 | if (query.dictionary_attributes_list == nullptr || query.dictionary_attributes_list->children.empty()) |
397 | throw Exception("Cannot create dictionary with empty attributes list" , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
398 | |
399 | if (query.dictionary->layout == nullptr) |
400 | throw Exception("Cannot create dictionary with empty layout" , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
401 | |
402 | if (query.dictionary->lifetime == nullptr) |
403 | throw Exception("Cannot create dictionary with empty lifetime" , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
404 | |
405 | if (query.dictionary->primary_key == nullptr) |
406 | throw Exception("Cannot create dictionary without primary key" , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
407 | |
408 | if (query.dictionary->source == nullptr) |
409 | throw Exception("Cannot create dictionary with empty source" , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
410 | |
411 | /// Range can be empty |
412 | } |
413 | |
414 | void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs) |
415 | { |
416 | for (const auto & key_attr : key_attrs) |
417 | if (all_attrs.count(key_attr) == 0) |
418 | throw Exception("Unknown key attribute '" + key_attr + "'" , ErrorCodes::INCORRECT_DICTIONARY_DEFINITION); |
419 | } |
420 | |
421 | } |
422 | |
423 | |
424 | DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query) |
425 | { |
426 | checkAST(query); |
427 | |
428 | AutoPtr<Poco::XML::Document> xml_document(new Poco::XML::Document()); |
429 | AutoPtr<Poco::XML::Element> document_root(xml_document->createElement("dictionaries" )); |
430 | xml_document->appendChild(document_root); |
431 | AutoPtr<Poco::XML::Element> current_dictionary(xml_document->createElement("dictionary" )); |
432 | document_root->appendChild(current_dictionary); |
433 | AutoPtr<Poco::Util::XMLConfiguration> conf(new Poco::Util::XMLConfiguration()); |
434 | |
435 | AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name" )); |
436 | current_dictionary->appendChild(name_element); |
437 | AutoPtr<Text> name(xml_document->createTextNode(query.table)); |
438 | name_element->appendChild(name); |
439 | |
440 | AutoPtr<Poco::XML::Element> database_element(xml_document->createElement("database" )); |
441 | current_dictionary->appendChild(database_element); |
442 | AutoPtr<Text> database(xml_document->createTextNode(query.database)); |
443 | database_element->appendChild(database); |
444 | |
445 | AutoPtr<Element> structure_element(xml_document->createElement("structure" )); |
446 | current_dictionary->appendChild(structure_element); |
447 | Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key); |
448 | auto dictionary_layout = query.dictionary->layout; |
449 | |
450 | bool complex = DictionaryFactory::instance().isComplex(dictionary_layout->layout_type); |
451 | |
452 | auto all_attr_names_and_types = buildDictionaryAttributesConfiguration(xml_document, structure_element, query.dictionary_attributes_list, pk_attrs); |
453 | checkPrimaryKey(all_attr_names_and_types, pk_attrs); |
454 | |
455 | buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list); |
456 | |
457 | buildLayoutConfiguration(xml_document, current_dictionary, dictionary_layout); |
458 | buildSourceConfiguration(xml_document, current_dictionary, query.dictionary->source); |
459 | buildLifetimeConfiguration(xml_document, current_dictionary, query.dictionary->lifetime); |
460 | |
461 | if (query.dictionary->range) |
462 | buildRangeConfiguration(xml_document, structure_element, query.dictionary->range, all_attr_names_and_types); |
463 | |
464 | conf->load(xml_document); |
465 | return conf; |
466 | } |
467 | |
468 | } |
469 | |