1#include <Dictionaries/getDictionaryConfigurationFromAST.h>
2
3#include <Poco/DOM/AutoPtr.h>
4#include <Poco/DOM/Document.h>
5#include <Poco/DOM/Element.h>
6#include <Poco/DOM/Text.h>
7#include <Poco/Util/AbstractConfiguration.h>
8#include <Poco/Util/XMLConfiguration.h>
9#include <IO/WriteHelpers.h>
10#include <Parsers/queryToString.h>
11#include <Parsers/ASTIdentifier.h>
12#include <Parsers/ASTFunction.h>
13#include <Core/Names.h>
14#include <Parsers/ASTFunctionWithKeyValueArguments.h>
15#include <Parsers/ASTDictionaryAttributeDeclaration.h>
16#include <Dictionaries/DictionaryFactory.h>
17
18namespace DB
19{
20
21namespace ErrorCodes
22{
23 extern const int INCORRECT_DICTIONARY_DEFINITION;
24}
25
26/// There are a lot of code, but it's very simple and straightforward
27/// We just convert
28namespace
29{
30
31using NamesToTypeNames = std::unordered_map<std::string, std::string>;
32/// Get value from field and convert it to string.
33/// Also remove quotes from strings.
34String getUnescapedFieldString(const Field & field)
35{
36 String string = applyVisitor(FieldVisitorToString(), field);
37 if (!string.empty() && string.front() == '\'' && string.back() == '\'')
38 return string.substr(1, string.size() - 2);
39 return string;
40}
41
42
43using namespace Poco;
44using namespace Poco::XML;
45/*
46 * Transforms next definition
47 * LIFETIME(MIN 10, MAX 100)
48 * to the next configuration
49 * <lifetime>
50 * <min>10</min>
51 * <max>100</max>
52 * </lifetime>
53 */
54void buildLifetimeConfiguration(
55 AutoPtr<Document> doc,
56 AutoPtr<Element> root,
57 const ASTDictionaryLifetime * lifetime)
58{
59
60 AutoPtr<Element> lifetime_element(doc->createElement("lifetime"));
61 AutoPtr<Element> min_element(doc->createElement("min"));
62 AutoPtr<Element> max_element(doc->createElement("max"));
63 AutoPtr<Text> min_sec(doc->createTextNode(toString(lifetime->min_sec)));
64 min_element->appendChild(min_sec);
65 AutoPtr<Text> max_sec(doc->createTextNode(toString(lifetime->max_sec)));
66 max_element->appendChild(max_sec);
67 lifetime_element->appendChild(min_element);
68 lifetime_element->appendChild(max_element);
69 root->appendChild(lifetime_element);
70}
71
72/*
73 * Transforms next definition
74 * LAYOUT(FLAT())
75 * to the next configuration
76 * <layout>
77 * <flat/>
78 * </layout>
79 *
80 * And next definition
81 * LAYOUT(CACHE(SIZE_IN_CELLS 1000))
82 * to the next one
83 * <layout>
84 * <cache>
85 * <size_in_cells>1000</size_in_cells>
86 * </cache>
87 * </layout>
88 */
89void buildLayoutConfiguration(
90 AutoPtr<Document> doc,
91 AutoPtr<Element> root,
92 const ASTDictionaryLayout * layout)
93{
94 AutoPtr<Element> layout_element(doc->createElement("layout"));
95 root->appendChild(layout_element);
96 AutoPtr<Element> layout_type_element(doc->createElement(layout->layout_type));
97 layout_element->appendChild(layout_type_element);
98 if (layout->parameter.has_value())
99 {
100 const auto & param = layout->parameter;
101 AutoPtr<Element> layout_type_parameter_element(doc->createElement(param->first));
102 const ASTLiteral & literal = param->second->as<const ASTLiteral &>();
103 AutoPtr<Text> value(doc->createTextNode(toString(literal.value.get<UInt64>())));
104 layout_type_parameter_element->appendChild(value);
105 layout_type_element->appendChild(layout_type_parameter_element);
106 }
107}
108
109/*
110 * Transforms next definition
111 * RANGE(MIN StartDate, MAX EndDate)
112 * to the next configuration
113 * <range_min><name>StartDate</name></range_min>
114 * <range_max><name>EndDate</name></range_max>
115 */
116void buildRangeConfiguration(AutoPtr<Document> doc, AutoPtr<Element> root, const ASTDictionaryRange * range, const NamesToTypeNames & all_attrs)
117{
118 // appends <key><name>value</name></key> to root
119 auto appendElem = [&doc, &root](const std::string & key, const std::string & name, const std::string & type)
120 {
121 AutoPtr<Element> element(doc->createElement(key));
122 AutoPtr<Element> name_node(doc->createElement("name"));
123 AutoPtr<Text> name_text(doc->createTextNode(name));
124 name_node->appendChild(name_text);
125 element->appendChild(name_node);
126
127 AutoPtr<Element> type_node(doc->createElement("type"));
128 AutoPtr<Text> type_text(doc->createTextNode(type));
129 type_node->appendChild(type_text);
130 element->appendChild(type_node);
131
132 root->appendChild(element);
133 };
134
135 appendElem("range_min", range->min_attr_name, all_attrs.at(range->min_attr_name));
136 appendElem("range_max", range->max_attr_name, all_attrs.at(range->max_attr_name));
137}
138
139
140/// Get primary key columns names from AST
141Names getPrimaryKeyColumns(const ASTExpressionList * primary_key)
142{
143 Names result;
144 const auto & children = primary_key->children;
145
146 for (size_t index = 0; index != children.size(); ++index)
147 {
148 const ASTIdentifier * key_part = children[index]->as<const ASTIdentifier>();
149 result.push_back(key_part->name);
150 }
151 return result;
152}
153
154/**
155 * Transofrms single dictionary attribute to configuration
156 * third_column UInt8 DEFAULT 2 EXPRESSION rand() % 100 * 77
157 * to
158 * <attribute>
159 * <name>third_column</name>
160 * <type>UInt8</type>
161 * <null_value>2</null_value>
162 * <expression>(rand() % 100) * 77</expression>
163 * </attribute>
164 */
165void buildSingleAttribute(
166 AutoPtr<Document> doc,
167 AutoPtr<Element> root,
168 const ASTDictionaryAttributeDeclaration * dict_attr)
169{
170 AutoPtr<Element> attribute_element(doc->createElement("attribute"));
171 root->appendChild(attribute_element);
172
173 AutoPtr<Element> name_element(doc->createElement("name"));
174 AutoPtr<Text> name(doc->createTextNode(dict_attr->name));
175 name_element->appendChild(name);
176 attribute_element->appendChild(name_element);
177
178 AutoPtr<Element> type_element(doc->createElement("type"));
179 AutoPtr<Text> type(doc->createTextNode(queryToString(dict_attr->type)));
180 type_element->appendChild(type);
181 attribute_element->appendChild(type_element);
182
183 AutoPtr<Element> null_value_element(doc->createElement("null_value"));
184 String null_value_str;
185 if (dict_attr->default_value)
186 null_value_str = getUnescapedFieldString(dict_attr->default_value->as<ASTLiteral>()->value);
187 AutoPtr<Text> null_value(doc->createTextNode(null_value_str));
188 null_value_element->appendChild(null_value);
189 attribute_element->appendChild(null_value_element);
190
191 if (dict_attr->expression != nullptr)
192 {
193 AutoPtr<Element> expression_element(doc->createElement("expression"));
194
195 /// EXPRESSION PROPERTY should be expression or string
196 String expression_str;
197 if (const auto * literal = dict_attr->expression->as<ASTLiteral>();
198 literal && literal->value.getType() == Field::Types::String)
199 {
200 expression_str = getUnescapedFieldString(literal->value);
201 }
202 else
203 expression_str = queryToString(dict_attr->expression);
204
205
206 AutoPtr<Text> expression(doc->createTextNode(expression_str));
207 expression_element->appendChild(expression);
208 attribute_element->appendChild(expression_element);
209 }
210
211 if (dict_attr->hierarchical)
212 {
213 AutoPtr<Element> hierarchical_element(doc->createElement("hierarchical"));
214 AutoPtr<Text> hierarchical(doc->createTextNode("true"));
215 hierarchical_element->appendChild(hierarchical);
216 attribute_element->appendChild(hierarchical_element);
217 }
218
219 if (dict_attr->injective)
220 {
221 AutoPtr<Element> injective_element(doc->createElement("injective"));
222 AutoPtr<Text> injective(doc->createTextNode("true"));
223 injective_element->appendChild(injective);
224 attribute_element->appendChild(injective_element);
225 }
226
227 if (dict_attr->is_object_id)
228 {
229 AutoPtr<Element> is_object_id_element(doc->createElement("is_object_id"));
230 AutoPtr<Text> is_object_id(doc->createTextNode("true"));
231 is_object_id_element->appendChild(is_object_id);
232 attribute_element->appendChild(is_object_id_element);
233 }
234}
235
236
237/**
238 * Transforms
239 * PRIMARY KEY Attr1 ,..., AttrN
240 * to the next configuration
241 * <id><name>Attr1</name></id>
242 * or
243 * <key>
244 * <attribute>
245 * <name>Attr1</name>
246 * <type>UInt8</type>
247 * </attribute>
248 * ...
249 * <attribute> fe
250 * </key>
251 *
252 */
253void buildPrimaryKeyConfiguration(
254 AutoPtr<Document> doc,
255 AutoPtr<Element> root,
256 bool complex,
257 const Names & key_names,
258 const ASTExpressionList * dictionary_attributes)
259{
260 if (!complex)
261 {
262 if (key_names.size() != 1)
263 throw Exception("Primary key for simple dictionary must contain exactly one element",
264 ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
265
266 AutoPtr<Element> id_element(doc->createElement("id"));
267 root->appendChild(id_element);
268 AutoPtr<Element> name_element(doc->createElement("name"));
269 id_element->appendChild(name_element);
270 AutoPtr<Text> name(doc->createTextNode(*key_names.begin()));
271 name_element->appendChild(name);
272 }
273 else
274 {
275 const auto & children = dictionary_attributes->children;
276 if (children.size() < key_names.size())
277 throw Exception(
278 "Primary key fields count is more, than dictionary attributes count.", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
279
280 AutoPtr<Element> key_element(doc->createElement("key"));
281 root->appendChild(key_element);
282 for (const auto & key_name : key_names)
283 {
284 bool found = false;
285 for (const auto & attr : children)
286 {
287 const ASTDictionaryAttributeDeclaration * dict_attr = attr->as<const ASTDictionaryAttributeDeclaration>();
288 if (dict_attr->name == key_name)
289 {
290 found = true;
291 buildSingleAttribute(doc, key_element, dict_attr);
292 break;
293 }
294 }
295 if (!found)
296 throw Exception(
297 "Primary key field '" + key_name + "' not found among attributes.", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
298 }
299 }
300}
301
302
303/**
304 * Transforms list of ASTDictionaryAttributeDeclarations to list of dictionary attributes
305 */
306NamesToTypeNames buildDictionaryAttributesConfiguration(
307 AutoPtr<Document> doc,
308 AutoPtr<Element> root,
309 const ASTExpressionList * dictionary_attributes,
310 const Names & key_columns)
311{
312 const auto & children = dictionary_attributes->children;
313 NamesToTypeNames attributes_names_and_types;
314 for (size_t i = 0; i < children.size(); ++i)
315 {
316 const ASTDictionaryAttributeDeclaration * dict_attr = children[i]->as<const ASTDictionaryAttributeDeclaration>();
317 if (!dict_attr->type)
318 throw Exception("Dictionary attribute must has type", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
319
320 attributes_names_and_types.emplace(dict_attr->name, queryToString(dict_attr->type));
321 if (std::find(key_columns.begin(), key_columns.end(), dict_attr->name) == key_columns.end())
322 buildSingleAttribute(doc, root, dict_attr);
323 }
324 return attributes_names_and_types;
325}
326
327/** Transform function with key-value arguments to configuration
328 * (used for source transformation)
329 */
330void buildConfigurationFromFunctionWithKeyValueArguments(
331 AutoPtr<Document> doc,
332 AutoPtr<Element> root,
333 const ASTExpressionList * ast_expr_list)
334{
335 const auto & children = ast_expr_list->children;
336 for (size_t i = 0; i != children.size(); ++i)
337 {
338 const ASTPair * pair = children[i]->as<const ASTPair>();
339 AutoPtr<Element> current_xml_element(doc->createElement(pair->first));
340 root->appendChild(current_xml_element);
341
342 if (auto identifier = pair->second->as<const ASTIdentifier>(); identifier)
343 {
344 AutoPtr<Text> value(doc->createTextNode(identifier->name));
345 current_xml_element->appendChild(value);
346 }
347 else if (auto literal = pair->second->as<const ASTLiteral>(); literal)
348 {
349 AutoPtr<Text> value(doc->createTextNode(getUnescapedFieldString(literal->value)));
350 current_xml_element->appendChild(value);
351 }
352 else if (auto list = pair->second->as<const ASTExpressionList>(); list)
353 {
354 buildConfigurationFromFunctionWithKeyValueArguments(doc, current_xml_element, list);
355 }
356 else
357 {
358 throw Exception(
359 "Incorrect ASTPair contains wrong value, should be literal, identifier or list",
360 ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
361 }
362 }
363}
364
365/** Build source definition from ast.
366 * SOURCE(MYSQL(HOST 'localhost' PORT 9000 USER 'default' REPLICA(HOST '127.0.0.1' PRIORITY 1) PASSWORD ''))
367 * to
368 * <source>
369 * <mysql>
370 * <host>localhost</host>
371 * ...
372 * <replica>
373 * <host>127.0.0.1</host>
374 * ...
375 * </replica>
376 * </mysql>
377 * </source>
378 */
379void buildSourceConfiguration(AutoPtr<Document> doc, AutoPtr<Element> root, const ASTFunctionWithKeyValueArguments * source)
380{
381 AutoPtr<Element> outer_element(doc->createElement("source"));
382 root->appendChild(outer_element);
383 AutoPtr<Element> source_element(doc->createElement(source->name));
384 outer_element->appendChild(source_element);
385 buildConfigurationFromFunctionWithKeyValueArguments(doc, source_element, source->elements->as<const ASTExpressionList>());
386}
387
388/** Check all AST fields are filled, throws exception
389 * in other case
390 */
391void checkAST(const ASTCreateQuery & query)
392{
393 if (!query.is_dictionary || query.dictionary == nullptr)
394 throw Exception("Cannot convert dictionary to configuration from non-dictionary AST.", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
395
396 if (query.dictionary_attributes_list == nullptr || query.dictionary_attributes_list->children.empty())
397 throw Exception("Cannot create dictionary with empty attributes list", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
398
399 if (query.dictionary->layout == nullptr)
400 throw Exception("Cannot create dictionary with empty layout", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
401
402 if (query.dictionary->lifetime == nullptr)
403 throw Exception("Cannot create dictionary with empty lifetime", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
404
405 if (query.dictionary->primary_key == nullptr)
406 throw Exception("Cannot create dictionary without primary key", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
407
408 if (query.dictionary->source == nullptr)
409 throw Exception("Cannot create dictionary with empty source", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
410
411 /// Range can be empty
412}
413
414void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs)
415{
416 for (const auto & key_attr : key_attrs)
417 if (all_attrs.count(key_attr) == 0)
418 throw Exception("Unknown key attribute '" + key_attr + "'", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
419}
420
421}
422
423
424DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query)
425{
426 checkAST(query);
427
428 AutoPtr<Poco::XML::Document> xml_document(new Poco::XML::Document());
429 AutoPtr<Poco::XML::Element> document_root(xml_document->createElement("dictionaries"));
430 xml_document->appendChild(document_root);
431 AutoPtr<Poco::XML::Element> current_dictionary(xml_document->createElement("dictionary"));
432 document_root->appendChild(current_dictionary);
433 AutoPtr<Poco::Util::XMLConfiguration> conf(new Poco::Util::XMLConfiguration());
434
435 AutoPtr<Poco::XML::Element> name_element(xml_document->createElement("name"));
436 current_dictionary->appendChild(name_element);
437 AutoPtr<Text> name(xml_document->createTextNode(query.table));
438 name_element->appendChild(name);
439
440 AutoPtr<Poco::XML::Element> database_element(xml_document->createElement("database"));
441 current_dictionary->appendChild(database_element);
442 AutoPtr<Text> database(xml_document->createTextNode(query.database));
443 database_element->appendChild(database);
444
445 AutoPtr<Element> structure_element(xml_document->createElement("structure"));
446 current_dictionary->appendChild(structure_element);
447 Names pk_attrs = getPrimaryKeyColumns(query.dictionary->primary_key);
448 auto dictionary_layout = query.dictionary->layout;
449
450 bool complex = DictionaryFactory::instance().isComplex(dictionary_layout->layout_type);
451
452 auto all_attr_names_and_types = buildDictionaryAttributesConfiguration(xml_document, structure_element, query.dictionary_attributes_list, pk_attrs);
453 checkPrimaryKey(all_attr_names_and_types, pk_attrs);
454
455 buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list);
456
457 buildLayoutConfiguration(xml_document, current_dictionary, dictionary_layout);
458 buildSourceConfiguration(xml_document, current_dictionary, query.dictionary->source);
459 buildLifetimeConfiguration(xml_document, current_dictionary, query.dictionary->lifetime);
460
461 if (query.dictionary->range)
462 buildRangeConfiguration(xml_document, structure_element, query.dictionary->range, all_attr_names_and_types);
463
464 conf->load(xml_document);
465 return conf;
466}
467
468}
469