1 | #pragma once |
2 | |
3 | #include <Parsers/IAST.h> |
4 | #include <Parsers/ASTSubquery.h> |
5 | #include <Parsers/ASTFunction.h> |
6 | #include <Parsers/ASTTablesInSelectQuery.h> |
7 | #include <Parsers/ASTSelectQuery.h> |
8 | #include <Parsers/ASTIdentifier.h> |
9 | #include <Interpreters/Context.h> |
10 | #include <Interpreters/interpretSubquery.h> |
11 | #include <Common/typeid_cast.h> |
12 | #include <Core/Block.h> |
13 | #include <Core/NamesAndTypes.h> |
14 | #include <Databases/IDatabase.h> |
15 | #include <Storages/StorageMemory.h> |
16 | #include <IO/WriteHelpers.h> |
17 | #include <Interpreters/InDepthNodeVisitor.h> |
18 | #include <Interpreters/IdentifierSemantic.h> |
19 | |
20 | namespace DB |
21 | { |
22 | |
23 | |
24 | class GlobalSubqueriesMatcher |
25 | { |
26 | public: |
27 | struct Data |
28 | { |
29 | const Context & context; |
30 | size_t subquery_depth; |
31 | bool is_remote; |
32 | size_t external_table_id; |
33 | Tables & external_tables; |
34 | SubqueriesForSets & subqueries_for_sets; |
35 | bool & has_global_subqueries; |
36 | |
37 | Data(const Context & context_, size_t subquery_depth_, bool is_remote_, |
38 | Tables & tables, SubqueriesForSets & subqueries_for_sets_, bool & has_global_subqueries_) |
39 | : context(context_), |
40 | subquery_depth(subquery_depth_), |
41 | is_remote(is_remote_), |
42 | external_table_id(1), |
43 | external_tables(tables), |
44 | subqueries_for_sets(subqueries_for_sets_), |
45 | has_global_subqueries(has_global_subqueries_) |
46 | {} |
47 | |
48 | void addExternalStorage(ASTPtr & ast, bool set_alias = false) |
49 | { |
50 | /// With nondistributed queries, creating temporary tables does not make sense. |
51 | if (!is_remote) |
52 | return; |
53 | |
54 | bool is_table = false; |
55 | ASTPtr subquery_or_table_name = ast; /// ASTIdentifier | ASTSubquery | ASTTableExpression |
56 | |
57 | if (const auto * ast_table_expr = ast->as<ASTTableExpression>()) |
58 | { |
59 | subquery_or_table_name = ast_table_expr->subquery; |
60 | |
61 | if (ast_table_expr->database_and_table_name) |
62 | { |
63 | subquery_or_table_name = ast_table_expr->database_and_table_name; |
64 | is_table = true; |
65 | } |
66 | } |
67 | else if (ast->as<ASTIdentifier>()) |
68 | is_table = true; |
69 | |
70 | if (!subquery_or_table_name) |
71 | throw Exception("Logical error: unknown AST element passed to ExpressionAnalyzer::addExternalStorage method" , |
72 | ErrorCodes::LOGICAL_ERROR); |
73 | |
74 | if (is_table) |
75 | { |
76 | /// If this is already an external table, you do not need to add anything. Just remember its presence. |
77 | if (external_tables.end() != external_tables.find(getIdentifierName(subquery_or_table_name))) |
78 | return; |
79 | } |
80 | |
81 | String external_table_name = subquery_or_table_name->tryGetAlias(); |
82 | if (external_table_name.empty()) |
83 | { |
84 | /// Generate the name for the external table. |
85 | external_table_name = "_data" + toString(external_table_id); |
86 | while (external_tables.count(external_table_name)) |
87 | { |
88 | ++external_table_id; |
89 | external_table_name = "_data" + toString(external_table_id); |
90 | } |
91 | } |
92 | |
93 | auto interpreter = interpretSubquery(subquery_or_table_name, context, subquery_depth, {}); |
94 | |
95 | Block sample = interpreter->getSampleBlock(); |
96 | NamesAndTypesList columns = sample.getNamesAndTypesList(); |
97 | |
98 | StoragePtr external_storage = StorageMemory::create("_external" , external_table_name, ColumnsDescription{columns}, ConstraintsDescription{}); |
99 | external_storage->startup(); |
100 | |
101 | /** We replace the subquery with the name of the temporary table. |
102 | * It is in this form, the request will go to the remote server. |
103 | * This temporary table will go to the remote server, and on its side, |
104 | * instead of doing a subquery, you just need to read it. |
105 | */ |
106 | |
107 | auto database_and_table_name = createTableIdentifier("" , external_table_name); |
108 | if (set_alias) |
109 | { |
110 | String alias = subquery_or_table_name->tryGetAlias(); |
111 | if (auto * table_name = subquery_or_table_name->as<ASTIdentifier>()) |
112 | if (alias.empty()) |
113 | alias = table_name->shortName(); |
114 | database_and_table_name->setAlias(alias); |
115 | } |
116 | |
117 | if (auto * ast_table_expr = ast->as<ASTTableExpression>()) |
118 | { |
119 | ast_table_expr->subquery.reset(); |
120 | ast_table_expr->database_and_table_name = database_and_table_name; |
121 | |
122 | ast_table_expr->children.clear(); |
123 | ast_table_expr->children.emplace_back(database_and_table_name); |
124 | } |
125 | else |
126 | ast = database_and_table_name; |
127 | |
128 | external_tables[external_table_name] = external_storage; |
129 | subqueries_for_sets[external_table_name].source = interpreter->execute().in; |
130 | subqueries_for_sets[external_table_name].table = external_storage; |
131 | |
132 | /** NOTE If it was written IN tmp_table - the existing temporary (but not external) table, |
133 | * then a new temporary table will be created (for example, _data1), |
134 | * and the data will then be copied to it. |
135 | * Maybe this can be avoided. |
136 | */ |
137 | } |
138 | }; |
139 | |
140 | static void visit(ASTPtr & ast, Data & data) |
141 | { |
142 | if (auto * t = ast->as<ASTFunction>()) |
143 | visit(*t, ast, data); |
144 | if (auto * t = ast->as<ASTTablesInSelectQueryElement>()) |
145 | visit(*t, ast, data); |
146 | } |
147 | |
148 | static bool needChildVisit(ASTPtr &, const ASTPtr & child) |
149 | { |
150 | /// We do not go into subqueries. |
151 | if (child->as<ASTSelectQuery>()) |
152 | return false; |
153 | return true; |
154 | } |
155 | |
156 | private: |
157 | /// GLOBAL IN |
158 | static void visit(ASTFunction & func, ASTPtr &, Data & data) |
159 | { |
160 | if (func.name == "globalIn" || func.name == "globalNotIn" ) |
161 | { |
162 | data.addExternalStorage(func.arguments->children[1]); |
163 | data.has_global_subqueries = true; |
164 | } |
165 | } |
166 | |
167 | /// GLOBAL JOIN |
168 | static void visit(ASTTablesInSelectQueryElement & table_elem, ASTPtr &, Data & data) |
169 | { |
170 | if (table_elem.table_join && table_elem.table_join->as<ASTTableJoin &>().locality == ASTTableJoin::Locality::Global) |
171 | { |
172 | data.addExternalStorage(table_elem.table_expression, true); |
173 | data.has_global_subqueries = true; |
174 | } |
175 | } |
176 | }; |
177 | |
178 | /// Converts GLOBAL subqueries to external tables; Puts them into the external_tables dictionary: name -> StoragePtr. |
179 | using GlobalSubqueriesVisitor = InDepthNodeVisitor<GlobalSubqueriesMatcher, false>; |
180 | |
181 | } |
182 | |