1 | #include <Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h> |
2 | #include <Storages/MergeTree/MergeTreeData.h> |
3 | #include <Parsers/formatAST.h> |
4 | #include <Parsers/parseQuery.h> |
5 | #include <Parsers/ExpressionListParsers.h> |
6 | #include <IO/Operators.h> |
7 | |
8 | |
9 | namespace DB |
10 | { |
11 | |
12 | namespace ErrorCodes |
13 | { |
14 | extern const int METADATA_MISMATCH; |
15 | } |
16 | |
17 | static String formattedAST(const ASTPtr & ast) |
18 | { |
19 | if (!ast) |
20 | return "" ; |
21 | std::stringstream ss; |
22 | formatAST(*ast, ss, false, true); |
23 | return ss.str(); |
24 | } |
25 | |
26 | ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTreeData & data) |
27 | { |
28 | if (data.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) |
29 | date_column = data.minmax_idx_columns[data.minmax_idx_date_column_pos]; |
30 | |
31 | const auto data_settings = data.getSettings(); |
32 | sampling_expression = formattedAST(data.sample_by_ast); |
33 | index_granularity = data_settings->index_granularity; |
34 | merging_params_mode = static_cast<int>(data.merging_params.mode); |
35 | sign_column = data.merging_params.sign_column; |
36 | |
37 | /// This code may looks strange, but previously we had only one entity: PRIMARY KEY (or ORDER BY, it doesn't matter) |
38 | /// Now we have two different entities ORDER BY and it's optional prefix -- PRIMARY KEY. |
39 | /// In most cases user doesn't specify PRIMARY KEY and semantically it's equal to ORDER BY. |
40 | /// So rules in zookeeper metadata is following: |
41 | /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata |
42 | /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata |
43 | if (!data.primary_key_ast) |
44 | primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast)); |
45 | else |
46 | { |
47 | primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.primary_key_ast)); |
48 | sorting_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast)); |
49 | } |
50 | |
51 | data_format_version = data.format_version; |
52 | |
53 | if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) |
54 | partition_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.partition_by_ast)); |
55 | |
56 | ttl_table = formattedAST(data.ttl_table_ast); |
57 | |
58 | std::ostringstream ttl_move_stream; |
59 | for (const auto & ttl_entry : data.move_ttl_entries) |
60 | { |
61 | if (ttl_move_stream.tellp() > 0) |
62 | ttl_move_stream << ", " ; |
63 | ttl_move_stream << formattedAST(ttl_entry.entry_ast); |
64 | } |
65 | ttl_move = ttl_move_stream.str(); |
66 | |
67 | skip_indices = data.getIndices().toString(); |
68 | if (data.canUseAdaptiveGranularity()) |
69 | index_granularity_bytes = data_settings->index_granularity_bytes; |
70 | else |
71 | index_granularity_bytes = 0; |
72 | |
73 | constraints = data.getConstraints().toString(); |
74 | } |
75 | |
76 | void ReplicatedMergeTreeTableMetadata::write(WriteBuffer & out) const |
77 | { |
78 | out << "metadata format version: 1" << "\n" |
79 | << "date column: " << date_column << "\n" |
80 | << "sampling expression: " << sampling_expression << "\n" |
81 | << "index granularity: " << index_granularity << "\n" |
82 | << "mode: " << merging_params_mode << "\n" |
83 | << "sign column: " << sign_column << "\n" |
84 | << "primary key: " << primary_key << "\n" ; |
85 | |
86 | if (data_format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) |
87 | { |
88 | out << "data format version: " << data_format_version.toUnderType() << "\n" |
89 | << "partition key: " << partition_key << "\n" ; |
90 | } |
91 | |
92 | if (!sorting_key.empty()) |
93 | out << "sorting key: " << sorting_key << "\n" ; |
94 | |
95 | if (!ttl_table.empty()) |
96 | out << "ttl: " << ttl_table << "\n" ; |
97 | |
98 | if (!ttl_move.empty()) |
99 | out << "move ttl: " << ttl_move << "\n" ; |
100 | |
101 | if (!skip_indices.empty()) |
102 | out << "indices: " << skip_indices << "\n" ; |
103 | |
104 | if (index_granularity_bytes != 0) |
105 | out << "granularity bytes: " << index_granularity_bytes << "\n" ; |
106 | |
107 | if (!constraints.empty()) |
108 | out << "constraints: " << constraints << "\n" ; |
109 | } |
110 | |
111 | String ReplicatedMergeTreeTableMetadata::toString() const |
112 | { |
113 | WriteBufferFromOwnString out; |
114 | write(out); |
115 | return out.str(); |
116 | } |
117 | |
118 | void ReplicatedMergeTreeTableMetadata::read(ReadBuffer & in) |
119 | { |
120 | in >> "metadata format version: 1\n" ; |
121 | in >> "date column: " >> date_column >> "\n" ; |
122 | in >> "sampling expression: " >> sampling_expression >> "\n" ; |
123 | in >> "index granularity: " >> index_granularity >> "\n" ; |
124 | in >> "mode: " >> merging_params_mode >> "\n" ; |
125 | in >> "sign column: " >> sign_column >> "\n" ; |
126 | in >> "primary key: " >> primary_key >> "\n" ; |
127 | |
128 | if (in.eof()) |
129 | data_format_version = 0; |
130 | else if (checkString("data format version: " , in)) |
131 | in >> data_format_version.toUnderType() >> "\n" ; |
132 | |
133 | if (data_format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) |
134 | in >> "partition key: " >> partition_key >> "\n" ; |
135 | |
136 | if (checkString("sorting key: " , in)) |
137 | in >> sorting_key >> "\n" ; |
138 | |
139 | if (checkString("ttl: " , in)) |
140 | in >> ttl_table >> "\n" ; |
141 | |
142 | if (checkString("move ttl: " , in)) |
143 | in >> ttl_move >> "\n" ; |
144 | |
145 | if (checkString("indices: " , in)) |
146 | in >> skip_indices >> "\n" ; |
147 | |
148 | if (checkString("granularity bytes: " , in)) |
149 | { |
150 | in >> index_granularity_bytes >> "\n" ; |
151 | index_granularity_bytes_found_in_zk = true; |
152 | } |
153 | else |
154 | index_granularity_bytes = 0; |
155 | |
156 | if (checkString("constraints: " , in)) |
157 | in >> constraints >> "\n" ; |
158 | } |
159 | |
160 | ReplicatedMergeTreeTableMetadata ReplicatedMergeTreeTableMetadata::parse(const String & s) |
161 | { |
162 | ReplicatedMergeTreeTableMetadata metadata; |
163 | ReadBufferFromString buf(s); |
164 | metadata.read(buf); |
165 | return metadata; |
166 | } |
167 | |
168 | ReplicatedMergeTreeTableMetadata::Diff |
169 | ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, bool allow_alter) const |
170 | { |
171 | Diff diff; |
172 | |
173 | if (data_format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) |
174 | { |
175 | if (date_column != from_zk.date_column) |
176 | throw Exception("Existing table metadata in ZooKeeper differs in date index column." |
177 | " Stored in ZooKeeper: " + from_zk.date_column + ", local: " + date_column, |
178 | ErrorCodes::METADATA_MISMATCH); |
179 | } |
180 | else if (!from_zk.date_column.empty()) |
181 | throw Exception( |
182 | "Existing table metadata in ZooKeeper differs in date index column." |
183 | " Stored in ZooKeeper: " + from_zk.date_column + ", local is custom-partitioned." , |
184 | ErrorCodes::METADATA_MISMATCH); |
185 | |
186 | if (sampling_expression != from_zk.sampling_expression) |
187 | throw Exception("Existing table metadata in ZooKeeper differs in sample expression." |
188 | " Stored in ZooKeeper: " + from_zk.sampling_expression + ", local: " + sampling_expression, |
189 | ErrorCodes::METADATA_MISMATCH); |
190 | |
191 | if (index_granularity != from_zk.index_granularity) |
192 | throw Exception("Existing table metadata in ZooKeeper differs in index granularity." |
193 | " Stored in ZooKeeper: " + DB::toString(from_zk.index_granularity) + ", local: " + DB::toString(index_granularity), |
194 | ErrorCodes::METADATA_MISMATCH); |
195 | |
196 | if (merging_params_mode != from_zk.merging_params_mode) |
197 | throw Exception("Existing table metadata in ZooKeeper differs in mode of merge operation." |
198 | " Stored in ZooKeeper: " + DB::toString(from_zk.merging_params_mode) + ", local: " |
199 | + DB::toString(merging_params_mode), |
200 | ErrorCodes::METADATA_MISMATCH); |
201 | |
202 | if (sign_column != from_zk.sign_column) |
203 | throw Exception("Existing table metadata in ZooKeeper differs in sign column." |
204 | " Stored in ZooKeeper: " + from_zk.sign_column + ", local: " + sign_column, |
205 | ErrorCodes::METADATA_MISMATCH); |
206 | |
207 | /// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes |
208 | /// in formatAST code. |
209 | if (primary_key != from_zk.primary_key) |
210 | throw Exception("Existing table metadata in ZooKeeper differs in primary key." |
211 | " Stored in ZooKeeper: " + from_zk.primary_key + ", local: " + primary_key, |
212 | ErrorCodes::METADATA_MISMATCH); |
213 | |
214 | if (data_format_version != from_zk.data_format_version) |
215 | throw Exception("Existing table metadata in ZooKeeper differs in data format version." |
216 | " Stored in ZooKeeper: " + DB::toString(from_zk.data_format_version.toUnderType()) + |
217 | ", local: " + DB::toString(data_format_version.toUnderType()), |
218 | ErrorCodes::METADATA_MISMATCH); |
219 | |
220 | if (partition_key != from_zk.partition_key) |
221 | throw Exception( |
222 | "Existing table metadata in ZooKeeper differs in partition key expression." |
223 | " Stored in ZooKeeper: " + from_zk.partition_key + ", local: " + partition_key, |
224 | ErrorCodes::METADATA_MISMATCH); |
225 | |
226 | if (sorting_key != from_zk.sorting_key) |
227 | { |
228 | if (allow_alter) |
229 | { |
230 | diff.sorting_key_changed = true; |
231 | diff.new_sorting_key = from_zk.sorting_key; |
232 | } |
233 | else |
234 | throw Exception( |
235 | "Existing table metadata in ZooKeeper differs in sorting key expression." |
236 | " Stored in ZooKeeper: " + from_zk.sorting_key + ", local: " + sorting_key, |
237 | ErrorCodes::METADATA_MISMATCH); |
238 | } |
239 | |
240 | if (ttl_table != from_zk.ttl_table) |
241 | { |
242 | if (allow_alter) |
243 | { |
244 | diff.ttl_table_changed = true; |
245 | diff.new_ttl_table = from_zk.ttl_table; |
246 | } |
247 | else |
248 | throw Exception( |
249 | "Existing table metadata in ZooKeeper differs in TTL." |
250 | " Stored in ZooKeeper: " + from_zk.ttl_table + |
251 | ", local: " + ttl_table, |
252 | ErrorCodes::METADATA_MISMATCH); |
253 | } |
254 | |
255 | if (ttl_move != from_zk.ttl_move) |
256 | { |
257 | if (allow_alter) |
258 | { |
259 | diff.ttl_move_changed = true; |
260 | diff.new_ttl_move = from_zk.ttl_move; |
261 | } |
262 | else |
263 | throw Exception( |
264 | "Existing table metadata in ZooKeeper differs in move TTL." |
265 | " Stored in ZooKeeper: " + from_zk.ttl_move + |
266 | ", local: " + ttl_move, |
267 | ErrorCodes::METADATA_MISMATCH); |
268 | } |
269 | |
270 | if (skip_indices != from_zk.skip_indices) |
271 | { |
272 | if (allow_alter) |
273 | { |
274 | diff.skip_indices_changed = true; |
275 | diff.new_skip_indices = from_zk.skip_indices; |
276 | } |
277 | else |
278 | throw Exception( |
279 | "Existing table metadata in ZooKeeper differs in skip indexes." |
280 | " Stored in ZooKeeper: " + from_zk.skip_indices + |
281 | ", local: " + skip_indices, |
282 | ErrorCodes::METADATA_MISMATCH); |
283 | } |
284 | |
285 | if (constraints != from_zk.constraints) |
286 | { |
287 | if (allow_alter) |
288 | { |
289 | diff.constraints_changed = true; |
290 | diff.new_constraints = from_zk.constraints; |
291 | } |
292 | else |
293 | throw Exception( |
294 | "Existing table metadata in ZooKeeper differs in constraints." |
295 | " Stored in ZooKeeper: " + from_zk.constraints + |
296 | ", local: " + constraints, |
297 | ErrorCodes::METADATA_MISMATCH); |
298 | } |
299 | |
300 | if (from_zk.index_granularity_bytes_found_in_zk && index_granularity_bytes != from_zk.index_granularity_bytes) |
301 | throw Exception("Existing table metadata in ZooKeeper differs in index granularity bytes." |
302 | " Stored in ZooKeeper: " + DB::toString(from_zk.index_granularity_bytes) + |
303 | ", local: " + DB::toString(index_granularity_bytes), |
304 | ErrorCodes::METADATA_MISMATCH); |
305 | |
306 | return diff; |
307 | } |
308 | |
309 | } |
310 | |