1#include <Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h>
2#include <Storages/MergeTree/MergeTreeData.h>
3#include <Parsers/formatAST.h>
4#include <Parsers/parseQuery.h>
5#include <Parsers/ExpressionListParsers.h>
6#include <IO/Operators.h>
7
8
9namespace DB
10{
11
12namespace ErrorCodes
13{
14 extern const int METADATA_MISMATCH;
15}
16
17static String formattedAST(const ASTPtr & ast)
18{
19 if (!ast)
20 return "";
21 std::stringstream ss;
22 formatAST(*ast, ss, false, true);
23 return ss.str();
24}
25
26ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTreeData & data)
27{
28 if (data.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
29 date_column = data.minmax_idx_columns[data.minmax_idx_date_column_pos];
30
31 const auto data_settings = data.getSettings();
32 sampling_expression = formattedAST(data.sample_by_ast);
33 index_granularity = data_settings->index_granularity;
34 merging_params_mode = static_cast<int>(data.merging_params.mode);
35 sign_column = data.merging_params.sign_column;
36
37 /// This code may looks strange, but previously we had only one entity: PRIMARY KEY (or ORDER BY, it doesn't matter)
38 /// Now we have two different entities ORDER BY and it's optional prefix -- PRIMARY KEY.
39 /// In most cases user doesn't specify PRIMARY KEY and semantically it's equal to ORDER BY.
40 /// So rules in zookeeper metadata is following:
41 /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata
42 /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata
43 if (!data.primary_key_ast)
44 primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast));
45 else
46 {
47 primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.primary_key_ast));
48 sorting_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast));
49 }
50
51 data_format_version = data.format_version;
52
53 if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
54 partition_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.partition_by_ast));
55
56 ttl_table = formattedAST(data.ttl_table_ast);
57
58 std::ostringstream ttl_move_stream;
59 for (const auto & ttl_entry : data.move_ttl_entries)
60 {
61 if (ttl_move_stream.tellp() > 0)
62 ttl_move_stream << ", ";
63 ttl_move_stream << formattedAST(ttl_entry.entry_ast);
64 }
65 ttl_move = ttl_move_stream.str();
66
67 skip_indices = data.getIndices().toString();
68 if (data.canUseAdaptiveGranularity())
69 index_granularity_bytes = data_settings->index_granularity_bytes;
70 else
71 index_granularity_bytes = 0;
72
73 constraints = data.getConstraints().toString();
74}
75
76void ReplicatedMergeTreeTableMetadata::write(WriteBuffer & out) const
77{
78 out << "metadata format version: 1" << "\n"
79 << "date column: " << date_column << "\n"
80 << "sampling expression: " << sampling_expression << "\n"
81 << "index granularity: " << index_granularity << "\n"
82 << "mode: " << merging_params_mode << "\n"
83 << "sign column: " << sign_column << "\n"
84 << "primary key: " << primary_key << "\n";
85
86 if (data_format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
87 {
88 out << "data format version: " << data_format_version.toUnderType() << "\n"
89 << "partition key: " << partition_key << "\n";
90 }
91
92 if (!sorting_key.empty())
93 out << "sorting key: " << sorting_key << "\n";
94
95 if (!ttl_table.empty())
96 out << "ttl: " << ttl_table << "\n";
97
98 if (!ttl_move.empty())
99 out << "move ttl: " << ttl_move << "\n";
100
101 if (!skip_indices.empty())
102 out << "indices: " << skip_indices << "\n";
103
104 if (index_granularity_bytes != 0)
105 out << "granularity bytes: " << index_granularity_bytes << "\n";
106
107 if (!constraints.empty())
108 out << "constraints: " << constraints << "\n";
109}
110
111String ReplicatedMergeTreeTableMetadata::toString() const
112{
113 WriteBufferFromOwnString out;
114 write(out);
115 return out.str();
116}
117
118void ReplicatedMergeTreeTableMetadata::read(ReadBuffer & in)
119{
120 in >> "metadata format version: 1\n";
121 in >> "date column: " >> date_column >> "\n";
122 in >> "sampling expression: " >> sampling_expression >> "\n";
123 in >> "index granularity: " >> index_granularity >> "\n";
124 in >> "mode: " >> merging_params_mode >> "\n";
125 in >> "sign column: " >> sign_column >> "\n";
126 in >> "primary key: " >> primary_key >> "\n";
127
128 if (in.eof())
129 data_format_version = 0;
130 else if (checkString("data format version: ", in))
131 in >> data_format_version.toUnderType() >> "\n";
132
133 if (data_format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
134 in >> "partition key: " >> partition_key >> "\n";
135
136 if (checkString("sorting key: ", in))
137 in >> sorting_key >> "\n";
138
139 if (checkString("ttl: ", in))
140 in >> ttl_table >> "\n";
141
142 if (checkString("move ttl: ", in))
143 in >> ttl_move >> "\n";
144
145 if (checkString("indices: ", in))
146 in >> skip_indices >> "\n";
147
148 if (checkString("granularity bytes: ", in))
149 {
150 in >> index_granularity_bytes >> "\n";
151 index_granularity_bytes_found_in_zk = true;
152 }
153 else
154 index_granularity_bytes = 0;
155
156 if (checkString("constraints: ", in))
157 in >> constraints >> "\n";
158}
159
160ReplicatedMergeTreeTableMetadata ReplicatedMergeTreeTableMetadata::parse(const String & s)
161{
162 ReplicatedMergeTreeTableMetadata metadata;
163 ReadBufferFromString buf(s);
164 metadata.read(buf);
165 return metadata;
166}
167
168ReplicatedMergeTreeTableMetadata::Diff
169ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, bool allow_alter) const
170{
171 Diff diff;
172
173 if (data_format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
174 {
175 if (date_column != from_zk.date_column)
176 throw Exception("Existing table metadata in ZooKeeper differs in date index column."
177 " Stored in ZooKeeper: " + from_zk.date_column + ", local: " + date_column,
178 ErrorCodes::METADATA_MISMATCH);
179 }
180 else if (!from_zk.date_column.empty())
181 throw Exception(
182 "Existing table metadata in ZooKeeper differs in date index column."
183 " Stored in ZooKeeper: " + from_zk.date_column + ", local is custom-partitioned.",
184 ErrorCodes::METADATA_MISMATCH);
185
186 if (sampling_expression != from_zk.sampling_expression)
187 throw Exception("Existing table metadata in ZooKeeper differs in sample expression."
188 " Stored in ZooKeeper: " + from_zk.sampling_expression + ", local: " + sampling_expression,
189 ErrorCodes::METADATA_MISMATCH);
190
191 if (index_granularity != from_zk.index_granularity)
192 throw Exception("Existing table metadata in ZooKeeper differs in index granularity."
193 " Stored in ZooKeeper: " + DB::toString(from_zk.index_granularity) + ", local: " + DB::toString(index_granularity),
194 ErrorCodes::METADATA_MISMATCH);
195
196 if (merging_params_mode != from_zk.merging_params_mode)
197 throw Exception("Existing table metadata in ZooKeeper differs in mode of merge operation."
198 " Stored in ZooKeeper: " + DB::toString(from_zk.merging_params_mode) + ", local: "
199 + DB::toString(merging_params_mode),
200 ErrorCodes::METADATA_MISMATCH);
201
202 if (sign_column != from_zk.sign_column)
203 throw Exception("Existing table metadata in ZooKeeper differs in sign column."
204 " Stored in ZooKeeper: " + from_zk.sign_column + ", local: " + sign_column,
205 ErrorCodes::METADATA_MISMATCH);
206
207 /// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes
208 /// in formatAST code.
209 if (primary_key != from_zk.primary_key)
210 throw Exception("Existing table metadata in ZooKeeper differs in primary key."
211 " Stored in ZooKeeper: " + from_zk.primary_key + ", local: " + primary_key,
212 ErrorCodes::METADATA_MISMATCH);
213
214 if (data_format_version != from_zk.data_format_version)
215 throw Exception("Existing table metadata in ZooKeeper differs in data format version."
216 " Stored in ZooKeeper: " + DB::toString(from_zk.data_format_version.toUnderType()) +
217 ", local: " + DB::toString(data_format_version.toUnderType()),
218 ErrorCodes::METADATA_MISMATCH);
219
220 if (partition_key != from_zk.partition_key)
221 throw Exception(
222 "Existing table metadata in ZooKeeper differs in partition key expression."
223 " Stored in ZooKeeper: " + from_zk.partition_key + ", local: " + partition_key,
224 ErrorCodes::METADATA_MISMATCH);
225
226 if (sorting_key != from_zk.sorting_key)
227 {
228 if (allow_alter)
229 {
230 diff.sorting_key_changed = true;
231 diff.new_sorting_key = from_zk.sorting_key;
232 }
233 else
234 throw Exception(
235 "Existing table metadata in ZooKeeper differs in sorting key expression."
236 " Stored in ZooKeeper: " + from_zk.sorting_key + ", local: " + sorting_key,
237 ErrorCodes::METADATA_MISMATCH);
238 }
239
240 if (ttl_table != from_zk.ttl_table)
241 {
242 if (allow_alter)
243 {
244 diff.ttl_table_changed = true;
245 diff.new_ttl_table = from_zk.ttl_table;
246 }
247 else
248 throw Exception(
249 "Existing table metadata in ZooKeeper differs in TTL."
250 " Stored in ZooKeeper: " + from_zk.ttl_table +
251 ", local: " + ttl_table,
252 ErrorCodes::METADATA_MISMATCH);
253 }
254
255 if (ttl_move != from_zk.ttl_move)
256 {
257 if (allow_alter)
258 {
259 diff.ttl_move_changed = true;
260 diff.new_ttl_move = from_zk.ttl_move;
261 }
262 else
263 throw Exception(
264 "Existing table metadata in ZooKeeper differs in move TTL."
265 " Stored in ZooKeeper: " + from_zk.ttl_move +
266 ", local: " + ttl_move,
267 ErrorCodes::METADATA_MISMATCH);
268 }
269
270 if (skip_indices != from_zk.skip_indices)
271 {
272 if (allow_alter)
273 {
274 diff.skip_indices_changed = true;
275 diff.new_skip_indices = from_zk.skip_indices;
276 }
277 else
278 throw Exception(
279 "Existing table metadata in ZooKeeper differs in skip indexes."
280 " Stored in ZooKeeper: " + from_zk.skip_indices +
281 ", local: " + skip_indices,
282 ErrorCodes::METADATA_MISMATCH);
283 }
284
285 if (constraints != from_zk.constraints)
286 {
287 if (allow_alter)
288 {
289 diff.constraints_changed = true;
290 diff.new_constraints = from_zk.constraints;
291 }
292 else
293 throw Exception(
294 "Existing table metadata in ZooKeeper differs in constraints."
295 " Stored in ZooKeeper: " + from_zk.constraints +
296 ", local: " + constraints,
297 ErrorCodes::METADATA_MISMATCH);
298 }
299
300 if (from_zk.index_granularity_bytes_found_in_zk && index_granularity_bytes != from_zk.index_granularity_bytes)
301 throw Exception("Existing table metadata in ZooKeeper differs in index granularity bytes."
302 " Stored in ZooKeeper: " + DB::toString(from_zk.index_granularity_bytes) +
303 ", local: " + DB::toString(index_granularity_bytes),
304 ErrorCodes::METADATA_MISMATCH);
305
306 return diff;
307}
308
309}
310