1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// This module contains the logical parquet-cpp types (independent of Thrift
19// structures), schema nodes, and related type tools
20
21#ifndef PARQUET_SCHEMA_TYPES_H
22#define PARQUET_SCHEMA_TYPES_H
23
24#include <cstdint>
25#include <memory>
26#include <ostream>
27#include <string>
28#include <unordered_map>
29#include <vector>
30
31#include "parquet/platform.h"
32#include "parquet/types.h"
33
34namespace parquet {
35
36class SchemaDescriptor;
37
38namespace schema {
39
40class Node;
41
42// List encodings: using the terminology from Impala to define different styles
43// of representing logical lists (a.k.a. ARRAY types) in Parquet schemas. Since
44// the converted type named in the Parquet metadata is ConvertedType::LIST we
45// use that terminology here. It also helps distinguish from the *_ARRAY
46// primitive types.
47//
48// One-level encoding: Only allows required lists with required cells
49// repeated value_type name
50//
51// Two-level encoding: Enables optional lists with only required cells
52// <required/optional> group list
53// repeated value_type item
54//
55// Three-level encoding: Enables optional lists with optional cells
56// <required/optional> group bag
57// repeated group list
58// <required/optional> value_type item
59//
60// 2- and 1-level encoding are respectively equivalent to 3-level encoding with
61// the non-repeated nodes set to required.
62//
63// The "official" encoding recommended in the Parquet spec is the 3-level, and
64// we use that as the default when creating list types. For semantic completeness
65// we allow the other two. Since all types of encodings will occur "in the
66// wild" we need to be able to interpret the associated definition levels in
67// the context of the actual encoding used in the file.
68//
69// NB: Some Parquet writers may not set ConvertedType::LIST on the repeated
70// SchemaElement, which could make things challenging if we are trying to infer
71// that a sequence of nodes semantically represents an array according to one
72// of these encodings (versus a struct containing an array). We should refuse
73// the temptation to guess, as they say.
74struct ListEncoding {
75 enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL };
76};
77
78class PARQUET_EXPORT ColumnPath {
79 public:
80 ColumnPath() : path_() {}
81 explicit ColumnPath(const std::vector<std::string>& path) : path_(path) {}
82 explicit ColumnPath(std::vector<std::string>&& path) : path_(path) {}
83
84 static std::shared_ptr<ColumnPath> FromDotString(const std::string& dotstring);
85 static std::shared_ptr<ColumnPath> FromNode(const Node& node);
86
87 std::shared_ptr<ColumnPath> extend(const std::string& node_name) const;
88 std::string ToDotString() const;
89 const std::vector<std::string>& ToDotVector() const;
90
91 protected:
92 std::vector<std::string> path_;
93};
94
95// Base class for logical schema types. A type has a name, repetition level,
96// and optionally a logical type (ConvertedType in Parquet metadata parlance)
97class PARQUET_EXPORT Node {
98 public:
99 enum type { PRIMITIVE, GROUP };
100
101 Node(Node::type type, const std::string& name, Repetition::type repetition,
102 ConvertedType::type converted_type = ConvertedType::NONE, int id = -1)
103 : type_(type),
104 name_(name),
105 repetition_(repetition),
106 converted_type_(converted_type),
107 id_(id),
108 parent_(NULLPTR) {}
109
110 Node(Node::type type, const std::string& name, Repetition::type repetition,
111 std::shared_ptr<const LogicalType> logical_type, int id = -1)
112 : type_(type),
113 name_(name),
114 repetition_(repetition),
115 logical_type_(logical_type),
116 id_(id),
117 parent_(NULLPTR) {}
118
119 virtual ~Node() {}
120
121 bool is_primitive() const { return type_ == Node::PRIMITIVE; }
122
123 bool is_group() const { return type_ == Node::GROUP; }
124
125 bool is_optional() const { return repetition_ == Repetition::OPTIONAL; }
126
127 bool is_repeated() const { return repetition_ == Repetition::REPEATED; }
128
129 bool is_required() const { return repetition_ == Repetition::REQUIRED; }
130
131 virtual bool Equals(const Node* other) const = 0;
132
133 const std::string& name() const { return name_; }
134
135 Node::type node_type() const { return type_; }
136
137 Repetition::type repetition() const { return repetition_; }
138
139 ConvertedType::type converted_type() const { return converted_type_; }
140
141 const std::shared_ptr<const LogicalType>& logical_type() const { return logical_type_; }
142
143 int id() const { return id_; }
144
145 const Node* parent() const { return parent_; }
146
147 const std::shared_ptr<ColumnPath> path() const;
148
149 virtual void ToParquet(void* element) const = 0;
150
151 // Node::Visitor abstract class for walking schemas with the visitor pattern
152 class Visitor {
153 public:
154 virtual ~Visitor() {}
155
156 virtual void Visit(Node* node) = 0;
157 };
158 class ConstVisitor {
159 public:
160 virtual ~ConstVisitor() {}
161
162 virtual void Visit(const Node* node) = 0;
163 };
164
165 virtual void Visit(Visitor* visitor) = 0;
166 virtual void VisitConst(ConstVisitor* visitor) const = 0;
167
168 protected:
169 friend class GroupNode;
170
171 Node::type type_;
172 std::string name_;
173 Repetition::type repetition_;
174 ConvertedType::type converted_type_;
175 std::shared_ptr<const LogicalType> logical_type_;
176 int id_;
177 // Nodes should not be shared, they have a single parent.
178 const Node* parent_;
179
180 bool EqualsInternal(const Node* other) const;
181 void SetParent(const Node* p_parent);
182
183 private:
184 PARQUET_DISALLOW_COPY_AND_ASSIGN(Node);
185};
186
187// Save our breath all over the place with these typedefs
188typedef std::shared_ptr<Node> NodePtr;
189typedef std::vector<NodePtr> NodeVector;
190
191// A type that is one of the primitive Parquet storage types. In addition to
192// the other type metadata (name, repetition level, logical type), also has the
193// physical storage type and their type-specific metadata (byte width, decimal
194// parameters)
195class PARQUET_EXPORT PrimitiveNode : public Node {
196 public:
197 static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id);
198
199 static inline NodePtr Make(const std::string& name, Repetition::type repetition,
200 Type::type type,
201 ConvertedType::type converted_type = ConvertedType::NONE,
202 int length = -1, int precision = -1, int scale = -1) {
203 return NodePtr(new PrimitiveNode(name, repetition, type, converted_type, length,
204 precision, scale));
205 }
206
207 static inline NodePtr Make(const std::string& name, Repetition::type repetition,
208 std::shared_ptr<const LogicalType> logical_type,
209 Type::type primitive_type, int primitive_length = -1) {
210 return NodePtr(new PrimitiveNode(name, repetition, logical_type, primitive_type,
211 primitive_length));
212 }
213
214 bool Equals(const Node* other) const override;
215
216 Type::type physical_type() const { return physical_type_; }
217
218 ColumnOrder column_order() const { return column_order_; }
219
220 void SetColumnOrder(ColumnOrder column_order) { column_order_ = column_order; }
221
222 int32_t type_length() const { return type_length_; }
223
224 const DecimalMetadata& decimal_metadata() const { return decimal_metadata_; }
225
226 void ToParquet(void* element) const override;
227 void Visit(Visitor* visitor) override;
228 void VisitConst(ConstVisitor* visitor) const override;
229
230 private:
231 PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type,
232 ConvertedType::type converted_type = ConvertedType::NONE, int length = -1,
233 int precision = -1, int scale = -1, int id = -1);
234
235 PrimitiveNode(const std::string& name, Repetition::type repetition,
236 std::shared_ptr<const LogicalType> logical_type,
237 Type::type primitive_type, int primitive_length = -1, int id = -1);
238
239 Type::type physical_type_;
240 int32_t type_length_;
241 DecimalMetadata decimal_metadata_;
242 ColumnOrder column_order_;
243
244 // For FIXED_LEN_BYTE_ARRAY
245 void SetTypeLength(int32_t length) { type_length_ = length; }
246
247 bool EqualsInternal(const PrimitiveNode* other) const;
248
249 FRIEND_TEST(TestPrimitiveNode, Attrs);
250 FRIEND_TEST(TestPrimitiveNode, Equals);
251 FRIEND_TEST(TestPrimitiveNode, PhysicalLogicalMapping);
252 FRIEND_TEST(TestPrimitiveNode, FromParquet);
253};
254
255class PARQUET_EXPORT GroupNode : public Node {
256 public:
257 static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id,
258 const NodeVector& fields);
259
260 static inline NodePtr Make(const std::string& name, Repetition::type repetition,
261 const NodeVector& fields,
262 ConvertedType::type converted_type = ConvertedType::NONE) {
263 return NodePtr(new GroupNode(name, repetition, fields, converted_type));
264 }
265
266 static inline NodePtr Make(const std::string& name, Repetition::type repetition,
267 const NodeVector& fields,
268 std::shared_ptr<const LogicalType> logical_type) {
269 return NodePtr(new GroupNode(name, repetition, fields, logical_type));
270 }
271
272 bool Equals(const Node* other) const override;
273
274 NodePtr field(int i) const { return fields_[i]; }
275 // Get the index of a field by its name, or negative value if not found.
276 // If several fields share the same name, it is unspecified which one
277 // is returned.
278 int FieldIndex(const std::string& name) const;
279 // Get the index of a field by its node, or negative value if not found.
280 int FieldIndex(const Node& node) const;
281
282 int field_count() const { return static_cast<int>(fields_.size()); }
283
284 void ToParquet(void* element) const override;
285 void Visit(Visitor* visitor) override;
286 void VisitConst(ConstVisitor* visitor) const override;
287
288 private:
289 GroupNode(const std::string& name, Repetition::type repetition,
290 const NodeVector& fields,
291 ConvertedType::type converted_type = ConvertedType::NONE, int id = -1);
292
293 GroupNode(const std::string& name, Repetition::type repetition,
294 const NodeVector& fields, std::shared_ptr<const LogicalType> logical_type,
295 int id = -1);
296
297 NodeVector fields_;
298 bool EqualsInternal(const GroupNode* other) const;
299
300 // Mapping between field name to the field index
301 std::unordered_multimap<std::string, int> field_name_to_idx_;
302
303 FRIEND_TEST(TestGroupNode, Attrs);
304 FRIEND_TEST(TestGroupNode, Equals);
305 FRIEND_TEST(TestGroupNode, FieldIndex);
306 FRIEND_TEST(TestGroupNode, FieldIndexDuplicateName);
307};
308
309// ----------------------------------------------------------------------
310// Convenience primitive type factory functions
311
312#define PRIMITIVE_FACTORY(FuncName, TYPE) \
313 static inline NodePtr FuncName(const std::string& name, \
314 Repetition::type repetition = Repetition::OPTIONAL) { \
315 return PrimitiveNode::Make(name, repetition, Type::TYPE); \
316 }
317
318PRIMITIVE_FACTORY(Boolean, BOOLEAN);
319PRIMITIVE_FACTORY(Int32, INT32);
320PRIMITIVE_FACTORY(Int64, INT64);
321PRIMITIVE_FACTORY(Int96, INT96);
322PRIMITIVE_FACTORY(Float, FLOAT);
323PRIMITIVE_FACTORY(Double, DOUBLE);
324PRIMITIVE_FACTORY(ByteArray, BYTE_ARRAY);
325
326void PARQUET_EXPORT PrintSchema(const schema::Node* schema, std::ostream& stream,
327 int indent_width = 2);
328
329} // namespace schema
330
331// The ColumnDescriptor encapsulates information necessary to interpret
332// primitive column data in the context of a particular schema. We have to
333// examine the node structure of a column's path to the root in the schema tree
334// to be able to reassemble the nested structure from the repetition and
335// definition levels.
336class PARQUET_EXPORT ColumnDescriptor {
337 public:
338 ColumnDescriptor(const schema::NodePtr& node, int16_t max_definition_level,
339 int16_t max_repetition_level,
340 const SchemaDescriptor* schema_descr = NULLPTR);
341
342 bool Equals(const ColumnDescriptor& other) const;
343
344 int16_t max_definition_level() const { return max_definition_level_; }
345
346 int16_t max_repetition_level() const { return max_repetition_level_; }
347
348 Type::type physical_type() const { return primitive_node_->physical_type(); }
349
350 ConvertedType::type converted_type() const { return primitive_node_->converted_type(); }
351
352 const std::shared_ptr<const LogicalType>& logical_type() const {
353 return primitive_node_->logical_type();
354 }
355
356 ColumnOrder column_order() const { return primitive_node_->column_order(); }
357
358 SortOrder::type sort_order() const {
359 auto la = logical_type();
360 auto pt = physical_type();
361 return la ? GetSortOrder(la, pt) : GetSortOrder(converted_type(), pt);
362 }
363
364 const std::string& name() const { return primitive_node_->name(); }
365
366 const std::shared_ptr<schema::ColumnPath> path() const;
367
368 const schema::NodePtr& schema_node() const { return node_; }
369
370 std::string ToString() const;
371
372 int type_length() const;
373
374 int type_precision() const;
375
376 int type_scale() const;
377
378 private:
379 schema::NodePtr node_;
380 const schema::PrimitiveNode* primitive_node_;
381
382 int16_t max_definition_level_;
383 int16_t max_repetition_level_;
384};
385
386// Container for the converted Parquet schema with a computed information from
387// the schema analysis needed for file reading
388//
389// * Column index to Node
390// * Max repetition / definition levels for each primitive node
391//
392// The ColumnDescriptor objects produced by this class can be used to assist in
393// the reconstruction of fully materialized data structures from the
394// repetition-definition level encoding of nested data
395//
396// TODO(wesm): this object can be recomputed from a Schema
397class PARQUET_EXPORT SchemaDescriptor {
398 public:
399 SchemaDescriptor() {}
400 ~SchemaDescriptor() {}
401
402 // Analyze the schema
403 void Init(std::unique_ptr<schema::Node> schema);
404 void Init(const schema::NodePtr& schema);
405
406 const ColumnDescriptor* Column(int i) const;
407
408 // Get the index of a column by its dotstring path, or negative value if not found.
409 // If several columns share the same dotstring path, it is unspecified which one
410 // is returned.
411 int ColumnIndex(const std::string& node_path) const;
412 // Get the index of a column by its node, or negative value if not found.
413 int ColumnIndex(const schema::Node& node) const;
414
415 bool Equals(const SchemaDescriptor& other) const;
416
417 // The number of physical columns appearing in the file
418 int num_columns() const { return static_cast<int>(leaves_.size()); }
419
420 const schema::NodePtr& schema_root() const { return schema_; }
421
422 const schema::GroupNode* group_node() const { return group_node_; }
423
424 // Returns the root (child of the schema root) node of the leaf(column) node
425 const schema::Node* GetColumnRoot(int i) const;
426
427 const std::string& name() const { return group_node_->name(); }
428
429 std::string ToString() const;
430
431 void updateColumnOrders(const std::vector<ColumnOrder>& column_orders);
432
433 /// \brief Return column index corresponding to a particular
434 /// PrimitiveNode. Returns -1 if not found
435 int GetColumnIndex(const schema::PrimitiveNode& node) const;
436
437 private:
438 friend class ColumnDescriptor;
439
440 // Root Node
441 schema::NodePtr schema_;
442 // Root Node
443 const schema::GroupNode* group_node_;
444
445 void BuildTree(const schema::NodePtr& node, int16_t max_def_level,
446 int16_t max_rep_level, const schema::NodePtr& base);
447
448 // Result of leaf node / tree analysis
449 std::vector<ColumnDescriptor> leaves_;
450
451 std::unordered_map<const schema::PrimitiveNode*, int> node_to_leaf_index_;
452
453 // Mapping between leaf nodes and root group of leaf (first node
454 // below the schema's root group)
455 //
456 // For example, the leaf `a.b.c.d` would have a link back to `a`
457 //
458 // -- a <------
459 // -- -- b |
460 // -- -- -- c |
461 // -- -- -- -- d
462 std::unordered_map<int, const schema::NodePtr> leaf_to_base_;
463
464 // Mapping between ColumnPath DotString to the leaf index
465 std::unordered_multimap<std::string, int> leaf_to_idx_;
466};
467
468} // namespace parquet
469
470#endif // PARQUET_SCHEMA_TYPES_H
471