1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// This module contains the logical parquet-cpp types (independent of Thrift
19// structures), schema nodes, and related type tools
20
21#ifndef PARQUET_SCHEMA_TYPES_H
22#define PARQUET_SCHEMA_TYPES_H
23
24#include <cstdint>
25#include <memory>
26#include <ostream>
27#include <string>
28#include <unordered_map>
29#include <vector>
30
31#include "parquet/types.h"
32#include "parquet/util/macros.h"
33#include "parquet/util/visibility.h"
34
35namespace parquet {
36
37class SchemaDescriptor;
38
39namespace schema {
40
41class Node;
42
43// List encodings: using the terminology from Impala to define different styles
44// of representing logical lists (a.k.a. ARRAY types) in Parquet schemas. Since
45// the converted type named in the Parquet metadata is ConvertedType::LIST we
46// use that terminology here. It also helps distinguish from the *_ARRAY
47// primitive types.
48//
49// One-level encoding: Only allows required lists with required cells
50// repeated value_type name
51//
52// Two-level encoding: Enables optional lists with only required cells
53// <required/optional> group list
54// repeated value_type item
55//
56// Three-level encoding: Enables optional lists with optional cells
57// <required/optional> group bag
58// repeated group list
59// <required/optional> value_type item
60//
61// 2- and 1-level encoding are respectively equivalent to 3-level encoding with
62// the non-repeated nodes set to required.
63//
64// The "official" encoding recommended in the Parquet spec is the 3-level, and
65// we use that as the default when creating list types. For semantic completeness
66// we allow the other two. Since all types of encodings will occur "in the
67// wild" we need to be able to interpret the associated definition levels in
68// the context of the actual encoding used in the file.
69//
70// NB: Some Parquet writers may not set ConvertedType::LIST on the repeated
71// SchemaElement, which could make things challenging if we are trying to infer
72// that a sequence of nodes semantically represents an array according to one
73// of these encodings (versus a struct containing an array). We should refuse
74// the temptation to guess, as they say.
75struct ListEncoding {
76 enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL };
77};
78
79struct DecimalMetadata {
80 bool isset;
81 int32_t scale;
82 int32_t precision;
83};
84
85class PARQUET_EXPORT ColumnPath {
86 public:
87 ColumnPath() : path_() {}
88 explicit ColumnPath(const std::vector<std::string>& path) : path_(path) {}
89 explicit ColumnPath(std::vector<std::string>&& path) : path_(path) {}
90
91 static std::shared_ptr<ColumnPath> FromDotString(const std::string& dotstring);
92 static std::shared_ptr<ColumnPath> FromNode(const Node& node);
93
94 std::shared_ptr<ColumnPath> extend(const std::string& node_name) const;
95 std::string ToDotString() const;
96 const std::vector<std::string>& ToDotVector() const;
97
98 protected:
99 std::vector<std::string> path_;
100};
101
102class GroupNode;
103
104// Base class for logical schema types. A type has a name, repetition level,
105// and optionally a logical type (ConvertedType in Parquet metadata parlance)
106class PARQUET_EXPORT Node {
107 public:
108 enum type { PRIMITIVE, GROUP };
109
110 Node(Node::type type, const std::string& name, Repetition::type repetition,
111 LogicalType::type logical_type = LogicalType::NONE, int id = -1)
112 : type_(type),
113 name_(name),
114 repetition_(repetition),
115 logical_type_(logical_type),
116 id_(id),
117 parent_(NULLPTR) {}
118
119 virtual ~Node() {}
120
121 bool is_primitive() const { return type_ == Node::PRIMITIVE; }
122
123 bool is_group() const { return type_ == Node::GROUP; }
124
125 bool is_optional() const { return repetition_ == Repetition::OPTIONAL; }
126
127 bool is_repeated() const { return repetition_ == Repetition::REPEATED; }
128
129 bool is_required() const { return repetition_ == Repetition::REQUIRED; }
130
131 virtual bool Equals(const Node* other) const = 0;
132
133 const std::string& name() const { return name_; }
134
135 Node::type node_type() const { return type_; }
136
137 Repetition::type repetition() const { return repetition_; }
138
139 LogicalType::type logical_type() const { return logical_type_; }
140
141 int id() const { return id_; }
142
143 const Node* parent() const { return parent_; }
144
145 const std::shared_ptr<ColumnPath> path() const;
146
147 // ToParquet returns an opaque void* to avoid exporting
148 // parquet::SchemaElement into the public API
149 virtual void ToParquet(void* opaque_element) const = 0;
150
151 // Node::Visitor abstract class for walking schemas with the visitor pattern
152 class Visitor {
153 public:
154 virtual ~Visitor() {}
155
156 virtual void Visit(Node* node) = 0;
157 };
158 class ConstVisitor {
159 public:
160 virtual ~ConstVisitor() {}
161
162 virtual void Visit(const Node* node) = 0;
163 };
164
165 virtual void Visit(Visitor* visitor) = 0;
166 virtual void VisitConst(ConstVisitor* visitor) const = 0;
167
168 protected:
169 friend class GroupNode;
170
171 Node::type type_;
172 std::string name_;
173 Repetition::type repetition_;
174 LogicalType::type logical_type_;
175 int id_;
176 // Nodes should not be shared, they have a single parent.
177 const Node* parent_;
178
179 bool EqualsInternal(const Node* other) const;
180 void SetParent(const Node* p_parent);
181
182 private:
183 PARQUET_DISALLOW_COPY_AND_ASSIGN(Node);
184};
185
186// Save our breath all over the place with these typedefs
187typedef std::shared_ptr<Node> NodePtr;
188typedef std::vector<NodePtr> NodeVector;
189
190// A type that is one of the primitive Parquet storage types. In addition to
191// the other type metadata (name, repetition level, logical type), also has the
192// physical storage type and their type-specific metadata (byte width, decimal
193// parameters)
194class PARQUET_EXPORT PrimitiveNode : public Node {
195 public:
196 // FromParquet accepts an opaque void* to avoid exporting
197 // parquet::SchemaElement into the public API
198 static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id);
199
200 static inline NodePtr Make(const std::string& name, Repetition::type repetition,
201 Type::type type,
202 LogicalType::type logical_type = LogicalType::NONE,
203 int length = -1, int precision = -1, int scale = -1) {
204 return NodePtr(new PrimitiveNode(name, repetition, type, logical_type, length,
205 precision, scale));
206 }
207
208 bool Equals(const Node* other) const override;
209
210 Type::type physical_type() const { return physical_type_; }
211
212 ColumnOrder column_order() const { return column_order_; }
213
214 void SetColumnOrder(ColumnOrder column_order) { column_order_ = column_order; }
215
216 int32_t type_length() const { return type_length_; }
217
218 const DecimalMetadata& decimal_metadata() const { return decimal_metadata_; }
219
220 void ToParquet(void* opaque_element) const override;
221 void Visit(Visitor* visitor) override;
222 void VisitConst(ConstVisitor* visitor) const override;
223
224 private:
225 PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type,
226 LogicalType::type logical_type = LogicalType::NONE, int length = -1,
227 int precision = -1, int scale = -1, int id = -1);
228
229 Type::type physical_type_;
230 int32_t type_length_;
231 DecimalMetadata decimal_metadata_;
232 ColumnOrder column_order_;
233
234 // For FIXED_LEN_BYTE_ARRAY
235 void SetTypeLength(int32_t length) { type_length_ = length; }
236
237 // For Decimal logical type: Precision and scale
238 void SetDecimalMetadata(int32_t scale, int32_t precision) {
239 decimal_metadata_.scale = scale;
240 decimal_metadata_.precision = precision;
241 }
242
243 bool EqualsInternal(const PrimitiveNode* other) const;
244
245 FRIEND_TEST(TestPrimitiveNode, Attrs);
246 FRIEND_TEST(TestPrimitiveNode, Equals);
247 FRIEND_TEST(TestPrimitiveNode, PhysicalLogicalMapping);
248 FRIEND_TEST(TestPrimitiveNode, FromParquet);
249};
250
251class PARQUET_EXPORT GroupNode : public Node {
252 public:
253 // Like PrimitiveNode, GroupNode::FromParquet accepts an opaque void* to avoid exporting
254 // parquet::SchemaElement into the public API
255 static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id,
256 const NodeVector& fields);
257
258 static inline NodePtr Make(const std::string& name, Repetition::type repetition,
259 const NodeVector& fields,
260 LogicalType::type logical_type = LogicalType::NONE) {
261 return NodePtr(new GroupNode(name, repetition, fields, logical_type));
262 }
263
264 bool Equals(const Node* other) const override;
265
266 NodePtr field(int i) const { return fields_[i]; }
267 // Get the index of a field by its name, or negative value if not found.
268 // If several fields share the same name, it is unspecified which one
269 // is returned.
270 int FieldIndex(const std::string& name) const;
271 // Get the index of a field by its node, or negative value if not found.
272 int FieldIndex(const Node& node) const;
273
274 int field_count() const { return static_cast<int>(fields_.size()); }
275
276 void ToParquet(void* opaque_element) const override;
277 void Visit(Visitor* visitor) override;
278 void VisitConst(ConstVisitor* visitor) const override;
279
280 private:
281 GroupNode(const std::string& name, Repetition::type repetition,
282 const NodeVector& fields, LogicalType::type logical_type = LogicalType::NONE,
283 int id = -1)
284 : Node(Node::GROUP, name, repetition, logical_type, id), fields_(fields) {
285 field_name_to_idx_.clear();
286 auto field_idx = 0;
287 for (NodePtr& field : fields_) {
288 field->SetParent(this);
289 field_name_to_idx_.emplace(field->name(), field_idx++);
290 }
291 }
292
293 NodeVector fields_;
294 bool EqualsInternal(const GroupNode* other) const;
295
296 // Mapping between field name to the field index
297 std::unordered_multimap<std::string, int> field_name_to_idx_;
298
299 FRIEND_TEST(TestGroupNode, Attrs);
300 FRIEND_TEST(TestGroupNode, Equals);
301 FRIEND_TEST(TestGroupNode, FieldIndex);
302 FRIEND_TEST(TestGroupNode, FieldIndexDuplicateName);
303};
304
305// ----------------------------------------------------------------------
306// Convenience primitive type factory functions
307
308#define PRIMITIVE_FACTORY(FuncName, TYPE) \
309 static inline NodePtr FuncName(const std::string& name, \
310 Repetition::type repetition = Repetition::OPTIONAL) { \
311 return PrimitiveNode::Make(name, repetition, Type::TYPE); \
312 }
313
314PRIMITIVE_FACTORY(Boolean, BOOLEAN);
315PRIMITIVE_FACTORY(Int32, INT32);
316PRIMITIVE_FACTORY(Int64, INT64);
317PRIMITIVE_FACTORY(Int96, INT96);
318PRIMITIVE_FACTORY(Float, FLOAT);
319PRIMITIVE_FACTORY(Double, DOUBLE);
320PRIMITIVE_FACTORY(ByteArray, BYTE_ARRAY);
321
322void PARQUET_EXPORT PrintSchema(const schema::Node* schema, std::ostream& stream,
323 int indent_width = 2);
324
325} // namespace schema
326
327// The ColumnDescriptor encapsulates information necessary to interpret
328// primitive column data in the context of a particular schema. We have to
329// examine the node structure of a column's path to the root in the schema tree
330// to be able to reassemble the nested structure from the repetition and
331// definition levels.
332class PARQUET_EXPORT ColumnDescriptor {
333 public:
334 ColumnDescriptor(const schema::NodePtr& node, int16_t max_definition_level,
335 int16_t max_repetition_level,
336 const SchemaDescriptor* schema_descr = NULLPTR);
337
338 bool Equals(const ColumnDescriptor& other) const;
339
340 int16_t max_definition_level() const { return max_definition_level_; }
341
342 int16_t max_repetition_level() const { return max_repetition_level_; }
343
344 Type::type physical_type() const { return primitive_node_->physical_type(); }
345
346 LogicalType::type logical_type() const { return primitive_node_->logical_type(); }
347
348 ColumnOrder column_order() const { return primitive_node_->column_order(); }
349
350 SortOrder::type sort_order() const {
351 return GetSortOrder(logical_type(), physical_type());
352 }
353
354 const std::string& name() const { return primitive_node_->name(); }
355
356 const std::shared_ptr<schema::ColumnPath> path() const;
357
358 const schema::NodePtr& schema_node() const { return node_; }
359
360 int type_length() const;
361
362 int type_precision() const;
363
364 int type_scale() const;
365
366 private:
367 schema::NodePtr node_;
368 const schema::PrimitiveNode* primitive_node_;
369
370 int16_t max_definition_level_;
371 int16_t max_repetition_level_;
372};
373
374// Container for the converted Parquet schema with a computed information from
375// the schema analysis needed for file reading
376//
377// * Column index to Node
378// * Max repetition / definition levels for each primitive node
379//
380// The ColumnDescriptor objects produced by this class can be used to assist in
381// the reconstruction of fully materialized data structures from the
382// repetition-definition level encoding of nested data
383//
384// TODO(wesm): this object can be recomputed from a Schema
385class PARQUET_EXPORT SchemaDescriptor {
386 public:
387 SchemaDescriptor() {}
388 ~SchemaDescriptor() {}
389
390 // Analyze the schema
391 void Init(std::unique_ptr<schema::Node> schema);
392 void Init(const schema::NodePtr& schema);
393
394 const ColumnDescriptor* Column(int i) const;
395
396 // Get the index of a column by its dotstring path, or negative value if not found.
397 // If several columns share the same dotstring path, it is unspecified which one
398 // is returned.
399 int ColumnIndex(const std::string& node_path) const;
400 // Get the index of a column by its node, or negative value if not found.
401 int ColumnIndex(const schema::Node& node) const;
402
403 bool Equals(const SchemaDescriptor& other) const;
404
405 // The number of physical columns appearing in the file
406 int num_columns() const { return static_cast<int>(leaves_.size()); }
407
408 const schema::NodePtr& schema_root() const { return schema_; }
409
410 const schema::GroupNode* group_node() const { return group_node_; }
411
412 // Returns the root (child of the schema root) node of the leaf(column) node
413 const schema::Node* GetColumnRoot(int i) const;
414
415 const std::string& name() const { return group_node_->name(); }
416
417 std::string ToString() const;
418
419 void updateColumnOrders(const std::vector<ColumnOrder>& column_orders);
420
421 private:
422 friend class ColumnDescriptor;
423
424 // Root Node
425 schema::NodePtr schema_;
426 // Root Node
427 const schema::GroupNode* group_node_;
428
429 void BuildTree(const schema::NodePtr& node, int16_t max_def_level,
430 int16_t max_rep_level, const schema::NodePtr& base);
431
432 // Result of leaf node / tree analysis
433 std::vector<ColumnDescriptor> leaves_;
434
435 // Mapping between leaf nodes and root group of leaf (first node
436 // below the schema's root group)
437 //
438 // For example, the leaf `a.b.c.d` would have a link back to `a`
439 //
440 // -- a <------
441 // -- -- b |
442 // -- -- -- c |
443 // -- -- -- -- d
444 std::unordered_map<int, const schema::NodePtr> leaf_to_base_;
445
446 // Mapping between ColumnPath DotString to the leaf index
447 std::unordered_multimap<std::string, int> leaf_to_idx_;
448};
449
450} // namespace parquet
451
452#endif // PARQUET_SCHEMA_TYPES_H
453