1#pragma once
2
3#include <DataStreams/IBlockInputStream.h>
4#include <DataStreams/MarkInCompressedFile.h>
5#include <Common/PODArray.h>
6
7namespace DB
8{
9
10class CompressedReadBufferFromFile;
11
12
13/** The Native format can contain a separately located index,
14 * which allows you to understand where what column is located,
15 * and skip unnecessary columns.
16 */
17
18/** The position of one piece of a single column. */
19struct IndexOfOneColumnForNativeFormat
20{
21 String name;
22 String type;
23 MarkInCompressedFile location;
24};
25
26/** The index for the data block. */
27struct IndexOfBlockForNativeFormat
28{
29 using Columns = std::vector<IndexOfOneColumnForNativeFormat>;
30
31 size_t num_columns;
32 size_t num_rows;
33 Columns columns;
34};
35
36/** The whole index. */
37struct IndexForNativeFormat
38{
39 using Blocks = std::vector<IndexOfBlockForNativeFormat>;
40 Blocks blocks;
41
42 IndexForNativeFormat() {}
43
44 IndexForNativeFormat(ReadBuffer & istr, const NameSet & required_columns)
45 {
46 read(istr, required_columns);
47 }
48
49 /// Read the index, only for the required columns.
50 void read(ReadBuffer & istr, const NameSet & required_columns);
51};
52
53
54/** Deserializes the stream of blocks from the native binary format (with names and column types).
55 * Designed for communication between servers.
56 *
57 * Can also be used to store data on disk.
58 * In this case, can use the index.
59 */
60class NativeBlockInputStream : public IBlockInputStream
61{
62public:
63 /// If a non-zero server_revision is specified, additional block information may be expected and read.
64 NativeBlockInputStream(ReadBuffer & istr_, UInt64 server_revision_);
65
66 /// For cases when data structure (header) is known in advance.
67 /// NOTE We may use header for data validation and/or type conversions. It is not implemented.
68 NativeBlockInputStream(ReadBuffer & istr_, const Block & header_, UInt64 server_revision_);
69
70 /// For cases when we have an index. It allows to skip columns. Only columns specified in the index will be read.
71 NativeBlockInputStream(ReadBuffer & istr_, UInt64 server_revision_,
72 IndexForNativeFormat::Blocks::const_iterator index_block_it_,
73 IndexForNativeFormat::Blocks::const_iterator index_block_end_);
74
75 String getName() const override { return "Native"; }
76
77 static void readData(const IDataType & type, IColumn & column, ReadBuffer & istr, size_t rows, double avg_value_size_hint);
78
79 Block getHeader() const override;
80
81 void resetParser();
82
83
84protected:
85 Block readImpl() override;
86
87private:
88 ReadBuffer & istr;
89 Block header;
90 UInt64 server_revision;
91
92 bool use_index = false;
93 IndexForNativeFormat::Blocks::const_iterator index_block_it;
94 IndexForNativeFormat::Blocks::const_iterator index_block_end;
95 IndexOfBlockForNativeFormat::Columns::const_iterator index_column_it;
96
97 /// If an index is specified, then `istr` must be CompressedReadBufferFromFile. Unused otherwise.
98 CompressedReadBufferFromFile * istr_concrete = nullptr;
99
100 PODArray<double> avg_value_size_hints;
101
102 void updateAvgValueSizeHints(const Block & block);
103};
104
105}
106