1 | #pragma once |
2 | |
3 | #include <DataStreams/IBlockInputStream.h> |
4 | #include <DataStreams/MarkInCompressedFile.h> |
5 | #include <Common/PODArray.h> |
6 | |
7 | namespace DB |
8 | { |
9 | |
10 | class CompressedReadBufferFromFile; |
11 | |
12 | |
13 | /** The Native format can contain a separately located index, |
14 | * which allows you to understand where what column is located, |
15 | * and skip unnecessary columns. |
16 | */ |
17 | |
18 | /** The position of one piece of a single column. */ |
19 | struct IndexOfOneColumnForNativeFormat |
20 | { |
21 | String name; |
22 | String type; |
23 | MarkInCompressedFile location; |
24 | }; |
25 | |
26 | /** The index for the data block. */ |
27 | struct IndexOfBlockForNativeFormat |
28 | { |
29 | using Columns = std::vector<IndexOfOneColumnForNativeFormat>; |
30 | |
31 | size_t num_columns; |
32 | size_t num_rows; |
33 | Columns columns; |
34 | }; |
35 | |
36 | /** The whole index. */ |
37 | struct IndexForNativeFormat |
38 | { |
39 | using Blocks = std::vector<IndexOfBlockForNativeFormat>; |
40 | Blocks blocks; |
41 | |
42 | IndexForNativeFormat() {} |
43 | |
44 | IndexForNativeFormat(ReadBuffer & istr, const NameSet & required_columns) |
45 | { |
46 | read(istr, required_columns); |
47 | } |
48 | |
49 | /// Read the index, only for the required columns. |
50 | void read(ReadBuffer & istr, const NameSet & required_columns); |
51 | }; |
52 | |
53 | |
54 | /** Deserializes the stream of blocks from the native binary format (with names and column types). |
55 | * Designed for communication between servers. |
56 | * |
57 | * Can also be used to store data on disk. |
58 | * In this case, can use the index. |
59 | */ |
60 | class NativeBlockInputStream : public IBlockInputStream |
61 | { |
62 | public: |
63 | /// If a non-zero server_revision is specified, additional block information may be expected and read. |
64 | NativeBlockInputStream(ReadBuffer & istr_, UInt64 server_revision_); |
65 | |
66 | /// For cases when data structure (header) is known in advance. |
67 | /// NOTE We may use header for data validation and/or type conversions. It is not implemented. |
68 | NativeBlockInputStream(ReadBuffer & istr_, const Block & , UInt64 server_revision_); |
69 | |
70 | /// For cases when we have an index. It allows to skip columns. Only columns specified in the index will be read. |
71 | NativeBlockInputStream(ReadBuffer & istr_, UInt64 server_revision_, |
72 | IndexForNativeFormat::Blocks::const_iterator index_block_it_, |
73 | IndexForNativeFormat::Blocks::const_iterator index_block_end_); |
74 | |
75 | String getName() const override { return "Native" ; } |
76 | |
77 | static void readData(const IDataType & type, IColumn & column, ReadBuffer & istr, size_t rows, double avg_value_size_hint); |
78 | |
79 | Block () const override; |
80 | |
81 | void resetParser(); |
82 | |
83 | |
84 | protected: |
85 | Block readImpl() override; |
86 | |
87 | private: |
88 | ReadBuffer & istr; |
89 | Block ; |
90 | UInt64 server_revision; |
91 | |
92 | bool use_index = false; |
93 | IndexForNativeFormat::Blocks::const_iterator index_block_it; |
94 | IndexForNativeFormat::Blocks::const_iterator index_block_end; |
95 | IndexOfBlockForNativeFormat::Columns::const_iterator index_column_it; |
96 | |
97 | /// If an index is specified, then `istr` must be CompressedReadBufferFromFile. Unused otherwise. |
98 | CompressedReadBufferFromFile * istr_concrete = nullptr; |
99 | |
100 | PODArray<double> avg_value_size_hints; |
101 | |
102 | void updateAvgValueSizeHints(const Block & block); |
103 | }; |
104 | |
105 | } |
106 | |