1 | #include <gtest/gtest.h> |
2 | #include <Core/Block.h> |
3 | #include <Columns/ColumnVector.h> |
4 | |
5 | // I know that inclusion of .cpp is not good at all |
6 | #include <Storages/MergeTree/IMergedBlockOutputStream.cpp> |
7 | |
8 | using namespace DB; |
9 | static Block getBlockWithSize(size_t required_size_in_bytes, size_t size_of_row_in_bytes) |
10 | { |
11 | |
12 | ColumnsWithTypeAndName cols; |
13 | size_t rows = required_size_in_bytes / size_of_row_in_bytes; |
14 | for (size_t i = 0; i < size_of_row_in_bytes; i += sizeof(UInt64)) |
15 | { |
16 | auto column = ColumnUInt64::create(rows, 0); |
17 | cols.emplace_back(std::move(column), std::make_shared<DataTypeUInt64>(), "column" + std::to_string(i)); |
18 | } |
19 | return Block(cols); |
20 | } |
21 | |
22 | TEST(AdaptiveIndexGranularity, FillGranularityToyTests) |
23 | { |
24 | auto block1 = getBlockWithSize(80, 8); |
25 | EXPECT_EQ(block1.bytes(), 80); |
26 | { /// Granularity bytes are not set. Take default index_granularity. |
27 | MergeTreeIndexGranularity index_granularity; |
28 | fillIndexGranularityImpl(block1, 0, 100, false, 0, index_granularity, false); |
29 | EXPECT_EQ(index_granularity.getMarksCount(), 1); |
30 | EXPECT_EQ(index_granularity.getMarkRows(0), 100); |
31 | } |
32 | |
33 | { /// Granule size is less than block size. Block contains multiple granules. |
34 | MergeTreeIndexGranularity index_granularity; |
35 | fillIndexGranularityImpl(block1, 16, 100, false, 0, index_granularity, true); |
36 | EXPECT_EQ(index_granularity.getMarksCount(), 5); /// First granule with 8 rows, and second with 1 row |
37 | for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) |
38 | EXPECT_EQ(index_granularity.getMarkRows(i), 2); |
39 | } |
40 | |
41 | { /// Granule size is more than block size. Whole block (and maybe more) can be placed in single granule. |
42 | |
43 | MergeTreeIndexGranularity index_granularity; |
44 | fillIndexGranularityImpl(block1, 512, 100, false, 0, index_granularity, true); |
45 | EXPECT_EQ(index_granularity.getMarksCount(), 1); |
46 | for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) |
47 | EXPECT_EQ(index_granularity.getMarkRows(i), 64); |
48 | } |
49 | |
50 | { /// Blocks with granule size |
51 | |
52 | MergeTreeIndexGranularity index_granularity; |
53 | fillIndexGranularityImpl(block1, 1, 100, true, 0, index_granularity, true); |
54 | EXPECT_EQ(index_granularity.getMarksCount(), 1); |
55 | for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) |
56 | EXPECT_EQ(index_granularity.getMarkRows(i), block1.rows()); |
57 | } |
58 | |
59 | { /// Shift in index offset |
60 | MergeTreeIndexGranularity index_granularity; |
61 | fillIndexGranularityImpl(block1, 16, 100, false, 6, index_granularity, true); |
62 | EXPECT_EQ(index_granularity.getMarksCount(), 2); |
63 | for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) |
64 | EXPECT_EQ(index_granularity.getMarkRows(i), 2); |
65 | } |
66 | } |
67 | |
68 | |
69 | TEST(AdaptiveIndexGranularity, FillGranularitySequenceOfBlocks) |
70 | { |
71 | { /// Three equal blocks |
72 | auto block1 = getBlockWithSize(65536, 8); |
73 | auto block2 = getBlockWithSize(65536, 8); |
74 | auto block3 = getBlockWithSize(65536, 8); |
75 | MergeTreeIndexGranularity index_granularity; |
76 | for (const auto & block : {block1, block2, block3}) |
77 | fillIndexGranularityImpl(block, 1024, 8192, false, 0, index_granularity, true); |
78 | |
79 | EXPECT_EQ(index_granularity.getMarksCount(), 192); /// granules |
80 | for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) |
81 | EXPECT_EQ(index_granularity.getMarkRows(i), 128); |
82 | } |
83 | { /// Three blocks of different size |
84 | auto block1 = getBlockWithSize(65536, 32); |
85 | auto block2 = getBlockWithSize(32768, 32); |
86 | auto block3 = getBlockWithSize(2048, 32); |
87 | EXPECT_EQ(block1.rows() + block2.rows() + block3.rows(), 3136); |
88 | MergeTreeIndexGranularity index_granularity; |
89 | for (const auto & block : {block1, block2, block3}) |
90 | fillIndexGranularityImpl(block, 1024, 8192, false, 0, index_granularity, true); |
91 | |
92 | EXPECT_EQ(index_granularity.getMarksCount(), 98); /// granules |
93 | for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) |
94 | EXPECT_EQ(index_granularity.getMarkRows(i), 32); |
95 | |
96 | } |
97 | { /// Three small blocks |
98 | auto block1 = getBlockWithSize(2048, 32); |
99 | auto block2 = getBlockWithSize(4096, 32); |
100 | auto block3 = getBlockWithSize(8192, 32); |
101 | |
102 | EXPECT_EQ(block1.rows() + block2.rows() + block3.rows(), (2048 + 4096 + 8192) / 32); |
103 | |
104 | MergeTreeIndexGranularity index_granularity; |
105 | size_t index_offset = 0; |
106 | for (const auto & block : {block1, block2, block3}) |
107 | { |
108 | fillIndexGranularityImpl(block, 16384, 8192, false, index_offset, index_granularity, true); |
109 | index_offset = index_granularity.getLastMarkRows() - block.rows(); |
110 | } |
111 | EXPECT_EQ(index_granularity.getMarksCount(), 1); /// granules |
112 | for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) |
113 | EXPECT_EQ(index_granularity.getMarkRows(i), 512); |
114 | } |
115 | |
116 | } |
117 | |
118 | TEST(AdaptiveIndexGranularity, TestIndexGranularityClass) |
119 | { |
120 | { |
121 | MergeTreeIndexGranularity index_granularity; |
122 | size_t sum_rows = 0; |
123 | size_t sum_marks = 0; |
124 | for (size_t i = 10; i <= 100; i+=10) |
125 | { |
126 | sum_rows += i; |
127 | sum_marks++; |
128 | index_granularity.appendMark(i); |
129 | } |
130 | EXPECT_EQ(index_granularity.getMarksCount(), sum_marks); |
131 | EXPECT_EQ(index_granularity.getTotalRows(), sum_rows); |
132 | EXPECT_EQ(index_granularity.getLastMarkRows(), 100); |
133 | EXPECT_EQ(index_granularity.getMarkStartingRow(0), 0); |
134 | EXPECT_EQ(index_granularity.getMarkStartingRow(1), 10); |
135 | EXPECT_EQ(index_granularity.getMarkStartingRow(2), 30); |
136 | EXPECT_EQ(index_granularity.getMarkStartingRow(3), 60); |
137 | |
138 | EXPECT_EQ(index_granularity.getRowsCountInRange({0, 10}), sum_rows); |
139 | EXPECT_EQ(index_granularity.getRowsCountInRange({0, 1}), 10); |
140 | EXPECT_EQ(index_granularity.getRowsCountInRange({2, 5}), 30 + 40 + 50); |
141 | |
142 | |
143 | EXPECT_EQ(index_granularity.getRowsCountInRanges({{2, 5}, {0, 1}, {0, 10}}), 10 + 30 + 40 + 50 + sum_rows); |
144 | } |
145 | } |
146 | |