1#include <gtest/gtest.h>
2#include <Core/Block.h>
3#include <Columns/ColumnVector.h>
4
5// I know that inclusion of .cpp is not good at all
6#include <Storages/MergeTree/IMergedBlockOutputStream.cpp>
7
8using namespace DB;
9static Block getBlockWithSize(size_t required_size_in_bytes, size_t size_of_row_in_bytes)
10{
11
12 ColumnsWithTypeAndName cols;
13 size_t rows = required_size_in_bytes / size_of_row_in_bytes;
14 for (size_t i = 0; i < size_of_row_in_bytes; i += sizeof(UInt64))
15 {
16 auto column = ColumnUInt64::create(rows, 0);
17 cols.emplace_back(std::move(column), std::make_shared<DataTypeUInt64>(), "column" + std::to_string(i));
18 }
19 return Block(cols);
20}
21
22TEST(AdaptiveIndexGranularity, FillGranularityToyTests)
23{
24 auto block1 = getBlockWithSize(80, 8);
25 EXPECT_EQ(block1.bytes(), 80);
26 { /// Granularity bytes are not set. Take default index_granularity.
27 MergeTreeIndexGranularity index_granularity;
28 fillIndexGranularityImpl(block1, 0, 100, false, 0, index_granularity, false);
29 EXPECT_EQ(index_granularity.getMarksCount(), 1);
30 EXPECT_EQ(index_granularity.getMarkRows(0), 100);
31 }
32
33 { /// Granule size is less than block size. Block contains multiple granules.
34 MergeTreeIndexGranularity index_granularity;
35 fillIndexGranularityImpl(block1, 16, 100, false, 0, index_granularity, true);
36 EXPECT_EQ(index_granularity.getMarksCount(), 5); /// First granule with 8 rows, and second with 1 row
37 for (size_t i = 0; i < index_granularity.getMarksCount(); ++i)
38 EXPECT_EQ(index_granularity.getMarkRows(i), 2);
39 }
40
41 { /// Granule size is more than block size. Whole block (and maybe more) can be placed in single granule.
42
43 MergeTreeIndexGranularity index_granularity;
44 fillIndexGranularityImpl(block1, 512, 100, false, 0, index_granularity, true);
45 EXPECT_EQ(index_granularity.getMarksCount(), 1);
46 for (size_t i = 0; i < index_granularity.getMarksCount(); ++i)
47 EXPECT_EQ(index_granularity.getMarkRows(i), 64);
48 }
49
50 { /// Blocks with granule size
51
52 MergeTreeIndexGranularity index_granularity;
53 fillIndexGranularityImpl(block1, 1, 100, true, 0, index_granularity, true);
54 EXPECT_EQ(index_granularity.getMarksCount(), 1);
55 for (size_t i = 0; i < index_granularity.getMarksCount(); ++i)
56 EXPECT_EQ(index_granularity.getMarkRows(i), block1.rows());
57 }
58
59 { /// Shift in index offset
60 MergeTreeIndexGranularity index_granularity;
61 fillIndexGranularityImpl(block1, 16, 100, false, 6, index_granularity, true);
62 EXPECT_EQ(index_granularity.getMarksCount(), 2);
63 for (size_t i = 0; i < index_granularity.getMarksCount(); ++i)
64 EXPECT_EQ(index_granularity.getMarkRows(i), 2);
65 }
66}
67
68
69TEST(AdaptiveIndexGranularity, FillGranularitySequenceOfBlocks)
70{
71 { /// Three equal blocks
72 auto block1 = getBlockWithSize(65536, 8);
73 auto block2 = getBlockWithSize(65536, 8);
74 auto block3 = getBlockWithSize(65536, 8);
75 MergeTreeIndexGranularity index_granularity;
76 for (const auto & block : {block1, block2, block3})
77 fillIndexGranularityImpl(block, 1024, 8192, false, 0, index_granularity, true);
78
79 EXPECT_EQ(index_granularity.getMarksCount(), 192); /// granules
80 for (size_t i = 0; i < index_granularity.getMarksCount(); ++i)
81 EXPECT_EQ(index_granularity.getMarkRows(i), 128);
82 }
83 { /// Three blocks of different size
84 auto block1 = getBlockWithSize(65536, 32);
85 auto block2 = getBlockWithSize(32768, 32);
86 auto block3 = getBlockWithSize(2048, 32);
87 EXPECT_EQ(block1.rows() + block2.rows() + block3.rows(), 3136);
88 MergeTreeIndexGranularity index_granularity;
89 for (const auto & block : {block1, block2, block3})
90 fillIndexGranularityImpl(block, 1024, 8192, false, 0, index_granularity, true);
91
92 EXPECT_EQ(index_granularity.getMarksCount(), 98); /// granules
93 for (size_t i = 0; i < index_granularity.getMarksCount(); ++i)
94 EXPECT_EQ(index_granularity.getMarkRows(i), 32);
95
96 }
97 { /// Three small blocks
98 auto block1 = getBlockWithSize(2048, 32);
99 auto block2 = getBlockWithSize(4096, 32);
100 auto block3 = getBlockWithSize(8192, 32);
101
102 EXPECT_EQ(block1.rows() + block2.rows() + block3.rows(), (2048 + 4096 + 8192) / 32);
103
104 MergeTreeIndexGranularity index_granularity;
105 size_t index_offset = 0;
106 for (const auto & block : {block1, block2, block3})
107 {
108 fillIndexGranularityImpl(block, 16384, 8192, false, index_offset, index_granularity, true);
109 index_offset = index_granularity.getLastMarkRows() - block.rows();
110 }
111 EXPECT_EQ(index_granularity.getMarksCount(), 1); /// granules
112 for (size_t i = 0; i < index_granularity.getMarksCount(); ++i)
113 EXPECT_EQ(index_granularity.getMarkRows(i), 512);
114 }
115
116}
117
118TEST(AdaptiveIndexGranularity, TestIndexGranularityClass)
119{
120 {
121 MergeTreeIndexGranularity index_granularity;
122 size_t sum_rows = 0;
123 size_t sum_marks = 0;
124 for (size_t i = 10; i <= 100; i+=10)
125 {
126 sum_rows += i;
127 sum_marks++;
128 index_granularity.appendMark(i);
129 }
130 EXPECT_EQ(index_granularity.getMarksCount(), sum_marks);
131 EXPECT_EQ(index_granularity.getTotalRows(), sum_rows);
132 EXPECT_EQ(index_granularity.getLastMarkRows(), 100);
133 EXPECT_EQ(index_granularity.getMarkStartingRow(0), 0);
134 EXPECT_EQ(index_granularity.getMarkStartingRow(1), 10);
135 EXPECT_EQ(index_granularity.getMarkStartingRow(2), 30);
136 EXPECT_EQ(index_granularity.getMarkStartingRow(3), 60);
137
138 EXPECT_EQ(index_granularity.getRowsCountInRange({0, 10}), sum_rows);
139 EXPECT_EQ(index_granularity.getRowsCountInRange({0, 1}), 10);
140 EXPECT_EQ(index_granularity.getRowsCountInRange({2, 5}), 30 + 40 + 50);
141
142
143 EXPECT_EQ(index_granularity.getRowsCountInRanges({{2, 5}, {0, 1}, {0, 10}}), 10 + 30 + 40 + 50 + sum_rows);
144 }
145}
146