1#include <string.h>
2#include <IO/BitHelpers.h>
3
4#include <Core/Types.h>
5#include <IO/MemoryReadWriteBuffer.h>
6#include <IO/ReadBufferFromMemory.h>
7#include <Common/BitHelpers.h>
8#include <Common/PODArray.h>
9
10#include <cmath>
11#include <iomanip>
12#include <memory>
13#include <bitset>
14#include <string>
15#include <vector>
16#include <typeinfo>
17#include <iostream>
18#pragma GCC diagnostic ignored "-Wmissing-declarations"
19#include <gtest/gtest.h>
20
21using namespace DB;
22
23// Intentionally asymmetric both byte and word-size to detect read and write inconsistencies
24// each prime bit is set to 0.
25// v-61 v-53 v-47 v-41 v-37 v-31 v-23 v-17 v-11 v-5
26const UInt64 BIT_PATTERN = 0b11101011'11101111'10111010'11101111'10101111'10111010'11101011'10101001;
27const UInt8 PRIMES[] = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61};
28
29template <typename T>
30std::string bin(const T & value, size_t bits = sizeof(T)*8)
31{
32 static const UInt8 MAX_BITS = sizeof(T)*8;
33 assert(bits <= MAX_BITS);
34
35 return std::bitset<sizeof(T) * 8>(static_cast<unsigned long long>(value))
36 .to_string().substr(MAX_BITS - bits, bits);
37}
38
39template <typename T>
40T getBits(UInt8 bits, const T & value)
41{
42 const T mask = ((static_cast<T>(1) << static_cast<T>(bits)) - 1);
43 return value & mask;
44}
45
46template <typename T>
47std::ostream & dumpBuffer(const T begin,
48 const T end,
49 std::ostream * destination,
50 const char* col_sep = " ",
51 const char* row_sep = "\n",
52 const size_t cols_in_row = 8,
53 UInt32 max_bytes = 0xFFFFFFFF)
54{
55 size_t col = 0;
56 for (auto p = begin; p < end && p - begin < max_bytes; ++p)
57 {
58 *destination << bin(*p);
59 if (++col % cols_in_row == 0)
60 {
61 if (row_sep)
62 *destination << row_sep;
63 }
64 else if (col_sep)
65 {
66 *destination << col_sep;
67 }
68 }
69
70 return *destination;
71}
72
73template <typename T>
74std::string dumpContents(const T& container,
75 const char* col_sep = " ",
76 const char* row_sep = "\n",
77 const size_t cols_in_row = 8)
78
79{
80 std::stringstream sstr;
81 dumpBuffer(std::begin(container), std::end(container), &sstr, col_sep, row_sep, cols_in_row);
82
83 return sstr.str();
84}
85
86struct TestCaseParameter
87{
88 std::vector<std::pair<UInt8, UInt64>> bits_and_vals;
89 std::string expected_buffer_binary;
90
91 explicit TestCaseParameter(std::vector<std::pair<UInt8, UInt64>> vals, std::string binary = std::string{})
92 : bits_and_vals(std::move(vals)),
93 expected_buffer_binary(binary)
94 {}
95};
96
97class BitIO : public ::testing::TestWithParam<TestCaseParameter>
98{};
99
100TEST_P(BitIO, WriteAndRead)
101{
102 const auto & param = GetParam();
103 const auto & bits_and_vals = param.bits_and_vals;
104 const auto & expected_buffer_binary = param.expected_buffer_binary;
105
106 UInt64 max_buffer_size = 0;
107 for (const auto & bv : bits_and_vals)
108 {
109 max_buffer_size += bv.first;
110 }
111 max_buffer_size = (max_buffer_size + 7) / 8;
112 SCOPED_TRACE(max_buffer_size);
113
114 PODArray<char> data(max_buffer_size);
115
116 {
117 WriteBuffer write_buffer(data.data(), data.size());
118 BitWriter writer(write_buffer);
119 for (const auto & bv : bits_and_vals)
120 {
121 writer.writeBits(bv.first, bv.second);
122 }
123 writer.flush();
124 }
125
126 {
127 ReadBufferFromMemory read_buffer(data.data(), data.size());
128// auto memory_read_buffer = memory_write_buffer.tryGetReadBuffer();
129
130 if (expected_buffer_binary != std::string{})
131 {
132 const auto actual_buffer_binary = dumpContents(data, " ", " ");
133 ASSERT_EQ(expected_buffer_binary, actual_buffer_binary);
134 }
135
136 BitReader reader(read_buffer);
137
138 int item = 0;
139 for (const auto & bv : bits_and_vals)
140 {
141 SCOPED_TRACE(::testing::Message()
142 << "item #" << item << ", width: " << static_cast<UInt32>(bv.first)
143 << ", value: " << bin(bv.second)
144 << ".\n\n\nBuffer memory:\n" << dumpContents(data));
145
146 //EXPECT_EQ(getBits(bv.first, bv.second), reader.peekBits(bv.first));
147 EXPECT_EQ(getBits(bv.first, bv.second), reader.readBits(bv.first));
148
149 ++item;
150 }
151 }
152}
153
154INSTANTIATE_TEST_CASE_P(Simple,
155 BitIO,
156 ::testing::Values(
157 TestCaseParameter(
158 {{9, 0xFFFFFFFF}, {9, 0x00}, {9, 0xFFFFFFFF}, {9, 0x00}, {9, 0xFFFFFFFF}},
159 "11111111 10000000 00111111 11100000 00001111 11111000 "),
160 TestCaseParameter(
161 {{7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {7, 0x3f}, {3, 0xFFFF}},
162 "01111110 11111101 11111011 11110111 11101111 11011111 10111111 01111111 11000000 "),
163 TestCaseParameter({{33, 0xFF110d0b07050300}, {33, 0xAAEE29251f1d1713}}),
164 TestCaseParameter({{33, BIT_PATTERN}, {33, BIT_PATTERN}}),
165 TestCaseParameter({{24, 0xFFFFFFFF}},
166 "11111111 11111111 11111111 ")
167),);
168
169TestCaseParameter primes_case(UInt8 repeat_times, UInt64 pattern)
170{
171 std::vector<std::pair<UInt8, UInt64>> test_data;
172
173 {
174 for (UInt8 r = 0; r < repeat_times; ++r)
175 {
176 for (const auto p : PRIMES)
177 {
178 test_data.emplace_back(p, pattern);
179 }
180 }
181 }
182
183 return TestCaseParameter(test_data);
184}
185
186INSTANTIATE_TEST_CASE_P(Primes,
187 BitIO,
188 ::testing::Values(
189 primes_case(11, 0xFFFFFFFFFFFFFFFFULL),
190 primes_case(11, BIT_PATTERN)
191),);
192
193TEST(BitHelpers, maskLowBits)
194{
195 EXPECT_EQ(0b00000111, ::maskLowBits<UInt8>(3));
196 EXPECT_EQ(0b01111111, ::maskLowBits<UInt8>(7));
197 EXPECT_EQ(0b0000000001111111, ::maskLowBits<UInt16>(7));
198 EXPECT_EQ(0b0001111111111111, ::maskLowBits<UInt16>(13));
199 EXPECT_EQ(0b00000111111111111111111111111111, ::maskLowBits<UInt32>(27));
200 EXPECT_EQ(0b111111111111111111111111111111111, ::maskLowBits<UInt64>(33));
201 EXPECT_EQ(0b11111111111111111111111111111111111, ::maskLowBits<UInt64>(35));
202
203 EXPECT_EQ(0xFF, ::maskLowBits<UInt8>(8));
204 EXPECT_EQ(0xFFFF, ::maskLowBits<UInt16>(16));
205 EXPECT_EQ(0xFFFFFFFF, ::maskLowBits<UInt32>(32));
206 EXPECT_EQ(0xFFFFFFFFFFFFFFFF, ::maskLowBits<UInt64>(64));
207}
208