1 | #include "StorageSystemParts.h" |
2 | |
3 | #include <Common/escapeForFileName.h> |
4 | #include <Columns/ColumnString.h> |
5 | #include <DataTypes/DataTypeString.h> |
6 | #include <DataTypes/DataTypesNumber.h> |
7 | #include <DataTypes/DataTypeDateTime.h> |
8 | #include <DataTypes/DataTypeDate.h> |
9 | #include <DataStreams/OneBlockInputStream.h> |
10 | #include <Storages/VirtualColumnUtils.h> |
11 | #include <Databases/IDatabase.h> |
12 | #include <Common/hex.h> |
13 | |
14 | namespace DB |
15 | { |
16 | |
17 | StorageSystemParts::StorageSystemParts(const std::string & name_) |
18 | : StorageSystemPartsBase(name_, |
19 | { |
20 | {"partition" , std::make_shared<DataTypeString>()}, |
21 | {"name" , std::make_shared<DataTypeString>()}, |
22 | {"active" , std::make_shared<DataTypeUInt8>()}, |
23 | {"marks" , std::make_shared<DataTypeUInt64>()}, |
24 | {"rows" , std::make_shared<DataTypeUInt64>()}, |
25 | {"bytes_on_disk" , std::make_shared<DataTypeUInt64>()}, |
26 | {"data_compressed_bytes" , std::make_shared<DataTypeUInt64>()}, |
27 | {"data_uncompressed_bytes" , std::make_shared<DataTypeUInt64>()}, |
28 | {"marks_bytes" , std::make_shared<DataTypeUInt64>()}, |
29 | {"modification_time" , std::make_shared<DataTypeDateTime>()}, |
30 | {"remove_time" , std::make_shared<DataTypeDateTime>()}, |
31 | {"refcount" , std::make_shared<DataTypeUInt32>()}, |
32 | {"min_date" , std::make_shared<DataTypeDate>()}, |
33 | {"max_date" , std::make_shared<DataTypeDate>()}, |
34 | {"min_time" , std::make_shared<DataTypeDateTime>()}, |
35 | {"max_time" , std::make_shared<DataTypeDateTime>()}, |
36 | {"partition_id" , std::make_shared<DataTypeString>()}, |
37 | {"min_block_number" , std::make_shared<DataTypeInt64>()}, |
38 | {"max_block_number" , std::make_shared<DataTypeInt64>()}, |
39 | {"level" , std::make_shared<DataTypeUInt32>()}, |
40 | {"data_version" , std::make_shared<DataTypeUInt64>()}, |
41 | {"primary_key_bytes_in_memory" , std::make_shared<DataTypeUInt64>()}, |
42 | {"primary_key_bytes_in_memory_allocated" , std::make_shared<DataTypeUInt64>()}, |
43 | {"is_frozen" , std::make_shared<DataTypeUInt8>()}, |
44 | |
45 | {"database" , std::make_shared<DataTypeString>()}, |
46 | {"table" , std::make_shared<DataTypeString>()}, |
47 | {"engine" , std::make_shared<DataTypeString>()}, |
48 | {"disk_name" , std::make_shared<DataTypeString>()}, |
49 | {"path" , std::make_shared<DataTypeString>()}, |
50 | |
51 | {"hash_of_all_files" , std::make_shared<DataTypeString>()}, |
52 | {"hash_of_uncompressed_files" , std::make_shared<DataTypeString>()}, |
53 | {"uncompressed_hash_of_compressed_files" , std::make_shared<DataTypeString>()} |
54 | } |
55 | ) |
56 | { |
57 | } |
58 | |
59 | void StorageSystemParts::processNextStorage(MutableColumns & columns_, const StoragesInfo & info, bool has_state_column) |
60 | { |
61 | using State = MergeTreeDataPart::State; |
62 | MergeTreeData::DataPartStateVector all_parts_state; |
63 | MergeTreeData::DataPartsVector all_parts; |
64 | |
65 | all_parts = info.getParts(all_parts_state, has_state_column); |
66 | |
67 | for (size_t part_number = 0; part_number < all_parts.size(); ++part_number) |
68 | { |
69 | const auto & part = all_parts[part_number]; |
70 | auto part_state = all_parts_state[part_number]; |
71 | |
72 | ColumnSize columns_size = part->getTotalColumnsSize(); |
73 | |
74 | size_t i = 0; |
75 | { |
76 | WriteBufferFromOwnString out; |
77 | part->partition.serializeText(*info.data, out, format_settings); |
78 | columns_[i++]->insert(out.str()); |
79 | } |
80 | columns_[i++]->insert(part->name); |
81 | columns_[i++]->insert(part_state == State::Committed); |
82 | columns_[i++]->insert(part->getMarksCount()); |
83 | columns_[i++]->insert(part->rows_count); |
84 | columns_[i++]->insert(part->bytes_on_disk.load(std::memory_order_relaxed)); |
85 | columns_[i++]->insert(columns_size.data_compressed); |
86 | columns_[i++]->insert(columns_size.data_uncompressed); |
87 | columns_[i++]->insert(columns_size.marks); |
88 | columns_[i++]->insert(static_cast<UInt64>(part->modification_time)); |
89 | |
90 | time_t remove_time = part->remove_time.load(std::memory_order_relaxed); |
91 | columns_[i++]->insert(static_cast<UInt64>(remove_time == std::numeric_limits<time_t>::max() ? 0 : remove_time)); |
92 | |
93 | /// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts. |
94 | columns_[i++]->insert(static_cast<UInt64>(part.use_count() - 1)); |
95 | |
96 | columns_[i++]->insert(part->getMinDate()); |
97 | columns_[i++]->insert(part->getMaxDate()); |
98 | columns_[i++]->insert(static_cast<UInt32>(part->getMinTime())); |
99 | columns_[i++]->insert(static_cast<UInt32>(part->getMaxTime())); |
100 | columns_[i++]->insert(part->info.partition_id); |
101 | columns_[i++]->insert(part->info.min_block); |
102 | columns_[i++]->insert(part->info.max_block); |
103 | columns_[i++]->insert(part->info.level); |
104 | columns_[i++]->insert(static_cast<UInt64>(part->info.getDataVersion())); |
105 | columns_[i++]->insert(part->getIndexSizeInBytes()); |
106 | columns_[i++]->insert(part->getIndexSizeInAllocatedBytes()); |
107 | columns_[i++]->insert(part->is_frozen.load(std::memory_order_relaxed)); |
108 | |
109 | columns_[i++]->insert(info.database); |
110 | columns_[i++]->insert(info.table); |
111 | columns_[i++]->insert(info.engine); |
112 | columns_[i++]->insert(part->disk->getName()); |
113 | columns_[i++]->insert(part->getFullPath()); |
114 | |
115 | if (has_state_column) |
116 | columns_[i++]->insert(part->stateString()); |
117 | |
118 | MinimalisticDataPartChecksums helper; |
119 | { |
120 | /// TODO: MergeTreeDataPart structure is too error-prone. |
121 | std::shared_lock<std::shared_mutex> lock(part->columns_lock); |
122 | helper.computeTotalChecksums(part->checksums); |
123 | } |
124 | |
125 | auto checksum = helper.hash_of_all_files; |
126 | columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); |
127 | |
128 | checksum = helper.hash_of_uncompressed_files; |
129 | columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); |
130 | |
131 | checksum = helper.uncompressed_hash_of_compressed_files; |
132 | columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second)); |
133 | } |
134 | } |
135 | |
136 | } |
137 | |