1#include "StorageSystemParts.h"
2
3#include <Common/escapeForFileName.h>
4#include <Columns/ColumnString.h>
5#include <DataTypes/DataTypeString.h>
6#include <DataTypes/DataTypesNumber.h>
7#include <DataTypes/DataTypeDateTime.h>
8#include <DataTypes/DataTypeDate.h>
9#include <DataStreams/OneBlockInputStream.h>
10#include <Storages/VirtualColumnUtils.h>
11#include <Databases/IDatabase.h>
12#include <Common/hex.h>
13
14namespace DB
15{
16
17StorageSystemParts::StorageSystemParts(const std::string & name_)
18 : StorageSystemPartsBase(name_,
19 {
20 {"partition", std::make_shared<DataTypeString>()},
21 {"name", std::make_shared<DataTypeString>()},
22 {"active", std::make_shared<DataTypeUInt8>()},
23 {"marks", std::make_shared<DataTypeUInt64>()},
24 {"rows", std::make_shared<DataTypeUInt64>()},
25 {"bytes_on_disk", std::make_shared<DataTypeUInt64>()},
26 {"data_compressed_bytes", std::make_shared<DataTypeUInt64>()},
27 {"data_uncompressed_bytes", std::make_shared<DataTypeUInt64>()},
28 {"marks_bytes", std::make_shared<DataTypeUInt64>()},
29 {"modification_time", std::make_shared<DataTypeDateTime>()},
30 {"remove_time", std::make_shared<DataTypeDateTime>()},
31 {"refcount", std::make_shared<DataTypeUInt32>()},
32 {"min_date", std::make_shared<DataTypeDate>()},
33 {"max_date", std::make_shared<DataTypeDate>()},
34 {"min_time", std::make_shared<DataTypeDateTime>()},
35 {"max_time", std::make_shared<DataTypeDateTime>()},
36 {"partition_id", std::make_shared<DataTypeString>()},
37 {"min_block_number", std::make_shared<DataTypeInt64>()},
38 {"max_block_number", std::make_shared<DataTypeInt64>()},
39 {"level", std::make_shared<DataTypeUInt32>()},
40 {"data_version", std::make_shared<DataTypeUInt64>()},
41 {"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
42 {"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
43 {"is_frozen", std::make_shared<DataTypeUInt8>()},
44
45 {"database", std::make_shared<DataTypeString>()},
46 {"table", std::make_shared<DataTypeString>()},
47 {"engine", std::make_shared<DataTypeString>()},
48 {"disk_name", std::make_shared<DataTypeString>()},
49 {"path", std::make_shared<DataTypeString>()},
50
51 {"hash_of_all_files", std::make_shared<DataTypeString>()},
52 {"hash_of_uncompressed_files", std::make_shared<DataTypeString>()},
53 {"uncompressed_hash_of_compressed_files", std::make_shared<DataTypeString>()}
54 }
55 )
56{
57}
58
59void StorageSystemParts::processNextStorage(MutableColumns & columns_, const StoragesInfo & info, bool has_state_column)
60{
61 using State = MergeTreeDataPart::State;
62 MergeTreeData::DataPartStateVector all_parts_state;
63 MergeTreeData::DataPartsVector all_parts;
64
65 all_parts = info.getParts(all_parts_state, has_state_column);
66
67 for (size_t part_number = 0; part_number < all_parts.size(); ++part_number)
68 {
69 const auto & part = all_parts[part_number];
70 auto part_state = all_parts_state[part_number];
71
72 ColumnSize columns_size = part->getTotalColumnsSize();
73
74 size_t i = 0;
75 {
76 WriteBufferFromOwnString out;
77 part->partition.serializeText(*info.data, out, format_settings);
78 columns_[i++]->insert(out.str());
79 }
80 columns_[i++]->insert(part->name);
81 columns_[i++]->insert(part_state == State::Committed);
82 columns_[i++]->insert(part->getMarksCount());
83 columns_[i++]->insert(part->rows_count);
84 columns_[i++]->insert(part->bytes_on_disk.load(std::memory_order_relaxed));
85 columns_[i++]->insert(columns_size.data_compressed);
86 columns_[i++]->insert(columns_size.data_uncompressed);
87 columns_[i++]->insert(columns_size.marks);
88 columns_[i++]->insert(static_cast<UInt64>(part->modification_time));
89
90 time_t remove_time = part->remove_time.load(std::memory_order_relaxed);
91 columns_[i++]->insert(static_cast<UInt64>(remove_time == std::numeric_limits<time_t>::max() ? 0 : remove_time));
92
93 /// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
94 columns_[i++]->insert(static_cast<UInt64>(part.use_count() - 1));
95
96 columns_[i++]->insert(part->getMinDate());
97 columns_[i++]->insert(part->getMaxDate());
98 columns_[i++]->insert(static_cast<UInt32>(part->getMinTime()));
99 columns_[i++]->insert(static_cast<UInt32>(part->getMaxTime()));
100 columns_[i++]->insert(part->info.partition_id);
101 columns_[i++]->insert(part->info.min_block);
102 columns_[i++]->insert(part->info.max_block);
103 columns_[i++]->insert(part->info.level);
104 columns_[i++]->insert(static_cast<UInt64>(part->info.getDataVersion()));
105 columns_[i++]->insert(part->getIndexSizeInBytes());
106 columns_[i++]->insert(part->getIndexSizeInAllocatedBytes());
107 columns_[i++]->insert(part->is_frozen.load(std::memory_order_relaxed));
108
109 columns_[i++]->insert(info.database);
110 columns_[i++]->insert(info.table);
111 columns_[i++]->insert(info.engine);
112 columns_[i++]->insert(part->disk->getName());
113 columns_[i++]->insert(part->getFullPath());
114
115 if (has_state_column)
116 columns_[i++]->insert(part->stateString());
117
118 MinimalisticDataPartChecksums helper;
119 {
120 /// TODO: MergeTreeDataPart structure is too error-prone.
121 std::shared_lock<std::shared_mutex> lock(part->columns_lock);
122 helper.computeTotalChecksums(part->checksums);
123 }
124
125 auto checksum = helper.hash_of_all_files;
126 columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
127
128 checksum = helper.hash_of_uncompressed_files;
129 columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
130
131 checksum = helper.uncompressed_hash_of_compressed_files;
132 columns_[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
133 }
134}
135
136}
137