1 | #include "DiskLocal.h" |
2 | #include "DiskFactory.h" |
3 | |
4 | #include <Interpreters/Context.h> |
5 | #include <Common/filesystemHelpers.h> |
6 | #include <Common/quoteString.h> |
7 | |
8 | |
9 | namespace DB |
10 | { |
11 | std::mutex DiskLocal::mutex; |
12 | |
13 | ReservationPtr DiskLocal::reserve(UInt64 bytes) |
14 | { |
15 | if (!tryReserve(bytes)) |
16 | return {}; |
17 | return std::make_unique<DiskLocalReservation>(std::static_pointer_cast<DiskLocal>(shared_from_this()), bytes); |
18 | } |
19 | |
20 | bool DiskLocal::tryReserve(UInt64 bytes) |
21 | { |
22 | std::lock_guard lock(mutex); |
23 | if (bytes == 0) |
24 | { |
25 | LOG_DEBUG(&Logger::get("DiskLocal" ), "Reserving 0 bytes on disk " << backQuote(name)); |
26 | ++reservation_count; |
27 | return true; |
28 | } |
29 | |
30 | auto available_space = getAvailableSpace(); |
31 | UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes); |
32 | if (unreserved_space >= bytes) |
33 | { |
34 | LOG_DEBUG( |
35 | &Logger::get("DiskLocal" ), |
36 | "Reserving " << formatReadableSizeWithBinarySuffix(bytes) << " on disk " << backQuote(name) << ", having unreserved " |
37 | << formatReadableSizeWithBinarySuffix(unreserved_space) << "." ); |
38 | ++reservation_count; |
39 | reserved_bytes += bytes; |
40 | return true; |
41 | } |
42 | return false; |
43 | } |
44 | |
45 | UInt64 DiskLocal::getTotalSpace() const |
46 | { |
47 | auto fs = getStatVFS(disk_path); |
48 | UInt64 total_size = fs.f_blocks * fs.f_bsize; |
49 | if (total_size < keep_free_space_bytes) |
50 | return 0; |
51 | return total_size - keep_free_space_bytes; |
52 | } |
53 | |
54 | UInt64 DiskLocal::getAvailableSpace() const |
55 | { |
56 | /// we use f_bavail, because part of b_free space is |
57 | /// available for superuser only and for system purposes |
58 | auto fs = getStatVFS(disk_path); |
59 | UInt64 total_size = fs.f_bavail * fs.f_bsize; |
60 | if (total_size < keep_free_space_bytes) |
61 | return 0; |
62 | return total_size - keep_free_space_bytes; |
63 | } |
64 | |
65 | UInt64 DiskLocal::getUnreservedSpace() const |
66 | { |
67 | std::lock_guard lock(mutex); |
68 | auto available_space = getAvailableSpace(); |
69 | available_space -= std::min(available_space, reserved_bytes); |
70 | return available_space; |
71 | } |
72 | |
73 | bool DiskLocal::exists(const String & path) const |
74 | { |
75 | return Poco::File(disk_path + path).exists(); |
76 | } |
77 | |
78 | bool DiskLocal::isFile(const String & path) const |
79 | { |
80 | return Poco::File(disk_path + path).isFile(); |
81 | } |
82 | |
83 | bool DiskLocal::isDirectory(const String & path) const |
84 | { |
85 | return Poco::File(disk_path + path).isDirectory(); |
86 | } |
87 | |
88 | void DiskLocal::createDirectory(const String & path) |
89 | { |
90 | Poco::File(disk_path + path).createDirectory(); |
91 | } |
92 | |
93 | void DiskLocal::createDirectories(const String & path) |
94 | { |
95 | Poco::File(disk_path + path).createDirectories(); |
96 | } |
97 | |
98 | DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) |
99 | { |
100 | return std::make_unique<DiskLocalDirectoryIterator>(disk_path + path); |
101 | } |
102 | |
103 | void DiskLocal::moveFile(const String & from_path, const String & to_path) |
104 | { |
105 | Poco::File(disk_path + from_path).renameTo(disk_path + to_path); |
106 | } |
107 | |
108 | void DiskLocal::copyFile(const String & from_path, const String & to_path) |
109 | { |
110 | Poco::File(disk_path + from_path).copyTo(disk_path + to_path); |
111 | } |
112 | |
113 | std::unique_ptr<ReadBuffer> DiskLocal::readFile(const String & path) const |
114 | { |
115 | return std::make_unique<ReadBufferFromFile>(disk_path + path); |
116 | } |
117 | |
118 | std::unique_ptr<WriteBuffer> DiskLocal::writeFile(const String & path) |
119 | { |
120 | return std::make_unique<WriteBufferFromFile>(disk_path + path); |
121 | } |
122 | |
123 | |
124 | void DiskLocalReservation::update(UInt64 new_size) |
125 | { |
126 | std::lock_guard lock(DiskLocal::mutex); |
127 | disk->reserved_bytes -= size; |
128 | size = new_size; |
129 | disk->reserved_bytes += size; |
130 | } |
131 | |
132 | DiskLocalReservation::~DiskLocalReservation() |
133 | { |
134 | try |
135 | { |
136 | std::lock_guard lock(DiskLocal::mutex); |
137 | if (disk->reserved_bytes < size) |
138 | { |
139 | disk->reserved_bytes = 0; |
140 | LOG_ERROR(&Logger::get("DiskLocal" ), "Unbalanced reservations size for disk '" + disk->getName() + "'." ); |
141 | } |
142 | else |
143 | { |
144 | disk->reserved_bytes -= size; |
145 | } |
146 | |
147 | if (disk->reservation_count == 0) |
148 | LOG_ERROR(&Logger::get("DiskLocal" ), "Unbalanced reservation count for disk '" + disk->getName() + "'." ); |
149 | else |
150 | --disk->reservation_count; |
151 | } |
152 | catch (...) |
153 | { |
154 | tryLogCurrentException(__PRETTY_FUNCTION__); |
155 | } |
156 | } |
157 | |
158 | |
159 | void registerDiskLocal(DiskFactory & factory) |
160 | { |
161 | auto creator = [](const String & name, |
162 | const Poco::Util::AbstractConfiguration & config, |
163 | const String & config_prefix, |
164 | const Context & context) -> DiskPtr { |
165 | String path = config.getString(config_prefix + ".path" , "" ); |
166 | if (name == "default" ) |
167 | { |
168 | if (!path.empty()) |
169 | throw Exception( |
170 | "\"default\" disk path should be provided in <path> not it <storage_configuration>" , |
171 | ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); |
172 | path = context.getPath(); |
173 | } |
174 | else |
175 | { |
176 | if (path.empty()) |
177 | throw Exception("Disk path can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); |
178 | if (path.back() != '/') |
179 | throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); |
180 | } |
181 | |
182 | bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio" ); |
183 | |
184 | if (config.has(config_prefix + ".keep_free_space_bytes" ) && has_space_ratio) |
185 | throw Exception( |
186 | "Only one of 'keep_free_space_bytes' and 'keep_free_space_ratio' can be specified" , |
187 | ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); |
188 | |
189 | UInt64 keep_free_space_bytes = config.getUInt64(config_prefix + ".keep_free_space_bytes" , 0); |
190 | |
191 | if (has_space_ratio) |
192 | { |
193 | auto ratio = config.getDouble(config_prefix + ".keep_free_space_ratio" ); |
194 | if (ratio < 0 || ratio > 1) |
195 | throw Exception("'keep_free_space_ratio' have to be between 0 and 1" , ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); |
196 | String tmp_path = path; |
197 | if (tmp_path.empty()) |
198 | tmp_path = context.getPath(); |
199 | |
200 | // Create tmp disk for getting total disk space. |
201 | keep_free_space_bytes = static_cast<UInt64>(DiskLocal("tmp" , tmp_path, 0).getTotalSpace() * ratio); |
202 | } |
203 | |
204 | return std::make_shared<DiskLocal>(name, path, keep_free_space_bytes); |
205 | }; |
206 | factory.registerDiskType("local" , creator); |
207 | } |
208 | |
209 | } |
210 | |