1#include "DiskLocal.h"
2#include "DiskFactory.h"
3
4#include <Interpreters/Context.h>
5#include <Common/filesystemHelpers.h>
6#include <Common/quoteString.h>
7
8
9namespace DB
10{
11std::mutex DiskLocal::mutex;
12
13ReservationPtr DiskLocal::reserve(UInt64 bytes)
14{
15 if (!tryReserve(bytes))
16 return {};
17 return std::make_unique<DiskLocalReservation>(std::static_pointer_cast<DiskLocal>(shared_from_this()), bytes);
18}
19
20bool DiskLocal::tryReserve(UInt64 bytes)
21{
22 std::lock_guard lock(mutex);
23 if (bytes == 0)
24 {
25 LOG_DEBUG(&Logger::get("DiskLocal"), "Reserving 0 bytes on disk " << backQuote(name));
26 ++reservation_count;
27 return true;
28 }
29
30 auto available_space = getAvailableSpace();
31 UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes);
32 if (unreserved_space >= bytes)
33 {
34 LOG_DEBUG(
35 &Logger::get("DiskLocal"),
36 "Reserving " << formatReadableSizeWithBinarySuffix(bytes) << " on disk " << backQuote(name) << ", having unreserved "
37 << formatReadableSizeWithBinarySuffix(unreserved_space) << ".");
38 ++reservation_count;
39 reserved_bytes += bytes;
40 return true;
41 }
42 return false;
43}
44
45UInt64 DiskLocal::getTotalSpace() const
46{
47 auto fs = getStatVFS(disk_path);
48 UInt64 total_size = fs.f_blocks * fs.f_bsize;
49 if (total_size < keep_free_space_bytes)
50 return 0;
51 return total_size - keep_free_space_bytes;
52}
53
54UInt64 DiskLocal::getAvailableSpace() const
55{
56 /// we use f_bavail, because part of b_free space is
57 /// available for superuser only and for system purposes
58 auto fs = getStatVFS(disk_path);
59 UInt64 total_size = fs.f_bavail * fs.f_bsize;
60 if (total_size < keep_free_space_bytes)
61 return 0;
62 return total_size - keep_free_space_bytes;
63}
64
65UInt64 DiskLocal::getUnreservedSpace() const
66{
67 std::lock_guard lock(mutex);
68 auto available_space = getAvailableSpace();
69 available_space -= std::min(available_space, reserved_bytes);
70 return available_space;
71}
72
73bool DiskLocal::exists(const String & path) const
74{
75 return Poco::File(disk_path + path).exists();
76}
77
78bool DiskLocal::isFile(const String & path) const
79{
80 return Poco::File(disk_path + path).isFile();
81}
82
83bool DiskLocal::isDirectory(const String & path) const
84{
85 return Poco::File(disk_path + path).isDirectory();
86}
87
88void DiskLocal::createDirectory(const String & path)
89{
90 Poco::File(disk_path + path).createDirectory();
91}
92
93void DiskLocal::createDirectories(const String & path)
94{
95 Poco::File(disk_path + path).createDirectories();
96}
97
98DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path)
99{
100 return std::make_unique<DiskLocalDirectoryIterator>(disk_path + path);
101}
102
103void DiskLocal::moveFile(const String & from_path, const String & to_path)
104{
105 Poco::File(disk_path + from_path).renameTo(disk_path + to_path);
106}
107
108void DiskLocal::copyFile(const String & from_path, const String & to_path)
109{
110 Poco::File(disk_path + from_path).copyTo(disk_path + to_path);
111}
112
113std::unique_ptr<ReadBuffer> DiskLocal::readFile(const String & path) const
114{
115 return std::make_unique<ReadBufferFromFile>(disk_path + path);
116}
117
118std::unique_ptr<WriteBuffer> DiskLocal::writeFile(const String & path)
119{
120 return std::make_unique<WriteBufferFromFile>(disk_path + path);
121}
122
123
124void DiskLocalReservation::update(UInt64 new_size)
125{
126 std::lock_guard lock(DiskLocal::mutex);
127 disk->reserved_bytes -= size;
128 size = new_size;
129 disk->reserved_bytes += size;
130}
131
132DiskLocalReservation::~DiskLocalReservation()
133{
134 try
135 {
136 std::lock_guard lock(DiskLocal::mutex);
137 if (disk->reserved_bytes < size)
138 {
139 disk->reserved_bytes = 0;
140 LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '" + disk->getName() + "'.");
141 }
142 else
143 {
144 disk->reserved_bytes -= size;
145 }
146
147 if (disk->reservation_count == 0)
148 LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '" + disk->getName() + "'.");
149 else
150 --disk->reservation_count;
151 }
152 catch (...)
153 {
154 tryLogCurrentException(__PRETTY_FUNCTION__);
155 }
156}
157
158
159void registerDiskLocal(DiskFactory & factory)
160{
161 auto creator = [](const String & name,
162 const Poco::Util::AbstractConfiguration & config,
163 const String & config_prefix,
164 const Context & context) -> DiskPtr {
165 String path = config.getString(config_prefix + ".path", "");
166 if (name == "default")
167 {
168 if (!path.empty())
169 throw Exception(
170 "\"default\" disk path should be provided in <path> not it <storage_configuration>",
171 ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
172 path = context.getPath();
173 }
174 else
175 {
176 if (path.empty())
177 throw Exception("Disk path can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
178 if (path.back() != '/')
179 throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
180 }
181
182 bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio");
183
184 if (config.has(config_prefix + ".keep_free_space_bytes") && has_space_ratio)
185 throw Exception(
186 "Only one of 'keep_free_space_bytes' and 'keep_free_space_ratio' can be specified",
187 ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
188
189 UInt64 keep_free_space_bytes = config.getUInt64(config_prefix + ".keep_free_space_bytes", 0);
190
191 if (has_space_ratio)
192 {
193 auto ratio = config.getDouble(config_prefix + ".keep_free_space_ratio");
194 if (ratio < 0 || ratio > 1)
195 throw Exception("'keep_free_space_ratio' have to be between 0 and 1", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
196 String tmp_path = path;
197 if (tmp_path.empty())
198 tmp_path = context.getPath();
199
200 // Create tmp disk for getting total disk space.
201 keep_free_space_bytes = static_cast<UInt64>(DiskLocal("tmp", tmp_path, 0).getTotalSpace() * ratio);
202 }
203
204 return std::make_shared<DiskLocal>(name, path, keep_free_space_bytes);
205 };
206 factory.registerDiskType("local", creator);
207}
208
209}
210