1#include <common/JSON.h>
2#include <Poco/Path.h>
3#include <IO/WriteBufferFromFile.h>
4#include <IO/ReadBufferFromFile.h>
5#include <IO/WriteBufferFromString.h>
6#include <IO/WriteHelpers.h>
7#include <IO/ReadHelpers.h>
8#include <Common/escapeForFileName.h>
9
10#include <Common/FileChecker.h>
11
12
13namespace DB
14{
15
16
17FileChecker::FileChecker(const std::string & file_info_path_)
18{
19 setPath(file_info_path_);
20}
21
22void FileChecker::setPath(const std::string & file_info_path_)
23{
24 files_info_path = file_info_path_;
25
26 Poco::Path path(files_info_path);
27 tmp_files_info_path = path.parent().toString() + "tmp_" + path.getFileName();
28}
29
30void FileChecker::update(const Poco::File & file)
31{
32 initialize();
33 updateImpl(file);
34 save();
35}
36
37void FileChecker::update(const Files::const_iterator & begin, const Files::const_iterator & end)
38{
39 initialize();
40 for (auto it = begin; it != end; ++it)
41 updateImpl(*it);
42 save();
43}
44
45CheckResults FileChecker::check() const
46{
47 /** Read the files again every time you call `check` - so as not to violate the constancy.
48 * `check` method is rarely called.
49 */
50
51 CheckResults results;
52 Map local_map;
53 load(local_map, files_info_path);
54
55 if (local_map.empty())
56 return {};
57
58 for (const auto & name_size : local_map)
59 {
60 Poco::Path path = Poco::Path(files_info_path).parent().toString() + "/" + name_size.first;
61 Poco::File file(path);
62 if (!file.exists())
63 {
64 results.emplace_back(path.getFileName(), false, "File " + file.path() + " doesn't exist");
65 break;
66 }
67
68
69 size_t real_size = file.getSize();
70 if (real_size != name_size.second)
71 {
72 results.emplace_back(path.getFileName(), false, "Size of " + file.path() + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second));
73 break;
74 }
75 results.emplace_back(path.getFileName(), true, "");
76 }
77
78 return results;
79}
80
81void FileChecker::initialize()
82{
83 if (initialized)
84 return;
85
86 load(map, files_info_path);
87 initialized = true;
88}
89
90void FileChecker::updateImpl(const Poco::File & file)
91{
92 map[Poco::Path(file.path()).getFileName()] = file.getSize();
93}
94
95void FileChecker::save() const
96{
97 {
98 WriteBufferFromFile out(tmp_files_info_path);
99
100 /// So complex JSON structure - for compatibility with the old format.
101 writeCString("{\"yandex\":{", out);
102
103 auto settings = FormatSettings();
104 for (auto it = map.begin(); it != map.end(); ++it)
105 {
106 if (it != map.begin())
107 writeString(",", out);
108
109 /// `escapeForFileName` is not really needed. But it is left for compatibility with the old code.
110 writeJSONString(escapeForFileName(it->first), out, settings);
111 writeString(":{\"size\":\"", out);
112 writeIntText(it->second, out);
113 writeString("\"}", out);
114 }
115
116 writeCString("}}", out);
117 out.next();
118 }
119
120 Poco::File current_file(files_info_path);
121
122 if (current_file.exists())
123 {
124 std::string old_file_name = files_info_path + ".old";
125 current_file.renameTo(old_file_name);
126 Poco::File(tmp_files_info_path).renameTo(files_info_path);
127 Poco::File(old_file_name).remove();
128 }
129 else
130 Poco::File(tmp_files_info_path).renameTo(files_info_path);
131}
132
133void FileChecker::load(Map & local_map, const std::string & path)
134{
135 local_map.clear();
136
137 if (!Poco::File(path).exists())
138 return;
139
140 ReadBufferFromFile in(path);
141 WriteBufferFromOwnString out;
142
143 /// The JSON library does not support whitespace. We delete them. Inefficient.
144 while (!in.eof())
145 {
146 char c;
147 readChar(c, in);
148 if (!isspace(c))
149 writeChar(c, out);
150 }
151 JSON json(out.str());
152
153 JSON files = json["yandex"];
154 for (const JSON name_value : files)
155 local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt();
156}
157
158}
159