1 | #include <common/JSON.h> |
2 | #include <Poco/Path.h> |
3 | #include <IO/WriteBufferFromFile.h> |
4 | #include <IO/ReadBufferFromFile.h> |
5 | #include <IO/WriteBufferFromString.h> |
6 | #include <IO/WriteHelpers.h> |
7 | #include <IO/ReadHelpers.h> |
8 | #include <Common/escapeForFileName.h> |
9 | |
10 | #include <Common/FileChecker.h> |
11 | |
12 | |
13 | namespace DB |
14 | { |
15 | |
16 | |
17 | FileChecker::FileChecker(const std::string & file_info_path_) |
18 | { |
19 | setPath(file_info_path_); |
20 | } |
21 | |
22 | void FileChecker::setPath(const std::string & file_info_path_) |
23 | { |
24 | files_info_path = file_info_path_; |
25 | |
26 | Poco::Path path(files_info_path); |
27 | tmp_files_info_path = path.parent().toString() + "tmp_" + path.getFileName(); |
28 | } |
29 | |
30 | void FileChecker::update(const Poco::File & file) |
31 | { |
32 | initialize(); |
33 | updateImpl(file); |
34 | save(); |
35 | } |
36 | |
37 | void FileChecker::update(const Files::const_iterator & begin, const Files::const_iterator & end) |
38 | { |
39 | initialize(); |
40 | for (auto it = begin; it != end; ++it) |
41 | updateImpl(*it); |
42 | save(); |
43 | } |
44 | |
45 | CheckResults FileChecker::check() const |
46 | { |
47 | /** Read the files again every time you call `check` - so as not to violate the constancy. |
48 | * `check` method is rarely called. |
49 | */ |
50 | |
51 | CheckResults results; |
52 | Map local_map; |
53 | load(local_map, files_info_path); |
54 | |
55 | if (local_map.empty()) |
56 | return {}; |
57 | |
58 | for (const auto & name_size : local_map) |
59 | { |
60 | Poco::Path path = Poco::Path(files_info_path).parent().toString() + "/" + name_size.first; |
61 | Poco::File file(path); |
62 | if (!file.exists()) |
63 | { |
64 | results.emplace_back(path.getFileName(), false, "File " + file.path() + " doesn't exist" ); |
65 | break; |
66 | } |
67 | |
68 | |
69 | size_t real_size = file.getSize(); |
70 | if (real_size != name_size.second) |
71 | { |
72 | results.emplace_back(path.getFileName(), false, "Size of " + file.path() + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second)); |
73 | break; |
74 | } |
75 | results.emplace_back(path.getFileName(), true, "" ); |
76 | } |
77 | |
78 | return results; |
79 | } |
80 | |
81 | void FileChecker::initialize() |
82 | { |
83 | if (initialized) |
84 | return; |
85 | |
86 | load(map, files_info_path); |
87 | initialized = true; |
88 | } |
89 | |
90 | void FileChecker::updateImpl(const Poco::File & file) |
91 | { |
92 | map[Poco::Path(file.path()).getFileName()] = file.getSize(); |
93 | } |
94 | |
95 | void FileChecker::save() const |
96 | { |
97 | { |
98 | WriteBufferFromFile out(tmp_files_info_path); |
99 | |
100 | /// So complex JSON structure - for compatibility with the old format. |
101 | writeCString("{\"yandex\":{" , out); |
102 | |
103 | auto settings = FormatSettings(); |
104 | for (auto it = map.begin(); it != map.end(); ++it) |
105 | { |
106 | if (it != map.begin()) |
107 | writeString("," , out); |
108 | |
109 | /// `escapeForFileName` is not really needed. But it is left for compatibility with the old code. |
110 | writeJSONString(escapeForFileName(it->first), out, settings); |
111 | writeString(":{\"size\":\"" , out); |
112 | writeIntText(it->second, out); |
113 | writeString("\"}" , out); |
114 | } |
115 | |
116 | writeCString("}}" , out); |
117 | out.next(); |
118 | } |
119 | |
120 | Poco::File current_file(files_info_path); |
121 | |
122 | if (current_file.exists()) |
123 | { |
124 | std::string old_file_name = files_info_path + ".old" ; |
125 | current_file.renameTo(old_file_name); |
126 | Poco::File(tmp_files_info_path).renameTo(files_info_path); |
127 | Poco::File(old_file_name).remove(); |
128 | } |
129 | else |
130 | Poco::File(tmp_files_info_path).renameTo(files_info_path); |
131 | } |
132 | |
133 | void FileChecker::load(Map & local_map, const std::string & path) |
134 | { |
135 | local_map.clear(); |
136 | |
137 | if (!Poco::File(path).exists()) |
138 | return; |
139 | |
140 | ReadBufferFromFile in(path); |
141 | WriteBufferFromOwnString out; |
142 | |
143 | /// The JSON library does not support whitespace. We delete them. Inefficient. |
144 | while (!in.eof()) |
145 | { |
146 | char c; |
147 | readChar(c, in); |
148 | if (!isspace(c)) |
149 | writeChar(c, out); |
150 | } |
151 | JSON json(out.str()); |
152 | |
153 | JSON files = json["yandex" ]; |
154 | for (const JSON name_value : files) |
155 | local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size" ].toUInt(); |
156 | } |
157 | |
158 | } |
159 | |