1/**************************************************************************/
2/* remote_filesystem_client.cpp */
3/**************************************************************************/
4/* This file is part of: */
5/* GODOT ENGINE */
6/* https://godotengine.org */
7/**************************************************************************/
8/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10/* */
11/* Permission is hereby granted, free of charge, to any person obtaining */
12/* a copy of this software and associated documentation files (the */
13/* "Software"), to deal in the Software without restriction, including */
14/* without limitation the rights to use, copy, modify, merge, publish, */
15/* distribute, sublicense, and/or sell copies of the Software, and to */
16/* permit persons to whom the Software is furnished to do so, subject to */
17/* the following conditions: */
18/* */
19/* The above copyright notice and this permission notice shall be */
20/* included in all copies or substantial portions of the Software. */
21/* */
22/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29/**************************************************************************/
30
31#include "remote_filesystem_client.h"
32
33#include "core/io/dir_access.h"
34#include "core/io/file_access.h"
35#include "core/io/stream_peer_tcp.h"
36#include "core/string/string_builder.h"
37
38#define FILESYSTEM_CACHE_VERSION 1
39#define FILESYSTEM_PROTOCOL_VERSION 1
40#define PASSWORD_LENGTH 32
41
42#define FILES_SUBFOLDER "remote_filesystem_files"
43#define FILES_CACHE_FILE "remote_filesystem.cache"
44
45Vector<RemoteFilesystemClient::FileCache> RemoteFilesystemClient::_load_cache_file() {
46 Ref<FileAccess> fa = FileAccess::open(cache_path.path_join(FILES_CACHE_FILE), FileAccess::READ);
47 if (!fa.is_valid()) {
48 return Vector<FileCache>(); // No cache, return empty
49 }
50
51 int version = fa->get_line().to_int();
52 if (version != FILESYSTEM_CACHE_VERSION) {
53 return Vector<FileCache>(); // Version mismatch, ignore everything.
54 }
55
56 String file_path = cache_path.path_join(FILES_SUBFOLDER);
57
58 Vector<FileCache> file_cache;
59
60 while (!fa->eof_reached()) {
61 String l = fa->get_line();
62 Vector<String> fields = l.split("::");
63 if (fields.size() != 3) {
64 break;
65 }
66 FileCache fc;
67 fc.path = fields[0];
68 fc.server_modified_time = fields[1].to_int();
69 fc.modified_time = fields[2].to_int();
70
71 String full_path = file_path.path_join(fc.path);
72 if (!FileAccess::exists(full_path)) {
73 continue; // File is gone.
74 }
75
76 if (FileAccess::get_modified_time(full_path) != fc.modified_time) {
77 DirAccess::remove_absolute(full_path); // Take the chance to remove this file and assume we no longer have it.
78 continue;
79 }
80
81 file_cache.push_back(fc);
82 }
83
84 return file_cache;
85}
86
87Error RemoteFilesystemClient::_store_file(const String &p_path, const LocalVector<uint8_t> &p_file, uint64_t &modified_time) {
88 modified_time = 0;
89 String full_path = cache_path.path_join(FILES_SUBFOLDER).path_join(p_path);
90 String base_file_dir = full_path.get_base_dir();
91
92 if (!validated_directories.has(base_file_dir)) {
93 // Verify that path exists before writing file, but only verify once for performance.
94 DirAccess::make_dir_recursive_absolute(base_file_dir);
95 validated_directories.insert(base_file_dir);
96 }
97
98 Ref<FileAccess> f = FileAccess::open(full_path, FileAccess::WRITE);
99 ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open file for writing to remote filesystem cache: " + p_path);
100 f->store_buffer(p_file.ptr(), p_file.size());
101 Error err = f->get_error();
102 if (err) {
103 return err;
104 }
105 f.unref(); // Unref to ensure file is not locked and modified time can be obtained.
106
107 modified_time = FileAccess::get_modified_time(full_path);
108 return OK;
109}
110
111Error RemoteFilesystemClient::_remove_file(const String &p_path) {
112 return DirAccess::remove_absolute(cache_path.path_join(FILES_SUBFOLDER).path_join(p_path));
113}
114Error RemoteFilesystemClient::_store_cache_file(const Vector<FileCache> &p_cache) {
115 String full_path = cache_path.path_join(FILES_CACHE_FILE);
116 String base_file_dir = full_path.get_base_dir();
117 Error err = DirAccess::make_dir_recursive_absolute(base_file_dir);
118 ERR_FAIL_COND_V_MSG(err != OK && err != ERR_ALREADY_EXISTS, err, "Unable to create base directory to store cache file: " + base_file_dir);
119
120 Ref<FileAccess> f = FileAccess::open(full_path, FileAccess::WRITE);
121 ERR_FAIL_COND_V_MSG(f.is_null(), ERR_FILE_CANT_OPEN, "Unable to open the remote cache file for writing: " + full_path);
122 f->store_line(itos(FILESYSTEM_CACHE_VERSION));
123 for (int i = 0; i < p_cache.size(); i++) {
124 String l = p_cache[i].path + "::" + itos(p_cache[i].server_modified_time) + "::" + itos(p_cache[i].modified_time);
125 f->store_line(l);
126 }
127 return OK;
128}
129
130Error RemoteFilesystemClient::synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) {
131 Error err = _synchronize_with_server(p_host, p_port, p_password, r_cache_path);
132 // Ensure no memory is kept
133 validated_directories.reset();
134 cache_path = String();
135 return err;
136}
137
138void RemoteFilesystemClient::_update_cache_path(String &r_cache_path) {
139 r_cache_path = cache_path.path_join(FILES_SUBFOLDER);
140}
141
142Error RemoteFilesystemClient::_synchronize_with_server(const String &p_host, int p_port, const String &p_password, String &r_cache_path) {
143 cache_path = r_cache_path;
144 {
145 Ref<DirAccess> dir = DirAccess::create(DirAccess::ACCESS_FILESYSTEM);
146 dir->change_dir(cache_path);
147 cache_path = dir->get_current_dir();
148 }
149
150 Ref<StreamPeerTCP> tcp_client;
151 tcp_client.instantiate();
152
153 IPAddress ip = p_host.is_valid_ip_address() ? IPAddress(p_host) : IP::get_singleton()->resolve_hostname(p_host);
154 ERR_FAIL_COND_V_MSG(!ip.is_valid(), ERR_INVALID_PARAMETER, "Unable to resolve remote filesystem server hostname: " + p_host);
155 print_verbose(vformat("Remote Filesystem: Connecting to host %s, port %d.", ip, p_port));
156 Error err = tcp_client->connect_to_host(ip, p_port);
157 ERR_FAIL_COND_V_MSG(err != OK, err, "Unable to open connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed.");
158
159 while (tcp_client->get_status() == StreamPeerTCP::STATUS_CONNECTING) {
160 tcp_client->poll();
161 OS::get_singleton()->delay_usec(100);
162 }
163
164 if (tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED) {
165 ERR_FAIL_V_MSG(ERR_CANT_CONNECT, "Connection to remote file server (" + String(p_host) + ", port " + itos(p_port) + ") failed.");
166 }
167
168 // Connection OK, now send the current file state.
169 print_verbose("Remote Filesystem: Connection OK.");
170
171 // Header (GRFS) - Godot Remote File System
172 print_verbose("Remote Filesystem: Sending header");
173 tcp_client->put_u8('G');
174 tcp_client->put_u8('R');
175 tcp_client->put_u8('F');
176 tcp_client->put_u8('S');
177 // Protocol version
178 tcp_client->put_32(FILESYSTEM_PROTOCOL_VERSION);
179 print_verbose("Remote Filesystem: Sending password");
180 uint8_t password[PASSWORD_LENGTH]; // Send fixed size password, since it's easier and safe to validate.
181 for (int i = 0; i < PASSWORD_LENGTH; i++) {
182 if (i < p_password.length()) {
183 password[i] = p_password[i];
184 } else {
185 password[i] = 0;
186 }
187 }
188 tcp_client->put_data(password, PASSWORD_LENGTH);
189 print_verbose("Remote Filesystem: Tags.");
190 Vector<String> tags;
191 {
192 tags.push_back(OS::get_singleton()->get_identifier());
193 switch (OS::get_singleton()->get_preferred_texture_format()) {
194 case OS::PREFERRED_TEXTURE_FORMAT_S3TC_BPTC: {
195 tags.push_back("bptc");
196 tags.push_back("s3tc");
197 } break;
198 case OS::PREFERRED_TEXTURE_FORMAT_ETC2_ASTC: {
199 tags.push_back("etc2");
200 tags.push_back("astc");
201 } break;
202 }
203 }
204
205 tcp_client->put_32(tags.size());
206 for (int i = 0; i < tags.size(); i++) {
207 tcp_client->put_utf8_string(tags[i]);
208 }
209 // Size of compressed list of files
210 print_verbose("Remote Filesystem: Sending file list");
211
212 Vector<FileCache> file_cache = _load_cache_file();
213
214 // Encode file cache to send it via network.
215 Vector<uint8_t> file_cache_buffer;
216 if (file_cache.size()) {
217 StringBuilder sbuild;
218 for (int i = 0; i < file_cache.size(); i++) {
219 sbuild.append(file_cache[i].path);
220 sbuild.append("::");
221 sbuild.append(itos(file_cache[i].server_modified_time));
222 sbuild.append("\n");
223 }
224 String s = sbuild.as_string();
225 CharString cs = s.utf8();
226 file_cache_buffer.resize(Compression::get_max_compressed_buffer_size(cs.length(), Compression::MODE_ZSTD));
227 int res_len = Compression::compress(file_cache_buffer.ptrw(), (const uint8_t *)cs.ptr(), cs.length(), Compression::MODE_ZSTD);
228 file_cache_buffer.resize(res_len);
229
230 tcp_client->put_32(cs.length()); // Size of buffer uncompressed
231 tcp_client->put_32(file_cache_buffer.size()); // Size of buffer compressed
232 tcp_client->put_data(file_cache_buffer.ptr(), file_cache_buffer.size()); // Buffer
233 } else {
234 tcp_client->put_32(0); // No file cache buffer
235 }
236
237 tcp_client->poll();
238 ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected after sending header.");
239
240 uint32_t file_count = tcp_client->get_32();
241
242 ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file list");
243
244 LocalVector<uint8_t> file_buffer;
245
246 Vector<FileCache> temp_file_cache;
247
248 HashSet<String> files_processed;
249 for (uint32_t i = 0; i < file_count; i++) {
250 String file = tcp_client->get_utf8_string();
251 ERR_FAIL_COND_V_MSG(file == String(), ERR_CONNECTION_ERROR, "Invalid file name received from remote filesystem.");
252 uint64_t server_modified_time = tcp_client->get_u64();
253 ERR_FAIL_COND_V_MSG(tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED, ERR_CONNECTION_ERROR, "Remote filesystem server disconnected while waiting for file info.");
254
255 FileCache fc;
256 fc.path = file;
257 fc.server_modified_time = server_modified_time;
258 temp_file_cache.push_back(fc);
259
260 files_processed.insert(file);
261 }
262
263 Vector<FileCache> new_file_cache;
264
265 // Get the actual files. As a robustness measure, if the connection is interrupted here, any file not yet received will be considered removed.
266 // Since the file changed anyway, this makes it the easiest way to keep robustness.
267
268 bool server_disconnected = false;
269 for (uint32_t i = 0; i < file_count; i++) {
270 String file = temp_file_cache[i].path;
271
272 if (temp_file_cache[i].server_modified_time == 0 || server_disconnected) {
273 // File was removed, or server disconnected before transferring it. Since it's no longer valid, remove anyway.
274 _remove_file(file);
275 continue;
276 }
277
278 uint64_t file_size = tcp_client->get_u64();
279 file_buffer.resize(file_size);
280
281 err = tcp_client->get_data(file_buffer.ptr(), file_size);
282 if (err != OK) {
283 ERR_PRINT("Error retrieving file from remote filesystem: " + file);
284 server_disconnected = true;
285 }
286
287 if (tcp_client->get_status() != StreamPeerTCP::STATUS_CONNECTED) {
288 // Early disconnect, stop accepting files.
289 server_disconnected = true;
290 }
291
292 if (server_disconnected) {
293 // No more server, transfer is invalid, remove this file.
294 _remove_file(file);
295 continue;
296 }
297
298 uint64_t modified_time = 0;
299 err = _store_file(file, file_buffer, modified_time);
300 if (err != OK) {
301 server_disconnected = true;
302 continue;
303 }
304 FileCache fc = temp_file_cache[i];
305 fc.modified_time = modified_time;
306 new_file_cache.push_back(fc);
307 }
308
309 print_verbose("Remote Filesystem: Updating the cache file.");
310
311 // Go through the list of local files read initially (file_cache) and see which ones are
312 // unchanged (not sent again from the server).
313 // These need to be re-saved in the new list (new_file_cache).
314
315 for (int i = 0; i < file_cache.size(); i++) {
316 if (files_processed.has(file_cache[i].path)) {
317 continue; // This was either added or removed, so skip.
318 }
319 new_file_cache.push_back(file_cache[i]);
320 }
321
322 err = _store_cache_file(new_file_cache);
323 ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Error writing the remote filesystem file cache.");
324
325 print_verbose("Remote Filesystem: Update success.");
326
327 _update_cache_path(r_cache_path);
328 return OK;
329}
330