1/**************************************************************************/
2/* compression.cpp */
3/**************************************************************************/
4/* This file is part of: */
5/* GODOT ENGINE */
6/* https://godotengine.org */
7/**************************************************************************/
8/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10/* */
11/* Permission is hereby granted, free of charge, to any person obtaining */
12/* a copy of this software and associated documentation files (the */
13/* "Software"), to deal in the Software without restriction, including */
14/* without limitation the rights to use, copy, modify, merge, publish, */
15/* distribute, sublicense, and/or sell copies of the Software, and to */
16/* permit persons to whom the Software is furnished to do so, subject to */
17/* the following conditions: */
18/* */
19/* The above copyright notice and this permission notice shall be */
20/* included in all copies or substantial portions of the Software. */
21/* */
22/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29/**************************************************************************/
30
31#include "compression.h"
32
33#include "core/config/project_settings.h"
34#include "core/io/zip_io.h"
35
36#include "thirdparty/misc/fastlz.h"
37
38#include <zlib.h>
39#include <zstd.h>
40
41#ifdef BROTLI_ENABLED
42#include <brotli/decode.h>
43#endif
44
45int Compression::compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode) {
46 switch (p_mode) {
47 case MODE_BROTLI: {
48 ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
49 } break;
50 case MODE_FASTLZ: {
51 if (p_src_size < 16) {
52 uint8_t src[16];
53 memset(&src[p_src_size], 0, 16 - p_src_size);
54 memcpy(src, p_src, p_src_size);
55 return fastlz_compress(src, 16, p_dst);
56 } else {
57 return fastlz_compress(p_src, p_src_size, p_dst);
58 }
59
60 } break;
61 case MODE_DEFLATE:
62 case MODE_GZIP: {
63 int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
64
65 z_stream strm;
66 strm.zalloc = zipio_alloc;
67 strm.zfree = zipio_free;
68 strm.opaque = Z_NULL;
69 int level = p_mode == MODE_DEFLATE ? zlib_level : gzip_level;
70 int err = deflateInit2(&strm, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
71 if (err != Z_OK) {
72 return -1;
73 }
74
75 strm.avail_in = p_src_size;
76 int aout = deflateBound(&strm, p_src_size);
77 strm.avail_out = aout;
78 strm.next_in = (Bytef *)p_src;
79 strm.next_out = p_dst;
80 deflate(&strm, Z_FINISH);
81 aout = aout - strm.avail_out;
82 deflateEnd(&strm);
83 return aout;
84
85 } break;
86 case MODE_ZSTD: {
87 ZSTD_CCtx *cctx = ZSTD_createCCtx();
88 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, zstd_level);
89 if (zstd_long_distance_matching) {
90 ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1);
91 ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, zstd_window_log_size);
92 }
93 int max_dst_size = get_max_compressed_buffer_size(p_src_size, MODE_ZSTD);
94 int ret = ZSTD_compressCCtx(cctx, p_dst, max_dst_size, p_src, p_src_size, zstd_level);
95 ZSTD_freeCCtx(cctx);
96 return ret;
97 } break;
98 }
99
100 ERR_FAIL_V(-1);
101}
102
103int Compression::get_max_compressed_buffer_size(int p_src_size, Mode p_mode) {
104 switch (p_mode) {
105 case MODE_BROTLI: {
106 ERR_FAIL_V_MSG(-1, "Only brotli decompression is supported.");
107 } break;
108 case MODE_FASTLZ: {
109 int ss = p_src_size + p_src_size * 6 / 100;
110 if (ss < 66) {
111 ss = 66;
112 }
113 return ss;
114
115 } break;
116 case MODE_DEFLATE:
117 case MODE_GZIP: {
118 int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
119
120 z_stream strm;
121 strm.zalloc = zipio_alloc;
122 strm.zfree = zipio_free;
123 strm.opaque = Z_NULL;
124 int err = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
125 if (err != Z_OK) {
126 return -1;
127 }
128 int aout = deflateBound(&strm, p_src_size);
129 deflateEnd(&strm);
130 return aout;
131 } break;
132 case MODE_ZSTD: {
133 return ZSTD_compressBound(p_src_size);
134 } break;
135 }
136
137 ERR_FAIL_V(-1);
138}
139
140int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
141 switch (p_mode) {
142 case MODE_BROTLI: {
143#ifdef BROTLI_ENABLED
144 size_t ret_size = p_dst_max_size;
145 BrotliDecoderResult res = BrotliDecoderDecompress(p_src_size, p_src, &ret_size, p_dst);
146 ERR_FAIL_COND_V(res != BROTLI_DECODER_RESULT_SUCCESS, -1);
147 return ret_size;
148#else
149 ERR_FAIL_V_MSG(-1, "Godot was compiled without brotli support.");
150#endif
151 } break;
152 case MODE_FASTLZ: {
153 int ret_size = 0;
154
155 if (p_dst_max_size < 16) {
156 uint8_t dst[16];
157 fastlz_decompress(p_src, p_src_size, dst, 16);
158 memcpy(p_dst, dst, p_dst_max_size);
159 ret_size = p_dst_max_size;
160 } else {
161 ret_size = fastlz_decompress(p_src, p_src_size, p_dst, p_dst_max_size);
162 }
163 return ret_size;
164 } break;
165 case MODE_DEFLATE:
166 case MODE_GZIP: {
167 int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
168
169 z_stream strm;
170 strm.zalloc = zipio_alloc;
171 strm.zfree = zipio_free;
172 strm.opaque = Z_NULL;
173 strm.avail_in = 0;
174 strm.next_in = Z_NULL;
175 int err = inflateInit2(&strm, window_bits);
176 ERR_FAIL_COND_V(err != Z_OK, -1);
177
178 strm.avail_in = p_src_size;
179 strm.avail_out = p_dst_max_size;
180 strm.next_in = (Bytef *)p_src;
181 strm.next_out = p_dst;
182
183 err = inflate(&strm, Z_FINISH);
184 int total = strm.total_out;
185 inflateEnd(&strm);
186 ERR_FAIL_COND_V(err != Z_STREAM_END, -1);
187 return total;
188 } break;
189 case MODE_ZSTD: {
190 ZSTD_DCtx *dctx = ZSTD_createDCtx();
191 if (zstd_long_distance_matching) {
192 ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, zstd_window_log_size);
193 }
194 int ret = ZSTD_decompressDCtx(dctx, p_dst, p_dst_max_size, p_src, p_src_size);
195 ZSTD_freeDCtx(dctx);
196 return ret;
197 } break;
198 }
199
200 ERR_FAIL_V(-1);
201}
202
203/**
204 This will handle both Gzip and Deflate streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
205 This is required for compressed data whose final uncompressed size is unknown, as is the case for HTTP response bodies.
206 This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
207*/
208int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
209 uint8_t *dst = nullptr;
210 int out_mark = 0;
211
212 ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
213
214 if (p_mode == MODE_BROTLI) {
215#ifdef BROTLI_ENABLED
216 BrotliDecoderResult ret;
217 BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
218 ERR_FAIL_NULL_V(state, Z_DATA_ERROR);
219
220 // Setup the stream inputs.
221 const uint8_t *next_in = p_src;
222 size_t avail_in = p_src_size;
223 uint8_t *next_out = nullptr;
224 size_t avail_out = 0;
225 size_t total_out = 0;
226
227 // Ensure the destination buffer is empty.
228 p_dst_vect->clear();
229
230 // Decompress until stream ends or end of file.
231 do {
232 // Add another chunk size to the output buffer.
233 // This forces a copy of the whole buffer.
234 p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
235 // Get pointer to the actual output buffer.
236 dst = p_dst_vect->ptrw();
237
238 // Set the stream to the new output stream.
239 // Since it was copied, we need to reset the stream to the new buffer.
240 next_out = &(dst[out_mark]);
241 avail_out += gzip_chunk;
242
243 ret = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, &total_out);
244 if (ret == BROTLI_DECODER_RESULT_ERROR) {
245 WARN_PRINT(BrotliDecoderErrorString(BrotliDecoderGetErrorCode(state)));
246 BrotliDecoderDestroyInstance(state);
247 p_dst_vect->clear();
248 return Z_DATA_ERROR;
249 }
250
251 out_mark += gzip_chunk - avail_out;
252
253 // Enforce max output size.
254 if (p_max_dst_size > -1 && total_out > (uint64_t)p_max_dst_size) {
255 BrotliDecoderDestroyInstance(state);
256 p_dst_vect->clear();
257 return Z_BUF_ERROR;
258 }
259 } while (ret != BROTLI_DECODER_RESULT_SUCCESS);
260
261 // If all done successfully, resize the output if it's larger than the actual output.
262 if ((unsigned long)p_dst_vect->size() > total_out) {
263 p_dst_vect->resize(total_out);
264 }
265
266 // Clean up and return.
267 BrotliDecoderDestroyInstance(state);
268 return Z_OK;
269#else
270 ERR_FAIL_V_MSG(Z_ERRNO, "Godot was compiled without brotli support.");
271#endif
272 } else {
273 // This function only supports GZip and Deflate.
274 ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
275
276 int ret;
277 z_stream strm;
278 int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
279
280 // Initialize the stream.
281 strm.zalloc = Z_NULL;
282 strm.zfree = Z_NULL;
283 strm.opaque = Z_NULL;
284 strm.avail_in = 0;
285 strm.next_in = Z_NULL;
286
287 int err = inflateInit2(&strm, window_bits);
288 ERR_FAIL_COND_V(err != Z_OK, -1);
289
290 // Setup the stream inputs.
291 strm.next_in = (Bytef *)p_src;
292 strm.avail_in = p_src_size;
293
294 // Ensure the destination buffer is empty.
295 p_dst_vect->clear();
296
297 // Decompress until deflate stream ends or end of file.
298 do {
299 // Add another chunk size to the output buffer.
300 // This forces a copy of the whole buffer.
301 p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
302 // Get pointer to the actual output buffer.
303 dst = p_dst_vect->ptrw();
304
305 // Set the stream to the new output stream.
306 // Since it was copied, we need to reset the stream to the new buffer.
307 strm.next_out = &(dst[out_mark]);
308 strm.avail_out = gzip_chunk;
309
310 // Run inflate() on input until output buffer is full and needs to be resized or input runs out.
311 do {
312 ret = inflate(&strm, Z_SYNC_FLUSH);
313
314 switch (ret) {
315 case Z_NEED_DICT:
316 ret = Z_DATA_ERROR;
317 [[fallthrough]];
318 case Z_DATA_ERROR:
319 case Z_MEM_ERROR:
320 case Z_STREAM_ERROR:
321 case Z_BUF_ERROR:
322 if (strm.msg) {
323 WARN_PRINT(strm.msg);
324 }
325 (void)inflateEnd(&strm);
326 p_dst_vect->clear();
327 return ret;
328 }
329 } while (strm.avail_out > 0 && strm.avail_in > 0);
330
331 out_mark += gzip_chunk;
332
333 // Enforce max output size.
334 if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
335 (void)inflateEnd(&strm);
336 p_dst_vect->clear();
337 return Z_BUF_ERROR;
338 }
339 } while (ret != Z_STREAM_END);
340
341 // If all done successfully, resize the output if it's larger than the actual output.
342 if ((unsigned long)p_dst_vect->size() > strm.total_out) {
343 p_dst_vect->resize(strm.total_out);
344 }
345
346 // Clean up and return.
347 (void)inflateEnd(&strm);
348 return Z_OK;
349 }
350}
351
352int Compression::zlib_level = Z_DEFAULT_COMPRESSION;
353int Compression::gzip_level = Z_DEFAULT_COMPRESSION;
354int Compression::zstd_level = 3;
355bool Compression::zstd_long_distance_matching = false;
356int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT
357int Compression::gzip_chunk = 16384;
358