1 | /* |
2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. |
3 | * All rights reserved. |
4 | * |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found |
7 | * in the COPYING file in the root directory of this source tree). |
8 | * You may select, at your option, one of the above-listed licenses. |
9 | */ |
10 | #if defined (__cplusplus) |
11 | extern "C" { |
12 | #endif |
13 | |
14 | #ifndef ZSTD_H_235446 |
15 | #define ZSTD_H_235446 |
16 | |
17 | /* ====== Dependency ======*/ |
18 | #include <stddef.h> /* size_t */ |
19 | |
20 | |
21 | /* ===== ZSTDLIB_API : control library symbols visibility ===== */ |
22 | #ifndef ZSTDLIB_VISIBILITY |
23 | # if defined(__GNUC__) && (__GNUC__ >= 4) |
24 | # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) |
25 | # else |
26 | # define ZSTDLIB_VISIBILITY |
27 | # endif |
28 | #endif |
29 | #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) |
30 | # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY |
31 | #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) |
32 | # define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ |
33 | #else |
34 | # define ZSTDLIB_API ZSTDLIB_VISIBILITY |
35 | #endif |
36 | |
37 | |
38 | /******************************************************************************************************* |
39 | Introduction |
40 | |
41 | zstd, short for Zstandard, is a fast lossless compression algorithm, |
42 | targeting real-time compression scenarios at zlib-level and better compression ratios. |
43 | The zstd compression library provides in-memory compression and decompression functions. |
44 | The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22. |
45 | Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory. |
46 | Compression can be done in: |
47 | - a single step (described as Simple API) |
48 | - a single step, reusing a context (described as Explicit context) |
49 | - unbounded multiple steps (described as Streaming compression) |
50 | The compression ratio achievable on small data can be highly improved using a dictionary in: |
51 | - a single step (described as Simple dictionary API) |
52 | - a single step, reusing a dictionary (described as Bulk-processing dictionary API) |
53 | |
54 | Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. |
55 | Advanced experimental APIs shall never be used with a dynamic library. |
56 | They are not "stable", their definition may change in the future. Only static linking is allowed. |
57 | *********************************************************************************************************/ |
58 | |
59 | /*------ Version ------*/ |
60 | #define ZSTD_VERSION_MAJOR 1 |
61 | #define ZSTD_VERSION_MINOR 3 |
62 | #define ZSTD_VERSION_RELEASE 4 |
63 | |
64 | #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) |
65 | ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */ |
66 | |
67 | #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE |
68 | #define ZSTD_QUOTE(str) #str |
69 | #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) |
70 | #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) |
71 | ZSTDLIB_API const char* ZSTD_versionString(void); /* added in v1.3.0 */ |
72 | |
73 | |
74 | /*************************************** |
75 | * Simple API |
76 | ***************************************/ |
77 | /*! ZSTD_compress() : |
78 | * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. |
79 | * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. |
80 | * @return : compressed size written into `dst` (<= `dstCapacity), |
81 | * or an error code if it fails (which can be tested using ZSTD_isError()). */ |
82 | ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, |
83 | const void* src, size_t srcSize, |
84 | int compressionLevel); |
85 | |
86 | /*! ZSTD_decompress() : |
87 | * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. |
88 | * `dstCapacity` is an upper bound of originalSize to regenerate. |
89 | * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. |
90 | * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), |
91 | * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ |
92 | ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, |
93 | const void* src, size_t compressedSize); |
94 | |
95 | /*! ZSTD_getFrameContentSize() : added in v1.3.0 |
96 | * `src` should point to the start of a ZSTD encoded frame. |
97 | * `srcSize` must be at least as large as the frame header. |
98 | * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. |
99 | * @return : - decompressed size of the frame in `src`, if known |
100 | * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined |
101 | * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) |
102 | * note 1 : a 0 return value means the frame is valid but "empty". |
103 | * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. |
104 | * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. |
105 | * In which case, it's necessary to use streaming mode to decompress data. |
106 | * Optionally, application can rely on some implicit limit, |
107 | * as ZSTD_decompress() only needs an upper bound of decompressed size. |
108 | * (For example, data could be necessarily cut into blocks <= 16 KB). |
109 | * note 3 : decompressed size is always present when compression is done with ZSTD_compress() |
110 | * note 4 : decompressed size can be very large (64-bits value), |
111 | * potentially larger than what local system can handle as a single memory segment. |
112 | * In which case, it's necessary to use streaming mode to decompress data. |
113 | * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. |
114 | * Always ensure return value fits within application's authorized limits. |
115 | * Each application can set its own limits. |
116 | * note 6 : This function replaces ZSTD_getDecompressedSize() */ |
117 | #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) |
118 | #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) |
119 | ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); |
120 | |
121 | /*! ZSTD_getDecompressedSize() : |
122 | * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). |
123 | * Both functions work the same way, but ZSTD_getDecompressedSize() blends |
124 | * "empty", "unknown" and "error" results to the same return value (0), |
125 | * while ZSTD_getFrameContentSize() gives them separate return values. |
126 | * `src` is the start of a zstd compressed frame. |
127 | * @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. */ |
128 | ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); |
129 | |
130 | |
131 | /*====== Helper functions ======*/ |
132 | #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ |
133 | ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ |
134 | ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ |
135 | ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ |
136 | ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ |
137 | |
138 | |
139 | /*************************************** |
140 | * Explicit context |
141 | ***************************************/ |
142 | /*= Compression context |
143 | * When compressing many times, |
144 | * it is recommended to allocate a context just once, and re-use it for each successive compression operation. |
145 | * This will make workload friendlier for system's memory. |
146 | * Use one context per thread for parallel execution in multi-threaded environments. */ |
147 | typedef struct ZSTD_CCtx_s ZSTD_CCtx; |
148 | ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); |
149 | ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); |
150 | |
151 | /*! ZSTD_compressCCtx() : |
152 | * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */ |
153 | ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, |
154 | void* dst, size_t dstCapacity, |
155 | const void* src, size_t srcSize, |
156 | int compressionLevel); |
157 | |
158 | /*= Decompression context |
159 | * When decompressing many times, |
160 | * it is recommended to allocate a context only once, |
161 | * and re-use it for each successive compression operation. |
162 | * This will make workload friendlier for system's memory. |
163 | * Use one context per thread for parallel execution. */ |
164 | typedef struct ZSTD_DCtx_s ZSTD_DCtx; |
165 | ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); |
166 | ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); |
167 | |
168 | /*! ZSTD_decompressDCtx() : |
169 | * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ |
170 | ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, |
171 | void* dst, size_t dstCapacity, |
172 | const void* src, size_t srcSize); |
173 | |
174 | |
175 | /************************** |
176 | * Simple dictionary API |
177 | ***************************/ |
178 | /*! ZSTD_compress_usingDict() : |
179 | * Compression using a predefined Dictionary (see dictBuilder/zdict.h). |
180 | * Note : This function loads the dictionary, resulting in significant startup delay. |
181 | * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ |
182 | ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, |
183 | void* dst, size_t dstCapacity, |
184 | const void* src, size_t srcSize, |
185 | const void* dict,size_t dictSize, |
186 | int compressionLevel); |
187 | |
188 | /*! ZSTD_decompress_usingDict() : |
189 | * Decompression using a predefined Dictionary (see dictBuilder/zdict.h). |
190 | * Dictionary must be identical to the one used during compression. |
191 | * Note : This function loads the dictionary, resulting in significant startup delay. |
192 | * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ |
193 | ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, |
194 | void* dst, size_t dstCapacity, |
195 | const void* src, size_t srcSize, |
196 | const void* dict,size_t dictSize); |
197 | |
198 | |
199 | /********************************** |
200 | * Bulk processing dictionary API |
201 | *********************************/ |
202 | typedef struct ZSTD_CDict_s ZSTD_CDict; |
203 | |
204 | /*! ZSTD_createCDict() : |
205 | * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. |
206 | * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. |
207 | * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. |
208 | * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */ |
209 | ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, |
210 | int compressionLevel); |
211 | |
212 | /*! ZSTD_freeCDict() : |
213 | * Function frees memory allocated by ZSTD_createCDict(). */ |
214 | ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); |
215 | |
216 | /*! ZSTD_compress_usingCDict() : |
217 | * Compression using a digested Dictionary. |
218 | * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. |
219 | * Note that compression level is decided during dictionary creation. |
220 | * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ |
221 | ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, |
222 | void* dst, size_t dstCapacity, |
223 | const void* src, size_t srcSize, |
224 | const ZSTD_CDict* cdict); |
225 | |
226 | |
227 | typedef struct ZSTD_DDict_s ZSTD_DDict; |
228 | |
229 | /*! ZSTD_createDDict() : |
230 | * Create a digested dictionary, ready to start decompression operation without startup delay. |
231 | * dictBuffer can be released after DDict creation, as its content is copied inside DDict */ |
232 | ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); |
233 | |
234 | /*! ZSTD_freeDDict() : |
235 | * Function frees memory allocated with ZSTD_createDDict() */ |
236 | ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); |
237 | |
238 | /*! ZSTD_decompress_usingDDict() : |
239 | * Decompression using a digested Dictionary. |
240 | * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */ |
241 | ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, |
242 | void* dst, size_t dstCapacity, |
243 | const void* src, size_t srcSize, |
244 | const ZSTD_DDict* ddict); |
245 | |
246 | |
247 | /**************************** |
248 | * Streaming |
249 | ****************************/ |
250 | |
251 | typedef struct ZSTD_inBuffer_s { |
252 | const void* src; /**< start of input buffer */ |
253 | size_t size; /**< size of input buffer */ |
254 | size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ |
255 | } ZSTD_inBuffer; |
256 | |
257 | typedef struct ZSTD_outBuffer_s { |
258 | void* dst; /**< start of output buffer */ |
259 | size_t size; /**< size of output buffer */ |
260 | size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ |
261 | } ZSTD_outBuffer; |
262 | |
263 | |
264 | |
265 | /*-*********************************************************************** |
266 | * Streaming compression - HowTo |
267 | * |
268 | * A ZSTD_CStream object is required to track streaming operation. |
269 | * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. |
270 | * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. |
271 | * It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively, |
272 | * since it will play nicer with system's memory, by re-using already allocated memory. |
273 | * Use one separate ZSTD_CStream per thread for parallel execution. |
274 | * |
275 | * Start a new compression by initializing ZSTD_CStream. |
276 | * Use ZSTD_initCStream() to start a new compression operation. |
277 | * Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section) |
278 | * |
279 | * Use ZSTD_compressStream() repetitively to consume input stream. |
280 | * The function will automatically update both `pos` fields. |
281 | * Note that it may not consume the entire input, in which case `pos < size`, |
282 | * and it's up to the caller to present again remaining data. |
283 | * @return : a size hint, preferred nb of bytes to use as input for next function call |
284 | * or an error code, which can be tested using ZSTD_isError(). |
285 | * Note 1 : it's just a hint, to help latency a little, any other value will work fine. |
286 | * Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() |
287 | * |
288 | * At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream(). |
289 | * `output->pos` will be updated. |
290 | * Note that some content might still be left within internal buffer if `output->size` is too small. |
291 | * @return : nb of bytes still present within internal buffer (0 if it's empty) |
292 | * or an error code, which can be tested using ZSTD_isError(). |
293 | * |
294 | * ZSTD_endStream() instructs to finish a frame. |
295 | * It will perform a flush and write frame epilogue. |
296 | * The epilogue is required for decoders to consider a frame completed. |
297 | * ZSTD_endStream() may not be able to flush full data if `output->size` is too small. |
298 | * In which case, call again ZSTD_endStream() to complete the flush. |
299 | * @return : 0 if frame fully completed and fully flushed, |
300 | or >0 if some data is still present within internal buffer |
301 | (value is minimum size estimation for remaining data to flush, but it could be more) |
302 | * or an error code, which can be tested using ZSTD_isError(). |
303 | * |
304 | * *******************************************************************/ |
305 | |
306 | typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ |
307 | /* Continue to distinguish them for compatibility with versions <= v1.2.0 */ |
308 | /*===== ZSTD_CStream management functions =====*/ |
309 | ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); |
310 | ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); |
311 | |
312 | /*===== Streaming compression functions =====*/ |
313 | ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); |
314 | ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); |
315 | ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); |
316 | ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); |
317 | |
318 | ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ |
319 | ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ |
320 | |
321 | |
322 | |
323 | /*-*************************************************************************** |
324 | * Streaming decompression - HowTo |
325 | * |
326 | * A ZSTD_DStream object is required to track streaming operations. |
327 | * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. |
328 | * ZSTD_DStream objects can be re-used multiple times. |
329 | * |
330 | * Use ZSTD_initDStream() to start a new decompression operation, |
331 | * or ZSTD_initDStream_usingDict() if decompression requires a dictionary. |
332 | * @return : recommended first input size |
333 | * |
334 | * Use ZSTD_decompressStream() repetitively to consume your input. |
335 | * The function will update both `pos` fields. |
336 | * If `input.pos < input.size`, some input has not been consumed. |
337 | * It's up to the caller to present again remaining data. |
338 | * If `output.pos < output.size`, decoder has flushed everything it could. |
339 | * @return : 0 when a frame is completely decoded and fully flushed, |
340 | * an error code, which can be tested using ZSTD_isError(), |
341 | * any other value > 0, which means there is still some decoding to do to complete current frame. |
342 | * The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame. |
343 | * *******************************************************************************/ |
344 | |
345 | typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ |
346 | /* For compatibility with versions <= v1.2.0, continue to consider them separated. */ |
347 | /*===== ZSTD_DStream management functions =====*/ |
348 | ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); |
349 | ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); |
350 | |
351 | /*===== Streaming decompression functions =====*/ |
352 | ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); |
353 | ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); |
354 | |
355 | ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ |
356 | ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ |
357 | |
358 | #endif /* ZSTD_H_235446 */ |
359 | |
360 | |
361 | |
362 | /**************************************************************************************** |
363 | * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS |
364 | * The definitions in this section are considered experimental. |
365 | * They should never be used with a dynamic library, as prototypes may change in the future. |
366 | * They are provided for advanced scenarios. |
367 | * Use them only in association with static linking. |
368 | * ***************************************************************************************/ |
369 | |
370 | #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) |
371 | #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY |
372 | |
373 | /* --- Constants ---*/ |
374 | #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ |
375 | #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U |
376 | #define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* >= v0.7.0 */ |
377 | |
378 | #define ZSTD_WINDOWLOG_MAX_32 30 |
379 | #define ZSTD_WINDOWLOG_MAX_64 31 |
380 | #define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) |
381 | #define ZSTD_WINDOWLOG_MIN 10 |
382 | #define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) |
383 | #define ZSTD_HASHLOG_MIN 6 |
384 | #define ZSTD_CHAINLOG_MAX_32 29 |
385 | #define ZSTD_CHAINLOG_MAX_64 30 |
386 | #define ZSTD_CHAINLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) |
387 | #define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN |
388 | #define ZSTD_HASHLOG3_MAX 17 |
389 | #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) |
390 | #define ZSTD_SEARCHLOG_MIN 1 |
391 | #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ |
392 | #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ |
393 | #define ZSTD_TARGETLENGTH_MIN 1 /* only used by btopt, btultra and btfast */ |
394 | #define ZSTD_LDM_MINMATCH_MIN 4 |
395 | #define ZSTD_LDM_MINMATCH_MAX 4096 |
396 | #define ZSTD_LDM_BUCKETSIZELOG_MAX 8 |
397 | |
398 | #define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */ |
399 | #define ZSTD_FRAMEHEADERSIZE_MIN 6 |
400 | #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ |
401 | static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX; |
402 | static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; |
403 | static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; |
404 | static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ |
405 | |
406 | |
407 | /*--- Advanced types ---*/ |
408 | typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, |
409 | ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */ |
410 | |
411 | typedef struct { |
412 | unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ |
413 | unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ |
414 | unsigned hashLog; /**< dispatch table : larger == faster, more memory */ |
415 | unsigned searchLog; /**< nb of searches : larger == more compression, slower */ |
416 | unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */ |
417 | unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ |
418 | ZSTD_strategy strategy; |
419 | } ZSTD_compressionParameters; |
420 | |
421 | typedef struct { |
422 | unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */ |
423 | unsigned checksumFlag; /**< 1: generate a 32-bits checksum at end of frame, for error detection */ |
424 | unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (if dictionary compression) */ |
425 | } ZSTD_frameParameters; |
426 | |
427 | typedef struct { |
428 | ZSTD_compressionParameters cParams; |
429 | ZSTD_frameParameters fParams; |
430 | } ZSTD_parameters; |
431 | |
432 | typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; |
433 | |
434 | typedef enum { |
435 | ZSTD_dct_auto=0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ |
436 | ZSTD_dct_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ |
437 | ZSTD_dct_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */ |
438 | } ZSTD_dictContentType_e; |
439 | |
440 | typedef enum { |
441 | ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ |
442 | ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ |
443 | } ZSTD_dictLoadMethod_e; |
444 | |
445 | |
446 | |
447 | /*************************************** |
448 | * Frame size functions |
449 | ***************************************/ |
450 | |
451 | /*! ZSTD_findFrameCompressedSize() : |
452 | * `src` should point to the start of a ZSTD encoded frame or skippable frame |
453 | * `srcSize` must be >= first frame size |
454 | * @return : the compressed size of the first frame starting at `src`, |
455 | * suitable to pass to `ZSTD_decompress` or similar, |
456 | * or an error code if input is invalid */ |
457 | ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); |
458 | |
459 | /*! ZSTD_findDecompressedSize() : |
460 | * `src` should point the start of a series of ZSTD encoded and/or skippable frames |
461 | * `srcSize` must be the _exact_ size of this series |
462 | * (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`) |
463 | * @return : - decompressed size of all data in all successive frames |
464 | * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN |
465 | * - if an error occurred: ZSTD_CONTENTSIZE_ERROR |
466 | * |
467 | * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. |
468 | * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. |
469 | * In which case, it's necessary to use streaming mode to decompress data. |
470 | * note 2 : decompressed size is always present when compression is done with ZSTD_compress() |
471 | * note 3 : decompressed size can be very large (64-bits value), |
472 | * potentially larger than what local system can handle as a single memory segment. |
473 | * In which case, it's necessary to use streaming mode to decompress data. |
474 | * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. |
475 | * Always ensure result fits within application's authorized limits. |
476 | * Each application can set its own limits. |
477 | * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to |
478 | * read each contained frame header. This is fast as most of the data is skipped, |
479 | * however it does mean that all frame data must be present and valid. */ |
480 | ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); |
481 | |
482 | /*! ZSTD_frameHeaderSize() : |
483 | * `src` should point to the start of a ZSTD frame |
484 | * `srcSize` must be >= ZSTD_frameHeaderSize_prefix. |
485 | * @return : size of the Frame Header */ |
486 | ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); |
487 | |
488 | |
489 | /*************************************** |
490 | * Memory management |
491 | ***************************************/ |
492 | |
493 | /*! ZSTD_sizeof_*() : |
494 | * These functions give the current memory usage of selected object. |
495 | * Object memory usage can evolve when re-used. */ |
496 | ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); |
497 | ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); |
498 | ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); |
499 | ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); |
500 | ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); |
501 | ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); |
502 | |
503 | /*! ZSTD_estimate*() : |
504 | * These functions make it possible to estimate memory usage |
505 | * of a future {D,C}Ctx, before its creation. |
506 | * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. |
507 | * It will also consider src size to be arbitrarily "large", which is worst case. |
508 | * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. |
509 | * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. |
510 | * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1. |
511 | * Note : CCtx size estimation is only correct for single-threaded compression. */ |
512 | ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); |
513 | ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); |
514 | ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); |
515 | ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); |
516 | |
517 | /*! ZSTD_estimateCStreamSize() : |
518 | * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. |
519 | * It will also consider src size to be arbitrarily "large", which is worst case. |
520 | * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. |
521 | * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. |
522 | * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1. |
523 | * Note : CStream size estimation is only correct for single-threaded compression. |
524 | * ZSTD_DStream memory budget depends on window Size. |
525 | * This information can be passed manually, using ZSTD_estimateDStreamSize, |
526 | * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); |
527 | * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), |
528 | * an internal ?Dict will be created, which additional size is not estimated here. |
529 | * In this case, get total size by adding ZSTD_estimate?DictSize */ |
530 | ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); |
531 | ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); |
532 | ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); |
533 | ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); |
534 | ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); |
535 | |
536 | /*! ZSTD_estimate?DictSize() : |
537 | * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). |
538 | * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). |
539 | * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. |
540 | */ |
541 | ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); |
542 | ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); |
543 | ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); |
544 | |
545 | /*! ZSTD_initStatic*() : |
546 | * Initialize an object using a pre-allocated fixed-size buffer. |
547 | * workspace: The memory area to emplace the object into. |
548 | * Provided pointer *must be 8-bytes aligned*. |
549 | * Buffer must outlive object. |
550 | * workspaceSize: Use ZSTD_estimate*Size() to determine |
551 | * how large workspace must be to support target scenario. |
552 | * @return : pointer to object (same address as workspace, just different type), |
553 | * or NULL if error (size too small, incorrect alignment, etc.) |
554 | * Note : zstd will never resize nor malloc() when using a static buffer. |
555 | * If the object requires more memory than available, |
556 | * zstd will just error out (typically ZSTD_error_memory_allocation). |
557 | * Note 2 : there is no corresponding "free" function. |
558 | * Since workspace is allocated externally, it must be freed externally too. |
559 | * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level |
560 | * into its associated cParams. |
561 | * Limitation 1 : currently not compatible with internal dictionary creation, triggered by |
562 | * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). |
563 | * Limitation 2 : static cctx currently not compatible with multi-threading. |
564 | * Limitation 3 : static dctx is incompatible with legacy support. |
565 | */ |
566 | ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); |
567 | ZSTDLIB_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ |
568 | |
569 | ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); |
570 | ZSTDLIB_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ |
571 | |
572 | ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict( |
573 | void* workspace, size_t workspaceSize, |
574 | const void* dict, size_t dictSize, |
575 | ZSTD_dictLoadMethod_e dictLoadMethod, |
576 | ZSTD_dictContentType_e dictContentType, |
577 | ZSTD_compressionParameters cParams); |
578 | |
579 | ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( |
580 | void* workspace, size_t workspaceSize, |
581 | const void* dict, size_t dictSize, |
582 | ZSTD_dictLoadMethod_e dictLoadMethod, |
583 | ZSTD_dictContentType_e dictContentType); |
584 | |
585 | /*! Custom memory allocation : |
586 | * These prototypes make it possible to pass your own allocation/free functions. |
587 | * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. |
588 | * All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones. |
589 | */ |
590 | typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); |
591 | typedef void (*ZSTD_freeFunction) (void* opaque, void* address); |
592 | typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; |
593 | static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ |
594 | |
595 | ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); |
596 | ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); |
597 | ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); |
598 | ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); |
599 | |
600 | ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, |
601 | ZSTD_dictLoadMethod_e dictLoadMethod, |
602 | ZSTD_dictContentType_e dictContentType, |
603 | ZSTD_compressionParameters cParams, |
604 | ZSTD_customMem customMem); |
605 | |
606 | ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, |
607 | ZSTD_dictLoadMethod_e dictLoadMethod, |
608 | ZSTD_dictContentType_e dictContentType, |
609 | ZSTD_customMem customMem); |
610 | |
611 | |
612 | |
613 | /*************************************** |
614 | * Advanced compression functions |
615 | ***************************************/ |
616 | |
617 | /*! ZSTD_createCDict_byReference() : |
618 | * Create a digested dictionary for compression |
619 | * Dictionary content is simply referenced, and therefore stays in dictBuffer. |
620 | * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */ |
621 | ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); |
622 | |
623 | /*! ZSTD_getCParams() : |
624 | * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. |
625 | * `estimatedSrcSize` value is optional, select 0 if not known */ |
626 | ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); |
627 | |
628 | /*! ZSTD_getParams() : |
629 | * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. |
630 | * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ |
631 | ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); |
632 | |
633 | /*! ZSTD_checkCParams() : |
634 | * Ensure param values remain within authorized range */ |
635 | ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); |
636 | |
637 | /*! ZSTD_adjustCParams() : |
638 | * optimize params for a given `srcSize` and `dictSize`. |
639 | * both values are optional, select `0` if unknown. */ |
640 | ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); |
641 | |
642 | /*! ZSTD_compress_advanced() : |
643 | * Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */ |
644 | ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, |
645 | void* dst, size_t dstCapacity, |
646 | const void* src, size_t srcSize, |
647 | const void* dict,size_t dictSize, |
648 | ZSTD_parameters params); |
649 | |
650 | /*! ZSTD_compress_usingCDict_advanced() : |
651 | * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ |
652 | ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, |
653 | void* dst, size_t dstCapacity, |
654 | const void* src, size_t srcSize, |
655 | const ZSTD_CDict* cdict, ZSTD_frameParameters fParams); |
656 | |
657 | |
658 | /*--- Advanced decompression functions ---*/ |
659 | |
660 | /*! ZSTD_isFrame() : |
661 | * Tells if the content of `buffer` starts with a valid Frame Identifier. |
662 | * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. |
663 | * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. |
664 | * Note 3 : Skippable Frame Identifiers are considered valid. */ |
665 | ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); |
666 | |
667 | /*! ZSTD_createDDict_byReference() : |
668 | * Create a digested dictionary, ready to start decompression operation without startup delay. |
669 | * Dictionary content is referenced, and therefore stays in dictBuffer. |
670 | * It is important that dictBuffer outlives DDict, |
671 | * it must remain read accessible throughout the lifetime of DDict */ |
672 | ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); |
673 | |
674 | |
675 | /*! ZSTD_getDictID_fromDict() : |
676 | * Provides the dictID stored within dictionary. |
677 | * if @return == 0, the dictionary is not conformant with Zstandard specification. |
678 | * It can still be loaded, but as a content-only dictionary. */ |
679 | ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); |
680 | |
681 | /*! ZSTD_getDictID_fromDDict() : |
682 | * Provides the dictID of the dictionary loaded into `ddict`. |
683 | * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. |
684 | * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ |
685 | ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); |
686 | |
687 | /*! ZSTD_getDictID_fromFrame() : |
688 | * Provides the dictID required to decompressed the frame stored within `src`. |
689 | * If @return == 0, the dictID could not be decoded. |
690 | * This could for one of the following reasons : |
691 | * - The frame does not require a dictionary to be decoded (most common case). |
692 | * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. |
693 | * Note : this use case also happens when using a non-conformant dictionary. |
694 | * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). |
695 | * - This is not a Zstandard frame. |
696 | * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ |
697 | ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); |
698 | |
699 | |
700 | /******************************************************************** |
701 | * Advanced streaming functions |
702 | ********************************************************************/ |
703 | |
704 | /*===== Advanced Streaming compression functions =====*/ |
705 | ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */ |
706 | ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/ |
707 | ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, |
708 | ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ |
709 | ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ |
710 | ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */ |
711 | |
712 | /*! ZSTD_resetCStream() : |
713 | * start a new compression job, using same parameters from previous job. |
714 | * This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. |
715 | * Note that zcs must be init at least once before using ZSTD_resetCStream(). |
716 | * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. |
717 | * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. |
718 | * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, |
719 | * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. |
720 | * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ |
721 | ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); |
722 | |
723 | |
724 | typedef struct { |
725 | unsigned long long ingested; |
726 | unsigned long long consumed; |
727 | unsigned long long produced; |
728 | } ZSTD_frameProgression; |
729 | |
730 | /* ZSTD_getFrameProgression(): |
731 | * tells how much data has been ingested (read from input) |
732 | * consumed (input actually compressed) and produced (output) for current frame. |
733 | * Therefore, (ingested - consumed) is amount of input data buffered internally, not yet compressed. |
734 | * Can report progression inside worker threads (multi-threading and non-blocking mode). |
735 | */ |
736 | ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); |
737 | |
738 | |
739 | |
740 | /*===== Advanced Streaming decompression functions =====*/ |
741 | typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e; |
742 | ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); /* obsolete : this API will be removed in a future version */ |
743 | ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */ |
744 | ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */ |
745 | ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ |
746 | |
747 | |
748 | /********************************************************************* |
749 | * Buffer-less and synchronous inner streaming functions |
750 | * |
751 | * This is an advanced API, giving full control over buffer management, for users which need direct control over memory. |
752 | * But it's also a complex one, with several restrictions, documented below. |
753 | * Prefer normal streaming API for an easier experience. |
754 | ********************************************************************* */ |
755 | |
756 | /** |
757 | Buffer-less streaming compression (synchronous mode) |
758 | |
759 | A ZSTD_CCtx object is required to track streaming operations. |
760 | Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. |
761 | ZSTD_CCtx object can be re-used multiple times within successive compression operations. |
762 | |
763 | Start by initializing a context. |
764 | Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, |
765 | or ZSTD_compressBegin_advanced(), for finer parameter control. |
766 | It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() |
767 | |
768 | Then, consume your input using ZSTD_compressContinue(). |
769 | There are some important considerations to keep in mind when using this advanced function : |
770 | - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. |
771 | - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. |
772 | - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. |
773 | Worst case evaluation is provided by ZSTD_compressBound(). |
774 | ZSTD_compressContinue() doesn't guarantee recover after a failed compression. |
775 | - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). |
776 | It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) |
777 | - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. |
778 | In which case, it will "discard" the relevant memory section from its history. |
779 | |
780 | Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. |
781 | It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. |
782 | Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. |
783 | |
784 | `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. |
785 | */ |
786 | |
787 | /*===== Buffer-less streaming compression functions =====*/ |
788 | ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); |
789 | ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); |
790 | ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ |
791 | ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ |
792 | ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ |
793 | ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ |
794 | |
795 | ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
796 | ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
797 | |
798 | |
799 | /*- |
800 | Buffer-less streaming decompression (synchronous mode) |
801 | |
802 | A ZSTD_DCtx object is required to track streaming operations. |
803 | Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. |
804 | A ZSTD_DCtx object can be re-used multiple times. |
805 | |
806 | First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). |
807 | Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. |
808 | Data fragment must be large enough to ensure successful decoding. |
809 | `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. |
810 | @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. |
811 | >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. |
812 | errorCode, which can be tested using ZSTD_isError(). |
813 | |
814 | It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, |
815 | such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). |
816 | Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. |
817 | As a consequence, check that values remain within valid application range. |
818 | For example, do not allocate memory blindly, check that `windowSize` is within expectation. |
819 | Each application can set its own limits, depending on local restrictions. |
820 | For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. |
821 | |
822 | ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. |
823 | ZSTD_decompressContinue() is very sensitive to contiguity, |
824 | if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, |
825 | or that previous contiguous segment is large enough to properly handle maximum back-reference distance. |
826 | There are multiple ways to guarantee this condition. |
827 | |
828 | The most memory efficient way is to use a round buffer of sufficient size. |
829 | Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), |
830 | which can @return an error code if required value is too large for current system (in 32-bits mode). |
831 | In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, |
832 | up to the moment there is not enough room left in the buffer to guarantee decoding another full block, |
833 | which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. |
834 | At which point, decoding can resume from the beginning of the buffer. |
835 | Note that already decoded data stored in the buffer should be flushed before being overwritten. |
836 | |
837 | There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. |
838 | |
839 | Finally, if you control the compression process, you can also ignore all buffer size rules, |
840 | as long as the encoder and decoder progress in "lock-step", |
841 | aka use exactly the same buffer sizes, break contiguity at the same place, etc. |
842 | |
843 | Once buffers are setup, start decompression, with ZSTD_decompressBegin(). |
844 | If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). |
845 | |
846 | Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. |
847 | ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). |
848 | ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. |
849 | |
850 | @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). |
851 | It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. |
852 | It can also be an error code, which can be tested with ZSTD_isError(). |
853 | |
854 | A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. |
855 | Context can then be reset to start a new decompression. |
856 | |
857 | Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). |
858 | This information is not required to properly decode a frame. |
859 | |
860 | == Special case : skippable frames == |
861 | |
862 | Skippable frames allow integration of user-defined data into a flow of concatenated frames. |
863 | Skippable frames will be ignored (skipped) by decompressor. |
864 | The format of skippable frames is as follows : |
865 | a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F |
866 | b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits |
867 | c) Frame Content - any content (User Data) of length equal to Frame Size |
868 | For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. |
869 | For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. |
870 | */ |
871 | |
872 | /*===== Buffer-less streaming decompression functions =====*/ |
873 | typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; |
874 | typedef struct { |
875 | unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ |
876 | unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ |
877 | unsigned blockSizeMax; |
878 | ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ |
879 | unsigned headerSize; |
880 | unsigned dictID; |
881 | unsigned checksumFlag; |
882 | } ZSTD_frameHeader; |
883 | ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ |
884 | ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ |
885 | |
886 | ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); |
887 | ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); |
888 | ZSTDLIB_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); |
889 | |
890 | ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); |
891 | ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
892 | |
893 | /* misc */ |
894 | ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); |
895 | typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; |
896 | ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); |
897 | |
898 | |
899 | |
900 | /* ============================================ */ |
901 | /** New advanced API (experimental) */ |
902 | /* ============================================ */ |
903 | |
904 | /* notes on API design : |
905 | * In this proposal, parameters are pushed one by one into an existing context, |
906 | * and then applied on all subsequent compression jobs. |
907 | * When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT. |
908 | * |
909 | * This API is intended to replace all others advanced / experimental API entry points. |
910 | * But it stands a reasonable chance to become "stable", after a reasonable testing period. |
911 | */ |
912 | |
913 | /* note on naming convention : |
914 | * Initially, the API favored names like ZSTD_setCCtxParameter() . |
915 | * In this proposal, convention is changed towards ZSTD_CCtx_setParameter() . |
916 | * The main driver is that it identifies more clearly the target object type. |
917 | * It feels clearer when considering multiple targets : |
918 | * ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter()) |
919 | * ZSTD_CCtxParams_setParameter() (rather than ZSTD_setCCtxParamsParameter() ) |
920 | * etc... |
921 | */ |
922 | |
923 | /* note on enum design : |
924 | * All enum will be pinned to explicit values before reaching "stable API" status */ |
925 | |
926 | typedef enum { |
927 | /* Opened question : should we have a format ZSTD_f_auto ? |
928 | * Today, it would mean exactly the same as ZSTD_f_zstd1. |
929 | * But, in the future, should several formats become supported, |
930 | * on the compression side, it would mean "default format". |
931 | * On the decompression side, it would mean "automatic format detection", |
932 | * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames". |
933 | * Since meaning is a little different, another option could be to define different enums for compression and decompression. |
934 | * This question could be kept for later, when there are actually multiple formats to support, |
935 | * but there is also the question of pinning enum values, and pinning value `0` is especially important */ |
936 | ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ |
937 | ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number. |
938 | * Useful to save 4 bytes per generated frame. |
939 | * Decoder cannot recognise automatically this format, requiring instructions. */ |
940 | } ZSTD_format_e; |
941 | |
942 | typedef enum { |
943 | /* compression format */ |
944 | ZSTD_p_format = 10, /* See ZSTD_format_e enum definition. |
945 | * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */ |
946 | |
947 | /* compression parameters */ |
948 | ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table |
949 | * Default level is ZSTD_CLEVEL_DEFAULT==3. |
950 | * Special: value 0 means "do not change cLevel". |
951 | * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type. |
952 | * Note 2 : setting a level sets all default values of other compression parameters. |
953 | * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */ |
954 | ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2. |
955 | * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. |
956 | * Special: value 0 means "use default windowLog". |
957 | * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27) |
958 | * requires explicitly allowing such window size during decompression stage. */ |
959 | ZSTD_p_hashLog, /* Size of the probe table, as a power of 2. |
960 | * Resulting table size is (1 << (hashLog+2)). |
961 | * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. |
962 | * Larger tables improve compression ratio of strategies <= dFast, |
963 | * and improve speed of strategies > dFast. |
964 | * Special: value 0 means "use default hashLog". */ |
965 | ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2. |
966 | * Resulting table size is (1 << (chainLog+2)). |
967 | * Larger tables result in better and slower compression. |
968 | * This parameter is useless when using "fast" strategy. |
969 | * Special: value 0 means "use default chainLog". */ |
970 | ZSTD_p_searchLog, /* Number of search attempts, as a power of 2. |
971 | * More attempts result in better and slower compression. |
972 | * This parameter is useless when using "fast" and "dFast" strategies. |
973 | * Special: value 0 means "use default searchLog". */ |
974 | ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller). |
975 | * Larger values make faster compression and decompression, but decrease ratio. |
976 | * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX. |
977 | * Note that currently, for all strategies < btopt, effective minimum is 4. |
978 | * , for all strategies > fast, effective maximum is 6. |
979 | * Special: value 0 means "use default minMatchLength". */ |
980 | ZSTD_p_targetLength, /* Impact of this field depends on strategy. |
981 | * For strategies btopt & btultra: |
982 | * Length of Match considered "good enough" to stop search. |
983 | * Larger values make compression stronger, and slower. |
984 | * For strategy fast: |
985 | * Distance between match sampling. |
986 | * Larger values make compression faster, and weaker. |
987 | * Special: value 0 means "use default targetLength". */ |
988 | ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition. |
989 | * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility. |
990 | * The higher the value of selected strategy, the more complex it is, |
991 | * resulting in stronger and slower compression. |
992 | * Special: value 0 means "use default strategy". */ |
993 | |
994 | ZSTD_p_enableLongDistanceMatching=160, /* Enable long distance matching. |
995 | * This parameter is designed to improve compression ratio |
996 | * for large inputs, by finding large matches at long distance. |
997 | * It increases memory usage and window size. |
998 | * Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB |
999 | * except when expressly set to a different value. */ |
1000 | ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2. |
1001 | * Larger values increase memory usage and compression ratio, |
1002 | * but decrease compression speed. |
1003 | * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX |
1004 | * default: windowlog - 7. |
1005 | * Special: value 0 means "automatically determine hashlog". */ |
1006 | ZSTD_p_ldmMinMatch, /* Minimum match size for long distance matcher. |
1007 | * Larger/too small values usually decrease compression ratio. |
1008 | * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. |
1009 | * Special: value 0 means "use default value" (default: 64). */ |
1010 | ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution. |
1011 | * Larger values improve collision resolution but decrease compression speed. |
1012 | * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX . |
1013 | * Special: value 0 means "use default value" (default: 3). */ |
1014 | ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table. |
1015 | * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). |
1016 | * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. |
1017 | * Larger values improve compression speed. |
1018 | * Deviating far from default value will likely result in a compression ratio decrease. |
1019 | * Special: value 0 means "automatically determine hashEveryLog". */ |
1020 | |
1021 | /* frame parameters */ |
1022 | ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) |
1023 | * Content size must be known at the beginning of compression, |
1024 | * it is provided using ZSTD_CCtx_setPledgedSrcSize() */ |
1025 | ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */ |
1026 | ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */ |
1027 | |
1028 | /* multi-threading parameters */ |
1029 | /* These parameters are only useful if multi-threading is enabled (ZSTD_MULTITHREAD). |
1030 | * They return an error otherwise. */ |
1031 | ZSTD_p_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. |
1032 | * When nbWorkers >= 1, triggers asynchronous mode : |
1033 | * ZSTD_compress_generic() consumes some input, flush some output if possible, and immediately gives back control to caller, |
1034 | * while compression work is performed in parallel, within worker threads. |
1035 | * (note : a strong exception to this rule is when first invocation sets ZSTD_e_end : it becomes a blocking call). |
1036 | * More workers improve speed, but also increase memory usage. |
1037 | * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ |
1038 | ZSTD_p_jobSize, /* Size of a compression job. This value is enforced only in non-blocking mode. |
1039 | * Each compression job is completed in parallel, so this value indirectly controls the nb of active threads. |
1040 | * 0 means default, which is dynamically determined based on compression parameters. |
1041 | * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest. |
1042 | * The minimum size is automatically and transparently enforced */ |
1043 | ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job. |
1044 | * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */ |
1045 | |
1046 | /* =================================================================== */ |
1047 | /* experimental parameters - no stability guaranteed */ |
1048 | /* =================================================================== */ |
1049 | |
1050 | ZSTD_p_compressLiterals=1000, /* control huffman compression of literals (enabled) by default. |
1051 | * disabling it improves speed and decreases compression ratio by a large amount. |
1052 | * note : this setting is automatically updated when changing compression level. |
1053 | * positive compression levels set ZSTD_p_compressLiterals to 1. |
1054 | * negative compression levels set ZSTD_p_compressLiterals to 0. */ |
1055 | |
1056 | ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, |
1057 | * even when referencing into Dictionary content (default:0) */ |
1058 | |
1059 | } ZSTD_cParameter; |
1060 | |
1061 | |
1062 | /*! ZSTD_CCtx_setParameter() : |
1063 | * Set one compression parameter, selected by enum ZSTD_cParameter. |
1064 | * Setting a parameter is generally only possible during frame initialization (before starting compression), |
1065 | * except for a few exceptions which can be updated during compression: compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. |
1066 | * Note : when `value` is an enum, cast it to unsigned for proper type checking. |
1067 | * @result : informational value (typically, value being set clamped correctly), |
1068 | * or an error code (which can be tested with ZSTD_isError()). */ |
1069 | ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value); |
1070 | |
1071 | /*! ZSTD_CCtx_setPledgedSrcSize() : |
1072 | * Total input data size to be compressed as a single frame. |
1073 | * This value will be controlled at the end, and result in error if not respected. |
1074 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1075 | * Note 1 : 0 means zero, empty. |
1076 | * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. |
1077 | * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job. |
1078 | * Note 2 : If all data is provided and consumed in a single round, |
1079 | * this value is overriden by srcSize instead. */ |
1080 | ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); |
1081 | |
1082 | /*! ZSTD_CCtx_loadDictionary() : |
1083 | * Create an internal CDict from `dict` buffer. |
1084 | * Decompression will have to use same dictionary. |
1085 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1086 | * Special: Adding a NULL (or 0-size) dictionary invalidates previous dictionary, |
1087 | * meaning "return to no-dictionary mode". |
1088 | * Note 1 : Dictionary will be used for all future compression jobs. |
1089 | * To return to "no-dictionary" situation, load a NULL dictionary |
1090 | * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters. |
1091 | * For this reason, compression parameters cannot be changed anymore after loading a dictionary. |
1092 | * It's also a CPU consuming operation, with non-negligible impact on latency. |
1093 | * Note 3 :`dict` content will be copied internally. |
1094 | * Use ZSTD_CCtx_loadDictionary_byReference() to reference dictionary content instead. |
1095 | * In such a case, dictionary buffer must outlive its users. |
1096 | * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() |
1097 | * to precisely select how dictionary content must be interpreted. */ |
1098 | ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); |
1099 | ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); |
1100 | ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); |
1101 | |
1102 | |
1103 | /*! ZSTD_CCtx_refCDict() : |
1104 | * Reference a prepared dictionary, to be used for all next compression jobs. |
1105 | * Note that compression parameters are enforced from within CDict, |
1106 | * and supercede any compression parameter previously set within CCtx. |
1107 | * The dictionary will remain valid for future compression jobs using same CCtx. |
1108 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1109 | * Special : adding a NULL CDict means "return to no-dictionary mode". |
1110 | * Note 1 : Currently, only one dictionary can be managed. |
1111 | * Adding a new dictionary effectively "discards" any previous one. |
1112 | * Note 2 : CDict is just referenced, its lifetime must outlive CCtx. */ |
1113 | ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); |
1114 | |
1115 | /*! ZSTD_CCtx_refPrefix() : |
1116 | * Reference a prefix (single-usage dictionary) for next compression job. |
1117 | * Decompression need same prefix to properly regenerate data. |
1118 | * Prefix is **only used once**. Tables are discarded at end of compression job. |
1119 | * Subsequent compression jobs will be done without prefix (if none is explicitly referenced). |
1120 | * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead. |
1121 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1122 | * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary |
1123 | * Note 1 : Prefix buffer is referenced. It must outlive compression job. |
1124 | * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters. |
1125 | * It's a CPU consuming operation, with non-negligible impact on latency. |
1126 | * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). |
1127 | * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */ |
1128 | ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); |
1129 | ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); |
1130 | |
1131 | /*! ZSTD_CCtx_reset() : |
1132 | * Return a CCtx to clean state. |
1133 | * Useful after an error, or to interrupt an ongoing compression job and start a new one. |
1134 | * Any internal data not yet flushed is cancelled. |
1135 | * Dictionary (if any) is dropped. |
1136 | * All parameters are back to default values. |
1137 | * It's possible to modify compression parameters after a reset. |
1138 | */ |
1139 | ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); |
1140 | |
1141 | |
1142 | |
1143 | typedef enum { |
1144 | ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */ |
1145 | ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */ |
1146 | ZSTD_e_end /* flush any remaining data and close current frame. Any additional data starts a new frame. */ |
1147 | } ZSTD_EndDirective; |
1148 | |
1149 | /*! ZSTD_compress_generic() : |
1150 | * Behave about the same as ZSTD_compressStream. To note : |
1151 | * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter() |
1152 | * - Compression parameters cannot be changed once compression is started. |
1153 | * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize |
1154 | * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. |
1155 | * - In single-thread mode (default), function is blocking : it completed its job before returning to caller. |
1156 | * - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads, |
1157 | * and then immediately returns, just indicating that there is some data remaining to be flushed. |
1158 | * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. |
1159 | * - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller. |
1160 | * - @return provides a minimum amount of data remaining to be flushed from internal buffers |
1161 | * or an error code, which can be tested using ZSTD_isError(). |
1162 | * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. |
1163 | * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. |
1164 | * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. |
1165 | * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), |
1166 | * only ZSTD_e_end or ZSTD_e_flush operations are allowed. |
1167 | * Before starting a new compression job, or changing compression parameters, |
1168 | * it is required to fully flush internal buffers. |
1169 | */ |
1170 | ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, |
1171 | ZSTD_outBuffer* output, |
1172 | ZSTD_inBuffer* input, |
1173 | ZSTD_EndDirective endOp); |
1174 | |
1175 | |
1176 | /*! ZSTD_compress_generic_simpleArgs() : |
1177 | * Same as ZSTD_compress_generic(), |
1178 | * but using only integral types as arguments. |
1179 | * Argument list is larger than ZSTD_{in,out}Buffer, |
1180 | * but can be helpful for binders from dynamic languages |
1181 | * which have troubles handling structures containing memory pointers. |
1182 | */ |
1183 | ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs ( |
1184 | ZSTD_CCtx* cctx, |
1185 | void* dst, size_t dstCapacity, size_t* dstPos, |
1186 | const void* src, size_t srcSize, size_t* srcPos, |
1187 | ZSTD_EndDirective endOp); |
1188 | |
1189 | |
1190 | /*! ZSTD_CCtx_params : |
1191 | * Quick howto : |
1192 | * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure |
1193 | * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into |
1194 | * an existing ZSTD_CCtx_params structure. |
1195 | * This is similar to |
1196 | * ZSTD_CCtx_setParameter(). |
1197 | * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to |
1198 | * an existing CCtx. |
1199 | * These parameters will be applied to |
1200 | * all subsequent compression jobs. |
1201 | * - ZSTD_compress_generic() : Do compression using the CCtx. |
1202 | * - ZSTD_freeCCtxParams() : Free the memory. |
1203 | * |
1204 | * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() |
1205 | * for static allocation for single-threaded compression. |
1206 | */ |
1207 | ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); |
1208 | ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); |
1209 | |
1210 | |
1211 | /*! ZSTD_CCtxParams_reset() : |
1212 | * Reset params to default values. |
1213 | */ |
1214 | ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); |
1215 | |
1216 | /*! ZSTD_CCtxParams_init() : |
1217 | * Initializes the compression parameters of cctxParams according to |
1218 | * compression level. All other parameters are reset to their default values. |
1219 | */ |
1220 | ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); |
1221 | |
1222 | /*! ZSTD_CCtxParams_init_advanced() : |
1223 | * Initializes the compression and frame parameters of cctxParams according to |
1224 | * params. All other parameters are reset to their default values. |
1225 | */ |
1226 | ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); |
1227 | |
1228 | |
1229 | /*! ZSTD_CCtxParam_setParameter() : |
1230 | * Similar to ZSTD_CCtx_setParameter. |
1231 | * Set one compression parameter, selected by enum ZSTD_cParameter. |
1232 | * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). |
1233 | * Note : when `value` is an enum, cast it to unsigned for proper type checking. |
1234 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1235 | */ |
1236 | ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value); |
1237 | |
1238 | /*! ZSTD_CCtx_setParametersUsingCCtxParams() : |
1239 | * Apply a set of ZSTD_CCtx_params to the compression context. |
1240 | * This can be done even after compression is started, |
1241 | * if nbWorkers==0, this will have no impact until a new compression is started. |
1242 | * if nbWorkers>=1, new parameters will be picked up at next job, |
1243 | * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). |
1244 | */ |
1245 | ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( |
1246 | ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); |
1247 | |
1248 | |
1249 | /*=== Advanced parameters for decompression API ===*/ |
1250 | |
1251 | /* The following parameters must be set after creating a ZSTD_DCtx* (or ZSTD_DStream*) object, |
1252 | * but before starting decompression of a frame. |
1253 | */ |
1254 | |
1255 | /*! ZSTD_DCtx_loadDictionary() : |
1256 | * Create an internal DDict from dict buffer, |
1257 | * to be used to decompress next frames. |
1258 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1259 | * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, |
1260 | * meaning "return to no-dictionary mode". |
1261 | * Note 1 : `dict` content will be copied internally. |
1262 | * Use ZSTD_DCtx_loadDictionary_byReference() |
1263 | * to reference dictionary content instead. |
1264 | * In which case, the dictionary buffer must outlive its users. |
1265 | * Note 2 : Loading a dictionary involves building tables, |
1266 | * which has a non-negligible impact on CPU usage and latency. |
1267 | * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select |
1268 | * how dictionary content will be interpreted and loaded. |
1269 | */ |
1270 | ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); |
1271 | ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); |
1272 | ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); |
1273 | |
1274 | |
1275 | /*! ZSTD_DCtx_refDDict() : |
1276 | * Reference a prepared dictionary, to be used to decompress next frames. |
1277 | * The dictionary remains active for decompression of future frames using same DCtx. |
1278 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1279 | * Note 1 : Currently, only one dictionary can be managed. |
1280 | * Referencing a new dictionary effectively "discards" any previous one. |
1281 | * Special : adding a NULL DDict means "return to no-dictionary mode". |
1282 | * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. |
1283 | */ |
1284 | ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); |
1285 | |
1286 | |
1287 | /*! ZSTD_DCtx_refPrefix() : |
1288 | * Reference a prefix (single-usage dictionary) for next compression job. |
1289 | * Prefix is **only used once**. It must be explicitly referenced before each frame. |
1290 | * If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead. |
1291 | * @result : 0, or an error code (which can be tested with ZSTD_isError()). |
1292 | * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary |
1293 | * Note 2 : Prefix buffer is referenced. It must outlive compression job. |
1294 | * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). |
1295 | * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. |
1296 | * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. |
1297 | */ |
1298 | ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); |
1299 | ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); |
1300 | |
1301 | |
1302 | /*! ZSTD_DCtx_setMaxWindowSize() : |
1303 | * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. |
1304 | * This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario). |
1305 | * This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode. |
1306 | * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX) |
1307 | * @return : 0, or an error code (which can be tested using ZSTD_isError()). |
1308 | */ |
1309 | ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); |
1310 | |
1311 | |
1312 | /*! ZSTD_DCtx_setFormat() : |
1313 | * Instruct the decoder context about what kind of data to decode next. |
1314 | * This instruction is mandatory to decode data without a fully-formed header, |
1315 | * such ZSTD_f_zstd1_magicless for example. |
1316 | * @return : 0, or an error code (which can be tested using ZSTD_isError()). |
1317 | */ |
1318 | ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); |
1319 | |
1320 | |
1321 | /*! ZSTD_decompress_generic() : |
1322 | * Behave the same as ZSTD_decompressStream. |
1323 | * Decompression parameters cannot be changed once decompression is started. |
1324 | * @return : an error code, which can be tested using ZSTD_isError() |
1325 | * if >0, a hint, nb of expected input bytes for next invocation. |
1326 | * `0` means : a frame has just been fully decoded and flushed. |
1327 | */ |
1328 | ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, |
1329 | ZSTD_outBuffer* output, |
1330 | ZSTD_inBuffer* input); |
1331 | |
1332 | |
1333 | /*! ZSTD_decompress_generic_simpleArgs() : |
1334 | * Same as ZSTD_decompress_generic(), |
1335 | * but using only integral types as arguments. |
1336 | * Argument list is larger than ZSTD_{in,out}Buffer, |
1337 | * but can be helpful for binders from dynamic languages |
1338 | * which have troubles handling structures containing memory pointers. |
1339 | */ |
1340 | ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs ( |
1341 | ZSTD_DCtx* dctx, |
1342 | void* dst, size_t dstCapacity, size_t* dstPos, |
1343 | const void* src, size_t srcSize, size_t* srcPos); |
1344 | |
1345 | |
1346 | /*! ZSTD_DCtx_reset() : |
1347 | * Return a DCtx to clean state. |
1348 | * If a decompression was ongoing, any internal data not yet flushed is cancelled. |
1349 | * All parameters are back to default values, including sticky ones. |
1350 | * Dictionary (if any) is dropped. |
1351 | * Parameters can be modified again after a reset. |
1352 | */ |
1353 | ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx); |
1354 | |
1355 | |
1356 | |
1357 | /* ============================ */ |
1358 | /** Block level API */ |
1359 | /* ============================ */ |
1360 | |
1361 | /*! |
1362 | Block functions produce and decode raw zstd blocks, without frame metadata. |
1363 | Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). |
1364 | User will have to take in charge required information to regenerate data, such as compressed and content sizes. |
1365 | |
1366 | A few rules to respect : |
1367 | - Compressing and decompressing require a context structure |
1368 | + Use ZSTD_createCCtx() and ZSTD_createDCtx() |
1369 | - It is necessary to init context before starting |
1370 | + compression : any ZSTD_compressBegin*() variant, including with dictionary |
1371 | + decompression : any ZSTD_decompressBegin*() variant, including with dictionary |
1372 | + copyCCtx() and copyDCtx() can be used too |
1373 | - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB |
1374 | + If input is larger than a block size, it's necessary to split input data into multiple blocks |
1375 | + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead. |
1376 | Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. |
1377 | - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. |
1378 | In which case, nothing is produced into `dst`. |
1379 | + User must test for such outcome and deal directly with uncompressed data |
1380 | + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! |
1381 | + In case of multiple successive blocks, should some of them be uncompressed, |
1382 | decoder must be informed of their existence in order to follow proper history. |
1383 | Use ZSTD_insertBlock() for such a case. |
1384 | */ |
1385 | |
1386 | #define ZSTD_BLOCKSIZELOG_MAX 17 |
1387 | #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */ |
1388 | /*===== Raw zstd block functions =====*/ |
1389 | ZSTDLIB_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); |
1390 | ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
1391 | ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); |
1392 | ZSTDLIB_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ |
1393 | |
1394 | |
1395 | #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ |
1396 | |
1397 | #if defined (__cplusplus) |
1398 | } |
1399 | #endif |
1400 | |