1/* ******************************************************************
2 * huff0 huffman codec,
3 * part of Finite State Entropy library
4 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 *
6 * You can contact the author at :
7 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
8 *
9 * This source code is licensed under both the BSD-style license (found in the
10 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
11 * in the COPYING file in the root directory of this source tree).
12 * You may select, at your option, one of the above-listed licenses.
13****************************************************************** */
14
15#if defined (__cplusplus)
16extern "C" {
17#endif
18
19#ifndef HUF_H_298734234
20#define HUF_H_298734234
21
22/* *** Dependencies *** */
23#include "zstd_deps.h" /* size_t */
24#include "mem.h" /* U32 */
25#define FSE_STATIC_LINKING_ONLY
26#include "fse.h"
27
28
29/* *** Tool functions *** */
30#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
31size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
32
33/* Error Management */
34unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
35const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
36
37
38#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
39#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
40
41/* *** Constants *** */
42#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
43#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
44#define HUF_SYMBOLVALUE_MAX 255
45
46#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
47#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
48# error "HUF_TABLELOG_MAX is too large !"
49#endif
50
51
52/* ****************************************
53* Static allocation
54******************************************/
55/* HUF buffer bounds */
56#define HUF_CTABLEBOUND 129
57#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */
58#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
59
60/* static allocation of HUF's Compression Table */
61/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
62typedef size_t HUF_CElt; /* consider it an incomplete type */
63#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
64#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
65#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
66 HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
67
68/* static allocation of HUF's DTable */
69typedef U32 HUF_DTable;
70#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
71#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
72 HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
73#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
74 HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
75
76
77/* ****************************************
78* Advanced decompression functions
79******************************************/
80
81/**
82 * Huffman flags bitset.
83 * For all flags, 0 is the default value.
84 */
85typedef enum {
86 /**
87 * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
88 * Otherwise: Ignored.
89 */
90 HUF_flags_bmi2 = (1 << 0),
91 /**
92 * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
93 * If unset: Use heuristic to find the table depth.
94 */
95 HUF_flags_optimalDepth = (1 << 1),
96 /**
97 * If set: If the previous table can encode the input, always reuse the previous table.
98 * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
99 */
100 HUF_flags_preferRepeat = (1 << 2),
101 /**
102 * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
103 * If unset: Always histogram the entire input.
104 */
105 HUF_flags_suspectUncompressible = (1 << 3),
106 /**
107 * If set: Don't use assembly implementations
108 * If unset: Allow using assembly implementations
109 */
110 HUF_flags_disableAsm = (1 << 4),
111 /**
112 * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
113 * If unset: Use the fast decoding loop when possible.
114 */
115 HUF_flags_disableFast = (1 << 5)
116} HUF_flags_e;
117
118
119/* ****************************************
120 * HUF detailed API
121 * ****************************************/
122#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
123
124/*! HUF_compress() does the following:
125 * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
126 * 2. (optional) refine tableLog using HUF_optimalTableLog()
127 * 3. build Huffman table from count using HUF_buildCTable()
128 * 4. save Huffman table to memory buffer using HUF_writeCTable()
129 * 5. encode the data stream using HUF_compress4X_usingCTable()
130 *
131 * The following API allows targeting specific sub-functions for advanced tasks.
132 * For example, it's possible to compress several blocks using the same 'CTable',
133 * or to save and regenerate 'CTable' using external methods.
134 */
135unsigned HUF_minTableLog(unsigned symbolCardinality);
136unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
137unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
138 size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
139size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
140size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
141size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
142int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
143
144typedef enum {
145 HUF_repeat_none, /**< Cannot use the previous table */
146 HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
147 HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */
148 } HUF_repeat;
149
150/** HUF_compress4X_repeat() :
151 * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
152 * If it uses hufTable it does not modify hufTable or repeat.
153 * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
154 * If preferRepeat then the old table will always be used if valid.
155 * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
156size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
157 const void* src, size_t srcSize,
158 unsigned maxSymbolValue, unsigned tableLog,
159 void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
160 HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
161
162/** HUF_buildCTable_wksp() :
163 * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
164 * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
165 */
166#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
167#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
168size_t HUF_buildCTable_wksp (HUF_CElt* tree,
169 const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
170 void* workSpace, size_t wkspSize);
171
172/*! HUF_readStats() :
173 * Read compact Huffman tree, saved by HUF_writeCTable().
174 * `huffWeight` is destination buffer.
175 * @return : size read from `src` , or an error Code .
176 * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
177size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
178 U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
179 const void* src, size_t srcSize);
180
181/*! HUF_readStats_wksp() :
182 * Same as HUF_readStats() but takes an external workspace which must be
183 * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
184 * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
185 */
186#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
187#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
188size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
189 U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
190 const void* src, size_t srcSize,
191 void* workspace, size_t wkspSize,
192 int flags);
193
194/** HUF_readCTable() :
195 * Loading a CTable saved with HUF_writeCTable() */
196size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
197
198/** HUF_getNbBitsFromCTable() :
199 * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
200 * Note 1 : is not inlined, as HUF_CElt definition is private */
201U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
202
203/*
204 * HUF_decompress() does the following:
205 * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
206 * 2. build Huffman table from save, using HUF_readDTableX?()
207 * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
208 */
209
210/** HUF_selectDecoder() :
211 * Tells which decoder is likely to decode faster,
212 * based on a set of pre-computed metrics.
213 * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
214 * Assumption : 0 < dstSize <= 128 KB */
215U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
216
217/**
218 * The minimum workspace size for the `workSpace` used in
219 * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
220 *
221 * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
222 * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
223 * Buffer overflow errors may potentially occur if code modifications result in
224 * a required workspace size greater than that specified in the following
225 * macro.
226 */
227#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
228#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
229
230
231/* ====================== */
232/* single stream variants */
233/* ====================== */
234
235size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
236/** HUF_compress1X_repeat() :
237 * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
238 * If it uses hufTable it does not modify hufTable or repeat.
239 * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
240 * If preferRepeat then the old table will always be used if valid.
241 * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
242size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
243 const void* src, size_t srcSize,
244 unsigned maxSymbolValue, unsigned tableLog,
245 void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
246 HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
247
248size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
249#ifndef HUF_FORCE_DECOMPRESS_X1
250size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */
251#endif
252
253/* BMI2 variants.
254 * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
255 */
256size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
257#ifndef HUF_FORCE_DECOMPRESS_X2
258size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
259#endif
260size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
261size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
262#ifndef HUF_FORCE_DECOMPRESS_X2
263size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
264#endif
265#ifndef HUF_FORCE_DECOMPRESS_X1
266size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
267#endif
268
269#endif /* HUF_H_298734234 */
270
271#if defined (__cplusplus)
272}
273#endif
274