1/* inffast.c -- fast decoding
2 * Copyright (C) 1995-2017 Mark Adler
3 * For conditions of distribution and use, see copyright notice in zlib.h
4 */
5
6#include "zbuild.h"
7#include "zutil.h"
8#include "inftrees.h"
9#include "inflate.h"
10#include "inffast.h"
11#include "inflate_p.h"
12#include "functable.h"
13
14
15/* Load 64 bits from IN and place the bytes at offset BITS in the result. */
16static inline uint64_t load_64_bits(const unsigned char *in, unsigned bits) {
17 uint64_t chunk;
18 memcpy(dest: &chunk, src: in, n: sizeof(chunk));
19
20#if BYTE_ORDER == LITTLE_ENDIAN
21 return chunk << bits;
22#else
23 return ZSWAP64(chunk) << bits;
24#endif
25}
26/*
27 Decode literal, length, and distance codes and write out the resulting
28 literal and match bytes until either not enough input or output is
29 available, an end-of-block is encountered, or a data error is encountered.
30 When large enough input and output buffers are supplied to inflate(), for
31 example, a 16K input buffer and a 64K output buffer, more than 95% of the
32 inflate execution time is spent in this routine.
33
34 Entry assumptions:
35
36 state->mode == LEN
37 strm->avail_in >= INFLATE_FAST_MIN_HAVE
38 strm->avail_out >= INFLATE_FAST_MIN_LEFT
39 start >= strm->avail_out
40 state->bits < 8
41
42 On return, state->mode is one of:
43
44 LEN -- ran out of enough output space or enough available input
45 TYPE -- reached end of block code, inflate() to interpret next block
46 BAD -- error in block data
47
48 Notes:
49
50 - The maximum input bits used by a length/distance pair is 15 bits for the
51 length code, 5 bits for the length extra, 15 bits for the distance code,
52 and 13 bits for the distance extra. This totals 48 bits, or six bytes.
53 Therefore if strm->avail_in >= 6, then there is enough input to avoid
54 checking for available input while decoding.
55
56 - On some architectures, it can be significantly faster (e.g. up to 1.2x
57 faster on x86_64) to load from strm->next_in 64 bits, or 8 bytes, at a
58 time, so INFLATE_FAST_MIN_HAVE == 8.
59
60 - The maximum bytes that a single length/distance pair can output is 258
61 bytes, which is the maximum length that can be coded. inflate_fast()
62 requires strm->avail_out >= 258 for each loop to avoid checking for
63 output space.
64 */
65void Z_INTERNAL zng_inflate_fast(PREFIX3(stream) *strm, unsigned long start) {
66 /* start: inflate()'s starting value for strm->avail_out */
67 struct inflate_state *state;
68 z_const unsigned char *in; /* local strm->next_in */
69 const unsigned char *last; /* have enough input while in < last */
70 unsigned char *out; /* local strm->next_out */
71 unsigned char *beg; /* inflate()'s initial strm->next_out */
72 unsigned char *end; /* while out < end, enough space available */
73 unsigned char *safe; /* can use chunkcopy provided out < safe */
74#ifdef INFLATE_STRICT
75 unsigned dmax; /* maximum distance from zlib header */
76#endif
77 unsigned wsize; /* window size or zero if not using window */
78 unsigned whave; /* valid bytes in the window */
79 unsigned wnext; /* window write index */
80 unsigned char *window; /* allocated sliding window, if wsize != 0 */
81
82 /* hold is a local copy of strm->hold. By default, hold satisfies the same
83 invariants that strm->hold does, namely that (hold >> bits) == 0. This
84 invariant is kept by loading bits into hold one byte at a time, like:
85
86 hold |= next_byte_of_input << bits; in++; bits += 8;
87
88 If we need to ensure that bits >= 15 then this code snippet is simply
89 repeated. Over one iteration of the outermost do/while loop, this
90 happens up to six times (48 bits of input), as described in the NOTES
91 above.
92
93 However, on some little endian architectures, it can be significantly
94 faster to load 64 bits once instead of 8 bits six times:
95
96 if (bits <= 16) {
97 hold |= next_8_bytes_of_input << bits; in += 6; bits += 48;
98 }
99
100 Unlike the simpler one byte load, shifting the next_8_bytes_of_input
101 by bits will overflow and lose those high bits, up to 2 bytes' worth.
102 The conservative estimate is therefore that we have read only 6 bytes
103 (48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
104 rest of the iteration, and we will not need to load another 8 bytes.
105
106 Inside this function, we no longer satisfy (hold >> bits) == 0, but
107 this is not problematic, even if that overflow does not land on an 8 bit
108 byte boundary. Those excess bits will eventually shift down lower as the
109 Huffman decoder consumes input, and when new input bits need to be loaded
110 into the bits variable, the same input bits will be or'ed over those
111 existing bits. A bitwise or is idempotent: (a | b | b) equals (a | b).
112 Note that we therefore write that load operation as "hold |= etc" and not
113 "hold += etc".
114
115 Outside that loop, at the end of the function, hold is bitwise and'ed
116 with (1<<bits)-1 to drop those excess bits so that, on function exit, we
117 keep the invariant that (state->hold >> state->bits) == 0.
118 */
119 uint64_t hold; /* local strm->hold */
120 unsigned bits; /* local strm->bits */
121 code const *lcode; /* local strm->lencode */
122 code const *dcode; /* local strm->distcode */
123 unsigned lmask; /* mask for first level of length codes */
124 unsigned dmask; /* mask for first level of distance codes */
125 const code *here; /* retrieved table entry */
126 unsigned op; /* code bits, operation, extra bits, or */
127 /* window position, window bytes to copy */
128 unsigned len; /* match length, unused bytes */
129 unsigned dist; /* match distance */
130 unsigned char *from; /* where to copy match from */
131 unsigned extra_safe; /* copy chunks safely in all cases */
132
133 /* copy state to local variables */
134 state = (struct inflate_state *)strm->state;
135 in = strm->next_in;
136 last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
137 out = strm->next_out;
138 beg = out - (start - strm->avail_out);
139 end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
140 safe = out + strm->avail_out;
141#ifdef INFLATE_STRICT
142 dmax = state->dmax;
143#endif
144 wsize = state->wsize;
145 whave = state->whave;
146 wnext = state->wnext;
147 window = state->window;
148 hold = state->hold;
149 bits = state->bits;
150 lcode = state->lencode;
151 dcode = state->distcode;
152 lmask = (1U << state->lenbits) - 1;
153 dmask = (1U << state->distbits) - 1;
154
155 /* Detect if out and window point to the same memory allocation. In this instance it is
156 necessary to use safe chunk copy functions to prevent overwriting the window. If the
157 window is overwritten then future matches with far distances will fail to copy correctly. */
158 extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + wsize);
159
160 /* decode literals and length/distances until end-of-block or not enough
161 input data or output space */
162 do {
163 if (bits < 15) {
164 hold |= load_64_bits(in, bits);
165 in += 6;
166 bits += 48;
167 }
168 here = lcode + (hold & lmask);
169 dolen:
170 DROPBITS(here->bits);
171 op = here->op;
172 if (op == 0) { /* literal */
173 Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
174 "inflate: literal '%c'\n" :
175 "inflate: literal 0x%02x\n", here->val));
176 *out++ = (unsigned char)(here->val);
177 } else if (op & 16) { /* length base */
178 len = here->val;
179 op &= 15; /* number of extra bits */
180 if (bits < op) {
181 hold |= load_64_bits(in, bits);
182 in += 6;
183 bits += 48;
184 }
185 len += BITS(op);
186 DROPBITS(op);
187 Tracevv((stderr, "inflate: length %u\n", len));
188 if (bits < 15) {
189 hold |= load_64_bits(in, bits);
190 in += 6;
191 bits += 48;
192 }
193 here = dcode + (hold & dmask);
194 dodist:
195 DROPBITS(here->bits);
196 op = here->op;
197 if (op & 16) { /* distance base */
198 dist = here->val;
199 op &= 15; /* number of extra bits */
200 if (bits < op) {
201 hold |= load_64_bits(in, bits);
202 in += 6;
203 bits += 48;
204 }
205 dist += BITS(op);
206#ifdef INFLATE_STRICT
207 if (dist > dmax) {
208 SET_BAD("invalid distance too far back");
209 break;
210 }
211#endif
212 DROPBITS(op);
213 Tracevv((stderr, "inflate: distance %u\n", dist));
214 op = (unsigned)(out - beg); /* max distance in output */
215 if (dist > op) { /* see if copy from window */
216 op = dist - op; /* distance back in window */
217 if (op > whave) {
218 if (state->sane) {
219 SET_BAD("invalid distance too far back");
220 break;
221 }
222#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
223 if (len <= op - whave) {
224 do {
225 *out++ = 0;
226 } while (--len);
227 continue;
228 }
229 len -= op - whave;
230 do {
231 *out++ = 0;
232 } while (--op > whave);
233 if (op == 0) {
234 from = out - dist;
235 do {
236 *out++ = *from++;
237 } while (--len);
238 continue;
239 }
240#endif
241 }
242 from = window;
243 if (wnext == 0) { /* very common case */
244 from += wsize - op;
245 } else if (wnext >= op) { /* contiguous in window */
246 from += wnext - op;
247 } else { /* wrap around window */
248 op -= wnext;
249 from += wsize - op;
250 if (op < len) { /* some from end of window */
251 len -= op;
252 out = functable.chunkcopy_safe(out, from, op, safe);
253 from = window; /* more from start of window */
254 op = wnext;
255 /* This (rare) case can create a situation where
256 the first chunkcopy below must be checked.
257 */
258 }
259 }
260 if (op < len) { /* still need some from output */
261 len -= op;
262 out = functable.chunkcopy_safe(out, from, op, safe);
263 out = functable.chunkunroll(out, &dist, &len);
264 out = functable.chunkcopy_safe(out, out - dist, len, safe);
265 } else {
266 out = functable.chunkcopy_safe(out, from, len, safe);
267 }
268 } else if (extra_safe) {
269 /* Whole reference is in range of current output. */
270 if (dist >= len || dist >= state->chunksize)
271 out = functable.chunkcopy_safe(out, out - dist, len, safe);
272 else
273 out = functable.chunkmemset_safe(out, dist, len, (unsigned)((safe - out) + 1));
274 } else {
275 /* Whole reference is in range of current output. No range checks are
276 necessary because we start with room for at least 258 bytes of output,
277 so unroll and roundoff operations can write beyond `out+len` so long
278 as they stay within 258 bytes of `out`.
279 */
280 if (dist >= len || dist >= state->chunksize)
281 out = functable.chunkcopy(out, out - dist, len);
282 else
283 out = functable.chunkmemset(out, dist, len);
284 }
285 } else if ((op & 64) == 0) { /* 2nd level distance code */
286 here = dcode + here->val + BITS(op);
287 goto dodist;
288 } else {
289 SET_BAD("invalid distance code");
290 break;
291 }
292 } else if ((op & 64) == 0) { /* 2nd level length code */
293 here = lcode + here->val + BITS(op);
294 goto dolen;
295 } else if (op & 32) { /* end-of-block */
296 Tracevv((stderr, "inflate: end of block\n"));
297 state->mode = TYPE;
298 break;
299 } else {
300 SET_BAD("invalid literal/length code");
301 break;
302 }
303 } while (in < last && out < end);
304
305 /* return unused bytes (on entry, bits < 8, so in won't go too far back) */
306 len = bits >> 3;
307 in -= len;
308 bits -= len << 3;
309 hold &= (UINT64_C(1) << bits) - 1;
310
311 /* update state and return */
312 strm->next_in = in;
313 strm->next_out = out;
314 strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
315 : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
316 strm->avail_out = (unsigned)(out < end ? (INFLATE_FAST_MIN_LEFT - 1) + (end - out)
317 : (INFLATE_FAST_MIN_LEFT - 1) - (out - end));
318
319 Assert(bits <= 32, "Remaining bits greater than 32");
320 state->hold = (uint32_t)hold;
321 state->bits = bits;
322 return;
323}
324
325/*
326 inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
327 - Using bit fields for code structure
328 - Different op definition to avoid & for extra bits (do & for table bits)
329 - Three separate decoding do-loops for direct, window, and wnext == 0
330 - Special case for distance > 1 copies to do overlapped load and store copy
331 - Explicit branch predictions (based on measured branch probabilities)
332 - Deferring match copy and interspersed it with decoding subsequent codes
333 - Swapping literal/length else
334 - Swapping window/direct else
335 - Larger unrolled copy loops (three is about right)
336 - Moving len -= 3 statement into middle of loop
337 */
338