1/* inffast.c -- fast decoding
2 * Copyright (C) 1995-2017 Mark Adler
3 * For conditions of distribution and use, see copyright notice in zlib.h
4 */
5
6#include "zbuild.h"
7#include "zutil.h"
8#include "inftrees.h"
9#include "inflate.h"
10#include "inffast.h"
11#include "inflate_p.h"
12#include "memcopy.h"
13
14/*
15 Decode literal, length, and distance codes and write out the resulting
16 literal and match bytes until either not enough input or output is
17 available, an end-of-block is encountered, or a data error is encountered.
18 When large enough input and output buffers are supplied to inflate(), for
19 example, a 16K input buffer and a 64K output buffer, more than 95% of the
20 inflate execution time is spent in this routine.
21
22 Entry assumptions:
23
24 state->mode == LEN
25 strm->avail_in >= INFLATE_FAST_MIN_HAVE
26 strm->avail_out >= INFLATE_FAST_MIN_LEFT
27 start >= strm->avail_out
28 state->bits < 8
29
30 On return, state->mode is one of:
31
32 LEN -- ran out of enough output space or enough available input
33 TYPE -- reached end of block code, inflate() to interpret next block
34 BAD -- error in block data
35
36 Notes:
37
38 - The maximum input bits used by a length/distance pair is 15 bits for the
39 length code, 5 bits for the length extra, 15 bits for the distance code,
40 and 13 bits for the distance extra. This totals 48 bits, or six bytes.
41 Therefore if strm->avail_in >= 6, then there is enough input to avoid
42 checking for available input while decoding.
43
44 - On some architectures, it can be significantly faster (e.g. up to 1.2x
45 faster on x86_64) to load from strm->next_in 64 bits, or 8 bytes, at a
46 time, so INFLATE_FAST_MIN_HAVE == 8.
47
48 - The maximum bytes that a single length/distance pair can output is 258
49 bytes, which is the maximum length that can be coded. inflate_fast()
50 requires strm->avail_out >= 258 for each loop to avoid checking for
51 output space.
52 */
53void ZLIB_INTERNAL zng_inflate_fast(PREFIX3(stream) *strm, unsigned long start) {
54 /* start: inflate()'s starting value for strm->avail_out */
55 struct inflate_state *state;
56 const unsigned char *in; /* local strm->next_in */
57 const unsigned char *last; /* have enough input while in < last */
58 unsigned char *out; /* local strm->next_out */
59 unsigned char *beg; /* inflate()'s initial strm->next_out */
60 unsigned char *end; /* while out < end, enough space available */
61#ifdef INFFAST_CHUNKSIZE
62 unsigned char *safe; /* can use chunkcopy provided out < safe */
63#endif
64#ifdef INFLATE_STRICT
65 unsigned dmax; /* maximum distance from zlib header */
66#endif
67 unsigned wsize; /* window size or zero if not using window */
68 unsigned whave; /* valid bytes in the window */
69 unsigned wnext; /* window write index */
70 unsigned char *window; /* allocated sliding window, if wsize != 0 */
71
72 /* hold is a local copy of strm->hold. By default, hold satisfies the same
73 invariants that strm->hold does, namely that (hold >> bits) == 0. This
74 invariant is kept by loading bits into hold one byte at a time, like:
75
76 hold |= next_byte_of_input << bits; in++; bits += 8;
77
78 If we need to ensure that bits >= 15 then this code snippet is simply
79 repeated. Over one iteration of the outermost do/while loop, this
80 happens up to six times (48 bits of input), as described in the NOTES
81 above.
82
83 However, on some little endian architectures, it can be significantly
84 faster to load 64 bits once instead of 8 bits six times:
85
86 if (bits <= 16) {
87 hold |= next_8_bytes_of_input << bits; in += 6; bits += 48;
88 }
89
90 Unlike the simpler one byte load, shifting the next_8_bytes_of_input
91 by bits will overflow and lose those high bits, up to 2 bytes' worth.
92 The conservative estimate is therefore that we have read only 6 bytes
93 (48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
94 rest of the iteration, and we will not need to load another 8 bytes.
95
96 Inside this function, we no longer satisfy (hold >> bits) == 0, but
97 this is not problematic, even if that overflow does not land on an 8 bit
98 byte boundary. Those excess bits will eventually shift down lower as the
99 Huffman decoder consumes input, and when new input bits need to be loaded
100 into the bits variable, the same input bits will be or'ed over those
101 existing bits. A bitwise or is idempotent: (a | b | b) equals (a | b).
102 Note that we therefore write that load operation as "hold |= etc" and not
103 "hold += etc".
104
105 Outside that loop, at the end of the function, hold is bitwise and'ed
106 with (1<<bits)-1 to drop those excess bits so that, on function exit, we
107 keep the invariant that (state->hold >> state->bits) == 0.
108 */
109 uint64_t hold; /* local strm->hold */
110 unsigned bits; /* local strm->bits */
111 code const *lcode; /* local strm->lencode */
112 code const *dcode; /* local strm->distcode */
113 unsigned lmask; /* mask for first level of length codes */
114 unsigned dmask; /* mask for first level of distance codes */
115 const code *here; /* retrieved table entry */
116 unsigned op; /* code bits, operation, extra bits, or */
117 /* window position, window bytes to copy */
118 unsigned len; /* match length, unused bytes */
119 unsigned dist; /* match distance */
120 unsigned char *from; /* where to copy match from */
121
122 /* copy state to local variables */
123 state = (struct inflate_state *)strm->state;
124 in = strm->next_in;
125 last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
126 out = strm->next_out;
127 beg = out - (start - strm->avail_out);
128 end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
129
130#ifdef INFFAST_CHUNKSIZE
131 safe = out + strm->avail_out;
132#endif
133#ifdef INFLATE_STRICT
134 dmax = state->dmax;
135#endif
136 wsize = state->wsize;
137 whave = state->whave;
138 wnext = state->wnext;
139 window = state->window;
140 hold = state->hold;
141 bits = state->bits;
142 lcode = state->lencode;
143 dcode = state->distcode;
144 lmask = (1U << state->lenbits) - 1;
145 dmask = (1U << state->distbits) - 1;
146
147 /* decode literals and length/distances until end-of-block or not enough
148 input data or output space */
149 do {
150 if (bits < 15) {
151 hold |= load_64_bits(in, bits);
152 in += 6;
153 bits += 48;
154 }
155 here = lcode + (hold & lmask);
156 dolen:
157 DROPBITS(here->bits);
158 op = here->op;
159 if (op == 0) { /* literal */
160 Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
161 "inflate: literal '%c'\n" :
162 "inflate: literal 0x%02x\n", here->val));
163 *out++ = (unsigned char)(here->val);
164 } else if (op & 16) { /* length base */
165 len = here->val;
166 op &= 15; /* number of extra bits */
167 if (op) {
168 if (bits < op) {
169 hold |= load_64_bits(in, bits);
170 in += 6;
171 bits += 48;
172 }
173 len += BITS(op);
174 DROPBITS(op);
175 }
176 Tracevv((stderr, "inflate: length %u\n", len));
177 if (bits < 15) {
178 hold |= load_64_bits(in, bits);
179 in += 6;
180 bits += 48;
181 }
182 here = dcode + (hold & dmask);
183 dodist:
184 DROPBITS(here->bits);
185 op = here->op;
186 if (op & 16) { /* distance base */
187 dist = here->val;
188 op &= 15; /* number of extra bits */
189 if (bits < op) {
190 hold |= load_64_bits(in, bits);
191 in += 6;
192 bits += 48;
193 }
194 dist += BITS(op);
195#ifdef INFLATE_STRICT
196 if (dist > dmax) {
197 strm->msg = (char *)"invalid distance too far back";
198 state->mode = BAD;
199 break;
200 }
201#endif
202 DROPBITS(op);
203 Tracevv((stderr, "inflate: distance %u\n", dist));
204 op = (unsigned)(out - beg); /* max distance in output */
205 if (dist > op) { /* see if copy from window */
206 op = dist - op; /* distance back in window */
207 if (op > whave) {
208 if (state->sane) {
209 strm->msg = (char *)"invalid distance too far back";
210 state->mode = BAD;
211 break;
212 }
213#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
214 if (len <= op - whave) {
215 do {
216 *out++ = 0;
217 } while (--len);
218 continue;
219 }
220 len -= op - whave;
221 do {
222 *out++ = 0;
223 } while (--op > whave);
224 if (op == 0) {
225 from = out - dist;
226 do {
227 *out++ = *from++;
228 } while (--len);
229 continue;
230 }
231#endif
232 }
233#ifdef INFFAST_CHUNKSIZE
234 from = window;
235 if (wnext == 0) { /* very common case */
236 from += wsize - op;
237 } else if (wnext >= op) { /* contiguous in window */
238 from += wnext - op;
239 } else { /* wrap around window */
240 op -= wnext;
241 from += wsize - op;
242 if (op < len) { /* some from end of window */
243 len -= op;
244 out = chunkcopysafe(out, from, op, safe);
245 from = window; /* more from start of window */
246 op = wnext;
247 /* This (rare) case can create a situation where
248 the first chunkcopy below must be checked.
249 */
250 }
251 }
252 if (op < len) { /* still need some from output */
253 len -= op;
254 out = chunkcopysafe(out, from, op, safe);
255 out = chunkunroll(out, &dist, &len);
256 out = chunkcopysafe(out, out - dist, len, safe);
257 } else {
258 out = chunkcopysafe(out, from, len, safe);
259 }
260#else
261 from = window;
262 if (wnext == 0) { /* very common case */
263 from += wsize - op;
264 if (op < len) { /* some from window */
265 len -= op;
266 do {
267 *out++ = *from++;
268 } while (--op);
269 from = out - dist; /* rest from output */
270 }
271 } else if (wnext < op) { /* wrap around window */
272 from += wsize + wnext - op;
273 op -= wnext;
274 if (op < len) { /* some from end of window */
275 len -= op;
276 do {
277 *out++ = *from++;
278 } while (--op);
279 from = window;
280 if (wnext < len) { /* some from start of window */
281 op = wnext;
282 len -= op;
283 do {
284 *out++ = *from++;
285 } while (--op);
286 from = out - dist; /* rest from output */
287 }
288 }
289 } else { /* contiguous in window */
290 from += wnext - op;
291 if (op < len) { /* some from window */
292 len -= op;
293 do {
294 *out++ = *from++;
295 } while (--op);
296 from = out - dist; /* rest from output */
297 }
298 }
299
300 out = chunk_copy(out, from, (int) (out - from), len);
301#endif
302 } else {
303#ifdef INFFAST_CHUNKSIZE
304 /* Whole reference is in range of current output. No
305 range checks are necessary because we start with room
306 for at least 258 bytes of output, so unroll and roundoff
307 operations can write beyond `out+len` so long as they
308 stay within 258 bytes of `out`.
309 */
310 if (dist >= len || dist >= INFFAST_CHUNKSIZE)
311 out = chunkcopy(out, out - dist, len);
312 else
313 out = chunkmemset(out, dist, len);
314#else
315 if (len < sizeof(uint64_t))
316 out = set_bytes(out, out - dist, dist, len);
317 else if (dist == 1)
318 out = byte_memset(out, len);
319 else
320 out = chunk_memset(out, out - dist, dist, len);
321#endif
322 }
323 } else if ((op & 64) == 0) { /* 2nd level distance code */
324 here = dcode + here->val + BITS(op);
325 goto dodist;
326 } else {
327 strm->msg = (char *)"invalid distance code";
328 state->mode = BAD;
329 break;
330 }
331 } else if ((op & 64) == 0) { /* 2nd level length code */
332 here = lcode + here->val + BITS(op);
333 goto dolen;
334 } else if (op & 32) { /* end-of-block */
335 Tracevv((stderr, "inflate: end of block\n"));
336 state->mode = TYPE;
337 break;
338 } else {
339 strm->msg = (char *)"invalid literal/length code";
340 state->mode = BAD;
341 break;
342 }
343 } while (in < last && out < end);
344
345 /* return unused bytes (on entry, bits < 8, so in won't go too far back) */
346 len = bits >> 3;
347 in -= len;
348 bits -= len << 3;
349 hold &= (UINT64_C(1) << bits) - 1;
350
351 /* update state and return */
352 strm->next_in = in;
353 strm->next_out = out;
354 strm->avail_in =
355 (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
356 : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
357 strm->avail_out =
358 (unsigned)(out < end ? (INFLATE_FAST_MIN_LEFT - 1) + (end - out)
359 : (INFLATE_FAST_MIN_LEFT - 1) - (out - end));
360
361 Assert(bits <= 32, "Remaining bits greater than 32");
362 state->hold = (uint32_t)hold;
363 state->bits = bits;
364 return;
365}
366
367/*
368 inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
369 - Using bit fields for code structure
370 - Different op definition to avoid & for extra bits (do & for table bits)
371 - Three separate decoding do-loops for direct, window, and wnext == 0
372 - Special case for distance > 1 copies to do overlapped load and store copy
373 - Explicit branch predictions (based on measured branch probabilities)
374 - Deferring match copy and interspersed it with decoding subsequent codes
375 - Swapping literal/length else
376 - Swapping window/direct else
377 - Larger unrolled copy loops (three is about right)
378 - Moving len -= 3 statement into middle of loop
379 */
380