1#include "bitpacking.h"
2
3#include <cstdint>
4#include <type_traits>
5
6namespace duckdb_fastpforlib {
7namespace internal {
8
9// Used for uint8_t, uint16_t and uint32_t
10template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
11typename std::enable_if<(DELTA + SHR) < TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict in,
12 TYPE *__restrict out) {
13 *out = ((*in) >> SHR) % (1 << DELTA);
14}
15
16// Used for uint8_t, uint16_t and uint32_t
17template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
18typename std::enable_if<(DELTA + SHR) >= TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict &in,
19 TYPE *__restrict out) {
20 *out = (*in) >> SHR;
21 ++in;
22
23 static const TYPE NEXT_SHR = SHR + DELTA - TYPE_SIZE;
24 *out |= ((*in) % (1U << NEXT_SHR)) << (TYPE_SIZE - SHR);
25}
26
27template <uint8_t DELTA, uint8_t SHR>
28typename std::enable_if<(DELTA + SHR) < 32>::type unpack_single_out(const uint32_t *__restrict in,
29 uint64_t *__restrict out) {
30 *out = ((static_cast<uint64_t>(*in)) >> SHR) % (1ULL << DELTA);
31}
32
33template <uint8_t DELTA, uint8_t SHR>
34typename std::enable_if<(DELTA + SHR) >= 32 && (DELTA + SHR) < 64>::type
35unpack_single_out(const uint32_t *__restrict &in, uint64_t *__restrict out) {
36 *out = static_cast<uint64_t>(*in) >> SHR;
37 ++in;
38 if (DELTA + SHR > 32) {
39 static const uint8_t NEXT_SHR = SHR + DELTA - 32;
40 *out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (32 - SHR);
41 }
42}
43
44template <uint8_t DELTA, uint8_t SHR>
45typename std::enable_if<(DELTA + SHR) >= 64>::type unpack_single_out(const uint32_t *__restrict &in,
46 uint64_t *__restrict out) {
47 *out = static_cast<uint64_t>(*in) >> SHR;
48 ++in;
49
50 *out |= static_cast<uint64_t>(*in) << (32 - SHR);
51 ++in;
52
53 if (DELTA + SHR > 64) {
54 static const uint8_t NEXT_SHR = DELTA + SHR - 64;
55 *out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (64 - SHR);
56 }
57}
58
59// Used for uint8_t, uint16_t and uint32_t
60template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
61 typename std::enable_if < DELTA + SHL<TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict out) {
62 if (SHL == 0) {
63 *out = in & MASK;
64 } else {
65 *out |= (in & MASK) << SHL;
66 }
67}
68
69// Used for uint8_t, uint16_t and uint32_t
70template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
71typename std::enable_if<DELTA + SHL >= TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict &out) {
72 *out |= in << SHL;
73 ++out;
74
75 if (DELTA + SHL > TYPE_SIZE) {
76 *out = (in & MASK) >> (TYPE_SIZE - SHL);
77 }
78}
79
80template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
81 typename std::enable_if < DELTA + SHL<32>::type pack_single_in64(const uint64_t in, uint32_t *__restrict out) {
82 if (SHL == 0) {
83 *out = static_cast<uint32_t>(in & MASK);
84 } else {
85 *out |= (in & MASK) << SHL;
86 }
87}
88template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
89 typename std::enable_if < DELTA + SHL >= 32 &&
90 DELTA + SHL<64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
91 if (SHL == 0) {
92 *out = static_cast<uint32_t>(in & MASK);
93 } else {
94 *out |= (in & MASK) << SHL;
95 }
96
97 ++out;
98
99 if (DELTA + SHL > 32) {
100 *out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
101 }
102}
103template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
104typename std::enable_if<DELTA + SHL >= 64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
105 *out |= in << SHL;
106 ++out;
107
108 *out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
109 ++out;
110
111 if (DELTA + SHL > 64) {
112 *out = (in & MASK) >> (64 - SHL);
113 }
114}
115template <uint16_t DELTA, uint16_t OINDEX = 0>
116struct Unroller8 {
117 static void Unpack(const uint8_t *__restrict &in, uint8_t *__restrict out) {
118 unpack_single_out<DELTA, (DELTA * OINDEX) % 8>(in, out + OINDEX);
119
120 Unroller8<DELTA, OINDEX + 1>::Unpack(in, out);
121 }
122
123 static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
124 pack_single_in<uint8_t, DELTA, (DELTA * OINDEX) % 8, (1U << DELTA) - 1>(in[OINDEX], out);
125
126 Unroller8<DELTA, OINDEX + 1>::Pack(in, out);
127 }
128
129};\
130template <uint16_t DELTA>
131struct Unroller8<DELTA, 7> {
132 enum { SHIFT = (DELTA * 7) % 8 };
133
134 static void Unpack(const uint8_t *__restrict in, uint8_t *__restrict out) {
135 out[7] = (*in) >> SHIFT;
136 }
137
138 static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
139 *out |= (in[7] << SHIFT);
140 }
141};
142
143template <uint16_t DELTA, uint16_t OINDEX = 0>
144struct Unroller16 {
145 static void Unpack(const uint16_t *__restrict &in, uint16_t *__restrict out) {
146 unpack_single_out<DELTA, (DELTA * OINDEX) % 16>(in, out + OINDEX);
147
148 Unroller16<DELTA, OINDEX + 1>::Unpack(in, out);
149 }
150
151 static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
152 pack_single_in<uint16_t, DELTA, (DELTA * OINDEX) % 16, (1U << DELTA) - 1>(in[OINDEX], out);
153
154 Unroller16<DELTA, OINDEX + 1>::Pack(in, out);
155 }
156
157};
158
159template <uint16_t DELTA>
160struct Unroller16<DELTA, 15> {
161 enum { SHIFT = (DELTA * 15) % 16 };
162
163 static void Unpack(const uint16_t *__restrict in, uint16_t *__restrict out) {
164 out[15] = (*in) >> SHIFT;
165 }
166
167 static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
168 *out |= (in[15] << SHIFT);
169 }
170};
171
172template <uint16_t DELTA, uint16_t OINDEX = 0>
173struct Unroller {
174 static void Unpack(const uint32_t *__restrict &in, uint32_t *__restrict out) {
175 unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);
176
177 Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
178 }
179
180 static void Unpack(const uint32_t *__restrict &in, uint64_t *__restrict out) {
181 unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);
182
183 Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
184 }
185
186 static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
187 pack_single_in<uint32_t, DELTA, (DELTA * OINDEX) % 32, (1U << DELTA) - 1>(in[OINDEX], out);
188
189 Unroller<DELTA, OINDEX + 1>::Pack(in, out);
190 }
191
192 static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
193 pack_single_in64<DELTA, (DELTA * OINDEX) % 32, (1ULL << DELTA) - 1>(in[OINDEX], out);
194
195 Unroller<DELTA, OINDEX + 1>::Pack(in, out);
196 }
197};
198
199template <uint16_t DELTA>
200struct Unroller<DELTA, 31> {
201 enum { SHIFT = (DELTA * 31) % 32 };
202
203 static void Unpack(const uint32_t *__restrict in, uint32_t *__restrict out) {
204 out[31] = (*in) >> SHIFT;
205 }
206
207 static void Unpack(const uint32_t *__restrict in, uint64_t *__restrict out) {
208 out[31] = (*in) >> SHIFT;
209 if (DELTA > 32) {
210 ++in;
211 out[31] |= static_cast<uint64_t>(*in) << (32 - SHIFT);
212 }
213 }
214
215 static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
216 *out |= (in[31] << SHIFT);
217 }
218
219 static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
220 *out |= (in[31] << SHIFT);
221 if (DELTA > 32) {
222 ++out;
223 *out = static_cast<uint32_t>(in[31] >> (32 - SHIFT));
224 }
225 }
226};
227
228// Special cases
229void __fastunpack0(const uint8_t *__restrict, uint8_t *__restrict out) {
230 for (uint8_t i = 0; i < 8; ++i)
231 *(out++) = 0;
232}
233
234void __fastunpack0(const uint16_t *__restrict, uint16_t *__restrict out) {
235 for (uint16_t i = 0; i < 16; ++i)
236 *(out++) = 0;
237}
238
239void __fastunpack0(const uint32_t *__restrict, uint32_t *__restrict out) {
240 for (uint32_t i = 0; i < 32; ++i)
241 *(out++) = 0;
242}
243
244void __fastunpack0(const uint32_t *__restrict, uint64_t *__restrict out) {
245 for (uint32_t i = 0; i < 32; ++i)
246 *(out++) = 0;
247}
248
249void __fastpack0(const uint8_t *__restrict, uint8_t *__restrict) {
250}
251void __fastpack0(const uint16_t *__restrict, uint16_t *__restrict) {
252}
253void __fastpack0(const uint32_t *__restrict, uint32_t *__restrict) {
254}
255void __fastpack0(const uint64_t *__restrict, uint32_t *__restrict) {
256}
257
258// fastunpack for 8 bits
259void __fastunpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
260 Unroller8<1>::Unpack(in, out);
261}
262
263void __fastunpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
264 Unroller8<2>::Unpack(in, out);
265}
266
267void __fastunpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
268 Unroller8<3>::Unpack(in, out);
269}
270
271void __fastunpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
272 for (uint8_t outer = 0; outer < 4; ++outer) {
273 for (uint8_t inwordpointer = 0; inwordpointer < 8; inwordpointer += 4)
274 *(out++) = ((*in) >> inwordpointer) % (1U << 4);
275 ++in;
276 }
277}
278
279void __fastunpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
280 Unroller8<5>::Unpack(in, out);
281}
282
283void __fastunpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
284 Unroller8<6>::Unpack(in, out);
285}
286
287void __fastunpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
288 Unroller8<7>::Unpack(in, out);
289}
290
291void __fastunpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
292 for (int k = 0; k < 8; ++k)
293 out[k] = in[k];
294}
295
296
297// fastunpack for 16 bits
298void __fastunpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
299 Unroller16<1>::Unpack(in, out);
300}
301
302void __fastunpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
303 Unroller16<2>::Unpack(in, out);
304}
305
306void __fastunpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
307 Unroller16<3>::Unpack(in, out);
308}
309
310void __fastunpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
311 for (uint16_t outer = 0; outer < 4; ++outer) {
312 for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 4)
313 *(out++) = ((*in) >> inwordpointer) % (1U << 4);
314 ++in;
315 }
316}
317
318void __fastunpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
319 Unroller16<5>::Unpack(in, out);
320}
321
322void __fastunpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
323 Unroller16<6>::Unpack(in, out);
324}
325
326void __fastunpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
327 Unroller16<7>::Unpack(in, out);
328}
329
330void __fastunpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
331 for (uint16_t outer = 0; outer < 8; ++outer) {
332 for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 8)
333 *(out++) = ((*in) >> inwordpointer) % (1U << 8);
334 ++in;
335 }
336}
337
338void __fastunpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
339 Unroller16<9>::Unpack(in, out);
340}
341
342void __fastunpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
343 Unroller16<10>::Unpack(in, out);
344}
345
346void __fastunpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
347 Unroller16<11>::Unpack(in, out);
348}
349
350void __fastunpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
351 Unroller16<12>::Unpack(in, out);
352}
353
354void __fastunpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
355 Unroller16<13>::Unpack(in, out);
356}
357
358void __fastunpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
359 Unroller16<14>::Unpack(in, out);
360}
361
362void __fastunpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
363 Unroller16<15>::Unpack(in, out);
364}
365
366void __fastunpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
367 for (int k = 0; k < 16; ++k)
368 out[k] = in[k];
369}
370
371// fastunpack for 32 bits
372void __fastunpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
373 Unroller<1>::Unpack(in, out);
374}
375
376void __fastunpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
377 Unroller<2>::Unpack(in, out);
378}
379
380void __fastunpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
381 Unroller<3>::Unpack(in, out);
382}
383
384void __fastunpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
385 for (uint32_t outer = 0; outer < 4; ++outer) {
386 for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
387 *(out++) = ((*in) >> inwordpointer) % (1U << 4);
388 ++in;
389 }
390}
391
392void __fastunpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
393 Unroller<5>::Unpack(in, out);
394}
395
396void __fastunpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
397 Unroller<6>::Unpack(in, out);
398}
399
400void __fastunpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
401 Unroller<7>::Unpack(in, out);
402}
403
404void __fastunpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
405 for (uint32_t outer = 0; outer < 8; ++outer) {
406 for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8)
407 *(out++) = ((*in) >> inwordpointer) % (1U << 8);
408 ++in;
409 }
410}
411
412void __fastunpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
413 Unroller<9>::Unpack(in, out);
414}
415
416void __fastunpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
417 Unroller<10>::Unpack(in, out);
418}
419
420void __fastunpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
421 Unroller<11>::Unpack(in, out);
422}
423
424void __fastunpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
425 Unroller<12>::Unpack(in, out);
426}
427
428void __fastunpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
429 Unroller<13>::Unpack(in, out);
430}
431
432void __fastunpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
433 Unroller<14>::Unpack(in, out);
434}
435
436void __fastunpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
437 Unroller<15>::Unpack(in, out);
438}
439
440void __fastunpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
441 for (uint32_t outer = 0; outer < 16; ++outer) {
442 for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
443 *(out++) = ((*in) >> inwordpointer) % (1U << 16);
444 ++in;
445 }
446}
447
448void __fastunpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
449 Unroller<17>::Unpack(in, out);
450}
451
452void __fastunpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
453 Unroller<18>::Unpack(in, out);
454}
455
456void __fastunpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
457 Unroller<19>::Unpack(in, out);
458}
459
460void __fastunpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
461 Unroller<20>::Unpack(in, out);
462}
463
464void __fastunpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
465 Unroller<21>::Unpack(in, out);
466}
467
468void __fastunpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
469 Unroller<22>::Unpack(in, out);
470}
471
472void __fastunpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
473 Unroller<23>::Unpack(in, out);
474}
475
476void __fastunpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
477 Unroller<24>::Unpack(in, out);
478}
479
480void __fastunpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
481 Unroller<25>::Unpack(in, out);
482}
483
484void __fastunpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
485 Unroller<26>::Unpack(in, out);
486}
487
488void __fastunpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
489 Unroller<27>::Unpack(in, out);
490}
491
492void __fastunpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
493 Unroller<28>::Unpack(in, out);
494}
495
496void __fastunpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
497 Unroller<29>::Unpack(in, out);
498}
499
500void __fastunpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
501 Unroller<30>::Unpack(in, out);
502}
503
504void __fastunpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
505 Unroller<31>::Unpack(in, out);
506}
507
508void __fastunpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
509 for (int k = 0; k < 32; ++k)
510 out[k] = in[k];
511}
512
513// fastupack for 64 bits
514void __fastunpack1(const uint32_t *__restrict in, uint64_t *__restrict out) {
515 Unroller<1>::Unpack(in, out);
516}
517
518void __fastunpack2(const uint32_t *__restrict in, uint64_t *__restrict out) {
519 Unroller<2>::Unpack(in, out);
520}
521
522void __fastunpack3(const uint32_t *__restrict in, uint64_t *__restrict out) {
523 Unroller<3>::Unpack(in, out);
524}
525
526void __fastunpack4(const uint32_t *__restrict in, uint64_t *__restrict out) {
527 for (uint32_t outer = 0; outer < 4; ++outer) {
528 for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
529 *(out++) = ((*in) >> inwordpointer) % (1U << 4);
530 ++in;
531 }
532}
533
534void __fastunpack5(const uint32_t *__restrict in, uint64_t *__restrict out) {
535 Unroller<5>::Unpack(in, out);
536}
537
538void __fastunpack6(const uint32_t *__restrict in, uint64_t *__restrict out) {
539 Unroller<6>::Unpack(in, out);
540}
541
542void __fastunpack7(const uint32_t *__restrict in, uint64_t *__restrict out) {
543 Unroller<7>::Unpack(in, out);
544}
545
546void __fastunpack8(const uint32_t *__restrict in, uint64_t *__restrict out) {
547 for (uint32_t outer = 0; outer < 8; ++outer) {
548 for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8) {
549 *(out++) = ((*in) >> inwordpointer) % (1U << 8);
550 }
551 ++in;
552 }
553}
554
555void __fastunpack9(const uint32_t *__restrict in, uint64_t *__restrict out) {
556 Unroller<9>::Unpack(in, out);
557}
558
559void __fastunpack10(const uint32_t *__restrict in, uint64_t *__restrict out) {
560 Unroller<10>::Unpack(in, out);
561}
562
563void __fastunpack11(const uint32_t *__restrict in, uint64_t *__restrict out) {
564 Unroller<11>::Unpack(in, out);
565}
566
567void __fastunpack12(const uint32_t *__restrict in, uint64_t *__restrict out) {
568 Unroller<12>::Unpack(in, out);
569}
570
571void __fastunpack13(const uint32_t *__restrict in, uint64_t *__restrict out) {
572 Unroller<13>::Unpack(in, out);
573}
574
575void __fastunpack14(const uint32_t *__restrict in, uint64_t *__restrict out) {
576 Unroller<14>::Unpack(in, out);
577}
578
579void __fastunpack15(const uint32_t *__restrict in, uint64_t *__restrict out) {
580 Unroller<15>::Unpack(in, out);
581}
582
583void __fastunpack16(const uint32_t *__restrict in, uint64_t *__restrict out) {
584 for (uint32_t outer = 0; outer < 16; ++outer) {
585 for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
586 *(out++) = ((*in) >> inwordpointer) % (1U << 16);
587 ++in;
588 }
589}
590
591void __fastunpack17(const uint32_t *__restrict in, uint64_t *__restrict out) {
592 Unroller<17>::Unpack(in, out);
593}
594
595void __fastunpack18(const uint32_t *__restrict in, uint64_t *__restrict out) {
596 Unroller<18>::Unpack(in, out);
597}
598
599void __fastunpack19(const uint32_t *__restrict in, uint64_t *__restrict out) {
600 Unroller<19>::Unpack(in, out);
601}
602
603void __fastunpack20(const uint32_t *__restrict in, uint64_t *__restrict out) {
604 Unroller<20>::Unpack(in, out);
605}
606
607void __fastunpack21(const uint32_t *__restrict in, uint64_t *__restrict out) {
608 Unroller<21>::Unpack(in, out);
609}
610
611void __fastunpack22(const uint32_t *__restrict in, uint64_t *__restrict out) {
612 Unroller<22>::Unpack(in, out);
613}
614
615void __fastunpack23(const uint32_t *__restrict in, uint64_t *__restrict out) {
616 Unroller<23>::Unpack(in, out);
617}
618
619void __fastunpack24(const uint32_t *__restrict in, uint64_t *__restrict out) {
620 Unroller<24>::Unpack(in, out);
621}
622
623void __fastunpack25(const uint32_t *__restrict in, uint64_t *__restrict out) {
624 Unroller<25>::Unpack(in, out);
625}
626
627void __fastunpack26(const uint32_t *__restrict in, uint64_t *__restrict out) {
628 Unroller<26>::Unpack(in, out);
629}
630
631void __fastunpack27(const uint32_t *__restrict in, uint64_t *__restrict out) {
632 Unroller<27>::Unpack(in, out);
633}
634
635void __fastunpack28(const uint32_t *__restrict in, uint64_t *__restrict out) {
636 Unroller<28>::Unpack(in, out);
637}
638
639void __fastunpack29(const uint32_t *__restrict in, uint64_t *__restrict out) {
640 Unroller<29>::Unpack(in, out);
641}
642
643void __fastunpack30(const uint32_t *__restrict in, uint64_t *__restrict out) {
644 Unroller<30>::Unpack(in, out);
645}
646
647void __fastunpack31(const uint32_t *__restrict in, uint64_t *__restrict out) {
648 Unroller<31>::Unpack(in, out);
649}
650
651void __fastunpack32(const uint32_t *__restrict in, uint64_t *__restrict out) {
652 for (int k = 0; k < 32; ++k)
653 out[k] = in[k];
654}
655
656void __fastunpack33(const uint32_t *__restrict in, uint64_t *__restrict out) {
657 Unroller<33>::Unpack(in, out);
658}
659
660void __fastunpack34(const uint32_t *__restrict in, uint64_t *__restrict out) {
661 Unroller<34>::Unpack(in, out);
662}
663
664void __fastunpack35(const uint32_t *__restrict in, uint64_t *__restrict out) {
665 Unroller<35>::Unpack(in, out);
666}
667
668void __fastunpack36(const uint32_t *__restrict in, uint64_t *__restrict out) {
669 Unroller<36>::Unpack(in, out);
670}
671
672void __fastunpack37(const uint32_t *__restrict in, uint64_t *__restrict out) {
673 Unroller<37>::Unpack(in, out);
674}
675
676void __fastunpack38(const uint32_t *__restrict in, uint64_t *__restrict out) {
677 Unroller<38>::Unpack(in, out);
678}
679
680void __fastunpack39(const uint32_t *__restrict in, uint64_t *__restrict out) {
681 Unroller<39>::Unpack(in, out);
682}
683
684void __fastunpack40(const uint32_t *__restrict in, uint64_t *__restrict out) {
685 Unroller<40>::Unpack(in, out);
686}
687
688void __fastunpack41(const uint32_t *__restrict in, uint64_t *__restrict out) {
689 Unroller<41>::Unpack(in, out);
690}
691
692void __fastunpack42(const uint32_t *__restrict in, uint64_t *__restrict out) {
693 Unroller<42>::Unpack(in, out);
694}
695
696void __fastunpack43(const uint32_t *__restrict in, uint64_t *__restrict out) {
697 Unroller<43>::Unpack(in, out);
698}
699
700void __fastunpack44(const uint32_t *__restrict in, uint64_t *__restrict out) {
701 Unroller<44>::Unpack(in, out);
702}
703
704void __fastunpack45(const uint32_t *__restrict in, uint64_t *__restrict out) {
705 Unroller<45>::Unpack(in, out);
706}
707
708void __fastunpack46(const uint32_t *__restrict in, uint64_t *__restrict out) {
709 Unroller<46>::Unpack(in, out);
710}
711
712void __fastunpack47(const uint32_t *__restrict in, uint64_t *__restrict out) {
713 Unroller<47>::Unpack(in, out);
714}
715
716void __fastunpack48(const uint32_t *__restrict in, uint64_t *__restrict out) {
717 Unroller<48>::Unpack(in, out);
718}
719
720void __fastunpack49(const uint32_t *__restrict in, uint64_t *__restrict out) {
721 Unroller<49>::Unpack(in, out);
722}
723
724void __fastunpack50(const uint32_t *__restrict in, uint64_t *__restrict out) {
725 Unroller<50>::Unpack(in, out);
726}
727
728void __fastunpack51(const uint32_t *__restrict in, uint64_t *__restrict out) {
729 Unroller<51>::Unpack(in, out);
730}
731
732void __fastunpack52(const uint32_t *__restrict in, uint64_t *__restrict out) {
733 Unroller<52>::Unpack(in, out);
734}
735
736void __fastunpack53(const uint32_t *__restrict in, uint64_t *__restrict out) {
737 Unroller<53>::Unpack(in, out);
738}
739
740void __fastunpack54(const uint32_t *__restrict in, uint64_t *__restrict out) {
741 Unroller<54>::Unpack(in, out);
742}
743
744void __fastunpack55(const uint32_t *__restrict in, uint64_t *__restrict out) {
745 Unroller<55>::Unpack(in, out);
746}
747
748void __fastunpack56(const uint32_t *__restrict in, uint64_t *__restrict out) {
749 Unroller<56>::Unpack(in, out);
750}
751
752void __fastunpack57(const uint32_t *__restrict in, uint64_t *__restrict out) {
753 Unroller<57>::Unpack(in, out);
754}
755
756void __fastunpack58(const uint32_t *__restrict in, uint64_t *__restrict out) {
757 Unroller<58>::Unpack(in, out);
758}
759
760void __fastunpack59(const uint32_t *__restrict in, uint64_t *__restrict out) {
761 Unroller<59>::Unpack(in, out);
762}
763
764void __fastunpack60(const uint32_t *__restrict in, uint64_t *__restrict out) {
765 Unroller<60>::Unpack(in, out);
766}
767
768void __fastunpack61(const uint32_t *__restrict in, uint64_t *__restrict out) {
769 Unroller<61>::Unpack(in, out);
770}
771
772void __fastunpack62(const uint32_t *__restrict in, uint64_t *__restrict out) {
773 Unroller<62>::Unpack(in, out);
774}
775
776void __fastunpack63(const uint32_t *__restrict in, uint64_t *__restrict out) {
777 Unroller<63>::Unpack(in, out);
778}
779
780void __fastunpack64(const uint32_t *__restrict in, uint64_t *__restrict out) {
781 for (int k = 0; k < 32; ++k) {
782 out[k] = in[k * 2];
783 out[k] |= static_cast<uint64_t>(in[k * 2 + 1]) << 32;
784 }
785}
786
787// fastpack for 8 bits
788
789void __fastpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
790 Unroller8<1>::Pack(in, out);
791}
792
793void __fastpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
794 Unroller8<2>::Pack(in, out);
795}
796
797void __fastpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
798 Unroller8<3>::Pack(in, out);
799}
800
801void __fastpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
802 Unroller8<4>::Pack(in, out);
803}
804
805void __fastpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
806 Unroller8<5>::Pack(in, out);
807}
808
809void __fastpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
810 Unroller8<6>::Pack(in, out);
811}
812
813void __fastpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
814 Unroller8<7>::Pack(in, out);
815}
816
817void __fastpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
818 for (int k = 0; k < 8; ++k)
819 out[k] = in[k];
820}
821
822// fastpack for 16 bits
823
824void __fastpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
825 Unroller16<1>::Pack(in, out);
826}
827
828void __fastpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
829 Unroller16<2>::Pack(in, out);
830}
831
832void __fastpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
833 Unroller16<3>::Pack(in, out);
834}
835
836void __fastpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
837 Unroller16<4>::Pack(in, out);
838}
839
840void __fastpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
841 Unroller16<5>::Pack(in, out);
842}
843
844void __fastpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
845 Unroller16<6>::Pack(in, out);
846}
847
848void __fastpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
849 Unroller16<7>::Pack(in, out);
850}
851
852void __fastpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
853 Unroller16<8>::Pack(in, out);
854}
855
856void __fastpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
857 Unroller16<9>::Pack(in, out);
858}
859
860void __fastpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
861 Unroller16<10>::Pack(in, out);
862}
863
864void __fastpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
865 Unroller16<11>::Pack(in, out);
866}
867
868void __fastpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
869 Unroller16<12>::Pack(in, out);
870}
871
872void __fastpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
873 Unroller16<13>::Pack(in, out);
874}
875
876void __fastpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
877 Unroller16<14>::Pack(in, out);
878}
879
880void __fastpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
881 Unroller16<15>::Pack(in, out);
882}
883
884void __fastpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
885 for (int k = 0; k < 16; ++k)
886 out[k] = in[k];
887}
888
889
890// fastpack for 32 bits
891
892void __fastpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
893 Unroller<1>::Pack(in, out);
894}
895
896void __fastpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
897 Unroller<2>::Pack(in, out);
898}
899
900void __fastpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
901 Unroller<3>::Pack(in, out);
902}
903
904void __fastpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
905 Unroller<4>::Pack(in, out);
906}
907
908void __fastpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
909 Unroller<5>::Pack(in, out);
910}
911
912void __fastpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
913 Unroller<6>::Pack(in, out);
914}
915
916void __fastpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
917 Unroller<7>::Pack(in, out);
918}
919
920void __fastpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
921 Unroller<8>::Pack(in, out);
922}
923
924void __fastpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
925 Unroller<9>::Pack(in, out);
926}
927
928void __fastpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
929 Unroller<10>::Pack(in, out);
930}
931
932void __fastpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
933 Unroller<11>::Pack(in, out);
934}
935
936void __fastpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
937 Unroller<12>::Pack(in, out);
938}
939
940void __fastpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
941 Unroller<13>::Pack(in, out);
942}
943
944void __fastpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
945 Unroller<14>::Pack(in, out);
946}
947
948void __fastpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
949 Unroller<15>::Pack(in, out);
950}
951
952void __fastpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
953 Unroller<16>::Pack(in, out);
954}
955
956void __fastpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
957 Unroller<17>::Pack(in, out);
958}
959
960void __fastpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
961 Unroller<18>::Pack(in, out);
962}
963
964void __fastpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
965 Unroller<19>::Pack(in, out);
966}
967
968void __fastpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
969 Unroller<20>::Pack(in, out);
970}
971
972void __fastpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
973 Unroller<21>::Pack(in, out);
974}
975
976void __fastpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
977 Unroller<22>::Pack(in, out);
978}
979
980void __fastpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
981 Unroller<23>::Pack(in, out);
982}
983
984void __fastpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
985 Unroller<24>::Pack(in, out);
986}
987
988void __fastpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
989 Unroller<25>::Pack(in, out);
990}
991
992void __fastpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
993 Unroller<26>::Pack(in, out);
994}
995
996void __fastpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
997 Unroller<27>::Pack(in, out);
998}
999
1000void __fastpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
1001 Unroller<28>::Pack(in, out);
1002}
1003
1004void __fastpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
1005 Unroller<29>::Pack(in, out);
1006}
1007
1008void __fastpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
1009 Unroller<30>::Pack(in, out);
1010}
1011
1012void __fastpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
1013 Unroller<31>::Pack(in, out);
1014}
1015
1016void __fastpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
1017 for (int k = 0; k < 32; ++k)
1018 out[k] = in[k];
1019}
1020
1021// fastpack for 64 bits
1022
1023void __fastpack1(const uint64_t *__restrict in, uint32_t *__restrict out) {
1024 Unroller<1>::Pack(in, out);
1025}
1026
1027void __fastpack2(const uint64_t *__restrict in, uint32_t *__restrict out) {
1028 Unroller<2>::Pack(in, out);
1029}
1030
1031void __fastpack3(const uint64_t *__restrict in, uint32_t *__restrict out) {
1032 Unroller<3>::Pack(in, out);
1033}
1034
1035void __fastpack4(const uint64_t *__restrict in, uint32_t *__restrict out) {
1036 Unroller<4>::Pack(in, out);
1037}
1038
1039void __fastpack5(const uint64_t *__restrict in, uint32_t *__restrict out) {
1040 Unroller<5>::Pack(in, out);
1041}
1042
1043void __fastpack6(const uint64_t *__restrict in, uint32_t *__restrict out) {
1044 Unroller<6>::Pack(in, out);
1045}
1046
1047void __fastpack7(const uint64_t *__restrict in, uint32_t *__restrict out) {
1048 Unroller<7>::Pack(in, out);
1049}
1050
1051void __fastpack8(const uint64_t *__restrict in, uint32_t *__restrict out) {
1052 Unroller<8>::Pack(in, out);
1053}
1054
1055void __fastpack9(const uint64_t *__restrict in, uint32_t *__restrict out) {
1056 Unroller<9>::Pack(in, out);
1057}
1058
1059void __fastpack10(const uint64_t *__restrict in, uint32_t *__restrict out) {
1060 Unroller<10>::Pack(in, out);
1061}
1062
1063void __fastpack11(const uint64_t *__restrict in, uint32_t *__restrict out) {
1064 Unroller<11>::Pack(in, out);
1065}
1066
1067void __fastpack12(const uint64_t *__restrict in, uint32_t *__restrict out) {
1068 Unroller<12>::Pack(in, out);
1069}
1070
1071void __fastpack13(const uint64_t *__restrict in, uint32_t *__restrict out) {
1072 Unroller<13>::Pack(in, out);
1073}
1074
1075void __fastpack14(const uint64_t *__restrict in, uint32_t *__restrict out) {
1076 Unroller<14>::Pack(in, out);
1077}
1078
1079void __fastpack15(const uint64_t *__restrict in, uint32_t *__restrict out) {
1080 Unroller<15>::Pack(in, out);
1081}
1082
1083void __fastpack16(const uint64_t *__restrict in, uint32_t *__restrict out) {
1084 Unroller<16>::Pack(in, out);
1085}
1086
1087void __fastpack17(const uint64_t *__restrict in, uint32_t *__restrict out) {
1088 Unroller<17>::Pack(in, out);
1089}
1090
1091void __fastpack18(const uint64_t *__restrict in, uint32_t *__restrict out) {
1092 Unroller<18>::Pack(in, out);
1093}
1094
1095void __fastpack19(const uint64_t *__restrict in, uint32_t *__restrict out) {
1096 Unroller<19>::Pack(in, out);
1097}
1098
1099void __fastpack20(const uint64_t *__restrict in, uint32_t *__restrict out) {
1100 Unroller<20>::Pack(in, out);
1101}
1102
1103void __fastpack21(const uint64_t *__restrict in, uint32_t *__restrict out) {
1104 Unroller<21>::Pack(in, out);
1105}
1106
1107void __fastpack22(const uint64_t *__restrict in, uint32_t *__restrict out) {
1108 Unroller<22>::Pack(in, out);
1109}
1110
1111void __fastpack23(const uint64_t *__restrict in, uint32_t *__restrict out) {
1112 Unroller<23>::Pack(in, out);
1113}
1114
1115void __fastpack24(const uint64_t *__restrict in, uint32_t *__restrict out) {
1116 Unroller<24>::Pack(in, out);
1117}
1118
1119void __fastpack25(const uint64_t *__restrict in, uint32_t *__restrict out) {
1120 Unroller<25>::Pack(in, out);
1121}
1122
1123void __fastpack26(const uint64_t *__restrict in, uint32_t *__restrict out) {
1124 Unroller<26>::Pack(in, out);
1125}
1126
1127void __fastpack27(const uint64_t *__restrict in, uint32_t *__restrict out) {
1128 Unroller<27>::Pack(in, out);
1129}
1130
1131void __fastpack28(const uint64_t *__restrict in, uint32_t *__restrict out) {
1132 Unroller<28>::Pack(in, out);
1133}
1134
1135void __fastpack29(const uint64_t *__restrict in, uint32_t *__restrict out) {
1136 Unroller<29>::Pack(in, out);
1137}
1138
1139void __fastpack30(const uint64_t *__restrict in, uint32_t *__restrict out) {
1140 Unroller<30>::Pack(in, out);
1141}
1142
1143void __fastpack31(const uint64_t *__restrict in, uint32_t *__restrict out) {
1144 Unroller<31>::Pack(in, out);
1145}
1146
1147void __fastpack32(const uint64_t *__restrict in, uint32_t *__restrict out) {
1148 for (int k = 0; k < 32; ++k) {
1149 out[k] = static_cast<uint32_t>(in[k]);
1150 }
1151}
1152
1153void __fastpack33(const uint64_t *__restrict in, uint32_t *__restrict out) {
1154 Unroller<33>::Pack(in, out);
1155}
1156
1157void __fastpack34(const uint64_t *__restrict in, uint32_t *__restrict out) {
1158 Unroller<34>::Pack(in, out);
1159}
1160
1161void __fastpack35(const uint64_t *__restrict in, uint32_t *__restrict out) {
1162 Unroller<35>::Pack(in, out);
1163}
1164
1165void __fastpack36(const uint64_t *__restrict in, uint32_t *__restrict out) {
1166 Unroller<36>::Pack(in, out);
1167}
1168
1169void __fastpack37(const uint64_t *__restrict in, uint32_t *__restrict out) {
1170 Unroller<37>::Pack(in, out);
1171}
1172
1173void __fastpack38(const uint64_t *__restrict in, uint32_t *__restrict out) {
1174 Unroller<38>::Pack(in, out);
1175}
1176
1177void __fastpack39(const uint64_t *__restrict in, uint32_t *__restrict out) {
1178 Unroller<39>::Pack(in, out);
1179}
1180
1181void __fastpack40(const uint64_t *__restrict in, uint32_t *__restrict out) {
1182 Unroller<40>::Pack(in, out);
1183}
1184
1185void __fastpack41(const uint64_t *__restrict in, uint32_t *__restrict out) {
1186 Unroller<41>::Pack(in, out);
1187}
1188
1189void __fastpack42(const uint64_t *__restrict in, uint32_t *__restrict out) {
1190 Unroller<42>::Pack(in, out);
1191}
1192
1193void __fastpack43(const uint64_t *__restrict in, uint32_t *__restrict out) {
1194 Unroller<43>::Pack(in, out);
1195}
1196
1197void __fastpack44(const uint64_t *__restrict in, uint32_t *__restrict out) {
1198 Unroller<44>::Pack(in, out);
1199}
1200
1201void __fastpack45(const uint64_t *__restrict in, uint32_t *__restrict out) {
1202 Unroller<45>::Pack(in, out);
1203}
1204
1205void __fastpack46(const uint64_t *__restrict in, uint32_t *__restrict out) {
1206 Unroller<46>::Pack(in, out);
1207}
1208
1209void __fastpack47(const uint64_t *__restrict in, uint32_t *__restrict out) {
1210 Unroller<47>::Pack(in, out);
1211}
1212
1213void __fastpack48(const uint64_t *__restrict in, uint32_t *__restrict out) {
1214 Unroller<48>::Pack(in, out);
1215}
1216
1217void __fastpack49(const uint64_t *__restrict in, uint32_t *__restrict out) {
1218 Unroller<49>::Pack(in, out);
1219}
1220
1221void __fastpack50(const uint64_t *__restrict in, uint32_t *__restrict out) {
1222 Unroller<50>::Pack(in, out);
1223}
1224
1225void __fastpack51(const uint64_t *__restrict in, uint32_t *__restrict out) {
1226 Unroller<51>::Pack(in, out);
1227}
1228
1229void __fastpack52(const uint64_t *__restrict in, uint32_t *__restrict out) {
1230 Unroller<52>::Pack(in, out);
1231}
1232
1233void __fastpack53(const uint64_t *__restrict in, uint32_t *__restrict out) {
1234 Unroller<53>::Pack(in, out);
1235}
1236
1237void __fastpack54(const uint64_t *__restrict in, uint32_t *__restrict out) {
1238 Unroller<54>::Pack(in, out);
1239}
1240
1241void __fastpack55(const uint64_t *__restrict in, uint32_t *__restrict out) {
1242 Unroller<55>::Pack(in, out);
1243}
1244
1245void __fastpack56(const uint64_t *__restrict in, uint32_t *__restrict out) {
1246 Unroller<56>::Pack(in, out);
1247}
1248
1249void __fastpack57(const uint64_t *__restrict in, uint32_t *__restrict out) {
1250 Unroller<57>::Pack(in, out);
1251}
1252
1253void __fastpack58(const uint64_t *__restrict in, uint32_t *__restrict out) {
1254 Unroller<58>::Pack(in, out);
1255}
1256
1257void __fastpack59(const uint64_t *__restrict in, uint32_t *__restrict out) {
1258 Unroller<59>::Pack(in, out);
1259}
1260
1261void __fastpack60(const uint64_t *__restrict in, uint32_t *__restrict out) {
1262 Unroller<60>::Pack(in, out);
1263}
1264
1265void __fastpack61(const uint64_t *__restrict in, uint32_t *__restrict out) {
1266 Unroller<61>::Pack(in, out);
1267}
1268
1269void __fastpack62(const uint64_t *__restrict in, uint32_t *__restrict out) {
1270 Unroller<62>::Pack(in, out);
1271}
1272
1273void __fastpack63(const uint64_t *__restrict in, uint32_t *__restrict out) {
1274 Unroller<63>::Pack(in, out);
1275}
1276
1277void __fastpack64(const uint64_t *__restrict in, uint32_t *__restrict out) {
1278 for (int i = 0; i < 32; ++i) {
1279 out[2 * i] = static_cast<uint32_t>(in[i]);
1280 out[2 * i + 1] = in[i] >> 32;
1281 }
1282}
1283} // namespace internal
1284} // namespace duckdb_fastpforlib
1285