1 | // Copyright 2016 Google Inc. All Rights Reserved. |
2 | // |
3 | // Use of this source code is governed by a BSD-style license |
4 | // that can be found in the COPYING file in the root of the source |
5 | // tree. An additional intellectual property rights grant can be found |
6 | // in the file PATENTS. All contributing project authors may |
7 | // be found in the AUTHORS file in the root of the source tree. |
8 | // ----------------------------------------------------------------------------- |
9 | // |
10 | // SSE2 code common to several files. |
11 | // |
12 | // Author: Vincent Rabaud (vrabaud@google.com) |
13 | |
14 | #ifndef WEBP_DSP_COMMON_SSE2_H_ |
15 | #define WEBP_DSP_COMMON_SSE2_H_ |
16 | |
17 | #ifdef __cplusplus |
18 | extern "C" { |
19 | #endif |
20 | |
21 | #if defined(WEBP_USE_SSE2) |
22 | |
23 | #include <emmintrin.h> |
24 | |
25 | //------------------------------------------------------------------------------ |
26 | // Quite useful macro for debugging. Left here for convenience. |
27 | |
28 | #if 0 |
29 | #include <stdio.h> |
30 | static WEBP_INLINE void PrintReg(const __m128i r, const char* const name, |
31 | int size) { |
32 | int n; |
33 | union { |
34 | __m128i r; |
35 | uint8_t i8[16]; |
36 | uint16_t i16[8]; |
37 | uint32_t i32[4]; |
38 | uint64_t i64[2]; |
39 | } tmp; |
40 | tmp.r = r; |
41 | fprintf(stderr, "%s\t: " , name); |
42 | if (size == 8) { |
43 | for (n = 0; n < 16; ++n) fprintf(stderr, "%.2x " , tmp.i8[n]); |
44 | } else if (size == 16) { |
45 | for (n = 0; n < 8; ++n) fprintf(stderr, "%.4x " , tmp.i16[n]); |
46 | } else if (size == 32) { |
47 | for (n = 0; n < 4; ++n) fprintf(stderr, "%.8x " , tmp.i32[n]); |
48 | } else { |
49 | for (n = 0; n < 2; ++n) fprintf(stderr, "%.16lx " , tmp.i64[n]); |
50 | } |
51 | fprintf(stderr, "\n" ); |
52 | } |
53 | #endif |
54 | |
55 | //------------------------------------------------------------------------------ |
56 | // Math functions. |
57 | |
58 | // Return the sum of all the 8b in the register. |
59 | static WEBP_INLINE int VP8HorizontalAdd8b(const __m128i* const a) { |
60 | const __m128i zero = _mm_setzero_si128(); |
61 | const __m128i sad8x2 = _mm_sad_epu8(*a, zero); |
62 | // sum the two sads: sad8x2[0:1] + sad8x2[8:9] |
63 | const __m128i sum = _mm_add_epi32(sad8x2, _mm_shuffle_epi32(sad8x2, 2)); |
64 | return _mm_cvtsi128_si32(sum); |
65 | } |
66 | |
67 | // Transpose two 4x4 16b matrices horizontally stored in registers. |
68 | static WEBP_INLINE void VP8Transpose_2_4x4_16b( |
69 | const __m128i* const in0, const __m128i* const in1, |
70 | const __m128i* const in2, const __m128i* const in3, __m128i* const out0, |
71 | __m128i* const out1, __m128i* const out2, __m128i* const out3) { |
72 | // Transpose the two 4x4. |
73 | // a00 a01 a02 a03 b00 b01 b02 b03 |
74 | // a10 a11 a12 a13 b10 b11 b12 b13 |
75 | // a20 a21 a22 a23 b20 b21 b22 b23 |
76 | // a30 a31 a32 a33 b30 b31 b32 b33 |
77 | const __m128i transpose0_0 = _mm_unpacklo_epi16(*in0, *in1); |
78 | const __m128i transpose0_1 = _mm_unpacklo_epi16(*in2, *in3); |
79 | const __m128i transpose0_2 = _mm_unpackhi_epi16(*in0, *in1); |
80 | const __m128i transpose0_3 = _mm_unpackhi_epi16(*in2, *in3); |
81 | // a00 a10 a01 a11 a02 a12 a03 a13 |
82 | // a20 a30 a21 a31 a22 a32 a23 a33 |
83 | // b00 b10 b01 b11 b02 b12 b03 b13 |
84 | // b20 b30 b21 b31 b22 b32 b23 b33 |
85 | const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1); |
86 | const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3); |
87 | const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1); |
88 | const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3); |
89 | // a00 a10 a20 a30 a01 a11 a21 a31 |
90 | // b00 b10 b20 b30 b01 b11 b21 b31 |
91 | // a02 a12 a22 a32 a03 a13 a23 a33 |
92 | // b02 b12 a22 b32 b03 b13 b23 b33 |
93 | *out0 = _mm_unpacklo_epi64(transpose1_0, transpose1_1); |
94 | *out1 = _mm_unpackhi_epi64(transpose1_0, transpose1_1); |
95 | *out2 = _mm_unpacklo_epi64(transpose1_2, transpose1_3); |
96 | *out3 = _mm_unpackhi_epi64(transpose1_2, transpose1_3); |
97 | // a00 a10 a20 a30 b00 b10 b20 b30 |
98 | // a01 a11 a21 a31 b01 b11 b21 b31 |
99 | // a02 a12 a22 a32 b02 b12 b22 b32 |
100 | // a03 a13 a23 a33 b03 b13 b23 b33 |
101 | } |
102 | |
103 | //------------------------------------------------------------------------------ |
104 | // Channel mixing. |
105 | |
106 | // Function used several times in VP8PlanarTo24b. |
107 | // It samples the in buffer as follows: one every two unsigned char is stored |
108 | // at the beginning of the buffer, while the other half is stored at the end. |
109 | #define VP8PlanarTo24bHelper(IN, OUT) \ |
110 | do { \ |
111 | const __m128i v_mask = _mm_set1_epi16(0x00ff); \ |
112 | /* Take one every two upper 8b values.*/ \ |
113 | (OUT##0) = _mm_packus_epi16(_mm_and_si128((IN##0), v_mask), \ |
114 | _mm_and_si128((IN##1), v_mask)); \ |
115 | (OUT##1) = _mm_packus_epi16(_mm_and_si128((IN##2), v_mask), \ |
116 | _mm_and_si128((IN##3), v_mask)); \ |
117 | (OUT##2) = _mm_packus_epi16(_mm_and_si128((IN##4), v_mask), \ |
118 | _mm_and_si128((IN##5), v_mask)); \ |
119 | /* Take one every two lower 8b values.*/ \ |
120 | (OUT##3) = _mm_packus_epi16(_mm_srli_epi16((IN##0), 8), \ |
121 | _mm_srli_epi16((IN##1), 8)); \ |
122 | (OUT##4) = _mm_packus_epi16(_mm_srli_epi16((IN##2), 8), \ |
123 | _mm_srli_epi16((IN##3), 8)); \ |
124 | (OUT##5) = _mm_packus_epi16(_mm_srli_epi16((IN##4), 8), \ |
125 | _mm_srli_epi16((IN##5), 8)); \ |
126 | } while (0) |
127 | |
128 | // Pack the planar buffers |
129 | // rrrr... rrrr... gggg... gggg... bbbb... bbbb.... |
130 | // triplet by triplet in the output buffer rgb as rgbrgbrgbrgb ... |
131 | static WEBP_INLINE void VP8PlanarTo24b_SSE2( |
132 | __m128i* const in0, __m128i* const in1, __m128i* const in2, |
133 | __m128i* const in3, __m128i* const in4, __m128i* const in5) { |
134 | // The input is 6 registers of sixteen 8b but for the sake of explanation, |
135 | // let's take 6 registers of four 8b values. |
136 | // To pack, we will keep taking one every two 8b integer and move it |
137 | // around as follows: |
138 | // Input: |
139 | // r0r1r2r3 | r4r5r6r7 | g0g1g2g3 | g4g5g6g7 | b0b1b2b3 | b4b5b6b7 |
140 | // Split the 6 registers in two sets of 3 registers: the first set as the even |
141 | // 8b bytes, the second the odd ones: |
142 | // r0r2r4r6 | g0g2g4g6 | b0b2b4b6 | r1r3r5r7 | g1g3g5g7 | b1b3b5b7 |
143 | // Repeat the same permutations twice more: |
144 | // r0r4g0g4 | b0b4r1r5 | g1g5b1b5 | r2r6g2g6 | b2b6r3r7 | g3g7b3b7 |
145 | // r0g0b0r1 | g1b1r2g2 | b2r3g3b3 | r4g4b4r5 | g5b5r6g6 | b6r7g7b7 |
146 | __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; |
147 | VP8PlanarTo24bHelper(*in, tmp); |
148 | VP8PlanarTo24bHelper(tmp, *in); |
149 | VP8PlanarTo24bHelper(*in, tmp); |
150 | // We need to do it two more times than the example as we have sixteen bytes. |
151 | { |
152 | __m128i out0, out1, out2, out3, out4, out5; |
153 | VP8PlanarTo24bHelper(tmp, out); |
154 | VP8PlanarTo24bHelper(out, *in); |
155 | } |
156 | } |
157 | |
158 | #undef VP8PlanarTo24bHelper |
159 | |
160 | // Convert four packed four-channel buffers like argbargbargbargb... into the |
161 | // split channels aaaaa ... rrrr ... gggg .... bbbbb ...... |
162 | static WEBP_INLINE void VP8L32bToPlanar_SSE2(__m128i* const in0, |
163 | __m128i* const in1, |
164 | __m128i* const in2, |
165 | __m128i* const in3) { |
166 | // Column-wise transpose. |
167 | const __m128i A0 = _mm_unpacklo_epi8(*in0, *in1); |
168 | const __m128i A1 = _mm_unpackhi_epi8(*in0, *in1); |
169 | const __m128i A2 = _mm_unpacklo_epi8(*in2, *in3); |
170 | const __m128i A3 = _mm_unpackhi_epi8(*in2, *in3); |
171 | const __m128i B0 = _mm_unpacklo_epi8(A0, A1); |
172 | const __m128i B1 = _mm_unpackhi_epi8(A0, A1); |
173 | const __m128i B2 = _mm_unpacklo_epi8(A2, A3); |
174 | const __m128i B3 = _mm_unpackhi_epi8(A2, A3); |
175 | // C0 = g7 g6 ... g1 g0 | b7 b6 ... b1 b0 |
176 | // C1 = a7 a6 ... a1 a0 | r7 r6 ... r1 r0 |
177 | const __m128i C0 = _mm_unpacklo_epi8(B0, B1); |
178 | const __m128i C1 = _mm_unpackhi_epi8(B0, B1); |
179 | const __m128i C2 = _mm_unpacklo_epi8(B2, B3); |
180 | const __m128i C3 = _mm_unpackhi_epi8(B2, B3); |
181 | // Gather the channels. |
182 | *in0 = _mm_unpackhi_epi64(C1, C3); |
183 | *in1 = _mm_unpacklo_epi64(C1, C3); |
184 | *in2 = _mm_unpackhi_epi64(C0, C2); |
185 | *in3 = _mm_unpacklo_epi64(C0, C2); |
186 | } |
187 | |
188 | #endif // WEBP_USE_SSE2 |
189 | |
190 | #ifdef __cplusplus |
191 | } // extern "C" |
192 | #endif |
193 | |
194 | #endif // WEBP_DSP_COMMON_SSE2_H_ |
195 | |