1// Copyright 2015 Google Inc. All Rights Reserved.
2//
3// Use of this source code is governed by a BSD-style license
4// that can be found in the COPYING file in the root of the source
5// tree. An additional intellectual property rights grant can be found
6// in the file PATENTS. All contributing project authors may
7// be found in the AUTHORS file in the root of the source tree.
8// -----------------------------------------------------------------------------
9//
10// SSE2 Rescaling functions
11//
12// Author: Skal (pascal.massimino@gmail.com)
13
14#include "./dsp.h"
15
16#if defined(WEBP_USE_SSE2)
17#include <emmintrin.h>
18
19#include <assert.h>
20#include "../utils/rescaler_utils.h"
21#include "../utils/utils.h"
22
23//------------------------------------------------------------------------------
24// Implementations of critical functions ImportRow / ExportRow
25
26#define ROUNDER (WEBP_RESCALER_ONE >> 1)
27#define MULT_FIX(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
28
29// input: 8 bytes ABCDEFGH -> output: A0E0B0F0C0G0D0H0
30static void LoadTwoPixels(const uint8_t* const src, __m128i* out) {
31 const __m128i zero = _mm_setzero_si128();
32 const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH
33 const __m128i B = _mm_unpacklo_epi8(A, zero); // A0B0C0D0E0F0G0H0
34 const __m128i C = _mm_srli_si128(B, 8); // E0F0G0H0
35 *out = _mm_unpacklo_epi16(B, C);
36}
37
38// input: 8 bytes ABCDEFGH -> output: A0B0C0D0E0F0G0H0
39static void LoadHeightPixels(const uint8_t* const src, __m128i* out) {
40 const __m128i zero = _mm_setzero_si128();
41 const __m128i A = _mm_loadl_epi64((const __m128i*)(src)); // ABCDEFGH
42 *out = _mm_unpacklo_epi8(A, zero);
43}
44
45static void RescalerImportRowExpandSSE2(WebPRescaler* const wrk,
46 const uint8_t* src) {
47 rescaler_t* frow = wrk->frow;
48 const rescaler_t* const frow_end = frow + wrk->dst_width * wrk->num_channels;
49 const int x_add = wrk->x_add;
50 int accum = x_add;
51 __m128i cur_pixels;
52
53 assert(!WebPRescalerInputDone(wrk));
54 assert(wrk->x_expand);
55 if (wrk->num_channels == 4) {
56 if (wrk->src_width < 2) {
57 WebPRescalerImportRowExpandC(wrk, src);
58 return;
59 }
60 LoadTwoPixels(src, &cur_pixels);
61 src += 4;
62 while (1) {
63 const __m128i mult = _mm_set1_epi32(((x_add - accum) << 16) | accum);
64 const __m128i out = _mm_madd_epi16(cur_pixels, mult);
65 _mm_storeu_si128((__m128i*)frow, out);
66 frow += 4;
67 if (frow >= frow_end) break;
68 accum -= wrk->x_sub;
69 if (accum < 0) {
70 LoadTwoPixels(src, &cur_pixels);
71 src += 4;
72 accum += x_add;
73 }
74 }
75 } else {
76 int left;
77 const uint8_t* const src_limit = src + wrk->src_width - 8;
78 if (wrk->src_width < 8) {
79 WebPRescalerImportRowExpandC(wrk, src);
80 return;
81 }
82 LoadHeightPixels(src, &cur_pixels);
83 src += 7;
84 left = 7;
85 while (1) {
86 const __m128i mult = _mm_cvtsi32_si128(((x_add - accum) << 16) | accum);
87 const __m128i out = _mm_madd_epi16(cur_pixels, mult);
88 assert(sizeof(*frow) == sizeof(uint32_t));
89 WebPUint32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out));
90 frow += 1;
91 if (frow >= frow_end) break;
92 accum -= wrk->x_sub;
93 if (accum < 0) {
94 if (--left) {
95 cur_pixels = _mm_srli_si128(cur_pixels, 2);
96 } else if (src <= src_limit) {
97 LoadHeightPixels(src, &cur_pixels);
98 src += 7;
99 left = 7;
100 } else { // tail
101 cur_pixels = _mm_srli_si128(cur_pixels, 2);
102 cur_pixels = _mm_insert_epi16(cur_pixels, src[1], 1);
103 src += 1;
104 left = 1;
105 }
106 accum += x_add;
107 }
108 }
109 }
110 assert(accum == 0);
111}
112
113static void RescalerImportRowShrinkSSE2(WebPRescaler* const wrk,
114 const uint8_t* src) {
115 const int x_sub = wrk->x_sub;
116 int accum = 0;
117 const __m128i zero = _mm_setzero_si128();
118 const __m128i mult0 = _mm_set1_epi16(x_sub);
119 const __m128i mult1 = _mm_set1_epi32(wrk->fx_scale);
120 const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER);
121 __m128i sum = zero;
122 rescaler_t* frow = wrk->frow;
123 const rescaler_t* const frow_end = wrk->frow + 4 * wrk->dst_width;
124
125 if (wrk->num_channels != 4 || wrk->x_add > (x_sub << 7)) {
126 WebPRescalerImportRowShrinkC(wrk, src);
127 return;
128 }
129 assert(!WebPRescalerInputDone(wrk));
130 assert(!wrk->x_expand);
131
132 for (; frow < frow_end; frow += 4) {
133 __m128i base = zero;
134 accum += wrk->x_add;
135 while (accum > 0) {
136 const __m128i A = _mm_cvtsi32_si128(WebPMemToUint32(src));
137 src += 4;
138 base = _mm_unpacklo_epi8(A, zero);
139 // To avoid overflow, we need: base * x_add / x_sub < 32768
140 // => x_add < x_sub << 7. That's a 1/128 reduction ratio limit.
141 sum = _mm_add_epi16(sum, base);
142 accum -= x_sub;
143 }
144 { // Emit next horizontal pixel.
145 const __m128i mult = _mm_set1_epi16(-accum);
146 const __m128i frac0 = _mm_mullo_epi16(base, mult); // 16b x 16b -> 32b
147 const __m128i frac1 = _mm_mulhi_epu16(base, mult);
148 const __m128i frac = _mm_unpacklo_epi16(frac0, frac1); // frac is 32b
149 const __m128i A0 = _mm_mullo_epi16(sum, mult0);
150 const __m128i A1 = _mm_mulhi_epu16(sum, mult0);
151 const __m128i B0 = _mm_unpacklo_epi16(A0, A1); // sum * x_sub
152 const __m128i frow_out = _mm_sub_epi32(B0, frac); // sum * x_sub - frac
153 const __m128i D0 = _mm_srli_epi64(frac, 32);
154 const __m128i D1 = _mm_mul_epu32(frac, mult1); // 32b x 16b -> 64b
155 const __m128i D2 = _mm_mul_epu32(D0, mult1);
156 const __m128i E1 = _mm_add_epi64(D1, rounder);
157 const __m128i E2 = _mm_add_epi64(D2, rounder);
158 const __m128i F1 = _mm_shuffle_epi32(E1, 1 | (3 << 2));
159 const __m128i F2 = _mm_shuffle_epi32(E2, 1 | (3 << 2));
160 const __m128i G = _mm_unpacklo_epi32(F1, F2);
161 sum = _mm_packs_epi32(G, zero);
162 _mm_storeu_si128((__m128i*)frow, frow_out);
163 }
164 }
165 assert(accum == 0);
166}
167
168//------------------------------------------------------------------------------
169// Row export
170
171// load *src as epi64, multiply by mult and store result in [out0 ... out3]
172static WEBP_INLINE void LoadDispatchAndMult(const rescaler_t* const src,
173 const __m128i* const mult,
174 __m128i* const out0,
175 __m128i* const out1,
176 __m128i* const out2,
177 __m128i* const out3) {
178 const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + 0));
179 const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + 4));
180 const __m128i A2 = _mm_srli_epi64(A0, 32);
181 const __m128i A3 = _mm_srli_epi64(A1, 32);
182 if (mult != NULL) {
183 *out0 = _mm_mul_epu32(A0, *mult);
184 *out1 = _mm_mul_epu32(A1, *mult);
185 *out2 = _mm_mul_epu32(A2, *mult);
186 *out3 = _mm_mul_epu32(A3, *mult);
187 } else {
188 *out0 = A0;
189 *out1 = A1;
190 *out2 = A2;
191 *out3 = A3;
192 }
193}
194
195static WEBP_INLINE void ProcessRow(const __m128i* const A0,
196 const __m128i* const A1,
197 const __m128i* const A2,
198 const __m128i* const A3,
199 const __m128i* const mult,
200 uint8_t* const dst) {
201 const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER);
202 const __m128i mask = _mm_set_epi32(0xffffffffu, 0, 0xffffffffu, 0);
203 const __m128i B0 = _mm_mul_epu32(*A0, *mult);
204 const __m128i B1 = _mm_mul_epu32(*A1, *mult);
205 const __m128i B2 = _mm_mul_epu32(*A2, *mult);
206 const __m128i B3 = _mm_mul_epu32(*A3, *mult);
207 const __m128i C0 = _mm_add_epi64(B0, rounder);
208 const __m128i C1 = _mm_add_epi64(B1, rounder);
209 const __m128i C2 = _mm_add_epi64(B2, rounder);
210 const __m128i C3 = _mm_add_epi64(B3, rounder);
211 const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX);
212 const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX);
213#if (WEBP_RESCALER_FIX < 32)
214 const __m128i D2 =
215 _mm_and_si128(_mm_slli_epi64(C2, 32 - WEBP_RESCALER_RFIX), mask);
216 const __m128i D3 =
217 _mm_and_si128(_mm_slli_epi64(C3, 32 - WEBP_RESCALER_RFIX), mask);
218#else
219 const __m128i D2 = _mm_and_si128(C2, mask);
220 const __m128i D3 = _mm_and_si128(C3, mask);
221#endif
222 const __m128i E0 = _mm_or_si128(D0, D2);
223 const __m128i E1 = _mm_or_si128(D1, D3);
224 const __m128i F = _mm_packs_epi32(E0, E1);
225 const __m128i G = _mm_packus_epi16(F, F);
226 _mm_storel_epi64((__m128i*)dst, G);
227}
228
229static void RescalerExportRowExpandSSE2(WebPRescaler* const wrk) {
230 int x_out;
231 uint8_t* const dst = wrk->dst;
232 rescaler_t* const irow = wrk->irow;
233 const int x_out_max = wrk->dst_width * wrk->num_channels;
234 const rescaler_t* const frow = wrk->frow;
235 const __m128i mult = _mm_set_epi32(0, wrk->fy_scale, 0, wrk->fy_scale);
236
237 assert(!WebPRescalerOutputDone(wrk));
238 assert(wrk->y_accum <= 0 && wrk->y_sub + wrk->y_accum >= 0);
239 assert(wrk->y_expand);
240 if (wrk->y_accum == 0) {
241 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
242 __m128i A0, A1, A2, A3;
243 LoadDispatchAndMult(frow + x_out, NULL, &A0, &A1, &A2, &A3);
244 ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out);
245 }
246 for (; x_out < x_out_max; ++x_out) {
247 const uint32_t J = frow[x_out];
248 const int v = (int)MULT_FIX(J, wrk->fy_scale);
249 assert(v >= 0 && v <= 255);
250 dst[x_out] = v;
251 }
252 } else {
253 const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
254 const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B);
255 const __m128i mA = _mm_set_epi32(0, A, 0, A);
256 const __m128i mB = _mm_set_epi32(0, B, 0, B);
257 const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER);
258 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
259 __m128i A0, A1, A2, A3, B0, B1, B2, B3;
260 LoadDispatchAndMult(frow + x_out, &mA, &A0, &A1, &A2, &A3);
261 LoadDispatchAndMult(irow + x_out, &mB, &B0, &B1, &B2, &B3);
262 {
263 const __m128i C0 = _mm_add_epi64(A0, B0);
264 const __m128i C1 = _mm_add_epi64(A1, B1);
265 const __m128i C2 = _mm_add_epi64(A2, B2);
266 const __m128i C3 = _mm_add_epi64(A3, B3);
267 const __m128i D0 = _mm_add_epi64(C0, rounder);
268 const __m128i D1 = _mm_add_epi64(C1, rounder);
269 const __m128i D2 = _mm_add_epi64(C2, rounder);
270 const __m128i D3 = _mm_add_epi64(C3, rounder);
271 const __m128i E0 = _mm_srli_epi64(D0, WEBP_RESCALER_RFIX);
272 const __m128i E1 = _mm_srli_epi64(D1, WEBP_RESCALER_RFIX);
273 const __m128i E2 = _mm_srli_epi64(D2, WEBP_RESCALER_RFIX);
274 const __m128i E3 = _mm_srli_epi64(D3, WEBP_RESCALER_RFIX);
275 ProcessRow(&E0, &E1, &E2, &E3, &mult, dst + x_out);
276 }
277 }
278 for (; x_out < x_out_max; ++x_out) {
279 const uint64_t I = (uint64_t)A * frow[x_out]
280 + (uint64_t)B * irow[x_out];
281 const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
282 const int v = (int)MULT_FIX(J, wrk->fy_scale);
283 assert(v >= 0 && v <= 255);
284 dst[x_out] = v;
285 }
286 }
287}
288
289static void RescalerExportRowShrinkSSE2(WebPRescaler* const wrk) {
290 int x_out;
291 uint8_t* const dst = wrk->dst;
292 rescaler_t* const irow = wrk->irow;
293 const int x_out_max = wrk->dst_width * wrk->num_channels;
294 const rescaler_t* const frow = wrk->frow;
295 const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum);
296 assert(!WebPRescalerOutputDone(wrk));
297 assert(wrk->y_accum <= 0);
298 assert(!wrk->y_expand);
299 if (yscale) {
300 const int scale_xy = wrk->fxy_scale;
301 const __m128i mult_xy = _mm_set_epi32(0, scale_xy, 0, scale_xy);
302 const __m128i mult_y = _mm_set_epi32(0, yscale, 0, yscale);
303 const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER);
304 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
305 __m128i A0, A1, A2, A3, B0, B1, B2, B3;
306 LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3);
307 LoadDispatchAndMult(frow + x_out, &mult_y, &B0, &B1, &B2, &B3);
308 {
309 const __m128i C0 = _mm_add_epi64(B0, rounder);
310 const __m128i C1 = _mm_add_epi64(B1, rounder);
311 const __m128i C2 = _mm_add_epi64(B2, rounder);
312 const __m128i C3 = _mm_add_epi64(B3, rounder);
313 const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); // = frac
314 const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX);
315 const __m128i D2 = _mm_srli_epi64(C2, WEBP_RESCALER_RFIX);
316 const __m128i D3 = _mm_srli_epi64(C3, WEBP_RESCALER_RFIX);
317 const __m128i E0 = _mm_sub_epi64(A0, D0); // irow[x] - frac
318 const __m128i E1 = _mm_sub_epi64(A1, D1);
319 const __m128i E2 = _mm_sub_epi64(A2, D2);
320 const __m128i E3 = _mm_sub_epi64(A3, D3);
321 const __m128i F2 = _mm_slli_epi64(D2, 32);
322 const __m128i F3 = _mm_slli_epi64(D3, 32);
323 const __m128i G0 = _mm_or_si128(D0, F2);
324 const __m128i G1 = _mm_or_si128(D1, F3);
325 _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0);
326 _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1);
327 ProcessRow(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out);
328 }
329 }
330 for (; x_out < x_out_max; ++x_out) {
331 const uint32_t frac = (int)MULT_FIX(frow[x_out], yscale);
332 const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale);
333 assert(v >= 0 && v <= 255);
334 dst[x_out] = v;
335 irow[x_out] = frac; // new fractional start
336 }
337 } else {
338 const uint32_t scale = wrk->fxy_scale;
339 const __m128i mult = _mm_set_epi32(0, scale, 0, scale);
340 const __m128i zero = _mm_setzero_si128();
341 for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) {
342 __m128i A0, A1, A2, A3;
343 LoadDispatchAndMult(irow + x_out, NULL, &A0, &A1, &A2, &A3);
344 _mm_storeu_si128((__m128i*)(irow + x_out + 0), zero);
345 _mm_storeu_si128((__m128i*)(irow + x_out + 4), zero);
346 ProcessRow(&A0, &A1, &A2, &A3, &mult, dst + x_out);
347 }
348 for (; x_out < x_out_max; ++x_out) {
349 const int v = (int)MULT_FIX(irow[x_out], scale);
350 assert(v >= 0 && v <= 255);
351 dst[x_out] = v;
352 irow[x_out] = 0;
353 }
354 }
355}
356
357#undef MULT_FIX
358#undef ROUNDER
359
360//------------------------------------------------------------------------------
361
362extern void WebPRescalerDspInitSSE2(void);
363
364WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitSSE2(void) {
365 WebPRescalerImportRowExpand = RescalerImportRowExpandSSE2;
366 WebPRescalerImportRowShrink = RescalerImportRowShrinkSSE2;
367 WebPRescalerExportRowExpand = RescalerExportRowExpandSSE2;
368 WebPRescalerExportRowShrink = RescalerExportRowShrinkSSE2;
369}
370
371#else // !WEBP_USE_SSE2
372
373WEBP_DSP_INIT_STUB(WebPRescalerDspInitSSE2)
374
375#endif // WEBP_USE_SSE2
376