1// Copyright 2015 Google Inc. All Rights Reserved.
2//
3// Use of this source code is governed by a BSD-style license
4// that can be found in the COPYING file in the root of the source
5// tree. An additional intellectual property rights grant can be found
6// in the file PATENTS. All contributing project authors may
7// be found in the AUTHORS file in the root of the source tree.
8// -----------------------------------------------------------------------------
9//
10// SSE2 variant of alpha filters
11//
12// Author: Skal (pascal.massimino@gmail.com)
13
14#include "src/dsp/dsp.h"
15
16#if defined(WEBP_USE_SSE2)
17
18#include <assert.h>
19#include <emmintrin.h>
20#include <stdlib.h>
21#include <string.h>
22
23//------------------------------------------------------------------------------
24// Helpful macro.
25
26# define SANITY_CHECK(in, out) \
27 assert((in) != NULL); \
28 assert((out) != NULL); \
29 assert(width > 0); \
30 assert(height > 0); \
31 assert(stride >= width); \
32 assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \
33 (void)height; // Silence unused warning.
34
35static void PredictLineTop_SSE2(const uint8_t* src, const uint8_t* pred,
36 uint8_t* dst, int length) {
37 int i;
38 const int max_pos = length & ~31;
39 assert(length >= 0);
40 for (i = 0; i < max_pos; i += 32) {
41 const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]);
42 const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]);
43 const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]);
44 const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]);
45 const __m128i C0 = _mm_sub_epi8(A0, B0);
46 const __m128i C1 = _mm_sub_epi8(A1, B1);
47 _mm_storeu_si128((__m128i*)&dst[i + 0], C0);
48 _mm_storeu_si128((__m128i*)&dst[i + 16], C1);
49 }
50 for (; i < length; ++i) dst[i] = src[i] - pred[i];
51}
52
53// Special case for left-based prediction (when preds==dst-1 or preds==src-1).
54static void PredictLineLeft_SSE2(const uint8_t* src, uint8_t* dst, int length) {
55 int i;
56 const int max_pos = length & ~31;
57 assert(length >= 0);
58 for (i = 0; i < max_pos; i += 32) {
59 const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 ));
60 const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1));
61 const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 ));
62 const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1));
63 const __m128i C0 = _mm_sub_epi8(A0, B0);
64 const __m128i C1 = _mm_sub_epi8(A1, B1);
65 _mm_storeu_si128((__m128i*)(dst + i + 0), C0);
66 _mm_storeu_si128((__m128i*)(dst + i + 16), C1);
67 }
68 for (; i < length; ++i) dst[i] = src[i] - src[i - 1];
69}
70
71//------------------------------------------------------------------------------
72// Horizontal filter.
73
74static WEBP_INLINE void DoHorizontalFilter_SSE2(const uint8_t* in,
75 int width, int height,
76 int stride,
77 int row, int num_rows,
78 uint8_t* out) {
79 const size_t start_offset = row * stride;
80 const int last_row = row + num_rows;
81 SANITY_CHECK(in, out);
82 in += start_offset;
83 out += start_offset;
84
85 if (row == 0) {
86 // Leftmost pixel is the same as input for topmost scanline.
87 out[0] = in[0];
88 PredictLineLeft_SSE2(in + 1, out + 1, width - 1);
89 row = 1;
90 in += stride;
91 out += stride;
92 }
93
94 // Filter line-by-line.
95 while (row < last_row) {
96 // Leftmost pixel is predicted from above.
97 out[0] = in[0] - in[-stride];
98 PredictLineLeft_SSE2(in + 1, out + 1, width - 1);
99 ++row;
100 in += stride;
101 out += stride;
102 }
103}
104
105//------------------------------------------------------------------------------
106// Vertical filter.
107
108static WEBP_INLINE void DoVerticalFilter_SSE2(const uint8_t* in,
109 int width, int height, int stride,
110 int row, int num_rows,
111 uint8_t* out) {
112 const size_t start_offset = row * stride;
113 const int last_row = row + num_rows;
114 SANITY_CHECK(in, out);
115 in += start_offset;
116 out += start_offset;
117
118 if (row == 0) {
119 // Very first top-left pixel is copied.
120 out[0] = in[0];
121 // Rest of top scan-line is left-predicted.
122 PredictLineLeft_SSE2(in + 1, out + 1, width - 1);
123 row = 1;
124 in += stride;
125 out += stride;
126 }
127
128 // Filter line-by-line.
129 while (row < last_row) {
130 PredictLineTop_SSE2(in, in - stride, out, width);
131 ++row;
132 in += stride;
133 out += stride;
134 }
135}
136
137//------------------------------------------------------------------------------
138// Gradient filter.
139
140static WEBP_INLINE int GradientPredictor_SSE2(uint8_t a, uint8_t b, uint8_t c) {
141 const int g = a + b - c;
142 return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit
143}
144
145static void GradientPredictDirect_SSE2(const uint8_t* const row,
146 const uint8_t* const top,
147 uint8_t* const out, int length) {
148 const int max_pos = length & ~7;
149 int i;
150 const __m128i zero = _mm_setzero_si128();
151 for (i = 0; i < max_pos; i += 8) {
152 const __m128i A0 = _mm_loadl_epi64((const __m128i*)&row[i - 1]);
153 const __m128i B0 = _mm_loadl_epi64((const __m128i*)&top[i]);
154 const __m128i C0 = _mm_loadl_epi64((const __m128i*)&top[i - 1]);
155 const __m128i D = _mm_loadl_epi64((const __m128i*)&row[i]);
156 const __m128i A1 = _mm_unpacklo_epi8(A0, zero);
157 const __m128i B1 = _mm_unpacklo_epi8(B0, zero);
158 const __m128i C1 = _mm_unpacklo_epi8(C0, zero);
159 const __m128i E = _mm_add_epi16(A1, B1);
160 const __m128i F = _mm_sub_epi16(E, C1);
161 const __m128i G = _mm_packus_epi16(F, zero);
162 const __m128i H = _mm_sub_epi8(D, G);
163 _mm_storel_epi64((__m128i*)(out + i), H);
164 }
165 for (; i < length; ++i) {
166 const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]);
167 out[i] = (uint8_t)(row[i] - delta);
168 }
169}
170
171static WEBP_INLINE void DoGradientFilter_SSE2(const uint8_t* in,
172 int width, int height, int stride,
173 int row, int num_rows,
174 uint8_t* out) {
175 const size_t start_offset = row * stride;
176 const int last_row = row + num_rows;
177 SANITY_CHECK(in, out);
178 in += start_offset;
179 out += start_offset;
180
181 // left prediction for top scan-line
182 if (row == 0) {
183 out[0] = in[0];
184 PredictLineLeft_SSE2(in + 1, out + 1, width - 1);
185 row = 1;
186 in += stride;
187 out += stride;
188 }
189
190 // Filter line-by-line.
191 while (row < last_row) {
192 out[0] = (uint8_t)(in[0] - in[-stride]);
193 GradientPredictDirect_SSE2(in + 1, in + 1 - stride, out + 1, width - 1);
194 ++row;
195 in += stride;
196 out += stride;
197 }
198}
199
200#undef SANITY_CHECK
201
202//------------------------------------------------------------------------------
203
204static void HorizontalFilter_SSE2(const uint8_t* data, int width, int height,
205 int stride, uint8_t* filtered_data) {
206 DoHorizontalFilter_SSE2(data, width, height, stride, 0, height,
207 filtered_data);
208}
209
210static void VerticalFilter_SSE2(const uint8_t* data, int width, int height,
211 int stride, uint8_t* filtered_data) {
212 DoVerticalFilter_SSE2(data, width, height, stride, 0, height, filtered_data);
213}
214
215static void GradientFilter_SSE2(const uint8_t* data, int width, int height,
216 int stride, uint8_t* filtered_data) {
217 DoGradientFilter_SSE2(data, width, height, stride, 0, height, filtered_data);
218}
219
220//------------------------------------------------------------------------------
221// Inverse transforms
222
223static void HorizontalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in,
224 uint8_t* out, int width) {
225 int i;
226 __m128i last;
227 out[0] = (uint8_t)(in[0] + (prev == NULL ? 0 : prev[0]));
228 if (width <= 1) return;
229 last = _mm_set_epi32(0, 0, 0, out[0]);
230 for (i = 1; i + 8 <= width; i += 8) {
231 const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i));
232 const __m128i A1 = _mm_add_epi8(A0, last);
233 const __m128i A2 = _mm_slli_si128(A1, 1);
234 const __m128i A3 = _mm_add_epi8(A1, A2);
235 const __m128i A4 = _mm_slli_si128(A3, 2);
236 const __m128i A5 = _mm_add_epi8(A3, A4);
237 const __m128i A6 = _mm_slli_si128(A5, 4);
238 const __m128i A7 = _mm_add_epi8(A5, A6);
239 _mm_storel_epi64((__m128i*)(out + i), A7);
240 last = _mm_srli_epi64(A7, 56);
241 }
242 for (; i < width; ++i) out[i] = (uint8_t)(in[i] + out[i - 1]);
243}
244
245static void VerticalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in,
246 uint8_t* out, int width) {
247 if (prev == NULL) {
248 HorizontalUnfilter_SSE2(NULL, in, out, width);
249 } else {
250 int i;
251 const int max_pos = width & ~31;
252 assert(width >= 0);
253 for (i = 0; i < max_pos; i += 32) {
254 const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i + 0]);
255 const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]);
256 const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i + 0]);
257 const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]);
258 const __m128i C0 = _mm_add_epi8(A0, B0);
259 const __m128i C1 = _mm_add_epi8(A1, B1);
260 _mm_storeu_si128((__m128i*)&out[i + 0], C0);
261 _mm_storeu_si128((__m128i*)&out[i + 16], C1);
262 }
263 for (; i < width; ++i) out[i] = (uint8_t)(in[i] + prev[i]);
264 }
265}
266
267static void GradientPredictInverse_SSE2(const uint8_t* const in,
268 const uint8_t* const top,
269 uint8_t* const row, int length) {
270 if (length > 0) {
271 int i;
272 const int max_pos = length & ~7;
273 const __m128i zero = _mm_setzero_si128();
274 __m128i A = _mm_set_epi32(0, 0, 0, row[-1]); // left sample
275 for (i = 0; i < max_pos; i += 8) {
276 const __m128i tmp0 = _mm_loadl_epi64((const __m128i*)&top[i]);
277 const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]);
278 const __m128i B = _mm_unpacklo_epi8(tmp0, zero);
279 const __m128i C = _mm_unpacklo_epi8(tmp1, zero);
280 const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]); // base input
281 const __m128i E = _mm_sub_epi16(B, C); // unclipped gradient basis B - C
282 __m128i out = zero; // accumulator for output
283 __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff);
284 int k = 8;
285 while (1) {
286 const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C
287 const __m128i tmp4 = _mm_packus_epi16(tmp3, zero); // saturate delta
288 const __m128i tmp5 = _mm_add_epi8(tmp4, D); // add to in[]
289 A = _mm_and_si128(tmp5, mask_hi); // 1-complement clip
290 out = _mm_or_si128(out, A); // accumulate output
291 if (--k == 0) break;
292 A = _mm_slli_si128(A, 1); // rotate left sample
293 mask_hi = _mm_slli_si128(mask_hi, 1); // rotate mask
294 A = _mm_unpacklo_epi8(A, zero); // convert 8b->16b
295 }
296 A = _mm_srli_si128(A, 7); // prepare left sample for next iteration
297 _mm_storel_epi64((__m128i*)&row[i], out);
298 }
299 for (; i < length; ++i) {
300 const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]);
301 row[i] = (uint8_t)(in[i] + delta);
302 }
303 }
304}
305
306static void GradientUnfilter_SSE2(const uint8_t* prev, const uint8_t* in,
307 uint8_t* out, int width) {
308 if (prev == NULL) {
309 HorizontalUnfilter_SSE2(NULL, in, out, width);
310 } else {
311 out[0] = (uint8_t)(in[0] + prev[0]); // predict from above
312 GradientPredictInverse_SSE2(in + 1, prev + 1, out + 1, width - 1);
313 }
314}
315
316//------------------------------------------------------------------------------
317// Entry point
318
319extern void VP8FiltersInitSSE2(void);
320
321WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) {
322 WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_SSE2;
323#if defined(CHROMIUM)
324 // TODO(crbug.com/654974)
325 (void)VerticalUnfilter_SSE2;
326#else
327 WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_SSE2;
328#endif
329 WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_SSE2;
330
331 WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_SSE2;
332 WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter_SSE2;
333 WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter_SSE2;
334}
335
336#else // !WEBP_USE_SSE2
337
338WEBP_DSP_INIT_STUB(VP8FiltersInitSSE2)
339
340#endif // WEBP_USE_SSE2
341