1 | // Copyright 2015 Google Inc. All Rights Reserved. |
2 | // |
3 | // Use of this source code is governed by a BSD-style license |
4 | // that can be found in the COPYING file in the root of the source |
5 | // tree. An additional intellectual property rights grant can be found |
6 | // in the file PATENTS. All contributing project authors may |
7 | // be found in the AUTHORS file in the root of the source tree. |
8 | // ----------------------------------------------------------------------------- |
9 | // |
10 | // SSE2 variant of alpha filters |
11 | // |
12 | // Author: Skal (pascal.massimino@gmail.com) |
13 | |
14 | #include "src/dsp/dsp.h" |
15 | |
16 | #if defined(WEBP_USE_SSE2) |
17 | |
18 | #include <assert.h> |
19 | #include <emmintrin.h> |
20 | #include <stdlib.h> |
21 | #include <string.h> |
22 | |
23 | //------------------------------------------------------------------------------ |
24 | // Helpful macro. |
25 | |
26 | # define SANITY_CHECK(in, out) \ |
27 | assert((in) != NULL); \ |
28 | assert((out) != NULL); \ |
29 | assert(width > 0); \ |
30 | assert(height > 0); \ |
31 | assert(stride >= width); \ |
32 | assert(row >= 0 && num_rows > 0 && row + num_rows <= height); \ |
33 | (void)height; // Silence unused warning. |
34 | |
35 | static void PredictLineTop_SSE2(const uint8_t* src, const uint8_t* pred, |
36 | uint8_t* dst, int length) { |
37 | int i; |
38 | const int max_pos = length & ~31; |
39 | assert(length >= 0); |
40 | for (i = 0; i < max_pos; i += 32) { |
41 | const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); |
42 | const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); |
43 | const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); |
44 | const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); |
45 | const __m128i C0 = _mm_sub_epi8(A0, B0); |
46 | const __m128i C1 = _mm_sub_epi8(A1, B1); |
47 | _mm_storeu_si128((__m128i*)&dst[i + 0], C0); |
48 | _mm_storeu_si128((__m128i*)&dst[i + 16], C1); |
49 | } |
50 | for (; i < length; ++i) dst[i] = src[i] - pred[i]; |
51 | } |
52 | |
53 | // Special case for left-based prediction (when preds==dst-1 or preds==src-1). |
54 | static void PredictLineLeft_SSE2(const uint8_t* src, uint8_t* dst, int length) { |
55 | int i; |
56 | const int max_pos = length & ~31; |
57 | assert(length >= 0); |
58 | for (i = 0; i < max_pos; i += 32) { |
59 | const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 )); |
60 | const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1)); |
61 | const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 )); |
62 | const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1)); |
63 | const __m128i C0 = _mm_sub_epi8(A0, B0); |
64 | const __m128i C1 = _mm_sub_epi8(A1, B1); |
65 | _mm_storeu_si128((__m128i*)(dst + i + 0), C0); |
66 | _mm_storeu_si128((__m128i*)(dst + i + 16), C1); |
67 | } |
68 | for (; i < length; ++i) dst[i] = src[i] - src[i - 1]; |
69 | } |
70 | |
71 | //------------------------------------------------------------------------------ |
72 | // Horizontal filter. |
73 | |
74 | static WEBP_INLINE void DoHorizontalFilter_SSE2(const uint8_t* in, |
75 | int width, int height, |
76 | int stride, |
77 | int row, int num_rows, |
78 | uint8_t* out) { |
79 | const size_t start_offset = row * stride; |
80 | const int last_row = row + num_rows; |
81 | SANITY_CHECK(in, out); |
82 | in += start_offset; |
83 | out += start_offset; |
84 | |
85 | if (row == 0) { |
86 | // Leftmost pixel is the same as input for topmost scanline. |
87 | out[0] = in[0]; |
88 | PredictLineLeft_SSE2(in + 1, out + 1, width - 1); |
89 | row = 1; |
90 | in += stride; |
91 | out += stride; |
92 | } |
93 | |
94 | // Filter line-by-line. |
95 | while (row < last_row) { |
96 | // Leftmost pixel is predicted from above. |
97 | out[0] = in[0] - in[-stride]; |
98 | PredictLineLeft_SSE2(in + 1, out + 1, width - 1); |
99 | ++row; |
100 | in += stride; |
101 | out += stride; |
102 | } |
103 | } |
104 | |
105 | //------------------------------------------------------------------------------ |
106 | // Vertical filter. |
107 | |
108 | static WEBP_INLINE void DoVerticalFilter_SSE2(const uint8_t* in, |
109 | int width, int height, int stride, |
110 | int row, int num_rows, |
111 | uint8_t* out) { |
112 | const size_t start_offset = row * stride; |
113 | const int last_row = row + num_rows; |
114 | SANITY_CHECK(in, out); |
115 | in += start_offset; |
116 | out += start_offset; |
117 | |
118 | if (row == 0) { |
119 | // Very first top-left pixel is copied. |
120 | out[0] = in[0]; |
121 | // Rest of top scan-line is left-predicted. |
122 | PredictLineLeft_SSE2(in + 1, out + 1, width - 1); |
123 | row = 1; |
124 | in += stride; |
125 | out += stride; |
126 | } |
127 | |
128 | // Filter line-by-line. |
129 | while (row < last_row) { |
130 | PredictLineTop_SSE2(in, in - stride, out, width); |
131 | ++row; |
132 | in += stride; |
133 | out += stride; |
134 | } |
135 | } |
136 | |
137 | //------------------------------------------------------------------------------ |
138 | // Gradient filter. |
139 | |
140 | static WEBP_INLINE int GradientPredictor_SSE2(uint8_t a, uint8_t b, uint8_t c) { |
141 | const int g = a + b - c; |
142 | return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit |
143 | } |
144 | |
145 | static void GradientPredictDirect_SSE2(const uint8_t* const row, |
146 | const uint8_t* const top, |
147 | uint8_t* const out, int length) { |
148 | const int max_pos = length & ~7; |
149 | int i; |
150 | const __m128i zero = _mm_setzero_si128(); |
151 | for (i = 0; i < max_pos; i += 8) { |
152 | const __m128i A0 = _mm_loadl_epi64((const __m128i*)&row[i - 1]); |
153 | const __m128i B0 = _mm_loadl_epi64((const __m128i*)&top[i]); |
154 | const __m128i C0 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); |
155 | const __m128i D = _mm_loadl_epi64((const __m128i*)&row[i]); |
156 | const __m128i A1 = _mm_unpacklo_epi8(A0, zero); |
157 | const __m128i B1 = _mm_unpacklo_epi8(B0, zero); |
158 | const __m128i C1 = _mm_unpacklo_epi8(C0, zero); |
159 | const __m128i E = _mm_add_epi16(A1, B1); |
160 | const __m128i F = _mm_sub_epi16(E, C1); |
161 | const __m128i G = _mm_packus_epi16(F, zero); |
162 | const __m128i H = _mm_sub_epi8(D, G); |
163 | _mm_storel_epi64((__m128i*)(out + i), H); |
164 | } |
165 | for (; i < length; ++i) { |
166 | const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); |
167 | out[i] = (uint8_t)(row[i] - delta); |
168 | } |
169 | } |
170 | |
171 | static WEBP_INLINE void DoGradientFilter_SSE2(const uint8_t* in, |
172 | int width, int height, int stride, |
173 | int row, int num_rows, |
174 | uint8_t* out) { |
175 | const size_t start_offset = row * stride; |
176 | const int last_row = row + num_rows; |
177 | SANITY_CHECK(in, out); |
178 | in += start_offset; |
179 | out += start_offset; |
180 | |
181 | // left prediction for top scan-line |
182 | if (row == 0) { |
183 | out[0] = in[0]; |
184 | PredictLineLeft_SSE2(in + 1, out + 1, width - 1); |
185 | row = 1; |
186 | in += stride; |
187 | out += stride; |
188 | } |
189 | |
190 | // Filter line-by-line. |
191 | while (row < last_row) { |
192 | out[0] = (uint8_t)(in[0] - in[-stride]); |
193 | GradientPredictDirect_SSE2(in + 1, in + 1 - stride, out + 1, width - 1); |
194 | ++row; |
195 | in += stride; |
196 | out += stride; |
197 | } |
198 | } |
199 | |
200 | #undef SANITY_CHECK |
201 | |
202 | //------------------------------------------------------------------------------ |
203 | |
204 | static void HorizontalFilter_SSE2(const uint8_t* data, int width, int height, |
205 | int stride, uint8_t* filtered_data) { |
206 | DoHorizontalFilter_SSE2(data, width, height, stride, 0, height, |
207 | filtered_data); |
208 | } |
209 | |
210 | static void VerticalFilter_SSE2(const uint8_t* data, int width, int height, |
211 | int stride, uint8_t* filtered_data) { |
212 | DoVerticalFilter_SSE2(data, width, height, stride, 0, height, filtered_data); |
213 | } |
214 | |
215 | static void GradientFilter_SSE2(const uint8_t* data, int width, int height, |
216 | int stride, uint8_t* filtered_data) { |
217 | DoGradientFilter_SSE2(data, width, height, stride, 0, height, filtered_data); |
218 | } |
219 | |
220 | //------------------------------------------------------------------------------ |
221 | // Inverse transforms |
222 | |
223 | static void HorizontalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, |
224 | uint8_t* out, int width) { |
225 | int i; |
226 | __m128i last; |
227 | out[0] = (uint8_t)(in[0] + (prev == NULL ? 0 : prev[0])); |
228 | if (width <= 1) return; |
229 | last = _mm_set_epi32(0, 0, 0, out[0]); |
230 | for (i = 1; i + 8 <= width; i += 8) { |
231 | const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i)); |
232 | const __m128i A1 = _mm_add_epi8(A0, last); |
233 | const __m128i A2 = _mm_slli_si128(A1, 1); |
234 | const __m128i A3 = _mm_add_epi8(A1, A2); |
235 | const __m128i A4 = _mm_slli_si128(A3, 2); |
236 | const __m128i A5 = _mm_add_epi8(A3, A4); |
237 | const __m128i A6 = _mm_slli_si128(A5, 4); |
238 | const __m128i A7 = _mm_add_epi8(A5, A6); |
239 | _mm_storel_epi64((__m128i*)(out + i), A7); |
240 | last = _mm_srli_epi64(A7, 56); |
241 | } |
242 | for (; i < width; ++i) out[i] = (uint8_t)(in[i] + out[i - 1]); |
243 | } |
244 | |
245 | static void VerticalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, |
246 | uint8_t* out, int width) { |
247 | if (prev == NULL) { |
248 | HorizontalUnfilter_SSE2(NULL, in, out, width); |
249 | } else { |
250 | int i; |
251 | const int max_pos = width & ~31; |
252 | assert(width >= 0); |
253 | for (i = 0; i < max_pos; i += 32) { |
254 | const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i + 0]); |
255 | const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]); |
256 | const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i + 0]); |
257 | const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]); |
258 | const __m128i C0 = _mm_add_epi8(A0, B0); |
259 | const __m128i C1 = _mm_add_epi8(A1, B1); |
260 | _mm_storeu_si128((__m128i*)&out[i + 0], C0); |
261 | _mm_storeu_si128((__m128i*)&out[i + 16], C1); |
262 | } |
263 | for (; i < width; ++i) out[i] = (uint8_t)(in[i] + prev[i]); |
264 | } |
265 | } |
266 | |
267 | static void GradientPredictInverse_SSE2(const uint8_t* const in, |
268 | const uint8_t* const top, |
269 | uint8_t* const row, int length) { |
270 | if (length > 0) { |
271 | int i; |
272 | const int max_pos = length & ~7; |
273 | const __m128i zero = _mm_setzero_si128(); |
274 | __m128i A = _mm_set_epi32(0, 0, 0, row[-1]); // left sample |
275 | for (i = 0; i < max_pos; i += 8) { |
276 | const __m128i tmp0 = _mm_loadl_epi64((const __m128i*)&top[i]); |
277 | const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); |
278 | const __m128i B = _mm_unpacklo_epi8(tmp0, zero); |
279 | const __m128i C = _mm_unpacklo_epi8(tmp1, zero); |
280 | const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]); // base input |
281 | const __m128i E = _mm_sub_epi16(B, C); // unclipped gradient basis B - C |
282 | __m128i out = zero; // accumulator for output |
283 | __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff); |
284 | int k = 8; |
285 | while (1) { |
286 | const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C |
287 | const __m128i tmp4 = _mm_packus_epi16(tmp3, zero); // saturate delta |
288 | const __m128i tmp5 = _mm_add_epi8(tmp4, D); // add to in[] |
289 | A = _mm_and_si128(tmp5, mask_hi); // 1-complement clip |
290 | out = _mm_or_si128(out, A); // accumulate output |
291 | if (--k == 0) break; |
292 | A = _mm_slli_si128(A, 1); // rotate left sample |
293 | mask_hi = _mm_slli_si128(mask_hi, 1); // rotate mask |
294 | A = _mm_unpacklo_epi8(A, zero); // convert 8b->16b |
295 | } |
296 | A = _mm_srli_si128(A, 7); // prepare left sample for next iteration |
297 | _mm_storel_epi64((__m128i*)&row[i], out); |
298 | } |
299 | for (; i < length; ++i) { |
300 | const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); |
301 | row[i] = (uint8_t)(in[i] + delta); |
302 | } |
303 | } |
304 | } |
305 | |
306 | static void GradientUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, |
307 | uint8_t* out, int width) { |
308 | if (prev == NULL) { |
309 | HorizontalUnfilter_SSE2(NULL, in, out, width); |
310 | } else { |
311 | out[0] = (uint8_t)(in[0] + prev[0]); // predict from above |
312 | GradientPredictInverse_SSE2(in + 1, prev + 1, out + 1, width - 1); |
313 | } |
314 | } |
315 | |
316 | //------------------------------------------------------------------------------ |
317 | // Entry point |
318 | |
319 | extern void VP8FiltersInitSSE2(void); |
320 | |
321 | WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) { |
322 | WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_SSE2; |
323 | #if defined(CHROMIUM) |
324 | // TODO(crbug.com/654974) |
325 | (void)VerticalUnfilter_SSE2; |
326 | #else |
327 | WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_SSE2; |
328 | #endif |
329 | WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_SSE2; |
330 | |
331 | WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_SSE2; |
332 | WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter_SSE2; |
333 | WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter_SSE2; |
334 | } |
335 | |
336 | #else // !WEBP_USE_SSE2 |
337 | |
338 | WEBP_DSP_INIT_STUB(VP8FiltersInitSSE2) |
339 | |
340 | #endif // WEBP_USE_SSE2 |
341 | |