1 | // Copyright 2015 Google Inc. All Rights Reserved. |
2 | // |
3 | // Use of this source code is governed by a BSD-style license |
4 | // that can be found in the COPYING file in the root of the source |
5 | // tree. An additional intellectual property rights grant can be found |
6 | // in the file PATENTS. All contributing project authors may |
7 | // be found in the AUTHORS file in the root of the source tree. |
8 | // ----------------------------------------------------------------------------- |
9 | // |
10 | // NEON version of rescaling functions |
11 | // |
12 | // Author: Skal (pascal.massimino@gmail.com) |
13 | |
14 | #include "src/dsp/dsp.h" |
15 | |
16 | #if defined(WEBP_USE_NEON) && !defined(WEBP_REDUCE_SIZE) |
17 | |
18 | #include <arm_neon.h> |
19 | #include <assert.h> |
20 | #include "src/dsp/neon.h" |
21 | #include "src/utils/rescaler_utils.h" |
22 | |
23 | #define ROUNDER (WEBP_RESCALER_ONE >> 1) |
24 | #define MULT_FIX_C(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX) |
25 | #define MULT_FIX_FLOOR_C(x, y) (((uint64_t)(x) * (y)) >> WEBP_RESCALER_RFIX) |
26 | |
27 | #define LOAD_32x4(SRC, DST) const uint32x4_t DST = vld1q_u32((SRC)) |
28 | #define LOAD_32x8(SRC, DST0, DST1) \ |
29 | LOAD_32x4(SRC + 0, DST0); \ |
30 | LOAD_32x4(SRC + 4, DST1) |
31 | |
32 | #define STORE_32x8(SRC0, SRC1, DST) do { \ |
33 | vst1q_u32((DST) + 0, SRC0); \ |
34 | vst1q_u32((DST) + 4, SRC1); \ |
35 | } while (0); |
36 | |
37 | #if (WEBP_RESCALER_RFIX == 32) |
38 | #define MAKE_HALF_CST(C) vdupq_n_s32((int32_t)((C) >> 1)) |
39 | // note: B is actualy scale>>1. See MAKE_HALF_CST |
40 | #define MULT_FIX(A, B) \ |
41 | vreinterpretq_u32_s32(vqrdmulhq_s32(vreinterpretq_s32_u32((A)), (B))) |
42 | #define MULT_FIX_FLOOR(A, B) \ |
43 | vreinterpretq_u32_s32(vqdmulhq_s32(vreinterpretq_s32_u32((A)), (B))) |
44 | #else |
45 | #error "MULT_FIX/WEBP_RESCALER_RFIX need some more work" |
46 | #endif |
47 | |
48 | static uint32x4_t Interpolate_NEON(const rescaler_t* const frow, |
49 | const rescaler_t* const irow, |
50 | uint32_t A, uint32_t B) { |
51 | LOAD_32x4(frow, A0); |
52 | LOAD_32x4(irow, B0); |
53 | const uint64x2_t C0 = vmull_n_u32(vget_low_u32(A0), A); |
54 | const uint64x2_t C1 = vmull_n_u32(vget_high_u32(A0), A); |
55 | const uint64x2_t D0 = vmlal_n_u32(C0, vget_low_u32(B0), B); |
56 | const uint64x2_t D1 = vmlal_n_u32(C1, vget_high_u32(B0), B); |
57 | const uint32x4_t E = vcombine_u32( |
58 | vrshrn_n_u64(D0, WEBP_RESCALER_RFIX), |
59 | vrshrn_n_u64(D1, WEBP_RESCALER_RFIX)); |
60 | return E; |
61 | } |
62 | |
63 | static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) { |
64 | int x_out; |
65 | uint8_t* const dst = wrk->dst; |
66 | rescaler_t* const irow = wrk->irow; |
67 | const int x_out_max = wrk->dst_width * wrk->num_channels; |
68 | const int max_span = x_out_max & ~7; |
69 | const rescaler_t* const frow = wrk->frow; |
70 | const uint32_t fy_scale = wrk->fy_scale; |
71 | const int32x4_t fy_scale_half = MAKE_HALF_CST(fy_scale); |
72 | assert(!WebPRescalerOutputDone(wrk)); |
73 | assert(wrk->y_accum <= 0); |
74 | assert(wrk->y_expand); |
75 | assert(wrk->y_sub != 0); |
76 | if (wrk->y_accum == 0) { |
77 | for (x_out = 0; x_out < max_span; x_out += 8) { |
78 | LOAD_32x4(frow + x_out + 0, A0); |
79 | LOAD_32x4(frow + x_out + 4, A1); |
80 | const uint32x4_t B0 = MULT_FIX(A0, fy_scale_half); |
81 | const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half); |
82 | const uint16x4_t C0 = vmovn_u32(B0); |
83 | const uint16x4_t C1 = vmovn_u32(B1); |
84 | const uint8x8_t D = vqmovn_u16(vcombine_u16(C0, C1)); |
85 | vst1_u8(dst + x_out, D); |
86 | } |
87 | for (; x_out < x_out_max; ++x_out) { |
88 | const uint32_t J = frow[x_out]; |
89 | const int v = (int)MULT_FIX_C(J, fy_scale); |
90 | dst[x_out] = (v > 255) ? 255u : (uint8_t)v; |
91 | } |
92 | } else { |
93 | const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); |
94 | const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B); |
95 | for (x_out = 0; x_out < max_span; x_out += 8) { |
96 | const uint32x4_t C0 = |
97 | Interpolate_NEON(frow + x_out + 0, irow + x_out + 0, A, B); |
98 | const uint32x4_t C1 = |
99 | Interpolate_NEON(frow + x_out + 4, irow + x_out + 4, A, B); |
100 | const uint32x4_t D0 = MULT_FIX(C0, fy_scale_half); |
101 | const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half); |
102 | const uint16x4_t E0 = vmovn_u32(D0); |
103 | const uint16x4_t E1 = vmovn_u32(D1); |
104 | const uint8x8_t F = vqmovn_u16(vcombine_u16(E0, E1)); |
105 | vst1_u8(dst + x_out, F); |
106 | } |
107 | for (; x_out < x_out_max; ++x_out) { |
108 | const uint64_t I = (uint64_t)A * frow[x_out] |
109 | + (uint64_t)B * irow[x_out]; |
110 | const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); |
111 | const int v = (int)MULT_FIX_C(J, fy_scale); |
112 | dst[x_out] = (v > 255) ? 255u : (uint8_t)v; |
113 | } |
114 | } |
115 | } |
116 | |
117 | static void RescalerExportRowShrink_NEON(WebPRescaler* const wrk) { |
118 | int x_out; |
119 | uint8_t* const dst = wrk->dst; |
120 | rescaler_t* const irow = wrk->irow; |
121 | const int x_out_max = wrk->dst_width * wrk->num_channels; |
122 | const int max_span = x_out_max & ~7; |
123 | const rescaler_t* const frow = wrk->frow; |
124 | const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum); |
125 | const uint32_t fxy_scale = wrk->fxy_scale; |
126 | const uint32x4_t zero = vdupq_n_u32(0); |
127 | const int32x4_t yscale_half = MAKE_HALF_CST(yscale); |
128 | const int32x4_t fxy_scale_half = MAKE_HALF_CST(fxy_scale); |
129 | assert(!WebPRescalerOutputDone(wrk)); |
130 | assert(wrk->y_accum <= 0); |
131 | assert(!wrk->y_expand); |
132 | if (yscale) { |
133 | for (x_out = 0; x_out < max_span; x_out += 8) { |
134 | LOAD_32x8(frow + x_out, in0, in1); |
135 | LOAD_32x8(irow + x_out, in2, in3); |
136 | const uint32x4_t A0 = MULT_FIX_FLOOR(in0, yscale_half); |
137 | const uint32x4_t A1 = MULT_FIX_FLOOR(in1, yscale_half); |
138 | const uint32x4_t B0 = vqsubq_u32(in2, A0); |
139 | const uint32x4_t B1 = vqsubq_u32(in3, A1); |
140 | const uint32x4_t C0 = MULT_FIX(B0, fxy_scale_half); |
141 | const uint32x4_t C1 = MULT_FIX(B1, fxy_scale_half); |
142 | const uint16x4_t D0 = vmovn_u32(C0); |
143 | const uint16x4_t D1 = vmovn_u32(C1); |
144 | const uint8x8_t E = vqmovn_u16(vcombine_u16(D0, D1)); |
145 | vst1_u8(dst + x_out, E); |
146 | STORE_32x8(A0, A1, irow + x_out); |
147 | } |
148 | for (; x_out < x_out_max; ++x_out) { |
149 | const uint32_t frac = (uint32_t)MULT_FIX_FLOOR_C(frow[x_out], yscale); |
150 | const int v = (int)MULT_FIX_C(irow[x_out] - frac, fxy_scale); |
151 | dst[x_out] = (v > 255) ? 255u : (uint8_t)v; |
152 | irow[x_out] = frac; // new fractional start |
153 | } |
154 | } else { |
155 | for (x_out = 0; x_out < max_span; x_out += 8) { |
156 | LOAD_32x8(irow + x_out, in0, in1); |
157 | const uint32x4_t A0 = MULT_FIX(in0, fxy_scale_half); |
158 | const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half); |
159 | const uint16x4_t B0 = vmovn_u32(A0); |
160 | const uint16x4_t B1 = vmovn_u32(A1); |
161 | const uint8x8_t C = vqmovn_u16(vcombine_u16(B0, B1)); |
162 | vst1_u8(dst + x_out, C); |
163 | STORE_32x8(zero, zero, irow + x_out); |
164 | } |
165 | for (; x_out < x_out_max; ++x_out) { |
166 | const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale); |
167 | dst[x_out] = (v > 255) ? 255u : (uint8_t)v; |
168 | irow[x_out] = 0; |
169 | } |
170 | } |
171 | } |
172 | |
173 | #undef MULT_FIX_FLOOR_C |
174 | #undef MULT_FIX_C |
175 | #undef MULT_FIX_FLOOR |
176 | #undef MULT_FIX |
177 | #undef ROUNDER |
178 | |
179 | //------------------------------------------------------------------------------ |
180 | |
181 | extern void WebPRescalerDspInitNEON(void); |
182 | |
183 | WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitNEON(void) { |
184 | WebPRescalerExportRowExpand = RescalerExportRowExpand_NEON; |
185 | WebPRescalerExportRowShrink = RescalerExportRowShrink_NEON; |
186 | } |
187 | |
188 | #else // !WEBP_USE_NEON |
189 | |
190 | WEBP_DSP_INIT_STUB(WebPRescalerDspInitNEON) |
191 | |
192 | #endif // WEBP_USE_NEON |
193 | |