1// Copyright 2014 Google Inc. All Rights Reserved.
2//
3// Use of this source code is governed by a BSD-style license
4// that can be found in the COPYING file in the root of the source
5// tree. An additional intellectual property rights grant can be found
6// in the file PATENTS. All contributing project authors may
7// be found in the AUTHORS file in the root of the source tree.
8// -----------------------------------------------------------------------------
9//
10// MIPS version of dsp functions
11//
12// Author(s): Djordje Pesut (djordje.pesut@imgtec.com)
13// Jovan Zelincevic (jovan.zelincevic@imgtec.com)
14
15#include "src/dsp/dsp.h"
16
17#if defined(WEBP_USE_MIPS_DSP_R2)
18
19#include "src/dsp/mips_macro.h"
20
21static const int kC1 = 20091 + (1 << 16);
22static const int kC2 = 35468;
23
24#define MUL(a, b) (((a) * (b)) >> 16)
25
26static void TransformDC(const int16_t* in, uint8_t* dst) {
27 int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9, temp10;
28
29 __asm__ volatile (
30 LOAD_WITH_OFFSET_X4(temp1, temp2, temp3, temp4, dst,
31 0, 0, 0, 0,
32 0, 1, 2, 3,
33 BPS)
34 "lh %[temp5], 0(%[in]) \n\t"
35 "addiu %[temp5], %[temp5], 4 \n\t"
36 "ins %[temp5], %[temp5], 16, 16 \n\t"
37 "shra.ph %[temp5], %[temp5], 3 \n\t"
38 CONVERT_2_BYTES_TO_HALF(temp6, temp7, temp8, temp9, temp10, temp1, temp2,
39 temp3, temp1, temp2, temp3, temp4)
40 STORE_SAT_SUM_X2(temp6, temp7, temp8, temp9, temp10, temp1, temp2, temp3,
41 temp5, temp5, temp5, temp5, temp5, temp5, temp5, temp5,
42 dst, 0, 1, 2, 3, BPS)
43
44 OUTPUT_EARLY_CLOBBER_REGS_10()
45 : [in]"r"(in), [dst]"r"(dst)
46 : "memory"
47 );
48}
49
50static void TransformAC3(const int16_t* in, uint8_t* dst) {
51 const int a = in[0] + 4;
52 int c4 = MUL(in[4], kC2);
53 const int d4 = MUL(in[4], kC1);
54 const int c1 = MUL(in[1], kC2);
55 const int d1 = MUL(in[1], kC1);
56 int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9;
57 int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18;
58
59 __asm__ volatile (
60 "ins %[c4], %[d4], 16, 16 \n\t"
61 "replv.ph %[temp1], %[a] \n\t"
62 "replv.ph %[temp4], %[d1] \n\t"
63 ADD_SUB_HALVES(temp2, temp3, temp1, c4)
64 "replv.ph %[temp5], %[c1] \n\t"
65 SHIFT_R_SUM_X2(temp1, temp6, temp7, temp8, temp2, temp9, temp10, temp4,
66 temp2, temp2, temp3, temp3, temp4, temp5, temp4, temp5)
67 LOAD_WITH_OFFSET_X4(temp3, temp5, temp11, temp12, dst,
68 0, 0, 0, 0,
69 0, 1, 2, 3,
70 BPS)
71 CONVERT_2_BYTES_TO_HALF(temp13, temp14, temp3, temp15, temp5, temp16,
72 temp11, temp17, temp3, temp5, temp11, temp12)
73 PACK_2_HALVES_TO_WORD(temp12, temp18, temp7, temp6, temp1, temp8, temp2,
74 temp4, temp7, temp6, temp10, temp9)
75 STORE_SAT_SUM_X2(temp13, temp14, temp3, temp15, temp5, temp16, temp11,
76 temp17, temp12, temp18, temp1, temp8, temp2, temp4,
77 temp7, temp6, dst, 0, 1, 2, 3, BPS)
78
79 OUTPUT_EARLY_CLOBBER_REGS_18(),
80 [c4]"+&r"(c4)
81 : [dst]"r"(dst), [a]"r"(a), [d1]"r"(d1), [d4]"r"(d4), [c1]"r"(c1)
82 : "memory"
83 );
84}
85
86static void TransformOne(const int16_t* in, uint8_t* dst) {
87 int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9;
88 int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18;
89
90 __asm__ volatile (
91 "ulw %[temp1], 0(%[in]) \n\t"
92 "ulw %[temp2], 16(%[in]) \n\t"
93 LOAD_IN_X2(temp5, temp6, 24, 26)
94 ADD_SUB_HALVES(temp3, temp4, temp1, temp2)
95 LOAD_IN_X2(temp1, temp2, 8, 10)
96 MUL_SHIFT_SUM(temp7, temp8, temp9, temp10, temp11, temp12, temp13, temp14,
97 temp10, temp8, temp9, temp7, temp1, temp2, temp5, temp6,
98 temp13, temp11, temp14, temp12)
99 INSERT_HALF_X2(temp8, temp7, temp10, temp9)
100 "ulw %[temp17], 4(%[in]) \n\t"
101 "ulw %[temp18], 20(%[in]) \n\t"
102 ADD_SUB_HALVES(temp1, temp2, temp3, temp8)
103 ADD_SUB_HALVES(temp5, temp6, temp4, temp7)
104 ADD_SUB_HALVES(temp7, temp8, temp17, temp18)
105 LOAD_IN_X2(temp17, temp18, 12, 14)
106 LOAD_IN_X2(temp9, temp10, 28, 30)
107 MUL_SHIFT_SUM(temp11, temp12, temp13, temp14, temp15, temp16, temp4, temp17,
108 temp12, temp14, temp11, temp13, temp17, temp18, temp9, temp10,
109 temp15, temp4, temp16, temp17)
110 INSERT_HALF_X2(temp11, temp12, temp13, temp14)
111 ADD_SUB_HALVES(temp17, temp8, temp8, temp11)
112 ADD_SUB_HALVES(temp3, temp4, temp7, temp12)
113
114 // horizontal
115 SRA_16(temp9, temp10, temp11, temp12, temp1, temp2, temp5, temp6)
116 INSERT_HALF_X2(temp1, temp6, temp5, temp2)
117 SRA_16(temp13, temp14, temp15, temp16, temp3, temp4, temp17, temp8)
118 "repl.ph %[temp2], 0x4 \n\t"
119 INSERT_HALF_X2(temp3, temp8, temp17, temp4)
120 "addq.ph %[temp1], %[temp1], %[temp2] \n\t"
121 "addq.ph %[temp6], %[temp6], %[temp2] \n\t"
122 ADD_SUB_HALVES(temp2, temp4, temp1, temp3)
123 ADD_SUB_HALVES(temp5, temp7, temp6, temp8)
124 MUL_SHIFT_SUM(temp1, temp3, temp6, temp8, temp9, temp13, temp17, temp18,
125 temp3, temp13, temp1, temp9, temp9, temp13, temp11, temp15,
126 temp6, temp17, temp8, temp18)
127 MUL_SHIFT_SUM(temp6, temp8, temp18, temp17, temp11, temp15, temp12, temp16,
128 temp8, temp15, temp6, temp11, temp12, temp16, temp10, temp14,
129 temp18, temp12, temp17, temp16)
130 INSERT_HALF_X2(temp1, temp3, temp9, temp13)
131 INSERT_HALF_X2(temp6, temp8, temp11, temp15)
132 SHIFT_R_SUM_X2(temp9, temp10, temp11, temp12, temp13, temp14, temp15,
133 temp16, temp2, temp4, temp5, temp7, temp3, temp1, temp8,
134 temp6)
135 PACK_2_HALVES_TO_WORD(temp1, temp2, temp3, temp4, temp9, temp12, temp13,
136 temp16, temp11, temp10, temp15, temp14)
137 LOAD_WITH_OFFSET_X4(temp10, temp11, temp14, temp15, dst,
138 0, 0, 0, 0,
139 0, 1, 2, 3,
140 BPS)
141 CONVERT_2_BYTES_TO_HALF(temp5, temp6, temp7, temp8, temp17, temp18, temp10,
142 temp11, temp10, temp11, temp14, temp15)
143 STORE_SAT_SUM_X2(temp5, temp6, temp7, temp8, temp17, temp18, temp10, temp11,
144 temp9, temp12, temp1, temp2, temp13, temp16, temp3, temp4,
145 dst, 0, 1, 2, 3, BPS)
146
147 OUTPUT_EARLY_CLOBBER_REGS_18()
148 : [dst]"r"(dst), [in]"r"(in), [kC1]"r"(kC1), [kC2]"r"(kC2)
149 : "memory", "hi", "lo"
150 );
151}
152
153static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) {
154 TransformOne(in, dst);
155 if (do_two) {
156 TransformOne(in + 16, dst + 4);
157 }
158}
159
160static WEBP_INLINE void FilterLoop26(uint8_t* p,
161 int hstride, int vstride, int size,
162 int thresh, int ithresh, int hev_thresh) {
163 const int thresh2 = 2 * thresh + 1;
164 int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9;
165 int temp10, temp11, temp12, temp13, temp14, temp15;
166
167 __asm__ volatile (
168 ".set push \n\t"
169 ".set noreorder \n\t"
170 "1: \n\t"
171 "negu %[temp1], %[hstride] \n\t"
172 "addiu %[size], %[size], -1 \n\t"
173 "sll %[temp2], %[hstride], 1 \n\t"
174 "sll %[temp3], %[temp1], 1 \n\t"
175 "addu %[temp4], %[temp2], %[hstride] \n\t"
176 "addu %[temp5], %[temp3], %[temp1] \n\t"
177 "lbu %[temp7], 0(%[p]) \n\t"
178 "sll %[temp6], %[temp3], 1 \n\t"
179 "lbux %[temp8], %[temp5](%[p]) \n\t"
180 "lbux %[temp9], %[temp3](%[p]) \n\t"
181 "lbux %[temp10], %[temp1](%[p]) \n\t"
182 "lbux %[temp11], %[temp6](%[p]) \n\t"
183 "lbux %[temp12], %[hstride](%[p]) \n\t"
184 "lbux %[temp13], %[temp2](%[p]) \n\t"
185 "lbux %[temp14], %[temp4](%[p]) \n\t"
186 "subu %[temp1], %[temp10], %[temp7] \n\t"
187 "subu %[temp2], %[temp9], %[temp12] \n\t"
188 "absq_s.w %[temp3], %[temp1] \n\t"
189 "absq_s.w %[temp4], %[temp2] \n\t"
190 "negu %[temp1], %[temp1] \n\t"
191 "sll %[temp3], %[temp3], 2 \n\t"
192 "addu %[temp15], %[temp3], %[temp4] \n\t"
193 "subu %[temp3], %[temp15], %[thresh2] \n\t"
194 "sll %[temp6], %[temp1], 1 \n\t"
195 "bgtz %[temp3], 3f \n\t"
196 " subu %[temp4], %[temp11], %[temp8] \n\t"
197 "absq_s.w %[temp4], %[temp4] \n\t"
198 "shll_s.w %[temp2], %[temp2], 24 \n\t"
199 "subu %[temp4], %[temp4], %[ithresh] \n\t"
200 "bgtz %[temp4], 3f \n\t"
201 " subu %[temp3], %[temp8], %[temp9] \n\t"
202 "absq_s.w %[temp3], %[temp3] \n\t"
203 "subu %[temp3], %[temp3], %[ithresh] \n\t"
204 "bgtz %[temp3], 3f \n\t"
205 " subu %[temp5], %[temp9], %[temp10] \n\t"
206 "absq_s.w %[temp3], %[temp5] \n\t"
207 "absq_s.w %[temp5], %[temp5] \n\t"
208 "subu %[temp3], %[temp3], %[ithresh] \n\t"
209 "bgtz %[temp3], 3f \n\t"
210 " subu %[temp3], %[temp14], %[temp13] \n\t"
211 "absq_s.w %[temp3], %[temp3] \n\t"
212 "slt %[temp5], %[hev_thresh], %[temp5] \n\t"
213 "subu %[temp3], %[temp3], %[ithresh] \n\t"
214 "bgtz %[temp3], 3f \n\t"
215 " subu %[temp3], %[temp13], %[temp12] \n\t"
216 "absq_s.w %[temp3], %[temp3] \n\t"
217 "sra %[temp4], %[temp2], 24 \n\t"
218 "subu %[temp3], %[temp3], %[ithresh] \n\t"
219 "bgtz %[temp3], 3f \n\t"
220 " subu %[temp15], %[temp12], %[temp7] \n\t"
221 "absq_s.w %[temp3], %[temp15] \n\t"
222 "absq_s.w %[temp15], %[temp15] \n\t"
223 "subu %[temp3], %[temp3], %[ithresh] \n\t"
224 "bgtz %[temp3], 3f \n\t"
225 " slt %[temp15], %[hev_thresh], %[temp15] \n\t"
226 "addu %[temp3], %[temp6], %[temp1] \n\t"
227 "or %[temp2], %[temp5], %[temp15] \n\t"
228 "addu %[temp5], %[temp4], %[temp3] \n\t"
229 "beqz %[temp2], 4f \n\t"
230 " shra_r.w %[temp1], %[temp5], 3 \n\t"
231 "addiu %[temp2], %[temp5], 3 \n\t"
232 "sra %[temp2], %[temp2], 3 \n\t"
233 "shll_s.w %[temp1], %[temp1], 27 \n\t"
234 "shll_s.w %[temp2], %[temp2], 27 \n\t"
235 "subu %[temp3], %[p], %[hstride] \n\t"
236 "sra %[temp1], %[temp1], 27 \n\t"
237 "sra %[temp2], %[temp2], 27 \n\t"
238 "subu %[temp1], %[temp7], %[temp1] \n\t"
239 "addu %[temp2], %[temp10], %[temp2] \n\t"
240 "lbux %[temp2], %[temp2](%[VP8kclip1]) \n\t"
241 "lbux %[temp1], %[temp1](%[VP8kclip1]) \n\t"
242 "sb %[temp2], 0(%[temp3]) \n\t"
243 "j 3f \n\t"
244 " sb %[temp1], 0(%[p]) \n\t"
245 "4: \n\t"
246 "shll_s.w %[temp5], %[temp5], 24 \n\t"
247 "subu %[temp14], %[p], %[hstride] \n\t"
248 "subu %[temp11], %[temp14], %[hstride] \n\t"
249 "sra %[temp6], %[temp5], 24 \n\t"
250 "sll %[temp1], %[temp6], 3 \n\t"
251 "subu %[temp15], %[temp11], %[hstride] \n\t"
252 "addu %[temp2], %[temp6], %[temp1] \n\t"
253 "sll %[temp3], %[temp2], 1 \n\t"
254 "addu %[temp4], %[temp3], %[temp2] \n\t"
255 "addiu %[temp2], %[temp2], 63 \n\t"
256 "addiu %[temp3], %[temp3], 63 \n\t"
257 "addiu %[temp4], %[temp4], 63 \n\t"
258 "sra %[temp2], %[temp2], 7 \n\t"
259 "sra %[temp3], %[temp3], 7 \n\t"
260 "sra %[temp4], %[temp4], 7 \n\t"
261 "addu %[temp1], %[temp8], %[temp2] \n\t"
262 "addu %[temp5], %[temp9], %[temp3] \n\t"
263 "addu %[temp6], %[temp10], %[temp4] \n\t"
264 "subu %[temp8], %[temp7], %[temp4] \n\t"
265 "subu %[temp7], %[temp12], %[temp3] \n\t"
266 "addu %[temp10], %[p], %[hstride] \n\t"
267 "subu %[temp9], %[temp13], %[temp2] \n\t"
268 "addu %[temp12], %[temp10], %[hstride] \n\t"
269 "lbux %[temp2], %[temp1](%[VP8kclip1]) \n\t"
270 "lbux %[temp3], %[temp5](%[VP8kclip1]) \n\t"
271 "lbux %[temp4], %[temp6](%[VP8kclip1]) \n\t"
272 "lbux %[temp5], %[temp8](%[VP8kclip1]) \n\t"
273 "lbux %[temp6], %[temp7](%[VP8kclip1]) \n\t"
274 "lbux %[temp8], %[temp9](%[VP8kclip1]) \n\t"
275 "sb %[temp2], 0(%[temp15]) \n\t"
276 "sb %[temp3], 0(%[temp11]) \n\t"
277 "sb %[temp4], 0(%[temp14]) \n\t"
278 "sb %[temp5], 0(%[p]) \n\t"
279 "sb %[temp6], 0(%[temp10]) \n\t"
280 "sb %[temp8], 0(%[temp12]) \n\t"
281 "3: \n\t"
282 "bgtz %[size], 1b \n\t"
283 " addu %[p], %[p], %[vstride] \n\t"
284 ".set pop \n\t"
285 : [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),[temp3]"=&r"(temp3),
286 [temp4]"=&r"(temp4), [temp5]"=&r"(temp5), [temp6]"=&r"(temp6),
287 [temp7]"=&r"(temp7),[temp8]"=&r"(temp8),[temp9]"=&r"(temp9),
288 [temp10]"=&r"(temp10),[temp11]"=&r"(temp11),[temp12]"=&r"(temp12),
289 [temp13]"=&r"(temp13),[temp14]"=&r"(temp14),[temp15]"=&r"(temp15),
290 [size]"+&r"(size), [p]"+&r"(p)
291 : [hstride]"r"(hstride), [thresh2]"r"(thresh2),
292 [ithresh]"r"(ithresh),[vstride]"r"(vstride), [hev_thresh]"r"(hev_thresh),
293 [VP8kclip1]"r"(VP8kclip1)
294 : "memory"
295 );
296}
297
298static WEBP_INLINE void FilterLoop24(uint8_t* p,
299 int hstride, int vstride, int size,
300 int thresh, int ithresh, int hev_thresh) {
301 int p0, q0, p1, q1, p2, q2, p3, q3;
302 int step1, step2, temp1, temp2, temp3, temp4;
303 uint8_t* pTemp0;
304 uint8_t* pTemp1;
305 const int thresh2 = 2 * thresh + 1;
306
307 __asm__ volatile (
308 ".set push \n\t"
309 ".set noreorder \n\t"
310 "bltz %[size], 3f \n\t"
311 " nop \n\t"
312 "2: \n\t"
313 "negu %[step1], %[hstride] \n\t"
314 "lbu %[q0], 0(%[p]) \n\t"
315 "lbux %[p0], %[step1](%[p]) \n\t"
316 "subu %[step1], %[step1], %[hstride] \n\t"
317 "lbux %[q1], %[hstride](%[p]) \n\t"
318 "subu %[temp1], %[p0], %[q0] \n\t"
319 "lbux %[p1], %[step1](%[p]) \n\t"
320 "addu %[step2], %[hstride], %[hstride] \n\t"
321 "absq_s.w %[temp2], %[temp1] \n\t"
322 "subu %[temp3], %[p1], %[q1] \n\t"
323 "absq_s.w %[temp4], %[temp3] \n\t"
324 "sll %[temp2], %[temp2], 2 \n\t"
325 "addu %[temp2], %[temp2], %[temp4] \n\t"
326 "subu %[temp4], %[temp2], %[thresh2] \n\t"
327 "subu %[step1], %[step1], %[hstride] \n\t"
328 "bgtz %[temp4], 0f \n\t"
329 " lbux %[p2], %[step1](%[p]) \n\t"
330 "subu %[step1], %[step1], %[hstride] \n\t"
331 "lbux %[q2], %[step2](%[p]) \n\t"
332 "lbux %[p3], %[step1](%[p]) \n\t"
333 "subu %[temp4], %[p2], %[p1] \n\t"
334 "addu %[step2], %[step2], %[hstride] \n\t"
335 "subu %[temp2], %[p3], %[p2] \n\t"
336 "absq_s.w %[temp4], %[temp4] \n\t"
337 "absq_s.w %[temp2], %[temp2] \n\t"
338 "lbux %[q3], %[step2](%[p]) \n\t"
339 "subu %[temp4], %[temp4], %[ithresh] \n\t"
340 "negu %[temp1], %[temp1] \n\t"
341 "bgtz %[temp4], 0f \n\t"
342 " subu %[temp2], %[temp2], %[ithresh] \n\t"
343 "subu %[p3], %[p1], %[p0] \n\t"
344 "bgtz %[temp2], 0f \n\t"
345 " absq_s.w %[p3], %[p3] \n\t"
346 "subu %[temp4], %[q3], %[q2] \n\t"
347 "subu %[pTemp0], %[p], %[hstride] \n\t"
348 "absq_s.w %[temp4], %[temp4] \n\t"
349 "subu %[temp2], %[p3], %[ithresh] \n\t"
350 "sll %[step1], %[temp1], 1 \n\t"
351 "bgtz %[temp2], 0f \n\t"
352 " subu %[temp4], %[temp4], %[ithresh] \n\t"
353 "subu %[temp2], %[q2], %[q1] \n\t"
354 "bgtz %[temp4], 0f \n\t"
355 " absq_s.w %[temp2], %[temp2] \n\t"
356 "subu %[q3], %[q1], %[q0] \n\t"
357 "absq_s.w %[q3], %[q3] \n\t"
358 "subu %[temp2], %[temp2], %[ithresh] \n\t"
359 "addu %[temp1], %[temp1], %[step1] \n\t"
360 "bgtz %[temp2], 0f \n\t"
361 " subu %[temp4], %[q3], %[ithresh] \n\t"
362 "slt %[p3], %[hev_thresh], %[p3] \n\t"
363 "bgtz %[temp4], 0f \n\t"
364 " slt %[q3], %[hev_thresh], %[q3] \n\t"
365 "or %[q3], %[q3], %[p3] \n\t"
366 "bgtz %[q3], 1f \n\t"
367 " shra_r.w %[temp2], %[temp1], 3 \n\t"
368 "addiu %[temp1], %[temp1], 3 \n\t"
369 "sra %[temp1], %[temp1], 3 \n\t"
370 "shll_s.w %[temp2], %[temp2], 27 \n\t"
371 "shll_s.w %[temp1], %[temp1], 27 \n\t"
372 "addu %[pTemp1], %[p], %[hstride] \n\t"
373 "sra %[temp2], %[temp2], 27 \n\t"
374 "sra %[temp1], %[temp1], 27 \n\t"
375 "addiu %[step1], %[temp2], 1 \n\t"
376 "sra %[step1], %[step1], 1 \n\t"
377 "addu %[p0], %[p0], %[temp1] \n\t"
378 "addu %[p1], %[p1], %[step1] \n\t"
379 "subu %[q0], %[q0], %[temp2] \n\t"
380 "subu %[q1], %[q1], %[step1] \n\t"
381 "lbux %[temp2], %[p0](%[VP8kclip1]) \n\t"
382 "lbux %[temp3], %[q0](%[VP8kclip1]) \n\t"
383 "lbux %[temp4], %[q1](%[VP8kclip1]) \n\t"
384 "sb %[temp2], 0(%[pTemp0]) \n\t"
385 "lbux %[temp1], %[p1](%[VP8kclip1]) \n\t"
386 "subu %[pTemp0], %[pTemp0], %[hstride] \n\t"
387 "sb %[temp3], 0(%[p]) \n\t"
388 "sb %[temp4], 0(%[pTemp1]) \n\t"
389 "j 0f \n\t"
390 " sb %[temp1], 0(%[pTemp0]) \n\t"
391 "1: \n\t"
392 "shll_s.w %[temp3], %[temp3], 24 \n\t"
393 "sra %[temp3], %[temp3], 24 \n\t"
394 "addu %[temp1], %[temp1], %[temp3] \n\t"
395 "shra_r.w %[temp2], %[temp1], 3 \n\t"
396 "addiu %[temp1], %[temp1], 3 \n\t"
397 "shll_s.w %[temp2], %[temp2], 27 \n\t"
398 "sra %[temp1], %[temp1], 3 \n\t"
399 "shll_s.w %[temp1], %[temp1], 27 \n\t"
400 "sra %[temp2], %[temp2], 27 \n\t"
401 "sra %[temp1], %[temp1], 27 \n\t"
402 "addu %[p0], %[p0], %[temp1] \n\t"
403 "subu %[q0], %[q0], %[temp2] \n\t"
404 "lbux %[temp1], %[p0](%[VP8kclip1]) \n\t"
405 "lbux %[temp2], %[q0](%[VP8kclip1]) \n\t"
406 "sb %[temp2], 0(%[p]) \n\t"
407 "sb %[temp1], 0(%[pTemp0]) \n\t"
408 "0: \n\t"
409 "subu %[size], %[size], 1 \n\t"
410 "bgtz %[size], 2b \n\t"
411 " addu %[p], %[p], %[vstride] \n\t"
412 "3: \n\t"
413 ".set pop \n\t"
414 : [p0]"=&r"(p0), [q0]"=&r"(q0), [p1]"=&r"(p1), [q1]"=&r"(q1),
415 [p2]"=&r"(p2), [q2]"=&r"(q2), [p3]"=&r"(p3), [q3]"=&r"(q3),
416 [step2]"=&r"(step2), [step1]"=&r"(step1), [temp1]"=&r"(temp1),
417 [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [temp4]"=&r"(temp4),
418 [pTemp0]"=&r"(pTemp0), [pTemp1]"=&r"(pTemp1), [p]"+&r"(p),
419 [size]"+&r"(size)
420 : [vstride]"r"(vstride), [ithresh]"r"(ithresh),
421 [hev_thresh]"r"(hev_thresh), [hstride]"r"(hstride),
422 [VP8kclip1]"r"(VP8kclip1), [thresh2]"r"(thresh2)
423 : "memory"
424 );
425}
426
427// on macroblock edges
428static void VFilter16(uint8_t* p, int stride,
429 int thresh, int ithresh, int hev_thresh) {
430 FilterLoop26(p, stride, 1, 16, thresh, ithresh, hev_thresh);
431}
432
433static void HFilter16(uint8_t* p, int stride,
434 int thresh, int ithresh, int hev_thresh) {
435 FilterLoop26(p, 1, stride, 16, thresh, ithresh, hev_thresh);
436}
437
438// 8-pixels wide variant, for chroma filtering
439static void VFilter8(uint8_t* u, uint8_t* v, int stride,
440 int thresh, int ithresh, int hev_thresh) {
441 FilterLoop26(u, stride, 1, 8, thresh, ithresh, hev_thresh);
442 FilterLoop26(v, stride, 1, 8, thresh, ithresh, hev_thresh);
443}
444
445static void HFilter8(uint8_t* u, uint8_t* v, int stride,
446 int thresh, int ithresh, int hev_thresh) {
447 FilterLoop26(u, 1, stride, 8, thresh, ithresh, hev_thresh);
448 FilterLoop26(v, 1, stride, 8, thresh, ithresh, hev_thresh);
449}
450
451// on three inner edges
452static void VFilter16i(uint8_t* p, int stride,
453 int thresh, int ithresh, int hev_thresh) {
454 int k;
455 for (k = 3; k > 0; --k) {
456 p += 4 * stride;
457 FilterLoop24(p, stride, 1, 16, thresh, ithresh, hev_thresh);
458 }
459}
460
461static void HFilter16i(uint8_t* p, int stride,
462 int thresh, int ithresh, int hev_thresh) {
463 int k;
464 for (k = 3; k > 0; --k) {
465 p += 4;
466 FilterLoop24(p, 1, stride, 16, thresh, ithresh, hev_thresh);
467 }
468}
469
470static void VFilter8i(uint8_t* u, uint8_t* v, int stride,
471 int thresh, int ithresh, int hev_thresh) {
472 FilterLoop24(u + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh);
473 FilterLoop24(v + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh);
474}
475
476static void HFilter8i(uint8_t* u, uint8_t* v, int stride,
477 int thresh, int ithresh, int hev_thresh) {
478 FilterLoop24(u + 4, 1, stride, 8, thresh, ithresh, hev_thresh);
479 FilterLoop24(v + 4, 1, stride, 8, thresh, ithresh, hev_thresh);
480}
481
482#undef MUL
483
484//------------------------------------------------------------------------------
485// Simple In-loop filtering (Paragraph 15.2)
486
487static void SimpleVFilter16(uint8_t* p, int stride, int thresh) {
488 int i;
489 const int thresh2 = 2 * thresh + 1;
490 int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8;
491 uint8_t* p1 = p - stride;
492 __asm__ volatile (
493 ".set push \n\t"
494 ".set noreorder \n\t"
495 "li %[i], 16 \n\t"
496 "0: \n\t"
497 "negu %[temp4], %[stride] \n\t"
498 "sll %[temp5], %[temp4], 1 \n\t"
499 "lbu %[temp2], 0(%[p]) \n\t"
500 "lbux %[temp3], %[stride](%[p]) \n\t"
501 "lbux %[temp1], %[temp4](%[p]) \n\t"
502 "lbux %[temp0], %[temp5](%[p]) \n\t"
503 "subu %[temp7], %[temp1], %[temp2] \n\t"
504 "subu %[temp6], %[temp0], %[temp3] \n\t"
505 "absq_s.w %[temp4], %[temp7] \n\t"
506 "absq_s.w %[temp5], %[temp6] \n\t"
507 "sll %[temp4], %[temp4], 2 \n\t"
508 "subu %[temp5], %[temp5], %[thresh2] \n\t"
509 "addu %[temp5], %[temp4], %[temp5] \n\t"
510 "negu %[temp8], %[temp7] \n\t"
511 "bgtz %[temp5], 1f \n\t"
512 " addiu %[i], %[i], -1 \n\t"
513 "sll %[temp4], %[temp8], 1 \n\t"
514 "shll_s.w %[temp5], %[temp6], 24 \n\t"
515 "addu %[temp3], %[temp4], %[temp8] \n\t"
516 "sra %[temp5], %[temp5], 24 \n\t"
517 "addu %[temp3], %[temp3], %[temp5] \n\t"
518 "addiu %[temp7], %[temp3], 3 \n\t"
519 "sra %[temp7], %[temp7], 3 \n\t"
520 "shra_r.w %[temp8], %[temp3], 3 \n\t"
521 "shll_s.w %[temp0], %[temp7], 27 \n\t"
522 "shll_s.w %[temp4], %[temp8], 27 \n\t"
523 "sra %[temp0], %[temp0], 27 \n\t"
524 "sra %[temp4], %[temp4], 27 \n\t"
525 "addu %[temp7], %[temp1], %[temp0] \n\t"
526 "subu %[temp2], %[temp2], %[temp4] \n\t"
527 "lbux %[temp3], %[temp7](%[VP8kclip1]) \n\t"
528 "lbux %[temp4], %[temp2](%[VP8kclip1]) \n\t"
529 "sb %[temp3], 0(%[p1]) \n\t"
530 "sb %[temp4], 0(%[p]) \n\t"
531 "1: \n\t"
532 "addiu %[p1], %[p1], 1 \n\t"
533 "bgtz %[i], 0b \n\t"
534 " addiu %[p], %[p], 1 \n\t"
535 " .set pop \n\t"
536 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
537 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
538 [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
539 [p]"+&r"(p), [i]"=&r"(i), [p1]"+&r"(p1)
540 : [stride]"r"(stride), [VP8kclip1]"r"(VP8kclip1), [thresh2]"r"(thresh2)
541 : "memory"
542 );
543}
544
545// TEMP0 = SRC[A + A1 * BPS]
546// TEMP1 = SRC[B + B1 * BPS]
547// TEMP2 = SRC[C + C1 * BPS]
548// TEMP3 = SRC[D + D1 * BPS]
549#define LOAD_4_BYTES(TEMP0, TEMP1, TEMP2, TEMP3, \
550 A, A1, B, B1, C, C1, D, D1, SRC) \
551 "lbu %[" #TEMP0 "], " #A "+" #A1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \
552 "lbu %[" #TEMP1 "], " #B "+" #B1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \
553 "lbu %[" #TEMP2 "], " #C "+" #C1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \
554 "lbu %[" #TEMP3 "], " #D "+" #D1 "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \
555
556static void SimpleHFilter16(uint8_t* p, int stride, int thresh) {
557 int i;
558 const int thresh2 = 2 * thresh + 1;
559 int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8;
560 __asm__ volatile (
561 ".set push \n\t"
562 ".set noreorder \n\t"
563 "li %[i], 16 \n\t"
564 "0: \n\t"
565 LOAD_4_BYTES(temp0, temp1, temp2, temp3, -2, 0, -1, 0, 0, 0, 1, 0, p)
566 "subu %[temp7], %[temp1], %[temp2] \n\t"
567 "subu %[temp6], %[temp0], %[temp3] \n\t"
568 "absq_s.w %[temp4], %[temp7] \n\t"
569 "absq_s.w %[temp5], %[temp6] \n\t"
570 "sll %[temp4], %[temp4], 2 \n\t"
571 "addu %[temp5], %[temp4], %[temp5] \n\t"
572 "subu %[temp5], %[temp5], %[thresh2] \n\t"
573 "negu %[temp8], %[temp7] \n\t"
574 "bgtz %[temp5], 1f \n\t"
575 " addiu %[i], %[i], -1 \n\t"
576 "sll %[temp4], %[temp8], 1 \n\t"
577 "shll_s.w %[temp5], %[temp6], 24 \n\t"
578 "addu %[temp3], %[temp4], %[temp8] \n\t"
579 "sra %[temp5], %[temp5], 24 \n\t"
580 "addu %[temp3], %[temp3], %[temp5] \n\t"
581 "addiu %[temp7], %[temp3], 3 \n\t"
582 "sra %[temp7], %[temp7], 3 \n\t"
583 "shra_r.w %[temp8], %[temp3], 3 \n\t"
584 "shll_s.w %[temp0], %[temp7], 27 \n\t"
585 "shll_s.w %[temp4], %[temp8], 27 \n\t"
586 "sra %[temp0], %[temp0], 27 \n\t"
587 "sra %[temp4], %[temp4], 27 \n\t"
588 "addu %[temp7], %[temp1], %[temp0] \n\t"
589 "subu %[temp2], %[temp2], %[temp4] \n\t"
590 "lbux %[temp3], %[temp7](%[VP8kclip1]) \n\t"
591 "lbux %[temp4], %[temp2](%[VP8kclip1]) \n\t"
592 "sb %[temp3], -1(%[p]) \n\t"
593 "sb %[temp4], 0(%[p]) \n\t"
594 "1: \n\t"
595 "bgtz %[i], 0b \n\t"
596 " addu %[p], %[p], %[stride] \n\t"
597 ".set pop \n\t"
598 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
599 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
600 [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
601 [p]"+&r"(p), [i]"=&r"(i)
602 : [stride]"r"(stride), [VP8kclip1]"r"(VP8kclip1), [thresh2]"r"(thresh2)
603 : "memory"
604 );
605}
606
607static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) {
608 int k;
609 for (k = 3; k > 0; --k) {
610 p += 4 * stride;
611 SimpleVFilter16(p, stride, thresh);
612 }
613}
614
615static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) {
616 int k;
617 for (k = 3; k > 0; --k) {
618 p += 4;
619 SimpleHFilter16(p, stride, thresh);
620 }
621}
622
623// DST[A * BPS] = TEMP0
624// DST[B + C * BPS] = TEMP1
625#define STORE_8_BYTES(TEMP0, TEMP1, A, B, C, DST) \
626 "usw %[" #TEMP0 "], " #A "*" XSTR(BPS) "(%[" #DST "]) \n\t" \
627 "usw %[" #TEMP1 "], " #B "+" #C "*" XSTR(BPS) "(%[" #DST "]) \n\t"
628
629static void VE4(uint8_t* dst) { // vertical
630 const uint8_t* top = dst - BPS;
631 int temp0, temp1, temp2, temp3, temp4, temp5, temp6;
632 __asm__ volatile (
633 "ulw %[temp0], -1(%[top]) \n\t"
634 "ulh %[temp1], 3(%[top]) \n\t"
635 "preceu.ph.qbr %[temp2], %[temp0] \n\t"
636 "preceu.ph.qbl %[temp3], %[temp0] \n\t"
637 "preceu.ph.qbr %[temp4], %[temp1] \n\t"
638 "packrl.ph %[temp5], %[temp3], %[temp2] \n\t"
639 "packrl.ph %[temp6], %[temp4], %[temp3] \n\t"
640 "shll.ph %[temp5], %[temp5], 1 \n\t"
641 "shll.ph %[temp6], %[temp6], 1 \n\t"
642 "addq.ph %[temp2], %[temp5], %[temp2] \n\t"
643 "addq.ph %[temp6], %[temp6], %[temp4] \n\t"
644 "addq.ph %[temp2], %[temp2], %[temp3] \n\t"
645 "addq.ph %[temp6], %[temp6], %[temp3] \n\t"
646 "shra_r.ph %[temp2], %[temp2], 2 \n\t"
647 "shra_r.ph %[temp6], %[temp6], 2 \n\t"
648 "precr.qb.ph %[temp4], %[temp6], %[temp2] \n\t"
649 STORE_8_BYTES(temp4, temp4, 0, 0, 1, dst)
650 STORE_8_BYTES(temp4, temp4, 2, 0, 3, dst)
651 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
652 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
653 [temp6]"=&r"(temp6)
654 : [top]"r"(top), [dst]"r"(dst)
655 : "memory"
656 );
657}
658
659static void DC4(uint8_t* dst) { // DC
660 int temp0, temp1, temp2, temp3, temp4;
661 __asm__ volatile (
662 "ulw %[temp0], -1*" XSTR(BPS) "(%[dst]) \n\t"
663 LOAD_4_BYTES(temp1, temp2, temp3, temp4, -1, 0, -1, 1, -1, 2, -1, 3, dst)
664 "ins %[temp1], %[temp2], 8, 8 \n\t"
665 "ins %[temp1], %[temp3], 16, 8 \n\t"
666 "ins %[temp1], %[temp4], 24, 8 \n\t"
667 "raddu.w.qb %[temp0], %[temp0] \n\t"
668 "raddu.w.qb %[temp1], %[temp1] \n\t"
669 "addu %[temp0], %[temp0], %[temp1] \n\t"
670 "shra_r.w %[temp0], %[temp0], 3 \n\t"
671 "replv.qb %[temp0], %[temp0] \n\t"
672 STORE_8_BYTES(temp0, temp0, 0, 0, 1, dst)
673 STORE_8_BYTES(temp0, temp0, 2, 0, 3, dst)
674 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
675 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4)
676 : [dst]"r"(dst)
677 : "memory"
678 );
679}
680
681static void RD4(uint8_t* dst) { // Down-right
682 int temp0, temp1, temp2, temp3, temp4;
683 int temp5, temp6, temp7, temp8;
684 __asm__ volatile (
685 LOAD_4_BYTES(temp0, temp1, temp2, temp3, -1, 0, -1, 1, -1, 2, -1, 3, dst)
686 "ulw %[temp7], -1-" XSTR(BPS) "(%[dst]) \n\t"
687 "ins %[temp1], %[temp0], 16, 16 \n\t"
688 "preceu.ph.qbr %[temp5], %[temp7] \n\t"
689 "ins %[temp2], %[temp1], 16, 16 \n\t"
690 "preceu.ph.qbl %[temp4], %[temp7] \n\t"
691 "ins %[temp3], %[temp2], 16, 16 \n\t"
692 "shll.ph %[temp2], %[temp2], 1 \n\t"
693 "addq.ph %[temp3], %[temp3], %[temp1] \n\t"
694 "packrl.ph %[temp6], %[temp5], %[temp1] \n\t"
695 "addq.ph %[temp3], %[temp3], %[temp2] \n\t"
696 "addq.ph %[temp1], %[temp1], %[temp5] \n\t"
697 "shll.ph %[temp6], %[temp6], 1 \n\t"
698 "addq.ph %[temp1], %[temp1], %[temp6] \n\t"
699 "packrl.ph %[temp0], %[temp4], %[temp5] \n\t"
700 "addq.ph %[temp8], %[temp5], %[temp4] \n\t"
701 "shra_r.ph %[temp3], %[temp3], 2 \n\t"
702 "shll.ph %[temp0], %[temp0], 1 \n\t"
703 "shra_r.ph %[temp1], %[temp1], 2 \n\t"
704 "addq.ph %[temp8], %[temp0], %[temp8] \n\t"
705 "lbu %[temp5], 3-" XSTR(BPS) "(%[dst]) \n\t"
706 "precrq.ph.w %[temp7], %[temp7], %[temp7] \n\t"
707 "shra_r.ph %[temp8], %[temp8], 2 \n\t"
708 "ins %[temp7], %[temp5], 0, 8 \n\t"
709 "precr.qb.ph %[temp2], %[temp1], %[temp3] \n\t"
710 "raddu.w.qb %[temp4], %[temp7] \n\t"
711 "precr.qb.ph %[temp6], %[temp8], %[temp1] \n\t"
712 "shra_r.w %[temp4], %[temp4], 2 \n\t"
713 STORE_8_BYTES(temp2, temp6, 3, 0, 1, dst)
714 "prepend %[temp2], %[temp8], 8 \n\t"
715 "prepend %[temp6], %[temp4], 8 \n\t"
716 STORE_8_BYTES(temp2, temp6, 2, 0, 0, dst)
717 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
718 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
719 [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8)
720 : [dst]"r"(dst)
721 : "memory"
722 );
723}
724
725// TEMP0 = SRC[A * BPS]
726// TEMP1 = SRC[B + C * BPS]
727#define LOAD_8_BYTES(TEMP0, TEMP1, A, B, C, SRC) \
728 "ulw %[" #TEMP0 "], " #A "*" XSTR(BPS) "(%[" #SRC "]) \n\t" \
729 "ulw %[" #TEMP1 "], " #B "+" #C "*" XSTR(BPS) "(%[" #SRC "]) \n\t"
730
731static void LD4(uint8_t* dst) { // Down-Left
732 int temp0, temp1, temp2, temp3, temp4;
733 int temp5, temp6, temp7, temp8, temp9;
734 __asm__ volatile (
735 LOAD_8_BYTES(temp0, temp1, -1, 4, -1, dst)
736 "preceu.ph.qbl %[temp2], %[temp0] \n\t"
737 "preceu.ph.qbr %[temp3], %[temp0] \n\t"
738 "preceu.ph.qbr %[temp4], %[temp1] \n\t"
739 "preceu.ph.qbl %[temp5], %[temp1] \n\t"
740 "packrl.ph %[temp6], %[temp2], %[temp3] \n\t"
741 "packrl.ph %[temp7], %[temp4], %[temp2] \n\t"
742 "packrl.ph %[temp8], %[temp5], %[temp4] \n\t"
743 "shll.ph %[temp6], %[temp6], 1 \n\t"
744 "addq.ph %[temp9], %[temp2], %[temp6] \n\t"
745 "shll.ph %[temp7], %[temp7], 1 \n\t"
746 "addq.ph %[temp9], %[temp9], %[temp3] \n\t"
747 "shll.ph %[temp8], %[temp8], 1 \n\t"
748 "shra_r.ph %[temp9], %[temp9], 2 \n\t"
749 "addq.ph %[temp3], %[temp4], %[temp7] \n\t"
750 "addq.ph %[temp0], %[temp5], %[temp8] \n\t"
751 "addq.ph %[temp3], %[temp3], %[temp2] \n\t"
752 "addq.ph %[temp0], %[temp0], %[temp4] \n\t"
753 "shra_r.ph %[temp3], %[temp3], 2 \n\t"
754 "shra_r.ph %[temp0], %[temp0], 2 \n\t"
755 "srl %[temp1], %[temp1], 24 \n\t"
756 "sll %[temp1], %[temp1], 1 \n\t"
757 "raddu.w.qb %[temp5], %[temp5] \n\t"
758 "precr.qb.ph %[temp9], %[temp3], %[temp9] \n\t"
759 "precr.qb.ph %[temp3], %[temp0], %[temp3] \n\t"
760 "addu %[temp1], %[temp1], %[temp5] \n\t"
761 "shra_r.w %[temp1], %[temp1], 2 \n\t"
762 STORE_8_BYTES(temp9, temp3, 0, 0, 2, dst)
763 "prepend %[temp9], %[temp0], 8 \n\t"
764 "prepend %[temp3], %[temp1], 8 \n\t"
765 STORE_8_BYTES(temp9, temp3, 1, 0, 3, dst)
766 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
767 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
768 [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
769 [temp9]"=&r"(temp9)
770 : [dst]"r"(dst)
771 : "memory"
772 );
773}
774
775//------------------------------------------------------------------------------
776// Chroma
777
778static void DC8uv(uint8_t* dst) { // DC
779 int temp0, temp1, temp2, temp3, temp4;
780 int temp5, temp6, temp7, temp8, temp9;
781 __asm__ volatile (
782 LOAD_8_BYTES(temp0, temp1, -1, 4, -1, dst)
783 LOAD_4_BYTES(temp2, temp3, temp4, temp5, -1, 0, -1, 1, -1, 2, -1, 3, dst)
784 LOAD_4_BYTES(temp6, temp7, temp8, temp9, -1, 4, -1, 5, -1, 6, -1, 7, dst)
785 "raddu.w.qb %[temp0], %[temp0] \n\t"
786 "raddu.w.qb %[temp1], %[temp1] \n\t"
787 "addu %[temp2], %[temp2], %[temp3] \n\t"
788 "addu %[temp4], %[temp4], %[temp5] \n\t"
789 "addu %[temp6], %[temp6], %[temp7] \n\t"
790 "addu %[temp8], %[temp8], %[temp9] \n\t"
791 "addu %[temp0], %[temp0], %[temp1] \n\t"
792 "addu %[temp2], %[temp2], %[temp4] \n\t"
793 "addu %[temp6], %[temp6], %[temp8] \n\t"
794 "addu %[temp0], %[temp0], %[temp2] \n\t"
795 "addu %[temp0], %[temp0], %[temp6] \n\t"
796 "shra_r.w %[temp0], %[temp0], 4 \n\t"
797 "replv.qb %[temp0], %[temp0] \n\t"
798 STORE_8_BYTES(temp0, temp0, 0, 4, 0, dst)
799 STORE_8_BYTES(temp0, temp0, 1, 4, 1, dst)
800 STORE_8_BYTES(temp0, temp0, 2, 4, 2, dst)
801 STORE_8_BYTES(temp0, temp0, 3, 4, 3, dst)
802 STORE_8_BYTES(temp0, temp0, 4, 4, 4, dst)
803 STORE_8_BYTES(temp0, temp0, 5, 4, 5, dst)
804 STORE_8_BYTES(temp0, temp0, 6, 4, 6, dst)
805 STORE_8_BYTES(temp0, temp0, 7, 4, 7, dst)
806 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
807 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
808 [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
809 [temp9]"=&r"(temp9)
810 : [dst]"r"(dst)
811 : "memory"
812 );
813}
814
815static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples
816 int temp0, temp1;
817 __asm__ volatile (
818 LOAD_8_BYTES(temp0, temp1, -1, 4, -1, dst)
819 "raddu.w.qb %[temp0], %[temp0] \n\t"
820 "raddu.w.qb %[temp1], %[temp1] \n\t"
821 "addu %[temp0], %[temp0], %[temp1] \n\t"
822 "shra_r.w %[temp0], %[temp0], 3 \n\t"
823 "replv.qb %[temp0], %[temp0] \n\t"
824 STORE_8_BYTES(temp0, temp0, 0, 4, 0, dst)
825 STORE_8_BYTES(temp0, temp0, 1, 4, 1, dst)
826 STORE_8_BYTES(temp0, temp0, 2, 4, 2, dst)
827 STORE_8_BYTES(temp0, temp0, 3, 4, 3, dst)
828 STORE_8_BYTES(temp0, temp0, 4, 4, 4, dst)
829 STORE_8_BYTES(temp0, temp0, 5, 4, 5, dst)
830 STORE_8_BYTES(temp0, temp0, 6, 4, 6, dst)
831 STORE_8_BYTES(temp0, temp0, 7, 4, 7, dst)
832 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1)
833 : [dst]"r"(dst)
834 : "memory"
835 );
836}
837
838static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples
839 int temp0, temp1, temp2, temp3, temp4;
840 int temp5, temp6, temp7, temp8;
841 __asm__ volatile (
842 LOAD_4_BYTES(temp2, temp3, temp4, temp5, -1, 0, -1, 1, -1, 2, -1, 3, dst)
843 LOAD_4_BYTES(temp6, temp7, temp8, temp1, -1, 4, -1, 5, -1, 6, -1, 7, dst)
844 "addu %[temp2], %[temp2], %[temp3] \n\t"
845 "addu %[temp4], %[temp4], %[temp5] \n\t"
846 "addu %[temp6], %[temp6], %[temp7] \n\t"
847 "addu %[temp8], %[temp8], %[temp1] \n\t"
848 "addu %[temp2], %[temp2], %[temp4] \n\t"
849 "addu %[temp6], %[temp6], %[temp8] \n\t"
850 "addu %[temp0], %[temp6], %[temp2] \n\t"
851 "shra_r.w %[temp0], %[temp0], 3 \n\t"
852 "replv.qb %[temp0], %[temp0] \n\t"
853 STORE_8_BYTES(temp0, temp0, 0, 4, 0, dst)
854 STORE_8_BYTES(temp0, temp0, 1, 4, 1, dst)
855 STORE_8_BYTES(temp0, temp0, 2, 4, 2, dst)
856 STORE_8_BYTES(temp0, temp0, 3, 4, 3, dst)
857 STORE_8_BYTES(temp0, temp0, 4, 4, 4, dst)
858 STORE_8_BYTES(temp0, temp0, 5, 4, 5, dst)
859 STORE_8_BYTES(temp0, temp0, 6, 4, 6, dst)
860 STORE_8_BYTES(temp0, temp0, 7, 4, 7, dst)
861 : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
862 [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
863 [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8)
864 : [dst]"r"(dst)
865 : "memory"
866 );
867}
868
869#undef LOAD_8_BYTES
870#undef STORE_8_BYTES
871#undef LOAD_4_BYTES
872
873#define CLIPPING(SIZE) \
874 "preceu.ph.qbl %[temp2], %[temp0] \n\t" \
875 "preceu.ph.qbr %[temp0], %[temp0] \n\t" \
876".if " #SIZE " == 8 \n\t" \
877 "preceu.ph.qbl %[temp3], %[temp1] \n\t" \
878 "preceu.ph.qbr %[temp1], %[temp1] \n\t" \
879".endif \n\t" \
880 "addu.ph %[temp2], %[temp2], %[dst_1] \n\t" \
881 "addu.ph %[temp0], %[temp0], %[dst_1] \n\t" \
882".if " #SIZE " == 8 \n\t" \
883 "addu.ph %[temp3], %[temp3], %[dst_1] \n\t" \
884 "addu.ph %[temp1], %[temp1], %[dst_1] \n\t" \
885".endif \n\t" \
886 "shll_s.ph %[temp2], %[temp2], 7 \n\t" \
887 "shll_s.ph %[temp0], %[temp0], 7 \n\t" \
888".if " #SIZE " == 8 \n\t" \
889 "shll_s.ph %[temp3], %[temp3], 7 \n\t" \
890 "shll_s.ph %[temp1], %[temp1], 7 \n\t" \
891".endif \n\t" \
892 "precrqu_s.qb.ph %[temp0], %[temp2], %[temp0] \n\t" \
893".if " #SIZE " == 8 \n\t" \
894 "precrqu_s.qb.ph %[temp1], %[temp3], %[temp1] \n\t" \
895".endif \n\t"
896
897
898#define CLIP_8B_TO_DST(DST, TOP, SIZE) do { \
899 int dst_1 = ((int)(DST)[-1] << 16) + (DST)[-1]; \
900 int temp0, temp1, temp2, temp3; \
901 __asm__ volatile ( \
902 ".if " #SIZE " < 8 \n\t" \
903 "ulw %[temp0], 0(%[top]) \n\t" \
904 "subu.ph %[dst_1], %[dst_1], %[top_1] \n\t" \
905 CLIPPING(4) \
906 "usw %[temp0], 0(%[dst]) \n\t" \
907 ".else \n\t" \
908 "ulw %[temp0], 0(%[top]) \n\t" \
909 "ulw %[temp1], 4(%[top]) \n\t" \
910 "subu.ph %[dst_1], %[dst_1], %[top_1] \n\t" \
911 CLIPPING(8) \
912 "usw %[temp0], 0(%[dst]) \n\t" \
913 "usw %[temp1], 4(%[dst]) \n\t" \
914 ".if " #SIZE " == 16 \n\t" \
915 "ulw %[temp0], 8(%[top]) \n\t" \
916 "ulw %[temp1], 12(%[top]) \n\t" \
917 CLIPPING(8) \
918 "usw %[temp0], 8(%[dst]) \n\t" \
919 "usw %[temp1], 12(%[dst]) \n\t" \
920 ".endif \n\t" \
921 ".endif \n\t" \
922 : [dst_1]"+&r"(dst_1), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), \
923 [temp2]"=&r"(temp2), [temp3]"=&r"(temp3) \
924 : [top_1]"r"(top_1), [top]"r"((TOP)), [dst]"r"((DST)) \
925 : "memory" \
926 ); \
927} while (0)
928
929#define CLIP_TO_DST(DST, SIZE) do { \
930 int y; \
931 const uint8_t* top = (DST) - BPS; \
932 const int top_1 = ((int)top[-1] << 16) + top[-1]; \
933 for (y = 0; y < (SIZE); ++y) { \
934 CLIP_8B_TO_DST((DST), top, (SIZE)); \
935 (DST) += BPS; \
936 } \
937} while (0)
938
939#define TRUE_MOTION(DST, SIZE) \
940static void TrueMotion##SIZE(uint8_t* (DST)) { \
941 CLIP_TO_DST((DST), (SIZE)); \
942}
943
944TRUE_MOTION(dst, 4)
945TRUE_MOTION(dst, 8)
946TRUE_MOTION(dst, 16)
947
948#undef TRUE_MOTION
949#undef CLIP_TO_DST
950#undef CLIP_8B_TO_DST
951#undef CLIPPING
952
953//------------------------------------------------------------------------------
954// Entry point
955
956extern void VP8DspInitMIPSdspR2(void);
957
958WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMIPSdspR2(void) {
959 VP8TransformDC = TransformDC;
960 VP8TransformAC3 = TransformAC3;
961 VP8Transform = TransformTwo;
962
963 VP8VFilter16 = VFilter16;
964 VP8HFilter16 = HFilter16;
965 VP8VFilter8 = VFilter8;
966 VP8HFilter8 = HFilter8;
967 VP8VFilter16i = VFilter16i;
968 VP8HFilter16i = HFilter16i;
969 VP8VFilter8i = VFilter8i;
970 VP8HFilter8i = HFilter8i;
971 VP8SimpleVFilter16 = SimpleVFilter16;
972 VP8SimpleHFilter16 = SimpleHFilter16;
973 VP8SimpleVFilter16i = SimpleVFilter16i;
974 VP8SimpleHFilter16i = SimpleHFilter16i;
975
976 VP8PredLuma4[0] = DC4;
977 VP8PredLuma4[1] = TrueMotion4;
978 VP8PredLuma4[2] = VE4;
979 VP8PredLuma4[4] = RD4;
980 VP8PredLuma4[6] = LD4;
981
982 VP8PredChroma8[0] = DC8uv;
983 VP8PredChroma8[1] = TrueMotion8;
984 VP8PredChroma8[4] = DC8uvNoTop;
985 VP8PredChroma8[5] = DC8uvNoLeft;
986
987 VP8PredLuma16[1] = TrueMotion16;
988}
989
990#else // !WEBP_USE_MIPS_DSP_R2
991
992WEBP_DSP_INIT_STUB(VP8DspInitMIPSdspR2)
993
994#endif // WEBP_USE_MIPS_DSP_R2
995