1 | // Copyright 2018 Google Inc. All Rights Reserved. |
2 | // |
3 | // Use of this source code is governed by a BSD-style license |
4 | // that can be found in the COPYING file in the root of the source |
5 | // tree. An additional intellectual property rights grant can be found |
6 | // in the file PATENTS. All contributing project authors may |
7 | // be found in the AUTHORS file in the root of the source tree. |
8 | // ----------------------------------------------------------------------------- |
9 | // |
10 | // ARM NEON version of cost functions |
11 | |
12 | #include "src/dsp/dsp.h" |
13 | |
14 | #if defined(WEBP_USE_NEON) |
15 | |
16 | #include "src/dsp/neon.h" |
17 | #include "src/enc/cost_enc.h" |
18 | |
19 | static const uint8_t position[16] = { 1, 2, 3, 4, 5, 6, 7, 8, |
20 | 9, 10, 11, 12, 13, 14, 15, 16 }; |
21 | |
22 | static void SetResidualCoeffs_NEON(const int16_t* const coeffs, |
23 | VP8Residual* const res) { |
24 | const int16x8_t minus_one = vdupq_n_s16(-1); |
25 | const int16x8_t coeffs_0 = vld1q_s16(coeffs); |
26 | const int16x8_t coeffs_1 = vld1q_s16(coeffs + 8); |
27 | const uint16x8_t eob_0 = vtstq_s16(coeffs_0, minus_one); |
28 | const uint16x8_t eob_1 = vtstq_s16(coeffs_1, minus_one); |
29 | const uint8x16_t eob = vcombine_u8(vqmovn_u16(eob_0), vqmovn_u16(eob_1)); |
30 | const uint8x16_t masked = vandq_u8(eob, vld1q_u8(position)); |
31 | |
32 | #ifdef __aarch64__ |
33 | res->last = vmaxvq_u8(masked) - 1; |
34 | #else |
35 | const uint8x8_t eob_8x8 = vmax_u8(vget_low_u8(masked), vget_high_u8(masked)); |
36 | const uint16x8_t eob_16x8 = vmovl_u8(eob_8x8); |
37 | const uint16x4_t eob_16x4 = |
38 | vmax_u16(vget_low_u16(eob_16x8), vget_high_u16(eob_16x8)); |
39 | const uint32x4_t eob_32x4 = vmovl_u16(eob_16x4); |
40 | uint32x2_t eob_32x2 = |
41 | vmax_u32(vget_low_u32(eob_32x4), vget_high_u32(eob_32x4)); |
42 | eob_32x2 = vpmax_u32(eob_32x2, eob_32x2); |
43 | |
44 | vst1_lane_s32(&res->last, vreinterpret_s32_u32(eob_32x2), 0); |
45 | --res->last; |
46 | #endif // __aarch64__ |
47 | |
48 | res->coeffs = coeffs; |
49 | } |
50 | |
51 | static int GetResidualCost_NEON(int ctx0, const VP8Residual* const res) { |
52 | uint8_t levels[16], ctxs[16]; |
53 | uint16_t abs_levels[16]; |
54 | int n = res->first; |
55 | // should be prob[VP8EncBands[n]], but it's equivalent for n=0 or 1 |
56 | const int p0 = res->prob[n][ctx0][0]; |
57 | CostArrayPtr const costs = res->costs; |
58 | const uint16_t* t = costs[n][ctx0]; |
59 | // bit_cost(1, p0) is already incorporated in t[] tables, but only if ctx != 0 |
60 | // (as required by the syntax). For ctx0 == 0, we need to add it here or it'll |
61 | // be missing during the loop. |
62 | int cost = (ctx0 == 0) ? VP8BitCost(1, p0) : 0; |
63 | |
64 | if (res->last < 0) { |
65 | return VP8BitCost(0, p0); |
66 | } |
67 | |
68 | { // precompute clamped levels and contexts, packed to 8b. |
69 | const uint8x16_t kCst2 = vdupq_n_u8(2); |
70 | const uint8x16_t kCst67 = vdupq_n_u8(MAX_VARIABLE_LEVEL); |
71 | const int16x8_t c0 = vld1q_s16(res->coeffs); |
72 | const int16x8_t c1 = vld1q_s16(res->coeffs + 8); |
73 | const uint16x8_t E0 = vreinterpretq_u16_s16(vabsq_s16(c0)); |
74 | const uint16x8_t E1 = vreinterpretq_u16_s16(vabsq_s16(c1)); |
75 | const uint8x16_t F = vcombine_u8(vqmovn_u16(E0), vqmovn_u16(E1)); |
76 | const uint8x16_t G = vminq_u8(F, kCst2); // context = 0,1,2 |
77 | const uint8x16_t H = vminq_u8(F, kCst67); // clamp_level in [0..67] |
78 | |
79 | vst1q_u8(ctxs, G); |
80 | vst1q_u8(levels, H); |
81 | |
82 | vst1q_u16(abs_levels, E0); |
83 | vst1q_u16(abs_levels + 8, E1); |
84 | } |
85 | for (; n < res->last; ++n) { |
86 | const int ctx = ctxs[n]; |
87 | const int level = levels[n]; |
88 | const int flevel = abs_levels[n]; // full level |
89 | cost += VP8LevelFixedCosts[flevel] + t[level]; // simplified VP8LevelCost() |
90 | t = costs[n + 1][ctx]; |
91 | } |
92 | // Last coefficient is always non-zero |
93 | { |
94 | const int level = levels[n]; |
95 | const int flevel = abs_levels[n]; |
96 | assert(flevel != 0); |
97 | cost += VP8LevelFixedCosts[flevel] + t[level]; |
98 | if (n < 15) { |
99 | const int b = VP8EncBands[n + 1]; |
100 | const int ctx = ctxs[n]; |
101 | const int last_p0 = res->prob[b][ctx][0]; |
102 | cost += VP8BitCost(0, last_p0); |
103 | } |
104 | } |
105 | return cost; |
106 | } |
107 | |
108 | //------------------------------------------------------------------------------ |
109 | // Entry point |
110 | |
111 | extern void VP8EncDspCostInitNEON(void); |
112 | |
113 | WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInitNEON(void) { |
114 | VP8SetResidualCoeffs = SetResidualCoeffs_NEON; |
115 | VP8GetResidualCost = GetResidualCost_NEON; |
116 | } |
117 | |
118 | #else // !WEBP_USE_NEON |
119 | |
120 | WEBP_DSP_INIT_STUB(VP8EncDspCostInitNEON) |
121 | |
122 | #endif // WEBP_USE_NEON |
123 | |