1/*
2 * Copyright 2018 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8#include "src/gpu/effects/GrYUVtoRGBEffect.h"
9
10#include "src/core/SkYUVMath.h"
11#include "src/gpu/GrTexture.h"
12#include "src/gpu/effects/GrMatrixEffect.h"
13#include "src/gpu/glsl/GrGLSLFragmentProcessor.h"
14#include "src/gpu/glsl/GrGLSLFragmentShaderBuilder.h"
15#include "src/gpu/glsl/GrGLSLProgramBuilder.h"
16#include "src/sksl/SkSLCPP.h"
17#include "src/sksl/SkSLUtil.h"
18
19static void border_colors(SkYUVColorSpace cs,
20 const SkYUVAIndex yuvaIndices[4],
21 float planeBorders[4][4]) {
22 float m[20];
23 SkColorMatrix_RGB2YUV(cs, m);
24 for (int i = 0; i < 4; ++i) {
25 if (yuvaIndices[i].fIndex == -1) {
26 return;
27 }
28 auto c = static_cast<int>(yuvaIndices[i].fChannel);
29 planeBorders[yuvaIndices[i].fIndex][c] = m[i*5 + 4];
30 }
31}
32
33std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::Make(GrSurfaceProxyView views[],
34 const SkYUVAIndex yuvaIndices[4],
35 SkYUVColorSpace yuvColorSpace,
36 GrSamplerState samplerState,
37 const GrCaps& caps,
38 const SkMatrix& localMatrix,
39 const SkRect* subset,
40 const SkRect* domain) {
41 int numPlanes;
42 SkAssertResult(SkYUVAIndex::AreValidIndices(yuvaIndices, &numPlanes));
43
44 const SkISize yDimensions =
45 views[yuvaIndices[SkYUVAIndex::kY_Index].fIndex].proxy()->dimensions();
46
47 bool usesBorder = samplerState.wrapModeX() == GrSamplerState::WrapMode::kClampToBorder ||
48 samplerState.wrapModeY() == GrSamplerState::WrapMode::kClampToBorder;
49 float planeBorders[4][4] = {};
50 if (usesBorder) {
51 border_colors(yuvColorSpace, yuvaIndices, planeBorders);
52 }
53
54 bool snap[2] = {false, false};
55 std::unique_ptr<GrFragmentProcessor> planeFPs[4];
56 for (int i = 0; i < numPlanes; ++i) {
57 SkISize dimensions = views[i].proxy()->dimensions();
58 SkTCopyOnFirstWrite<SkMatrix> planeMatrix(&SkMatrix::I());
59 SkRect planeSubset;
60 SkRect planeDomain;
61 bool makeLinearWithSnap = false;
62 float sx = 1.f,
63 sy = 1.f;
64 if (dimensions != yDimensions) {
65 // JPEG chroma subsampling of odd dimensions produces U and V planes with the ceiling of
66 // the image size divided by the subsampling factor (2). Our API for creating YUVA
67 // doesn't capture the intended subsampling (and we should fix that). This fixes up 2x
68 // subsampling for images with odd widths/heights (e.g. JPEG 420 or 422).
69 sx = (float)dimensions.width() / yDimensions.width();
70 sy = (float)dimensions.height() / yDimensions.height();
71 if ((yDimensions.width() & 0b1) && dimensions.width() == yDimensions.width() / 2 + 1) {
72 sx = 0.5f;
73 }
74 if ((yDimensions.height() & 0b1) &&
75 dimensions.height() == yDimensions.height() / 2 + 1) {
76 sy = 0.5f;
77 }
78 *planeMatrix.writable() = SkMatrix::Scale(sx, sy);
79 if (subset) {
80 planeSubset = {subset->fLeft * sx,
81 subset->fTop * sy,
82 subset->fRight * sx,
83 subset->fBottom * sy};
84 }
85 if (domain) {
86 planeDomain = {domain->fLeft * sx,
87 domain->fTop * sy,
88 domain->fRight * sx,
89 domain->fBottom * sy};
90 }
91 // This promotion of nearest to linear filtering for UV planes exists to mimic
92 // libjpeg[-turbo]'s do_fancy_upsampling option. We will filter the subsampled plane,
93 // however we want to filter at a fixed point for each logical image pixel to simulate
94 // nearest neighbor.
95 if (samplerState.filter() == GrSamplerState::Filter::kNearest) {
96 bool snapX = (sx != 1.f),
97 snapY = (sy != 1.f);
98 makeLinearWithSnap = snapX || snapY;
99 snap[0] |= snapX;
100 snap[1] |= snapY;
101 if (domain) {
102 // The outer YUVToRGB effect will ensure sampling happens at pixel centers
103 // within this plane.
104 planeDomain = {std::floor(planeDomain.fLeft) + 0.5f,
105 std::floor(planeDomain.fTop) + 0.5f,
106 std::floor(planeDomain.fRight) + 0.5f,
107 std::floor(planeDomain.fBottom) + 0.5f};
108 }
109 }
110 } else {
111 if (subset) {
112 planeSubset = *subset;
113 }
114 if (domain) {
115 planeDomain = *domain;
116 }
117 }
118 if (subset) {
119 SkASSERT(samplerState.mipmapped() == GrMipmapped::kNo);
120 if (makeLinearWithSnap) {
121 // The plane is subsampled and we have an overall subset on the image. We're
122 // emulating do_fancy_upsampling using linear filtering but snapping look ups to the
123 // y-plane pixel centers. Consider a logical image pixel at the edge of the subset.
124 // When computing the logical pixel color value we should use a 50/50 blend of two
125 // values from the subsampled plane. Depending on where the subset edge falls in
126 // actual subsampled plane, one of those values may come from outside the subset.
127 // Hence, we use this custom inset factory which applies the wrap mode to
128 // planeSubset but allows linear filtering to read pixels from the plane that are
129 // just outside planeSubset.
130 SkRect* domainRect = domain ? &planeDomain : nullptr;
131 planeFPs[i] = GrTextureEffect::MakeCustomLinearFilterInset(
132 views[i], kUnknown_SkAlphaType, *planeMatrix, samplerState.wrapModeX(),
133 samplerState.wrapModeY(), planeSubset, domainRect, {sx / 2.f, sy / 2.f},
134 caps, planeBorders[i]);
135 } else if (domain) {
136 planeFPs[i] = GrTextureEffect::MakeSubset(views[i], kUnknown_SkAlphaType,
137 *planeMatrix, samplerState, planeSubset,
138 planeDomain, caps, planeBorders[i]);
139 } else {
140 planeFPs[i] = GrTextureEffect::MakeSubset(views[i], kUnknown_SkAlphaType,
141 *planeMatrix, samplerState, planeSubset,
142 caps, planeBorders[i]);
143 }
144 } else {
145 GrSamplerState planeSampler = samplerState;
146 if (makeLinearWithSnap) {
147 planeSampler.setFilterMode(GrSamplerState::Filter::kLinear);
148 }
149 planeFPs[i] = GrTextureEffect::Make(views[i], kUnknown_SkAlphaType, *planeMatrix,
150 planeSampler, caps, planeBorders[i]);
151 }
152 }
153 auto fp = std::unique_ptr<GrFragmentProcessor>(
154 new GrYUVtoRGBEffect(planeFPs, numPlanes, yuvaIndices, snap, yuvColorSpace));
155 return GrMatrixEffect::Make(localMatrix, std::move(fp));
156}
157
158static SkAlphaType alpha_type(const SkYUVAIndex yuvaIndices[4]) {
159 return yuvaIndices[3].fIndex >= 0 ? kPremul_SkAlphaType : kOpaque_SkAlphaType;
160}
161
162GrYUVtoRGBEffect::GrYUVtoRGBEffect(std::unique_ptr<GrFragmentProcessor> planeFPs[4],
163 int numPlanes,
164 const SkYUVAIndex yuvaIndices[4],
165 const bool snap[2],
166 SkYUVColorSpace yuvColorSpace)
167 : GrFragmentProcessor(kGrYUVtoRGBEffect_ClassID,
168 ModulateForClampedSamplerOptFlags(alpha_type(yuvaIndices)))
169 , fYUVColorSpace(yuvColorSpace) {
170 std::copy_n(yuvaIndices, 4, fYUVAIndices);
171 std::copy_n(snap, 2, fSnap);
172
173 if (fSnap[0] || fSnap[1]) {
174 // Need this so that we can access coords in SKSL to perform snapping.
175 this->setUsesSampleCoordsDirectly();
176 for (int i = 0; i < numPlanes; ++i) {
177 this->registerChild(std::move(planeFPs[i]), SkSL::SampleUsage::Explicit());
178 }
179 } else {
180 for (int i = 0; i < numPlanes; ++i) {
181 this->registerChild(std::move(planeFPs[i]));
182 }
183 }
184}
185
186#if GR_TEST_UTILS
187SkString GrYUVtoRGBEffect::onDumpInfo() const {
188 SkString str("(");
189 for (int i = 0; i < 4; ++i) {
190 str.appendf("YUVAIndices[%d]=%d %d, ",
191 i, fYUVAIndices[i].fIndex, static_cast<int>(fYUVAIndices[i].fChannel));
192 }
193 str.appendf("YUVColorSpace=%d, snap=(%d, %d))",
194 static_cast<int>(fYUVColorSpace), fSnap[0], fSnap[1]);
195 return str;
196}
197#endif
198
199GrGLSLFragmentProcessor* GrYUVtoRGBEffect::onCreateGLSLInstance() const {
200 class GrGLSLYUVtoRGBEffect : public GrGLSLFragmentProcessor {
201 public:
202 GrGLSLYUVtoRGBEffect() {}
203
204 void emitCode(EmitArgs& args) override {
205 GrGLSLFPFragmentBuilder* fragBuilder = args.fFragBuilder;
206 const GrYUVtoRGBEffect& yuvEffect = args.fFp.cast<GrYUVtoRGBEffect>();
207
208 int numPlanes = yuvEffect.numChildProcessors();
209
210 const char* sampleCoords = "";
211 if (yuvEffect.fSnap[0] || yuvEffect.fSnap[1]) {
212 fragBuilder->codeAppendf("float2 snappedCoords = %s;", args.fSampleCoord);
213 if (yuvEffect.fSnap[0]) {
214 fragBuilder->codeAppend("snappedCoords.x = floor(snappedCoords.x) + 0.5;");
215 }
216 if (yuvEffect.fSnap[1]) {
217 fragBuilder->codeAppend("snappedCoords.y = floor(snappedCoords.y) + 0.5;");
218 }
219 sampleCoords = "snappedCoords";
220 }
221
222 fragBuilder->codeAppendf("half4 planes[%d];", numPlanes);
223 for (int i = 0; i < numPlanes; ++i) {
224 SkString tempVar = this->invokeChild(i, args, sampleCoords);
225 fragBuilder->codeAppendf("planes[%d] = %s;", i, tempVar.c_str());
226 }
227
228 bool hasAlpha = yuvEffect.fYUVAIndices[3].fIndex >= 0;
229 SkString rgba[4];
230 rgba[3] = "1";
231 for (int i = 0; i < (hasAlpha ? 4 : 3); ++i) {
232 auto info = yuvEffect.fYUVAIndices[i];
233 auto letter = "rgba"[static_cast<int>(info.fChannel)];
234 rgba[i].printf("planes[%d].%c", info.fIndex, letter);
235 }
236
237 fragBuilder->codeAppendf("half4 color = half4(%s, %s, %s, %s);",
238 rgba[0].c_str(), rgba[1].c_str(), rgba[2].c_str(), rgba[3].c_str());
239
240 if (kIdentity_SkYUVColorSpace != yuvEffect.fYUVColorSpace) {
241 fColorSpaceMatrixVar = args.fUniformHandler->addUniform(&yuvEffect,
242 kFragment_GrShaderFlag, kHalf3x3_GrSLType, "colorSpaceMatrix");
243 fColorSpaceTranslateVar = args.fUniformHandler->addUniform(&yuvEffect,
244 kFragment_GrShaderFlag, kHalf3_GrSLType, "colorSpaceTranslate");
245 fragBuilder->codeAppendf(
246 "color.rgb = saturate(color.rgb * %s + %s);",
247 args.fUniformHandler->getUniformCStr(fColorSpaceMatrixVar),
248 args.fUniformHandler->getUniformCStr(fColorSpaceTranslateVar));
249 }
250
251 if (hasAlpha) {
252 // premultiply alpha
253 fragBuilder->codeAppendf("color.rgb *= color.a;");
254 }
255 fragBuilder->codeAppendf("%s = color;", args.fOutputColor);
256 }
257
258 private:
259 void onSetData(const GrGLSLProgramDataManager& pdman,
260 const GrFragmentProcessor& proc) override {
261 const GrYUVtoRGBEffect& yuvEffect = proc.cast<GrYUVtoRGBEffect>();
262
263 if (yuvEffect.fYUVColorSpace != kIdentity_SkYUVColorSpace) {
264 SkASSERT(fColorSpaceMatrixVar.isValid());
265 float yuvM[20];
266 SkColorMatrix_YUV2RGB(yuvEffect.fYUVColorSpace, yuvM);
267 // We drop the fourth column entirely since the transformation
268 // should not depend on alpha. The fifth column is sent as a separate
269 // vector. The fourth row is also dropped entirely because alpha should
270 // never be modified.
271 SkASSERT(yuvM[3] == 0 && yuvM[8] == 0 && yuvM[13] == 0 && yuvM[18] == 1);
272 SkASSERT(yuvM[15] == 0 && yuvM[16] == 0 && yuvM[17] == 0 && yuvM[19] == 0);
273 float mtx[9] = {
274 yuvM[ 0], yuvM[ 1], yuvM[ 2],
275 yuvM[ 5], yuvM[ 6], yuvM[ 7],
276 yuvM[10], yuvM[11], yuvM[12],
277 };
278 float v[3] = {yuvM[4], yuvM[9], yuvM[14]};
279 pdman.setMatrix3f(fColorSpaceMatrixVar, mtx);
280 pdman.set3fv(fColorSpaceTranslateVar, 1, v);
281 }
282 }
283
284 UniformHandle fColorSpaceMatrixVar;
285 UniformHandle fColorSpaceTranslateVar;
286 };
287
288 return new GrGLSLYUVtoRGBEffect;
289}
290void GrYUVtoRGBEffect::onGetGLSLProcessorKey(const GrShaderCaps& caps,
291 GrProcessorKeyBuilder* b) const {
292 uint32_t packed = 0;
293 for (int i = 0; i < 4; ++i) {
294 if (fYUVAIndices[i].fIndex < 0) {
295 continue;
296 }
297
298 uint8_t index = fYUVAIndices[i].fIndex;
299 uint8_t chann = static_cast<int>(fYUVAIndices[i].fChannel);
300
301 SkASSERT(index < 4 && chann < 4);
302
303 packed |= (index | (chann << 2)) << (i * 4);
304 }
305 if (fYUVColorSpace == kIdentity_SkYUVColorSpace) {
306 packed |= 1 << 16;
307 }
308 if (fSnap[0]) {
309 packed |= 1 << 17;
310 }
311 if (fSnap[1]) {
312 packed |= 1 << 18;
313 }
314 b->add32(packed);
315}
316
317bool GrYUVtoRGBEffect::onIsEqual(const GrFragmentProcessor& other) const {
318 const GrYUVtoRGBEffect& that = other.cast<GrYUVtoRGBEffect>();
319
320 return std::equal(fYUVAIndices, fYUVAIndices + 4, that.fYUVAIndices) &&
321 std::equal(fSnap, fSnap + 2, that.fSnap) &&
322 fYUVColorSpace == that.fYUVColorSpace;
323}
324
325GrYUVtoRGBEffect::GrYUVtoRGBEffect(const GrYUVtoRGBEffect& src)
326 : GrFragmentProcessor(kGrYUVtoRGBEffect_ClassID, src.optimizationFlags())
327 , fYUVColorSpace(src.fYUVColorSpace) {
328 this->cloneAndRegisterAllChildProcessors(src);
329 if (src.fSnap[0] || src.fSnap[1]) {
330 this->setUsesSampleCoordsDirectly();
331 }
332 std::copy_n(src.fYUVAIndices, this->numChildProcessors(), fYUVAIndices);
333 std::copy_n(src.fSnap, 2, fSnap);
334}
335
336std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::clone() const {
337 return std::unique_ptr<GrFragmentProcessor>(new GrYUVtoRGBEffect(*this));
338}
339