1// Copyright 2013 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "flutter/lib/ui/painting/image_encoding.h"
6
7#include <memory>
8#include <utility>
9
10#include "flutter/common/task_runners.h"
11#include "flutter/fml/build_config.h"
12#include "flutter/fml/make_copyable.h"
13#include "flutter/fml/trace_event.h"
14#include "flutter/lib/ui/painting/image.h"
15#include "flutter/lib/ui/ui_dart_state.h"
16#include "third_party/skia/include/core/SkCanvas.h"
17#include "third_party/skia/include/core/SkEncodedImageFormat.h"
18#include "third_party/skia/include/core/SkImage.h"
19#include "third_party/skia/include/core/SkSurface.h"
20#include "third_party/tonic/dart_persistent_value.h"
21#include "third_party/tonic/logging/dart_invoke.h"
22#include "third_party/tonic/typed_data/typed_list.h"
23
24using tonic::DartInvoke;
25using tonic::DartPersistentValue;
26using tonic::ToDart;
27
28namespace flutter {
29namespace {
30
31// This must be kept in sync with the enum in painting.dart
32enum ImageByteFormat {
33 kRawRGBA,
34 kRawUnmodified,
35 kPNG,
36};
37
38void FinalizeSkData(void* isolate_callback_data,
39 Dart_WeakPersistentHandle handle,
40 void* peer) {
41 SkData* buffer = reinterpret_cast<SkData*>(peer);
42 buffer->unref();
43}
44
45void InvokeDataCallback(std::unique_ptr<DartPersistentValue> callback,
46 sk_sp<SkData> buffer) {
47 std::shared_ptr<tonic::DartState> dart_state = callback->dart_state().lock();
48 if (!dart_state) {
49 return;
50 }
51 tonic::DartState::Scope scope(dart_state);
52 if (!buffer) {
53 DartInvoke(callback->value(), {Dart_Null()});
54 return;
55 }
56 // Skia will not modify the buffer, and it is backed by memory that is
57 // read/write, so Dart can be given direct access to the buffer through an
58 // external Uint8List.
59 void* bytes = const_cast<void*>(buffer->data());
60 const intptr_t length = buffer->size();
61 void* peer = reinterpret_cast<void*>(buffer.release());
62 Dart_Handle dart_data = Dart_NewExternalTypedDataWithFinalizer(
63 Dart_TypedData_kUint8, bytes, length, peer, length, FinalizeSkData);
64 DartInvoke(callback->value(), {dart_data});
65}
66
67sk_sp<SkImage> ConvertToRasterUsingResourceContext(
68 sk_sp<SkImage> image,
69 GrDirectContext* resource_context) {
70 sk_sp<SkSurface> surface;
71 SkImageInfo surface_info = SkImageInfo::MakeN32Premul(image->dimensions());
72 if (resource_context) {
73 surface = SkSurface::MakeRenderTarget(resource_context, SkBudgeted::kNo,
74 surface_info);
75 } else {
76 surface = SkSurface::MakeRaster(surface_info);
77 }
78
79 if (surface == nullptr || surface->getCanvas() == nullptr) {
80 FML_LOG(ERROR) << "Could not create a surface to copy the texture into.";
81 return nullptr;
82 }
83
84 surface->getCanvas()->drawImage(image, 0, 0);
85 surface->getCanvas()->flush();
86
87 auto snapshot = surface->makeImageSnapshot();
88
89 if (snapshot == nullptr) {
90 FML_LOG(ERROR) << "Could not snapshot image to encode.";
91 return nullptr;
92 }
93
94 return snapshot->makeRasterImage();
95}
96
97void ConvertImageToRaster(sk_sp<SkImage> image,
98 std::function<void(sk_sp<SkImage>)> encode_task,
99 fml::RefPtr<fml::TaskRunner> raster_task_runner,
100 fml::RefPtr<fml::TaskRunner> io_task_runner,
101 GrDirectContext* resource_context,
102 fml::WeakPtr<SnapshotDelegate> snapshot_delegate) {
103 // Check validity of the image.
104 if (image == nullptr) {
105 FML_LOG(ERROR) << "Image was null.";
106 encode_task(nullptr);
107 return;
108 }
109
110 auto dimensions = image->dimensions();
111
112 if (dimensions.isEmpty()) {
113 FML_LOG(ERROR) << "Image dimensions were empty.";
114 encode_task(nullptr);
115 return;
116 }
117
118 SkPixmap pixmap;
119 if (image->peekPixels(&pixmap)) {
120 // This is already a raster image.
121 encode_task(image);
122 return;
123 }
124
125 if (sk_sp<SkImage> raster_image = image->makeRasterImage()) {
126 // The image can be converted to a raster image.
127 encode_task(raster_image);
128 return;
129 }
130
131 // Cross-context images do not support makeRasterImage. Convert these images
132 // by drawing them into a surface. This must be done on the raster thread
133 // to prevent concurrent usage of the image on both the IO and raster threads.
134 raster_task_runner->PostTask([image, encode_task = std::move(encode_task),
135 resource_context, snapshot_delegate,
136 io_task_runner]() {
137 sk_sp<SkImage> raster_image =
138 snapshot_delegate->ConvertToRasterImage(image);
139
140 io_task_runner->PostTask([image, encode_task = std::move(encode_task),
141 raster_image = std::move(raster_image),
142 resource_context]() mutable {
143 if (!raster_image) {
144 // The rasterizer was unable to render the cross-context image
145 // (presumably because it does not have a GrContext). In that case,
146 // convert the image on the IO thread using the resource context.
147 raster_image =
148 ConvertToRasterUsingResourceContext(image, resource_context);
149 }
150 encode_task(raster_image);
151 });
152 });
153}
154
155sk_sp<SkData> CopyImageByteData(sk_sp<SkImage> raster_image,
156 SkColorType color_type) {
157 FML_DCHECK(raster_image);
158
159 SkPixmap pixmap;
160
161 if (!raster_image->peekPixels(&pixmap)) {
162 FML_LOG(ERROR) << "Could not copy pixels from the raster image.";
163 return nullptr;
164 }
165
166 // The color types already match. No need to swizzle. Return early.
167 if (pixmap.colorType() == color_type) {
168 return SkData::MakeWithCopy(pixmap.addr(), pixmap.computeByteSize());
169 }
170
171 // Perform swizzle if the type doesnt match the specification.
172 auto surface = SkSurface::MakeRaster(
173 SkImageInfo::Make(raster_image->width(), raster_image->height(),
174 color_type, kPremul_SkAlphaType, nullptr));
175
176 if (!surface) {
177 FML_LOG(ERROR) << "Could not setup the surface for swizzle.";
178 return nullptr;
179 }
180
181 surface->writePixels(pixmap, 0, 0);
182
183 if (!surface->peekPixels(&pixmap)) {
184 FML_LOG(ERROR) << "Pixel address is not available.";
185 return nullptr;
186 }
187
188 return SkData::MakeWithCopy(pixmap.addr(), pixmap.computeByteSize());
189}
190
191sk_sp<SkData> EncodeImage(sk_sp<SkImage> raster_image, ImageByteFormat format) {
192 TRACE_EVENT0("flutter", __FUNCTION__);
193
194 if (!raster_image) {
195 return nullptr;
196 }
197
198 switch (format) {
199 case kPNG: {
200 auto png_image =
201 raster_image->encodeToData(SkEncodedImageFormat::kPNG, 0);
202
203 if (png_image == nullptr) {
204 FML_LOG(ERROR) << "Could not convert raster image to PNG.";
205 return nullptr;
206 };
207 return png_image;
208 } break;
209 case kRawRGBA: {
210 return CopyImageByteData(raster_image, kRGBA_8888_SkColorType);
211 } break;
212 case kRawUnmodified: {
213 return CopyImageByteData(raster_image, raster_image->colorType());
214 } break;
215 }
216
217 FML_LOG(ERROR) << "Unknown error encoding image.";
218 return nullptr;
219}
220
221void EncodeImageAndInvokeDataCallback(
222 sk_sp<SkImage> image,
223 std::unique_ptr<DartPersistentValue> callback,
224 ImageByteFormat format,
225 fml::RefPtr<fml::TaskRunner> ui_task_runner,
226 fml::RefPtr<fml::TaskRunner> raster_task_runner,
227 fml::RefPtr<fml::TaskRunner> io_task_runner,
228 GrDirectContext* resource_context,
229 fml::WeakPtr<SnapshotDelegate> snapshot_delegate) {
230 auto callback_task = fml::MakeCopyable(
231 [callback = std::move(callback)](sk_sp<SkData> encoded) mutable {
232 InvokeDataCallback(std::move(callback), std::move(encoded));
233 });
234
235 auto encode_task = [callback_task = std::move(callback_task), format,
236 ui_task_runner](sk_sp<SkImage> raster_image) {
237 sk_sp<SkData> encoded = EncodeImage(std::move(raster_image), format);
238 ui_task_runner->PostTask([callback_task = std::move(callback_task),
239 encoded = std::move(encoded)]() mutable {
240 callback_task(std::move(encoded));
241 });
242 };
243
244 ConvertImageToRaster(std::move(image), encode_task, raster_task_runner,
245 io_task_runner, resource_context, snapshot_delegate);
246}
247
248} // namespace
249
250Dart_Handle EncodeImage(CanvasImage* canvas_image,
251 int format,
252 Dart_Handle callback_handle) {
253 if (!canvas_image) {
254 return ToDart("encode called with non-genuine Image.");
255 }
256
257 if (!Dart_IsClosure(callback_handle)) {
258 return ToDart("Callback must be a function.");
259 }
260
261 ImageByteFormat image_format = static_cast<ImageByteFormat>(format);
262
263 auto callback = std::make_unique<DartPersistentValue>(
264 tonic::DartState::Current(), callback_handle);
265
266 const auto& task_runners = UIDartState::Current()->GetTaskRunners();
267
268 task_runners.GetIOTaskRunner()->PostTask(fml::MakeCopyable(
269 [callback = std::move(callback), image = canvas_image->image(),
270 image_format, ui_task_runner = task_runners.GetUITaskRunner(),
271 raster_task_runner = task_runners.GetRasterTaskRunner(),
272 io_task_runner = task_runners.GetIOTaskRunner(),
273 io_manager = UIDartState::Current()->GetIOManager(),
274 snapshot_delegate =
275 UIDartState::Current()->GetSnapshotDelegate()]() mutable {
276 EncodeImageAndInvokeDataCallback(
277 std::move(image), std::move(callback), image_format,
278 std::move(ui_task_runner), std::move(raster_task_runner),
279 std::move(io_task_runner), io_manager->GetResourceContext().get(),
280 std::move(snapshot_delegate));
281 }));
282
283 return Dart_Null();
284}
285
286} // namespace flutter
287