1// Copyright 2013 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef SHELL_COMMON_RASTERIZER_H_
6#define SHELL_COMMON_RASTERIZER_H_
7
8#include <memory>
9#include <optional>
10
11#include "flutter/common/settings.h"
12#include "flutter/common/task_runners.h"
13#include "flutter/flow/compositor_context.h"
14#include "flutter/flow/layers/layer_tree.h"
15#include "flutter/flow/surface.h"
16#include "flutter/fml/closure.h"
17#include "flutter/fml/memory/weak_ptr.h"
18#include "flutter/fml/raster_thread_merger.h"
19#include "flutter/fml/synchronization/sync_switch.h"
20#include "flutter/fml/synchronization/waitable_event.h"
21#include "flutter/fml/time/time_delta.h"
22#include "flutter/fml/time/time_point.h"
23#include "flutter/lib/ui/snapshot_delegate.h"
24#include "flutter/shell/common/pipeline.h"
25
26namespace flutter {
27
28//------------------------------------------------------------------------------
29/// The rasterizer is a component owned by the shell that resides on the GPU
30/// task runner. Each shell owns exactly one instance of a rasterizer. The
31/// rasterizer may only be created, used and collected on the GPU task runner.
32///
33/// The rasterizer owns the instance of the currently active on-screen render
34/// surface. On this surface, it renders the contents of layer trees submitted
35/// to it by the `Engine` (which lives on the UI task runner).
36///
37/// The primary components owned by the rasterizer are the compositor context
38/// and the on-screen render surface. The compositor context has all the GPU
39/// state necessary to render frames to the render surface.
40///
41class Rasterizer final : public SnapshotDelegate {
42 public:
43 //----------------------------------------------------------------------------
44 /// @brief Used to forward events from the rasterizer to interested
45 /// subsystems. Currently, the shell sets itself up as the
46 /// rasterizer delegate to listen for frame rasterization events.
47 /// It can then forward these events to the engine.
48 ///
49 /// Like all rasterizer operation, the rasterizer delegate call
50 /// are made on the GPU task runner. Any delegate must ensure that
51 /// they can handle the threading implications.
52 ///
53 class Delegate {
54 public:
55 //--------------------------------------------------------------------------
56 /// @brief Notifies the delegate that a frame has been rendered. The
57 /// rasterizer collects profiling information for each part of
58 /// the frame workload. This profiling information is made
59 /// available to the delegate for forwarding to subsystems
60 /// interested in collecting such profiles. Currently, the shell
61 /// (the delegate) forwards this to the engine where Dart code
62 /// can react to this information.
63 ///
64 /// @see `FrameTiming`
65 ///
66 /// @param[in] frame_timing Instrumentation information for each phase of
67 /// the frame workload.
68 ///
69 virtual void OnFrameRasterized(const FrameTiming& frame_timing) = 0;
70
71 /// Time limit for a smooth frame. See `Engine::GetDisplayRefreshRate`.
72 virtual fml::Milliseconds GetFrameBudget() = 0;
73
74 /// Target time for the latest frame. See also `Shell::OnAnimatorBeginFrame`
75 /// for when this time gets updated.
76 virtual fml::TimePoint GetLatestFrameTargetTime() const = 0;
77
78 /// Task runners used by the shell.
79 virtual const TaskRunners& GetTaskRunners() const = 0;
80
81 /// Accessor for the shell's GPU sync switch, which determines whether GPU
82 /// operations are allowed on the current thread.
83 ///
84 /// For example, on some platforms when the application is backgrounded it
85 /// is critical that GPU operations are not processed.
86 virtual std::shared_ptr<fml::SyncSwitch> GetIsGpuDisabledSyncSwitch()
87 const = 0;
88 };
89
90 //----------------------------------------------------------------------------
91 /// @brief Creates a new instance of a rasterizer. Rasterizers may only
92 /// be created on the GPU task runner. Rasterizers are currently
93 /// only created by the shell (which also sets itself up as the
94 /// rasterizer delegate).
95 ///
96 /// @param[in] delegate The rasterizer delegate.
97 ///
98 Rasterizer(Delegate& delegate);
99
100 //----------------------------------------------------------------------------
101 /// @brief Creates a new instance of a rasterizer. Rasterizers may only
102 /// be created on the GPU task runner. Rasterizers are currently
103 /// only created by the shell (which also sets itself up as the
104 /// rasterizer delegate).
105 ///
106 /// @param[in] delegate The rasterizer delegate.
107 /// @param[in] compositor_context The compositor context used to hold all
108 /// the GPU state used by the rasterizer.
109 ///
110 Rasterizer(Delegate& delegate,
111 std::unique_ptr<flutter::CompositorContext> compositor_context);
112
113 //----------------------------------------------------------------------------
114 /// @brief Destroys the rasterizer. This must happen on the GPU task
115 /// runner. All GPU resources are collected before this call
116 /// returns. Any context setup by the embedder to hold these
117 /// resources can be immediately collected as well.
118 ///
119 ~Rasterizer();
120
121 //----------------------------------------------------------------------------
122 /// @brief Rasterizers may be created well before an on-screen surface is
123 /// available for rendering. Shells usually create a rasterizer in
124 /// their constructors. Once an on-screen surface is available
125 /// however, one may be provided to the rasterizer using this
126 /// call. No rendering may occur before this call. The surface is
127 /// held till the balancing call to `Rasterizer::Teardown` is
128 /// made. Calling a setup before tearing down the previous surface
129 /// (if this is not the first time the surface has been setup) is
130 /// user error.
131 ///
132 /// @see `Rasterizer::Teardown`
133 ///
134 /// @param[in] surface The on-screen render surface.
135 ///
136 void Setup(std::unique_ptr<Surface> surface);
137
138 //----------------------------------------------------------------------------
139 /// @brief Releases the previously setup on-screen render surface and
140 /// collects associated resources. No more rendering may occur
141 /// till the next call to `Rasterizer::Setup` with a new render
142 /// surface. Calling a teardown without a setup is user error.
143 ///
144 void Teardown();
145
146 //----------------------------------------------------------------------------
147 /// @brief Notifies the rasterizer that there is a low memory situation
148 /// and it must purge as many unnecessary resources as possible.
149 /// Currently, the Skia context associated with onscreen rendering
150 /// is told to free GPU resources.
151 ///
152 void NotifyLowMemoryWarning() const;
153
154 //----------------------------------------------------------------------------
155 /// @brief Gets a weak pointer to the rasterizer. The rasterizer may only
156 /// be accessed on the GPU task runner.
157 ///
158 /// @return The weak pointer to the rasterizer.
159 ///
160 fml::TaskRunnerAffineWeakPtr<Rasterizer> GetWeakPtr() const;
161
162 fml::TaskRunnerAffineWeakPtr<SnapshotDelegate> GetSnapshotDelegate() const;
163
164 //----------------------------------------------------------------------------
165 /// @brief Sometimes, it may be necessary to render the same frame again
166 /// without having to wait for the framework to build a whole new
167 /// layer tree describing the same contents. One such case is when
168 /// external textures (video or camera streams for example) are
169 /// updated in an otherwise static layer tree. To support this use
170 /// case, the rasterizer holds onto the last rendered layer tree.
171 ///
172 /// @bug https://github.com/flutter/flutter/issues/33939
173 ///
174 /// @return A pointer to the last layer or `nullptr` if this rasterizer
175 /// has never rendered a frame.
176 ///
177 flutter::LayerTree* GetLastLayerTree();
178
179 //----------------------------------------------------------------------------
180 /// @brief Draws a last layer tree to the render surface. This may seem
181 /// entirely redundant at first glance. After all, on surface loss
182 /// and re-acquisition, the framework generates a new layer tree.
183 /// Otherwise, why render the same contents to the screen again?
184 /// This is used as an optimization in cases where there are
185 /// external textures (video or camera streams for example) in
186 /// referenced in the layer tree. These textures may be updated at
187 /// a cadence different from that of the Flutter application.
188 /// Flutter can re-render the layer tree with just the updated
189 /// textures instead of waiting for the framework to do the work
190 /// to generate the layer tree describing the same contents.
191 ///
192 void DrawLastLayerTree();
193
194 //----------------------------------------------------------------------------
195 /// @brief Gets the registry of external textures currently in use by the
196 /// rasterizer. These textures may be updated at a cadence
197 /// different from that of the Flutter application. When an
198 /// external texture is referenced in the Flutter layer tree, that
199 /// texture is composited within the Flutter layer tree.
200 ///
201 /// @return A pointer to the external texture registry.
202 ///
203 flutter::TextureRegistry* GetTextureRegistry();
204
205 //----------------------------------------------------------------------------
206 /// @brief Takes the next item from the layer tree pipeline and executes
207 /// the raster thread frame workload for that pipeline item to
208 /// render a frame on the on-screen surface.
209 ///
210 /// Why does the draw call take a layer tree pipeline and not the
211 /// layer tree directly?
212 ///
213 /// The pipeline is the way book-keeping of frame workloads
214 /// distributed across the multiple threads is managed. The
215 /// rasterizer deals with the pipelines directly (instead of layer
216 /// trees which is what it actually renders) because the pipeline
217 /// consumer's workload must be accounted for within the pipeline
218 /// itself. If the rasterizer took the layer tree directly, it
219 /// would have to be taken out of the pipeline. That would signal
220 /// the end of the frame workload and the pipeline would be ready
221 /// for new frames. But the last frame has not been rendered by
222 /// the frame yet! On the other hand, the pipeline must own the
223 /// layer tree it renders because it keeps a reference to the last
224 /// layer tree around till a new frame is rendered. So a simple
225 /// reference wont work either. The `Rasterizer::DoDraw` method
226 /// actually performs the GPU operations within the layer tree
227 /// pipeline.
228 ///
229 /// @see `Rasterizer::DoDraw`
230 ///
231 /// @param[in] pipeline The layer tree pipeline to take the next layer tree
232 /// to render from.
233 ///
234 void Draw(fml::RefPtr<Pipeline<flutter::LayerTree>> pipeline);
235
236 //----------------------------------------------------------------------------
237 /// @brief The type of the screenshot to obtain of the previously
238 /// rendered layer tree.
239 ///
240 enum class ScreenshotType {
241 //--------------------------------------------------------------------------
242 /// A format used to denote a Skia picture. A Skia picture is a serialized
243 /// representation of an `SkPicture` that can be used to introspect the
244 /// series of commands used to draw that picture.
245 ///
246 /// Skia pictures are typically stored as files with the .skp extension on
247 /// disk. These files may be viewed in an interactive debugger available at
248 /// https://debugger.skia.org/
249 ///
250 SkiaPicture,
251
252 //--------------------------------------------------------------------------
253 /// A format used to denote uncompressed image data. This format
254 /// is 32 bits per pixel, 8 bits per component and
255 /// denoted by the `kN32_SkColorType ` Skia color type.
256 ///
257 UncompressedImage,
258
259 //--------------------------------------------------------------------------
260 /// A format used to denote compressed image data. The PNG compressed
261 /// container is used.
262 ///
263 CompressedImage,
264 };
265
266 //----------------------------------------------------------------------------
267 /// @brief A POD type used to return the screenshot data along with the
268 /// size of the frame.
269 ///
270 struct Screenshot {
271 //--------------------------------------------------------------------------
272 /// The data used to describe the screenshot. The data format depends on the
273 /// type of screenshot taken and any further encoding done to the same.
274 ///
275 /// @see `ScreenshotType`
276 ///
277 sk_sp<SkData> data;
278
279 //--------------------------------------------------------------------------
280 /// The size of the screenshot in texels.
281 ///
282 SkISize frame_size = SkISize::MakeEmpty();
283
284 //--------------------------------------------------------------------------
285 /// @brief Creates an empty screenshot
286 ///
287 Screenshot();
288
289 //--------------------------------------------------------------------------
290 /// @brief Creates a screenshot with the specified data and size.
291 ///
292 /// @param[in] p_data The screenshot data
293 /// @param[in] p_size The screenshot size.
294 ///
295 Screenshot(sk_sp<SkData> p_data, SkISize p_size);
296
297 //--------------------------------------------------------------------------
298 /// @brief The copy constructor for a screenshot.
299 ///
300 /// @param[in] other The screenshot to copy from.
301 ///
302 Screenshot(const Screenshot& other);
303
304 //--------------------------------------------------------------------------
305 /// @brief Destroys the screenshot object and releases underlying data.
306 ///
307 ~Screenshot();
308 };
309
310 //----------------------------------------------------------------------------
311 /// @brief Screenshots the last layer tree to one of the supported
312 /// screenshot types and optionally Base 64 encodes that data for
313 /// easier transmission and packaging (usually over the service
314 /// protocol for instrumentation tools running on the host).
315 ///
316 /// @param[in] type The type of the screenshot to gather.
317 /// @param[in] base64_encode Whether Base 64 encoding must be applied to the
318 /// data after a screenshot has been captured.
319 ///
320 /// @return A non-empty screenshot if one could be captured. A screenshot
321 /// capture may fail if there were no layer trees previously
322 /// rendered by this rasterizer, or, due to an unspecified
323 /// internal error. Internal error will be logged to the console.
324 ///
325 Screenshot ScreenshotLastLayerTree(ScreenshotType type, bool base64_encode);
326
327 //----------------------------------------------------------------------------
328 /// @brief Sets a callback that will be executed when the next layer tree
329 /// in rendered to the on-screen surface. This is used by
330 /// embedders to listen for one time operations like listening for
331 /// when the first frame is rendered so that they may hide splash
332 /// screens.
333 ///
334 /// The callback is only executed once and dropped on the GPU
335 /// thread when executed (lambda captures must be able to deal
336 /// with the threading repercussions of this behavior).
337 ///
338 /// @param[in] callback The callback to execute when the next layer tree is
339 /// rendered on-screen.
340 ///
341 void SetNextFrameCallback(const fml::closure& callback);
342
343 //----------------------------------------------------------------------------
344 /// @brief Returns a pointer to the compositor context used by this
345 /// rasterizer. This pointer will never be `nullptr`.
346 ///
347 /// @return The compositor context used by this rasterizer.
348 ///
349 flutter::CompositorContext* compositor_context() {
350 return compositor_context_.get();
351 }
352
353 //----------------------------------------------------------------------------
354 /// @brief Skia has no notion of time. To work around the performance
355 /// implications of this, it may cache GPU resources to reference
356 /// them from one frame to the next. Using this call, embedders
357 /// may set the maximum bytes cached by Skia in its caches
358 /// dedicated to on-screen rendering.
359 ///
360 /// @attention This cache setting will be invalidated when the surface is
361 /// torn down via `Rasterizer::Teardown`. This call must be made
362 /// again with new limits after surface re-acquisition.
363 ///
364 /// @attention This cache does not describe the entirety of GPU resources
365 /// that may be cached. The `RasterCache` also holds very large
366 /// GPU resources.
367 ///
368 /// @see `RasterCache`
369 ///
370 /// @param[in] max_bytes The maximum byte size of resource that may be
371 /// cached for GPU rendering.
372 /// @param[in] from_user Whether this request was from user code, e.g. via
373 /// the flutter/skia message channel, in which case
374 /// it should not be overridden by the platform.
375 ///
376 void SetResourceCacheMaxBytes(size_t max_bytes, bool from_user);
377
378 //----------------------------------------------------------------------------
379 /// @brief The current value of Skia's resource cache size, if a surface
380 /// is present.
381 ///
382 /// @attention This cache does not describe the entirety of GPU resources
383 /// that may be cached. The `RasterCache` also holds very large
384 /// GPU resources.
385 ///
386 /// @see `RasterCache`
387 ///
388 /// @return The size of Skia's resource cache, if available.
389 ///
390 std::optional<size_t> GetResourceCacheMaxBytes() const;
391
392 private:
393 Delegate& delegate_;
394 std::unique_ptr<Surface> surface_;
395 std::unique_ptr<flutter::CompositorContext> compositor_context_;
396 // This is the last successfully rasterized layer tree.
397 std::unique_ptr<flutter::LayerTree> last_layer_tree_;
398 // Set when we need attempt to rasterize the layer tree again. This layer_tree
399 // has not successfully rasterized. This can happen due to the change in the
400 // thread configuration. This will be inserted to the front of the pipeline.
401 std::unique_ptr<flutter::LayerTree> resubmitted_layer_tree_;
402 fml::closure next_frame_callback_;
403 bool user_override_resource_cache_bytes_;
404 std::optional<size_t> max_cache_bytes_;
405 fml::TaskRunnerAffineWeakPtrFactory<Rasterizer> weak_factory_;
406 fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger_;
407
408 // |SnapshotDelegate|
409 sk_sp<SkImage> MakeRasterSnapshot(sk_sp<SkPicture> picture,
410 SkISize picture_size) override;
411
412 // |SnapshotDelegate|
413 sk_sp<SkImage> ConvertToRasterImage(sk_sp<SkImage> image) override;
414
415 sk_sp<SkData> ScreenshotLayerTreeAsImage(
416 flutter::LayerTree* tree,
417 flutter::CompositorContext& compositor_context,
418 GrDirectContext* surface_context,
419 bool compressed);
420
421 sk_sp<SkImage> DoMakeRasterSnapshot(
422 SkISize size,
423 std::function<void(SkCanvas*)> draw_callback);
424
425 RasterStatus DoDraw(std::unique_ptr<flutter::LayerTree> layer_tree);
426
427 RasterStatus DrawToSurface(flutter::LayerTree& layer_tree);
428
429 void FireNextFrameCallbackIfPresent();
430
431 FML_DISALLOW_COPY_AND_ASSIGN(Rasterizer);
432};
433
434} // namespace flutter
435
436#endif // SHELL_COMMON_RASTERIZER_H_
437