1 | // Copyright 2013 The Flutter Authors. All rights reserved. |
2 | // Use of this source code is governed by a BSD-style license that can be |
3 | // found in the LICENSE file. |
4 | |
5 | #ifndef SHELL_COMMON_RASTERIZER_H_ |
6 | #define SHELL_COMMON_RASTERIZER_H_ |
7 | |
8 | #include <memory> |
9 | #include <optional> |
10 | |
11 | #include "flutter/common/settings.h" |
12 | #include "flutter/common/task_runners.h" |
13 | #include "flutter/flow/compositor_context.h" |
14 | #include "flutter/flow/layers/layer_tree.h" |
15 | #include "flutter/flow/surface.h" |
16 | #include "flutter/fml/closure.h" |
17 | #include "flutter/fml/memory/weak_ptr.h" |
18 | #include "flutter/fml/raster_thread_merger.h" |
19 | #include "flutter/fml/synchronization/sync_switch.h" |
20 | #include "flutter/fml/synchronization/waitable_event.h" |
21 | #include "flutter/fml/time/time_delta.h" |
22 | #include "flutter/fml/time/time_point.h" |
23 | #include "flutter/lib/ui/snapshot_delegate.h" |
24 | #include "flutter/shell/common/pipeline.h" |
25 | |
26 | namespace flutter { |
27 | |
28 | //------------------------------------------------------------------------------ |
29 | /// The rasterizer is a component owned by the shell that resides on the GPU |
30 | /// task runner. Each shell owns exactly one instance of a rasterizer. The |
31 | /// rasterizer may only be created, used and collected on the GPU task runner. |
32 | /// |
33 | /// The rasterizer owns the instance of the currently active on-screen render |
34 | /// surface. On this surface, it renders the contents of layer trees submitted |
35 | /// to it by the `Engine` (which lives on the UI task runner). |
36 | /// |
37 | /// The primary components owned by the rasterizer are the compositor context |
38 | /// and the on-screen render surface. The compositor context has all the GPU |
39 | /// state necessary to render frames to the render surface. |
40 | /// |
41 | class Rasterizer final : public SnapshotDelegate { |
42 | public: |
43 | //---------------------------------------------------------------------------- |
44 | /// @brief Used to forward events from the rasterizer to interested |
45 | /// subsystems. Currently, the shell sets itself up as the |
46 | /// rasterizer delegate to listen for frame rasterization events. |
47 | /// It can then forward these events to the engine. |
48 | /// |
49 | /// Like all rasterizer operation, the rasterizer delegate call |
50 | /// are made on the GPU task runner. Any delegate must ensure that |
51 | /// they can handle the threading implications. |
52 | /// |
53 | class Delegate { |
54 | public: |
55 | //-------------------------------------------------------------------------- |
56 | /// @brief Notifies the delegate that a frame has been rendered. The |
57 | /// rasterizer collects profiling information for each part of |
58 | /// the frame workload. This profiling information is made |
59 | /// available to the delegate for forwarding to subsystems |
60 | /// interested in collecting such profiles. Currently, the shell |
61 | /// (the delegate) forwards this to the engine where Dart code |
62 | /// can react to this information. |
63 | /// |
64 | /// @see `FrameTiming` |
65 | /// |
66 | /// @param[in] frame_timing Instrumentation information for each phase of |
67 | /// the frame workload. |
68 | /// |
69 | virtual void OnFrameRasterized(const FrameTiming& frame_timing) = 0; |
70 | |
71 | /// Time limit for a smooth frame. See `Engine::GetDisplayRefreshRate`. |
72 | virtual fml::Milliseconds GetFrameBudget() = 0; |
73 | |
74 | /// Target time for the latest frame. See also `Shell::OnAnimatorBeginFrame` |
75 | /// for when this time gets updated. |
76 | virtual fml::TimePoint GetLatestFrameTargetTime() const = 0; |
77 | |
78 | /// Task runners used by the shell. |
79 | virtual const TaskRunners& GetTaskRunners() const = 0; |
80 | |
81 | /// Accessor for the shell's GPU sync switch, which determines whether GPU |
82 | /// operations are allowed on the current thread. |
83 | /// |
84 | /// For example, on some platforms when the application is backgrounded it |
85 | /// is critical that GPU operations are not processed. |
86 | virtual std::shared_ptr<fml::SyncSwitch> GetIsGpuDisabledSyncSwitch() |
87 | const = 0; |
88 | }; |
89 | |
90 | //---------------------------------------------------------------------------- |
91 | /// @brief Creates a new instance of a rasterizer. Rasterizers may only |
92 | /// be created on the GPU task runner. Rasterizers are currently |
93 | /// only created by the shell (which also sets itself up as the |
94 | /// rasterizer delegate). |
95 | /// |
96 | /// @param[in] delegate The rasterizer delegate. |
97 | /// |
98 | Rasterizer(Delegate& delegate); |
99 | |
100 | //---------------------------------------------------------------------------- |
101 | /// @brief Creates a new instance of a rasterizer. Rasterizers may only |
102 | /// be created on the GPU task runner. Rasterizers are currently |
103 | /// only created by the shell (which also sets itself up as the |
104 | /// rasterizer delegate). |
105 | /// |
106 | /// @param[in] delegate The rasterizer delegate. |
107 | /// @param[in] compositor_context The compositor context used to hold all |
108 | /// the GPU state used by the rasterizer. |
109 | /// |
110 | Rasterizer(Delegate& delegate, |
111 | std::unique_ptr<flutter::CompositorContext> compositor_context); |
112 | |
113 | //---------------------------------------------------------------------------- |
114 | /// @brief Destroys the rasterizer. This must happen on the GPU task |
115 | /// runner. All GPU resources are collected before this call |
116 | /// returns. Any context setup by the embedder to hold these |
117 | /// resources can be immediately collected as well. |
118 | /// |
119 | ~Rasterizer(); |
120 | |
121 | //---------------------------------------------------------------------------- |
122 | /// @brief Rasterizers may be created well before an on-screen surface is |
123 | /// available for rendering. Shells usually create a rasterizer in |
124 | /// their constructors. Once an on-screen surface is available |
125 | /// however, one may be provided to the rasterizer using this |
126 | /// call. No rendering may occur before this call. The surface is |
127 | /// held till the balancing call to `Rasterizer::Teardown` is |
128 | /// made. Calling a setup before tearing down the previous surface |
129 | /// (if this is not the first time the surface has been setup) is |
130 | /// user error. |
131 | /// |
132 | /// @see `Rasterizer::Teardown` |
133 | /// |
134 | /// @param[in] surface The on-screen render surface. |
135 | /// |
136 | void Setup(std::unique_ptr<Surface> surface); |
137 | |
138 | //---------------------------------------------------------------------------- |
139 | /// @brief Releases the previously setup on-screen render surface and |
140 | /// collects associated resources. No more rendering may occur |
141 | /// till the next call to `Rasterizer::Setup` with a new render |
142 | /// surface. Calling a teardown without a setup is user error. |
143 | /// |
144 | void Teardown(); |
145 | |
146 | //---------------------------------------------------------------------------- |
147 | /// @brief Notifies the rasterizer that there is a low memory situation |
148 | /// and it must purge as many unnecessary resources as possible. |
149 | /// Currently, the Skia context associated with onscreen rendering |
150 | /// is told to free GPU resources. |
151 | /// |
152 | void NotifyLowMemoryWarning() const; |
153 | |
154 | //---------------------------------------------------------------------------- |
155 | /// @brief Gets a weak pointer to the rasterizer. The rasterizer may only |
156 | /// be accessed on the GPU task runner. |
157 | /// |
158 | /// @return The weak pointer to the rasterizer. |
159 | /// |
160 | fml::TaskRunnerAffineWeakPtr<Rasterizer> GetWeakPtr() const; |
161 | |
162 | fml::TaskRunnerAffineWeakPtr<SnapshotDelegate> GetSnapshotDelegate() const; |
163 | |
164 | //---------------------------------------------------------------------------- |
165 | /// @brief Sometimes, it may be necessary to render the same frame again |
166 | /// without having to wait for the framework to build a whole new |
167 | /// layer tree describing the same contents. One such case is when |
168 | /// external textures (video or camera streams for example) are |
169 | /// updated in an otherwise static layer tree. To support this use |
170 | /// case, the rasterizer holds onto the last rendered layer tree. |
171 | /// |
172 | /// @bug https://github.com/flutter/flutter/issues/33939 |
173 | /// |
174 | /// @return A pointer to the last layer or `nullptr` if this rasterizer |
175 | /// has never rendered a frame. |
176 | /// |
177 | flutter::LayerTree* GetLastLayerTree(); |
178 | |
179 | //---------------------------------------------------------------------------- |
180 | /// @brief Draws a last layer tree to the render surface. This may seem |
181 | /// entirely redundant at first glance. After all, on surface loss |
182 | /// and re-acquisition, the framework generates a new layer tree. |
183 | /// Otherwise, why render the same contents to the screen again? |
184 | /// This is used as an optimization in cases where there are |
185 | /// external textures (video or camera streams for example) in |
186 | /// referenced in the layer tree. These textures may be updated at |
187 | /// a cadence different from that of the Flutter application. |
188 | /// Flutter can re-render the layer tree with just the updated |
189 | /// textures instead of waiting for the framework to do the work |
190 | /// to generate the layer tree describing the same contents. |
191 | /// |
192 | void DrawLastLayerTree(); |
193 | |
194 | //---------------------------------------------------------------------------- |
195 | /// @brief Gets the registry of external textures currently in use by the |
196 | /// rasterizer. These textures may be updated at a cadence |
197 | /// different from that of the Flutter application. When an |
198 | /// external texture is referenced in the Flutter layer tree, that |
199 | /// texture is composited within the Flutter layer tree. |
200 | /// |
201 | /// @return A pointer to the external texture registry. |
202 | /// |
203 | flutter::TextureRegistry* GetTextureRegistry(); |
204 | |
205 | //---------------------------------------------------------------------------- |
206 | /// @brief Takes the next item from the layer tree pipeline and executes |
207 | /// the raster thread frame workload for that pipeline item to |
208 | /// render a frame on the on-screen surface. |
209 | /// |
210 | /// Why does the draw call take a layer tree pipeline and not the |
211 | /// layer tree directly? |
212 | /// |
213 | /// The pipeline is the way book-keeping of frame workloads |
214 | /// distributed across the multiple threads is managed. The |
215 | /// rasterizer deals with the pipelines directly (instead of layer |
216 | /// trees which is what it actually renders) because the pipeline |
217 | /// consumer's workload must be accounted for within the pipeline |
218 | /// itself. If the rasterizer took the layer tree directly, it |
219 | /// would have to be taken out of the pipeline. That would signal |
220 | /// the end of the frame workload and the pipeline would be ready |
221 | /// for new frames. But the last frame has not been rendered by |
222 | /// the frame yet! On the other hand, the pipeline must own the |
223 | /// layer tree it renders because it keeps a reference to the last |
224 | /// layer tree around till a new frame is rendered. So a simple |
225 | /// reference wont work either. The `Rasterizer::DoDraw` method |
226 | /// actually performs the GPU operations within the layer tree |
227 | /// pipeline. |
228 | /// |
229 | /// @see `Rasterizer::DoDraw` |
230 | /// |
231 | /// @param[in] pipeline The layer tree pipeline to take the next layer tree |
232 | /// to render from. |
233 | /// |
234 | void Draw(fml::RefPtr<Pipeline<flutter::LayerTree>> pipeline); |
235 | |
236 | //---------------------------------------------------------------------------- |
237 | /// @brief The type of the screenshot to obtain of the previously |
238 | /// rendered layer tree. |
239 | /// |
240 | enum class ScreenshotType { |
241 | //-------------------------------------------------------------------------- |
242 | /// A format used to denote a Skia picture. A Skia picture is a serialized |
243 | /// representation of an `SkPicture` that can be used to introspect the |
244 | /// series of commands used to draw that picture. |
245 | /// |
246 | /// Skia pictures are typically stored as files with the .skp extension on |
247 | /// disk. These files may be viewed in an interactive debugger available at |
248 | /// https://debugger.skia.org/ |
249 | /// |
250 | SkiaPicture, |
251 | |
252 | //-------------------------------------------------------------------------- |
253 | /// A format used to denote uncompressed image data. This format |
254 | /// is 32 bits per pixel, 8 bits per component and |
255 | /// denoted by the `kN32_SkColorType ` Skia color type. |
256 | /// |
257 | UncompressedImage, |
258 | |
259 | //-------------------------------------------------------------------------- |
260 | /// A format used to denote compressed image data. The PNG compressed |
261 | /// container is used. |
262 | /// |
263 | CompressedImage, |
264 | }; |
265 | |
266 | //---------------------------------------------------------------------------- |
267 | /// @brief A POD type used to return the screenshot data along with the |
268 | /// size of the frame. |
269 | /// |
270 | struct Screenshot { |
271 | //-------------------------------------------------------------------------- |
272 | /// The data used to describe the screenshot. The data format depends on the |
273 | /// type of screenshot taken and any further encoding done to the same. |
274 | /// |
275 | /// @see `ScreenshotType` |
276 | /// |
277 | sk_sp<SkData> data; |
278 | |
279 | //-------------------------------------------------------------------------- |
280 | /// The size of the screenshot in texels. |
281 | /// |
282 | SkISize frame_size = SkISize::MakeEmpty(); |
283 | |
284 | //-------------------------------------------------------------------------- |
285 | /// @brief Creates an empty screenshot |
286 | /// |
287 | Screenshot(); |
288 | |
289 | //-------------------------------------------------------------------------- |
290 | /// @brief Creates a screenshot with the specified data and size. |
291 | /// |
292 | /// @param[in] p_data The screenshot data |
293 | /// @param[in] p_size The screenshot size. |
294 | /// |
295 | Screenshot(sk_sp<SkData> p_data, SkISize p_size); |
296 | |
297 | //-------------------------------------------------------------------------- |
298 | /// @brief The copy constructor for a screenshot. |
299 | /// |
300 | /// @param[in] other The screenshot to copy from. |
301 | /// |
302 | Screenshot(const Screenshot& other); |
303 | |
304 | //-------------------------------------------------------------------------- |
305 | /// @brief Destroys the screenshot object and releases underlying data. |
306 | /// |
307 | ~Screenshot(); |
308 | }; |
309 | |
310 | //---------------------------------------------------------------------------- |
311 | /// @brief Screenshots the last layer tree to one of the supported |
312 | /// screenshot types and optionally Base 64 encodes that data for |
313 | /// easier transmission and packaging (usually over the service |
314 | /// protocol for instrumentation tools running on the host). |
315 | /// |
316 | /// @param[in] type The type of the screenshot to gather. |
317 | /// @param[in] base64_encode Whether Base 64 encoding must be applied to the |
318 | /// data after a screenshot has been captured. |
319 | /// |
320 | /// @return A non-empty screenshot if one could be captured. A screenshot |
321 | /// capture may fail if there were no layer trees previously |
322 | /// rendered by this rasterizer, or, due to an unspecified |
323 | /// internal error. Internal error will be logged to the console. |
324 | /// |
325 | Screenshot ScreenshotLastLayerTree(ScreenshotType type, bool base64_encode); |
326 | |
327 | //---------------------------------------------------------------------------- |
328 | /// @brief Sets a callback that will be executed when the next layer tree |
329 | /// in rendered to the on-screen surface. This is used by |
330 | /// embedders to listen for one time operations like listening for |
331 | /// when the first frame is rendered so that they may hide splash |
332 | /// screens. |
333 | /// |
334 | /// The callback is only executed once and dropped on the GPU |
335 | /// thread when executed (lambda captures must be able to deal |
336 | /// with the threading repercussions of this behavior). |
337 | /// |
338 | /// @param[in] callback The callback to execute when the next layer tree is |
339 | /// rendered on-screen. |
340 | /// |
341 | void SetNextFrameCallback(const fml::closure& callback); |
342 | |
343 | //---------------------------------------------------------------------------- |
344 | /// @brief Returns a pointer to the compositor context used by this |
345 | /// rasterizer. This pointer will never be `nullptr`. |
346 | /// |
347 | /// @return The compositor context used by this rasterizer. |
348 | /// |
349 | flutter::CompositorContext* compositor_context() { |
350 | return compositor_context_.get(); |
351 | } |
352 | |
353 | //---------------------------------------------------------------------------- |
354 | /// @brief Skia has no notion of time. To work around the performance |
355 | /// implications of this, it may cache GPU resources to reference |
356 | /// them from one frame to the next. Using this call, embedders |
357 | /// may set the maximum bytes cached by Skia in its caches |
358 | /// dedicated to on-screen rendering. |
359 | /// |
360 | /// @attention This cache setting will be invalidated when the surface is |
361 | /// torn down via `Rasterizer::Teardown`. This call must be made |
362 | /// again with new limits after surface re-acquisition. |
363 | /// |
364 | /// @attention This cache does not describe the entirety of GPU resources |
365 | /// that may be cached. The `RasterCache` also holds very large |
366 | /// GPU resources. |
367 | /// |
368 | /// @see `RasterCache` |
369 | /// |
370 | /// @param[in] max_bytes The maximum byte size of resource that may be |
371 | /// cached for GPU rendering. |
372 | /// @param[in] from_user Whether this request was from user code, e.g. via |
373 | /// the flutter/skia message channel, in which case |
374 | /// it should not be overridden by the platform. |
375 | /// |
376 | void SetResourceCacheMaxBytes(size_t max_bytes, bool from_user); |
377 | |
378 | //---------------------------------------------------------------------------- |
379 | /// @brief The current value of Skia's resource cache size, if a surface |
380 | /// is present. |
381 | /// |
382 | /// @attention This cache does not describe the entirety of GPU resources |
383 | /// that may be cached. The `RasterCache` also holds very large |
384 | /// GPU resources. |
385 | /// |
386 | /// @see `RasterCache` |
387 | /// |
388 | /// @return The size of Skia's resource cache, if available. |
389 | /// |
390 | std::optional<size_t> GetResourceCacheMaxBytes() const; |
391 | |
392 | private: |
393 | Delegate& delegate_; |
394 | std::unique_ptr<Surface> surface_; |
395 | std::unique_ptr<flutter::CompositorContext> compositor_context_; |
396 | // This is the last successfully rasterized layer tree. |
397 | std::unique_ptr<flutter::LayerTree> last_layer_tree_; |
398 | // Set when we need attempt to rasterize the layer tree again. This layer_tree |
399 | // has not successfully rasterized. This can happen due to the change in the |
400 | // thread configuration. This will be inserted to the front of the pipeline. |
401 | std::unique_ptr<flutter::LayerTree> resubmitted_layer_tree_; |
402 | fml::closure next_frame_callback_; |
403 | bool user_override_resource_cache_bytes_; |
404 | std::optional<size_t> max_cache_bytes_; |
405 | fml::TaskRunnerAffineWeakPtrFactory<Rasterizer> weak_factory_; |
406 | fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger_; |
407 | |
408 | // |SnapshotDelegate| |
409 | sk_sp<SkImage> MakeRasterSnapshot(sk_sp<SkPicture> picture, |
410 | SkISize picture_size) override; |
411 | |
412 | // |SnapshotDelegate| |
413 | sk_sp<SkImage> ConvertToRasterImage(sk_sp<SkImage> image) override; |
414 | |
415 | sk_sp<SkData> ScreenshotLayerTreeAsImage( |
416 | flutter::LayerTree* tree, |
417 | flutter::CompositorContext& compositor_context, |
418 | GrDirectContext* surface_context, |
419 | bool compressed); |
420 | |
421 | sk_sp<SkImage> DoMakeRasterSnapshot( |
422 | SkISize size, |
423 | std::function<void(SkCanvas*)> draw_callback); |
424 | |
425 | RasterStatus DoDraw(std::unique_ptr<flutter::LayerTree> layer_tree); |
426 | |
427 | RasterStatus DrawToSurface(flutter::LayerTree& layer_tree); |
428 | |
429 | void FireNextFrameCallbackIfPresent(); |
430 | |
431 | FML_DISALLOW_COPY_AND_ASSIGN(Rasterizer); |
432 | }; |
433 | |
434 | } // namespace flutter |
435 | |
436 | #endif // SHELL_COMMON_RASTERIZER_H_ |
437 | |