| Flutter iOS Embedder |

FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterMetalLayer.h"
6
7 #include <CoreMedia/CoreMedia.h>
8 #include <IOSurface/IOSurfaceObjC.h>
9 #include <Metal/Metal.h>
10 #include <UIKit/UIKit.h>
11
12 #include "flutter/fml/logging.h"
13 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
14
15 FLUTTER_ASSERT_ARC
16
17 @interface DisplayLinkManager : NSObject
18 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
19 + (double)displayRefreshRate;
20 @end
21
22 @class FlutterTexture;
23 @class FlutterDrawable;
24
25 extern CFTimeInterval display_link_target;
26
27 @interface FlutterMetalLayer () {
28 id<MTLDevice> _preferredDevice;
29 CGSize _drawableSize;
30
31 NSUInteger _nextDrawableId;
32
33 NSMutableSet<FlutterTexture*>* _availableTextures;
34 NSUInteger _totalTextures;
35
36 FlutterTexture* _front;
37
38 // There must be a CADisplayLink scheduled *on main thread* otherwise
39 // core animation only updates layers 60 times a second.
40 CADisplayLink* _displayLink;
41 NSUInteger _displayLinkPauseCountdown;
42
43 // Used to track whether the content was set during this display link.
44 // When unlocking phone the layer (main thread) display link and raster thread
45 // display link get out of sync for several seconds. Even worse, layer display
46 // link does not seem to reflect actual vsync. Forcing the layer link
47 // to max rate (instead range) temporarily seems to fix the issue.
48 BOOL _didSetContentsDuringThisDisplayLinkPeriod;
49
50 // Whether layer displayLink is forced to max rate.
51 BOOL _displayLinkForcedMaxRate;
52 }
53
54 - (void)onDisplayLink:(CADisplayLink*)link;
55 - (void)presentTexture:(FlutterTexture*)texture;
56 - (void)returnTexture:(FlutterTexture*)texture;
57
58 @end
59
60 @interface FlutterTexture : NSObject
61
62 @property(readonly, nonatomic) id<MTLTexture> texture;
63 @property(readonly, nonatomic) IOSurface* surface;
64 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
65 @property(readwrite, atomic) BOOL waitingForCompletion;
66
67 @end
68
69 @implementation FlutterTexture
70
71 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
72 if (self = [super init]) {
73 _texture = texture;
74 _surface = surface;
75 }
76 return self;
77 }
78
79 @end
80
81 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
82 FlutterTexture* _texture;
83 __weak FlutterMetalLayer* _layer;
84 NSUInteger _drawableId;
85 BOOL _presented;
86 }
87
88 - (instancetype)initWithTexture:(FlutterTexture*)texture
89 layer:(FlutterMetalLayer*)layer
90 drawableId:(NSUInteger)drawableId;
91
92 @end
93
94 @implementation FlutterDrawable
95
96 - (instancetype)initWithTexture:(FlutterTexture*)texture
97 layer:(FlutterMetalLayer*)layer
98 drawableId:(NSUInteger)drawableId {
99 if (self = [super init]) {
100 _texture = texture;
101 _layer = layer;
102 _drawableId = drawableId;
103 }
104 return self;
105 }
106
107 - (id<MTLTexture>)texture {
108 return self->_texture.texture;
109 }
110
111 #pragma clang diagnostic push
112 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
113 - (CAMetalLayer*)layer {
114 return (id)self->_layer;
115 }
116 #pragma clang diagnostic pop
117
118 - (NSUInteger)drawableID {
119 return self->_drawableId;
120 }
121
122 - (CFTimeInterval)presentedTime {
123 return 0;
124 }
125
126 - (void)present {
127 [_layer presentTexture:self->_texture];
128 self->_presented = YES;
129 }
130
131 - (void)dealloc {
132 if (!_presented) {
133 [_layer returnTexture:self->_texture];
134 }
135 }
136
137 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
138 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement addPresentedHandler:";
139 }
140
141 - (void)presentAtTime:(CFTimeInterval)presentationTime {
142 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAtTime:";
143 }
144
145 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
146 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:";
147 }
148
149 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
150 FlutterTexture* texture = _texture;
151 texture.waitingForCompletion = YES;
152 [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
153 texture.waitingForCompletion = NO;
154 }];
155 }
156
157 @end
158
159 @interface FlutterMetalLayerDisplayLinkProxy : NSObject {
160 __weak FlutterMetalLayer* _layer;
161 }
162
163 @end
164
165 @implementation FlutterMetalLayerDisplayLinkProxy
166 - (instancetype)initWithLayer:(FlutterMetalLayer*)layer {
167 if (self = [super init]) {
168 _layer = layer;
169 }
170 return self;
171 }
172
173 - (void)onDisplayLink:(CADisplayLink*)link {
174 [_layer onDisplayLink:link];
175 }
176
177 @end
178
179 @implementation FlutterMetalLayer
180
181 - (instancetype)init {
182 if (self = [super init]) {
183 _preferredDevice = MTLCreateSystemDefaultDevice();
184 self.device = self.preferredDevice;
185 self.pixelFormat = MTLPixelFormatBGRA8Unorm;
186 _availableTextures = [[NSMutableSet alloc] init];
187
188 FlutterMetalLayerDisplayLinkProxy* proxy =
189 [[FlutterMetalLayerDisplayLinkProxy alloc] initWithLayer:self];
190 _displayLink = [CADisplayLink displayLinkWithTarget:proxy selector:@selector(onDisplayLink:)];
191 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
192 [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
193 [[NSNotificationCenter defaultCenter] addObserver:self
194 selector:@selector(didEnterBackground:)
195 name:UIApplicationDidEnterBackgroundNotification
196 object:nil];
197 }
198 return self;
199 }
200
201 - (void)dealloc {
202 [_displayLink invalidate];
203 [[NSNotificationCenter defaultCenter] removeObserver:self];
204 }
205
206 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
207 // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
208 // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
209 // has it's own displaylink scheduled on main thread, which is used to trigger core animation
210 // frame allowing for 120hz updates.
211 if (!DisplayLinkManager.maxRefreshRateEnabledOnIPhone) {
212 return;
213 }
214 double maxFrameRate = fmax(refreshRate, 60);
215 double minFrameRate = fmax(maxFrameRate / 2, 60);
216 if (@available(iOS 15.0, *)) {
217 _displayLink.preferredFrameRateRange =
218 CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
219 } else {
220 _displayLink.preferredFramesPerSecond = maxFrameRate;
221 }
222 }
223
224 - (void)onDisplayLink:(CADisplayLink*)link {
225 _didSetContentsDuringThisDisplayLinkPeriod = NO;
226 // Do not pause immediately, this seems to prevent 120hz while touching.
227 if (_displayLinkPauseCountdown == 3) {
228 _displayLink.paused = YES;
229 if (_displayLinkForcedMaxRate) {
230 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
231 _displayLinkForcedMaxRate = NO;
232 }
233 } else {
234 ++_displayLinkPauseCountdown;
235 }
236 }
237
238 - (BOOL)isKindOfClass:(Class)aClass {
239 #pragma clang diagnostic push
240 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
241 // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
242 if ([aClass isEqual:[CAMetalLayer class]]) {
243 return YES;
244 }
245 #pragma clang diagnostic pop
246 return [super isKindOfClass:aClass];
247 }
248
249 - (void)setDrawableSize:(CGSize)drawableSize {
250 [_availableTextures removeAllObjects];
251 _front = nil;
252 _totalTextures = 0;
253 _drawableSize = drawableSize;
254 }
255
256 - (void)didEnterBackground:(id)notification {
257 [_availableTextures removeAllObjects];
258 _totalTextures = _front != nil ? 1 : 0;
259 _displayLink.paused = YES;
260 }
261
262 - (CGSize)drawableSize {
263 return _drawableSize;
264 }
265
266 - (IOSurface*)createIOSurface {
267 unsigned pixelFormat;
268 unsigned bytesPerElement;
269 if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
270 pixelFormat = kCVPixelFormatType_64RGBAHalf;
271 bytesPerElement = 8;
272 } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
273 pixelFormat = kCVPixelFormatType_32BGRA;
274 bytesPerElement = 4;
275 } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
276 pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
277 bytesPerElement = 8;
278 } else {
279 FML_LOG(ERROR) << "Unsupported pixel format: " << self.pixelFormat;
280 return nil;
281 }
282 size_t bytesPerRow =
283 IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
284 size_t totalBytes =
285 IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
286 NSDictionary* options = @{
287 (id)kIOSurfaceWidth : @(_drawableSize.width),
288 (id)kIOSurfaceHeight : @(_drawableSize.height),
289 (id)kIOSurfacePixelFormat : @(pixelFormat),
290 (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
291 (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
292 (id)kIOSurfaceAllocSize : @(totalBytes),
293 };
294
295 IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
296 if (res == nil) {
297 FML_LOG(ERROR) << "Failed to create IOSurface with options "
298 << options.debugDescription.UTF8String;
299 return nil;
300 }
301
302 if (self.colorspace != nil) {
303 CFStringRef name = CGColorSpaceGetName(self.colorspace);
304 IOSurfaceSetValue(res, kIOSurfaceColorSpace, name);
305 } else {
306 IOSurfaceSetValue(res, kIOSurfaceColorSpace, kCGColorSpaceSRGB);
307 }
308 return (__bridge_transfer IOSurface*)res;
309 }
310
311 - (FlutterTexture*)nextTexture {
312 CFTimeInterval start = CACurrentMediaTime();
313 while (true) {
314 FlutterTexture* texture = [self tryNextTexture];
315 if (texture != nil) {
316 return texture;
317 }
318 CFTimeInterval elapsed = CACurrentMediaTime() - start;
319 if (elapsed > 1.0) {
320 NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
321 return nil;
322 }
323 }
324 }
325
326 - (FlutterTexture*)tryNextTexture {
327 @synchronized(self) {
328 if (_front != nil && _front.waitingForCompletion) {
329 return nil;
330 }
331 if (_totalTextures < 3) {
332 ++_totalTextures;
333 IOSurface* surface = [self createIOSurface];
334 if (surface == nil) {
335 return nil;
336 }
337 MTLTextureDescriptor* textureDescriptor =
338 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
339 width:_drawableSize.width
340 height:_drawableSize.height
341 mipmapped:NO];
342
343 if (_framebufferOnly) {
344 textureDescriptor.usage = MTLTextureUsageRenderTarget;
345 } else {
346 textureDescriptor.usage =
347 MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
348 }
349 id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
350 iosurface:(__bridge IOSurfaceRef)surface
351 plane:0];
352 FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
353 surface:surface];
354 return flutterTexture;
355 } else {
356 // Prefer surface that is not in use and has been presented the longest
357 // time ago.
358 // When isInUse is false, the surface is definitely not used by the compositor.
359 // When isInUse is true, the surface may be used by the compositor.
360 // When both surfaces are in use, the one presented earlier will be returned.
361 // The assumption here is that the compositor is already aware of the
362 // newer texture and is unlikely to read from the older one, even though it
363 // has not decreased the use count yet (there seems to be certain latency).
364 FlutterTexture* res = nil;
365 for (FlutterTexture* texture in _availableTextures) {
366 if (res == nil) {
367 res = texture;
368 } else if (res.surface.isInUse && !texture.surface.isInUse) {
369 // prefer texture that is not in use.
370 res = texture;
371 } else if (res.surface.isInUse == texture.surface.isInUse &&
372 texture.presentedTime < res.presentedTime) {
373 // prefer texture with older presented time.
374 res = texture;
375 }
376 }
377 if (res != nil) {
378 [_availableTextures removeObject:res];
379 }
380 return res;
381 }
382 }
383 }
384
385 - (id<CAMetalDrawable>)nextDrawable {
386 FlutterTexture* texture = [self nextTexture];
387 if (texture == nil) {
388 return nil;
389 }
390 FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
391 layer:self
392 drawableId:_nextDrawableId++];
393 return drawable;
394 }
395
396 - (void)presentOnMainThread:(FlutterTexture*)texture {
397 // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
398 // Might also be placebo
399 [self setNeedsDisplay];
400
401 [CATransaction begin];
402 [CATransaction setDisableActions:YES];
403 self.contents = texture.surface;
404 [CATransaction commit];
405 _displayLink.paused = NO;
406 _displayLinkPauseCountdown = 0;
407 if (!_didSetContentsDuringThisDisplayLinkPeriod) {
408 _didSetContentsDuringThisDisplayLinkPeriod = YES;
409 } else if (!_displayLinkForcedMaxRate) {
410 _displayLinkForcedMaxRate = YES;
411 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:YES];
412 }
413 }
414
415 - (void)presentTexture:(FlutterTexture*)texture {
416 @synchronized(self) {
417 if (_front != nil) {
418 [_availableTextures addObject:_front];
419 }
420 _front = texture;
421 texture.presentedTime = CACurrentMediaTime();
422 if ([NSThread isMainThread]) {
423 [self presentOnMainThread:texture];
424 } else {
425 // Core animation layers can only be updated on main thread.
426 dispatch_async(dispatch_get_main_queue(), ^{
427 [self presentOnMainThread:texture];
428 });
429 }
430 }
431 }
432
433 - (void)returnTexture:(FlutterTexture*)texture {
434 @synchronized(self) {
435 [_availableTextures addObject:texture];
436 }
437 }
438
439 + (BOOL)enabled {
440 static BOOL enabled = YES;
441 static BOOL didCheckInfoPlist = NO;
442 if (!didCheckInfoPlist) {
443 didCheckInfoPlist = YES;
444 NSNumber* use_flutter_metal_layer =
445 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
446 if (use_flutter_metal_layer != nil && ![use_flutter_metal_layer boolValue]) {
447 enabled = NO;
448 }
449 }
450 return enabled;
451 }
452
453 @end
FlutterMacros.h
FlutterMetalLayer.h
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:28
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:34
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:33
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:48
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:40
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:51
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:36
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:29
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:41
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:31
DisplayLinkManager
Definition: FlutterMetalLayer.mm:18
DisplayLinkManager::maxRefreshRateEnabledOnIPhone
BOOL maxRefreshRateEnabledOnIPhone
Whether the max refresh rate on iPhone ProMotion devices are enabled. This reflects the value of CADi...
Definition: FlutterMetalLayer.mm:18
DisplayLinkManager::displayRefreshRate
double displayRefreshRate
The display refresh rate used for reporting purposes. The engine does not care about this for frame s...
Definition: vsync_waiter_ios.h:43
FlutterDrawable
Definition: FlutterMetalLayer.mm:81
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:82
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:84
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:83
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:85
FlutterMetalLayerDisplayLinkProxy
Definition: FlutterMetalLayer.mm:159
FlutterMetalLayerDisplayLinkProxy::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:160
FlutterMetalLayer
Definition: FlutterMetalLayer.h:16
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:439
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:25
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:385
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:21
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:23
FlutterTexture
Definition: FlutterMetalLayer.mm:61
FlutterTexture::waitingForCompletion
BOOL waitingForCompletion
Definition: FlutterMetalLayer.mm:65
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:63
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:62
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:64
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
FlutterMetalDrawable-p
Definition: FlutterMetalLayer.h:37
_displayLink
CADisplayLink * _displayLink
Definition: vsync_waiter_ios.mm:66