| Flutter iOS Embedder |

FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterMetalLayer.h"
6
7 #include <CoreMedia/CoreMedia.h>
8 #include <IOSurface/IOSurfaceObjC.h>
9 #include <Metal/Metal.h>
10 #include <UIKit/UIKit.h>
11
12 #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h"
13 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
14
15 FLUTTER_ASSERT_ARC
16
17 @interface DisplayLinkManager : NSObject
18 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
19 + (double)displayRefreshRate;
20 @end
21
22 @class FlutterTexture;
23 @class FlutterDrawable;
24
25 extern CFTimeInterval display_link_target;
26
27 @interface FlutterMetalLayer () {
28 id<MTLDevice> _preferredDevice;
29 CGSize _drawableSize;
30
31 NSUInteger _nextDrawableId;
32
33 // Access to these variables must be synchronized.
34 NSMutableSet<FlutterTexture*>* _availableTextures;
35 NSUInteger _totalTextures;
36 FlutterTexture* _front;
37
38 // There must be a CADisplayLink scheduled *on main thread* otherwise
39 // core animation only updates layers 60 times a second.
40 CADisplayLink* _displayLink;
41 NSUInteger _displayLinkPauseCountdown;
42
43 // Used to track whether the content was set during this display link.
44 // When unlocking phone the layer (main thread) display link and raster thread
45 // display link get out of sync for several seconds. Even worse, layer display
46 // link does not seem to reflect actual vsync. Forcing the layer link
47 // to max rate (instead range) temporarily seems to fix the issue.
48 BOOL _didSetContentsDuringThisDisplayLinkPeriod;
49
50 // Whether layer displayLink is forced to max rate.
51 BOOL _displayLinkForcedMaxRate;
52 }
53
54 - (void)onDisplayLink:(CADisplayLink*)link;
55 - (void)presentTexture:(FlutterTexture*)texture;
56 - (void)returnTexture:(FlutterTexture*)texture;
57
58 @end
59
60 @interface FlutterTexture : NSObject
61
62 @property(readonly, nonatomic) id<MTLTexture> texture;
63 @property(readonly, nonatomic) IOSurface* surface;
64 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
65 @property(readwrite, atomic) BOOL waitingForCompletion;
66
67 @end
68
69 @implementation FlutterTexture
70
71 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
72 if (self = [super init]) {
73 _texture = texture;
74 _surface = surface;
75 }
76 return self;
77 }
78
79 @end
80
81 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
82 FlutterTexture* _texture;
83 __weak FlutterMetalLayer* _layer;
84 NSUInteger _drawableId;
85 BOOL _presented;
86 }
87
88 - (instancetype)initWithTexture:(FlutterTexture*)texture
89 layer:(FlutterMetalLayer*)layer
90 drawableId:(NSUInteger)drawableId;
91
92 @end
93
94 @implementation FlutterDrawable
95
96 - (instancetype)initWithTexture:(FlutterTexture*)texture
97 layer:(FlutterMetalLayer*)layer
98 drawableId:(NSUInteger)drawableId {
99 if (self = [super init]) {
100 _texture = texture;
101 _layer = layer;
102 _drawableId = drawableId;
103 }
104 return self;
105 }
106
107 - (id<MTLTexture>)texture {
108 return self->_texture.texture;
109 }
110
111 #pragma clang diagnostic push
112 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
113 - (CAMetalLayer*)layer {
114 return (id)self->_layer;
115 }
116 #pragma clang diagnostic pop
117
118 - (NSUInteger)drawableID {
119 return self->_drawableId;
120 }
121
122 - (CFTimeInterval)presentedTime {
123 return 0;
124 }
125
126 - (void)present {
127 [_layer presentTexture:self->_texture];
128 self->_presented = YES;
129 }
130
131 - (void)dealloc {
132 if (!_presented) {
133 [_layer returnTexture:self->_texture];
134 }
135 }
136
137 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
138 [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement addPresentedHandler:"];
139 }
140
141 - (void)presentAtTime:(CFTimeInterval)presentationTime {
142 [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement presentAtTime:"];
143 }
144
145 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
146 [FlutterLogger
147 logWarning:@"FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:"];
148 }
149
150 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
151 FlutterTexture* texture = _texture;
152 texture.waitingForCompletion = YES;
153 [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
154 texture.waitingForCompletion = NO;
155 }];
156 }
157
158 @end
159
160 @interface FlutterMetalLayerDisplayLinkProxy : NSObject {
161 __weak FlutterMetalLayer* _layer;
162 }
163
164 @end
165
166 @implementation FlutterMetalLayerDisplayLinkProxy
167 - (instancetype)initWithLayer:(FlutterMetalLayer*)layer {
168 if (self = [super init]) {
169 _layer = layer;
170 }
171 return self;
172 }
173
174 - (void)onDisplayLink:(CADisplayLink*)link {
175 [_layer onDisplayLink:link];
176 }
177
178 @end
179
180 @implementation FlutterMetalLayer
181
182 - (instancetype)init {
183 if (self = [super init]) {
184 _preferredDevice = MTLCreateSystemDefaultDevice();
185 self.device = self.preferredDevice;
186 self.pixelFormat = MTLPixelFormatBGRA8Unorm;
187 _availableTextures = [[NSMutableSet alloc] init];
188
189 FlutterMetalLayerDisplayLinkProxy* proxy =
190 [[FlutterMetalLayerDisplayLinkProxy alloc] initWithLayer:self];
191 _displayLink = [CADisplayLink displayLinkWithTarget:proxy selector:@selector(onDisplayLink:)];
192 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
193 [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
194 [[NSNotificationCenter defaultCenter] addObserver:self
195 selector:@selector(didEnterBackground:)
196 name:UIApplicationDidEnterBackgroundNotification
197 object:nil];
198 }
199 return self;
200 }
201
202 - (void)dealloc {
203 [_displayLink invalidate];
204 [[NSNotificationCenter defaultCenter] removeObserver:self];
205 }
206
207 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
208 // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
209 // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
210 // has it's own displaylink scheduled on main thread, which is used to trigger core animation
211 // frame allowing for 120hz updates.
212 if (!DisplayLinkManager.maxRefreshRateEnabledOnIPhone) {
213 return;
214 }
215 double maxFrameRate = fmax(refreshRate, 60);
216 double minFrameRate = fmax(maxFrameRate / 2, 60);
217 if (@available(iOS 15.0, *)) {
218 _displayLink.preferredFrameRateRange =
219 CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
220 } else {
221 _displayLink.preferredFramesPerSecond = maxFrameRate;
222 }
223 }
224
225 - (void)onDisplayLink:(CADisplayLink*)link {
226 _didSetContentsDuringThisDisplayLinkPeriod = NO;
227 // Do not pause immediately, this seems to prevent 120hz while touching.
228 if (_displayLinkPauseCountdown == 3) {
229 _displayLink.paused = YES;
230 if (_displayLinkForcedMaxRate) {
231 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
232 _displayLinkForcedMaxRate = NO;
233 }
234 } else {
235 ++_displayLinkPauseCountdown;
236 }
237 }
238
239 - (BOOL)isKindOfClass:(Class)aClass {
240 #pragma clang diagnostic push
241 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
242 // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
243 if ([aClass isEqual:[CAMetalLayer class]]) {
244 return YES;
245 }
246 #pragma clang diagnostic pop
247 return [super isKindOfClass:aClass];
248 }
249
250 - (void)setDrawableSize:(CGSize)drawableSize {
251 @synchronized(self) {
252 [_availableTextures removeAllObjects];
253 _front = nil;
254 _totalTextures = 0;
255 _drawableSize = drawableSize;
256 }
257 }
258
259 - (void)didEnterBackground:(id)notification {
260 @synchronized(self) {
261 [_availableTextures removeAllObjects];
262 _totalTextures = _front != nil ? 1 : 0;
263 }
264 _displayLink.paused = YES;
265 }
266
267 - (CGSize)drawableSize {
268 @synchronized(self) {
269 return _drawableSize;
270 }
271 }
272
273 - (IOSurface*)createIOSurface {
274 unsigned pixelFormat;
275 unsigned bytesPerElement;
276 if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
277 pixelFormat = kCVPixelFormatType_64RGBAHalf;
278 bytesPerElement = 8;
279 } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
280 pixelFormat = kCVPixelFormatType_32BGRA;
281 bytesPerElement = 4;
282 } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
283 pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
284 bytesPerElement = 8;
285 } else {
286 NSString* errorMessage =
287 [NSString stringWithFormat:@"Unsupported pixel format: %lu", self.pixelFormat];
288 [FlutterLogger logError:errorMessage];
289 return nil;
290 }
291 size_t bytesPerRow =
292 IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
293 size_t totalBytes =
294 IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
295 NSDictionary* options = @{
296 (id)kIOSurfaceWidth : @(_drawableSize.width),
297 (id)kIOSurfaceHeight : @(_drawableSize.height),
298 (id)kIOSurfacePixelFormat : @(pixelFormat),
299 (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
300 (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
301 (id)kIOSurfaceAllocSize : @(totalBytes),
302 };
303
304 IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
305 if (res == nil) {
306 NSString* errorMessage = [NSString
307 stringWithFormat:@"Failed to create IOSurface with options %@", options.debugDescription];
308 [FlutterLogger logError:errorMessage];
309 return nil;
310 }
311
312 if (self.colorspace != nil) {
313 CFStringRef name = CGColorSpaceGetName(self.colorspace);
314 IOSurfaceSetValue(res, kIOSurfaceColorSpace, name);
315 } else {
316 IOSurfaceSetValue(res, kIOSurfaceColorSpace, kCGColorSpaceSRGB);
317 }
318 return (__bridge_transfer IOSurface*)res;
319 }
320
321 - (FlutterTexture*)nextTexture {
322 CFTimeInterval start = CACurrentMediaTime();
323 while (true) {
324 FlutterTexture* texture = [self tryNextTexture];
325 if (texture != nil) {
326 return texture;
327 }
328 CFTimeInterval elapsed = CACurrentMediaTime() - start;
329 if (elapsed > 1.0) {
330 NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
331 return nil;
332 }
333 }
334 }
335
336 - (FlutterTexture*)tryNextTexture {
337 @synchronized(self) {
338 if (_front != nil && _front.waitingForCompletion) {
339 return nil;
340 }
341 if (_totalTextures < 3) {
342 ++_totalTextures;
343 IOSurface* surface = [self createIOSurface];
344 if (surface == nil) {
345 return nil;
346 }
347 MTLTextureDescriptor* textureDescriptor =
348 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
349 width:_drawableSize.width
350 height:_drawableSize.height
351 mipmapped:NO];
352
353 if (_framebufferOnly) {
354 textureDescriptor.usage = MTLTextureUsageRenderTarget;
355 } else {
356 textureDescriptor.usage =
357 MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
358 }
359 id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
360 iosurface:(__bridge IOSurfaceRef)surface
361 plane:0];
362 FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
363 surface:surface];
364 return flutterTexture;
365 } else {
366 // Prefer surface that is not in use and has been presented the longest
367 // time ago.
368 // When isInUse is false, the surface is definitely not used by the compositor.
369 // When isInUse is true, the surface may be used by the compositor.
370 // When both surfaces are in use, the one presented earlier will be returned.
371 // The assumption here is that the compositor is already aware of the
372 // newer texture and is unlikely to read from the older one, even though it
373 // has not decreased the use count yet (there seems to be certain latency).
374 FlutterTexture* res = nil;
375 for (FlutterTexture* texture in _availableTextures) {
376 if (res == nil) {
377 res = texture;
378 } else if (res.surface.isInUse && !texture.surface.isInUse) {
379 // prefer texture that is not in use.
380 res = texture;
381 } else if (res.surface.isInUse == texture.surface.isInUse &&
382 texture.presentedTime < res.presentedTime) {
383 // prefer texture with older presented time.
384 res = texture;
385 }
386 }
387 if (res != nil) {
388 [_availableTextures removeObject:res];
389 }
390 return res;
391 }
392 }
393 }
394
395 - (id<CAMetalDrawable>)nextDrawable {
396 FlutterTexture* texture = [self nextTexture];
397 if (texture == nil) {
398 return nil;
399 }
400 FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
401 layer:self
402 drawableId:_nextDrawableId++];
403 return drawable;
404 }
405
406 - (void)presentOnMainThread:(FlutterTexture*)texture {
407 if (texture.texture.width != _drawableSize.width ||
408 texture.texture.height != _drawableSize.height) {
409 // This texture was created with an old size, but the view has since been
410 // resized. Do not present this stale frame to avoid distortion. The texture
411 // will be correctly recycled on the next frame.
412 return;
413 }
414
415 // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
416 // Might also be placebo
417 [self setNeedsDisplay];
418
419 [CATransaction begin];
420 [CATransaction setDisableActions:YES];
421 self.contents = texture.surface;
422 [CATransaction commit];
423 _displayLink.paused = NO;
424 _displayLinkPauseCountdown = 0;
425 if (!_didSetContentsDuringThisDisplayLinkPeriod) {
426 _didSetContentsDuringThisDisplayLinkPeriod = YES;
427 } else if (!_displayLinkForcedMaxRate) {
428 _displayLinkForcedMaxRate = YES;
429 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:YES];
430 }
431 }
432
433 - (void)presentTexture:(FlutterTexture*)texture {
434 @synchronized(self) {
435 if (texture.texture.width != _drawableSize.width ||
436 texture.texture.height != _drawableSize.height) {
437 return;
438 }
439 if (_front != nil) {
440 [_availableTextures addObject:_front];
441 }
442 _front = texture;
443 texture.presentedTime = CACurrentMediaTime();
444 if ([NSThread isMainThread]) {
445 [self presentOnMainThread:texture];
446 } else {
447 // Core animation layers can only be updated on main thread.
448 dispatch_async(dispatch_get_main_queue(), ^{
449 [self presentOnMainThread:texture];
450 });
451 }
452 }
453 }
454
455 - (void)returnTexture:(FlutterTexture*)texture {
456 if (texture == nil) {
457 return;
458 }
459 @synchronized(self) {
460 if (texture.texture.width == _drawableSize.width &&
461 texture.texture.height == _drawableSize.height) {
462 [_availableTextures addObject:texture];
463 }
464 }
465 }
466
467 + (BOOL)enabled {
468 static BOOL enabled = YES;
469 static BOOL didCheckInfoPlist = NO;
470 if (!didCheckInfoPlist) {
471 didCheckInfoPlist = YES;
472 NSNumber* use_flutter_metal_layer =
473 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
474 if (use_flutter_metal_layer != nil && ![use_flutter_metal_layer boolValue]) {
475 enabled = NO;
476 }
477 }
478 return enabled;
479 }
480
481 @end
FlutterMacros.h
FlutterMetalLayer.h
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:28
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:35
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:34
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:48
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:40
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:51
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:36
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:29
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:41
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:31
DisplayLinkManager
Definition: FlutterMetalLayer.mm:18
DisplayLinkManager::maxRefreshRateEnabledOnIPhone
BOOL maxRefreshRateEnabledOnIPhone
Whether the max refresh rate on iPhone ProMotion devices are enabled. This reflects the value of CADi...
Definition: FlutterMetalLayer.mm:18
DisplayLinkManager::displayRefreshRate
double displayRefreshRate
The display refresh rate used for reporting purposes. The engine does not care about this for frame s...
Definition: vsync_waiter_ios.h:43
FlutterDrawable
Definition: FlutterMetalLayer.mm:81
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:82
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:84
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:83
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:85
FlutterMetalLayerDisplayLinkProxy
Definition: FlutterMetalLayer.mm:160
FlutterMetalLayerDisplayLinkProxy::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:161
FlutterMetalLayer
Definition: FlutterMetalLayer.h:16
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:467
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:26
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:395
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:22
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:24
FlutterTexture
Definition: FlutterMetalLayer.mm:61
FlutterTexture::waitingForCompletion
BOOL waitingForCompletion
Definition: FlutterMetalLayer.mm:65
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:63
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:62
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:64
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
FlutterMetalDrawable-p
Definition: FlutterMetalLayer.h:38
_displayLink
CADisplayLink * _displayLink
Definition: vsync_waiter_ios.mm:66