| Flutter iOS Embedder |

FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterMetalLayer.h"
6
7 #include <CoreMedia/CoreMedia.h>
8 #include <IOSurface/IOSurfaceObjC.h>
9 #include <Metal/Metal.h>
10 #include <UIKit/UIKit.h>
11
12 #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h"
13 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
14
15 FLUTTER_ASSERT_ARC
16
17 @interface DisplayLinkManager : NSObject
18 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
19 + (double)displayRefreshRate;
20 @end
21
22 @class FlutterTexture;
23 @class FlutterDrawable;
24
25 extern CFTimeInterval display_link_target;
26
27 @interface FlutterMetalLayer () {
28 id<MTLDevice> _preferredDevice;
29 CGSize _drawableSize;
30
31 NSUInteger _nextDrawableId;
32
33 NSMutableSet<FlutterTexture*>* _availableTextures;
34 NSUInteger _totalTextures;
35
36 FlutterTexture* _front;
37
38 // There must be a CADisplayLink scheduled *on main thread* otherwise
39 // core animation only updates layers 60 times a second.
40 CADisplayLink* _displayLink;
41 NSUInteger _displayLinkPauseCountdown;
42
43 // Used to track whether the content was set during this display link.
44 // When unlocking phone the layer (main thread) display link and raster thread
45 // display link get out of sync for several seconds. Even worse, layer display
46 // link does not seem to reflect actual vsync. Forcing the layer link
47 // to max rate (instead range) temporarily seems to fix the issue.
48 BOOL _didSetContentsDuringThisDisplayLinkPeriod;
49
50 // Whether layer displayLink is forced to max rate.
51 BOOL _displayLinkForcedMaxRate;
52 }
53
54 - (void)onDisplayLink:(CADisplayLink*)link;
55 - (void)presentTexture:(FlutterTexture*)texture;
56 - (void)returnTexture:(FlutterTexture*)texture;
57
58 @end
59
60 @interface FlutterTexture : NSObject
61
62 @property(readonly, nonatomic) id<MTLTexture> texture;
63 @property(readonly, nonatomic) IOSurface* surface;
64 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
65 @property(readwrite, atomic) BOOL waitingForCompletion;
66
67 @end
68
69 @implementation FlutterTexture
70
71 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
72 if (self = [super init]) {
73 _texture = texture;
74 _surface = surface;
75 }
76 return self;
77 }
78
79 @end
80
81 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
82 FlutterTexture* _texture;
83 __weak FlutterMetalLayer* _layer;
84 NSUInteger _drawableId;
85 BOOL _presented;
86 }
87
88 - (instancetype)initWithTexture:(FlutterTexture*)texture
89 layer:(FlutterMetalLayer*)layer
90 drawableId:(NSUInteger)drawableId;
91
92 @end
93
94 @implementation FlutterDrawable
95
96 - (instancetype)initWithTexture:(FlutterTexture*)texture
97 layer:(FlutterMetalLayer*)layer
98 drawableId:(NSUInteger)drawableId {
99 if (self = [super init]) {
100 _texture = texture;
101 _layer = layer;
102 _drawableId = drawableId;
103 }
104 return self;
105 }
106
107 - (id<MTLTexture>)texture {
108 return self->_texture.texture;
109 }
110
111 #pragma clang diagnostic push
112 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
113 - (CAMetalLayer*)layer {
114 return (id)self->_layer;
115 }
116 #pragma clang diagnostic pop
117
118 - (NSUInteger)drawableID {
119 return self->_drawableId;
120 }
121
122 - (CFTimeInterval)presentedTime {
123 return 0;
124 }
125
126 - (void)present {
127 [_layer presentTexture:self->_texture];
128 self->_presented = YES;
129 }
130
131 - (void)dealloc {
132 if (!_presented) {
133 [_layer returnTexture:self->_texture];
134 }
135 }
136
137 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
138 [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement addPresentedHandler:"];
139 }
140
141 - (void)presentAtTime:(CFTimeInterval)presentationTime {
142 [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement presentAtTime:"];
143 }
144
145 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
146 [FlutterLogger
147 logWarning:@"FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:"];
148 }
149
150 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
151 FlutterTexture* texture = _texture;
152 texture.waitingForCompletion = YES;
153 [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
154 texture.waitingForCompletion = NO;
155 }];
156 }
157
158 @end
159
160 @interface FlutterMetalLayerDisplayLinkProxy : NSObject {
161 __weak FlutterMetalLayer* _layer;
162 }
163
164 @end
165
166 @implementation FlutterMetalLayerDisplayLinkProxy
167 - (instancetype)initWithLayer:(FlutterMetalLayer*)layer {
168 if (self = [super init]) {
169 _layer = layer;
170 }
171 return self;
172 }
173
174 - (void)onDisplayLink:(CADisplayLink*)link {
175 [_layer onDisplayLink:link];
176 }
177
178 @end
179
180 @implementation FlutterMetalLayer
181
182 - (instancetype)init {
183 if (self = [super init]) {
184 _preferredDevice = MTLCreateSystemDefaultDevice();
185 self.device = self.preferredDevice;
186 self.pixelFormat = MTLPixelFormatBGRA8Unorm;
187 _availableTextures = [[NSMutableSet alloc] init];
188
189 FlutterMetalLayerDisplayLinkProxy* proxy =
190 [[FlutterMetalLayerDisplayLinkProxy alloc] initWithLayer:self];
191 _displayLink = [CADisplayLink displayLinkWithTarget:proxy selector:@selector(onDisplayLink:)];
192 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
193 [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
194 [[NSNotificationCenter defaultCenter] addObserver:self
195 selector:@selector(didEnterBackground:)
196 name:UIApplicationDidEnterBackgroundNotification
197 object:nil];
198 }
199 return self;
200 }
201
202 - (void)dealloc {
203 [_displayLink invalidate];
204 [[NSNotificationCenter defaultCenter] removeObserver:self];
205 }
206
207 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
208 // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
209 // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
210 // has it's own displaylink scheduled on main thread, which is used to trigger core animation
211 // frame allowing for 120hz updates.
212 if (!DisplayLinkManager.maxRefreshRateEnabledOnIPhone) {
213 return;
214 }
215 double maxFrameRate = fmax(refreshRate, 60);
216 double minFrameRate = fmax(maxFrameRate / 2, 60);
217 if (@available(iOS 15.0, *)) {
218 _displayLink.preferredFrameRateRange =
219 CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
220 } else {
221 _displayLink.preferredFramesPerSecond = maxFrameRate;
222 }
223 }
224
225 - (void)onDisplayLink:(CADisplayLink*)link {
226 _didSetContentsDuringThisDisplayLinkPeriod = NO;
227 // Do not pause immediately, this seems to prevent 120hz while touching.
228 if (_displayLinkPauseCountdown == 3) {
229 _displayLink.paused = YES;
230 if (_displayLinkForcedMaxRate) {
231 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
232 _displayLinkForcedMaxRate = NO;
233 }
234 } else {
235 ++_displayLinkPauseCountdown;
236 }
237 }
238
239 - (BOOL)isKindOfClass:(Class)aClass {
240 #pragma clang diagnostic push
241 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
242 // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
243 if ([aClass isEqual:[CAMetalLayer class]]) {
244 return YES;
245 }
246 #pragma clang diagnostic pop
247 return [super isKindOfClass:aClass];
248 }
249
250 - (void)setDrawableSize:(CGSize)drawableSize {
251 [_availableTextures removeAllObjects];
252 _front = nil;
253 _totalTextures = 0;
254 _drawableSize = drawableSize;
255 }
256
257 - (void)didEnterBackground:(id)notification {
258 [_availableTextures removeAllObjects];
259 _totalTextures = _front != nil ? 1 : 0;
260 _displayLink.paused = YES;
261 }
262
263 - (CGSize)drawableSize {
264 return _drawableSize;
265 }
266
267 - (IOSurface*)createIOSurface {
268 unsigned pixelFormat;
269 unsigned bytesPerElement;
270 if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
271 pixelFormat = kCVPixelFormatType_64RGBAHalf;
272 bytesPerElement = 8;
273 } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
274 pixelFormat = kCVPixelFormatType_32BGRA;
275 bytesPerElement = 4;
276 } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
277 pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
278 bytesPerElement = 8;
279 } else {
280 NSString* errorMessage =
281 [NSString stringWithFormat:@"Unsupported pixel format: %lu", self.pixelFormat];
282 [FlutterLogger logError:errorMessage];
283 return nil;
284 }
285 size_t bytesPerRow =
286 IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
287 size_t totalBytes =
288 IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
289 NSDictionary* options = @{
290 (id)kIOSurfaceWidth : @(_drawableSize.width),
291 (id)kIOSurfaceHeight : @(_drawableSize.height),
292 (id)kIOSurfacePixelFormat : @(pixelFormat),
293 (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
294 (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
295 (id)kIOSurfaceAllocSize : @(totalBytes),
296 };
297
298 IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
299 if (res == nil) {
300 NSString* errorMessage = [NSString
301 stringWithFormat:@"Failed to create IOSurface with options %@", options.debugDescription];
302 [FlutterLogger logError:errorMessage];
303 return nil;
304 }
305
306 if (self.colorspace != nil) {
307 CFStringRef name = CGColorSpaceGetName(self.colorspace);
308 IOSurfaceSetValue(res, kIOSurfaceColorSpace, name);
309 } else {
310 IOSurfaceSetValue(res, kIOSurfaceColorSpace, kCGColorSpaceSRGB);
311 }
312 return (__bridge_transfer IOSurface*)res;
313 }
314
315 - (FlutterTexture*)nextTexture {
316 CFTimeInterval start = CACurrentMediaTime();
317 while (true) {
318 FlutterTexture* texture = [self tryNextTexture];
319 if (texture != nil) {
320 return texture;
321 }
322 CFTimeInterval elapsed = CACurrentMediaTime() - start;
323 if (elapsed > 1.0) {
324 NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
325 return nil;
326 }
327 }
328 }
329
330 - (FlutterTexture*)tryNextTexture {
331 @synchronized(self) {
332 if (_front != nil && _front.waitingForCompletion) {
333 return nil;
334 }
335 if (_totalTextures < 3) {
336 ++_totalTextures;
337 IOSurface* surface = [self createIOSurface];
338 if (surface == nil) {
339 return nil;
340 }
341 MTLTextureDescriptor* textureDescriptor =
342 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
343 width:_drawableSize.width
344 height:_drawableSize.height
345 mipmapped:NO];
346
347 if (_framebufferOnly) {
348 textureDescriptor.usage = MTLTextureUsageRenderTarget;
349 } else {
350 textureDescriptor.usage =
351 MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
352 }
353 id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
354 iosurface:(__bridge IOSurfaceRef)surface
355 plane:0];
356 FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
357 surface:surface];
358 return flutterTexture;
359 } else {
360 // Prefer surface that is not in use and has been presented the longest
361 // time ago.
362 // When isInUse is false, the surface is definitely not used by the compositor.
363 // When isInUse is true, the surface may be used by the compositor.
364 // When both surfaces are in use, the one presented earlier will be returned.
365 // The assumption here is that the compositor is already aware of the
366 // newer texture and is unlikely to read from the older one, even though it
367 // has not decreased the use count yet (there seems to be certain latency).
368 FlutterTexture* res = nil;
369 for (FlutterTexture* texture in _availableTextures) {
370 if (res == nil) {
371 res = texture;
372 } else if (res.surface.isInUse && !texture.surface.isInUse) {
373 // prefer texture that is not in use.
374 res = texture;
375 } else if (res.surface.isInUse == texture.surface.isInUse &&
376 texture.presentedTime < res.presentedTime) {
377 // prefer texture with older presented time.
378 res = texture;
379 }
380 }
381 if (res != nil) {
382 [_availableTextures removeObject:res];
383 }
384 return res;
385 }
386 }
387 }
388
389 - (id<CAMetalDrawable>)nextDrawable {
390 FlutterTexture* texture = [self nextTexture];
391 if (texture == nil) {
392 return nil;
393 }
394 FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
395 layer:self
396 drawableId:_nextDrawableId++];
397 return drawable;
398 }
399
400 - (void)presentOnMainThread:(FlutterTexture*)texture {
401 // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
402 // Might also be placebo
403 [self setNeedsDisplay];
404
405 [CATransaction begin];
406 [CATransaction setDisableActions:YES];
407 self.contents = texture.surface;
408 [CATransaction commit];
409 _displayLink.paused = NO;
410 _displayLinkPauseCountdown = 0;
411 if (!_didSetContentsDuringThisDisplayLinkPeriod) {
412 _didSetContentsDuringThisDisplayLinkPeriod = YES;
413 } else if (!_displayLinkForcedMaxRate) {
414 _displayLinkForcedMaxRate = YES;
415 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:YES];
416 }
417 }
418
419 - (void)presentTexture:(FlutterTexture*)texture {
420 @synchronized(self) {
421 if (_front != nil) {
422 [_availableTextures addObject:_front];
423 }
424 _front = texture;
425 texture.presentedTime = CACurrentMediaTime();
426 if ([NSThread isMainThread]) {
427 [self presentOnMainThread:texture];
428 } else {
429 // Core animation layers can only be updated on main thread.
430 dispatch_async(dispatch_get_main_queue(), ^{
431 [self presentOnMainThread:texture];
432 });
433 }
434 }
435 }
436
437 - (void)returnTexture:(FlutterTexture*)texture {
438 @synchronized(self) {
439 [_availableTextures addObject:texture];
440 }
441 }
442
443 + (BOOL)enabled {
444 static BOOL enabled = YES;
445 static BOOL didCheckInfoPlist = NO;
446 if (!didCheckInfoPlist) {
447 didCheckInfoPlist = YES;
448 NSNumber* use_flutter_metal_layer =
449 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
450 if (use_flutter_metal_layer != nil && ![use_flutter_metal_layer boolValue]) {
451 enabled = NO;
452 }
453 }
454 return enabled;
455 }
456
457 @end
FlutterMacros.h
FlutterMetalLayer.h
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:28
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:34
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:33
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:48
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:40
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:51
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:36
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:29
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:41
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:31
DisplayLinkManager
Definition: FlutterMetalLayer.mm:18
DisplayLinkManager::maxRefreshRateEnabledOnIPhone
BOOL maxRefreshRateEnabledOnIPhone
Whether the max refresh rate on iPhone ProMotion devices are enabled. This reflects the value of CADi...
Definition: FlutterMetalLayer.mm:18
DisplayLinkManager::displayRefreshRate
double displayRefreshRate
The display refresh rate used for reporting purposes. The engine does not care about this for frame s...
Definition: vsync_waiter_ios.h:43
FlutterDrawable
Definition: FlutterMetalLayer.mm:81
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:82
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:84
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:83
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:85
FlutterMetalLayerDisplayLinkProxy
Definition: FlutterMetalLayer.mm:160
FlutterMetalLayerDisplayLinkProxy::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:161
FlutterMetalLayer
Definition: FlutterMetalLayer.h:16
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:443
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:25
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:389
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:21
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:23
FlutterTexture
Definition: FlutterMetalLayer.mm:61
FlutterTexture::waitingForCompletion
BOOL waitingForCompletion
Definition: FlutterMetalLayer.mm:65
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:63
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:62
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:64
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
FlutterMetalDrawable-p
Definition: FlutterMetalLayer.h:37
_displayLink
CADisplayLink * _displayLink
Definition: vsync_waiter_ios.mm:66