| Flutter iOS Embedder |

FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterMetalLayer.h"
6
7 #include <IOSurface/IOSurfaceObjC.h>
8 #include <Metal/Metal.h>
9 #include <UIKit/UIKit.h>
10
11 #include "flutter/fml/logging.h"
12 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
13
14 FLUTTER_ASSERT_ARC
15
16 @interface DisplayLinkManager : NSObject
17 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
18 + (double)displayRefreshRate;
19 @end
20
21 @class FlutterTexture;
22 @class FlutterDrawable;
23
24 extern CFTimeInterval display_link_target;
25
26 @interface FlutterMetalLayer () {
27 id<MTLDevice> _preferredDevice;
28 CGSize _drawableSize;
29
30 NSUInteger _nextDrawableId;
31
32 NSMutableSet<FlutterTexture*>* _availableTextures;
33 NSUInteger _totalTextures;
34
35 FlutterTexture* _front;
36
37 // There must be a CADisplayLink scheduled *on main thread* otherwise
38 // core animation only updates layers 60 times a second.
39 CADisplayLink* _displayLink;
40 NSUInteger _displayLinkPauseCountdown;
41
42 // Used to track whether the content was set during this display link.
43 // When unlocking phone the layer (main thread) display link and raster thread
44 // display link get out of sync for several seconds. Even worse, layer display
45 // link does not seem to reflect actual vsync. Forcing the layer link
46 // to max rate (instead range) temporarily seems to fix the issue.
47 BOOL _didSetContentsDuringThisDisplayLinkPeriod;
48
49 // Whether layer displayLink is forced to max rate.
50 BOOL _displayLinkForcedMaxRate;
51 }
52
53 - (void)onDisplayLink:(CADisplayLink*)link;
54 - (void)presentTexture:(FlutterTexture*)texture;
55 - (void)returnTexture:(FlutterTexture*)texture;
56
57 @end
58
59 @interface FlutterTexture : NSObject {
60 id<MTLTexture> _texture;
61 IOSurface* _surface;
62 CFTimeInterval _presentedTime;
63 }
64
65 @property(readonly, nonatomic) id<MTLTexture> texture;
66 @property(readonly, nonatomic) IOSurface* surface;
67 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
68 @property(readwrite, atomic) BOOL waitingForCompletion;
69
70 @end
71
72 @implementation FlutterTexture
73
74 @synthesize texture = _texture;
75 @synthesize surface = _surface;
76 @synthesize presentedTime = _presentedTime;
77 @synthesize waitingForCompletion;
78
79 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
80 if (self = [super init]) {
81 _texture = texture;
82 _surface = surface;
83 }
84 return self;
85 }
86
87 @end
88
89 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
90 FlutterTexture* _texture;
91 __weak FlutterMetalLayer* _layer;
92 NSUInteger _drawableId;
93 BOOL _presented;
94 }
95
96 - (instancetype)initWithTexture:(FlutterTexture*)texture
97 layer:(FlutterMetalLayer*)layer
98 drawableId:(NSUInteger)drawableId;
99
100 @end
101
102 @implementation FlutterDrawable
103
104 - (instancetype)initWithTexture:(FlutterTexture*)texture
105 layer:(FlutterMetalLayer*)layer
106 drawableId:(NSUInteger)drawableId {
107 if (self = [super init]) {
108 _texture = texture;
109 _layer = layer;
110 _drawableId = drawableId;
111 }
112 return self;
113 }
114
115 - (id<MTLTexture>)texture {
116 return self->_texture.texture;
117 }
118
119 #pragma clang diagnostic push
120 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
121 - (CAMetalLayer*)layer {
122 return (id)self->_layer;
123 }
124 #pragma clang diagnostic pop
125
126 - (NSUInteger)drawableID {
127 return self->_drawableId;
128 }
129
130 - (CFTimeInterval)presentedTime {
131 return 0;
132 }
133
134 - (void)present {
135 [_layer presentTexture:self->_texture];
136 self->_presented = YES;
137 }
138
139 - (void)dealloc {
140 if (!_presented) {
141 [_layer returnTexture:self->_texture];
142 }
143 }
144
145 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
146 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement addPresentedHandler:";
147 }
148
149 - (void)presentAtTime:(CFTimeInterval)presentationTime {
150 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAtTime:";
151 }
152
153 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
154 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:";
155 }
156
157 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
158 FlutterTexture* texture = _texture;
159 texture.waitingForCompletion = YES;
160 [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
161 texture.waitingForCompletion = NO;
162 }];
163 }
164
165 @end
166
167 @interface FlutterMetalLayerDisplayLinkProxy : NSObject {
168 __weak FlutterMetalLayer* _layer;
169 }
170
171 @end
172
173 @implementation FlutterMetalLayerDisplayLinkProxy
174 - (instancetype)initWithLayer:(FlutterMetalLayer*)layer {
175 if (self = [super init]) {
176 _layer = layer;
177 }
178 return self;
179 }
180
181 - (void)onDisplayLink:(CADisplayLink*)link {
182 [_layer onDisplayLink:link];
183 }
184
185 @end
186
187 @implementation FlutterMetalLayer
188
189 @synthesize preferredDevice = _preferredDevice;
190 @synthesize device = _device;
191 @synthesize pixelFormat = _pixelFormat;
192 @synthesize framebufferOnly = _framebufferOnly;
193 @synthesize colorspace = _colorspace;
194 @synthesize wantsExtendedDynamicRangeContent = _wantsExtendedDynamicRangeContent;
195
196 - (instancetype)init {
197 if (self = [super init]) {
198 _preferredDevice = MTLCreateSystemDefaultDevice();
199 self.device = self.preferredDevice;
200 self.pixelFormat = MTLPixelFormatBGRA8Unorm;
201 _availableTextures = [[NSMutableSet alloc] init];
202
203 FlutterMetalLayerDisplayLinkProxy* proxy =
204 [[FlutterMetalLayerDisplayLinkProxy alloc] initWithLayer:self];
205 _displayLink = [CADisplayLink displayLinkWithTarget:proxy selector:@selector(onDisplayLink:)];
206 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
207 [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
208 [[NSNotificationCenter defaultCenter] addObserver:self
209 selector:@selector(didEnterBackground:)
210 name:UIApplicationDidEnterBackgroundNotification
211 object:nil];
212 }
213 return self;
214 }
215
216 - (void)dealloc {
217 [_displayLink invalidate];
218 [[NSNotificationCenter defaultCenter] removeObserver:self];
219 }
220
221 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
222 // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
223 // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
224 // has it's own displaylink scheduled on main thread, which is used to trigger core animation
225 // frame allowing for 120hz updates.
226 if (!DisplayLinkManager.maxRefreshRateEnabledOnIPhone) {
227 return;
228 }
229 double maxFrameRate = fmax(refreshRate, 60);
230 double minFrameRate = fmax(maxFrameRate / 2, 60);
231 if (@available(iOS 15.0, *)) {
232 _displayLink.preferredFrameRateRange =
233 CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
234 } else {
235 _displayLink.preferredFramesPerSecond = maxFrameRate;
236 }
237 }
238
239 - (void)onDisplayLink:(CADisplayLink*)link {
240 _didSetContentsDuringThisDisplayLinkPeriod = NO;
241 // Do not pause immediately, this seems to prevent 120hz while touching.
242 if (_displayLinkPauseCountdown == 3) {
243 _displayLink.paused = YES;
244 if (_displayLinkForcedMaxRate) {
245 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
246 _displayLinkForcedMaxRate = NO;
247 }
248 } else {
249 ++_displayLinkPauseCountdown;
250 }
251 }
252
253 - (BOOL)isKindOfClass:(Class)aClass {
254 #pragma clang diagnostic push
255 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
256 // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
257 if ([aClass isEqual:[CAMetalLayer class]]) {
258 return YES;
259 }
260 #pragma clang diagnostic pop
261 return [super isKindOfClass:aClass];
262 }
263
264 - (void)setDrawableSize:(CGSize)drawableSize {
265 [_availableTextures removeAllObjects];
266 _front = nil;
267 _totalTextures = 0;
268 _drawableSize = drawableSize;
269 }
270
271 - (void)didEnterBackground:(id)notification {
272 [_availableTextures removeAllObjects];
273 _totalTextures = _front != nil ? 1 : 0;
274 _displayLink.paused = YES;
275 }
276
277 - (CGSize)drawableSize {
278 return _drawableSize;
279 }
280
281 - (IOSurface*)createIOSurface {
282 unsigned pixelFormat;
283 unsigned bytesPerElement;
284 if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
285 pixelFormat = kCVPixelFormatType_64RGBAHalf;
286 bytesPerElement = 8;
287 } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
288 pixelFormat = kCVPixelFormatType_32BGRA;
289 bytesPerElement = 4;
290 } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
291 pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
292 bytesPerElement = 8;
293 } else {
294 FML_LOG(ERROR) << "Unsupported pixel format: " << self.pixelFormat;
295 return nil;
296 }
297 size_t bytesPerRow =
298 IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
299 size_t totalBytes =
300 IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
301 NSDictionary* options = @{
302 (id)kIOSurfaceWidth : @(_drawableSize.width),
303 (id)kIOSurfaceHeight : @(_drawableSize.height),
304 (id)kIOSurfacePixelFormat : @(pixelFormat),
305 (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
306 (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
307 (id)kIOSurfaceAllocSize : @(totalBytes),
308 };
309
310 IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
311 if (res == nil) {
312 FML_LOG(ERROR) << "Failed to create IOSurface with options "
313 << options.debugDescription.UTF8String;
314 return nil;
315 }
316
317 if (self.colorspace != nil) {
318 CFStringRef name = CGColorSpaceGetName(self.colorspace);
319 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), name);
320 } else {
321 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), kCGColorSpaceSRGB);
322 }
323 return (__bridge_transfer IOSurface*)res;
324 }
325
326 - (FlutterTexture*)nextTexture {
327 CFTimeInterval start = CACurrentMediaTime();
328 while (true) {
329 FlutterTexture* texture = [self tryNextTexture];
330 if (texture != nil) {
331 return texture;
332 }
333 CFTimeInterval elapsed = CACurrentMediaTime() - start;
334 if (elapsed > 1.0) {
335 NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
336 return nil;
337 }
338 }
339 }
340
341 - (FlutterTexture*)tryNextTexture {
342 @synchronized(self) {
343 if (_front != nil && _front.waitingForCompletion) {
344 return nil;
345 }
346 if (_totalTextures < 3) {
347 ++_totalTextures;
348 IOSurface* surface = [self createIOSurface];
349 if (surface == nil) {
350 return nil;
351 }
352 MTLTextureDescriptor* textureDescriptor =
353 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
354 width:_drawableSize.width
355 height:_drawableSize.height
356 mipmapped:NO];
357
358 if (_framebufferOnly) {
359 textureDescriptor.usage = MTLTextureUsageRenderTarget;
360 } else {
361 textureDescriptor.usage =
362 MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
363 }
364 id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
365 iosurface:(__bridge IOSurfaceRef)surface
366 plane:0];
367 FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
368 surface:surface];
369 return flutterTexture;
370 } else {
371 // Prefer surface that is not in use and has been presented the longest
372 // time ago.
373 // When isInUse is false, the surface is definitely not used by the compositor.
374 // When isInUse is true, the surface may be used by the compositor.
375 // When both surfaces are in use, the one presented earlier will be returned.
376 // The assumption here is that the compositor is already aware of the
377 // newer texture and is unlikely to read from the older one, even though it
378 // has not decreased the use count yet (there seems to be certain latency).
379 FlutterTexture* res = nil;
380 for (FlutterTexture* texture in _availableTextures) {
381 if (res == nil) {
382 res = texture;
383 } else if (res.surface.isInUse && !texture.surface.isInUse) {
384 // prefer texture that is not in use.
385 res = texture;
386 } else if (res.surface.isInUse == texture.surface.isInUse &&
387 texture.presentedTime < res.presentedTime) {
388 // prefer texture with older presented time.
389 res = texture;
390 }
391 }
392 if (res != nil) {
393 [_availableTextures removeObject:res];
394 }
395 return res;
396 }
397 }
398 }
399
400 - (id<CAMetalDrawable>)nextDrawable {
401 FlutterTexture* texture = [self nextTexture];
402 if (texture == nil) {
403 return nil;
404 }
405 FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
406 layer:self
407 drawableId:_nextDrawableId++];
408 return drawable;
409 }
410
411 - (void)presentOnMainThread:(FlutterTexture*)texture {
412 // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
413 // Might also be placebo
414 [self setNeedsDisplay];
415
416 [CATransaction begin];
417 [CATransaction setDisableActions:YES];
418 self.contents = texture.surface;
419 [CATransaction commit];
420 _displayLink.paused = NO;
421 _displayLinkPauseCountdown = 0;
422 if (!_didSetContentsDuringThisDisplayLinkPeriod) {
423 _didSetContentsDuringThisDisplayLinkPeriod = YES;
424 } else if (!_displayLinkForcedMaxRate) {
425 _displayLinkForcedMaxRate = YES;
426 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:YES];
427 }
428 }
429
430 - (void)presentTexture:(FlutterTexture*)texture {
431 @synchronized(self) {
432 if (_front != nil) {
433 [_availableTextures addObject:_front];
434 }
435 _front = texture;
436 texture.presentedTime = CACurrentMediaTime();
437 if ([NSThread isMainThread]) {
438 [self presentOnMainThread:texture];
439 } else {
440 // Core animation layers can only be updated on main thread.
441 dispatch_async(dispatch_get_main_queue(), ^{
442 [self presentOnMainThread:texture];
443 });
444 }
445 }
446 }
447
448 - (void)returnTexture:(FlutterTexture*)texture {
449 @synchronized(self) {
450 [_availableTextures addObject:texture];
451 }
452 }
453
454 + (BOOL)enabled {
455 static BOOL enabled = YES;
456 static BOOL didCheckInfoPlist = NO;
457 if (!didCheckInfoPlist) {
458 didCheckInfoPlist = YES;
459 NSNumber* use_flutter_metal_layer =
460 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
461 if (use_flutter_metal_layer != nil && ![use_flutter_metal_layer boolValue]) {
462 enabled = NO;
463 }
464 }
465 return enabled;
466 }
467
468 @end
FlutterMetalLayer::wantsExtendedDynamicRangeContent
BOOL wantsExtendedDynamicRangeContent
Definition: FlutterMetalLayer.h:21
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:454
DisplayLinkManager::maxRefreshRateEnabledOnIPhone
BOOL maxRefreshRateEnabledOnIPhone
Definition: FlutterMetalLayer.mm:17
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:90
FlutterTexture::_surface
IOSurface * _surface
Definition: FlutterMetalLayer.mm:61
FlutterDrawable
Definition: FlutterMetalLayer.mm:89
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:27
FlutterMetalLayerDisplayLinkProxy
Definition: FlutterMetalLayer.mm:167
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:92
DisplayLinkManager
Definition: FlutterMetalLayer.mm:16
DisplayLinkManager::displayRefreshRate
double displayRefreshRate
The display refresh rate used for reporting purposes. The engine does not care about this for frame s...
Definition: vsync_waiter_ios.h:32
FlutterTexture::waitingForCompletion
BOOL waitingForCompletion
Definition: FlutterMetalLayer.mm:68
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:47
FlutterMacros.h
FlutterMetalLayer::framebufferOnly
BOOL framebufferOnly
Definition: FlutterMetalLayer.h:17
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:33
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:35
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:20
FlutterTexture::_presentedTime
CFTimeInterval _presentedTime
Definition: FlutterMetalLayer.mm:62
FlutterMetalDrawable-p
Definition: FlutterMetalLayer.h:33
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:28
_displayLink
CADisplayLink * _displayLink
Definition: vsync_waiter_ios.mm:64
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:40
FlutterMetalLayer::preferredDevice
id< MTLDevice > preferredDevice
Definition: FlutterMetalLayer.h:15
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:32
FlutterTexture
Definition: FlutterMetalLayer.mm:59
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayerDisplayLinkProxy::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:168
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:30
FlutterTexture::_texture
id< MTLTexture > _texture
Definition: FlutterMetalLayer.mm:60
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:67
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:39
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:66
FlutterMetalLayer::device
id< MTLDevice > device
Definition: FlutterMetalLayer.h:14
FlutterMetalLayer.h
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:93
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:400
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:16
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:50
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:18
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:91
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:65
FlutterMetalLayer
Definition: FlutterMetalLayer.h:12