| Flutter iOS Embedder |

FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterMetalLayer.h"
6
7 #include <IOSurface/IOSurfaceObjC.h>
8 #include <Metal/Metal.h>
9 #include <UIKit/UIKit.h>
10
11 #include "flutter/fml/logging.h"
12 #import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
13
14 FLUTTER_ASSERT_ARC
15
16 @interface DisplayLinkManager : NSObject
17 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
18 + (double)displayRefreshRate;
19 @end
20
21 @class FlutterTexture;
22 @class FlutterDrawable;
23
24 extern CFTimeInterval display_link_target;
25
26 @interface FlutterMetalLayer () {
27 id<MTLDevice> _preferredDevice;
28 CGSize _drawableSize;
29
30 NSUInteger _nextDrawableId;
31
32 NSMutableSet<FlutterTexture*>* _availableTextures;
33 NSUInteger _totalTextures;
34
35 FlutterTexture* _front;
36
37 // There must be a CADisplayLink scheduled *on main thread* otherwise
38 // core animation only updates layers 60 times a second.
39 CADisplayLink* _displayLink;
40 NSUInteger _displayLinkPauseCountdown;
41
42 // Used to track whether the content was set during this display link.
43 // When unlocking phone the layer (main thread) display link and raster thread
44 // display link get out of sync for several seconds. Even worse, layer display
45 // link does not seem to reflect actual vsync. Forcing the layer link
46 // to max rate (instead range) temporarily seems to fix the issue.
47 BOOL _didSetContentsDuringThisDisplayLinkPeriod;
48
49 // Whether layer displayLink is forced to max rate.
50 BOOL _displayLinkForcedMaxRate;
51 }
52
53 - (void)presentTexture:(FlutterTexture*)texture;
54 - (void)returnTexture:(FlutterTexture*)texture;
55
56 @end
57
58 @interface FlutterTexture : NSObject {
59 id<MTLTexture> _texture;
60 IOSurface* _surface;
61 CFTimeInterval _presentedTime;
62 }
63
64 @property(readonly, nonatomic) id<MTLTexture> texture;
65 @property(readonly, nonatomic) IOSurface* surface;
66 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
67 @property(readwrite, atomic) BOOL waitingForCompletion;
68
69 @end
70
71 @implementation FlutterTexture
72
73 @synthesize texture = _texture;
74 @synthesize surface = _surface;
75 @synthesize presentedTime = _presentedTime;
76 @synthesize waitingForCompletion;
77
78 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
79 if (self = [super init]) {
80 _texture = texture;
81 _surface = surface;
82 }
83 return self;
84 }
85
86 @end
87
88 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
89 FlutterTexture* _texture;
90 __weak FlutterMetalLayer* _layer;
91 NSUInteger _drawableId;
92 BOOL _presented;
93 }
94
95 - (instancetype)initWithTexture:(FlutterTexture*)texture
96 layer:(FlutterMetalLayer*)layer
97 drawableId:(NSUInteger)drawableId;
98
99 @end
100
101 @implementation FlutterDrawable
102
103 - (instancetype)initWithTexture:(FlutterTexture*)texture
104 layer:(FlutterMetalLayer*)layer
105 drawableId:(NSUInteger)drawableId {
106 if (self = [super init]) {
107 _texture = texture;
108 _layer = layer;
109 _drawableId = drawableId;
110 }
111 return self;
112 }
113
114 - (id<MTLTexture>)texture {
115 return self->_texture.texture;
116 }
117
118 #pragma clang diagnostic push
119 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
120 - (CAMetalLayer*)layer {
121 return (id)self->_layer;
122 }
123 #pragma clang diagnostic pop
124
125 - (NSUInteger)drawableID {
126 return self->_drawableId;
127 }
128
129 - (CFTimeInterval)presentedTime {
130 return 0;
131 }
132
133 - (void)present {
134 [_layer presentTexture:self->_texture];
135 self->_presented = YES;
136 }
137
138 - (void)dealloc {
139 if (!_presented) {
140 [_layer returnTexture:self->_texture];
141 }
142 }
143
144 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
145 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement addPresentedHandler:";
146 }
147
148 - (void)presentAtTime:(CFTimeInterval)presentationTime {
149 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAtTime:";
150 }
151
152 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
153 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:";
154 }
155
156 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
157 FlutterTexture* texture = _texture;
158 texture.waitingForCompletion = YES;
159 [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
160 texture.waitingForCompletion = NO;
161 }];
162 }
163
164 @end
165
166 @implementation FlutterMetalLayer
167
168 @synthesize preferredDevice = _preferredDevice;
169 @synthesize device = _device;
170 @synthesize pixelFormat = _pixelFormat;
171 @synthesize framebufferOnly = _framebufferOnly;
172 @synthesize colorspace = _colorspace;
173 @synthesize wantsExtendedDynamicRangeContent = _wantsExtendedDynamicRangeContent;
174
175 - (instancetype)init {
176 if (self = [super init]) {
177 _preferredDevice = MTLCreateSystemDefaultDevice();
178 self.device = self.preferredDevice;
179 self.pixelFormat = MTLPixelFormatBGRA8Unorm;
180 _availableTextures = [[NSMutableSet alloc] init];
181
182 _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(onDisplayLink:)];
183 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
184 [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
185 [[NSNotificationCenter defaultCenter] addObserver:self
186 selector:@selector(didEnterBackground:)
187 name:UIApplicationDidEnterBackgroundNotification
188 object:nil];
189 }
190 return self;
191 }
192
193 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
194 // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
195 // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
196 // has it's own displaylink scheduled on main thread, which is used to trigger core animation
197 // frame allowing for 120hz updates.
198 if (!DisplayLinkManager.maxRefreshRateEnabledOnIPhone) {
199 return;
200 }
201 double maxFrameRate = fmax(refreshRate, 60);
202 double minFrameRate = fmax(maxFrameRate / 2, 60);
203 if (@available(iOS 15.0, *)) {
204 _displayLink.preferredFrameRateRange =
205 CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
206 } else {
207 _displayLink.preferredFramesPerSecond = maxFrameRate;
208 }
209 }
210
211 - (void)onDisplayLink:(CADisplayLink*)link {
212 _didSetContentsDuringThisDisplayLinkPeriod = NO;
213 // Do not pause immediately, this seems to prevent 120hz while touching.
214 if (_displayLinkPauseCountdown == 3) {
215 _displayLink.paused = YES;
216 if (_displayLinkForcedMaxRate) {
217 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:NO];
218 _displayLinkForcedMaxRate = NO;
219 }
220 } else {
221 ++_displayLinkPauseCountdown;
222 }
223 }
224
225 - (BOOL)isKindOfClass:(Class)aClass {
226 #pragma clang diagnostic push
227 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
228 // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
229 if ([aClass isEqual:[CAMetalLayer class]]) {
230 return YES;
231 }
232 #pragma clang diagnostic pop
233 return [super isKindOfClass:aClass];
234 }
235
236 - (void)setDrawableSize:(CGSize)drawableSize {
237 [_availableTextures removeAllObjects];
238 _front = nil;
239 _totalTextures = 0;
240 _drawableSize = drawableSize;
241 }
242
243 - (void)didEnterBackground:(id)notification {
244 [_availableTextures removeAllObjects];
245 _totalTextures = _front != nil ? 1 : 0;
246 _displayLink.paused = YES;
247 }
248
249 - (CGSize)drawableSize {
250 return _drawableSize;
251 }
252
253 - (IOSurface*)createIOSurface {
254 unsigned pixelFormat;
255 unsigned bytesPerElement;
256 if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
257 pixelFormat = kCVPixelFormatType_64RGBAHalf;
258 bytesPerElement = 8;
259 } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
260 pixelFormat = kCVPixelFormatType_32BGRA;
261 bytesPerElement = 4;
262 } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
263 pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
264 bytesPerElement = 8;
265 } else {
266 FML_LOG(ERROR) << "Unsupported pixel format: " << self.pixelFormat;
267 return nil;
268 }
269 size_t bytesPerRow =
270 IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
271 size_t totalBytes =
272 IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
273 NSDictionary* options = @{
274 (id)kIOSurfaceWidth : @(_drawableSize.width),
275 (id)kIOSurfaceHeight : @(_drawableSize.height),
276 (id)kIOSurfacePixelFormat : @(pixelFormat),
277 (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
278 (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
279 (id)kIOSurfaceAllocSize : @(totalBytes),
280 };
281
282 IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
283 if (res == nil) {
284 FML_LOG(ERROR) << "Failed to create IOSurface with options "
285 << options.debugDescription.UTF8String;
286 return nil;
287 }
288
289 if (self.colorspace != nil) {
290 CFStringRef name = CGColorSpaceGetName(self.colorspace);
291 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), name);
292 } else {
293 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), kCGColorSpaceSRGB);
294 }
295 return (__bridge_transfer IOSurface*)res;
296 }
297
298 - (FlutterTexture*)nextTexture {
299 CFTimeInterval start = CACurrentMediaTime();
300 while (true) {
301 FlutterTexture* texture = [self tryNextTexture];
302 if (texture != nil) {
303 return texture;
304 }
305 CFTimeInterval elapsed = CACurrentMediaTime() - start;
306 if (elapsed > 1.0) {
307 NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
308 return nil;
309 }
310 }
311 }
312
313 - (FlutterTexture*)tryNextTexture {
314 @synchronized(self) {
315 if (_front != nil && _front.waitingForCompletion) {
316 return nil;
317 }
318 if (_totalTextures < 3) {
319 ++_totalTextures;
320 IOSurface* surface = [self createIOSurface];
321 if (surface == nil) {
322 return nil;
323 }
324 MTLTextureDescriptor* textureDescriptor =
325 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
326 width:_drawableSize.width
327 height:_drawableSize.height
328 mipmapped:NO];
329
330 if (_framebufferOnly) {
331 textureDescriptor.usage = MTLTextureUsageRenderTarget;
332 } else {
333 textureDescriptor.usage =
334 MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
335 }
336 id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
337 iosurface:(__bridge IOSurfaceRef)surface
338 plane:0];
339 FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
340 surface:surface];
341 return flutterTexture;
342 } else {
343 // Prefer surface that is not in use and has been presented the longest
344 // time ago.
345 // When isInUse is false, the surface is definitely not used by the compositor.
346 // When isInUse is true, the surface may be used by the compositor.
347 // When both surfaces are in use, the one presented earlier will be returned.
348 // The assumption here is that the compositor is already aware of the
349 // newer texture and is unlikely to read from the older one, even though it
350 // has not decreased the use count yet (there seems to be certain latency).
351 FlutterTexture* res = nil;
352 for (FlutterTexture* texture in _availableTextures) {
353 if (res == nil) {
354 res = texture;
355 } else if (res.surface.isInUse && !texture.surface.isInUse) {
356 // prefer texture that is not in use.
357 res = texture;
358 } else if (res.surface.isInUse == texture.surface.isInUse &&
359 texture.presentedTime < res.presentedTime) {
360 // prefer texture with older presented time.
361 res = texture;
362 }
363 }
364 if (res != nil) {
365 [_availableTextures removeObject:res];
366 }
367 return res;
368 }
369 }
370 }
371
372 - (id<CAMetalDrawable>)nextDrawable {
373 FlutterTexture* texture = [self nextTexture];
374 if (texture == nil) {
375 return nil;
376 }
377 FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
378 layer:self
379 drawableId:_nextDrawableId++];
380 return drawable;
381 }
382
383 - (void)presentOnMainThread:(FlutterTexture*)texture {
384 // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
385 // Might also be placebo
386 [self setNeedsDisplay];
387
388 [CATransaction begin];
389 [CATransaction setDisableActions:YES];
390 self.contents = texture.surface;
391 [CATransaction commit];
392 _displayLink.paused = NO;
393 _displayLinkPauseCountdown = 0;
394 if (!_didSetContentsDuringThisDisplayLinkPeriod) {
395 _didSetContentsDuringThisDisplayLinkPeriod = YES;
396 } else if (!_displayLinkForcedMaxRate) {
397 _displayLinkForcedMaxRate = YES;
398 [self setMaxRefreshRate:DisplayLinkManager.displayRefreshRate forceMax:YES];
399 }
400 }
401
402 - (void)presentTexture:(FlutterTexture*)texture {
403 @synchronized(self) {
404 if (_front != nil) {
405 [_availableTextures addObject:_front];
406 }
407 _front = texture;
408 texture.presentedTime = CACurrentMediaTime();
409 if ([NSThread isMainThread]) {
410 [self presentOnMainThread:texture];
411 } else {
412 // Core animation layers can only be updated on main thread.
413 dispatch_async(dispatch_get_main_queue(), ^{
414 [self presentOnMainThread:texture];
415 });
416 }
417 }
418 }
419
420 - (void)returnTexture:(FlutterTexture*)texture {
421 @synchronized(self) {
422 [_availableTextures addObject:texture];
423 }
424 }
425
426 + (BOOL)enabled {
427 static BOOL enabled = NO;
428 static BOOL didCheckInfoPlist = NO;
429 if (!didCheckInfoPlist) {
430 didCheckInfoPlist = YES;
431 NSNumber* use_flutter_metal_layer =
432 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
433 if (use_flutter_metal_layer != nil && [use_flutter_metal_layer boolValue]) {
434 enabled = YES;
435 FML_LOG(WARNING) << "Using FlutterMetalLayer. This is an experimental feature.";
436 }
437 }
438 return enabled;
439 }
440
441 @end
FlutterMetalLayer::wantsExtendedDynamicRangeContent
BOOL wantsExtendedDynamicRangeContent
Definition: FlutterMetalLayer.h:21
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:426
DisplayLinkManager::maxRefreshRateEnabledOnIPhone
BOOL maxRefreshRateEnabledOnIPhone
Definition: FlutterMetalLayer.mm:17
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:89
FlutterTexture::_surface
IOSurface * _surface
Definition: FlutterMetalLayer.mm:60
FlutterDrawable
Definition: FlutterMetalLayer.mm:88
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:27
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:91
DisplayLinkManager
Definition: FlutterMetalLayer.mm:16
DisplayLinkManager::displayRefreshRate
double displayRefreshRate
The display refresh rate used for reporting purposes. The engine does not care about this for frame s...
Definition: vsync_waiter_ios.h:32
FlutterTexture::waitingForCompletion
BOOL waitingForCompletion
Definition: FlutterMetalLayer.mm:67
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:47
FlutterMacros.h
FlutterMetalLayer::framebufferOnly
BOOL framebufferOnly
Definition: FlutterMetalLayer.h:17
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:33
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:35
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:20
FlutterTexture::_presentedTime
CFTimeInterval _presentedTime
Definition: FlutterMetalLayer.mm:61
FlutterMetalDrawable-p
Definition: FlutterMetalLayer.h:33
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:28
_displayLink
CADisplayLink * _displayLink
Definition: vsync_waiter_ios.mm:64
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:40
FlutterMetalLayer::preferredDevice
id< MTLDevice > preferredDevice
Definition: FlutterMetalLayer.h:15
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:32
FlutterTexture
Definition: FlutterMetalLayer.mm:58
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:30
FlutterTexture::_texture
id< MTLTexture > _texture
Definition: FlutterMetalLayer.mm:59
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:66
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:39
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:65
FlutterMetalLayer::device
id< MTLDevice > device
Definition: FlutterMetalLayer.h:14
FlutterMetalLayer.h
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:92
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:372
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:16
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:50
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:18
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:90
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:64
FlutterMetalLayer
Definition: FlutterMetalLayer.h:12