| Flutter iOS Embedder |

FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <IOSurface/IOSurfaceObjC.h>
6 #include <Metal/Metal.h>
7 #include <UIKit/UIKit.h>
8
9 #include "flutter/fml/logging.h"
10 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterMetalLayer.h"
11
12 @interface DisplayLinkManager : NSObject
13 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
14 + (double)displayRefreshRate;
15 @end
16
17 @class FlutterTexture;
18 @class FlutterDrawable;
19
20 extern CFTimeInterval display_link_target;
21
22 @interface FlutterMetalLayer () {
23 id<MTLDevice> _preferredDevice;
24 CGSize _drawableSize;
25
26 NSUInteger _nextDrawableId;
27
28 NSMutableSet<FlutterTexture*>* _availableTextures;
29 NSUInteger _totalTextures;
30
31 FlutterTexture* _front;
32
33 // There must be a CADisplayLink scheduled *on main thread* otherwise
34 // core animation only updates layers 60 times a second.
35 CADisplayLink* _displayLink;
36 NSUInteger _displayLinkPauseCountdown;
37
38 // Used to track whether the content was set during this display link.
39 // When unlocking phone the layer (main thread) display link and raster thread
40 // display link get out of sync for several seconds. Even worse, layer display
41 // link does not seem to reflect actual vsync. Forcing the layer link
42 // to max rate (instead range) temporarily seems to fix the issue.
43 BOOL _didSetContentsDuringThisDisplayLinkPeriod;
44
45 // Whether layer displayLink is forced to max rate.
46 BOOL _displayLinkForcedMaxRate;
47 }
48
49 - (void)presentTexture:(FlutterTexture*)texture;
50 - (void)returnTexture:(FlutterTexture*)texture;
51
52 @end
53
54 @interface FlutterTexture : NSObject {
55 id<MTLTexture> _texture;
56 IOSurface* _surface;
57 CFTimeInterval _presentedTime;
58 }
59
60 @property(readonly, nonatomic) id<MTLTexture> texture;
61 @property(readonly, nonatomic) IOSurface* surface;
62 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
63 @property(readwrite, atomic) BOOL waitingForCompletion;
64
65 @end
66
67 @implementation FlutterTexture
68
69 @synthesize texture = _texture;
70 @synthesize surface = _surface;
71 @synthesize presentedTime = _presentedTime;
72 @synthesize waitingForCompletion;
73
74 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
75 if (self = [super init]) {
76 _texture = texture;
77 _surface = surface;
78 }
79 return self;
80 }
81
82 @end
83
84 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
85 FlutterTexture* _texture;
86 __weak FlutterMetalLayer* _layer;
87 NSUInteger _drawableId;
88 BOOL _presented;
89 }
90
91 - (instancetype)initWithTexture:(FlutterTexture*)texture
92 layer:(FlutterMetalLayer*)layer
93 drawableId:(NSUInteger)drawableId;
94
95 @end
96
97 @implementation FlutterDrawable
98
99 - (instancetype)initWithTexture:(FlutterTexture*)texture
100 layer:(FlutterMetalLayer*)layer
101 drawableId:(NSUInteger)drawableId {
102 if (self = [super init]) {
103 _texture = texture;
104 _layer = layer;
105 _drawableId = drawableId;
106 }
107 return self;
108 }
109
110 - (id<MTLTexture>)texture {
111 return self->_texture.texture;
112 }
113
114 #pragma clang diagnostic push
115 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
116 - (CAMetalLayer*)layer {
117 return (id)self->_layer;
118 }
119 #pragma clang diagnostic pop
120
121 - (NSUInteger)drawableID {
122 return self->_drawableId;
123 }
124
125 - (CFTimeInterval)presentedTime {
126 return 0;
127 }
128
129 - (void)present {
130 [_layer presentTexture:self->_texture];
131 self->_presented = YES;
132 }
133
134 - (void)dealloc {
135 if (!_presented) {
136 [_layer returnTexture:self->_texture];
137 }
138 }
139
140 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
141 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement addPresentedHandler:";
142 }
143
144 - (void)presentAtTime:(CFTimeInterval)presentationTime {
145 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAtTime:";
146 }
147
148 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
149 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:";
150 }
151
152 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
153 FlutterTexture* texture = _texture;
154 texture.waitingForCompletion = YES;
155 [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
156 texture.waitingForCompletion = NO;
157 }];
158 }
159
160 @end
161
162 @implementation FlutterMetalLayer
163
164 @synthesize preferredDevice = _preferredDevice;
165 @synthesize device = _device;
166 @synthesize pixelFormat = _pixelFormat;
167 @synthesize framebufferOnly = _framebufferOnly;
168 @synthesize colorspace = _colorspace;
169 @synthesize wantsExtendedDynamicRangeContent = _wantsExtendedDynamicRangeContent;
170
171 - (instancetype)init {
172 if (self = [super init]) {
173 _preferredDevice = MTLCreateSystemDefaultDevice();
174 self.device = self.preferredDevice;
175 self.pixelFormat = MTLPixelFormatBGRA8Unorm;
176 _availableTextures = [[NSMutableSet alloc] init];
177
178 _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(onDisplayLink:)];
179 [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:NO];
180 [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
181 [[NSNotificationCenter defaultCenter] addObserver:self
182 selector:@selector(didEnterBackground:)
183 name:UIApplicationDidEnterBackgroundNotification
184 object:nil];
185 }
186 return self;
187 }
188
189 - (void)dealloc {
190 [[NSNotificationCenter defaultCenter] removeObserver:self];
191 }
192
193 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
194 // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
195 // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
196 // has it's own displaylink scheduled on main thread, which is used to trigger core animation
197 // frame allowing for 120hz updates.
198 if (!DisplayLinkManager.maxRefreshRateEnabledOnIPhone) {
199 return;
200 }
201 double maxFrameRate = fmax(refreshRate, 60);
202 double minFrameRate = fmax(maxFrameRate / 2, 60);
203 if (@available(iOS 15.0, *)) {
204 _displayLink.preferredFrameRateRange =
205 CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
206 } else {
207 _displayLink.preferredFramesPerSecond = maxFrameRate;
208 }
209 }
210
211 - (void)onDisplayLink:(CADisplayLink*)link {
212 _didSetContentsDuringThisDisplayLinkPeriod = NO;
213 // Do not pause immediately, this seems to prevent 120hz while touching.
214 if (_displayLinkPauseCountdown == 3) {
215 _displayLink.paused = YES;
216 if (_displayLinkForcedMaxRate) {
217 [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:NO];
218 _displayLinkForcedMaxRate = NO;
219 }
220 } else {
221 ++_displayLinkPauseCountdown;
222 }
223 }
224
225 - (BOOL)isKindOfClass:(Class)aClass {
226 #pragma clang diagnostic push
227 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
228 // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
229 if ([aClass isEqual:[CAMetalLayer class]]) {
230 return YES;
231 }
232 #pragma clang diagnostic pop
233 return [super isKindOfClass:aClass];
234 }
235
236 - (void)setDrawableSize:(CGSize)drawableSize {
237 [_availableTextures removeAllObjects];
238 _front = nil;
239 _totalTextures = 0;
240 _drawableSize = drawableSize;
241 }
242
243 - (void)didEnterBackground:(id)notification {
244 [_availableTextures removeAllObjects];
245 _totalTextures = _front != nil ? 1 : 0;
246 _displayLink.paused = YES;
247 }
248
249 - (CGSize)drawableSize {
250 return _drawableSize;
251 }
252
253 - (IOSurface*)createIOSurface {
254 unsigned pixelFormat;
255 unsigned bytesPerElement;
256 if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
257 pixelFormat = kCVPixelFormatType_64RGBAHalf;
258 bytesPerElement = 8;
259 } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
260 pixelFormat = kCVPixelFormatType_32BGRA;
261 bytesPerElement = 4;
262 } else {
263 FML_LOG(ERROR) << "Unsupported pixel format: " << self.pixelFormat;
264 return nil;
265 }
266 size_t bytesPerRow =
267 IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
268 size_t totalBytes =
269 IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
270 NSDictionary* options = @{
271 (id)kIOSurfaceWidth : @(_drawableSize.width),
272 (id)kIOSurfaceHeight : @(_drawableSize.height),
273 (id)kIOSurfacePixelFormat : @(pixelFormat),
274 (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
275 (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
276 (id)kIOSurfaceAllocSize : @(totalBytes),
277 };
278
279 IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
280 if (res == nil) {
281 FML_LOG(ERROR) << "Failed to create IOSurface with options "
282 << options.debugDescription.UTF8String;
283 return nil;
284 }
285
286 if (self.colorspace != nil) {
287 CFStringRef name = CGColorSpaceGetName(self.colorspace);
288 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), name);
289 } else {
290 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), kCGColorSpaceSRGB);
291 }
292 return (__bridge_transfer IOSurface*)res;
293 }
294
295 - (FlutterTexture*)nextTexture {
296 CFTimeInterval start = CACurrentMediaTime();
297 while (true) {
298 FlutterTexture* texture = [self tryNextTexture];
299 if (texture != nil) {
300 return texture;
301 }
302 CFTimeInterval elapsed = CACurrentMediaTime() - start;
303 if (elapsed > 1.0) {
304 NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
305 return nil;
306 }
307 }
308 }
309
310 - (FlutterTexture*)tryNextTexture {
311 @synchronized(self) {
312 if (_front != nil && _front.waitingForCompletion) {
313 return nil;
314 }
315 if (_totalTextures < 3) {
316 ++_totalTextures;
317 IOSurface* surface = [self createIOSurface];
318 if (surface == nil) {
319 return nil;
320 }
321 MTLTextureDescriptor* textureDescriptor =
322 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
323 width:_drawableSize.width
324 height:_drawableSize.height
325 mipmapped:NO];
326
327 if (_framebufferOnly) {
328 textureDescriptor.usage = MTLTextureUsageRenderTarget;
329 } else {
330 textureDescriptor.usage =
331 MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
332 }
333 id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
334 iosurface:(__bridge IOSurfaceRef)surface
335 plane:0];
336 FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
337 surface:surface];
338 return flutterTexture;
339 } else {
340 // Prefer surface that is not in use and has been presented the longest
341 // time ago.
342 // When isInUse is false, the surface is definitely not used by the compositor.
343 // When isInUse is true, the surface may be used by the compositor.
344 // When both surfaces are in use, the one presented earlier will be returned.
345 // The assumption here is that the compositor is already aware of the
346 // newer texture and is unlikely to read from the older one, even though it
347 // has not decreased the use count yet (there seems to be certain latency).
348 FlutterTexture* res = nil;
349 for (FlutterTexture* texture in _availableTextures) {
350 if (res == nil) {
351 res = texture;
352 } else if (res.surface.isInUse && !texture.surface.isInUse) {
353 // prefer texture that is not in use.
354 res = texture;
355 } else if (res.surface.isInUse == texture.surface.isInUse &&
356 texture.presentedTime < res.presentedTime) {
357 // prefer texture with older presented time.
358 res = texture;
359 }
360 }
361 if (res != nil) {
362 [_availableTextures removeObject:res];
363 }
364 return res;
365 }
366 }
367 }
368
369 - (id<CAMetalDrawable>)nextDrawable {
370 FlutterTexture* texture = [self nextTexture];
371 if (texture == nil) {
372 return nil;
373 }
374 FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
375 layer:self
376 drawableId:_nextDrawableId++];
377 return drawable;
378 }
379
380 - (void)presentOnMainThread:(FlutterTexture*)texture {
381 // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
382 // Might also be placebo
383 [self setNeedsDisplay];
384
385 [CATransaction begin];
386 [CATransaction setDisableActions:YES];
387 self.contents = texture.surface;
388 [CATransaction commit];
389 _displayLink.paused = NO;
390 _displayLinkPauseCountdown = 0;
391 if (!_didSetContentsDuringThisDisplayLinkPeriod) {
392 _didSetContentsDuringThisDisplayLinkPeriod = YES;
393 } else if (!_displayLinkForcedMaxRate) {
394 _displayLinkForcedMaxRate = YES;
395 [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:YES];
396 }
397 }
398
399 - (void)presentTexture:(FlutterTexture*)texture {
400 @synchronized(self) {
401 if (_front != nil) {
402 [_availableTextures addObject:_front];
403 }
404 _front = texture;
405 texture.presentedTime = CACurrentMediaTime();
406 if ([NSThread isMainThread]) {
407 [self presentOnMainThread:texture];
408 } else {
409 // Core animation layers can only be updated on main thread.
410 dispatch_async(dispatch_get_main_queue(), ^{
411 [self presentOnMainThread:texture];
412 });
413 }
414 }
415 }
416
417 - (void)returnTexture:(FlutterTexture*)texture {
418 @synchronized(self) {
419 [_availableTextures addObject:texture];
420 }
421 }
422
423 + (BOOL)enabled {
424 static BOOL enabled = NO;
425 static BOOL didCheckInfoPlist = NO;
426 if (!didCheckInfoPlist) {
427 didCheckInfoPlist = YES;
428 NSNumber* use_flutter_metal_layer =
429 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
430 if (use_flutter_metal_layer != nil && [use_flutter_metal_layer boolValue]) {
431 enabled = YES;
432 FML_LOG(WARNING) << "Using FlutterMetalLayer. This is an experimental feature.";
433 }
434 }
435 return enabled;
436 }
437
438 @end
FlutterMetalLayer::wantsExtendedDynamicRangeContent
BOOL wantsExtendedDynamicRangeContent
Definition: FlutterMetalLayer.h:21
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:423
DisplayLinkManager::maxRefreshRateEnabledOnIPhone
BOOL maxRefreshRateEnabledOnIPhone
Definition: FlutterMetalLayer.mm:13
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:85
FlutterTexture::_surface
IOSurface * _surface
Definition: FlutterMetalLayer.mm:56
FlutterDrawable
Definition: FlutterMetalLayer.mm:84
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:23
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:87
DisplayLinkManager
Definition: FlutterMetalLayer.mm:12
FlutterTexture::waitingForCompletion
BOOL waitingForCompletion
Definition: FlutterMetalLayer.mm:63
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:43
FlutterMetalLayer::framebufferOnly
BOOL framebufferOnly
Definition: FlutterMetalLayer.h:17
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:29
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:31
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:20
FlutterTexture::_presentedTime
CFTimeInterval _presentedTime
Definition: FlutterMetalLayer.mm:57
FlutterMetalDrawable-p
Definition: FlutterMetalLayer.h:33
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:24
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:36
FlutterMetalLayer::preferredDevice
id< MTLDevice > preferredDevice
Definition: FlutterMetalLayer.h:15
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:28
FlutterTexture
Definition: FlutterMetalLayer.mm:54
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:26
FlutterTexture::_texture
id< MTLTexture > _texture
Definition: FlutterMetalLayer.mm:55
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:62
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:35
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:61
FlutterMetalLayer::device
id< MTLDevice > device
Definition: FlutterMetalLayer.h:14
FlutterMetalLayer.h
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:88
+[DisplayLinkManager displayRefreshRate]
double displayRefreshRate()
Definition: vsync_waiter_ios.mm:160
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:369
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:16
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:46
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:18
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:86
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:60
FlutterMetalLayer
Definition: FlutterMetalLayer.h:12