| Flutter iOS Embedder |

FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <IOSurface/IOSurfaceObjC.h>
6 #include <Metal/Metal.h>
7 #include <UIKit/UIKit.h>
8
9 #include "flutter/fml/logging.h"
10 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterMetalLayer.h"
11
12 @interface DisplayLinkManager : NSObject
13 @property(class, nonatomic, readonly) BOOL maxRefreshRateEnabledOnIPhone;
14 + (double)displayRefreshRate;
15 @end
16
17 @class FlutterTexture;
18 @class FlutterDrawable;
19
20 extern CFTimeInterval display_link_target;
21
22 @interface FlutterMetalLayer () {
23 id<MTLDevice> _preferredDevice;
24 CGSize _drawableSize;
25
26 NSUInteger _nextDrawableId;
27
28 NSMutableSet<FlutterTexture*>* _availableTextures;
29 NSUInteger _totalTextures;
30
31 FlutterTexture* _front;
32
33 // There must be a CADisplayLink scheduled *on main thread* otherwise
34 // core animation only updates layers 60 times a second.
35 CADisplayLink* _displayLink;
36 NSUInteger _displayLinkPauseCountdown;
37
38 // Used to track whether the content was set during this display link.
39 // When unlocking phone the layer (main thread) display link and raster thread
40 // display link get out of sync for several seconds. Even worse, layer display
41 // link does not seem to reflect actual vsync. Forcing the layer link
42 // to max rate (instead range) temporarily seems to fix the issue.
43 BOOL _didSetContentsDuringThisDisplayLinkPeriod;
44
45 // Whether layer displayLink is forced to max rate.
46 BOOL _displayLinkForcedMaxRate;
47 }
48
49 - (void)presentTexture:(FlutterTexture*)texture;
50 - (void)returnTexture:(FlutterTexture*)texture;
51
52 @end
53
54 @interface FlutterTexture : NSObject {
55 id<MTLTexture> _texture;
56 IOSurface* _surface;
57 CFTimeInterval _presentedTime;
58 }
59
60 @property(readonly, nonatomic) id<MTLTexture> texture;
61 @property(readonly, nonatomic) IOSurface* surface;
62 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
63
64 @end
65
66 @implementation FlutterTexture
67
68 @synthesize texture = _texture;
69 @synthesize surface = _surface;
70 @synthesize presentedTime = _presentedTime;
71
72 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
73 if (self = [super init]) {
74 _texture = texture;
75 _surface = surface;
76 }
77 return self;
78 }
79
80 @end
81
82 @interface FlutterDrawable : NSObject <CAMetalDrawable> {
83 FlutterTexture* _texture;
84 __weak FlutterMetalLayer* _layer;
85 NSUInteger _drawableId;
86 BOOL _presented;
87 }
88
89 - (instancetype)initWithTexture:(FlutterTexture*)texture
90 layer:(FlutterMetalLayer*)layer
91 drawableId:(NSUInteger)drawableId;
92
93 @end
94
95 @implementation FlutterDrawable
96
97 - (instancetype)initWithTexture:(FlutterTexture*)texture
98 layer:(FlutterMetalLayer*)layer
99 drawableId:(NSUInteger)drawableId {
100 if (self = [super init]) {
101 _texture = texture;
102 _layer = layer;
103 _drawableId = drawableId;
104 }
105 return self;
106 }
107
108 - (id<MTLTexture>)texture {
109 return self->_texture.texture;
110 }
111
112 #pragma clang diagnostic push
113 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
114 - (CAMetalLayer*)layer {
115 return (id)self->_layer;
116 }
117 #pragma clang diagnostic pop
118
119 - (NSUInteger)drawableID {
120 return self->_drawableId;
121 }
122
123 - (CFTimeInterval)presentedTime {
124 return 0;
125 }
126
127 - (void)present {
128 [_layer presentTexture:self->_texture];
129 self->_presented = YES;
130 }
131
132 - (void)dealloc {
133 if (!_presented) {
134 [_layer returnTexture:self->_texture];
135 }
136 }
137
138 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
139 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement addPresentedHandler:";
140 }
141
142 - (void)presentAtTime:(CFTimeInterval)presentationTime {
143 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAtTime:";
144 }
145
146 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
147 FML_LOG(WARNING) << "FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:";
148 }
149
150 @end
151
152 @implementation FlutterMetalLayer
153
154 @synthesize preferredDevice = _preferredDevice;
155 @synthesize device = _device;
156 @synthesize pixelFormat = _pixelFormat;
157 @synthesize framebufferOnly = _framebufferOnly;
158 @synthesize colorspace = _colorspace;
159 @synthesize wantsExtendedDynamicRangeContent = _wantsExtendedDynamicRangeContent;
160
161 - (instancetype)init {
162 if (self = [super init]) {
163 _preferredDevice = MTLCreateSystemDefaultDevice();
164 self.device = self.preferredDevice;
165 self.pixelFormat = MTLPixelFormatBGRA8Unorm;
166 _availableTextures = [[NSMutableSet alloc] init];
167
168 _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(onDisplayLink:)];
169 [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:NO];
170 [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes];
171 [[NSNotificationCenter defaultCenter] addObserver:self
172 selector:@selector(didEnterBackground:)
173 name:UIApplicationDidEnterBackgroundNotification
174 object:nil];
175 }
176 return self;
177 }
178
179 - (void)dealloc {
180 [[NSNotificationCenter defaultCenter] removeObserver:self];
181 }
182
183 - (void)setMaxRefreshRate:(double)refreshRate forceMax:(BOOL)forceMax {
184 // This is copied from vsync_waiter_ios.mm. The vsync waiter has display link scheduled on UI
185 // thread which does not trigger actual core animation frame. As a workaround FlutterMetalLayer
186 // has it's own displaylink scheduled on main thread, which is used to trigger core animation
187 // frame allowing for 120hz updates.
188 if (!DisplayLinkManager.maxRefreshRateEnabledOnIPhone) {
189 return;
190 }
191 double maxFrameRate = fmax(refreshRate, 60);
192 double minFrameRate = fmax(maxFrameRate / 2, 60);
193 if (@available(iOS 15.0, *)) {
194 _displayLink.preferredFrameRateRange =
195 CAFrameRateRangeMake(forceMax ? maxFrameRate : minFrameRate, maxFrameRate, maxFrameRate);
196 } else {
197 _displayLink.preferredFramesPerSecond = maxFrameRate;
198 }
199 }
200
201 - (void)onDisplayLink:(CADisplayLink*)link {
202 _didSetContentsDuringThisDisplayLinkPeriod = NO;
203 // Do not pause immediately, this seems to prevent 120hz while touching.
204 if (_displayLinkPauseCountdown == 3) {
205 _displayLink.paused = YES;
206 if (_displayLinkForcedMaxRate) {
207 [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:NO];
208 _displayLinkForcedMaxRate = NO;
209 }
210 } else {
211 ++_displayLinkPauseCountdown;
212 }
213 }
214
215 - (BOOL)isKindOfClass:(Class)aClass {
216 #pragma clang diagnostic push
217 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
218 // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
219 if ([aClass isEqual:[CAMetalLayer class]]) {
220 return YES;
221 }
222 #pragma clang diagnostic pop
223 return [super isKindOfClass:aClass];
224 }
225
226 - (void)setDrawableSize:(CGSize)drawableSize {
227 [_availableTextures removeAllObjects];
228 _front = nil;
229 _totalTextures = 0;
230 _drawableSize = drawableSize;
231 }
232
233 - (void)didEnterBackground:(id)notification {
234 [_availableTextures removeAllObjects];
235 _totalTextures = _front != nil ? 1 : 0;
236 _displayLink.paused = YES;
237 }
238
239 - (CGSize)drawableSize {
240 return _drawableSize;
241 }
242
243 - (IOSurface*)createIOSurface {
244 unsigned pixelFormat;
245 unsigned bytesPerElement;
246 if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
247 pixelFormat = kCVPixelFormatType_64RGBAHalf;
248 bytesPerElement = 8;
249 } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
250 pixelFormat = kCVPixelFormatType_32BGRA;
251 bytesPerElement = 4;
252 } else {
253 FML_LOG(ERROR) << "Unsupported pixel format: " << self.pixelFormat;
254 return nil;
255 }
256 size_t bytesPerRow =
257 IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
258 size_t totalBytes =
259 IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
260 NSDictionary* options = @{
261 (id)kIOSurfaceWidth : @(_drawableSize.width),
262 (id)kIOSurfaceHeight : @(_drawableSize.height),
263 (id)kIOSurfacePixelFormat : @(pixelFormat),
264 (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
265 (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
266 (id)kIOSurfaceAllocSize : @(totalBytes),
267 };
268
269 IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
270 if (res == nil) {
271 FML_LOG(ERROR) << "Failed to create IOSurface with options "
272 << options.debugDescription.UTF8String;
273 return nil;
274 }
275
276 if (self.colorspace != nil) {
277 CFStringRef name = CGColorSpaceGetName(self.colorspace);
278 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), name);
279 } else {
280 IOSurfaceSetValue(res, CFSTR("IOSurfaceColorSpace"), kCGColorSpaceSRGB);
281 }
282 return (__bridge_transfer IOSurface*)res;
283 }
284
285 - (FlutterTexture*)nextTexture {
286 @synchronized(self) {
287 if (_totalTextures < 3) {
288 ++_totalTextures;
289 IOSurface* surface = [self createIOSurface];
290 if (surface == nil) {
291 return nil;
292 }
293 MTLTextureDescriptor* textureDescriptor =
294 [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
295 width:_drawableSize.width
296 height:_drawableSize.height
297 mipmapped:NO];
298
299 if (_framebufferOnly) {
300 textureDescriptor.usage = MTLTextureUsageRenderTarget;
301 } else {
302 textureDescriptor.usage =
303 MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
304 }
305 id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
306 iosurface:(__bridge IOSurfaceRef)surface
307 plane:0];
308 FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
309 surface:surface];
310 return flutterTexture;
311 } else {
312 // Make sure raster thread doesn't have too many drawables in flight.
313 if (_availableTextures.count == 0) {
314 CFTimeInterval start = CACurrentMediaTime();
315 while (_availableTextures.count == 0 && CACurrentMediaTime() - start < 1.0) {
316 usleep(100);
317 }
318 CFTimeInterval elapsed = CACurrentMediaTime() - start;
319 if (_availableTextures.count == 0) {
320 NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
321 return nil;
322 } else {
323 NSLog(@"Had to wait %f seconds for a drawable", elapsed);
324 }
325 }
326
327 // Prefer surface that is not in use and has been presented the longest
328 // time ago.
329 // When isInUse is false, the surface is definitely not used by the compositor.
330 // When isInUse is true, the surface may be used by the compositor.
331 // When both surfaces are in use, the one presented earlier will be returned.
332 // The assumption here is that the compositor is already aware of the
333 // newer texture and is unlikely to read from the older one, even though it
334 // has not decreased the use count yet (there seems to be certain latency).
335 FlutterTexture* res = nil;
336 for (FlutterTexture* texture in _availableTextures) {
337 if (res == nil) {
338 res = texture;
339 } else if (res.surface.isInUse && !texture.surface.isInUse) {
340 // prefer texture that is not in use.
341 res = texture;
342 } else if (res.surface.isInUse == texture.surface.isInUse &&
343 texture.presentedTime < res.presentedTime) {
344 // prefer texture with older presented time.
345 res = texture;
346 }
347 }
348 [_availableTextures removeObject:res];
349 return res;
350 }
351 }
352 }
353
354 - (id<CAMetalDrawable>)nextDrawable {
355 FlutterTexture* texture = [self nextTexture];
356 if (texture == nil) {
357 return nil;
358 }
359 FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
360 layer:self
361 drawableId:_nextDrawableId++];
362 return drawable;
363 }
364
365 - (void)presentOnMainThread:(FlutterTexture*)texture {
366 // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
367 // Might also be placebo
368 [self setNeedsDisplay];
369
370 [CATransaction begin];
371 [CATransaction setDisableActions:YES];
372 self.contents = texture.surface;
373 texture.presentedTime = CACurrentMediaTime();
374 [CATransaction commit];
375 _displayLink.paused = NO;
376 _displayLinkPauseCountdown = 0;
377 if (!_didSetContentsDuringThisDisplayLinkPeriod) {
378 _didSetContentsDuringThisDisplayLinkPeriod = YES;
379 } else if (!_displayLinkForcedMaxRate) {
380 _displayLinkForcedMaxRate = YES;
381 [self setMaxRefreshRate:[DisplayLinkManager displayRefreshRate] forceMax:YES];
382 }
383 }
384
385 - (void)presentTexture:(FlutterTexture*)texture {
386 @synchronized(self) {
387 if (_front != nil) {
388 [_availableTextures addObject:_front];
389 }
390 _front = texture;
391 if ([NSThread isMainThread]) {
392 [self presentOnMainThread:texture];
393 } else {
394 // Core animation layers can only be updated on main thread.
395 dispatch_async(dispatch_get_main_queue(), ^{
396 [self presentOnMainThread:texture];
397 });
398 }
399 }
400 }
401
402 - (void)returnTexture:(FlutterTexture*)texture {
403 @synchronized(self) {
404 [_availableTextures addObject:texture];
405 }
406 }
407
408 + (BOOL)enabled {
409 static BOOL enabled = NO;
410 static BOOL didCheckInfoPlist = NO;
411 if (!didCheckInfoPlist) {
412 didCheckInfoPlist = YES;
413 NSNumber* use_flutter_metal_layer =
414 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
415 if (use_flutter_metal_layer != nil && [use_flutter_metal_layer boolValue]) {
416 enabled = YES;
417 FML_LOG(WARNING) << "Using FlutterMetalLayer. This is an experimental feature.";
418 }
419 }
420 return enabled;
421 }
422
423 @end
FlutterMetalLayer::wantsExtendedDynamicRangeContent
BOOL wantsExtendedDynamicRangeContent
Definition: FlutterMetalLayer.h:18
+[FlutterMetalLayer enabled]
BOOL enabled()
Definition: FlutterMetalLayer.mm:408
DisplayLinkManager::maxRefreshRateEnabledOnIPhone
BOOL maxRefreshRateEnabledOnIPhone
Definition: FlutterMetalLayer.mm:13
FlutterDrawable::_texture
FlutterTexture * _texture
Definition: FlutterMetalLayer.mm:83
FlutterTexture::_surface
IOSurface * _surface
Definition: FlutterMetalLayer.mm:56
FlutterDrawable
Definition: FlutterMetalLayer.mm:82
FlutterMetalLayer()::_preferredDevice
id< MTLDevice > _preferredDevice
Definition: FlutterMetalLayer.mm:23
FlutterDrawable::_drawableId
NSUInteger _drawableId
Definition: FlutterMetalLayer.mm:85
DisplayLinkManager
Definition: FlutterMetalLayer.mm:12
FlutterMetalLayer()::_didSetContentsDuringThisDisplayLinkPeriod
BOOL _didSetContentsDuringThisDisplayLinkPeriod
Definition: FlutterMetalLayer.mm:43
FlutterMetalLayer::framebufferOnly
BOOL framebufferOnly
Definition: FlutterMetalLayer.h:14
FlutterMetalLayer()::_totalTextures
NSUInteger _totalTextures
Definition: FlutterMetalLayer.mm:29
FlutterMetalLayer()::_front
FlutterTexture * _front
Definition: FlutterMetalLayer.mm:31
FlutterMetalLayer::colorspace
CGColorSpaceRef colorspace
Definition: FlutterMetalLayer.h:17
FlutterTexture::_presentedTime
CFTimeInterval _presentedTime
Definition: FlutterMetalLayer.mm:57
FlutterMetalLayer()::_drawableSize
CGSize _drawableSize
Definition: FlutterMetalLayer.mm:24
FlutterMetalLayer()::_displayLinkPauseCountdown
NSUInteger _displayLinkPauseCountdown
Definition: FlutterMetalLayer.mm:36
FlutterMetalLayer::preferredDevice
id< MTLDevice > preferredDevice
Definition: FlutterMetalLayer.h:12
FlutterMetalLayer()::_availableTextures
NSMutableSet< FlutterTexture * > * _availableTextures
Definition: FlutterMetalLayer.mm:28
FlutterTexture
Definition: FlutterMetalLayer.mm:54
display_link_target
CFTimeInterval display_link_target
FlutterMetalLayer()::_nextDrawableId
NSUInteger _nextDrawableId
Definition: FlutterMetalLayer.mm:26
FlutterTexture::_texture
id< MTLTexture > _texture
Definition: FlutterMetalLayer.mm:55
FlutterTexture::presentedTime
CFTimeInterval presentedTime
Definition: FlutterMetalLayer.mm:62
FlutterMetalLayer()::_displayLink
CADisplayLink * _displayLink
Definition: FlutterMetalLayer.mm:35
FlutterTexture::surface
IOSurface * surface
Definition: FlutterMetalLayer.mm:61
FlutterMetalLayer::device
id< MTLDevice > device
Definition: FlutterMetalLayer.h:11
FlutterMetalLayer.h
FlutterDrawable::_presented
BOOL _presented
Definition: FlutterMetalLayer.mm:86
+[DisplayLinkManager displayRefreshRate]
double displayRefreshRate()
Definition: vsync_waiter_ios.mm:174
-[FlutterMetalLayer nextDrawable]
nullable id< CAMetalDrawable > nextDrawable()
Definition: FlutterMetalLayer.mm:354
FlutterMetalLayer::pixelFormat
MTLPixelFormat pixelFormat
Definition: FlutterMetalLayer.h:13
FlutterMetalLayer()::_displayLinkForcedMaxRate
BOOL _displayLinkForcedMaxRate
Definition: FlutterMetalLayer.mm:46
FlutterMetalLayer::drawableSize
CGSize drawableSize
Definition: FlutterMetalLayer.h:15
FlutterDrawable::_layer
__weak FlutterMetalLayer * _layer
Definition: FlutterMetalLayer.mm:84
FlutterTexture::texture
id< MTLTexture > texture
Definition: FlutterMetalLayer.mm:60
FlutterMetalLayer
Definition: FlutterMetalLayer.h:9