| Flutter iOS Embedder |

accessibility_bridge.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
6
7 #include <utility>
8
9 #include "flutter/fml/logging.h"
10 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h"
11 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h"
12 #import "flutter/shell/platform/darwin/ios/framework/Source/TextInputSemanticsObject.h"
13 #import "flutter/shell/platform/darwin/ios/platform_view_ios.h"
14
15 #include "flutter/common/constants.h"
16
17 #pragma GCC diagnostic error "-Wundeclared-selector"
18
19 FLUTTER_ASSERT_ARC
20
21 namespace flutter {
22 namespace {
23
24 constexpr int32_t kSemanticObjectIdInvalid = -1;
25
26 class DefaultIosDelegate : public AccessibilityBridge::IosDelegate {
27 public:
28 bool IsFlutterViewControllerPresentingModalViewController(
29 FlutterViewController* view_controller) override {
30 if (view_controller) {
31 return view_controller.isPresentingViewController;
32 } else {
33 return false;
34 }
35 }
36
37 void PostAccessibilityNotification(UIAccessibilityNotifications notification,
38 id argument) override {
39 UIAccessibilityPostNotification(notification, argument);
40 }
41 };
42 } // namespace
43
44 AccessibilityBridge::AccessibilityBridge(
45 FlutterViewController* view_controller,
46 PlatformViewIOS* platform_view,
47 __weak FlutterPlatformViewsController* platform_views_controller,
48 std::unique_ptr<IosDelegate> ios_delegate)
49 : view_controller_(view_controller),
50 platform_view_(platform_view),
51 platform_views_controller_(platform_views_controller),
52 last_focused_semantics_object_id_(kSemanticObjectIdInvalid),
53 objects_([[NSMutableDictionary alloc] init]),
54 previous_routes_({}),
55 ios_delegate_(ios_delegate ? std::move(ios_delegate)
56 : std::make_unique<DefaultIosDelegate>()),
57 weak_factory_(this) {
58 accessibility_channel_ = [[FlutterBasicMessageChannel alloc]
59 initWithName:@"flutter/accessibility"
60 binaryMessenger:platform_view->GetOwnerViewController().engine.binaryMessenger
61 codec:[FlutterStandardMessageCodec sharedInstance]];
62 [accessibility_channel_ setMessageHandler:^(id message, FlutterReply reply) {
63 HandleEvent((NSDictionary*)message);
64 }];
65 }
66
67 AccessibilityBridge::~AccessibilityBridge() {
68 [accessibility_channel_ setMessageHandler:nil];
69 clearState();
70 }
71
72 UIView<UITextInput>* AccessibilityBridge::textInputView() {
73 return [[platform_view_->GetOwnerViewController().engine textInputPlugin] textInputView];
74 }
75
76 void AccessibilityBridge::AccessibilityObjectDidBecomeFocused(int32_t id) {
77 last_focused_semantics_object_id_ = id;
78 [accessibility_channel_ sendMessage:@{@"type" : @"didGainFocus", @"nodeId" : @(id)}];
79 }
80
81 void AccessibilityBridge::AccessibilityObjectDidLoseFocus(int32_t id) {
82 if (last_focused_semantics_object_id_ == id) {
83 last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
84 }
85 }
86
87 void AccessibilityBridge::UpdateSemantics(
88 flutter::SemanticsNodeUpdates nodes,
89 const flutter::CustomAccessibilityActionUpdates& actions) {
90 BOOL layoutChanged = NO;
91 BOOL scrollOccured = NO;
92 BOOL needsAnnouncement = NO;
93 for (const auto& entry : actions) {
94 const flutter::CustomAccessibilityAction& action = entry.second;
95 actions_[action.id] = action;
96 }
97 for (const auto& entry : nodes) {
98 const flutter::SemanticsNode& node = entry.second;
99 SemanticsObject* object = GetOrCreateObject(node.id, nodes);
100 layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
101 scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
102 needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
103 [object setSemanticsNode:&node];
104 NSUInteger newChildCount = node.childrenInTraversalOrder.size();
105 NSMutableArray* newChildren = [[NSMutableArray alloc] initWithCapacity:newChildCount];
106 for (NSUInteger i = 0; i < newChildCount; ++i) {
107 SemanticsObject* child = GetOrCreateObject(node.childrenInTraversalOrder[i], nodes);
108 [newChildren addObject:child];
109 }
110 NSMutableArray* newChildrenInHitTestOrder =
111 [[NSMutableArray alloc] initWithCapacity:newChildCount];
112 for (NSUInteger i = 0; i < newChildCount; ++i) {
113 SemanticsObject* child = GetOrCreateObject(node.childrenInHitTestOrder[i], nodes);
114 [newChildrenInHitTestOrder addObject:child];
115 }
116 object.children = newChildren;
117 object.childrenInHitTestOrder = newChildrenInHitTestOrder;
118 if (!node.customAccessibilityActions.empty()) {
119 NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
120 [[NSMutableArray alloc] init];
121 for (int32_t action_id : node.customAccessibilityActions) {
122 flutter::CustomAccessibilityAction& action = actions_[action_id];
123 if (action.overrideId != -1) {
124 // iOS does not support overriding standard actions, so we ignore any
125 // custom actions that have an override id provided.
126 continue;
127 }
128 NSString* label = @(action.label.data());
129 SEL selector = @selector(onCustomAccessibilityAction:);
130 FlutterCustomAccessibilityAction* customAction =
131 [[FlutterCustomAccessibilityAction alloc] initWithName:label
132 target:object
133 selector:selector];
134 customAction.uid = action_id;
135 [accessibilityCustomActions addObject:customAction];
136 }
137 object.accessibilityCustomActions = accessibilityCustomActions;
138 }
139
140 if (needsAnnouncement) {
141 // Try to be more polite - iOS 11+ supports
142 // UIAccessibilitySpeechAttributeQueueAnnouncement which should avoid
143 // interrupting system notifications or other elements.
144 // Expectation: roughly match the behavior of polite announcements on
145 // Android.
146 NSString* announcement = [[NSString alloc] initWithUTF8String:object.node.label.c_str()];
147 UIAccessibilityPostNotification(
148 UIAccessibilityAnnouncementNotification,
149 [[NSAttributedString alloc] initWithString:announcement
150 attributes:@{
151 UIAccessibilitySpeechAttributeQueueAnnouncement : @YES
152 }]);
153 }
154 }
155
156 SemanticsObject* root = objects_[@(kRootNodeId)];
157
158 bool routeChanged = false;
159 SemanticsObject* lastAdded = nil;
160
161 if (root) {
162 if (!view_controller_.view.accessibilityElements) {
163 view_controller_.view.accessibilityElements =
164 @[ [root accessibilityContainer] ?: [NSNull null] ];
165 }
166 NSMutableArray<SemanticsObject*>* newRoutes = [[NSMutableArray alloc] init];
167 [root collectRoutes:newRoutes];
168 // Finds the last route that is not in the previous routes.
169 for (SemanticsObject* route in newRoutes) {
170 if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) ==
171 previous_routes_.end()) {
172 lastAdded = route;
173 }
174 }
175 // If all the routes are in the previous route, get the last route.
176 if (lastAdded == nil && [newRoutes count] > 0) {
177 int index = [newRoutes count] - 1;
178 lastAdded = [newRoutes objectAtIndex:index];
179 }
180 // There are two cases if lastAdded != nil
181 // 1. lastAdded is not in previous routes. In this case,
182 // [lastAdded uid] != previous_route_id_
183 // 2. All new routes are in previous routes and
184 // lastAdded = newRoutes.last.
185 // In the first case, we need to announce new route. In the second case,
186 // we need to announce if one list is shorter than the other.
187 if (lastAdded != nil &&
188 ([lastAdded uid] != previous_route_id_ || [newRoutes count] != previous_routes_.size())) {
189 previous_route_id_ = [lastAdded uid];
190 routeChanged = true;
191 }
192 previous_routes_.clear();
193 for (SemanticsObject* route in newRoutes) {
194 previous_routes_.push_back([route uid]);
195 }
196 } else {
197 view_controller_.viewIfLoaded.accessibilityElements = nil;
198 }
199
200 NSMutableArray<NSNumber*>* doomed_uids = [NSMutableArray arrayWithArray:objects_.allKeys];
201 if (root) {
202 VisitObjectsRecursivelyAndRemove(root, doomed_uids);
203 }
204 [objects_ removeObjectsForKeys:doomed_uids];
205
206 for (SemanticsObject* object in objects_.allValues) {
207 [object accessibilityBridgeDidFinishUpdate];
208 }
209
210 if (!ios_delegate_->IsFlutterViewControllerPresentingModalViewController(view_controller_)) {
211 layoutChanged = layoutChanged || [doomed_uids count] > 0;
212
213 if (routeChanged) {
214 NSString* routeName = [lastAdded routeName];
215 ios_delegate_->PostAccessibilityNotification(UIAccessibilityScreenChangedNotification,
216 routeName);
217 }
218
219 if (layoutChanged) {
220 SemanticsObject* next = FindNextFocusableIfNecessary();
221 SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
222 // Only specify the focus item if the new focus is different, avoiding double focuses on the
223 // same item. See: https://github.com/flutter/flutter/issues/104176. If there is a route
224 // change, we always refocus.
225 ios_delegate_->PostAccessibilityNotification(
226 UIAccessibilityLayoutChangedNotification,
227 (routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
228 } else if (scrollOccured) {
229 // TODO(chunhtai): figure out what string to use for notification. At this
230 // point, it is guarantee the previous focused object is still in the tree
231 // so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
232 ios_delegate_->PostAccessibilityNotification(
233 UIAccessibilityPageScrolledNotification,
234 FindNextFocusableIfNecessary().nativeAccessibility);
235 }
236 }
237 }
238
239 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
240 flutter::SemanticsAction action) {
241 // TODO(team-ios): Remove implicit view assumption.
242 // https://github.com/flutter/flutter/issues/142845
243 platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action, {});
244 }
245
246 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
247 flutter::SemanticsAction action,
248 fml::MallocMapping args) {
249 // TODO(team-ios): Remove implicit view assumption.
250 // https://github.com/flutter/flutter/issues/142845
251 platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action,
252 std::move(args));
253 }
254
255 static void ReplaceSemanticsObject(SemanticsObject* oldObject,
256 SemanticsObject* newObject,
257 NSMutableDictionary<NSNumber*, SemanticsObject*>* objects) {
258 // `newObject` should represent the same id as `oldObject`.
259 FML_DCHECK(oldObject.node.id == newObject.uid);
260 NSNumber* nodeId = @(oldObject.node.id);
261 NSUInteger positionInChildlist = [oldObject.parent.children indexOfObject:oldObject];
262 oldObject.children = @[];
263 [oldObject.parent replaceChildAtIndex:positionInChildlist withChild:newObject];
264 [objects removeObjectForKey:nodeId];
265 objects[nodeId] = newObject;
266 }
267
268 static SemanticsObject* CreateObject(const flutter::SemanticsNode& node,
269 const fml::WeakPtr<AccessibilityBridge>& weak_ptr) {
270 if (node.flags.isTextField && !node.flags.isReadOnly) {
271 // Text fields are backed by objects that implement UITextInput.
272 return [[TextInputSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
273 } else if (!node.flags.isInMutuallyExclusiveGroup &&
274 (node.flags.hasToggledState || node.flags.hasCheckedState)) {
275 return [[FlutterSwitchSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
276 } else if (node.flags.hasImplicitScrolling) {
277 return [[FlutterScrollableSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
278 } else if (node.IsPlatformViewNode()) {
279 FlutterPlatformViewsController* platformViewsController =
280 weak_ptr->GetPlatformViewsController();
281 FlutterTouchInterceptingView* touchInterceptingView =
282 [platformViewsController flutterTouchInterceptingViewForId:node.platformViewId];
283 return [[FlutterPlatformViewSemanticsContainer alloc] initWithBridge:weak_ptr
284 uid:node.id
285 platformView:touchInterceptingView];
286 } else {
287 return [[FlutterSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
288 }
289 }
290
291 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
292 flutter::SemanticsNodeUpdates& updates) {
293 SemanticsObject* object = objects_[@(uid)];
294 if (!object) {
295 object = CreateObject(updates[uid], GetWeakPtr());
296 objects_[@(uid)] = object;
297 } else {
298 // Existing node case
299 auto nodeEntry = updates.find(object.node.id);
300 if (nodeEntry != updates.end()) {
301 // There's an update for this node
302 flutter::SemanticsNode node = nodeEntry->second;
303 if (object.node.flags.isTextField != node.flags.isTextField ||
304 object.node.flags.isReadOnly != node.flags.isReadOnly ||
305 object.node.flags.hasCheckedState != node.flags.hasCheckedState ||
306 object.node.flags.hasToggledState != node.flags.hasToggledState ||
307 object.node.flags.hasImplicitScrolling != node.flags.hasImplicitScrolling
308
309 ) {
310 // The node changed its type. In this case, we cannot reuse the existing
311 // SemanticsObject implementation. Instead, we replace it with a new
312 // instance.
313 SemanticsObject* newSemanticsObject = CreateObject(node, GetWeakPtr());
314 ReplaceSemanticsObject(object, newSemanticsObject, objects_);
315 object = newSemanticsObject;
316 }
317 }
318 }
319 return object;
320 }
321
322 void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* object,
323 NSMutableArray<NSNumber*>* doomed_uids) {
324 [doomed_uids removeObject:@(object.uid)];
325 for (SemanticsObject* child in [object children])
326 VisitObjectsRecursivelyAndRemove(child, doomed_uids);
327 }
328
329 SemanticsObject* AccessibilityBridge::FindNextFocusableIfNecessary() {
330 // This property will be -1 if the focus is outside of the flutter
331 // application. In this case, we should not refocus anything.
332 if (last_focused_semantics_object_id_ == kSemanticObjectIdInvalid) {
333 return nil;
334 }
335
336 // Tries to refocus the previous focused semantics object to avoid random jumps.
337 return FindFirstFocusable(objects_[@(last_focused_semantics_object_id_)]);
338 }
339
340 SemanticsObject* AccessibilityBridge::FindFirstFocusable(SemanticsObject* parent) {
341 SemanticsObject* currentObject = parent ?: objects_[@(kRootNodeId)];
342 if (!currentObject) {
343 return nil;
344 }
345 if (currentObject.isAccessibilityElement) {
346 return currentObject;
347 }
348
349 for (SemanticsObject* child in [currentObject children]) {
350 SemanticsObject* candidate = FindFirstFocusable(child);
351 if (candidate) {
352 return candidate;
353 }
354 }
355 return nil;
356 }
357
358 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
359 NSString* type = annotatedEvent[@"type"];
360 if ([type isEqualToString:@"announce"]) {
361 NSString* message = annotatedEvent[@"data"][@"message"];
362 ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message);
363 }
364 if ([type isEqualToString:@"focus"]) {
365 SemanticsObject* node = objects_[annotatedEvent[@"nodeId"]];
366 ios_delegate_->PostAccessibilityNotification(UIAccessibilityLayoutChangedNotification, node);
367 }
368 }
369
370 fml::WeakPtr<AccessibilityBridge> AccessibilityBridge::GetWeakPtr() {
371 return weak_factory_.GetWeakPtr();
372 }
373
374 void AccessibilityBridge::clearState() {
375 [objects_ removeAllObjects];
376 previous_route_id_ = 0;
377 previous_routes_.clear();
378 view_controller_.viewIfLoaded.accessibilityElements = nil;
379 }
380
381 } // namespace flutter
FlutterReply
NS_ASSUME_NONNULL_BEGIN typedef void(^ FlutterReply)(id _Nullable reply)
FlutterEngine_Internal.h
platform_view
std::unique_ptr< flutter::PlatformViewIOS > platform_view
Definition: FlutterEnginePlatformViewTest.mm:68
textInputPlugin
FlutterTextInputPlugin * textInputPlugin
Definition: FlutterTextInputPluginTest.mm:137
FlutterViewController_Internal.h
kRootNodeId
constexpr int32_t kRootNodeId
Definition: SemanticsObject.h:16
TextInputSemanticsObject.h
FlutterTouchInterceptingView
Definition: FlutterPlatformViews.mm:513
flutter::AccessibilityBridge::AccessibilityBridge
AccessibilityBridge()
Creates a new instance of a accessibility bridge.
Definition: accessibility_bridge.cc:23
accessibility_bridge.h
FlutterBasicMessageChannel
Definition: FlutterChannels.h:38
FlutterCustomAccessibilityAction
Definition: SemanticsObject.h:136
FlutterCustomAccessibilityAction::uid
int32_t uid
Definition: SemanticsObject.h:140
FlutterPlatformViewSemanticsContainer
Definition: SemanticsObject.h:170
FlutterPlatformViewsController
Definition: FlutterPlatformViewsController.h:31
FlutterScrollableSemanticsObject
Definition: SemanticsObject.h:190
FlutterSemanticsObject
Definition: SemanticsObject.h:156
FlutterStandardMessageCodec
Definition: FlutterCodecs.h:209
FlutterSwitchSemanticsObject
Definition: SemanticsObject.h:184
FlutterViewController
Definition: FlutterViewController.h:57
SemanticsObject
Definition: SemanticsObject.h:32
SemanticsObject::parent
SemanticsObject * parent
Definition: SemanticsObject.h:42
SemanticsObject::nativeAccessibility
id nativeAccessibility
Definition: SemanticsObject.h:83
SemanticsObject::uid
int32_t uid
Definition: SemanticsObject.h:36
SemanticsObject::node
flutter::SemanticsNode node
Definition: SemanticsObject.h:57
SemanticsObject::children
NSArray< SemanticsObject * > * children
Definition: SemanticsObject.h:68
TextInputSemanticsObject
Definition: TextInputSemanticsObject.h:20
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
flutter
Definition: accessibility_bridge.h:26
platform_view_ios.h