| Flutter iOS Embedder |

accessibility_bridge.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
6
7 #include <utility>
8
9 #include "flutter/fml/logging.h"
10 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h"
11 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h"
12 #import "flutter/shell/platform/darwin/ios/framework/Source/TextInputSemanticsObject.h"
13 #import "flutter/shell/platform/darwin/ios/platform_view_ios.h"
14
15 #include "flutter/common/constants.h"
16
17 #pragma GCC diagnostic error "-Wundeclared-selector"
18
19 FLUTTER_ASSERT_ARC
20
21 namespace flutter {
22 namespace {
23
24 constexpr int32_t kSemanticObjectIdInvalid = -1;
25
26 class DefaultIosDelegate : public AccessibilityBridge::IosDelegate {
27 public:
28 bool IsFlutterViewControllerPresentingModalViewController(
29 FlutterViewController* view_controller) override {
30 if (view_controller) {
31 return view_controller.isPresentingViewController;
32 } else {
33 return false;
34 }
35 }
36
37 void PostAccessibilityNotification(UIAccessibilityNotifications notification,
38 id argument) override {
39 UIAccessibilityPostNotification(notification, argument);
40 }
41 };
42 } // namespace
43
44 AccessibilityBridge::AccessibilityBridge(
45 FlutterViewController* view_controller,
46 PlatformViewIOS* platform_view,
47 __weak FlutterPlatformViewsController* platform_views_controller,
48 std::unique_ptr<IosDelegate> ios_delegate)
49 : view_controller_(view_controller),
50 platform_view_(platform_view),
51 platform_views_controller_(platform_views_controller),
52 last_focused_semantics_object_id_(kSemanticObjectIdInvalid),
53 objects_([[NSMutableDictionary alloc] init]),
54 previous_routes_({}),
55 ios_delegate_(ios_delegate ? std::move(ios_delegate)
56 : std::make_unique<DefaultIosDelegate>()),
57 weak_factory_(this) {
58 accessibility_channel_ = [[FlutterBasicMessageChannel alloc]
59 initWithName:@"flutter/accessibility"
60 binaryMessenger:platform_view->GetOwnerViewController().engine.binaryMessenger
61 codec:[FlutterStandardMessageCodec sharedInstance]];
62 [accessibility_channel_ setMessageHandler:^(id message, FlutterReply reply) {
63 HandleEvent((NSDictionary*)message);
64 }];
65 }
66
67 AccessibilityBridge::~AccessibilityBridge() {
68 [accessibility_channel_ setMessageHandler:nil];
69 clearState();
70 }
71
72 UIView<UITextInput>* AccessibilityBridge::textInputView() {
73 return [[platform_view_->GetOwnerViewController().engine textInputPlugin] textInputView];
74 }
75
76 void AccessibilityBridge::AccessibilityObjectDidBecomeFocused(int32_t id) {
77 last_focused_semantics_object_id_ = id;
78 [accessibility_channel_ sendMessage:@{@"type" : @"didGainFocus", @"nodeId" : @(id)}];
79 }
80
81 void AccessibilityBridge::AccessibilityObjectDidLoseFocus(int32_t id) {
82 if (last_focused_semantics_object_id_ == id) {
83 last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
84 }
85 }
86
87 void AccessibilityBridge::UpdateSemantics(
88 flutter::SemanticsNodeUpdates nodes,
89 const flutter::CustomAccessibilityActionUpdates& actions) {
90 BOOL layoutChanged = NO;
91 BOOL scrollOccured = NO;
92 BOOL needsAnnouncement = NO;
93 for (const auto& entry : actions) {
94 const flutter::CustomAccessibilityAction& action = entry.second;
95 actions_[action.id] = action;
96 }
97 for (const auto& entry : nodes) {
98 const flutter::SemanticsNode& node = entry.second;
99 SemanticsObject* object = GetOrCreateObject(node.id, nodes);
100 layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
101 scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
102 needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
103 [object setSemanticsNode:&node];
104 NSUInteger newChildCount = node.childrenInTraversalOrder.size();
105 NSMutableArray* newChildren = [[NSMutableArray alloc] initWithCapacity:newChildCount];
106 for (NSUInteger i = 0; i < newChildCount; ++i) {
107 SemanticsObject* child = GetOrCreateObject(node.childrenInTraversalOrder[i], nodes);
108 [newChildren addObject:child];
109 }
110 NSMutableArray* newChildrenInHitTestOrder =
111 [[NSMutableArray alloc] initWithCapacity:newChildCount];
112 for (NSUInteger i = 0; i < newChildCount; ++i) {
113 SemanticsObject* child = GetOrCreateObject(node.childrenInHitTestOrder[i], nodes);
114 [newChildrenInHitTestOrder addObject:child];
115 }
116 object.children = newChildren;
117 object.childrenInHitTestOrder = newChildrenInHitTestOrder;
118 if (!node.customAccessibilityActions.empty()) {
119 NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
120 [[NSMutableArray alloc] init];
121 for (int32_t action_id : node.customAccessibilityActions) {
122 flutter::CustomAccessibilityAction& action = actions_[action_id];
123 if (action.overrideId != -1) {
124 // iOS does not support overriding standard actions, so we ignore any
125 // custom actions that have an override id provided.
126 continue;
127 }
128 NSString* label = @(action.label.data());
129 SEL selector = @selector(onCustomAccessibilityAction:);
130 FlutterCustomAccessibilityAction* customAction =
131 [[FlutterCustomAccessibilityAction alloc] initWithName:label
132 target:object
133 selector:selector];
134 customAction.uid = action_id;
135 [accessibilityCustomActions addObject:customAction];
136 }
137 object.accessibilityCustomActions = accessibilityCustomActions;
138 }
139
140 if (needsAnnouncement) {
141 // Try to be more polite - iOS 11+ supports
142 // UIAccessibilitySpeechAttributeQueueAnnouncement which should avoid
143 // interrupting system notifications or other elements.
144 // Expectation: roughly match the behavior of polite announcements on
145 // Android.
146 NSString* announcement = [[NSString alloc] initWithUTF8String:object.node.label.c_str()];
147 UIAccessibilityPostNotification(
148 UIAccessibilityAnnouncementNotification,
149 [[NSAttributedString alloc] initWithString:announcement
150 attributes:@{
151 UIAccessibilitySpeechAttributeQueueAnnouncement : @YES
152 }]);
153 }
154 }
155
156 SemanticsObject* root = objects_[@(kRootNodeId)];
157
158 bool routeChanged = false;
159 SemanticsObject* lastAdded = nil;
160
161 if (root) {
162 if (!view_controller_.view.accessibilityElements) {
163 view_controller_.view.accessibilityElements =
164 @[ [root accessibilityContainer] ?: [NSNull null] ];
165 }
166 NSMutableArray<SemanticsObject*>* newRoutes = [[NSMutableArray alloc] init];
167 [root collectRoutes:newRoutes];
168 // Finds the last route that is not in the previous routes.
169 for (SemanticsObject* route in newRoutes) {
170 if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) ==
171 previous_routes_.end()) {
172 lastAdded = route;
173 }
174 }
175 // If all the routes are in the previous route, get the last route.
176 if (lastAdded == nil && [newRoutes count] > 0) {
177 int index = [newRoutes count] - 1;
178 lastAdded = [newRoutes objectAtIndex:index];
179 }
180 // There are two cases if lastAdded != nil
181 // 1. lastAdded is not in previous routes. In this case,
182 // [lastAdded uid] != previous_route_id_
183 // 2. All new routes are in previous routes and
184 // lastAdded = newRoutes.last.
185 // In the first case, we need to announce new route. In the second case,
186 // we need to announce if one list is shorter than the other.
187 if (lastAdded != nil &&
188 ([lastAdded uid] != previous_route_id_ || [newRoutes count] != previous_routes_.size())) {
189 previous_route_id_ = [lastAdded uid];
190 routeChanged = true;
191 }
192 previous_routes_.clear();
193 for (SemanticsObject* route in newRoutes) {
194 previous_routes_.push_back([route uid]);
195 }
196 } else {
197 view_controller_.viewIfLoaded.accessibilityElements = nil;
198 }
199
200 NSMutableArray<NSNumber*>* doomed_uids = [NSMutableArray arrayWithArray:objects_.allKeys];
201 if (root) {
202 VisitObjectsRecursivelyAndRemove(root, doomed_uids);
203 }
204 [objects_ removeObjectsForKeys:doomed_uids];
205
206 for (SemanticsObject* object in objects_.allValues) {
207 [object accessibilityBridgeDidFinishUpdate];
208 }
209
210 if (!ios_delegate_->IsFlutterViewControllerPresentingModalViewController(view_controller_)) {
211 layoutChanged = layoutChanged || [doomed_uids count] > 0;
212
213 if (routeChanged) {
214 NSString* routeName = [lastAdded routeName];
215 ios_delegate_->PostAccessibilityNotification(UIAccessibilityScreenChangedNotification,
216 routeName);
217 }
218
219 if (layoutChanged) {
220 SemanticsObject* next = FindNextFocusableIfNecessary();
221 SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
222 // Only specify the focus item if the new focus is different, avoiding double focuses on the
223 // same item. See: https://github.com/flutter/flutter/issues/104176. If there is a route
224 // change, we always refocus.
225 ios_delegate_->PostAccessibilityNotification(
226 UIAccessibilityLayoutChangedNotification,
227 (routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
228 } else if (scrollOccured) {
229 // TODO(chunhtai): figure out what string to use for notification. At this
230 // point, it is guarantee the previous focused object is still in the tree
231 // so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
232 ios_delegate_->PostAccessibilityNotification(
233 UIAccessibilityPageScrolledNotification,
234 FindNextFocusableIfNecessary().nativeAccessibility);
235 }
236 }
237 }
238
239 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
240 flutter::SemanticsAction action) {
241 // TODO(team-ios): Remove implicit view assumption.
242 // https://github.com/flutter/flutter/issues/142845
243 platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action, {});
244 }
245
246 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
247 flutter::SemanticsAction action,
248 fml::MallocMapping args) {
249 // TODO(team-ios): Remove implicit view assumption.
250 // https://github.com/flutter/flutter/issues/142845
251 platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action,
252 std::move(args));
253 }
254
255 static void ReplaceSemanticsObject(SemanticsObject* oldObject,
256 SemanticsObject* newObject,
257 NSMutableDictionary<NSNumber*, SemanticsObject*>* objects) {
258 // `newObject` should represent the same id as `oldObject`.
259 FML_DCHECK(oldObject.node.id == newObject.uid);
260 NSNumber* nodeId = @(oldObject.node.id);
261 NSUInteger positionInChildlist = [oldObject.parent.children indexOfObject:oldObject];
262 oldObject.children = @[];
263 [oldObject.parent replaceChildAtIndex:positionInChildlist withChild:newObject];
264 [objects removeObjectForKey:nodeId];
265 objects[nodeId] = newObject;
266 }
267
268 static SemanticsObject* CreateObject(const flutter::SemanticsNode& node,
269 const fml::WeakPtr<AccessibilityBridge>& weak_ptr) {
270 if (node.HasFlag(flutter::SemanticsFlags::kIsTextField) &&
271 !node.HasFlag(flutter::SemanticsFlags::kIsReadOnly)) {
272 // Text fields are backed by objects that implement UITextInput.
273 return [[TextInputSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
274 } else if (!node.HasFlag(flutter::SemanticsFlags::kIsInMutuallyExclusiveGroup) &&
275 (node.HasFlag(flutter::SemanticsFlags::kHasToggledState) ||
276 node.HasFlag(flutter::SemanticsFlags::kHasCheckedState))) {
277 return [[FlutterSwitchSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
278 } else if (node.HasFlag(flutter::SemanticsFlags::kHasImplicitScrolling)) {
279 return [[FlutterScrollableSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
280 } else if (node.IsPlatformViewNode()) {
281 FlutterPlatformViewsController* platformViewsController =
282 weak_ptr->GetPlatformViewsController();
283 FlutterTouchInterceptingView* touchInterceptingView =
284 [platformViewsController flutterTouchInterceptingViewForId:node.platformViewId];
285 return [[FlutterPlatformViewSemanticsContainer alloc] initWithBridge:weak_ptr
286 uid:node.id
287 platformView:touchInterceptingView];
288 } else {
289 return [[FlutterSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
290 }
291 }
292
293 static bool DidFlagChange(const flutter::SemanticsNode& oldNode,
294 const flutter::SemanticsNode& newNode,
295 SemanticsFlags flag) {
296 return oldNode.HasFlag(flag) != newNode.HasFlag(flag);
297 }
298
299 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
300 flutter::SemanticsNodeUpdates& updates) {
301 SemanticsObject* object = objects_[@(uid)];
302 if (!object) {
303 object = CreateObject(updates[uid], GetWeakPtr());
304 objects_[@(uid)] = object;
305 } else {
306 // Existing node case
307 auto nodeEntry = updates.find(object.node.id);
308 if (nodeEntry != updates.end()) {
309 // There's an update for this node
310 flutter::SemanticsNode node = nodeEntry->second;
311 if (DidFlagChange(object.node, node, flutter::SemanticsFlags::kIsTextField) ||
312 DidFlagChange(object.node, node, flutter::SemanticsFlags::kIsReadOnly) ||
313 DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasCheckedState) ||
314 DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasToggledState) ||
315 DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasImplicitScrolling)) {
316 // The node changed its type. In this case, we cannot reuse the existing
317 // SemanticsObject implementation. Instead, we replace it with a new
318 // instance.
319 SemanticsObject* newSemanticsObject = CreateObject(node, GetWeakPtr());
320 ReplaceSemanticsObject(object, newSemanticsObject, objects_);
321 object = newSemanticsObject;
322 }
323 }
324 }
325 return object;
326 }
327
328 void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* object,
329 NSMutableArray<NSNumber*>* doomed_uids) {
330 [doomed_uids removeObject:@(object.uid)];
331 for (SemanticsObject* child in [object children])
332 VisitObjectsRecursivelyAndRemove(child, doomed_uids);
333 }
334
335 SemanticsObject* AccessibilityBridge::FindNextFocusableIfNecessary() {
336 // This property will be -1 if the focus is outside of the flutter
337 // application. In this case, we should not refocus anything.
338 if (last_focused_semantics_object_id_ == kSemanticObjectIdInvalid) {
339 return nil;
340 }
341
342 // Tries to refocus the previous focused semantics object to avoid random jumps.
343 return FindFirstFocusable(objects_[@(last_focused_semantics_object_id_)]);
344 }
345
346 SemanticsObject* AccessibilityBridge::FindFirstFocusable(SemanticsObject* parent) {
347 SemanticsObject* currentObject = parent ?: objects_[@(kRootNodeId)];
348 if (!currentObject) {
349 return nil;
350 }
351 if (currentObject.isAccessibilityElement) {
352 return currentObject;
353 }
354
355 for (SemanticsObject* child in [currentObject children]) {
356 SemanticsObject* candidate = FindFirstFocusable(child);
357 if (candidate) {
358 return candidate;
359 }
360 }
361 return nil;
362 }
363
364 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
365 NSString* type = annotatedEvent[@"type"];
366 if ([type isEqualToString:@"announce"]) {
367 NSString* message = annotatedEvent[@"data"][@"message"];
368 ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message);
369 }
370 if ([type isEqualToString:@"focus"]) {
371 SemanticsObject* node = objects_[annotatedEvent[@"nodeId"]];
372 ios_delegate_->PostAccessibilityNotification(UIAccessibilityLayoutChangedNotification, node);
373 }
374 }
375
376 fml::WeakPtr<AccessibilityBridge> AccessibilityBridge::GetWeakPtr() {
377 return weak_factory_.GetWeakPtr();
378 }
379
380 void AccessibilityBridge::clearState() {
381 [objects_ removeAllObjects];
382 previous_route_id_ = 0;
383 previous_routes_.clear();
384 view_controller_.viewIfLoaded.accessibilityElements = nil;
385 }
386
387 } // namespace flutter
FlutterReply
NS_ASSUME_NONNULL_BEGIN typedef void(^ FlutterReply)(id _Nullable reply)
FlutterEngine_Internal.h
platform_view
std::unique_ptr< flutter::PlatformViewIOS > platform_view
Definition: FlutterEnginePlatformViewTest.mm:68
textInputPlugin
FlutterTextInputPlugin * textInputPlugin
Definition: FlutterTextInputPluginTest.mm:93
FlutterViewController_Internal.h
kRootNodeId
constexpr int32_t kRootNodeId
Definition: SemanticsObject.h:16
TextInputSemanticsObject.h
FlutterTouchInterceptingView
Definition: FlutterPlatformViews.mm:515
flutter::AccessibilityBridge::AccessibilityBridge
AccessibilityBridge()
Creates a new instance of a accessibility bridge.
Definition: accessibility_bridge.cc:23
accessibility_bridge.h
FlutterBasicMessageChannel
Definition: FlutterChannels.h:38
FlutterCustomAccessibilityAction
Definition: SemanticsObject.h:136
FlutterCustomAccessibilityAction::uid
int32_t uid
Definition: SemanticsObject.h:140
FlutterPlatformViewSemanticsContainer
Definition: SemanticsObject.h:170
FlutterPlatformViewsController
Definition: FlutterPlatformViewsController.h:31
FlutterScrollableSemanticsObject
Definition: SemanticsObject.h:190
FlutterSemanticsObject
Definition: SemanticsObject.h:156
FlutterStandardMessageCodec
Definition: FlutterCodecs.h:209
FlutterSwitchSemanticsObject
Definition: SemanticsObject.h:184
FlutterViewController
Definition: FlutterViewController.h:57
SemanticsObject
Definition: SemanticsObject.h:32
SemanticsObject::parent
SemanticsObject * parent
Definition: SemanticsObject.h:42
SemanticsObject::nativeAccessibility
id nativeAccessibility
Definition: SemanticsObject.h:83
SemanticsObject::uid
int32_t uid
Definition: SemanticsObject.h:36
SemanticsObject::node
flutter::SemanticsNode node
Definition: SemanticsObject.h:57
SemanticsObject::children
NSArray< SemanticsObject * > * children
Definition: SemanticsObject.h:68
TextInputSemanticsObject
Definition: TextInputSemanticsObject.h:20
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
flutter
Definition: accessibility_bridge.h:26
platform_view_ios.h