| Flutter iOS Embedder |

accessibility_bridge.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
6
7 #include <utility>
8
9 #include "flutter/fml/logging.h"
10 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h"
11 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h"
12 #import "flutter/shell/platform/darwin/ios/framework/Source/TextInputSemanticsObject.h"
13 #import "flutter/shell/platform/darwin/ios/platform_view_ios.h"
14
15 #include "flutter/common/constants.h"
16
17 #pragma GCC diagnostic error "-Wundeclared-selector"
18
19 FLUTTER_ASSERT_ARC
20
21 namespace flutter {
22 namespace {
23
24 class DefaultIosDelegate : public AccessibilityBridge::IosDelegate {
25 public:
26 bool IsFlutterViewControllerPresentingModalViewController(
27 FlutterViewController* view_controller) override {
28 if (view_controller) {
29 return view_controller.isPresentingViewController;
30 } else {
31 return false;
32 }
33 }
34
35 void PostAccessibilityNotification(UIAccessibilityNotifications notification,
36 id argument) override {
37 UIAccessibilityPostNotification(notification, argument);
38 }
39 };
40 } // namespace
41
42 AccessibilityBridge::AccessibilityBridge(
43 FlutterViewController* view_controller,
44 PlatformViewIOS* platform_view,
45 __weak FlutterPlatformViewsController* platform_views_controller,
46 std::unique_ptr<IosDelegate> ios_delegate)
47 : view_controller_(view_controller),
48 platform_view_(platform_view),
49 platform_views_controller_(platform_views_controller),
50 objects_([[NSMutableDictionary alloc] init]),
51 previous_routes_({}),
52 ios_delegate_(ios_delegate ? std::move(ios_delegate)
53 : std::make_unique<DefaultIosDelegate>()),
54 weak_factory_(this) {
55 accessibility_channel_ = [[FlutterBasicMessageChannel alloc]
56 initWithName:@"flutter/accessibility"
57 binaryMessenger:platform_view->GetOwnerViewController().engine.binaryMessenger
58 codec:[FlutterStandardMessageCodec sharedInstance]];
59 [accessibility_channel_ setMessageHandler:^(id message, FlutterReply reply) {
60 HandleEvent((NSDictionary*)message);
61 }];
62 }
63
64 AccessibilityBridge::~AccessibilityBridge() {
65 [accessibility_channel_ setMessageHandler:nil];
66 clearState();
67 }
68
69 UIView<UITextInput>* AccessibilityBridge::textInputView() {
70 return [[platform_view_->GetOwnerViewController().engine textInputPlugin] textInputView];
71 }
72
73 void AccessibilityBridge::AccessibilityObjectDidBecomeFocused(int32_t id) {
74 last_focused_semantics_object_id_ = id;
75 [accessibility_channel_ sendMessage:@{@"type" : @"didGainFocus", @"nodeId" : @(id)}];
76 }
77
78 void AccessibilityBridge::AccessibilityObjectDidLoseFocus(int32_t id) {
79 if (last_focused_semantics_object_id_ == id) {
80 last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
81 }
82 }
83
84 void AccessibilityBridge::UpdateSemantics(
85 flutter::SemanticsNodeUpdates nodes,
86 const flutter::CustomAccessibilityActionUpdates& actions) {
87 BOOL layoutChanged = NO;
88 BOOL scrollOccured = NO;
89 BOOL needsAnnouncement = NO;
90 for (const auto& entry : actions) {
91 const flutter::CustomAccessibilityAction& action = entry.second;
92 actions_[action.id] = action;
93 }
94 for (const auto& entry : nodes) {
95 const flutter::SemanticsNode& node = entry.second;
96 SemanticsObject* object = GetOrCreateObject(node.id, nodes);
97 layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
98 scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
99 needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
100 [object setSemanticsNode:&node];
101 NSUInteger newChildCount = node.childrenInTraversalOrder.size();
102 NSMutableArray* newChildren = [[NSMutableArray alloc] initWithCapacity:newChildCount];
103 for (NSUInteger i = 0; i < newChildCount; ++i) {
104 SemanticsObject* child = GetOrCreateObject(node.childrenInTraversalOrder[i], nodes);
105 [newChildren addObject:child];
106 }
107 NSMutableArray* newChildrenInHitTestOrder =
108 [[NSMutableArray alloc] initWithCapacity:newChildCount];
109 for (NSUInteger i = 0; i < newChildCount; ++i) {
110 SemanticsObject* child = GetOrCreateObject(node.childrenInHitTestOrder[i], nodes);
111 [newChildrenInHitTestOrder addObject:child];
112 }
113 object.children = newChildren;
114 object.childrenInHitTestOrder = newChildrenInHitTestOrder;
115 if (!node.customAccessibilityActions.empty()) {
116 NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
117 [[NSMutableArray alloc] init];
118 for (int32_t action_id : node.customAccessibilityActions) {
119 flutter::CustomAccessibilityAction& action = actions_[action_id];
120 if (action.overrideId != -1) {
121 // iOS does not support overriding standard actions, so we ignore any
122 // custom actions that have an override id provided.
123 continue;
124 }
125 NSString* label = @(action.label.data());
126 SEL selector = @selector(onCustomAccessibilityAction:);
127 FlutterCustomAccessibilityAction* customAction =
128 [[FlutterCustomAccessibilityAction alloc] initWithName:label
129 target:object
130 selector:selector];
131 customAction.uid = action_id;
132 [accessibilityCustomActions addObject:customAction];
133 }
134 object.accessibilityCustomActions = accessibilityCustomActions;
135 }
136
137 if (needsAnnouncement) {
138 // Try to be more polite - iOS 11+ supports
139 // UIAccessibilitySpeechAttributeQueueAnnouncement which should avoid
140 // interrupting system notifications or other elements.
141 // Expectation: roughly match the behavior of polite announcements on
142 // Android.
143 NSString* announcement = [[NSString alloc] initWithUTF8String:object.node.label.c_str()];
144 UIAccessibilityPostNotification(
145 UIAccessibilityAnnouncementNotification,
146 [[NSAttributedString alloc] initWithString:announcement
147 attributes:@{
148 UIAccessibilitySpeechAttributeQueueAnnouncement : @YES
149 }]);
150 }
151 }
152
153 SemanticsObject* root = objects_[@(kRootNodeId)];
154
155 bool routeChanged = false;
156 SemanticsObject* lastAdded = nil;
157
158 if (root) {
159 if (!view_controller_.view.accessibilityElements) {
160 view_controller_.view.accessibilityElements =
161 @[ [root accessibilityContainer] ?: [NSNull null] ];
162 }
163 NSMutableArray<SemanticsObject*>* newRoutes = [[NSMutableArray alloc] init];
164 [root collectRoutes:newRoutes];
165 // Finds the last route that is not in the previous routes.
166 for (SemanticsObject* route in newRoutes) {
167 if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) ==
168 previous_routes_.end()) {
169 lastAdded = route;
170 }
171 }
172 // If all the routes are in the previous route, get the last route.
173 if (lastAdded == nil && [newRoutes count] > 0) {
174 int index = [newRoutes count] - 1;
175 lastAdded = [newRoutes objectAtIndex:index];
176 }
177 // There are two cases if lastAdded != nil
178 // 1. lastAdded is not in previous routes. In this case,
179 // [lastAdded uid] != previous_route_id_
180 // 2. All new routes are in previous routes and
181 // lastAdded = newRoutes.last.
182 // In the first case, we need to announce new route. In the second case,
183 // we need to announce if one list is shorter than the other.
184 if (lastAdded != nil &&
185 ([lastAdded uid] != previous_route_id_ || [newRoutes count] != previous_routes_.size())) {
186 previous_route_id_ = [lastAdded uid];
187 routeChanged = true;
188 }
189 previous_routes_.clear();
190 for (SemanticsObject* route in newRoutes) {
191 previous_routes_.push_back([route uid]);
192 }
193 } else {
194 view_controller_.viewIfLoaded.accessibilityElements = nil;
195 }
196
197 NSMutableArray<NSNumber*>* doomed_uids = [NSMutableArray arrayWithArray:objects_.allKeys];
198 if (root) {
199 VisitObjectsRecursivelyAndRemove(root, doomed_uids);
200 }
201 [objects_ removeObjectsForKeys:doomed_uids];
202
203 for (SemanticsObject* object in objects_.allValues) {
204 [object accessibilityBridgeDidFinishUpdate];
205 }
206
207 if (!ios_delegate_->IsFlutterViewControllerPresentingModalViewController(view_controller_)) {
208 layoutChanged = layoutChanged || [doomed_uids count] > 0;
209
210 if (routeChanged) {
211 NSString* routeName = [lastAdded routeName];
212 ios_delegate_->PostAccessibilityNotification(UIAccessibilityScreenChangedNotification,
213 routeName);
214 }
215
216 if (layoutChanged) {
217 SemanticsObject* next = FindNextFocusableIfNecessary();
218 SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
219 // Only specify the focus item if the new focus is different, avoiding double focuses on the
220 // same item. See: https://github.com/flutter/flutter/issues/104176. If there is a route
221 // change, we always refocus.
222 ios_delegate_->PostAccessibilityNotification(
223 UIAccessibilityLayoutChangedNotification,
224 (routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
225 } else if (scrollOccured) {
226 // TODO(chunhtai): figure out what string to use for notification. At this
227 // point, it is guarantee the previous focused object is still in the tree
228 // so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
229 ios_delegate_->PostAccessibilityNotification(
230 UIAccessibilityPageScrolledNotification,
231 FindNextFocusableIfNecessary().nativeAccessibility);
232 }
233 }
234 }
235
236 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
237 flutter::SemanticsAction action) {
238 // TODO(team-ios): Remove implicit view assumption.
239 // https://github.com/flutter/flutter/issues/142845
240 platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action, {});
241 }
242
243 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
244 flutter::SemanticsAction action,
245 fml::MallocMapping args) {
246 // TODO(team-ios): Remove implicit view assumption.
247 // https://github.com/flutter/flutter/issues/142845
248 platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action,
249 std::move(args));
250 }
251
252 static void ReplaceSemanticsObject(SemanticsObject* oldObject,
253 SemanticsObject* newObject,
254 NSMutableDictionary<NSNumber*, SemanticsObject*>* objects) {
255 // `newObject` should represent the same id as `oldObject`.
256 FML_DCHECK(oldObject.node.id == newObject.uid);
257 NSNumber* nodeId = @(oldObject.node.id);
258 NSUInteger positionInChildlist = [oldObject.parent.children indexOfObject:oldObject];
259 oldObject.children = @[];
260 [oldObject.parent replaceChildAtIndex:positionInChildlist withChild:newObject];
261 [objects removeObjectForKey:nodeId];
262 objects[nodeId] = newObject;
263 }
264
265 static SemanticsObject* CreateObject(const flutter::SemanticsNode& node,
266 const fml::WeakPtr<AccessibilityBridge>& weak_ptr) {
267 if (node.flags.isTextField && !node.flags.isReadOnly) {
268 // Text fields are backed by objects that implement UITextInput.
269 return [[TextInputSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
270 } else if (!node.flags.isInMutuallyExclusiveGroup &&
271 (node.flags.isToggled != flutter::SemanticsTristate::kNone ||
272 node.flags.isChecked != flutter::SemanticsCheckState::kNone)) {
273 return [[FlutterSwitchSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
274 } else if (node.flags.hasImplicitScrolling) {
275 return [[FlutterScrollableSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
276 } else if (node.IsPlatformViewNode()) {
277 FlutterPlatformViewsController* platformViewsController =
278 weak_ptr->GetPlatformViewsController();
279 FlutterTouchInterceptingView* touchInterceptingView =
280 [platformViewsController flutterTouchInterceptingViewForId:node.platformViewId];
281 return [[FlutterPlatformViewSemanticsContainer alloc] initWithBridge:weak_ptr
282 uid:node.id
283 platformView:touchInterceptingView];
284 } else {
285 return [[FlutterSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
286 }
287 }
288
289 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
290 flutter::SemanticsNodeUpdates& updates) {
291 SemanticsObject* object = objects_[@(uid)];
292 if (!object) {
293 object = CreateObject(updates[uid], GetWeakPtr());
294 objects_[@(uid)] = object;
295 } else {
296 // Existing node case
297 auto nodeEntry = updates.find(object.node.id);
298 if (nodeEntry != updates.end()) {
299 // There's an update for this node
300 flutter::SemanticsNode node = nodeEntry->second;
301 if (object.node.flags.isTextField != node.flags.isTextField ||
302 object.node.flags.isReadOnly != node.flags.isReadOnly ||
303 (object.node.flags.isChecked == flutter::SemanticsCheckState::kNone) !=
304 (node.flags.isChecked == flutter::SemanticsCheckState::kNone) ||
305 ((object.node.flags.isToggled == flutter::SemanticsTristate::kNone) !=
306 (node.flags.isToggled == flutter::SemanticsTristate::kNone)) ||
307 object.node.flags.hasImplicitScrolling != node.flags.hasImplicitScrolling
308
309 ) {
310 // The node changed its type. In this case, we cannot reuse the existing
311 // SemanticsObject implementation. Instead, we replace it with a new
312 // instance.
313 SemanticsObject* newSemanticsObject = CreateObject(node, GetWeakPtr());
314 ReplaceSemanticsObject(object, newSemanticsObject, objects_);
315 object = newSemanticsObject;
316 }
317 }
318 }
319 return object;
320 }
321
322 void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* object,
323 NSMutableArray<NSNumber*>* doomed_uids) {
324 [doomed_uids removeObject:@(object.uid)];
325 for (SemanticsObject* child in [object children])
326 VisitObjectsRecursivelyAndRemove(child, doomed_uids);
327 }
328
329 SemanticsObject* AccessibilityBridge::FindNextFocusableIfNecessary() {
330 // This property will be -1 if the focus is outside of the flutter
331 // application. In this case, we should not refocus anything.
332 if (last_focused_semantics_object_id_ == kSemanticObjectIdInvalid) {
333 return nil;
334 }
335
336 // Tries to refocus the previous focused semantics object to avoid random jumps.
337 return FindFirstFocusable(objects_[@(last_focused_semantics_object_id_)]);
338 }
339
340 SemanticsObject* AccessibilityBridge::FindFirstFocusable(SemanticsObject* parent) {
341 SemanticsObject* currentObject = parent ?: objects_[@(kRootNodeId)];
342 if (!currentObject) {
343 return nil;
344 }
345 if (currentObject.isAccessibilityElement) {
346 return currentObject;
347 }
348
349 for (SemanticsObject* child in [currentObject children]) {
350 SemanticsObject* candidate = FindFirstFocusable(child);
351 if (candidate) {
352 return candidate;
353 }
354 }
355 return nil;
356 }
357
358 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
359 NSString* type = annotatedEvent[@"type"];
360 if ([type isEqualToString:@"announce"]) {
361 NSString* message = annotatedEvent[@"data"][@"message"];
362 ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message);
363 }
364 if ([type isEqualToString:@"focus"]) {
365 SemanticsObject* node = objects_[annotatedEvent[@"nodeId"]];
366 ios_delegate_->PostAccessibilityNotification(UIAccessibilityLayoutChangedNotification, node);
367 }
368 }
369
370 fml::WeakPtr<AccessibilityBridge> AccessibilityBridge::GetWeakPtr() {
371 return weak_factory_.GetWeakPtr();
372 }
373
374 void AccessibilityBridge::clearState() {
375 [objects_ removeAllObjects];
376 previous_route_id_ = 0;
377 previous_routes_.clear();
378 view_controller_.viewIfLoaded.accessibilityElements = nil;
379 }
380
381 } // namespace flutter
FlutterReply
NS_ASSUME_NONNULL_BEGIN typedef void(^ FlutterReply)(id _Nullable reply)
FlutterEngine_Internal.h
platform_view
std::unique_ptr< flutter::PlatformViewIOS > platform_view
Definition: FlutterEnginePlatformViewTest.mm:68
textInputPlugin
FlutterTextInputPlugin * textInputPlugin
Definition: FlutterTextInputPluginTest.mm:137
FlutterViewController_Internal.h
kRootNodeId
constexpr int32_t kRootNodeId
Definition: SemanticsObject.h:16
TextInputSemanticsObject.h
FlutterTouchInterceptingView
Definition: FlutterPlatformViews.mm:513
flutter::AccessibilityBridge::AccessibilityBridge
AccessibilityBridge()
Creates a new instance of a accessibility bridge.
Definition: accessibility_bridge.cc:23
accessibility_bridge.h
FlutterBasicMessageChannel
Definition: FlutterChannels.h:38
FlutterCustomAccessibilityAction
Definition: SemanticsObject.h:136
FlutterCustomAccessibilityAction::uid
int32_t uid
Definition: SemanticsObject.h:140
FlutterPlatformViewSemanticsContainer
Definition: SemanticsObject.h:170
FlutterPlatformViewsController
Definition: FlutterPlatformViewsController.h:31
FlutterScrollableSemanticsObject
Definition: SemanticsObject.h:190
FlutterSemanticsObject
Definition: SemanticsObject.h:156
FlutterStandardMessageCodec
Definition: FlutterCodecs.h:209
FlutterSwitchSemanticsObject
Definition: SemanticsObject.h:184
FlutterViewController
Definition: FlutterViewController.h:57
SemanticsObject
Definition: SemanticsObject.h:32
SemanticsObject::parent
SemanticsObject * parent
Definition: SemanticsObject.h:42
SemanticsObject::nativeAccessibility
id nativeAccessibility
Definition: SemanticsObject.h:83
SemanticsObject::uid
int32_t uid
Definition: SemanticsObject.h:36
SemanticsObject::node
flutter::SemanticsNode node
Definition: SemanticsObject.h:57
SemanticsObject::children
NSArray< SemanticsObject * > * children
Definition: SemanticsObject.h:68
TextInputSemanticsObject
Definition: TextInputSemanticsObject.h:20
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
flutter
Definition: accessibility_bridge.h:26
platform_view_ios.h