| Flutter iOS Embedder |

accessibility_bridge.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
6
7 #include <utility>
8
9 #include "flutter/fml/logging.h"
10 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterEngine_Internal.h"
11 #import "flutter/shell/platform/darwin/ios/framework/Source/FlutterViewController_Internal.h"
12 #import "flutter/shell/platform/darwin/ios/framework/Source/TextInputSemanticsObject.h"
13 #import "flutter/shell/platform/darwin/ios/platform_view_ios.h"
14
15 #pragma GCC diagnostic error "-Wundeclared-selector"
16
17 FLUTTER_ASSERT_NOT_ARC
18
19 namespace flutter {
20 namespace {
21
22 constexpr int32_t kSemanticObjectIdInvalid = -1;
23
24 class DefaultIosDelegate : public AccessibilityBridge::IosDelegate {
25 public:
26 bool IsFlutterViewControllerPresentingModalViewController(
27 FlutterViewController* view_controller) override {
28 if (view_controller) {
29 return view_controller.isPresentingViewController;
30 } else {
31 return false;
32 }
33 }
34
35 void PostAccessibilityNotification(UIAccessibilityNotifications notification,
36 id argument) override {
37 UIAccessibilityPostNotification(notification, argument);
38 }
39 };
40 } // namespace
41
42 AccessibilityBridge::AccessibilityBridge(
43 FlutterViewController* view_controller,
44 PlatformViewIOS* platform_view,
45 std::shared_ptr<FlutterPlatformViewsController> platform_views_controller,
46 std::unique_ptr<IosDelegate> ios_delegate)
47 : view_controller_(view_controller),
48 platform_view_(platform_view),
49 platform_views_controller_(std::move(platform_views_controller)),
50 last_focused_semantics_object_id_(kSemanticObjectIdInvalid),
51 objects_([[NSMutableDictionary alloc] init]),
52 previous_routes_({}),
53 ios_delegate_(ios_delegate ? std::move(ios_delegate)
54 : std::make_unique<DefaultIosDelegate>()),
55 weak_factory_(this) {
56 accessibility_channel_.reset([[FlutterBasicMessageChannel alloc]
57 initWithName:@"flutter/accessibility"
58 binaryMessenger:platform_view->GetOwnerViewController().get().engine.binaryMessenger
59 codec:[FlutterStandardMessageCodec sharedInstance]]);
60 [accessibility_channel_.get() setMessageHandler:^(id message, FlutterReply reply) {
61 HandleEvent((NSDictionary*)message);
62 }];
63 }
64
65 AccessibilityBridge::~AccessibilityBridge() {
66 [accessibility_channel_.get() setMessageHandler:nil];
67 clearState();
68 view_controller_.viewIfLoaded.accessibilityElements = nil;
69 }
70
71 UIView<UITextInput>* AccessibilityBridge::textInputView() {
72 return [[platform_view_->GetOwnerViewController().get().engine textInputPlugin] textInputView];
73 }
74
75 void AccessibilityBridge::AccessibilityObjectDidBecomeFocused(int32_t id) {
76 last_focused_semantics_object_id_ = id;
77 [accessibility_channel_.get() sendMessage:@{@"type" : @"didGainFocus", @"nodeId" : @(id)}];
78 }
79
80 void AccessibilityBridge::AccessibilityObjectDidLoseFocus(int32_t id) {
81 if (last_focused_semantics_object_id_ == id) {
82 last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
83 }
84 }
85
86 void AccessibilityBridge::UpdateSemantics(
87 flutter::SemanticsNodeUpdates nodes,
88 const flutter::CustomAccessibilityActionUpdates& actions) {
89 BOOL layoutChanged = NO;
90 BOOL scrollOccured = NO;
91 BOOL needsAnnouncement = NO;
92 for (const auto& entry : actions) {
93 const flutter::CustomAccessibilityAction& action = entry.second;
94 actions_[action.id] = action;
95 }
96 for (const auto& entry : nodes) {
97 const flutter::SemanticsNode& node = entry.second;
98 SemanticsObject* object = GetOrCreateObject(node.id, nodes);
99 layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
100 scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
101 needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
102 [object setSemanticsNode:&node];
103 NSUInteger newChildCount = node.childrenInTraversalOrder.size();
104 NSMutableArray* newChildren =
105 [[[NSMutableArray alloc] initWithCapacity:newChildCount] autorelease];
106 for (NSUInteger i = 0; i < newChildCount; ++i) {
107 SemanticsObject* child = GetOrCreateObject(node.childrenInTraversalOrder[i], nodes);
108 [newChildren addObject:child];
109 }
110 NSMutableArray* newChildrenInHitTestOrder =
111 [[[NSMutableArray alloc] initWithCapacity:newChildCount] autorelease];
112 for (NSUInteger i = 0; i < newChildCount; ++i) {
113 SemanticsObject* child = GetOrCreateObject(node.childrenInHitTestOrder[i], nodes);
114 [newChildrenInHitTestOrder addObject:child];
115 }
116 object.children = newChildren;
117 object.childrenInHitTestOrder = newChildrenInHitTestOrder;
118 if (!node.customAccessibilityActions.empty()) {
119 NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
120 [[[NSMutableArray alloc] init] autorelease];
121 for (int32_t action_id : node.customAccessibilityActions) {
122 flutter::CustomAccessibilityAction& action = actions_[action_id];
123 if (action.overrideId != -1) {
124 // iOS does not support overriding standard actions, so we ignore any
125 // custom actions that have an override id provided.
126 continue;
127 }
128 NSString* label = @(action.label.data());
129 SEL selector = @selector(onCustomAccessibilityAction:);
130 FlutterCustomAccessibilityAction* customAction =
131 [[[FlutterCustomAccessibilityAction alloc] initWithName:label
132 target:object
133 selector:selector] autorelease];
134 customAction.uid = action_id;
135 [accessibilityCustomActions addObject:customAction];
136 }
137 object.accessibilityCustomActions = accessibilityCustomActions;
138 }
139
140 if (needsAnnouncement) {
141 // Try to be more polite - iOS 11+ supports
142 // UIAccessibilitySpeechAttributeQueueAnnouncement which should avoid
143 // interrupting system notifications or other elements.
144 // Expectation: roughly match the behavior of polite announcements on
145 // Android.
146 NSString* announcement =
147 [[[NSString alloc] initWithUTF8String:object.node.label.c_str()] autorelease];
148 UIAccessibilityPostNotification(
149 UIAccessibilityAnnouncementNotification,
150 [[[NSAttributedString alloc] initWithString:announcement
151 attributes:@{
152 UIAccessibilitySpeechAttributeQueueAnnouncement : @YES
153 }] autorelease]);
154 }
155 }
156
157 SemanticsObject* root = objects_.get()[@(kRootNodeId)];
158
159 bool routeChanged = false;
160 SemanticsObject* lastAdded = nil;
161
162 if (root) {
163 if (!view_controller_.view.accessibilityElements) {
164 view_controller_.view.accessibilityElements =
165 @[ [root accessibilityContainer] ?: [NSNull null] ];
166 }
167 NSMutableArray<SemanticsObject*>* newRoutes = [[[NSMutableArray alloc] init] autorelease];
168 [root collectRoutes:newRoutes];
169 // Finds the last route that is not in the previous routes.
170 for (SemanticsObject* route in newRoutes) {
171 if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) ==
172 previous_routes_.end()) {
173 lastAdded = route;
174 }
175 }
176 // If all the routes are in the previous route, get the last route.
177 if (lastAdded == nil && [newRoutes count] > 0) {
178 int index = [newRoutes count] - 1;
179 lastAdded = [newRoutes objectAtIndex:index];
180 }
181 // There are two cases if lastAdded != nil
182 // 1. lastAdded is not in previous routes. In this case,
183 // [lastAdded uid] != previous_route_id_
184 // 2. All new routes are in previous routes and
185 // lastAdded = newRoutes.last.
186 // In the first case, we need to announce new route. In the second case,
187 // we need to announce if one list is shorter than the other.
188 if (lastAdded != nil &&
189 ([lastAdded uid] != previous_route_id_ || [newRoutes count] != previous_routes_.size())) {
190 previous_route_id_ = [lastAdded uid];
191 routeChanged = true;
192 }
193 previous_routes_.clear();
194 for (SemanticsObject* route in newRoutes) {
195 previous_routes_.push_back([route uid]);
196 }
197 } else {
198 view_controller_.viewIfLoaded.accessibilityElements = nil;
199 }
200
201 NSMutableArray<NSNumber*>* doomed_uids = [NSMutableArray arrayWithArray:[objects_ allKeys]];
202 if (root) {
203 VisitObjectsRecursivelyAndRemove(root, doomed_uids);
204 }
205 [objects_ removeObjectsForKeys:doomed_uids];
206
207 for (SemanticsObject* object in [objects_ allValues]) {
208 [object accessibilityBridgeDidFinishUpdate];
209 }
210
211 if (!ios_delegate_->IsFlutterViewControllerPresentingModalViewController(view_controller_)) {
212 layoutChanged = layoutChanged || [doomed_uids count] > 0;
213
214 if (routeChanged) {
215 NSString* routeName = [lastAdded routeName];
216 ios_delegate_->PostAccessibilityNotification(UIAccessibilityScreenChangedNotification,
217 routeName);
218 }
219
220 if (layoutChanged) {
221 SemanticsObject* next = FindNextFocusableIfNecessary();
222 SemanticsObject* lastFocused =
223 [objects_.get() objectForKey:@(last_focused_semantics_object_id_)];
224 // Only specify the focus item if the new focus is different, avoiding double focuses on the
225 // same item. See: https://github.com/flutter/flutter/issues/104176. If there is a route
226 // change, we always refocus.
227 ios_delegate_->PostAccessibilityNotification(
228 UIAccessibilityLayoutChangedNotification,
229 (routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
230 } else if (scrollOccured) {
231 // TODO(chunhtai): figure out what string to use for notification. At this
232 // point, it is guarantee the previous focused object is still in the tree
233 // so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
234 ios_delegate_->PostAccessibilityNotification(
235 UIAccessibilityPageScrolledNotification,
236 FindNextFocusableIfNecessary().nativeAccessibility);
237 }
238 }
239 }
240
241 void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, flutter::SemanticsAction action) {
242 platform_view_->DispatchSemanticsAction(uid, action, {});
243 }
244
245 void AccessibilityBridge::DispatchSemanticsAction(int32_t uid,
246 flutter::SemanticsAction action,
247 fml::MallocMapping args) {
248 platform_view_->DispatchSemanticsAction(uid, action, std::move(args));
249 }
250
251 static void ReplaceSemanticsObject(SemanticsObject* oldObject,
252 SemanticsObject* newObject,
253 NSMutableDictionary<NSNumber*, SemanticsObject*>* objects) {
254 // `newObject` should represent the same id as `oldObject`.
255 FML_DCHECK(oldObject.node.id == newObject.uid);
256 NSNumber* nodeId = @(oldObject.node.id);
257 NSUInteger positionInChildlist = [oldObject.parent.children indexOfObject:oldObject];
258 [[oldObject retain] autorelease];
259 oldObject.children = @[];
260 [oldObject.parent replaceChildAtIndex:positionInChildlist withChild:newObject];
261 [objects removeObjectForKey:nodeId];
262 objects[nodeId] = newObject;
263 }
264
265 static SemanticsObject* CreateObject(const flutter::SemanticsNode& node,
266 const fml::WeakPtr<AccessibilityBridge>& weak_ptr) {
267 if (node.HasFlag(flutter::SemanticsFlags::kIsTextField) &&
268 !node.HasFlag(flutter::SemanticsFlags::kIsReadOnly)) {
269 // Text fields are backed by objects that implement UITextInput.
270 return [[[TextInputSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id] autorelease];
271 } else if (!node.HasFlag(flutter::SemanticsFlags::kIsInMutuallyExclusiveGroup) &&
272 (node.HasFlag(flutter::SemanticsFlags::kHasToggledState) ||
273 node.HasFlag(flutter::SemanticsFlags::kHasCheckedState))) {
274 return [[[FlutterSwitchSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id] autorelease];
275 } else if (node.HasFlag(flutter::SemanticsFlags::kHasImplicitScrolling)) {
276 return [[[FlutterScrollableSemanticsObject alloc] initWithBridge:weak_ptr
277 uid:node.id] autorelease];
278 } else if (node.IsPlatformViewNode()) {
279 return [[[FlutterPlatformViewSemanticsContainer alloc]
280 initWithBridge:weak_ptr
281 uid:node.id
282 platformView:weak_ptr->GetPlatformViewsController()->GetFlutterTouchInterceptingViewByID(
283 node.platformViewId)] autorelease];
284 } else {
285 return [[[FlutterSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id] autorelease];
286 }
287 }
288
289 static bool DidFlagChange(const flutter::SemanticsNode& oldNode,
290 const flutter::SemanticsNode& newNode,
291 SemanticsFlags flag) {
292 return oldNode.HasFlag(flag) != newNode.HasFlag(flag);
293 }
294
295 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
296 flutter::SemanticsNodeUpdates& updates) {
297 SemanticsObject* object = objects_.get()[@(uid)];
298 if (!object) {
299 object = CreateObject(updates[uid], GetWeakPtr());
300 objects_.get()[@(uid)] = object;
301 } else {
302 // Existing node case
303 auto nodeEntry = updates.find(object.node.id);
304 if (nodeEntry != updates.end()) {
305 // There's an update for this node
306 flutter::SemanticsNode node = nodeEntry->second;
307 if (DidFlagChange(object.node, node, flutter::SemanticsFlags::kIsTextField) ||
308 DidFlagChange(object.node, node, flutter::SemanticsFlags::kIsReadOnly) ||
309 DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasCheckedState) ||
310 DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasToggledState) ||
311 DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasImplicitScrolling)) {
312 // The node changed its type. In this case, we cannot reuse the existing
313 // SemanticsObject implementation. Instead, we replace it with a new
314 // instance.
315 SemanticsObject* newSemanticsObject = CreateObject(node, GetWeakPtr());
316 ReplaceSemanticsObject(object, newSemanticsObject, objects_.get());
317 object = newSemanticsObject;
318 }
319 }
320 }
321 return object;
322 }
323
324 void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* object,
325 NSMutableArray<NSNumber*>* doomed_uids) {
326 [doomed_uids removeObject:@(object.uid)];
327 for (SemanticsObject* child in [object children])
328 VisitObjectsRecursivelyAndRemove(child, doomed_uids);
329 }
330
331 SemanticsObject* AccessibilityBridge::FindNextFocusableIfNecessary() {
332 // This property will be -1 if the focus is outside of the flutter
333 // application. In this case, we should not refocus anything.
334 if (last_focused_semantics_object_id_ == kSemanticObjectIdInvalid) {
335 return nil;
336 }
337
338 // Tries to refocus the previous focused semantics object to avoid random jumps.
339 return FindFirstFocusable([objects_.get() objectForKey:@(last_focused_semantics_object_id_)]);
340 }
341
342 SemanticsObject* AccessibilityBridge::FindFirstFocusable(SemanticsObject* parent) {
343 SemanticsObject* currentObject = parent ?: objects_.get()[@(kRootNodeId)];
344 if (!currentObject) {
345 return nil;
346 }
347 if (currentObject.isAccessibilityElement) {
348 return currentObject;
349 }
350
351 for (SemanticsObject* child in [currentObject children]) {
352 SemanticsObject* candidate = FindFirstFocusable(child);
353 if (candidate) {
354 return candidate;
355 }
356 }
357 return nil;
358 }
359
360 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
361 NSString* type = annotatedEvent[@"type"];
362 if ([type isEqualToString:@"announce"]) {
363 NSString* message = annotatedEvent[@"data"][@"message"];
364 ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message);
365 }
366 if ([type isEqualToString:@"focus"]) {
367 SemanticsObject* node = objects_.get()[annotatedEvent[@"nodeId"]];
368 ios_delegate_->PostAccessibilityNotification(UIAccessibilityLayoutChangedNotification, node);
369 }
370 }
371
372 fml::WeakPtr<AccessibilityBridge> AccessibilityBridge::GetWeakPtr() {
373 return weak_factory_.GetWeakPtr();
374 }
375
376 void AccessibilityBridge::clearState() {
377 [objects_ removeAllObjects];
378 previous_route_id_ = 0;
379 previous_routes_.clear();
380 }
381
382 } // namespace flutter
FlutterBasicMessageChannel
Definition: FlutterChannels.h:37
SemanticsObject::parent
SemanticsObject * parent
Definition: SemanticsObject.h:41
FlutterViewController
Definition: FlutterViewController.h:56
FlutterCustomAccessibilityAction
Definition: SemanticsObject.h:134
flutter::AccessibilityBridge::AccessibilityObjectDidLoseFocus
void AccessibilityObjectDidLoseFocus(int32_t id) override
FlutterEngine_Internal.h
flutter::AccessibilityBridge::textInputView
UIView< UITextInput > * textInputView() override
flutter::AccessibilityBridge::clearState
void clearState()
TextInputSemanticsObject.h
platform_view
std::unique_ptr< flutter::PlatformViewIOS > platform_view
Definition: FlutterEnginePlatformViewTest.mm:65
FLUTTER_ASSERT_NOT_ARC
#define FLUTTER_ASSERT_NOT_ARC
Definition: FlutterMacros.h:45
FlutterCustomAccessibilityAction::uid
int32_t uid
Definition: SemanticsObject.h:139
FlutterStandardMessageCodec
Definition: FlutterCodecs.h:209
FlutterSemanticsObject
Definition: SemanticsObject.h:154
flutter::AccessibilityBridge::AccessibilityBridge
AccessibilityBridge()
Creates a new instance of a accessibility bridge.
Definition: accessibility_bridge.cc:23
flutter
Definition: accessibility_bridge.h:28
accessibility_bridge.h
SemanticsObject::node
flutter::SemanticsNode node
Definition: SemanticsObject.h:56
FlutterSwitchSemanticsObject
Definition: SemanticsObject.h:182
flutter::PlatformViewIOS::GetOwnerViewController
fml::WeakNSObject< FlutterViewController > GetOwnerViewController() const
Definition: platform_view_ios.mm:77
kRootNodeId
constexpr int32_t kRootNodeId
Definition: SemanticsObject.h:15
flutter::AccessibilityBridge::AccessibilityObjectDidBecomeFocused
void AccessibilityObjectDidBecomeFocused(int32_t id) override
FlutterReply
NS_ASSUME_NONNULL_BEGIN typedef void(^ FlutterReply)(id _Nullable reply)
SemanticsObject::children
NSArray< SemanticsObject * > * children
Definition: SemanticsObject.h:67
textInputPlugin
FlutterTextInputPlugin * textInputPlugin
Definition: FlutterTextInputPluginTest.mm:90
FlutterViewController_Internal.h
SemanticsObject::nativeAccessibility
id nativeAccessibility
Definition: SemanticsObject.h:82
TextInputSemanticsObject
Definition: TextInputSemanticsObject.h:20
FlutterPlatformViewSemanticsContainer
Definition: SemanticsObject.h:168
SemanticsObject::uid
int32_t uid
Definition: SemanticsObject.h:35
flutter::AccessibilityBridge::DispatchSemanticsAction
void DispatchSemanticsAction(int32_t id, flutter::SemanticsAction action) override
platform_view_ios.h
flutter::AccessibilityBridge::UpdateSemantics
void UpdateSemantics(flutter::SemanticsNodeUpdates nodes, const flutter::CustomAccessibilityActionUpdates &actions)
flutter::AccessibilityBridge::GetWeakPtr
fml::WeakPtr< AccessibilityBridge > GetWeakPtr()
flutter::AccessibilityBridge::HandleEvent
void HandleEvent(NSDictionary< NSString *, id > *annotatedEvent)
FlutterScrollableSemanticsObject
Definition: SemanticsObject.h:188
SemanticsObject
Definition: SemanticsObject.h:30
flutter::AccessibilityBridge::~AccessibilityBridge
~AccessibilityBridge()
Definition: accessibility_bridge.cc:34