| Flutter Windows Embedder |

accessibility_bridge_windows_unittests.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "flutter/shell/platform/windows/accessibility_bridge_windows.h"
6
7 #include <comdef.h>
8 #include <comutil.h>
9 #include <oleacc.h>
10
11 #include <vector>
12
13 #include "flutter/fml/macros.h"
14 #include "flutter/shell/platform/embedder/embedder.h"
15 #include "flutter/shell/platform/embedder/test_utils/proc_table_replacement.h"
16 #include "flutter/shell/platform/windows/flutter_platform_node_delegate_windows.h"
17 #include "flutter/shell/platform/windows/flutter_windows_engine.h"
18 #include "flutter/shell/platform/windows/flutter_windows_view.h"
19 #include "flutter/shell/platform/windows/testing/engine_modifier.h"
20 #include "flutter/shell/platform/windows/testing/mock_window_binding_handler.h"
21 #include "flutter/shell/platform/windows/testing/test_keyboard.h"
22 #include "gmock/gmock.h"
23 #include "gtest/gtest.h"
24
25 namespace flutter {
26 namespace testing {
27
28 namespace {
29 using ::testing::NiceMock;
30
31 // A structure representing a Win32 MSAA event targeting a specified node.
32 struct MsaaEvent {
33 std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate;
34 ax::mojom::Event event_type;
35 };
36
37 // Accessibility bridge delegate that captures events dispatched to the OS.
38 class AccessibilityBridgeWindowsSpy : public AccessibilityBridgeWindows {
39 public:
40 using AccessibilityBridgeWindows::OnAccessibilityEvent;
41
42 explicit AccessibilityBridgeWindowsSpy(FlutterWindowsEngine* engine,
43 FlutterWindowsView* view)
44 : AccessibilityBridgeWindows(view) {}
45
46 void DispatchWinAccessibilityEvent(
47 std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate,
48 ax::mojom::Event event_type) override {
49 dispatched_events_.push_back({node_delegate, event_type});
50 }
51
52 void SetFocus(std::shared_ptr<FlutterPlatformNodeDelegateWindows>
53 node_delegate) override {
54 focused_nodes_.push_back(std::move(node_delegate));
55 }
56
57 void ResetRecords() {
58 dispatched_events_.clear();
59 focused_nodes_.clear();
60 }
61
62 const std::vector<MsaaEvent>& dispatched_events() const {
63 return dispatched_events_;
64 }
65
66 const std::vector<int32_t> focused_nodes() const {
67 std::vector<int32_t> ids;
68 std::transform(focused_nodes_.begin(), focused_nodes_.end(),
69 std::back_inserter(ids),
70 [](std::shared_ptr<FlutterPlatformNodeDelegate> node) {
71 return node->GetAXNode()->id();
72 });
73 return ids;
74 }
75
76 protected:
77 std::weak_ptr<FlutterPlatformNodeDelegate> GetFocusedNode() override {
78 return focused_nodes_.back();
79 }
80
81 private:
82 std::vector<MsaaEvent> dispatched_events_;
83 std::vector<std::shared_ptr<FlutterPlatformNodeDelegate>> focused_nodes_;
84
85 FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridgeWindowsSpy);
86 };
87
88 // A FlutterWindowsView whose accessibility bridge is an
89 // AccessibilityBridgeWindowsSpy.
90 class FlutterWindowsViewSpy : public FlutterWindowsView {
91 public:
92 FlutterWindowsViewSpy(FlutterWindowsEngine* engine,
93 std::unique_ptr<WindowBindingHandler> handler)
94 : FlutterWindowsView(kImplicitViewId, engine, std::move(handler)) {}
95
96 protected:
97 virtual std::shared_ptr<AccessibilityBridgeWindows>
98 CreateAccessibilityBridge() override {
99 return std::make_shared<AccessibilityBridgeWindowsSpy>(GetEngine(), this);
100 }
101
102 private:
103 FML_DISALLOW_COPY_AND_ASSIGN(FlutterWindowsViewSpy);
104 };
105
106 // Returns an engine instance configured with dummy project path values, and
107 // overridden methods for sending platform messages, so that the engine can
108 // respond as if the framework were connected.
109 std::unique_ptr<FlutterWindowsEngine> GetTestEngine() {
110 FlutterDesktopEngineProperties properties = {};
111 properties.assets_path = L"C:\\foo\\flutter_assets";
112 properties.icu_data_path = L"C:\\foo\\icudtl.dat";
113 properties.aot_library_path = L"C:\\foo\\aot.so";
114 FlutterProjectBundle project(properties);
115 auto engine = std::make_unique<FlutterWindowsEngine>(project);
116
117 EngineModifier modifier(engine.get());
118 modifier.embedder_api().UpdateSemanticsEnabled =
119 [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) {
120 return kSuccess;
121 };
122
123 MockEmbedderApiForKeyboard(modifier,
124 std::make_shared<MockKeyResponseController>());
125
126 engine->Run();
127 return engine;
128 }
129
130 // Populates the AXTree associated with the specified bridge with test data.
131 //
132 // node0
133 // / \
134 // node1 node2
135 // / \
136 // node3 node4
137 //
138 // node0 and node2 are grouping nodes. node1 and node2 are static text nodes.
139 // node4 is a static text node with no text, and hence has the "ignored" state.
140 void PopulateAXTree(std::shared_ptr<AccessibilityBridge> bridge) {
141 // Add node 0: root.
142 FlutterSemanticsNode2 node0{sizeof(FlutterSemanticsNode2), 0};
143 auto empty_flags = FlutterSemanticsFlags{};
144 std::vector<int32_t> node0_children{1, 2};
145 node0.child_count = node0_children.size();
146 node0.children_in_traversal_order = node0_children.data();
147 node0.children_in_hit_test_order = node0_children.data();
148 node0.flags2 = &empty_flags;
149
150 // Add node 1: text child of node 0.
151 FlutterSemanticsNode2 node1{sizeof(FlutterSemanticsNode2), 1};
152 node1.label = "prefecture";
153 node1.value = "Kyoto";
154 node1.flags2 = &empty_flags;
155
156 // Add node 2: subtree child of node 0.
157 FlutterSemanticsNode2 node2{sizeof(FlutterSemanticsNode2), 2};
158 std::vector<int32_t> node2_children{3, 4};
159 node2.child_count = node2_children.size();
160 node2.children_in_traversal_order = node2_children.data();
161 node2.children_in_hit_test_order = node2_children.data();
162 node2.flags2 = &empty_flags;
163
164 // Add node 3: text child of node 2.
165 FlutterSemanticsNode2 node3{sizeof(FlutterSemanticsNode2), 3};
166 node3.label = "city";
167 node3.value = "Uji";
168 node3.flags2 = &empty_flags;
169
170 // Add node 4: text child (with no text) of node 2.
171 FlutterSemanticsNode2 node4{sizeof(FlutterSemanticsNode2), 4};
172 node4.flags2 = &empty_flags;
173
174 bridge->AddFlutterSemanticsNodeUpdate(node0);
175 bridge->AddFlutterSemanticsNodeUpdate(node1);
176 bridge->AddFlutterSemanticsNodeUpdate(node2);
177 bridge->AddFlutterSemanticsNodeUpdate(node3);
178 bridge->AddFlutterSemanticsNodeUpdate(node4);
179 bridge->CommitUpdates();
180 }
181
182 ui::AXNode* AXNodeFromID(std::shared_ptr<AccessibilityBridge> bridge,
183 int32_t id) {
184 auto node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(id).lock();
185 return node_delegate ? node_delegate->GetAXNode() : nullptr;
186 }
187
188 std::shared_ptr<AccessibilityBridgeWindowsSpy> GetAccessibilityBridgeSpy(
189 FlutterWindowsView& view) {
190 return std::static_pointer_cast<AccessibilityBridgeWindowsSpy>(
191 view.accessibility_bridge().lock());
192 }
193
194 void ExpectWinEventFromAXEvent(int32_t node_id,
195 ui::AXEventGenerator::Event ax_event,
196 ax::mojom::Event expected_event) {
197 auto engine = GetTestEngine();
198 FlutterWindowsViewSpy view{
199 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
200 EngineModifier modifier{engine.get()};
201 modifier.SetImplicitView(&view);
202 view.OnUpdateSemanticsEnabled(true);
203
204 auto bridge = GetAccessibilityBridgeSpy(view);
205 PopulateAXTree(bridge);
206
207 bridge->ResetRecords();
208 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
209 {ax_event, ax::mojom::EventFrom::kNone, {}}});
210 ASSERT_EQ(bridge->dispatched_events().size(), 1);
211 EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
212 }
213
214 void ExpectWinEventFromAXEventOnFocusNode(int32_t node_id,
215 ui::AXEventGenerator::Event ax_event,
216 ax::mojom::Event expected_event,
217 int32_t focus_id) {
218 auto engine = GetTestEngine();
219 FlutterWindowsViewSpy view{
220 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
221 EngineModifier modifier{engine.get()};
222 modifier.SetImplicitView(&view);
223 view.OnUpdateSemanticsEnabled(true);
224
225 auto bridge = GetAccessibilityBridgeSpy(view);
226 PopulateAXTree(bridge);
227
228 bridge->ResetRecords();
229 auto focus_delegate =
230 bridge->GetFlutterPlatformNodeDelegateFromID(focus_id).lock();
231 bridge->SetFocus(std::static_pointer_cast<FlutterPlatformNodeDelegateWindows>(
232 focus_delegate));
233 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
234 {ax_event, ax::mojom::EventFrom::kNone, {}}});
235 ASSERT_EQ(bridge->dispatched_events().size(), 1);
236 EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
237 EXPECT_EQ(bridge->dispatched_events()[0].node_delegate->GetAXNode()->id(),
238 focus_id);
239 }
240
241 } // namespace
242
243 TEST(AccessibilityBridgeWindows, GetParent) {
244 auto engine = GetTestEngine();
245 FlutterWindowsViewSpy view{
246 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
247 EngineModifier modifier{engine.get()};
248 modifier.SetImplicitView(&view);
249 view.OnUpdateSemanticsEnabled(true);
250
251 auto bridge = view.accessibility_bridge().lock();
252 PopulateAXTree(bridge);
253
254 auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
255 auto node1_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock();
256 EXPECT_EQ(node0_delegate->GetNativeViewAccessible(),
257 node1_delegate->GetParent());
258 }
259
260 TEST(AccessibilityBridgeWindows, GetParentOnRootRetunsNullptr) {
261 auto engine = GetTestEngine();
262 FlutterWindowsViewSpy view{
263 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
264 EngineModifier modifier{engine.get()};
265 modifier.SetImplicitView(&view);
266 view.OnUpdateSemanticsEnabled(true);
267
268 auto bridge = view.accessibility_bridge().lock();
269 PopulateAXTree(bridge);
270
271 auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
272 ASSERT_TRUE(node0_delegate->GetParent() == nullptr);
273 }
274
275 TEST(AccessibilityBridgeWindows, DispatchAccessibilityAction) {
276 auto engine = GetTestEngine();
277 FlutterWindowsViewSpy view{
278 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
279 EngineModifier modifier{engine.get()};
280 modifier.SetImplicitView(&view);
281 view.OnUpdateSemanticsEnabled(true);
282
283 auto bridge = view.accessibility_bridge().lock();
284 PopulateAXTree(bridge);
285
286 FlutterSemanticsAction actual_action = kFlutterSemanticsActionTap;
287 modifier.embedder_api().SendSemanticsAction = MOCK_ENGINE_PROC(
288 SendSemanticsAction,
289 ([&actual_action](FLUTTER_API_SYMBOL(FlutterEngine) engine,
290 const FlutterSendSemanticsActionInfo* info) {
291 actual_action = info->action;
292 return kSuccess;
293 }));
294
295 AccessibilityBridgeWindows delegate(&view);
296 delegate.DispatchAccessibilityAction(1, kFlutterSemanticsActionCopy, {});
297 EXPECT_EQ(actual_action, kFlutterSemanticsActionCopy);
298 }
299
300 TEST(AccessibilityBridgeWindows, OnAccessibilityEventAlert) {
301 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::ALERT,
302 ax::mojom::Event::kAlert);
303 }
304
305 TEST(AccessibilityBridgeWindows, OnAccessibilityEventChildrenChanged) {
306 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::CHILDREN_CHANGED,
307 ax::mojom::Event::kChildrenChanged);
308 }
309
310 TEST(AccessibilityBridgeWindows, OnAccessibilityEventFocusChanged) {
311 auto engine = GetTestEngine();
312 FlutterWindowsViewSpy view{
313 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
314 EngineModifier modifier{engine.get()};
315 modifier.SetImplicitView(&view);
316 view.OnUpdateSemanticsEnabled(true);
317
318 auto bridge = GetAccessibilityBridgeSpy(view);
319 PopulateAXTree(bridge);
320
321 bridge->ResetRecords();
322 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, 1),
323 {ui::AXEventGenerator::Event::FOCUS_CHANGED,
324 ax::mojom::EventFrom::kNone,
325 {}}});
326 ASSERT_EQ(bridge->dispatched_events().size(), 1);
327 EXPECT_EQ(bridge->dispatched_events()[0].event_type,
328 ax::mojom::Event::kFocus);
329
330 ASSERT_EQ(bridge->focused_nodes().size(), 1);
331 EXPECT_EQ(bridge->focused_nodes()[0], 1);
332 }
333
334 TEST(AccessibilityBridgeWindows, OnAccessibilityEventIgnoredChanged) {
335 // Static test nodes with no text, hint, or scrollability are ignored.
336 ExpectWinEventFromAXEvent(4, ui::AXEventGenerator::Event::IGNORED_CHANGED,
337 ax::mojom::Event::kHide);
338 }
339
340 TEST(AccessibilityBridgeWindows, OnAccessibilityImageAnnotationChanged) {
341 ExpectWinEventFromAXEvent(
342 1, ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED,
343 ax::mojom::Event::kTextChanged);
344 }
345
346 TEST(AccessibilityBridgeWindows, OnAccessibilityLiveRegionChanged) {
347 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::LIVE_REGION_CHANGED,
348 ax::mojom::Event::kLiveRegionChanged);
349 }
350
351 TEST(AccessibilityBridgeWindows, OnAccessibilityNameChanged) {
352 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::NAME_CHANGED,
353 ax::mojom::Event::kTextChanged);
354 }
355
356 TEST(AccessibilityBridgeWindows, OnAccessibilityHScrollPosChanged) {
357 ExpectWinEventFromAXEvent(
358 1, ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED,
359 ax::mojom::Event::kScrollPositionChanged);
360 }
361
362 TEST(AccessibilityBridgeWindows, OnAccessibilityVScrollPosChanged) {
363 ExpectWinEventFromAXEvent(
364 1, ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED,
365 ax::mojom::Event::kScrollPositionChanged);
366 }
367
368 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChanged) {
369 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::SELECTED_CHANGED,
370 ax::mojom::Event::kValueChanged);
371 }
372
373 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChildrenChanged) {
374 ExpectWinEventFromAXEvent(
375 2, ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED,
376 ax::mojom::Event::kSelectedChildrenChanged);
377 }
378
379 TEST(AccessibilityBridgeWindows, OnAccessibilitySubtreeCreated) {
380 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::SUBTREE_CREATED,
381 ax::mojom::Event::kShow);
382 }
383
384 TEST(AccessibilityBridgeWindows, OnAccessibilityValueChanged) {
385 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::VALUE_CHANGED,
386 ax::mojom::Event::kValueChanged);
387 }
388
389 TEST(AccessibilityBridgeWindows, OnAccessibilityStateChanged) {
390 ExpectWinEventFromAXEvent(
391 1, ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED,
392 ax::mojom::Event::kStateChanged);
393 }
394
395 TEST(AccessibilityBridgeWindows, OnDocumentSelectionChanged) {
396 ExpectWinEventFromAXEventOnFocusNode(
397 1, ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED,
398 ax::mojom::Event::kDocumentSelectionChanged, 2);
399 }
400
401 } // namespace testing
402 } // namespace flutter
accessibility_bridge_windows.h
node_delegate
std::shared_ptr< FlutterPlatformNodeDelegateWindows > node_delegate
Definition: accessibility_bridge_windows_unittests.cc:33
event_type
ax::mojom::Event event_type
Definition: accessibility_bridge_windows_unittests.cc:34
flutter::AccessibilityBridgeWindows
Definition: accessibility_bridge_windows.h:28
flutter::AccessibilityBridgeWindows::DispatchAccessibilityAction
void DispatchAccessibilityAction(AccessibilityNodeId target, FlutterSemanticsAction action, fml::MallocMapping data) override
Dispatch accessibility action back to the Flutter framework. These actions are generated in the nativ...
Definition: accessibility_bridge_windows.cc:162
flutter::AccessibilityBridgeWindows::OnAccessibilityEvent
void OnAccessibilityEvent(ui::AXEventGenerator::TargetedEvent targeted_event) override
Handle accessibility events generated due to accessibility tree changes. These events are needed to b...
Definition: accessibility_bridge_windows.cc:18
flutter::FlutterEngine
Definition: flutter_engine.h:28
flutter_platform_node_delegate_windows.h
flutter_windows_engine.h
flutter_windows_view.h
flutter::testing::TEST
TEST(AccessibilityBridgeWindows, GetParent)
Definition: accessibility_bridge_windows_unittests.cc:243
flutter
Definition: accessibility_bridge_windows.cc:11
flutter::kImplicitViewId
constexpr FlutterViewId kImplicitViewId
Definition: flutter_windows_engine.h:56
FlutterDesktopEngineProperties
Definition: flutter_windows.h:60
FlutterDesktopEngineProperties::icu_data_path
const wchar_t * icu_data_path
Definition: flutter_windows.h:69
FlutterDesktopEngineProperties::assets_path
const wchar_t * assets_path
Definition: flutter_windows.h:64
FlutterDesktopEngineProperties::aot_library_path
const wchar_t * aot_library_path
Definition: flutter_windows.h:75