| Flutter Windows Embedder |

accessibility_bridge_windows_unittests.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "flutter/shell/platform/windows/accessibility_bridge_windows.h"
6
7 #include <comdef.h>
8 #include <comutil.h>
9 #include <oleacc.h>
10
11 #include <vector>
12
13 #include "flutter/fml/macros.h"
14 #include "flutter/shell/platform/embedder/embedder.h"
15 #include "flutter/shell/platform/embedder/test_utils/proc_table_replacement.h"
16 #include "flutter/shell/platform/windows/flutter_platform_node_delegate_windows.h"
17 #include "flutter/shell/platform/windows/flutter_windows_engine.h"
18 #include "flutter/shell/platform/windows/flutter_windows_view.h"
19 #include "flutter/shell/platform/windows/testing/engine_modifier.h"
20 #include "flutter/shell/platform/windows/testing/mock_window_binding_handler.h"
21 #include "flutter/shell/platform/windows/testing/test_keyboard.h"
22 #include "gmock/gmock.h"
23 #include "gtest/gtest.h"
24
25 namespace flutter {
26 namespace testing {
27
28 namespace {
29
30 // A structure representing a Win32 MSAA event targeting a specified node.
31 struct MsaaEvent {
32 std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate;
33 ax::mojom::Event event_type;
34 };
35
36 // Accessibility bridge delegate that captures events dispatched to the OS.
37 class AccessibilityBridgeWindowsSpy : public AccessibilityBridgeWindows {
38 public:
39 using AccessibilityBridgeWindows::OnAccessibilityEvent;
40
41 explicit AccessibilityBridgeWindowsSpy(FlutterWindowsEngine* engine,
42 FlutterWindowsView* view)
43 : AccessibilityBridgeWindows(view) {}
44
45 void DispatchWinAccessibilityEvent(
46 std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate,
47 ax::mojom::Event event_type) override {
48 dispatched_events_.push_back({node_delegate, event_type});
49 }
50
51 void SetFocus(std::shared_ptr<FlutterPlatformNodeDelegateWindows>
52 node_delegate) override {
53 focused_nodes_.push_back(std::move(node_delegate));
54 }
55
56 void ResetRecords() {
57 dispatched_events_.clear();
58 focused_nodes_.clear();
59 }
60
61 const std::vector<MsaaEvent>& dispatched_events() const {
62 return dispatched_events_;
63 }
64
65 const std::vector<int32_t> focused_nodes() const {
66 std::vector<int32_t> ids;
67 std::transform(focused_nodes_.begin(), focused_nodes_.end(),
68 std::back_inserter(ids),
69 [](std::shared_ptr<FlutterPlatformNodeDelegate> node) {
70 return node->GetAXNode()->id();
71 });
72 return ids;
73 }
74
75 protected:
76 std::weak_ptr<FlutterPlatformNodeDelegate> GetFocusedNode() override {
77 return focused_nodes_.back();
78 }
79
80 private:
81 std::vector<MsaaEvent> dispatched_events_;
82 std::vector<std::shared_ptr<FlutterPlatformNodeDelegate>> focused_nodes_;
83
84 FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridgeWindowsSpy);
85 };
86
87 // A FlutterWindowsView whose accessibility bridge is an
88 // AccessibilityBridgeWindowsSpy.
89 class FlutterWindowsViewSpy : public FlutterWindowsView {
90 public:
91 explicit FlutterWindowsViewSpy(std::unique_ptr<WindowBindingHandler> handler)
92 : FlutterWindowsView(std::move(handler)) {}
93
94 protected:
95 virtual std::shared_ptr<AccessibilityBridgeWindows>
96 CreateAccessibilityBridge() override {
97 return std::make_shared<AccessibilityBridgeWindowsSpy>(GetEngine(), this);
98 }
99
100 private:
101 FML_DISALLOW_COPY_AND_ASSIGN(FlutterWindowsViewSpy);
102 };
103
104 // Returns an engine instance configured with dummy project path values, and
105 // overridden methods for sending platform messages, so that the engine can
106 // respond as if the framework were connected.
107 std::unique_ptr<FlutterWindowsEngine> GetTestEngine() {
108 FlutterDesktopEngineProperties properties = {};
109 properties.assets_path = L"C:\\foo\\flutter_assets";
110 properties.icu_data_path = L"C:\\foo\\icudtl.dat";
111 properties.aot_library_path = L"C:\\foo\\aot.so";
112 FlutterProjectBundle project(properties);
113 auto engine = std::make_unique<FlutterWindowsEngine>(project);
114
115 EngineModifier modifier(engine.get());
116 modifier.embedder_api().UpdateSemanticsEnabled =
117 [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) {
118 return kSuccess;
119 };
120
121 MockEmbedderApiForKeyboard(modifier,
122 std::make_shared<MockKeyResponseController>());
123
124 engine->Run();
125 return engine;
126 }
127
128 // Populates the AXTree associated with the specified bridge with test data.
129 //
130 // node0
131 // / \
132 // node1 node2
133 // / \
134 // node3 node4
135 //
136 // node0 and node2 are grouping nodes. node1 and node2 are static text nodes.
137 // node4 is a static text node with no text, and hence has the "ignored" state.
138 void PopulateAXTree(std::shared_ptr<AccessibilityBridge> bridge) {
139 // Add node 0: root.
140 FlutterSemanticsNode2 node0{sizeof(FlutterSemanticsNode2), 0};
141 std::vector<int32_t> node0_children{1, 2};
142 node0.child_count = node0_children.size();
143 node0.children_in_traversal_order = node0_children.data();
144 node0.children_in_hit_test_order = node0_children.data();
145
146 // Add node 1: text child of node 0.
147 FlutterSemanticsNode2 node1{sizeof(FlutterSemanticsNode2), 1};
148 node1.label = "prefecture";
149 node1.value = "Kyoto";
150
151 // Add node 2: subtree child of node 0.
152 FlutterSemanticsNode2 node2{sizeof(FlutterSemanticsNode2), 2};
153 std::vector<int32_t> node2_children{3, 4};
154 node2.child_count = node2_children.size();
155 node2.children_in_traversal_order = node2_children.data();
156 node2.children_in_hit_test_order = node2_children.data();
157
158 // Add node 3: text child of node 2.
159 FlutterSemanticsNode2 node3{sizeof(FlutterSemanticsNode2), 3};
160 node3.label = "city";
161 node3.value = "Uji";
162
163 // Add node 4: text child (with no text) of node 2.
164 FlutterSemanticsNode2 node4{sizeof(FlutterSemanticsNode2), 4};
165
166 bridge->AddFlutterSemanticsNodeUpdate(node0);
167 bridge->AddFlutterSemanticsNodeUpdate(node1);
168 bridge->AddFlutterSemanticsNodeUpdate(node2);
169 bridge->AddFlutterSemanticsNodeUpdate(node3);
170 bridge->AddFlutterSemanticsNodeUpdate(node4);
171 bridge->CommitUpdates();
172 }
173
174 ui::AXNode* AXNodeFromID(std::shared_ptr<AccessibilityBridge> bridge,
175 int32_t id) {
176 auto node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(id).lock();
177 return node_delegate ? node_delegate->GetAXNode() : nullptr;
178 }
179
180 std::shared_ptr<AccessibilityBridgeWindowsSpy> GetAccessibilityBridgeSpy(
181 FlutterWindowsView& view) {
182 return std::static_pointer_cast<AccessibilityBridgeWindowsSpy>(
183 view.accessibility_bridge().lock());
184 }
185
186 void ExpectWinEventFromAXEvent(int32_t node_id,
187 ui::AXEventGenerator::Event ax_event,
188 ax::mojom::Event expected_event) {
189 auto window_binding_handler =
190 std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
191 FlutterWindowsViewSpy view(std::move(window_binding_handler));
192 view.SetEngine(GetTestEngine());
193 view.OnUpdateSemanticsEnabled(true);
194
195 auto bridge = GetAccessibilityBridgeSpy(view);
196 PopulateAXTree(bridge);
197
198 bridge->ResetRecords();
199 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
200 {ax_event, ax::mojom::EventFrom::kNone, {}}});
201 ASSERT_EQ(bridge->dispatched_events().size(), 1);
202 EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
203 }
204
205 void ExpectWinEventFromAXEventOnFocusNode(int32_t node_id,
206 ui::AXEventGenerator::Event ax_event,
207 ax::mojom::Event expected_event,
208 int32_t focus_id) {
209 auto window_binding_handler =
210 std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
211 FlutterWindowsViewSpy view(std::move(window_binding_handler));
212 view.SetEngine(GetTestEngine());
213 view.OnUpdateSemanticsEnabled(true);
214
215 auto bridge = GetAccessibilityBridgeSpy(view);
216 PopulateAXTree(bridge);
217
218 bridge->ResetRecords();
219 auto focus_delegate =
220 bridge->GetFlutterPlatformNodeDelegateFromID(focus_id).lock();
221 bridge->SetFocus(std::static_pointer_cast<FlutterPlatformNodeDelegateWindows>(
222 focus_delegate));
223 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
224 {ax_event, ax::mojom::EventFrom::kNone, {}}});
225 ASSERT_EQ(bridge->dispatched_events().size(), 1);
226 EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
227 EXPECT_EQ(bridge->dispatched_events()[0].node_delegate->GetAXNode()->id(),
228 focus_id);
229 }
230
231 } // namespace
232
233 TEST(AccessibilityBridgeWindows, GetParent) {
234 auto window_binding_handler =
235 std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
236 FlutterWindowsViewSpy view(std::move(window_binding_handler));
237 view.SetEngine(GetTestEngine());
238 view.OnUpdateSemanticsEnabled(true);
239
240 auto bridge = view.accessibility_bridge().lock();
241 PopulateAXTree(bridge);
242
243 auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
244 auto node1_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock();
245 EXPECT_EQ(node0_delegate->GetNativeViewAccessible(),
246 node1_delegate->GetParent());
247 }
248
249 TEST(AccessibilityBridgeWindows, GetParentOnRootRetunsNullptr) {
250 auto window_binding_handler =
251 std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
252 FlutterWindowsViewSpy view(std::move(window_binding_handler));
253 view.SetEngine(GetTestEngine());
254 view.OnUpdateSemanticsEnabled(true);
255
256 auto bridge = view.accessibility_bridge().lock();
257 PopulateAXTree(bridge);
258
259 auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
260 ASSERT_TRUE(node0_delegate->GetParent() == nullptr);
261 }
262
263 TEST(AccessibilityBridgeWindows, DispatchAccessibilityAction) {
264 auto window_binding_handler =
265 std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
266 FlutterWindowsViewSpy view(std::move(window_binding_handler));
267 view.SetEngine(GetTestEngine());
268 view.OnUpdateSemanticsEnabled(true);
269
270 auto bridge = view.accessibility_bridge().lock();
271 PopulateAXTree(bridge);
272
273 FlutterSemanticsAction actual_action = kFlutterSemanticsActionTap;
274 EngineModifier modifier(view.GetEngine());
275 modifier.embedder_api().DispatchSemanticsAction = MOCK_ENGINE_PROC(
276 DispatchSemanticsAction,
277 ([&actual_action](FLUTTER_API_SYMBOL(FlutterEngine) engine, uint64_t id,
278 FlutterSemanticsAction action, const uint8_t* data,
279 size_t data_length) {
280 actual_action = action;
281 return kSuccess;
282 }));
283
284 AccessibilityBridgeWindows delegate(&view);
285 delegate.DispatchAccessibilityAction(1, kFlutterSemanticsActionCopy, {});
286 EXPECT_EQ(actual_action, kFlutterSemanticsActionCopy);
287 }
288
289 TEST(AccessibilityBridgeWindows, OnAccessibilityEventAlert) {
290 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::ALERT,
291 ax::mojom::Event::kAlert);
292 }
293
294 TEST(AccessibilityBridgeWindows, OnAccessibilityEventChildrenChanged) {
295 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::CHILDREN_CHANGED,
296 ax::mojom::Event::kChildrenChanged);
297 }
298
299 TEST(AccessibilityBridgeWindows, OnAccessibilityEventFocusChanged) {
300 auto window_binding_handler =
301 std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
302 FlutterWindowsViewSpy view(std::move(window_binding_handler));
303 view.SetEngine(GetTestEngine());
304 view.OnUpdateSemanticsEnabled(true);
305
306 auto bridge = GetAccessibilityBridgeSpy(view);
307 PopulateAXTree(bridge);
308
309 bridge->ResetRecords();
310 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, 1),
311 {ui::AXEventGenerator::Event::FOCUS_CHANGED,
312 ax::mojom::EventFrom::kNone,
313 {}}});
314 ASSERT_EQ(bridge->dispatched_events().size(), 1);
315 EXPECT_EQ(bridge->dispatched_events()[0].event_type,
316 ax::mojom::Event::kFocus);
317
318 ASSERT_EQ(bridge->focused_nodes().size(), 1);
319 EXPECT_EQ(bridge->focused_nodes()[0], 1);
320 }
321
322 TEST(AccessibilityBridgeWindows, OnAccessibilityEventIgnoredChanged) {
323 // Static test nodes with no text, hint, or scrollability are ignored.
324 ExpectWinEventFromAXEvent(4, ui::AXEventGenerator::Event::IGNORED_CHANGED,
325 ax::mojom::Event::kHide);
326 }
327
328 TEST(AccessibilityBridgeWindows, OnAccessibilityImageAnnotationChanged) {
329 ExpectWinEventFromAXEvent(
330 1, ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED,
331 ax::mojom::Event::kTextChanged);
332 }
333
334 TEST(AccessibilityBridgeWindows, OnAccessibilityLiveRegionChanged) {
335 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::LIVE_REGION_CHANGED,
336 ax::mojom::Event::kLiveRegionChanged);
337 }
338
339 TEST(AccessibilityBridgeWindows, OnAccessibilityNameChanged) {
340 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::NAME_CHANGED,
341 ax::mojom::Event::kTextChanged);
342 }
343
344 TEST(AccessibilityBridgeWindows, OnAccessibilityHScrollPosChanged) {
345 ExpectWinEventFromAXEvent(
346 1, ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED,
347 ax::mojom::Event::kScrollPositionChanged);
348 }
349
350 TEST(AccessibilityBridgeWindows, OnAccessibilityVScrollPosChanged) {
351 ExpectWinEventFromAXEvent(
352 1, ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED,
353 ax::mojom::Event::kScrollPositionChanged);
354 }
355
356 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChanged) {
357 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::SELECTED_CHANGED,
358 ax::mojom::Event::kValueChanged);
359 }
360
361 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChildrenChanged) {
362 ExpectWinEventFromAXEvent(
363 2, ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED,
364 ax::mojom::Event::kSelectedChildrenChanged);
365 }
366
367 TEST(AccessibilityBridgeWindows, OnAccessibilitySubtreeCreated) {
368 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::SUBTREE_CREATED,
369 ax::mojom::Event::kShow);
370 }
371
372 TEST(AccessibilityBridgeWindows, OnAccessibilityValueChanged) {
373 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::VALUE_CHANGED,
374 ax::mojom::Event::kValueChanged);
375 }
376
377 TEST(AccessibilityBridgeWindows, OnAccessibilityStateChanged) {
378 ExpectWinEventFromAXEvent(
379 1, ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED,
380 ax::mojom::Event::kStateChanged);
381 }
382
383 TEST(AccessibilityBridgeWindows, OnDocumentSelectionChanged) {
384 ExpectWinEventFromAXEventOnFocusNode(
385 1, ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED,
386 ax::mojom::Event::kDocumentSelectionChanged, 2);
387 }
388
389 } // namespace testing
390 } // namespace flutter
FlutterDesktopEngineProperties::aot_library_path
const wchar_t * aot_library_path
Definition: flutter_windows.h:51
FlutterDesktopEngineProperties
Definition: flutter_windows.h:36
flutter::FlutterEngine
Definition: flutter_engine.h:28
flutter::AccessibilityBridgeWindows::OnAccessibilityEvent
void OnAccessibilityEvent(ui::AXEventGenerator::TargetedEvent targeted_event) override
Handle accessibility events generated due to accessibility tree changes. These events are needed to b...
Definition: accessibility_bridge_windows.cc:18
flutter::AccessibilityBridgeWindows
Definition: accessibility_bridge_windows.h:27
FlutterDesktopEngineProperties::icu_data_path
const wchar_t * icu_data_path
Definition: flutter_windows.h:45
flutter_windows_view.h
accessibility_bridge_windows.h
flutter::AccessibilityBridgeWindows::DispatchAccessibilityAction
void DispatchAccessibilityAction(AccessibilityNodeId target, FlutterSemanticsAction action, fml::MallocMapping data) override
Dispatch accessibility action back to the Flutter framework. These actions are generated in the nativ...
Definition: accessibility_bridge_windows.cc:162
flutter
Definition: accessibility_bridge_windows.cc:11
flutter_windows_engine.h
flutter_platform_node_delegate_windows.h
flutter::testing::TEST
TEST(AccessibilityBridgeWindows, GetParent)
Definition: accessibility_bridge_windows_unittests.cc:233
action
int action
Definition: keyboard_key_handler_unittests.cc:116
event_type
ax::mojom::Event event_type
Definition: accessibility_bridge_windows_unittests.cc:33
FlutterDesktopEngineProperties::assets_path
const wchar_t * assets_path
Definition: flutter_windows.h:40
node_delegate
std::shared_ptr< FlutterPlatformNodeDelegateWindows > node_delegate
Definition: accessibility_bridge_windows_unittests.cc:32