| Flutter Windows Embedder |

accessibility_bridge_windows_unittests.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "flutter/shell/platform/windows/accessibility_bridge_windows.h"
6
7 #include <comdef.h>
8 #include <comutil.h>
9 #include <oleacc.h>
10
11 #include <vector>
12
13 #include "flutter/fml/macros.h"
14 #include "flutter/shell/platform/embedder/embedder.h"
15 #include "flutter/shell/platform/embedder/test_utils/proc_table_replacement.h"
16 #include "flutter/shell/platform/windows/flutter_platform_node_delegate_windows.h"
17 #include "flutter/shell/platform/windows/flutter_windows_engine.h"
18 #include "flutter/shell/platform/windows/flutter_windows_view.h"
19 #include "flutter/shell/platform/windows/testing/engine_modifier.h"
20 #include "flutter/shell/platform/windows/testing/mock_window_binding_handler.h"
21 #include "flutter/shell/platform/windows/testing/test_keyboard.h"
22 #include "gmock/gmock.h"
23 #include "gtest/gtest.h"
24
25 namespace flutter {
26 namespace testing {
27
28 namespace {
29 using ::testing::NiceMock;
30
31 // A structure representing a Win32 MSAA event targeting a specified node.
32 struct MsaaEvent {
33 std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate;
34 ax::mojom::Event event_type;
35 };
36
37 // Accessibility bridge delegate that captures events dispatched to the OS.
38 class AccessibilityBridgeWindowsSpy : public AccessibilityBridgeWindows {
39 public:
40 using AccessibilityBridgeWindows::OnAccessibilityEvent;
41
42 explicit AccessibilityBridgeWindowsSpy(FlutterWindowsEngine* engine,
43 FlutterWindowsView* view)
44 : AccessibilityBridgeWindows(view) {}
45
46 void DispatchWinAccessibilityEvent(
47 std::shared_ptr<FlutterPlatformNodeDelegateWindows> node_delegate,
48 ax::mojom::Event event_type) override {
49 dispatched_events_.push_back({node_delegate, event_type});
50 }
51
52 void SetFocus(std::shared_ptr<FlutterPlatformNodeDelegateWindows>
53 node_delegate) override {
54 focused_nodes_.push_back(std::move(node_delegate));
55 }
56
57 void ResetRecords() {
58 dispatched_events_.clear();
59 focused_nodes_.clear();
60 }
61
62 const std::vector<MsaaEvent>& dispatched_events() const {
63 return dispatched_events_;
64 }
65
66 const std::vector<int32_t> focused_nodes() const {
67 std::vector<int32_t> ids;
68 std::transform(focused_nodes_.begin(), focused_nodes_.end(),
69 std::back_inserter(ids),
70 [](std::shared_ptr<FlutterPlatformNodeDelegate> node) {
71 return node->GetAXNode()->id();
72 });
73 return ids;
74 }
75
76 protected:
77 std::weak_ptr<FlutterPlatformNodeDelegate> GetFocusedNode() override {
78 return focused_nodes_.back();
79 }
80
81 private:
82 std::vector<MsaaEvent> dispatched_events_;
83 std::vector<std::shared_ptr<FlutterPlatformNodeDelegate>> focused_nodes_;
84
85 FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridgeWindowsSpy);
86 };
87
88 // A FlutterWindowsView whose accessibility bridge is an
89 // AccessibilityBridgeWindowsSpy.
90 class FlutterWindowsViewSpy : public FlutterWindowsView {
91 public:
92 FlutterWindowsViewSpy(FlutterWindowsEngine* engine,
93 std::unique_ptr<WindowBindingHandler> handler)
94 : FlutterWindowsView(kImplicitViewId,
95 engine,
96 std::move(handler),
97 false,
98 BoxConstraints()) {}
99
100 protected:
101 virtual std::shared_ptr<AccessibilityBridgeWindows>
102 CreateAccessibilityBridge() override {
103 return std::make_shared<AccessibilityBridgeWindowsSpy>(GetEngine(), this);
104 }
105
106 private:
107 FML_DISALLOW_COPY_AND_ASSIGN(FlutterWindowsViewSpy);
108 };
109
110 // Returns an engine instance configured with dummy project path values, and
111 // overridden methods for sending platform messages, so that the engine can
112 // respond as if the framework were connected.
113 std::unique_ptr<FlutterWindowsEngine> GetTestEngine() {
114 FlutterDesktopEngineProperties properties = {};
115 properties.assets_path = L"C:\\foo\\flutter_assets";
116 properties.icu_data_path = L"C:\\foo\\icudtl.dat";
117 properties.aot_library_path = L"C:\\foo\\aot.so";
118 FlutterProjectBundle project(properties);
119 auto engine = std::make_unique<FlutterWindowsEngine>(project);
120
121 EngineModifier modifier(engine.get());
122 modifier.embedder_api().UpdateSemanticsEnabled =
123 [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) {
124 return kSuccess;
125 };
126
127 MockEmbedderApiForKeyboard(modifier,
128 std::make_shared<MockKeyResponseController>());
129
130 engine->Run();
131 return engine;
132 }
133
134 // Populates the AXTree associated with the specified bridge with test data.
135 //
136 // node0
137 // / \
138 // node1 node2
139 // / \
140 // node3 node4
141 //
142 // node0 and node2 are grouping nodes. node1 and node2 are static text nodes.
143 // node4 is a static text node with no text, and hence has the "ignored" state.
144 void PopulateAXTree(std::shared_ptr<AccessibilityBridge> bridge) {
145 // Add node 0: root.
146 FlutterSemanticsNode2 node0{sizeof(FlutterSemanticsNode2), 0};
147 auto empty_flags = FlutterSemanticsFlags{};
148 std::vector<int32_t> node0_children{1, 2};
149 node0.child_count = node0_children.size();
150 node0.children_in_traversal_order = node0_children.data();
151 node0.children_in_hit_test_order = node0_children.data();
152 node0.flags2 = &empty_flags;
153
154 // Add node 1: text child of node 0.
155 FlutterSemanticsNode2 node1{sizeof(FlutterSemanticsNode2), 1};
156 node1.label = "prefecture";
157 node1.value = "Kyoto";
158 node1.flags2 = &empty_flags;
159
160 // Add node 2: subtree child of node 0.
161 FlutterSemanticsNode2 node2{sizeof(FlutterSemanticsNode2), 2};
162 std::vector<int32_t> node2_children{3, 4};
163 node2.child_count = node2_children.size();
164 node2.children_in_traversal_order = node2_children.data();
165 node2.children_in_hit_test_order = node2_children.data();
166 node2.flags2 = &empty_flags;
167
168 // Add node 3: text child of node 2.
169 FlutterSemanticsNode2 node3{sizeof(FlutterSemanticsNode2), 3};
170 node3.label = "city";
171 node3.value = "Uji";
172 node3.flags2 = &empty_flags;
173
174 // Add node 4: text child (with no text) of node 2.
175 FlutterSemanticsNode2 node4{sizeof(FlutterSemanticsNode2), 4};
176 node4.flags2 = &empty_flags;
177
178 bridge->AddFlutterSemanticsNodeUpdate(node0);
179 bridge->AddFlutterSemanticsNodeUpdate(node1);
180 bridge->AddFlutterSemanticsNodeUpdate(node2);
181 bridge->AddFlutterSemanticsNodeUpdate(node3);
182 bridge->AddFlutterSemanticsNodeUpdate(node4);
183 bridge->CommitUpdates();
184 }
185
186 ui::AXNode* AXNodeFromID(std::shared_ptr<AccessibilityBridge> bridge,
187 int32_t id) {
188 auto node_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(id).lock();
189 return node_delegate ? node_delegate->GetAXNode() : nullptr;
190 }
191
192 std::shared_ptr<AccessibilityBridgeWindowsSpy> GetAccessibilityBridgeSpy(
193 FlutterWindowsView& view) {
194 return std::static_pointer_cast<AccessibilityBridgeWindowsSpy>(
195 view.accessibility_bridge().lock());
196 }
197
198 void ExpectWinEventFromAXEvent(int32_t node_id,
199 ui::AXEventGenerator::Event ax_event,
200 ax::mojom::Event expected_event) {
201 auto engine = GetTestEngine();
202 FlutterWindowsViewSpy view{
203 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
204 EngineModifier modifier{engine.get()};
205 modifier.SetImplicitView(&view);
206 view.OnUpdateSemanticsEnabled(true);
207
208 auto bridge = GetAccessibilityBridgeSpy(view);
209 PopulateAXTree(bridge);
210
211 bridge->ResetRecords();
212 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
213 {ax_event, ax::mojom::EventFrom::kNone, {}}});
214 ASSERT_EQ(bridge->dispatched_events().size(), 1);
215 EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
216 }
217
218 void ExpectWinEventFromAXEventOnFocusNode(int32_t node_id,
219 ui::AXEventGenerator::Event ax_event,
220 ax::mojom::Event expected_event,
221 int32_t focus_id) {
222 auto engine = GetTestEngine();
223 FlutterWindowsViewSpy view{
224 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
225 EngineModifier modifier{engine.get()};
226 modifier.SetImplicitView(&view);
227 view.OnUpdateSemanticsEnabled(true);
228
229 auto bridge = GetAccessibilityBridgeSpy(view);
230 PopulateAXTree(bridge);
231
232 bridge->ResetRecords();
233 auto focus_delegate =
234 bridge->GetFlutterPlatformNodeDelegateFromID(focus_id).lock();
235 bridge->SetFocus(std::static_pointer_cast<FlutterPlatformNodeDelegateWindows>(
236 focus_delegate));
237 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, node_id),
238 {ax_event, ax::mojom::EventFrom::kNone, {}}});
239 ASSERT_EQ(bridge->dispatched_events().size(), 1);
240 EXPECT_EQ(bridge->dispatched_events()[0].event_type, expected_event);
241 EXPECT_EQ(bridge->dispatched_events()[0].node_delegate->GetAXNode()->id(),
242 focus_id);
243 }
244
245 } // namespace
246
247 TEST(AccessibilityBridgeWindows, GetParent) {
248 auto engine = GetTestEngine();
249 FlutterWindowsViewSpy view{
250 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
251 EngineModifier modifier{engine.get()};
252 modifier.SetImplicitView(&view);
253 view.OnUpdateSemanticsEnabled(true);
254
255 auto bridge = view.accessibility_bridge().lock();
256 PopulateAXTree(bridge);
257
258 auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
259 auto node1_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(1).lock();
260 EXPECT_EQ(node0_delegate->GetNativeViewAccessible(),
261 node1_delegate->GetParent());
262 }
263
264 TEST(AccessibilityBridgeWindows, GetParentOnRootRetunsNullptr) {
265 auto engine = GetTestEngine();
266 FlutterWindowsViewSpy view{
267 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
268 EngineModifier modifier{engine.get()};
269 modifier.SetImplicitView(&view);
270 view.OnUpdateSemanticsEnabled(true);
271
272 auto bridge = view.accessibility_bridge().lock();
273 PopulateAXTree(bridge);
274
275 auto node0_delegate = bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
276 ASSERT_TRUE(node0_delegate->GetParent() == nullptr);
277 }
278
279 TEST(AccessibilityBridgeWindows, DispatchAccessibilityAction) {
280 auto engine = GetTestEngine();
281 FlutterWindowsViewSpy view{
282 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
283 EngineModifier modifier{engine.get()};
284 modifier.SetImplicitView(&view);
285 view.OnUpdateSemanticsEnabled(true);
286
287 auto bridge = view.accessibility_bridge().lock();
288 PopulateAXTree(bridge);
289
290 FlutterSemanticsAction actual_action = kFlutterSemanticsActionTap;
291 modifier.embedder_api().SendSemanticsAction = MOCK_ENGINE_PROC(
292 SendSemanticsAction,
293 ([&actual_action](FLUTTER_API_SYMBOL(FlutterEngine) engine,
294 const FlutterSendSemanticsActionInfo* info) {
295 actual_action = info->action;
296 return kSuccess;
297 }));
298
299 AccessibilityBridgeWindows delegate(&view);
300 delegate.DispatchAccessibilityAction(1, kFlutterSemanticsActionCopy, {});
301 EXPECT_EQ(actual_action, kFlutterSemanticsActionCopy);
302 }
303
304 TEST(AccessibilityBridgeWindows, OnAccessibilityEventAlert) {
305 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::ALERT,
306 ax::mojom::Event::kAlert);
307 }
308
309 TEST(AccessibilityBridgeWindows, OnAccessibilityEventChildrenChanged) {
310 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::CHILDREN_CHANGED,
311 ax::mojom::Event::kChildrenChanged);
312 }
313
314 TEST(AccessibilityBridgeWindows, OnAccessibilityEventFocusChanged) {
315 auto engine = GetTestEngine();
316 FlutterWindowsViewSpy view{
317 engine.get(), std::make_unique<NiceMock<MockWindowBindingHandler>>()};
318 EngineModifier modifier{engine.get()};
319 modifier.SetImplicitView(&view);
320 view.OnUpdateSemanticsEnabled(true);
321
322 auto bridge = GetAccessibilityBridgeSpy(view);
323 PopulateAXTree(bridge);
324
325 bridge->ResetRecords();
326 bridge->OnAccessibilityEvent({AXNodeFromID(bridge, 1),
327 {ui::AXEventGenerator::Event::FOCUS_CHANGED,
328 ax::mojom::EventFrom::kNone,
329 {}}});
330 ASSERT_EQ(bridge->dispatched_events().size(), 1);
331 EXPECT_EQ(bridge->dispatched_events()[0].event_type,
332 ax::mojom::Event::kFocus);
333
334 ASSERT_EQ(bridge->focused_nodes().size(), 1);
335 EXPECT_EQ(bridge->focused_nodes()[0], 1);
336 }
337
338 TEST(AccessibilityBridgeWindows, OnAccessibilityEventIgnoredChanged) {
339 // Static test nodes with no text, hint, or scrollability are ignored.
340 ExpectWinEventFromAXEvent(4, ui::AXEventGenerator::Event::IGNORED_CHANGED,
341 ax::mojom::Event::kHide);
342 }
343
344 TEST(AccessibilityBridgeWindows, OnAccessibilityImageAnnotationChanged) {
345 ExpectWinEventFromAXEvent(
346 1, ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED,
347 ax::mojom::Event::kTextChanged);
348 }
349
350 TEST(AccessibilityBridgeWindows, OnAccessibilityLiveRegionChanged) {
351 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::LIVE_REGION_CHANGED,
352 ax::mojom::Event::kLiveRegionChanged);
353 }
354
355 TEST(AccessibilityBridgeWindows, OnAccessibilityNameChanged) {
356 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::NAME_CHANGED,
357 ax::mojom::Event::kTextChanged);
358 }
359
360 TEST(AccessibilityBridgeWindows, OnAccessibilityHScrollPosChanged) {
361 ExpectWinEventFromAXEvent(
362 1, ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED,
363 ax::mojom::Event::kScrollPositionChanged);
364 }
365
366 TEST(AccessibilityBridgeWindows, OnAccessibilityVScrollPosChanged) {
367 ExpectWinEventFromAXEvent(
368 1, ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED,
369 ax::mojom::Event::kScrollPositionChanged);
370 }
371
372 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChanged) {
373 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::SELECTED_CHANGED,
374 ax::mojom::Event::kValueChanged);
375 }
376
377 TEST(AccessibilityBridgeWindows, OnAccessibilitySelectedChildrenChanged) {
378 ExpectWinEventFromAXEvent(
379 2, ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED,
380 ax::mojom::Event::kSelectedChildrenChanged);
381 }
382
383 TEST(AccessibilityBridgeWindows, OnAccessibilitySubtreeCreated) {
384 ExpectWinEventFromAXEvent(0, ui::AXEventGenerator::Event::SUBTREE_CREATED,
385 ax::mojom::Event::kShow);
386 }
387
388 TEST(AccessibilityBridgeWindows, OnAccessibilityValueChanged) {
389 ExpectWinEventFromAXEvent(1, ui::AXEventGenerator::Event::VALUE_CHANGED,
390 ax::mojom::Event::kValueChanged);
391 }
392
393 TEST(AccessibilityBridgeWindows, OnAccessibilityStateChanged) {
394 ExpectWinEventFromAXEvent(
395 1, ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED,
396 ax::mojom::Event::kStateChanged);
397 }
398
399 TEST(AccessibilityBridgeWindows, OnDocumentSelectionChanged) {
400 ExpectWinEventFromAXEventOnFocusNode(
401 1, ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED,
402 ax::mojom::Event::kDocumentSelectionChanged, 2);
403 }
404
405 } // namespace testing
406 } // namespace flutter
accessibility_bridge_windows.h
node_delegate
std::shared_ptr< FlutterPlatformNodeDelegateWindows > node_delegate
Definition: accessibility_bridge_windows_unittests.cc:33
event_type
ax::mojom::Event event_type
Definition: accessibility_bridge_windows_unittests.cc:34
flutter::AccessibilityBridgeWindows
Definition: accessibility_bridge_windows.h:28
flutter::AccessibilityBridgeWindows::DispatchAccessibilityAction
void DispatchAccessibilityAction(AccessibilityNodeId target, FlutterSemanticsAction action, fml::MallocMapping data) override
Dispatch accessibility action back to the Flutter framework. These actions are generated in the nativ...
Definition: accessibility_bridge_windows.cc:162
flutter::AccessibilityBridgeWindows::OnAccessibilityEvent
void OnAccessibilityEvent(ui::AXEventGenerator::TargetedEvent targeted_event) override
Handle accessibility events generated due to accessibility tree changes. These events are needed to b...
Definition: accessibility_bridge_windows.cc:18
flutter::FlutterEngine
Definition: flutter_engine.h:28
flutter_platform_node_delegate_windows.h
flutter_windows_engine.h
flutter_windows_view.h
flutter::testing::TEST
TEST(AccessibilityBridgeWindows, GetParent)
Definition: accessibility_bridge_windows_unittests.cc:247
flutter
Definition: accessibility_bridge_windows.cc:11
flutter::kImplicitViewId
constexpr FlutterViewId kImplicitViewId
Definition: flutter_windows_engine.h:56
FlutterDesktopEngineProperties
Definition: flutter_windows.h:64
FlutterDesktopEngineProperties::icu_data_path
const wchar_t * icu_data_path
Definition: flutter_windows.h:73
FlutterDesktopEngineProperties::assets_path
const wchar_t * assets_path
Definition: flutter_windows.h:68
FlutterDesktopEngineProperties::aot_library_path
const wchar_t * aot_library_path
Definition: flutter_windows.h:79