From 97154927afd1f0237a18bda8a67b5cee4d6ef5ee Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Thu, 21 Oct 2021 00:05:33 -0700 Subject: [PATCH 01/12] Win32 a11y bridge and platform node delegates This is the third in a series of patches adding accessibility support for the Windows embedder. This patch wires in the Accessibility bridge, and lands the core structure of the Windows FlutterPlatformNodeDelegate and AccessibilityBridgeDelegate classes, including: * Instantiating the AccessibilityBridge when the semantics tree is enabled. * Creating FlutterPlatformNodeDelegate wrappers for semantics tree nodes. * Building and updating the accessibility tree on semantics updates. * Returning the native IAccessible objects when queried. Breaking this out so that the follow-up patches can be reviewed and landed in smaller, independent chunks. Issue: https://github.com/flutter/flutter/issues/77838 --- ci/licenses_golden/licenses_flutter | 8 ++ shell/platform/windows/BUILD.gn | 14 +++- .../accessibility_bridge_delegate_win32.cc | 37 +++++++++ .../accessibility_bridge_delegate_win32.h | 47 ++++++++++++ .../accessibility_bridge_delegate_winuwp.cc | 37 +++++++++ .../accessibility_bridge_delegate_winuwp.h | 47 ++++++++++++ .../flutter_platform_node_delegate_win32.cc | 75 +++++++++++++++++++ .../flutter_platform_node_delegate_win32.h | 47 ++++++++++++ .../flutter_platform_node_delegate_winuwp.cc | 67 +++++++++++++++++ .../flutter_platform_node_delegate_winuwp.h | 47 ++++++++++++ .../platform/windows/flutter_window_win32.cc | 4 + shell/platform/windows/flutter_window_win32.h | 3 + .../windows/flutter_window_win32_unittests.cc | 1 + .../windows/flutter_windows_engine.cc | 50 +++++++++++++ .../platform/windows/flutter_windows_engine.h | 10 +++ .../platform/windows/flutter_windows_view.cc | 5 ++ shell/platform/windows/flutter_windows_view.h | 3 + .../windows/testing/mock_window_win32.h | 1 + .../windows/window_binding_handler_delegate.h | 4 + shell/platform/windows/window_win32.cc | 9 ++- shell/platform/windows/window_win32.h | 4 + third_party/accessibility/ax/ax_node.h | 5 ++ 22 files changed, 520 insertions(+), 5 deletions(-) create mode 100644 shell/platform/windows/accessibility_bridge_delegate_win32.cc create mode 100644 shell/platform/windows/accessibility_bridge_delegate_win32.h create mode 100644 shell/platform/windows/accessibility_bridge_delegate_winuwp.cc create mode 100644 shell/platform/windows/accessibility_bridge_delegate_winuwp.h create mode 100644 shell/platform/windows/flutter_platform_node_delegate_win32.cc create mode 100644 shell/platform/windows/flutter_platform_node_delegate_win32.h create mode 100644 shell/platform/windows/flutter_platform_node_delegate_winuwp.cc create mode 100644 shell/platform/windows/flutter_platform_node_delegate_winuwp.h diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index e0b43fca37cf5..d2355b90c6a72 100755 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -1693,6 +1693,10 @@ FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/fl_texture_regi FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/fl_value.h FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/fl_view.h FILE: ../../../flutter/shell/platform/linux/public/flutter_linux/flutter_linux.h +FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_delegate_win32.cc +FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_delegate_win32.h +FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_delegate_winuwp.cc +FILE: ../../../flutter/shell/platform/windows/accessibility_bridge_delegate_winuwp.h FILE: ../../../flutter/shell/platform/windows/angle_surface_manager.cc FILE: ../../../flutter/shell/platform/windows/angle_surface_manager.h FILE: ../../../flutter/shell/platform/windows/client_wrapper/dart_project_unittests.cc @@ -1717,6 +1721,10 @@ FILE: ../../../flutter/shell/platform/windows/dpi_utils_win32_unittests.cc FILE: ../../../flutter/shell/platform/windows/external_texture_gl.cc FILE: ../../../flutter/shell/platform/windows/external_texture_gl.h FILE: ../../../flutter/shell/platform/windows/flutter_key_map.cc +FILE: ../../../flutter/shell/platform/windows/flutter_platform_node_delegate_win32.cc +FILE: ../../../flutter/shell/platform/windows/flutter_platform_node_delegate_win32.h +FILE: ../../../flutter/shell/platform/windows/flutter_platform_node_delegate_winuwp.cc +FILE: ../../../flutter/shell/platform/windows/flutter_platform_node_delegate_winuwp.h FILE: ../../../flutter/shell/platform/windows/flutter_project_bundle.cc FILE: ../../../flutter/shell/platform/windows/flutter_project_bundle.h FILE: ../../../flutter/shell/platform/windows/flutter_project_bundle_unittests.cc diff --git a/shell/platform/windows/BUILD.gn b/shell/platform/windows/BUILD.gn index 3cfcfdb81f0cc..f8968126f078f 100644 --- a/shell/platform/windows/BUILD.gn +++ b/shell/platform/windows/BUILD.gn @@ -89,8 +89,12 @@ source_set("flutter_windows_source") { # Target-specific sources. if (target_os == "winuwp") { sources += [ + "accessibility_bridge_delegate_winuwp.cc", + "accessibility_bridge_delegate_winuwp.h", "display_helper_winuwp.cc", "display_helper_winuwp.h", + "flutter_platform_node_delegate_winuwp.cc", + "flutter_platform_node_delegate_winuwp.h", "flutter_window_winuwp.cc", "flutter_window_winuwp.h", "flutter_windows_winuwp.cc", @@ -106,8 +110,12 @@ source_set("flutter_windows_source") { ] } else { sources += [ + "accessibility_bridge_delegate_win32.cc", + "accessibility_bridge_delegate_win32.h", "dpi_utils_win32.cc", "dpi_utils_win32.h", + "flutter_platform_node_delegate_win32.cc", + "flutter_platform_node_delegate_win32.h", "flutter_window_win32.cc", "flutter_window_win32.h", "flutter_windows_win32.cc", @@ -153,7 +161,10 @@ source_set("flutter_windows_source") { defines = [ "FLUTTER_ENGINE_NO_PROTOTYPES" ] } - public_deps = [ ":string_conversion" ] + public_deps = [ + ":string_conversion", + "//flutter/shell/platform/common:common_cpp_accessibility", + ] deps = [ ":flutter_windows_headers", @@ -163,7 +174,6 @@ source_set("flutter_windows_source") { "//flutter/shell/platform/common/client_wrapper:client_wrapper", "//flutter/shell/platform/embedder:embedder_as_internal_library", "//flutter/shell/platform/windows/client_wrapper:client_wrapper_windows", - "//flutter/third_party/accessibility", "//third_party/angle:libEGL_static", # the order of libEGL_static and # libGLESv2_static is important.. if # reversed, will cause a linker error diff --git a/shell/platform/windows/accessibility_bridge_delegate_win32.cc b/shell/platform/windows/accessibility_bridge_delegate_win32.cc new file mode 100644 index 0000000000000..518e52d4a96e6 --- /dev/null +++ b/shell/platform/windows/accessibility_bridge_delegate_win32.cc @@ -0,0 +1,37 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/windows/accessibility_bridge_delegate_win32.h" + +#include "flutter/shell/platform/windows/flutter_platform_node_delegate_win32.h" +#include "flutter/shell/platform/windows/flutter_windows_view.h" +#include "flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate_base.h" + +namespace flutter { + +AccessibilityBridgeDelegateWin32::AccessibilityBridgeDelegateWin32( + FlutterWindowsEngine* engine) + : engine_(engine) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 + assert(engine_); +} + +void AccessibilityBridgeDelegateWin32::OnAccessibilityEvent( + ui::AXEventGenerator::TargetedEvent targeted_event) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 +} + +void AccessibilityBridgeDelegateWin32::DispatchAccessibilityAction( + AccessibilityNodeId target, + FlutterSemanticsAction action, + fml::MallocMapping data) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 +} + +std::shared_ptr +AccessibilityBridgeDelegateWin32::CreateFlutterPlatformNodeDelegate() { + return std::make_shared(engine_); +} + +} // namespace flutter diff --git a/shell/platform/windows/accessibility_bridge_delegate_win32.h b/shell/platform/windows/accessibility_bridge_delegate_win32.h new file mode 100644 index 0000000000000..2a8da186e0267 --- /dev/null +++ b/shell/platform/windows/accessibility_bridge_delegate_win32.h @@ -0,0 +1,47 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_BRIDGE_DELEGATE_H_ +#define FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_BRIDGE_DELEGATE_H_ + +#include "flutter/shell/platform/common/accessibility_bridge.h" +#include "flutter/shell/platform/windows/flutter_windows_engine.h" + +namespace flutter { + +class FlutterWindowsEngine; + +// The Win32 implementation of AccessibilityBridgeDelegate. +// +// Handles requests from the accessibility bridge to interact with Windows +// accessibility APIs. This includes routing accessibility events fired from +// the framework to Windows, routing native Windows accessibility events to the +// framework, and creating Windows-specific FlutterPlatformNodeDelegate objects +// for each node in the semantics tree. +class AccessibilityBridgeDelegateWin32 + : public AccessibilityBridge::AccessibilityBridgeDelegate { + public: + explicit AccessibilityBridgeDelegateWin32(FlutterWindowsEngine* engine); + virtual ~AccessibilityBridgeDelegateWin32() = default; + + // |AccessibilityBridge::AccessibilityBridgeDelegate| + void OnAccessibilityEvent( + ui::AXEventGenerator::TargetedEvent targeted_event) override; + + // |AccessibilityBridge::AccessibilityBridgeDelegate| + void DispatchAccessibilityAction(AccessibilityNodeId target, + FlutterSemanticsAction action, + fml::MallocMapping data) override; + + // |AccessibilityBridge::AccessibilityBridgeDelegate| + std::shared_ptr + CreateFlutterPlatformNodeDelegate() override; + + private: + FlutterWindowsEngine* engine_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_BRIDGE_DELEGATE_H_ diff --git a/shell/platform/windows/accessibility_bridge_delegate_winuwp.cc b/shell/platform/windows/accessibility_bridge_delegate_winuwp.cc new file mode 100644 index 0000000000000..2dd82d4b45753 --- /dev/null +++ b/shell/platform/windows/accessibility_bridge_delegate_winuwp.cc @@ -0,0 +1,37 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/shell/platform/windows/accessibility_bridge_delegate_winuwp.h" + +#include "flutter/shell/platform/windows/flutter_platform_node_delegate_winuwp.h" +#include "flutter/shell/platform/windows/flutter_windows_view.h" +#include "flutter/third_party/accessibility/ax/platform/ax_platform_node_delegate_base.h" + +namespace flutter { + +AccessibilityBridgeDelegateWinUWP::AccessibilityBridgeDelegateWinUWP( + FlutterWindowsEngine* engine) + : engine_(engine) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/93928 + assert(engine_); +} + +void AccessibilityBridgeDelegateWinUWP::OnAccessibilityEvent( + ui::AXEventGenerator::TargetedEvent targeted_event) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/93928 +} + +void AccessibilityBridgeDelegateWinUWP::DispatchAccessibilityAction( + AccessibilityNodeId target, + FlutterSemanticsAction action, + fml::MallocMapping data) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/93928 +} + +std::shared_ptr +AccessibilityBridgeDelegateWinUWP::CreateFlutterPlatformNodeDelegate() { + return std::make_shared(engine_); +} + +} // namespace flutter diff --git a/shell/platform/windows/accessibility_bridge_delegate_winuwp.h b/shell/platform/windows/accessibility_bridge_delegate_winuwp.h new file mode 100644 index 0000000000000..b1bad7141a002 --- /dev/null +++ b/shell/platform/windows/accessibility_bridge_delegate_winuwp.h @@ -0,0 +1,47 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_BRIDGE_DELEGATE_H_ +#define FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_BRIDGE_DELEGATE_H_ + +#include "flutter/shell/platform/common/accessibility_bridge.h" +#include "flutter/shell/platform/windows/flutter_windows_engine.h" + +namespace flutter { + +class FlutterWindowsEngine; + +// The Windows UWP implementation of AccessibilityBridgeDelegate. +// +// Handles requests from the accessibility bridge to interact with Windows +// accessibility APIs. This includes routing accessibility events fired from +// the framework to Windows, routing native Windows accessibility events to the +// framework, and creating Windows-specific FlutterPlatformNodeDelegate objects +// for each node in the semantics tree. +class AccessibilityBridgeDelegateWinUWP + : public AccessibilityBridge::AccessibilityBridgeDelegate { + public: + explicit AccessibilityBridgeDelegateWinUWP(FlutterWindowsEngine* engine); + virtual ~AccessibilityBridgeDelegateWinUWP() = default; + + // |AccessibilityBridge::AccessibilityBridgeDelegate| + void OnAccessibilityEvent( + ui::AXEventGenerator::TargetedEvent targeted_event) override; + + // |AccessibilityBridge::AccessibilityBridgeDelegate| + void DispatchAccessibilityAction(AccessibilityNodeId target, + FlutterSemanticsAction action, + fml::MallocMapping data) override; + + // |AccessibilityBridge::AccessibilityBridgeDelegate| + std::shared_ptr + CreateFlutterPlatformNodeDelegate() override; + + private: + FlutterWindowsEngine* engine_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_WINDOWS_ACCESSIBILITY_BRIDGE_DELEGATE_H_ diff --git a/shell/platform/windows/flutter_platform_node_delegate_win32.cc b/shell/platform/windows/flutter_platform_node_delegate_win32.cc new file mode 100644 index 0000000000000..e52680742ddae --- /dev/null +++ b/shell/platform/windows/flutter_platform_node_delegate_win32.cc @@ -0,0 +1,75 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include "flutter/shell/platform/windows/flutter_platform_node_delegate_win32.h" + +#include "flutter/shell/platform/windows/flutter_windows_view.h" + +namespace flutter { + +FlutterPlatformNodeDelegateWin32::FlutterPlatformNodeDelegateWin32( + FlutterWindowsEngine* engine) + : engine_(engine) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 + assert(engine_); +} + +FlutterPlatformNodeDelegateWin32::~FlutterPlatformNodeDelegateWin32() { + if (ax_platform_node_) { + ax_platform_node_->Destroy(); + } +} + +// |ui::AXPlatformNodeDelegate| +void FlutterPlatformNodeDelegateWin32::Init(std::weak_ptr bridge, + ui::AXNode* node) { + FlutterPlatformNodeDelegate::Init(bridge, node); + ax_platform_node_ = ui::AXPlatformNode::Create(this); + assert(ax_platform_node_); +} + +// |ui::AXPlatformNodeDelegate| +gfx::NativeViewAccessible +FlutterPlatformNodeDelegateWin32::GetNativeViewAccessible() { + assert(ax_platform_node_); + return ax_platform_node_->GetNativeViewAccessible(); +} + +// |FlutterPlatformNodeDelegate| +gfx::NativeViewAccessible FlutterPlatformNodeDelegateWin32::GetParent() { + gfx::NativeViewAccessible parent = FlutterPlatformNodeDelegate::GetParent(); + if (parent) { + return parent; + } + assert(engine_); + FlutterWindowsView* view = engine_->view(); + if (!view) { + return nullptr; + } + HWND hwnd = view->GetPlatformWindow(); + if (!hwnd) { + return nullptr; + } + + IAccessible* iaccessible_parent; + if (SUCCEEDED(::AccessibleObjectFromWindow( + hwnd, OBJID_WINDOW, IID_IAccessible, + reinterpret_cast(&iaccessible_parent)))) { + return iaccessible_parent; + } + return nullptr; +} + +// |FlutterPlatformNodeDelegate| +gfx::Rect FlutterPlatformNodeDelegateWin32::GetBoundsRect( + const ui::AXCoordinateSystem coordinate_system, + const ui::AXClippingBehavior clipping_behavior, + ui::AXOffscreenResult* offscreen_result) const { + // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 + return {}; +} + +} // namespace flutter diff --git a/shell/platform/windows/flutter_platform_node_delegate_win32.h b/shell/platform/windows/flutter_platform_node_delegate_win32.h new file mode 100644 index 0000000000000..964dcdf2cf510 --- /dev/null +++ b/shell/platform/windows/flutter_platform_node_delegate_win32.h @@ -0,0 +1,47 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_FLUTTER_PLATFORM_NODE_DELEGATE_H_ +#define FLUTTER_SHELL_PLATFORM_WINDOWS_FLUTTER_PLATFORM_NODE_DELEGATE_H_ + +#include "flutter/shell/platform/common/flutter_platform_node_delegate.h" +#include "flutter/shell/platform/windows/flutter_windows_engine.h" +#include "flutter/third_party/accessibility/ax/platform/ax_platform_node.h" + +namespace flutter { + +class FlutterWindowsEngine; + +// The Win32 implementation of FlutterPlatformNodeDelegate. +// +// This class implements a wrapper around the Win32 accessibility objects that +// compose the accessibility tree. +class FlutterPlatformNodeDelegateWin32 : public FlutterPlatformNodeDelegate { + public: + explicit FlutterPlatformNodeDelegateWin32(FlutterWindowsEngine* engine); + virtual ~FlutterPlatformNodeDelegateWin32(); + + // |ui::AXPlatformNodeDelegate| + void Init(std::weak_ptr bridge, ui::AXNode* node) override; + + // |ui::AXPlatformNodeDelegate| + gfx::NativeViewAccessible GetNativeViewAccessible() override; + + // |FlutterPlatformNodeDelegate| + gfx::NativeViewAccessible GetParent() override; + + // |FlutterPlatformNodeDelegate| + gfx::Rect GetBoundsRect( + const ui::AXCoordinateSystem coordinate_system, + const ui::AXClippingBehavior clipping_behavior, + ui::AXOffscreenResult* offscreen_result) const override; + + private: + ui::AXPlatformNode* ax_platform_node_; + FlutterWindowsEngine* engine_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_WINDOWS_FLUTTER_PLATFORM_NODE_DELEGATE_H_ diff --git a/shell/platform/windows/flutter_platform_node_delegate_winuwp.cc b/shell/platform/windows/flutter_platform_node_delegate_winuwp.cc new file mode 100644 index 0000000000000..9b102abf99998 --- /dev/null +++ b/shell/platform/windows/flutter_platform_node_delegate_winuwp.cc @@ -0,0 +1,67 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include "flutter/shell/platform/windows/flutter_platform_node_delegate_winuwp.h" + +#include "flutter/shell/platform/windows/flutter_windows_view.h" + +namespace flutter { + +FlutterPlatformNodeDelegateWinUWP::FlutterPlatformNodeDelegateWinUWP( + FlutterWindowsEngine* engine) + : engine_(engine) { + // TODO(cbracken): https://github.com/flutter/flutter/issues/93928 + assert(engine_); +} + +FlutterPlatformNodeDelegateWinUWP::~FlutterPlatformNodeDelegateWinUWP() { + if (ax_platform_node_) { + ax_platform_node_->Destroy(); + } +} + +// |ui::AXPlatformNodeDelegate| +void FlutterPlatformNodeDelegateWinUWP::Init(std::weak_ptr bridge, + ui::AXNode* node) { + FlutterPlatformNodeDelegate::Init(bridge, node); + ax_platform_node_ = ui::AXPlatformNode::Create(this); + assert(ax_platform_node_); +} + +// |ui::AXPlatformNodeDelegate| +gfx::NativeViewAccessible +FlutterPlatformNodeDelegateWinUWP::GetNativeViewAccessible() { + assert(ax_platform_node_); + return ax_platform_node_->GetNativeViewAccessible(); +} + +// |FlutterPlatformNodeDelegate| +gfx::NativeViewAccessible FlutterPlatformNodeDelegateWinUWP::GetParent() { + gfx::NativeViewAccessible parent = FlutterPlatformNodeDelegate::GetParent(); + if (parent) { + return parent; + } + assert(engine_); + FlutterWindowsView* view = engine_->view(); + if (!view) { + return nullptr; + } + // TODO(cbracken): https://github.com/flutter/flutter/issues/93928 + // Use FlutterWindowsView::GetPlatformView to get the root view, and return + // the associated accessibility object. + return nullptr; +} + +// |FlutterPlatformNodeDelegate| +gfx::Rect FlutterPlatformNodeDelegateWinUWP::GetBoundsRect( + const ui::AXCoordinateSystem coordinate_system, + const ui::AXClippingBehavior clipping_behavior, + ui::AXOffscreenResult* offscreen_result) const { + // TODO(cbracken): https://github.com/flutter/flutter/issues/93928 + return {}; +} + +} // namespace flutter diff --git a/shell/platform/windows/flutter_platform_node_delegate_winuwp.h b/shell/platform/windows/flutter_platform_node_delegate_winuwp.h new file mode 100644 index 0000000000000..ac1f5645c30e5 --- /dev/null +++ b/shell/platform/windows/flutter_platform_node_delegate_winuwp.h @@ -0,0 +1,47 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_FLUTTER_PLATFORM_NODE_DELEGATE_H_ +#define FLUTTER_SHELL_PLATFORM_WINDOWS_FLUTTER_PLATFORM_NODE_DELEGATE_H_ + +#include "flutter/shell/platform/common/flutter_platform_node_delegate.h" +#include "flutter/shell/platform/windows/flutter_windows_engine.h" +#include "flutter/third_party/accessibility/ax/platform/ax_platform_node.h" + +namespace flutter { + +class FlutterWindowsEngine; + +// The Windows UWP implementation of FlutterPlatformNodeDelegate. +// +// This class implements a wrapper around the Windows UWP accessibility objects +// that compose the accessibility tree. +class FlutterPlatformNodeDelegateWinUWP : public FlutterPlatformNodeDelegate { + public: + explicit FlutterPlatformNodeDelegateWinUWP(FlutterWindowsEngine* engine); + virtual ~FlutterPlatformNodeDelegateWinUWP(); + + // |ui::AXPlatformNodeDelegate| + void Init(std::weak_ptr bridge, ui::AXNode* node) override; + + // |ui::AXPlatformNodeDelegate| + gfx::NativeViewAccessible GetNativeViewAccessible() override; + + // |FlutterPlatformNodeDelegate| + gfx::NativeViewAccessible GetParent() override; + + // |FlutterPlatformNodeDelegate| + gfx::Rect GetBoundsRect( + const ui::AXCoordinateSystem coordinate_system, + const ui::AXClippingBehavior clipping_behavior, + ui::AXOffscreenResult* offscreen_result) const override; + + private: + ui::AXPlatformNode* ax_platform_node_; + FlutterWindowsEngine* engine_; +}; + +} // namespace flutter + +#endif // FLUTTER_SHELL_PLATFORM_WINDOWS_FLUTTER_PLATFORM_NODE_DELEGATE_H_ diff --git a/shell/platform/windows/flutter_window_win32.cc b/shell/platform/windows/flutter_window_win32.cc index 07a71e8ab6206..8c72d6b273094 100644 --- a/shell/platform/windows/flutter_window_win32.cc +++ b/shell/platform/windows/flutter_window_win32.cc @@ -258,4 +258,8 @@ bool FlutterWindowWin32::OnBitmapSurfaceUpdated(const void* allocation, return ret != 0; } +gfx::NativeViewAccessible FlutterWindowWin32::GetNativeViewAccessible() { + return binding_handler_delegate_->GetNativeViewAccessible(); +} + } // namespace flutter diff --git a/shell/platform/windows/flutter_window_win32.h b/shell/platform/windows/flutter_window_win32.h index bb867e3b93451..584daa4a932dc 100644 --- a/shell/platform/windows/flutter_window_win32.h +++ b/shell/platform/windows/flutter_window_win32.h @@ -101,6 +101,9 @@ class FlutterWindowWin32 : public WindowWin32, public WindowBindingHandler { FlutterPointerDeviceKind device_kind, int32_t device_id) override; + // |WindowWin32| + gfx::NativeViewAccessible GetNativeViewAccessible() override; + // |FlutterWindowBindingHandler| void SetView(WindowBindingHandlerDelegate* view) override; diff --git a/shell/platform/windows/flutter_window_win32_unittests.cc b/shell/platform/windows/flutter_window_win32_unittests.cc index 1df68eb7ca899..c61e6bfda3791 100644 --- a/shell/platform/windows/flutter_window_win32_unittests.cc +++ b/shell/platform/windows/flutter_window_win32_unittests.cc @@ -217,6 +217,7 @@ class MockWindowBindingHandlerDelegate : public WindowBindingHandlerDelegate { MOCK_METHOD0(OnComposeEnd, void()); MOCK_METHOD2(OnComposeChange, void(const std::u16string&, int)); MOCK_METHOD1(OnUpdateSemanticsEnabled, void(bool)); + MOCK_METHOD0(GetNativeViewAccessible, gfx::NativeViewAccessible()); MOCK_METHOD7(OnScroll, void(double, double, diff --git a/shell/platform/windows/flutter_windows_engine.cc b/shell/platform/windows/flutter_windows_engine.cc index bb020d52f4ccc..174f1a170f995 100644 --- a/shell/platform/windows/flutter_windows_engine.cc +++ b/shell/platform/windows/flutter_windows_engine.cc @@ -18,6 +18,12 @@ #include "flutter/shell/platform/windows/task_runner.h" #include "third_party/rapidjson/include/rapidjson/document.h" +#if defined(WINUWP) +#include "flutter/shell/platform/windows/accessibility_bridge_delegate_winuwp.h" +#else +#include "flutter/shell/platform/windows/accessibility_bridge_delegate_win32.h" +#endif // defined(WINUWP) + namespace flutter { namespace { @@ -260,6 +266,24 @@ bool FlutterWindowsEngine::RunWithEntrypoint(const char* entrypoint) { auto host = static_cast(user_data); host->view()->OnPreEngineRestart(); }; + args.update_semantics_node_callback = [](const FlutterSemanticsNode* node, + void* user_data) { + auto host = static_cast(user_data); + if (!node || node->id == kFlutterSemanticsNodeIdBatchEnd) { + host->accessibility_bridge_->CommitUpdates(); + return; + } + host->accessibility_bridge_->AddFlutterSemanticsNodeUpdate(node); + }; + args.update_semantics_custom_action_callback = + [](const FlutterSemanticsCustomAction* action, void* user_data) { + auto host = static_cast(user_data); + if (!action || action->id == kFlutterSemanticsNodeIdBatchEnd) { + host->accessibility_bridge_->CommitUpdates(); + return; + } + // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 + }; args.custom_task_runners = &custom_task_runners; @@ -452,10 +476,36 @@ bool FlutterWindowsEngine::DispatchSemanticsAction( } void FlutterWindowsEngine::UpdateSemanticsEnabled(bool enabled) { +#if defined(WINUWP) + using AccessibilityBridgeDelegateWindows = AccessibilityBridgeDelegateWinUWP; +#else + using AccessibilityBridgeDelegateWindows = AccessibilityBridgeDelegateWin32; +#endif // defined(WINUWP) + if (engine_ && semantics_enabled_ != enabled) { semantics_enabled_ = enabled; embedder_api_.UpdateSemanticsEnabled(engine_, enabled); + + if (!semantics_enabled_ && accessibility_bridge_) { + accessibility_bridge_.reset(); + } else if (semantics_enabled_ && !accessibility_bridge_) { + accessibility_bridge_ = std::make_shared( + std::make_unique(this)); + } + } +} + +gfx::NativeViewAccessible FlutterWindowsEngine::GetNativeAccessibleFromId( + AccessibilityNodeId id) { + if (!accessibility_bridge_) { + return nullptr; + } + std::shared_ptr node_delegate = + accessibility_bridge_->GetFlutterPlatformNodeDelegateFromID(id).lock(); + if (!node_delegate) { + return nullptr; } + return node_delegate->GetNativeViewAccessible(); } } // namespace flutter diff --git a/shell/platform/windows/flutter_windows_engine.h b/shell/platform/windows/flutter_windows_engine.h index 22e75149e9349..adb1a8455b7c1 100644 --- a/shell/platform/windows/flutter_windows_engine.h +++ b/shell/platform/windows/flutter_windows_engine.h @@ -10,6 +10,7 @@ #include #include +#include "flutter/shell/platform/common/accessibility_bridge.h" #include "flutter/shell/platform/common/client_wrapper/binary_messenger_impl.h" #include "flutter/shell/platform/common/client_wrapper/include/flutter/basic_message_channel.h" #include "flutter/shell/platform/common/incoming_message_dispatcher.h" @@ -93,6 +94,10 @@ class FlutterWindowsEngine { // rendering using software instead of OpenGL. AngleSurfaceManager* surface_manager() { return surface_manager_.get(); } + AccessibilityBridge* accessibility_bridge() { + return accessibility_bridge_.get(); + } + #ifndef WINUWP WindowProcDelegateManagerWin32* window_proc_delegate_manager() { return window_proc_delegate_manager_.get(); @@ -155,6 +160,9 @@ class FlutterWindowsEngine { // Returns true if the semantics tree is enabled. bool semantics_enabled() const { return semantics_enabled_; } + // Returns the native accessibility node with the given id. + gfx::NativeViewAccessible GetNativeAccessibleFromId(AccessibilityNodeId id); + private: // Allows swapping out embedder_api_ calls in tests. friend class EngineModifier; @@ -214,6 +222,8 @@ class FlutterWindowsEngine { bool semantics_enabled_ = false; + std::shared_ptr accessibility_bridge_; + #ifndef WINUWP // The manager for WindowProc delegate registration and callbacks. std::unique_ptr window_proc_delegate_manager_; diff --git a/shell/platform/windows/flutter_windows_view.cc b/shell/platform/windows/flutter_windows_view.cc index 91783fcdfadb3..0532cfcaa734f 100644 --- a/shell/platform/windows/flutter_windows_view.cc +++ b/shell/platform/windows/flutter_windows_view.cc @@ -6,6 +6,7 @@ #include +#include "flutter/shell/platform/common/accessibility_bridge.h" #include "flutter/shell/platform/windows/keyboard_key_channel_handler.h" #include "flutter/shell/platform/windows/keyboard_key_embedder_handler.h" #include "flutter/shell/platform/windows/text_input_plugin.h" @@ -252,6 +253,10 @@ void FlutterWindowsView::OnUpdateSemanticsEnabled(bool enabled) { engine_->UpdateSemanticsEnabled(enabled); } +gfx::NativeViewAccessible FlutterWindowsView::GetNativeViewAccessible() { + return engine_->GetNativeAccessibleFromId(AccessibilityBridge::kRootNodeId); +} + void FlutterWindowsView::OnCursorRectUpdated(const Rect& rect) { binding_handler_->OnCursorRectUpdated(rect); } diff --git a/shell/platform/windows/flutter_windows_view.h b/shell/platform/windows/flutter_windows_view.h index c18032e709eca..e9a542cb5970e 100644 --- a/shell/platform/windows/flutter_windows_view.h +++ b/shell/platform/windows/flutter_windows_view.h @@ -161,6 +161,9 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate, // |WindowBindingHandlerDelegate| virtual void OnUpdateSemanticsEnabled(bool enabled) override; + // |WindowBindingHandlerDelegate| + virtual gfx::NativeViewAccessible GetNativeViewAccessible() override; + // |TextInputPluginDelegate| void OnCursorRectUpdated(const Rect& rect) override; diff --git a/shell/platform/windows/testing/mock_window_win32.h b/shell/platform/windows/testing/mock_window_win32.h index 406cf1b9b04a7..6c77ab1e85cc6 100644 --- a/shell/platform/windows/testing/mock_window_win32.h +++ b/shell/platform/windows/testing/mock_window_win32.h @@ -46,6 +46,7 @@ class MockWin32Window : public WindowWin32, public MockMessageQueue { MOCK_METHOD1(OnText, void(const std::u16string&)); MOCK_METHOD6(OnKey, bool(int, int, int, char32_t, bool, bool)); MOCK_METHOD1(OnUpdateSemanticsEnabled, void(bool)); + MOCK_METHOD0(GetNativeViewAccessible, gfx::NativeViewAccessible()); MOCK_METHOD4(OnScroll, void(double, double, FlutterPointerDeviceKind, int32_t)); MOCK_METHOD0(OnComposeBegin, void()); diff --git a/shell/platform/windows/window_binding_handler_delegate.h b/shell/platform/windows/window_binding_handler_delegate.h index 3b58e4aed93e8..177724e596168 100644 --- a/shell/platform/windows/window_binding_handler_delegate.h +++ b/shell/platform/windows/window_binding_handler_delegate.h @@ -7,6 +7,7 @@ #include "flutter/shell/platform/common/geometry.h" #include "flutter/shell/platform/embedder/embedder.h" +#include "flutter/third_party/accessibility/gfx/native_widget_types.h" namespace flutter { @@ -102,6 +103,9 @@ class WindowBindingHandlerDelegate { // Notifies delegate that the Flutter semantics tree should be enabled or // disabled. virtual void OnUpdateSemanticsEnabled(bool enabled) = 0; + + // Returns the root view accessibility node, or nullptr if none. + virtual gfx::NativeViewAccessible GetNativeViewAccessible() = 0; }; } // namespace flutter diff --git a/shell/platform/windows/window_win32.cc b/shell/platform/windows/window_win32.cc index c1cebf8514bc1..3cee361545309 100644 --- a/shell/platform/windows/window_win32.cc +++ b/shell/platform/windows/window_win32.cc @@ -5,6 +5,7 @@ #include "flutter/shell/platform/windows/window_win32.h" #include +#include #include @@ -167,9 +168,11 @@ void WindowWin32::OnGetObject(UINT const message, // Microsoft Active Accessibility (MSAA) COM objects. OnUpdateSemanticsEnabled(true); - // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 - // Once AccessibilityBridge is wired up, look up the IAccessible - // representing the root view and call LresultFromObject. + // Return the IAccessible for the root view. + gfx::NativeViewAccessible root_iaccessible = GetNativeViewAccessible(); + if (root_iaccessible) { + LresultFromObject(IID_IAccessible, wparam, root_iaccessible); + } } } diff --git a/shell/platform/windows/window_win32.h b/shell/platform/windows/window_win32.h index c106b82819eb7..48f3ea8308353 100644 --- a/shell/platform/windows/window_win32.h +++ b/shell/platform/windows/window_win32.h @@ -16,6 +16,7 @@ #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/windows/sequential_id_generator.h" #include "flutter/shell/platform/windows/text_input_manager_win32.h" +#include "flutter/third_party/accessibility/gfx/native_widget_types.h" namespace flutter { @@ -218,6 +219,9 @@ class WindowWin32 { // Used to process key messages. Exposed for dependency injection. virtual uint32_t Win32MapVkToChar(uint32_t virtual_key); + // Returns the root view accessibility node, or nullptr if none. + virtual gfx::NativeViewAccessible GetNativeViewAccessible() = 0; + private: // Release OS resources associated with window. void Destroy(); diff --git a/third_party/accessibility/ax/ax_node.h b/third_party/accessibility/ax/ax_node.h index 9928095670f0b..0cae10f933da5 100644 --- a/third_party/accessibility/ax/ax_node.h +++ b/third_party/accessibility/ax/ax_node.h @@ -20,6 +20,11 @@ #include "gfx/geometry/rect.h" #include "gfx/transform.h" +#ifdef _WIN32 +// windowx.h defines GetNextSibling as a macro. +#undef GetNextSibling +#endif + namespace ui { class AXTableInfo; From 512013ccc0c92f8f3be939f15a287c4538659086 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Mon, 22 Nov 2021 11:44:56 -0800 Subject: [PATCH 02/12] review: return weak_ptr --- shell/platform/windows/flutter_windows_engine.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell/platform/windows/flutter_windows_engine.h b/shell/platform/windows/flutter_windows_engine.h index adb1a8455b7c1..5386a914b7e1d 100644 --- a/shell/platform/windows/flutter_windows_engine.h +++ b/shell/platform/windows/flutter_windows_engine.h @@ -94,8 +94,8 @@ class FlutterWindowsEngine { // rendering using software instead of OpenGL. AngleSurfaceManager* surface_manager() { return surface_manager_.get(); } - AccessibilityBridge* accessibility_bridge() { - return accessibility_bridge_.get(); + std::weak_ptr accessibility_bridge() { + return accessibility_bridge_; } #ifndef WINUWP From b23e78be026f6a17152a4087b7251ea1f7671873 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Mon, 22 Nov 2021 11:50:18 -0800 Subject: [PATCH 03/12] Add custom action update --- shell/platform/windows/flutter_windows_engine.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shell/platform/windows/flutter_windows_engine.cc b/shell/platform/windows/flutter_windows_engine.cc index 174f1a170f995..7b138f5ae7289 100644 --- a/shell/platform/windows/flutter_windows_engine.cc +++ b/shell/platform/windows/flutter_windows_engine.cc @@ -282,7 +282,8 @@ bool FlutterWindowsEngine::RunWithEntrypoint(const char* entrypoint) { host->accessibility_bridge_->CommitUpdates(); return; } - // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 + host->accessibility_bridge_->AddFlutterSemanticsCustomActionUpdate( + action); }; args.custom_task_runners = &custom_task_runners; From 3ab733c42a094ca279bed6aa1477b43b35669a07 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Wed, 24 Nov 2021 11:02:04 -0800 Subject: [PATCH 04/12] Implement FlutterPlatformNodeDelegateWin32::GetBoundsRect --- .../windows/flutter_platform_node_delegate_win32.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/shell/platform/windows/flutter_platform_node_delegate_win32.cc b/shell/platform/windows/flutter_platform_node_delegate_win32.cc index e52680742ddae..84cb9abd4b0e7 100644 --- a/shell/platform/windows/flutter_platform_node_delegate_win32.cc +++ b/shell/platform/windows/flutter_platform_node_delegate_win32.cc @@ -68,8 +68,15 @@ gfx::Rect FlutterPlatformNodeDelegateWin32::GetBoundsRect( const ui::AXCoordinateSystem coordinate_system, const ui::AXClippingBehavior clipping_behavior, ui::AXOffscreenResult* offscreen_result) const { - // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 - return {}; + gfx::Rect local_bounds = FlutterPlatformNodeDelegate::GetBoundsRect( + coordinate_system, clipping_behavior, offscreen_result); + POINT origin{local_bounds.x(), local_bounds.y()}; + POINT extent{local_bounds.x() + local_bounds.width(), + local_bounds.y() + local_bounds.height()}; + ClientToScreen(engine_->view()->GetPlatformWindow(), &origin); + ClientToScreen(engine_->view()->GetPlatformWindow(), &extent); + return gfx::Rect(origin.x, origin.y, extent.x - origin.x, + extent.y - origin.y); } } // namespace flutter From 384f0a0f9aee264ba3cb81226d8747c51ad9fa14 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Wed, 24 Nov 2021 08:55:08 -0800 Subject: [PATCH 05/12] Add UIA COM object lookup --- shell/platform/windows/window_win32.cc | 39 +++++++++++++++++++------- shell/platform/windows/window_win32.h | 6 ++-- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/shell/platform/windows/window_win32.cc b/shell/platform/windows/window_win32.cc index 3cee361545309..9c373a8e3d706 100644 --- a/shell/platform/windows/window_win32.cc +++ b/shell/platform/windows/window_win32.cc @@ -4,8 +4,13 @@ #include "flutter/shell/platform/windows/window_win32.h" +#include "base/win/atl.h" // NOLINT(build/include_order) + #include #include +#include +#include +#include #include @@ -147,17 +152,19 @@ void WindowWin32::TrackMouseLeaveEvent(HWND hwnd) { } } -void WindowWin32::OnGetObject(UINT const message, - WPARAM const wparam, - LPARAM const lparam) { +LRESULT WindowWin32::OnGetObject(UINT const message, + WPARAM const wparam, + LPARAM const lparam) { LRESULT reference_result = static_cast(0L); // Only the lower 32 bits of lparam are valid when checking the object id // because it sometimes gets sign-extended incorrectly (but not always). DWORD obj_id = static_cast(static_cast(lparam)); + bool is_uia_request = static_cast(UiaRootObjectId) == obj_id; bool is_msaa_request = static_cast(OBJID_CLIENT) == obj_id; - if (is_msaa_request) { + + if (is_uia_request || is_msaa_request) { // On Windows, we don't get a notification that the screen reader has been // enabled or disabled. There is an API to query for screen reader state, // but that state isn't set by all screen readers, including by Narrator, @@ -167,13 +174,21 @@ void WindowWin32::OnGetObject(UINT const message, // Instead, we enable semantics in Flutter if Windows issues queries for // Microsoft Active Accessibility (MSAA) COM objects. OnUpdateSemanticsEnabled(true); + } + gfx::NativeViewAccessible root_view = GetNativeViewAccessible(); + if (is_uia_request && root_view) { + Microsoft::WRL::ComPtr root; + root_view->QueryInterface(IID_PPV_ARGS(&root)); + LRESULT lresult = UiaReturnRawElementProvider(window_handle_, wparam, lparam, root.Get()); + return lresult; + } else if (is_msaa_request && root_view) { // Return the IAccessible for the root view. - gfx::NativeViewAccessible root_iaccessible = GetNativeViewAccessible(); - if (root_iaccessible) { - LresultFromObject(IID_IAccessible, wparam, root_iaccessible); - } + Microsoft::WRL::ComPtr root(root_view); + LRESULT lresult = LresultFromObject(IID_IAccessible, wparam, root.Get()); + return lresult; } + return 0; } void WindowWin32::OnImeSetContext(UINT const message, @@ -415,9 +430,13 @@ WindowWin32::HandleMessage(UINT const message, static_cast(WHEEL_DELTA)), 0.0, kFlutterPointerDeviceKindMouse, kDefaultPointerDeviceId); break; - case WM_GETOBJECT: - OnGetObject(message, wparam, lparam); + case WM_GETOBJECT: { + LRESULT lresult = OnGetObject(message, wparam, lparam); + if (lresult) { + return lresult; + } break; + } case WM_INPUTLANGCHANGE: // TODO(cbracken): pass this to TextInputManager to aid with // language-specific issues. diff --git a/shell/platform/windows/window_win32.h b/shell/platform/windows/window_win32.h index 48f3ea8308353..b8d97011d57aa 100644 --- a/shell/platform/windows/window_win32.h +++ b/shell/platform/windows/window_win32.h @@ -127,9 +127,9 @@ class WindowWin32 { // // The primary use of this function is to supply Windows with wrapped // semantics objects for use by Windows accessibility. - void OnGetObject(UINT const message, - WPARAM const wparam, - LPARAM const lparam); + LRESULT OnGetObject(UINT const message, + WPARAM const wparam, + LPARAM const lparam); // Called when IME composing begins. virtual void OnComposeBegin() = 0; From d58c4c74d0f36bb94d3ec7ec00e69997c7d61d9f Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Wed, 24 Nov 2021 11:19:53 -0800 Subject: [PATCH 06/12] Formatting --- shell/platform/windows/window_win32.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shell/platform/windows/window_win32.cc b/shell/platform/windows/window_win32.cc index 9c373a8e3d706..a9e37ef9ae664 100644 --- a/shell/platform/windows/window_win32.cc +++ b/shell/platform/windows/window_win32.cc @@ -180,7 +180,8 @@ LRESULT WindowWin32::OnGetObject(UINT const message, if (is_uia_request && root_view) { Microsoft::WRL::ComPtr root; root_view->QueryInterface(IID_PPV_ARGS(&root)); - LRESULT lresult = UiaReturnRawElementProvider(window_handle_, wparam, lparam, root.Get()); + LRESULT lresult = + UiaReturnRawElementProvider(window_handle_, wparam, lparam, root.Get()); return lresult; } else if (is_msaa_request && root_view) { // Return the IAccessible for the root view. From 1d040599ec6d34c3deaaf78092f06e761cddd0a6 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Wed, 24 Nov 2021 11:29:26 -0800 Subject: [PATCH 07/12] Eliminate ctor TODOs There's nothing to do here. --- shell/platform/windows/accessibility_bridge_delegate_win32.cc | 1 - shell/platform/windows/flutter_platform_node_delegate_win32.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/shell/platform/windows/accessibility_bridge_delegate_win32.cc b/shell/platform/windows/accessibility_bridge_delegate_win32.cc index 518e52d4a96e6..cb9361747bce3 100644 --- a/shell/platform/windows/accessibility_bridge_delegate_win32.cc +++ b/shell/platform/windows/accessibility_bridge_delegate_win32.cc @@ -13,7 +13,6 @@ namespace flutter { AccessibilityBridgeDelegateWin32::AccessibilityBridgeDelegateWin32( FlutterWindowsEngine* engine) : engine_(engine) { - // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 assert(engine_); } diff --git a/shell/platform/windows/flutter_platform_node_delegate_win32.cc b/shell/platform/windows/flutter_platform_node_delegate_win32.cc index 84cb9abd4b0e7..b3eafcda305c1 100644 --- a/shell/platform/windows/flutter_platform_node_delegate_win32.cc +++ b/shell/platform/windows/flutter_platform_node_delegate_win32.cc @@ -13,7 +13,6 @@ namespace flutter { FlutterPlatformNodeDelegateWin32::FlutterPlatformNodeDelegateWin32( FlutterWindowsEngine* engine) : engine_(engine) { - // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 assert(engine_); } From c405bf47170788544f9ef0feef7b3b6f44d15967 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Wed, 24 Nov 2021 16:39:35 -0800 Subject: [PATCH 08/12] Formatting --- .../windows/flutter_platform_node_delegate_win32.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/shell/platform/windows/flutter_platform_node_delegate_win32.cc b/shell/platform/windows/flutter_platform_node_delegate_win32.cc index b3eafcda305c1..05ed7b156eea5 100644 --- a/shell/platform/windows/flutter_platform_node_delegate_win32.cc +++ b/shell/platform/windows/flutter_platform_node_delegate_win32.cc @@ -67,11 +67,10 @@ gfx::Rect FlutterPlatformNodeDelegateWin32::GetBoundsRect( const ui::AXCoordinateSystem coordinate_system, const ui::AXClippingBehavior clipping_behavior, ui::AXOffscreenResult* offscreen_result) const { - gfx::Rect local_bounds = FlutterPlatformNodeDelegate::GetBoundsRect( + gfx::Rect bounds = FlutterPlatformNodeDelegate::GetBoundsRect( coordinate_system, clipping_behavior, offscreen_result); - POINT origin{local_bounds.x(), local_bounds.y()}; - POINT extent{local_bounds.x() + local_bounds.width(), - local_bounds.y() + local_bounds.height()}; + POINT origin{bounds.x(), bounds.y()}; + POINT extent{bounds.x() + bounds.width(), bounds.y() + bounds.height()}; ClientToScreen(engine_->view()->GetPlatformWindow(), &origin); ClientToScreen(engine_->view()->GetPlatformWindow(), &extent); return gfx::Rect(origin.x, origin.y, extent.x - origin.x, From c687518de6959f2e8c86af62f3778e6b7768d06a Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Mon, 29 Nov 2021 11:18:39 -0800 Subject: [PATCH 09/12] Add unittest to verify axtree construction Verify that a windows IAccessible COM object is generated. --- .../windows/flutter_windows_view_unittests.cc | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/shell/platform/windows/flutter_windows_view_unittests.cc b/shell/platform/windows/flutter_windows_view_unittests.cc index 77343522b2363..6bbbb4a14e48c 100644 --- a/shell/platform/windows/flutter_windows_view_unittests.cc +++ b/shell/platform/windows/flutter_windows_view_unittests.cc @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "flutter/shell/platform/windows/flutter_windows_view.h" + +#include +#include +#include + #include #include @@ -9,7 +15,6 @@ #include "flutter/shell/platform/embedder/test_utils/proc_table_replacement.h" #include "flutter/shell/platform/windows/flutter_windows_engine.h" #include "flutter/shell/platform/windows/flutter_windows_texture_registrar.h" -#include "flutter/shell/platform/windows/flutter_windows_view.h" #include "flutter/shell/platform/windows/testing/engine_modifier.h" #include "flutter/shell/platform/windows/testing/mock_window_binding_handler.h" #include "flutter/shell/platform/windows/testing/test_keyboard.h" @@ -145,5 +150,55 @@ TEST(FlutterWindowsViewTest, EnableSemantics) { EXPECT_TRUE(semantics_enabled); } +TEST(FlutterWindowsEngine, AddSemanticsNodeUpdate) { + std::unique_ptr engine = GetTestEngine(); + EngineModifier modifier(engine.get()); + modifier.embedder_api().UpdateSemanticsEnabled = + [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) { + return kSuccess; + }; + + auto window_binding_handler = + std::make_unique<::testing::NiceMock>(); + FlutterWindowsView view(std::move(window_binding_handler)); + view.SetEngine(std::move(engine)); + + // Enable semantics to instantiate accessibility bridge. + view.OnUpdateSemanticsEnabled(true); + + auto bridge = view.GetEngine()->accessibility_bridge().lock(); + ASSERT_TRUE(bridge); + + // Add root node. + FlutterSemanticsNode node{sizeof(FlutterSemanticsNode), 0}; + node.label = "name"; + node.hint = "hint"; + node.value = "value"; + node.platform_view_id = -1; + bridge->AddFlutterSemanticsNodeUpdate(&node); + bridge->CommitUpdates(); + + // Look up the root windows node delegate. + auto node_delegate = bridge + ->GetFlutterPlatformNodeDelegateFromID( + AccessibilityBridge::kRootNodeId) + .lock(); + ASSERT_TRUE(node_delegate); + EXPECT_EQ(node_delegate->GetChildCount(), 0); + + // Get the native IAccessible object. + IAccessible* native_view = node_delegate->GetNativeViewAccessible(); + ASSERT_TRUE(native_view != nullptr); + + // Verify node name matches our label. + BSTR bname = nullptr; + VARIANT varchild{}; + varchild.vt = VT_I4; + varchild.lVal = 0; + native_view->get_accName(varchild, &bname); + std::string name(_com_util::ConvertBSTRToString(bname)); + ASSERT_EQ(name, "name"); +} + } // namespace testing } // namespace flutter From 2c8b6cc1200618b1c8f361bf25d34800bb8da75d Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Mon, 29 Nov 2021 11:33:33 -0800 Subject: [PATCH 10/12] Test value as well --- .../windows/flutter_windows_view_unittests.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/shell/platform/windows/flutter_windows_view_unittests.cc b/shell/platform/windows/flutter_windows_view_unittests.cc index 6bbbb4a14e48c..bbd1ad6882071 100644 --- a/shell/platform/windows/flutter_windows_view_unittests.cc +++ b/shell/platform/windows/flutter_windows_view_unittests.cc @@ -172,7 +172,6 @@ TEST(FlutterWindowsEngine, AddSemanticsNodeUpdate) { // Add root node. FlutterSemanticsNode node{sizeof(FlutterSemanticsNode), 0}; node.label = "name"; - node.hint = "hint"; node.value = "value"; node.platform_view_id = -1; bridge->AddFlutterSemanticsNodeUpdate(&node); @@ -192,12 +191,19 @@ TEST(FlutterWindowsEngine, AddSemanticsNodeUpdate) { // Verify node name matches our label. BSTR bname = nullptr; - VARIANT varchild{}; - varchild.vt = VT_I4; - varchild.lVal = 0; - native_view->get_accName(varchild, &bname); + VARIANT varvalue{}; + varvalue.vt = VT_I4; + varvalue.lVal = 0; + native_view->get_accName(varvalue, &bname); std::string name(_com_util::ConvertBSTRToString(bname)); ASSERT_EQ(name, "name"); + + // Verify node value matches. + BSTR bvalue = nullptr; + varvalue.lVal = 0; + native_view->get_accValue(varvalue, &bvalue); + std::string value(_com_util::ConvertBSTRToString(bvalue)); + ASSERT_EQ(value, "value"); } } // namespace testing From 147081ef0daace1d021681b2579a7b97a549d449 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Mon, 29 Nov 2021 11:43:21 -0800 Subject: [PATCH 11/12] Check role --- .../windows/flutter_windows_view_unittests.cc | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/shell/platform/windows/flutter_windows_view_unittests.cc b/shell/platform/windows/flutter_windows_view_unittests.cc index bbd1ad6882071..04d9d8135509b 100644 --- a/shell/platform/windows/flutter_windows_view_unittests.cc +++ b/shell/platform/windows/flutter_windows_view_unittests.cc @@ -191,19 +191,24 @@ TEST(FlutterWindowsEngine, AddSemanticsNodeUpdate) { // Verify node name matches our label. BSTR bname = nullptr; - VARIANT varvalue{}; - varvalue.vt = VT_I4; - varvalue.lVal = 0; - native_view->get_accName(varvalue, &bname); + VARIANT varchild{}; + varchild.vt = VT_I4; + varchild.lVal = CHILDID_SELF; + ASSERT_EQ(native_view->get_accName(varchild, &bname), S_OK); std::string name(_com_util::ConvertBSTRToString(bname)); - ASSERT_EQ(name, "name"); + EXPECT_EQ(name, "name"); // Verify node value matches. BSTR bvalue = nullptr; - varvalue.lVal = 0; - native_view->get_accValue(varvalue, &bvalue); + ASSERT_EQ(native_view->get_accValue(varchild, &bvalue), S_OK); std::string value(_com_util::ConvertBSTRToString(bvalue)); - ASSERT_EQ(value, "value"); + EXPECT_EQ(value, "value"); + + // Verify node type is a group. + VARIANT varrole{}; + varrole.vt = VT_I4; + ASSERT_EQ(native_view->get_accRole(varchild, &varrole), S_OK); + EXPECT_EQ(varrole.lVal, ROLE_SYSTEM_STATICTEXT); } } // namespace testing From 51b18c27683a79666bfb732be34ad07135355a7c Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Mon, 29 Nov 2021 11:47:02 -0800 Subject: [PATCH 12/12] Minor cleanup/clarification for readability --- shell/platform/windows/flutter_windows_view_unittests.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shell/platform/windows/flutter_windows_view_unittests.cc b/shell/platform/windows/flutter_windows_view_unittests.cc index 04d9d8135509b..1ad32e87142db 100644 --- a/shell/platform/windows/flutter_windows_view_unittests.cc +++ b/shell/platform/windows/flutter_windows_view_unittests.cc @@ -189,11 +189,13 @@ TEST(FlutterWindowsEngine, AddSemanticsNodeUpdate) { IAccessible* native_view = node_delegate->GetNativeViewAccessible(); ASSERT_TRUE(native_view != nullptr); - // Verify node name matches our label. - BSTR bname = nullptr; + // Property lookups will be made against this node itself. VARIANT varchild{}; varchild.vt = VT_I4; varchild.lVal = CHILDID_SELF; + + // Verify node name matches our label. + BSTR bname = nullptr; ASSERT_EQ(native_view->get_accName(varchild, &bname), S_OK); std::string name(_com_util::ConvertBSTRToString(bname)); EXPECT_EQ(name, "name");