From 0693c3b2f8a2b04073be93f6229c14f58991c079 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Tue, 20 Nov 2018 15:19:40 -0800 Subject: [PATCH 1/7] Support in/out of list announcements on Android --- ci/licenses_golden/licenses_flutter | 1 + shell/platform/android/BUILD.gn | 1 + .../android/io/flutter/util/Predicate.java | 11 ++ .../io/flutter/view/AccessibilityBridge.java | 152 +++++++++++++++++- 4 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 shell/platform/android/io/flutter/util/Predicate.java diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 63b2ac899e260..7cf25b3228d94 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -445,6 +445,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/SingleV FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/VirtualDisplayController.java FILE: ../../../flutter/shell/platform/android/io/flutter/util/PathUtils.java FILE: ../../../flutter/shell/platform/android/io/flutter/util/Preconditions.java +FILE: ../../../flutter/shell/platform/android/io/flutter/util/Predicate.java FILE: ../../../flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java FILE: ../../../flutter/shell/platform/android/io/flutter/view/FlutterCallbackInformation.java FILE: ../../../flutter/shell/platform/android/io/flutter/view/FlutterMain.java diff --git a/shell/platform/android/BUILD.gn b/shell/platform/android/BUILD.gn index aad27b2f9dd37..b9df763dbb7aa 100644 --- a/shell/platform/android/BUILD.gn +++ b/shell/platform/android/BUILD.gn @@ -131,6 +131,7 @@ java_library("flutter_shell_java") { "io/flutter/plugin/platform/VirtualDisplayController.java", "io/flutter/util/PathUtils.java", "io/flutter/util/Preconditions.java", + "io/flutter/util/Predicate.java", "io/flutter/view/AccessibilityBridge.java", "io/flutter/view/FlutterCallbackInformation.java", "io/flutter/view/FlutterMain.java", diff --git a/shell/platform/android/io/flutter/util/Predicate.java b/shell/platform/android/io/flutter/util/Predicate.java new file mode 100644 index 0000000000000..96c6c1d1cd478 --- /dev/null +++ b/shell/platform/android/io/flutter/util/Predicate.java @@ -0,0 +1,11 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package io.flutter.util; + +// TODO(dnfield): remove this if/when we can use appcompat to support it. +// java.util.function.Predicate isn't available until API24 +public interface Predicate { + public abstract boolean test(T t); +} \ No newline at end of file diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index 4fc808e9eea5f..e4761747221c7 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -16,6 +16,7 @@ import android.view.accessibility.AccessibilityNodeProvider; import io.flutter.plugin.common.BasicMessageChannel; import io.flutter.plugin.common.StandardMessageCodec; +import io.flutter.util.Predicate; import java.nio.ByteBuffer; import java.util.*; @@ -103,6 +104,74 @@ enum Flag { } final int value; + + static String flagsToString(int value) { + if (value == 0) { + return "(no flags)"; + } + StringBuilder builder = new StringBuilder(); + if ((value & HAS_CHECKED_STATE.value) != 0) { + builder.append("HAS_CHECKED_STATE | "); + } + if ((value & IS_CHECKED.value) != 0) { + builder.append("IS_CHECKED | "); + } + if ((value & IS_SELECTED.value) != 0) { + builder.append("IS_SELECTED | "); + } + if ((value & IS_BUTTON.value) != 0) { + builder.append("IS_BUTTON | "); + } + if ((value & IS_TEXT_FIELD.value) != 0) { + builder.append("IS_TEXT_FIELD | "); + } + if ((value & IS_FOCUSED.value) != 0) { + builder.append("IS_FOCUSED | "); + } + if ((value & HAS_ENABLED_STATE.value) != 0) { + builder.append("HAS_ENABLED_STATE | "); + } + if ((value & IS_ENABLED.value) != 0) { + builder.append("IS_ENABLED | "); + } + if ((value & IS_IN_MUTUALLY_EXCLUSIVE_GROUP.value) != 0) { + builder.append("IS_IN_MUTUALLY_EXCLUSIVE_GROUP | "); + } + if ((value & IS_HEADER.value) != 0) { + builder.append("IS_HEADER | "); + } + if ((value & IS_OBSCURED.value) != 0) { + builder.append("IS_OBSCURED | "); + } + if ((value & SCOPES_ROUTE.value) != 0) { + builder.append("SCOPES_ROUTE | "); + } + if ((value & NAMES_ROUTE.value) != 0) { + builder.append("NAMES_ROUTE | "); + } + if ((value & IS_HIDDEN.value) != 0) { + builder.append("IS_HIDDEN | "); + } + if ((value & IS_IMAGE.value) != 0) { + builder.append("IS_IMAGE | "); + } + if ((value & IS_LIVE_REGION.value) != 0) { + builder.append("IS_LIVE_REGION | "); + } + if ((value & HAS_TOGGLED_STATE.value) != 0) { + builder.append("HAS_TOGGLED_STATE | "); + } + if ((value & IS_TOGGLED.value) != 0) { + builder.append("IS_TOGGLED | "); + } + if ((value & HAS_IMPLICIT_SCROLLING.value) != 0) { + builder.append("HAS_IMPLICIT_SCROLLING | "); + } + if (builder.length() > 3) { + builder.setLength(builder.length() - 3); + } + return builder.toString(); + } } AccessibilityBridge(FlutterView owner) { @@ -125,6 +194,36 @@ void setAccessibilityEnabled(boolean accessibilityEnabled) { } } + private boolean shouldSetCollectionInfo(SemanticsObject object) { + // TODO(dnfield): make these lambdas when Java 1.8 support lands. + Predicate parentMatcher = new Predicate() { + @Override + public boolean test(SemanticsObject o) { + return o == mA11yFocusedObject; + } + }; + + Predicate flagMatcher = new Predicate() { + @Override + public boolean test(SemanticsObject o) { + return o.hasFlag(Flag.HAS_IMPLICIT_SCROLLING); + } + }; + + // TalkBack expects a number of rows and/or columns greater than 0 to announce + // in list and out of list. For an infinite or growing list, you have to + // specify something > 0 to get "in list" announcements. + // TalkBack will also only track one list at a time, so we only want to set this + // for a list that contians the current a11y focused object - otherwise, if there + // are two lists or nested lists, we may end up with announcements for only the last + // one that is currently availalbe in the semantics tree. However, we also want + // to set it if we're exiting a list to a non-list, so that we can get the "out of list" + // announcement when A11y focus moves out of a list and not into another list. + return object.scrollChildren > 0 + && (hasSemanticsObjectAncestor(mA11yFocusedObject, parentMatcher) + || !hasSemanticsObjectAncestor(mA11yFocusedObject, flagMatcher)); + } + @Override @SuppressWarnings("deprecation") public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { @@ -266,14 +365,29 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { if (object.hasAction(Action.SCROLL_LEFT) || object.hasAction(Action.SCROLL_UP) || object.hasAction(Action.SCROLL_RIGHT) || object.hasAction(Action.SCROLL_DOWN)) { result.setScrollable(true); + // This tells Android's a11y to send scroll events when reaching the end of // the visible viewport of a scrollable, unless the node itself does not // allow implicit scrolling - then we leave the className as view.View. + // + // We should prefer setCollectionInfo to the class names, as this way we get "In List" + // and "Out of list" announcements. But we don't always know the counts, so we + // can fallback to the generic scroll view class names. + // TODO(dnfield): We should add semantics properties for rows and columns in 2 dimensional lists, e.g. + // GridView. Right now, we're only supporting ListViews and only if they have scroll children. if (object.hasFlag(Flag.HAS_IMPLICIT_SCROLLING)) { if (object.hasAction(Action.SCROLL_LEFT) || object.hasAction(Action.SCROLL_RIGHT)) { - result.setClassName("android.widget.HorizontalScrollView"); + if (shouldSetCollectionInfo(object)) { + result.setCollectionInfo(AccessibilityNodeInfo.CollectionInfo.obtain(0, object.scrollChildren, false)); + } else { + result.setClassName("android.widget.HorizontalScrollView"); + } } else { - result.setClassName("android.widget.ScrollView"); + if (shouldSetCollectionInfo(object)) { + result.setCollectionInfo(AccessibilityNodeInfo.CollectionInfo.obtain(object.scrollChildren, 0, false)); + } else { + result.setClassName("android.widget.ScrollView"); + } } } // TODO(ianh): Once we're on SDK v23+, call addAction to @@ -960,6 +1074,11 @@ boolean isStandardAction() { /// Value is derived from ACTION_TYPE_MASK in AccessibilityNodeInfo.java static int firstResourceId = 267386881; + + static boolean hasSemanticsObjectAncestor(SemanticsObject target, Predicate tester) { + return target != null && target.getAncestor(tester) != null; + } + private class SemanticsObject { SemanticsObject() {} @@ -1012,6 +1131,33 @@ private class SemanticsObject { private float[] globalTransform; private Rect globalRect; + SemanticsObject getAncestor(Predicate tester) { + SemanticsObject nextAncestor = parent; + while (nextAncestor != null) { + if (tester.test(nextAncestor)) { + return nextAncestor; + } + nextAncestor = nextAncestor.parent; + } + return null; + } + + // boolean hasAncestor(SemanticsObject ancestor) { + // SemanticsObject nextAncestor = parent; + // while (nextAncestor != null && nextAncestor != ancestor) { + // nextAncestor = nextAncestor.parent; + // } + // return nextAncestor == ancestor; + // } + + // boolean hasAncestorWithFlag(Flag flag) { + // SemanticsObject nextAncestor = parent; + // while (nextAncestor != null && !nextAncestor.hasFlag(flag)) { + // nextAncestor = nextAncestor.parent; + // } + // return nextAncestor != null && nextAncestor.hasFlag(flag); + // } + boolean hasAction(Action action) { return (actions & action.value) != 0; } @@ -1044,7 +1190,7 @@ boolean didChangeLabel() { void log(String indent, boolean recursive) { Log.i(TAG, indent + "SemanticsObject id=" + id + " label=" + label + " actions=" + actions - + " flags=" + flags + "\n" + indent + " +-- textDirection=" + + " flags=" + Flag.flagsToString(flags) + "\n" + indent + " +-- textDirection=" + textDirection + "\n" + indent + " +-- rect.ltrb=(" + left + ", " + top + ", " + right + ", " + bottom + ")\n" + indent + " +-- transform=" + Arrays.toString(transform) + "\n"); From b921edb18370fb3378a327b11a90e6e6c5b8a1fb Mon Sep 17 00:00:00 2001 From: Dan Field Date: Tue, 20 Nov 2018 15:28:08 -0800 Subject: [PATCH 2/7] remove unused code --- .../io/flutter/view/AccessibilityBridge.java | 86 +------------------ 1 file changed, 1 insertion(+), 85 deletions(-) diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index e4761747221c7..ab65e17acae37 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -104,74 +104,6 @@ enum Flag { } final int value; - - static String flagsToString(int value) { - if (value == 0) { - return "(no flags)"; - } - StringBuilder builder = new StringBuilder(); - if ((value & HAS_CHECKED_STATE.value) != 0) { - builder.append("HAS_CHECKED_STATE | "); - } - if ((value & IS_CHECKED.value) != 0) { - builder.append("IS_CHECKED | "); - } - if ((value & IS_SELECTED.value) != 0) { - builder.append("IS_SELECTED | "); - } - if ((value & IS_BUTTON.value) != 0) { - builder.append("IS_BUTTON | "); - } - if ((value & IS_TEXT_FIELD.value) != 0) { - builder.append("IS_TEXT_FIELD | "); - } - if ((value & IS_FOCUSED.value) != 0) { - builder.append("IS_FOCUSED | "); - } - if ((value & HAS_ENABLED_STATE.value) != 0) { - builder.append("HAS_ENABLED_STATE | "); - } - if ((value & IS_ENABLED.value) != 0) { - builder.append("IS_ENABLED | "); - } - if ((value & IS_IN_MUTUALLY_EXCLUSIVE_GROUP.value) != 0) { - builder.append("IS_IN_MUTUALLY_EXCLUSIVE_GROUP | "); - } - if ((value & IS_HEADER.value) != 0) { - builder.append("IS_HEADER | "); - } - if ((value & IS_OBSCURED.value) != 0) { - builder.append("IS_OBSCURED | "); - } - if ((value & SCOPES_ROUTE.value) != 0) { - builder.append("SCOPES_ROUTE | "); - } - if ((value & NAMES_ROUTE.value) != 0) { - builder.append("NAMES_ROUTE | "); - } - if ((value & IS_HIDDEN.value) != 0) { - builder.append("IS_HIDDEN | "); - } - if ((value & IS_IMAGE.value) != 0) { - builder.append("IS_IMAGE | "); - } - if ((value & IS_LIVE_REGION.value) != 0) { - builder.append("IS_LIVE_REGION | "); - } - if ((value & HAS_TOGGLED_STATE.value) != 0) { - builder.append("HAS_TOGGLED_STATE | "); - } - if ((value & IS_TOGGLED.value) != 0) { - builder.append("IS_TOGGLED | "); - } - if ((value & HAS_IMPLICIT_SCROLLING.value) != 0) { - builder.append("HAS_IMPLICIT_SCROLLING | "); - } - if (builder.length() > 3) { - builder.setLength(builder.length() - 3); - } - return builder.toString(); - } } AccessibilityBridge(FlutterView owner) { @@ -1142,22 +1074,6 @@ SemanticsObject getAncestor(Predicate tester) { return null; } - // boolean hasAncestor(SemanticsObject ancestor) { - // SemanticsObject nextAncestor = parent; - // while (nextAncestor != null && nextAncestor != ancestor) { - // nextAncestor = nextAncestor.parent; - // } - // return nextAncestor == ancestor; - // } - - // boolean hasAncestorWithFlag(Flag flag) { - // SemanticsObject nextAncestor = parent; - // while (nextAncestor != null && !nextAncestor.hasFlag(flag)) { - // nextAncestor = nextAncestor.parent; - // } - // return nextAncestor != null && nextAncestor.hasFlag(flag); - // } - boolean hasAction(Action action) { return (actions & action.value) != 0; } @@ -1190,7 +1106,7 @@ boolean didChangeLabel() { void log(String indent, boolean recursive) { Log.i(TAG, indent + "SemanticsObject id=" + id + " label=" + label + " actions=" + actions - + " flags=" + Flag.flagsToString(flags) + "\n" + indent + " +-- textDirection=" + + " flags=" + flags + "\n" + indent + " +-- textDirection=" + textDirection + "\n" + indent + " +-- rect.ltrb=(" + left + ", " + top + ", " + right + ", " + bottom + ")\n" + indent + " +-- transform=" + Arrays.toString(transform) + "\n"); From 088ff527521a1974344a21e738023d9c0b8cca82 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Tue, 20 Nov 2018 16:20:34 -0800 Subject: [PATCH 3/7] typo/nit --- shell/platform/android/io/flutter/util/Predicate.java | 2 +- .../platform/android/io/flutter/view/AccessibilityBridge.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shell/platform/android/io/flutter/util/Predicate.java b/shell/platform/android/io/flutter/util/Predicate.java index 96c6c1d1cd478..e24df12e2a386 100644 --- a/shell/platform/android/io/flutter/util/Predicate.java +++ b/shell/platform/android/io/flutter/util/Predicate.java @@ -8,4 +8,4 @@ // java.util.function.Predicate isn't available until API24 public interface Predicate { public abstract boolean test(T t); -} \ No newline at end of file +} diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index ab65e17acae37..0a64872ccd548 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -146,9 +146,9 @@ public boolean test(SemanticsObject o) { // in list and out of list. For an infinite or growing list, you have to // specify something > 0 to get "in list" announcements. // TalkBack will also only track one list at a time, so we only want to set this - // for a list that contians the current a11y focused object - otherwise, if there + // for a list that contains the current a11y focused object - otherwise, if there // are two lists or nested lists, we may end up with announcements for only the last - // one that is currently availalbe in the semantics tree. However, we also want + // one that is currently available in the semantics tree. However, we also want // to set it if we're exiting a list to a non-list, so that we can get the "out of list" // announcement when A11y focus moves out of a list and not into another list. return object.scrollChildren > 0 From 61d55f15ddc105d69a6a09bd239c5f5ae3de73f4 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Tue, 20 Nov 2018 21:01:28 -0800 Subject: [PATCH 4/7] annotate params --- .../android/io/flutter/view/AccessibilityBridge.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index 0a64872ccd548..093f77032234e 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -310,13 +310,19 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) { if (object.hasFlag(Flag.HAS_IMPLICIT_SCROLLING)) { if (object.hasAction(Action.SCROLL_LEFT) || object.hasAction(Action.SCROLL_RIGHT)) { if (shouldSetCollectionInfo(object)) { - result.setCollectionInfo(AccessibilityNodeInfo.CollectionInfo.obtain(0, object.scrollChildren, false)); + result.setCollectionInfo(AccessibilityNodeInfo.CollectionInfo.obtain( + 0, // rows + object.scrollChildren, // columns + false)); // hierarchical } else { result.setClassName("android.widget.HorizontalScrollView"); } } else { if (shouldSetCollectionInfo(object)) { - result.setCollectionInfo(AccessibilityNodeInfo.CollectionInfo.obtain(object.scrollChildren, 0, false)); + result.setCollectionInfo(AccessibilityNodeInfo.CollectionInfo.obtain( + object.scrollChildren, // rows + 0, // columns + false)); // hierarchical } else { result.setClassName("android.widget.ScrollView"); } From 2a578527ce95b0ed5a6d4b22ee2954106e054914 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Tue, 20 Nov 2018 23:50:16 -0800 Subject: [PATCH 5/7] check for right object --- shell/platform/android/io/flutter/view/AccessibilityBridge.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index 093f77032234e..8e1c6502594c7 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -131,7 +131,7 @@ private boolean shouldSetCollectionInfo(SemanticsObject object) { Predicate parentMatcher = new Predicate() { @Override public boolean test(SemanticsObject o) { - return o == mA11yFocusedObject; + return o == object; } }; From 6f1e5b319224548ba9ac1ad516eeb549a8d56962 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Wed, 21 Nov 2018 01:47:23 -0800 Subject: [PATCH 6/7] make it final --- shell/platform/android/io/flutter/view/AccessibilityBridge.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index 8e1c6502594c7..9269bb7ff56bf 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -126,7 +126,7 @@ void setAccessibilityEnabled(boolean accessibilityEnabled) { } } - private boolean shouldSetCollectionInfo(SemanticsObject object) { + private boolean shouldSetCollectionInfo(final SemanticsObject object) { // TODO(dnfield): make these lambdas when Java 1.8 support lands. Predicate parentMatcher = new Predicate() { @Override From 60ee7c67979706cc76e10d76dedf6defcd1e046f Mon Sep 17 00:00:00 2001 From: Dan Field Date: Mon, 7 Jan 2019 13:26:20 -0800 Subject: [PATCH 7/7] lambdas! --- .../io/flutter/view/AccessibilityBridge.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index b512a34c81bb1..d8c2de3749195 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -127,21 +127,6 @@ void setAccessibilityEnabled(boolean accessibilityEnabled) { } private boolean shouldSetCollectionInfo(final SemanticsObject object) { - // TODO(dnfield): make these lambdas when Java 1.8 support lands. - Predicate parentMatcher = new Predicate() { - @Override - public boolean test(SemanticsObject o) { - return o == object; - } - }; - - Predicate flagMatcher = new Predicate() { - @Override - public boolean test(SemanticsObject o) { - return o.hasFlag(Flag.HAS_IMPLICIT_SCROLLING); - } - }; - // TalkBack expects a number of rows and/or columns greater than 0 to announce // in list and out of list. For an infinite or growing list, you have to // specify something > 0 to get "in list" announcements. @@ -152,8 +137,8 @@ public boolean test(SemanticsObject o) { // to set it if we're exiting a list to a non-list, so that we can get the "out of list" // announcement when A11y focus moves out of a list and not into another list. return object.scrollChildren > 0 - && (hasSemanticsObjectAncestor(mA11yFocusedObject, parentMatcher) - || !hasSemanticsObjectAncestor(mA11yFocusedObject, flagMatcher)); + && (hasSemanticsObjectAncestor(mA11yFocusedObject, o -> o == object) + || !hasSemanticsObjectAncestor(mA11yFocusedObject, o -> o.hasFlag(Flag.HAS_IMPLICIT_SCROLLING))); } @Override