From 3179e4c1fa4ce1164ae4a2551770aa5980b92c91 Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Fri, 8 Sep 2023 11:05:57 -0700 Subject: [PATCH] Adds a GN template that invokes run_tests.py --- BUILD.gn | 188 +++++++++------- ci/BUILD.gn | 16 ++ display_list/BUILD.gn | 5 +- flow/BUILD.gn | 3 +- flutter_frontend_server/BUILD.gn | 10 + fml/BUILD.gn | 5 +- impeller/BUILD.gn | 15 ++ impeller/golden_tests/BUILD.gn | 8 + lib/ui/BUILD.gn | 3 +- runtime/BUILD.gn | 7 +- shell/common/BUILD.gn | 11 +- shell/gpu/BUILD.gn | 8 + .../android/external_view_embedder/BUILD.gn | 3 +- shell/platform/android/jni/BUILD.gn | 3 +- .../platform_view_android_delegate/BUILD.gn | 3 +- shell/platform/common/BUILD.gn | 7 +- shell/platform/common/client_wrapper/BUILD.gn | 3 +- shell/platform/darwin/common/BUILD.gn | 5 +- shell/platform/darwin/macos/BUILD.gn | 3 +- shell/platform/embedder/BUILD.gn | 13 +- shell/platform/glfw/client_wrapper/BUILD.gn | 3 +- testing/BUILD.gn | 14 +- testing/dart/BUILD.gn | 20 +- testing/litetest/BUILD.gn | 16 ++ testing/run_tests.gni | 212 ++++++++++++++++++ testing/run_tests.py | 58 +++-- testing/single_test.py.tmpl | 29 +++ testing/smoke_test_failure/BUILD.gn | 15 +- third_party/accessibility/BUILD.gn | 3 +- third_party/spring_animation/BUILD.gn | 3 +- third_party/tonic/tests/BUILD.gn | 3 +- third_party/txt/BUILD.gn | 5 +- tools/gn | 1 + 33 files changed, 549 insertions(+), 152 deletions(-) create mode 100644 ci/BUILD.gn create mode 100644 testing/litetest/BUILD.gn create mode 100644 testing/run_tests.gni create mode 100644 testing/single_test.py.tmpl diff --git a/BUILD.gn b/BUILD.gn index f21342d919a10..264dae2fd8eef 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -157,118 +157,132 @@ group("flutter") { } } -group("unittests") { - testonly = true - - public_deps = [] - if (is_android) { - public_deps += [ - "//flutter/impeller/toolkit/android:apk_unittests", - "//flutter/impeller/toolkit/android:unittests", - "//flutter/shell/platform/android:flutter_shell_native_unittests", +_unittest_targets = [] +_runable_tests = [] +_common_targets = [] +if (is_android) { + _unittest_targets += [ + "//flutter/impeller/toolkit/android:apk_unittests", + "//flutter/impeller/toolkit/android:unittests", + "//flutter/shell/platform/android:flutter_shell_native_unittests", + ] +} +if (is_ios) { + _unittest_targets += + [ "//flutter/shell/platform/darwin/ios:ios_test_flutter" ] +} +if (enable_unittests) { + _common_targets += [ + "//flutter/display_list:display_list_rendertests", + "//flutter/display_list:display_list_unittests", + "//flutter/flow:flow_unittests", + "//flutter/fml:fml_arc_unittests", + "//flutter/fml:fml_unittests", + "//flutter/lib/ui:ui_unittests", + "//flutter/runtime:dart_plugin_registrant_unittests", + "//flutter/runtime:no_dart_plugin_registrant_unittests", + "//flutter/runtime:runtime_unittests", + "//flutter/shell/common:shell_unittests", + "//flutter/shell/platform/embedder:embedder_a11y_unittests", + "//flutter/shell/platform/embedder:embedder_proctable_unittests", + "//flutter/shell/platform/embedder:embedder_unittests", + "//flutter/testing:testing_unittests", + "//flutter/testing/dart:dart", + "//flutter/testing/smoke_test_failure:smoke_test_failure", + "//flutter/third_party/tonic/tests:tonic_unittests", + "//flutter/third_party/txt:txt_unittests", + ] + if (is_mac || is_win) { + _common_targets += + [ "//flutter/third_party/accessibility:accessibility_unittests" ] + } + if (is_fuchsia) { + _unittest_targets += [ "//flutter/shell/platform/fuchsia:tests" ] + } + if (is_mac || is_linux || is_win) { + _common_targets += [ + "//flutter/impeller:impeller_dart_unittests", + "//flutter/impeller:impeller_unittests", ] } - - if (is_ios) { - public_deps += [ "//flutter/shell/platform/darwin/ios:ios_test_flutter" ] + if (is_mac) { + _common_targets += [ + "//flutter/impeller/golden_tests:impeller_golden_tests", + "//flutter/shell/gpu:gpu_surface_metal_unittests", + "//flutter/shell/platform/darwin/common:framework_common_unittests", + "//flutter/third_party/spring_animation:spring_animation_unittests", + ] + } + if (!is_win && !is_fuchsia) { + _common_targets += [ + "//flutter/shell/platform/android/external_view_embedder:android_external_view_embedder_unittests", + "//flutter/shell/platform/android/jni:jni_unittests", + "//flutter/shell/platform/android/platform_view_android_delegate:platform_view_android_delegate_unittests", + ] } - # Compile all unittests targets if enabled. - if (enable_unittests) { - public_deps += [ - "//flutter/display_list:display_list_rendertests", - "//flutter/display_list:display_list_unittests", - "//flutter/flow:flow_unittests", - "//flutter/fml:fml_arc_unittests", - "//flutter/fml:fml_unittests", - "//flutter/lib/ui:ui_unittests", - "//flutter/runtime:dart_plugin_registrant_unittests", - "//flutter/runtime:no_dart_plugin_registrant_unittests", - "//flutter/runtime:runtime_unittests", - "//flutter/shell/common:shell_unittests", - "//flutter/shell/platform/embedder:embedder_a11y_unittests", - "//flutter/shell/platform/embedder:embedder_proctable_unittests", - "//flutter/shell/platform/embedder:embedder_unittests", - "//flutter/testing:testing_unittests", - "//flutter/testing/dart", - "//flutter/testing/smoke_test_failure", - "//flutter/third_party/tonic/tests:tonic_unittests", - "//flutter/third_party/txt:txt_unittests", + # Unit tests for desktop embeddings should only be built if the desktop + # embeddings are being built. + if (enable_desktop_embeddings) { + _common_targets += [ + "//flutter/shell/platform/common:common_cpp_core_unittests", + "//flutter/shell/platform/common/client_wrapper:client_wrapper_unittests", ] - # The accessibility library only supports Mac and Windows at the moment. - if (is_mac || is_win) { - public_deps += - [ "//flutter/third_party/accessibility:accessibility_unittests" ] - } + if (!is_fuchsia) { + # These tests require the embedder and thus cannot run on Fuchsia. + _common_targets += + [ "//flutter/shell/platform/common:common_cpp_unittests" ] - if (is_fuchsia) { - public_deps += [ "//flutter/shell/platform/fuchsia:tests" ] + # These tests require GLFW and thus cannot run on fuchsia. + _common_targets += [ "//flutter/shell/platform/glfw/client_wrapper:client_wrapper_glfw_unittests" ] } - if (is_mac || is_linux || is_win) { - public_deps += [ - "//flutter/impeller:impeller_dart_unittests", - "//flutter/impeller:impeller_unittests", - ] + if (is_linux) { + _unittest_targets += + [ "//flutter/shell/platform/linux:flutter_linux_unittests" ] + if (build_glfw_shell) { + _unittest_targets += + [ "//flutter/shell/platform/glfw:flutter_glfw_unittests" ] + } } if (is_mac) { - public_deps += [ + _common_targets += [ "//flutter/impeller/golden_tests:impeller_golden_tests", "//flutter/shell/gpu:gpu_surface_metal_unittests", "//flutter/shell/platform/darwin/common:availability_version_check_unittests", "//flutter/shell/platform/darwin/common:framework_common_unittests", + "//flutter/shell/platform/darwin/macos:flutter_desktop_darwin_unittests", "//flutter/third_party/spring_animation:spring_animation_unittests", ] } - if (!is_win && !is_fuchsia) { - public_deps += [ - "//flutter/shell/platform/android/external_view_embedder:android_external_view_embedder_unittests", - "//flutter/shell/platform/android/jni:jni_unittests", - "//flutter/shell/platform/android/platform_view_android_delegate:platform_view_android_delegate_unittests", + if (is_win) { + _unittest_targets += [ + "//flutter/shell/platform/windows:flutter_windows_unittests", + "//flutter/shell/platform/windows/client_wrapper:client_wrapper_windows_unittests", ] } + } +} +_unittest_targets += _common_targets +_runable_tests += _common_targets - # Unit tests for desktop embeddings should only be built if the desktop - # embeddings are being built. - if (enable_desktop_embeddings) { - public_deps += [ - "//flutter/shell/platform/common:common_cpp_core_unittests", - "//flutter/shell/platform/common/client_wrapper:client_wrapper_unittests", - ] - - if (!is_fuchsia) { - # These tests require the embedder and thus cannot run on fuchsia. - # TODO(): Enable when embedder works on fuchsia. - public_deps += - [ "//flutter/shell/platform/common:common_cpp_unittests" ] - - # These tests require GLFW and thus cannot run on fuchsia. - public_deps += [ "//flutter/shell/platform/glfw/client_wrapper:client_wrapper_glfw_unittests" ] - } - - if (is_linux) { - public_deps += - [ "//flutter/shell/platform/linux:flutter_linux_unittests" ] - if (build_glfw_shell) { - public_deps += - [ "//flutter/shell/platform/glfw:flutter_glfw_unittests" ] - } - } +group("unittests") { + testonly = true - if (is_mac) { - public_deps += [ "//flutter/shell/platform/darwin/macos:flutter_desktop_darwin_unittests" ] - } + public_deps = _unittest_targets +} - if (is_win) { - public_deps += [ - "//flutter/shell/platform/windows:flutter_windows_unittests", - "//flutter/shell/platform/windows/client_wrapper:client_wrapper_windows_unittests", - ] - } - } +group("run_tests") { + testonly = true + public_deps = [] + foreach(test_target, _runable_tests) { + test_target_name = get_label_info(test_target, "name") + test_target_dir = get_label_info(test_target, "dir") + run_target = "$test_target_dir:_run_tests_py_$test_target_name" + public_deps += [ run_target ] } } diff --git a/ci/BUILD.gn b/ci/BUILD.gn new file mode 100644 index 0000000000000..9f3353b01d9fb --- /dev/null +++ b/ci/BUILD.gn @@ -0,0 +1,16 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//flutter/build/dart/rules.gni") +import("//flutter/testing/run_tests.gni") + +host_dart_test("format_test") { + test_name = "flutter/ci" + main_dart = "test/format_test.dart" + runtime_inputs = [ + "//.clang-format", + "//flutter/.clang-format", + "//flutter/.style.yapf", + ] +} diff --git a/display_list/BUILD.gn b/display_list/BUILD.gn index 2bbb1e545e4db..3f90e40a159e9 100644 --- a/display_list/BUILD.gn +++ b/display_list/BUILD.gn @@ -4,6 +4,7 @@ import("//flutter/common/config.gni") import("//flutter/impeller/tools/impeller.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") if (is_fuchsia) { @@ -105,7 +106,7 @@ test_fixtures("display_list_fixtures") { } if (enable_unittests) { - executable("display_list_unittests") { + engine_native_unittests("display_list_unittests") { testonly = true sources = [ @@ -149,7 +150,7 @@ if (enable_unittests) { } } - executable("display_list_rendertests") { + engine_native_unittests("display_list_rendertests") { testonly = true sources = [ "testing/dl_rendering_unittests.cc" ] diff --git a/flow/BUILD.gn b/flow/BUILD.gn index 4a1aade9b5f75..4031c0b6aed70 100644 --- a/flow/BUILD.gn +++ b/flow/BUILD.gn @@ -5,6 +5,7 @@ import("//flutter/common/config.gni") import("//flutter/impeller/tools/impeller.gni") import("//flutter/shell/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") if (is_fuchsia) { @@ -146,7 +147,7 @@ if (enable_unittests) { ] } - executable("flow_unittests") { + engine_native_unittests("flow_unittests") { testonly = true sources = [ diff --git a/flutter_frontend_server/BUILD.gn b/flutter_frontend_server/BUILD.gn index 7d7f0c2027694..7bbab69f76584 100644 --- a/flutter_frontend_server/BUILD.gn +++ b/flutter_frontend_server/BUILD.gn @@ -3,7 +3,9 @@ # found in the LICENSE file. import("//flutter/build/dart/dart.gni") +import("//flutter/build/dart/rules.gni") import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") copy("frontend_server") { if (flutter_prebuilt_dart_sdk) { @@ -16,3 +18,11 @@ copy("frontend_server") { sources = [ snapshot ] outputs = [ "$root_gen_dir/frontend_server_aot.dart.snapshot" ] } + +if (build_engine_artifacts) { + host_dart_test("to_string_test") { + test_name = "flutter/flutter_frontend_server" + main_dart = "test/to_string_test.dart" + runtime_deps = [ ":frontend_server" ] + } +} diff --git a/fml/BUILD.gn b/fml/BUILD.gn index d2d4a165c7686..414793226d9a6 100644 --- a/fml/BUILD.gn +++ b/fml/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") if (is_fuchsia) { @@ -333,7 +334,7 @@ if (enable_unittests) { ] } - executable("fml_unittests") { + engine_native_unittests("fml_unittests") { testonly = true sources = [ @@ -418,7 +419,7 @@ if (enable_unittests) { } } - executable("fml_arc_unittests") { + engine_native_unittests("fml_arc_unittests") { testonly = true if (is_mac || is_ios) { cflags_objcc = flutter_cflags_objc_arc diff --git a/impeller/BUILD.gn b/impeller/BUILD.gn index f5e2ce7bf3388..7883633dde6f1 100644 --- a/impeller/BUILD.gn +++ b/impeller/BUILD.gn @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//flutter/testing/run_tests.gni") import("//flutter/third_party/glfw/glfw_args.gni") import("tools/impeller.gni") @@ -123,6 +124,13 @@ impeller_component("impeller_unittests") { } } +run_tests_py("impeller_unittests") { + testonly = true + deps = [ ":impeller_unittests" ] + name = "impeller_unittests" + type = "engine" +} + if (impeller_supports_rendering) { impeller_component("impeller_dart_unittests") { target_type = "executable" @@ -131,4 +139,11 @@ if (impeller_supports_rendering) { deps = [ "renderer:renderer_dart_unittests" ] } + + run_tests_py("impeller_dart_unittests") { + testonly = true + deps = [ ":impeller_dart_unittests" ] + name = "impeller_dart_unittests" + type = "engine" + } } diff --git a/impeller/golden_tests/BUILD.gn b/impeller/golden_tests/BUILD.gn index 8c8b5548833e0..9e762508bc0af 100644 --- a/impeller/golden_tests/BUILD.gn +++ b/impeller/golden_tests/BUILD.gn @@ -4,6 +4,7 @@ import("//flutter/common/config.gni") import("//flutter/impeller/tools/impeller.gni") +import("//flutter/testing/run_tests.gni") impeller_component("golden_playground_test") { testonly = true @@ -89,4 +90,11 @@ if (is_mac) { "//flutter/third_party/swiftshader", ] } + + run_tests_py("impeller_golden_tests") { + testonly = true + deps = [ ":impeller_golden_tests" ] + name = "impeller_golden_tests" + type = "engine" + } } diff --git a/lib/ui/BUILD.gn b/lib/ui/BUILD.gn index 1d00f1dfeba50..cbbaeafbff48f 100644 --- a/lib/ui/BUILD.gn +++ b/lib/ui/BUILD.gn @@ -5,6 +5,7 @@ import("//flutter/common/config.gni") import("//flutter/impeller/tools/impeller.gni") import("//flutter/shell/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") if (is_fuchsia) { @@ -259,7 +260,7 @@ if (enable_unittests) { ] } - executable("ui_unittests") { + engine_native_unittests("ui_unittests") { testonly = true public_configs = [ "//flutter:export_dynamic_symbols" ] diff --git a/runtime/BUILD.gn b/runtime/BUILD.gn index 1ed968a4d62d4..7c018bed1223f 100644 --- a/runtime/BUILD.gn +++ b/runtime/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") source_set("test_font") { @@ -130,7 +131,7 @@ if (enable_unittests) { dart_main = "fixtures/runtime_test.dart" } - executable("runtime_unittests") { + engine_native_unittests("runtime_unittests") { testonly = true sources = [ @@ -165,7 +166,7 @@ if (enable_unittests) { use_target_as_artifact_prefix = true } - executable("no_dart_plugin_registrant_unittests") { + engine_native_unittests("no_dart_plugin_registrant_unittests") { testonly = true sources = [ "no_dart_plugin_registrant_unittests.cc" ] @@ -186,7 +187,7 @@ if (enable_unittests) { use_target_as_artifact_prefix = true } - executable("dart_plugin_registrant_unittests") { + engine_native_unittests("dart_plugin_registrant_unittests") { testonly = true sources = [ "dart_plugin_registrant_unittests.cc" ] diff --git a/shell/common/BUILD.gn b/shell/common/BUILD.gn index 8e2d9f2174b3a..27f7f3cb5c903 100644 --- a/shell/common/BUILD.gn +++ b/shell/common/BUILD.gn @@ -5,6 +5,7 @@ import("//flutter/common/config.gni") import("//flutter/impeller/tools/impeller.gni") import("//flutter/shell/gpu/gpu.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") if (is_fuchsia) { @@ -305,7 +306,7 @@ if (enable_unittests) { } } - shell_host_executable("shell_unittests") { + engine_native_unittests("shell_unittests") { testonly = true sources = [ @@ -362,5 +363,13 @@ if (enable_unittests) { "//flutter/third_party/swiftshader", ] } + + deps += [ + ":common", + "//flutter/lib/snapshot", + "//flutter/runtime:libdart", + ] + + public_configs = [ "//flutter:export_dynamic_symbols" ] } } diff --git a/shell/gpu/BUILD.gn b/shell/gpu/BUILD.gn index 0e7e68670551f..34461e9e44f02 100644 --- a/shell/gpu/BUILD.gn +++ b/shell/gpu/BUILD.gn @@ -5,6 +5,7 @@ import("//flutter/common/config.gni") import("//flutter/impeller/tools/impeller.gni") import("//flutter/shell/config.gni") +import("//flutter/testing/run_tests.gni") gpu_common_deps = [ "//flutter/common", @@ -110,4 +111,11 @@ if (is_mac) { "//flutter/testing", ] + gpu_common_deps } + + run_tests_py("gpu_surface_metal_unittests") { + testonly = true + deps = [ ":gpu_surface_metal_unittests" ] + name = "gpu_surface_metal_unittests" + type = "engine" + } } diff --git a/shell/platform/android/external_view_embedder/BUILD.gn b/shell/platform/android/external_view_embedder/BUILD.gn index 44ae1106df182..42458c79448ee 100644 --- a/shell/platform/android/external_view_embedder/BUILD.gn +++ b/shell/platform/android/external_view_embedder/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") source_set("external_view_embedder") { @@ -30,7 +31,7 @@ test_fixtures("external_view_embedder_fixtures") { fixtures = [] } -executable("android_external_view_embedder_unittests") { +engine_native_unittests("android_external_view_embedder_unittests") { testonly = true sources = [ diff --git a/shell/platform/android/jni/BUILD.gn b/shell/platform/android/jni/BUILD.gn index 7a05be011d2f7..bee37174d13dd 100644 --- a/shell/platform/android/jni/BUILD.gn +++ b/shell/platform/android/jni/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") source_set("jni") { @@ -36,7 +37,7 @@ test_fixtures("jni_fixtures") { fixtures = [] } -executable("jni_unittests") { +engine_native_unittests("jni_unittests") { testonly = true sources = [ "jni_mock_unittest.cc" ] diff --git a/shell/platform/android/platform_view_android_delegate/BUILD.gn b/shell/platform/android/platform_view_android_delegate/BUILD.gn index f01f6f8da46c1..52bc983354950 100644 --- a/shell/platform/android/platform_view_android_delegate/BUILD.gn +++ b/shell/platform/android/platform_view_android_delegate/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") source_set("platform_view_android_delegate") { @@ -27,7 +28,7 @@ test_fixtures("platform_view_android_delegate_fixtures") { fixtures = [] } -executable("platform_view_android_delegate_unittests") { +engine_native_unittests("platform_view_android_delegate_unittests") { testonly = true sources = [ "platform_view_android_delegate_unittests.cc" ] diff --git a/shell/platform/common/BUILD.gn b/shell/platform/common/BUILD.gn index 5ebfb2236d2c5..c5f6dbc968723 100644 --- a/shell/platform/common/BUILD.gn +++ b/shell/platform/common/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") config("desktop_library_implementation") { @@ -154,7 +155,7 @@ if (enable_unittests) { fixtures = [] } - executable("common_cpp_core_unittests") { + engine_native_unittests("common_cpp_core_unittests") { testonly = true sources = [ "path_utils_unittests.cc" ] @@ -173,7 +174,7 @@ if (enable_unittests) { fixtures = [] } - executable("common_cpp_unittests") { + engine_native_unittests("common_cpp_unittests") { testonly = true sources = [ @@ -198,7 +199,7 @@ if (enable_unittests) { "//flutter/testing", ] - # The accessibility bridge only supports MacOS for now. + # The accessibility bridge only supports macOS for now. if (is_mac || is_win) { sources += [ "accessibility_bridge_unittests.cc", diff --git a/shell/platform/common/client_wrapper/BUILD.gn b/shell/platform/common/client_wrapper/BUILD.gn index 91e7120b65734..bf12a1ef0fa00 100644 --- a/shell/platform/common/client_wrapper/BUILD.gn +++ b/shell/platform/common/client_wrapper/BUILD.gn @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") import("core_wrapper_files.gni") @@ -35,7 +36,7 @@ test_fixtures("client_wrapper_fixtures") { fixtures = [] } -executable("client_wrapper_unittests") { +engine_native_unittests("client_wrapper_unittests") { testonly = true sources = [ diff --git a/shell/platform/darwin/common/BUILD.gn b/shell/platform/darwin/common/BUILD.gn index f86af9017146c..4da20f1be20c4 100644 --- a/shell/platform/darwin/common/BUILD.gn +++ b/shell/platform/darwin/common/BUILD.gn @@ -5,6 +5,7 @@ assert(is_mac || is_ios) import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") import("framework_common.gni") @@ -54,7 +55,7 @@ test_fixtures("availability_version_check_fixtures") { fixtures = [] } -executable("availability_version_check_unittests") { +engine_native_unittests("availability_version_check_unittests") { testonly = true sources = [ "availability_version_check_unittests.cc" ] @@ -111,7 +112,7 @@ test_fixtures("framework_common_fixtures") { } # Unit tests for channels. -executable("framework_common_unittests") { +engine_native_unittests("framework_common_unittests") { testonly = true sources = [ diff --git a/shell/platform/darwin/macos/BUILD.gn b/shell/platform/darwin/macos/BUILD.gn index 401b63bc0b8b6..bcbad99cd4b66 100644 --- a/shell/platform/darwin/macos/BUILD.gn +++ b/shell/platform/darwin/macos/BUILD.gn @@ -9,6 +9,7 @@ import("//flutter/build/zip_bundle.gni") import("//flutter/shell/gpu/gpu.gni") import("//flutter/shell/platform/darwin/common/framework_common.gni") import("//flutter/shell/platform/glfw/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") shell_gpu_configuration("macos_gpu_configuration") { @@ -171,7 +172,7 @@ test_fixtures("flutter_desktop_darwin_fixtures") { fixtures = [ "//flutter/third_party/icu/common/icudtl.dat" ] } -executable("flutter_desktop_darwin_unittests") { +engine_native_unittests("flutter_desktop_darwin_unittests") { testonly = true sources = [ diff --git a/shell/platform/embedder/BUILD.gn b/shell/platform/embedder/BUILD.gn index ba7d1276f54a7..a875c03091d41 100644 --- a/shell/platform/embedder/BUILD.gn +++ b/shell/platform/embedder/BUILD.gn @@ -8,6 +8,7 @@ import("//flutter/common/config.gni") import("//flutter/impeller/tools/impeller.gni") import("//flutter/shell/gpu/gpu.gni") import("//flutter/shell/platform/embedder/embedder.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") declare_args() { @@ -344,10 +345,10 @@ if (enable_unittests) { } } - executable("embedder_unittests") { + engine_native_unittests("embedder_unittests") { testonly = true - configs += [ + configs = [ ":embedder_jit_snapshot_setup", ":embedder_gpu_configuration_config", "//flutter:export_dynamic_symbols", @@ -375,10 +376,10 @@ if (enable_unittests) { } } - executable("embedder_a11y_unittests") { + engine_native_unittests("embedder_a11y_unittests") { testonly = true - configs += [ + configs = [ ":embedder_gpu_configuration_config", "//flutter:export_dynamic_symbols", ] @@ -391,10 +392,10 @@ if (enable_unittests) { } # Tests that build in FLUTTER_ENGINE_NO_PROTOTYPES mode. - executable("embedder_proctable_unittests") { + engine_native_unittests("embedder_proctable_unittests") { testonly = true - configs += [ + configs = [ ":embedder_gpu_configuration_config", "//flutter:export_dynamic_symbols", ] diff --git a/shell/platform/glfw/client_wrapper/BUILD.gn b/shell/platform/glfw/client_wrapper/BUILD.gn index d9c24fb513307..eaaa5cf468c36 100644 --- a/shell/platform/glfw/client_wrapper/BUILD.gn +++ b/shell/platform/glfw/client_wrapper/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/shell/platform/common/client_wrapper/publish.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") _wrapper_includes = [ @@ -69,7 +70,7 @@ test_fixtures("client_wrapper_glfw_fixtures") { fixtures = [] } -executable("client_wrapper_glfw_unittests") { +engine_native_unittests("client_wrapper_glfw_unittests") { testonly = true sources = [ diff --git a/testing/BUILD.gn b/testing/BUILD.gn index 35251252f86b2..96a1520c51961 100644 --- a/testing/BUILD.gn +++ b/testing/BUILD.gn @@ -4,8 +4,20 @@ import("//flutter/common/config.gni") import("//flutter/shell/config.gni") +import("run_tests.gni") import("testing.gni") +declare_args() { + # Maximum number of test processes to run in parallel. + # + # To avoid out-of-memory errors we explicitly reduce the number of jobs. + flutter_concurrent_test_jobs = 1 +} + +pool("run_tests_pool") { + depth = flutter_concurrent_test_jobs +} + config("dynamic_symbols") { if (is_clang && is_linux) { ldflags = [ "-rdynamic" ] @@ -183,7 +195,7 @@ if (enable_unittests) { # The //flutter/testing library provides utility methods to other test targets. # This test target tests the testing utilities. - executable("testing_unittests") { + engine_native_unittests("testing_unittests") { testonly = true sources = [ "mock_canvas_unittests.cc" ] diff --git a/testing/dart/BUILD.gn b/testing/dart/BUILD.gn index f67dad80f0b4e..58b70ee17a837 100644 --- a/testing/dart/BUILD.gn +++ b/testing/dart/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/build/dart/rules.gni") +import("//flutter/testing/run_tests.gni") tests = [ "assets_test.dart", @@ -19,6 +20,7 @@ tests = [ "gesture_settings_test.dart", "gpu_test.dart", "gradient_test.dart", + "http_allow_http_connections_test.dart", "http_disallow_http_connections_test.dart", "image_descriptor_test.dart", @@ -57,17 +59,15 @@ foreach(test, tests) { flutter_build_dir = rebase_path("$root_gen_dir") flutter_src_dir = rebase_path("//flutter") skia_gold_work_dir = rebase_path("$root_gen_dir/skia_gold_$test") - flutter_frontend_server("compile_$test") { - main_dart = test - kernel_output = "$root_gen_dir/$test.dill" - extra_args = [ + flutter_dart_test(test) { + test_name = test + extra_build_args = [ "-DkFlutterSrcDirectory=$flutter_src_dir", "-DkFlutterBuildDirectory=$flutter_build_dir", "-DkSkiaGoldWorkDirectory=$skia_gold_work_dir", ] - package_config = ".dart_tool/package_config.json" deps = [ "//flutter/third_party/txt:txt_fixtures" ] - testonly = true + runtime_deps = [ "//flutter/lib/ui:ui_unittests_fixtures" ] } } @@ -78,3 +78,11 @@ group("dart") { deps += [ ":compile_$test" ] } } + +group("_run_tests_py_dart") { + testonly = true + deps = [] + foreach(test, tests) { + deps += [ ":_run_tests_py_${test}" ] + } +} diff --git a/testing/litetest/BUILD.gn b/testing/litetest/BUILD.gn new file mode 100644 index 0000000000000..f1d89d6c9c4c3 --- /dev/null +++ b/testing/litetest/BUILD.gn @@ -0,0 +1,16 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//flutter/build/dart/rules.gni") +import("//flutter/testing/run_tests.gni") + +host_dart_test("litetest_test") { + test_name = "flutter/testing/litetest" + main_dart = "test/litetest_test.dart" + runtime_inputs = [ + "//.clang-format", + "//flutter/.clang-format", + "//flutter/.style.yapf", + ] +} diff --git a/testing/run_tests.gni b/testing/run_tests.gni new file mode 100644 index 0000000000000..9544f094e5972 --- /dev/null +++ b/testing/run_tests.gni @@ -0,0 +1,212 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//flutter/build/dart/rules.gni") + +# TODO(zra): Add real docs + +_test_script_template = + read_file("//flutter/testing/single_test.py.tmpl", "string") + +template("_write_test_runner_script") { + generated_file("_script_for_$target_name") { + assert(defined(invoker.name), "The test name must be specified.") + assert(defined(invoker.type), "The test type must be specified.") + assert( + invoker.type == "dart" || invoker.type == "dart-host" || + invoker.type == "engine" || invoker.type == "java" || + invoker.type == "objc", + "The test type must be one of: 'dart', 'dart-host', 'engine', 'java', or 'objc'") + forward_variables_from(invoker, + [ + "visibility", + "testonly", + "deps", + ]) + + stamp_file = "$target_out_dir/${target_name}.tested" + output = "$target_out_dir/${target_name}.py" + outputs = [ output ] + + variant = get_path_info(root_out_dir, "name") + + variant_flag = "--variant" + if (invoker.type == "java") { + variant_flag = "--android-variant" + } else if (invoker.type == "objc") { + variant_flag = "--ios-variant" + } + + filter = invoker.name + + filter_flag = "--engine-filter" + if (invoker.type == "dart") { + filter_flag = "--dart-filter" + } else if (invoker.type == "dart-host") { + filter_flag = "--dart-host-filter" + } else if (invoker.type == "java") { + filter_flag = "--java-filter" + } else if (invoker.type == "objc") { + filter_flag = "--objc-filter" + } + + command = string_join(" ", + [ + "python3", + rebase_path("//flutter/testing/run_tests.py"), + "--quiet", + "--type", + invoker.type, + variant_flag, + variant, + filter_flag, + filter, + "--engine-capture-core-dump", + "--output", + rebase_path(stamp_file), + ]) + + contents = string_replace(_test_script_template, + "{{run_tests_py_command}}", + command) + } +} + +# Supported types: +# 'engine', +# 'dart', +# 'dart-host', +# 'java', +# 'objc', +template("run_tests_py") { + _write_test_runner_script(target_name) { + name = invoker.name + type = invoker.type + forward_variables_from(invoker, + [ + "visibility", + "testonly", + "deps", + ]) + } + + script_target = "_script_for_$target_name" + + action("_run_tests_py_$target_name") { + forward_variables_from(invoker, + [ + "visibility", + "testonly", + ]) + + if (defined(invoker.deps)) { + deps = invoker.deps + [ ":$script_target" ] + } else { + deps = [ ":$script_target" ] + } + test_runner_script_outputs = get_target_outputs(":$script_target") + test_runner_script = test_runner_script_outputs[0] + script = test_runner_script + output = "$target_out_dir/${target_name}.tested" + outputs = [ output ] + } +} + +template("engine_native_unittests") { + _test_name = target_name + executable(_test_name) { + forward_variables_from(invoker, + "*", + [ + "test_name", + "test_type", + "runtime_deps", + "testonly", + "configs", + ]) + if (defined(invoker.configs)) { + configs += invoker.configs + } + testonly = true + } + + run_tests_py(_test_name) { + forward_variables_from(invoker, [ "visibility" ]) + testonly = true + deps = [ ":$_test_name" ] + if (defined(invoker.runtime_deps)) { + deps += invoker.runtime_deps + } + name = _test_name + if (defined(invoker.test_name)) { + name = invoker.test_name + } + type = "engine" + } +} + +# Declares targets for building and running a test that runs in the +# command-line Dart VM. +# +# Required parameters: +# main_dart +# test_name +# runtime_deps +# runtime_inputs +# +template("host_dart_test") { + _test_name = target_name + application_snapshot(_test_name) { + snapshot_kind = "kernel" + main_dart = invoker.main_dart + package_config = ".dart_tool/package_config.json" + training_args = [] + } + + run_tests_py(_test_name) { + testonly = true + deps = [ + ":$_test_name", + "//flutter/build/dart:dart_sdk", + ] + if (defined(invoker.runtime_deps)) { + deps += invoker.runtime_deps + } + if (defined(invoker.runtime_inputs)) { + inputs += invoker.runtime_inputs + } + name = invoker.test_name + type = "dart-host" + } +} + +template("flutter_dart_test") { + _test_name = target_name + flutter_frontend_server("compile_$_test_name") { + forward_variables_from(invoker, [ "deps" ]) + testonly = true + main_dart = invoker.test_name + kernel_output = "$root_gen_dir/${invoker.test_name}.dill" + package_config = ".dart_tool/package_config.json" + if (defined(invoker.extra_build_args)) { + extra_args = invoker.extra_build_args + } + } + + run_tests_py(_test_name) { + testonly = true + deps = [ ":compile_$_test_name" ] + if (build_engine_artifacts) { + deps += [ + "//flutter/build/dart:dart_sdk", + "//flutter/shell/testing:testing", + ] + } + if (defined(invoker.runtime_deps)) { + deps += invoker.runtime_deps + } + name = invoker.test_name + type = "dart" + } +} diff --git a/testing/run_tests.py b/testing/run_tests.py index dc04e6d9849ee..b47082e104143 100755 --- a/testing/run_tests.py +++ b/testing/run_tests.py @@ -404,6 +404,7 @@ def make_test(name, flags=None, extra_env=None): extra_env = {} return (name, flags, extra_env) + icu_flags = ['--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat')] unittests = [ make_test('client_wrapper_glfw_unittests'), make_test('client_wrapper_unittests'), @@ -423,6 +424,7 @@ def make_test(name, flags=None, extra_env=None): make_test('tonic_unittests'), # The image release unit test can take a while on slow machines. make_test('ui_unittests', flags=repeat_flags + ['--timeout=90']), + make_test('txt_unittests', flags=repeat_flags + ['--'] + icu_flags), ] if not is_windows(): @@ -459,7 +461,6 @@ def make_test(name, flags=None, extra_env=None): '--golden-dir=%s' % GOLDEN_DIR, '--font-file=%s' % ROBOTO_FONT_PATH, ] - icu_flags = ['--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat')] unittests += [ make_test('flow_unittests', flags=repeat_flags + ['--'] + flow_flags), make_test('flutter_glfw_unittests'), @@ -1098,24 +1099,26 @@ def run_impeller_golden_tests(build_dir: str): run_cmd([dart_bin, '--disable-dart-dev', str(bin_path), temp_dir]) -def main(): +ALL_TYPES = [ + 'engine', + 'dart', + 'dart-host', + 'benchmarks', + 'java', + 'android', + 'objc', + 'font-subset', + 'impeller-golden', +] + + +def parse_args(): parser = argparse.ArgumentParser( description=""" In order to learn the details of running tests in the engine, please consult the Flutter Wiki page on the subject: https://github.com/flutter/flutter/wiki/Testing-the-engine """ ) - all_types = [ - 'engine', - 'dart', - 'dart-host', - 'benchmarks', - 'java', - 'android', - 'objc', - 'font-subset', - 'impeller-golden', - ] parser.add_argument( '--variant', @@ -1128,7 +1131,7 @@ def main(): '--type', type=str, default='all', - help='A list of test types, default is "all" (equivalent to "%s")' % (','.join(all_types)) + help='A list of test types, default is "all" (equivalent to "%s")' % (','.join(ALL_TYPES)) ) parser.add_argument( '--engine-filter', type=str, default='', help='A list of engine test executables to run.' @@ -1221,9 +1224,18 @@ def main(): type=str, help='The directory that verbose logs will be copied to in --quiet mode.', ) + parser.add_argument( + '--output', + dest='output', + type=str, + default=None, + help='A stamp file written when a test finishes successfully.' + ) + return parser.parse_args() - args = parser.parse_args() +def main(): + args = parse_args() logger.addHandler(console_logger_handler) logger.addHandler(file_logger_handler) logger.setLevel(logging.INFO) @@ -1234,7 +1246,7 @@ def main(): console_logger_handler.setLevel(logging.INFO) if args.type == 'all': - types = all_types + types = ALL_TYPES else: types = args.type.split(',') @@ -1264,20 +1276,18 @@ def main(): # Use this type to exclusively run impeller tests. if 'impeller' in types: - build_name = args.variant try: - xvfb.start_virtual_x(build_name, build_dir) - extra_env = vulkan_validation_env(build_dir) + xvfb.start_virtual_x(args.variant, build_dir) run_engine_executable( build_dir, 'impeller_unittests', engine_filter, shuffle_flags, coverage=args.coverage, - extra_env=extra_env + extra_env=vulkan_validation_env(build_dir) ) finally: - xvfb.stop_virtual_x(build_name) + xvfb.stop_virtual_x(args.variant) if 'dart' in types: dart_filter = args.dart_filter.split(',') if args.dart_filter else None @@ -1331,7 +1341,7 @@ def should_skip(variant): matches = [variant for variant in variants_to_skip if variant in args.variant] return len(matches) > 0 - if ('engine' in types or 'font-subset' in types) and not should_skip(args.variant): + if 'font-subset' in types and not should_skip(args.variant): cmd = ['python3', 'test.py', '--variant', args.variant] if 'arm64' in args.variant: cmd += ['--target-cpu', 'arm64'] @@ -1343,6 +1353,10 @@ def should_skip(variant): if args.quiet and args.logs_dir: shutil.copy(LOG_FILE, os.path.join(args.logs_dir, 'run_tests.log')) + if args.output: + with open(args.output, 'w') as output_file: + output_file.write('Done\n') + return 0 if success else 1 diff --git a/testing/single_test.py.tmpl b/testing/single_test.py.tmpl new file mode 100644 index 0000000000000..2508c9f8597b4 --- /dev/null +++ b/testing/single_test.py.tmpl @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import subprocess +import sys + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +BUILDROOT_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..')) + +def main(): + test_result = subprocess.run( + '{{run_tests_py_command}}', + check=False, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=BUILDROOT_DIR, + ) + if test_result.returncode != 0: + sys.stderr.write(test_result.stdout.decode()) + return 1 + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/testing/smoke_test_failure/BUILD.gn b/testing/smoke_test_failure/BUILD.gn index 1ce144ecc523a..f5db34e25d72b 100644 --- a/testing/smoke_test_failure/BUILD.gn +++ b/testing/smoke_test_failure/BUILD.gn @@ -3,14 +3,13 @@ # found in the LICENSE file. import("//flutter/build/dart/rules.gni") +import("//flutter/testing/run_tests.gni") tests = [ "fail_test.dart" ] foreach(test, tests) { - flutter_frontend_server("compile_$test") { - main_dart = test - kernel_output = "$root_gen_dir/$test.dill" - package_config = ".dart_tool/package_config.json" + flutter_dart_test(test) { + test_name = test } } @@ -21,3 +20,11 @@ group("smoke_test_failure") { deps += [ ":compile_$test" ] } } + +group("_run_tests_py_smoke_test_failure") { + testonly = true + deps = [] + foreach(test, tests) { + deps += [ ":_run_tests_py_${test}" ] + } +} diff --git a/third_party/accessibility/BUILD.gn b/third_party/accessibility/BUILD.gn index 19ba407c38220..9a7d50c87473f 100644 --- a/third_party/accessibility/BUILD.gn +++ b/third_party/accessibility/BUILD.gn @@ -3,6 +3,7 @@ # found in the LICENSE file. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") config("accessibility_config") { @@ -45,7 +46,7 @@ if (enable_unittests) { fixtures = [] } - executable("accessibility_unittests") { + engine_native_unittests("accessibility_unittests") { testonly = true public_configs = [ ":accessibility_config" ] diff --git a/third_party/spring_animation/BUILD.gn b/third_party/spring_animation/BUILD.gn index 32a6a128b5b16..97213ef0a6e05 100644 --- a/third_party/spring_animation/BUILD.gn +++ b/third_party/spring_animation/BUILD.gn @@ -4,6 +4,7 @@ if (is_ios || is_mac) { import("//flutter/common/config.gni") + import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") source_set("spring_animation") { @@ -19,7 +20,7 @@ if (is_ios || is_mac) { fixtures = [] } - executable("spring_animation_unittests") { + engine_native_unittests("spring_animation_unittests") { testonly = true sources = [ "SpringAnimationTest.mm" ] deps = [ diff --git a/third_party/tonic/tests/BUILD.gn b/third_party/tonic/tests/BUILD.gn index 45619e91395c7..82a115b5f26a2 100644 --- a/third_party/tonic/tests/BUILD.gn +++ b/third_party/tonic/tests/BUILD.gn @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") test_fixtures("tonic_fixtures") { @@ -9,7 +10,7 @@ test_fixtures("tonic_fixtures") { fixtures = [] } -executable("tonic_unittests") { +engine_native_unittests("tonic_unittests") { testonly = true public_configs = [ "//flutter:export_dynamic_symbols" ] diff --git a/third_party/txt/BUILD.gn b/third_party/txt/BUILD.gn index 9079ed1585300..3ec33f6325dd9 100644 --- a/third_party/txt/BUILD.gn +++ b/third_party/txt/BUILD.gn @@ -13,6 +13,7 @@ # limitations under the License. import("//flutter/common/config.gni") +import("//flutter/testing/run_tests.gni") import("//flutter/testing/testing.gni") if (is_fuchsia) { @@ -145,7 +146,7 @@ if (enable_unittests) { ] } - executable("txt_unittests") { + engine_native_unittests("txt_unittests") { testonly = true sources = [ @@ -161,7 +162,7 @@ if (enable_unittests) { public_configs = [ ":txt_config" ] - configs += [ ":allow_posix_names" ] + configs = [ ":allow_posix_names" ] deps = [ ":txt", diff --git a/tools/gn b/tools/gn index a6bc832497f49..027494053dce9 100755 --- a/tools/gn +++ b/tools/gn @@ -611,6 +611,7 @@ def to_gn_args(args): gn_args['dart_runtime_mode'] = runtime_mode gn_args['concurrent_dart_jobs'] = get_concurrent_jobs('1GB', '1GB') + gn_args['flutter_concurrent_test_jobs'] = get_concurrent_jobs('1GB', '100MB') # Hardcoding this avoids invoking a relatively expensive python script from # GN, but removes the ability to use git-worktrees in the Dart checkout from