diff --git a/Dockerfile.ci b/Dockerfile.ci index ec35aab734ed9..fe7c2cd7d66f6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1090,8 +1090,8 @@ if [[ ${REMOVE_ARM_PACKAGES:="false"} == "true" ]]; then python "${IN_CONTAINER_DIR}/remove_arm_packages.py" fi -declare -a SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ - ALL_TESTS ALL_PRESELECTED_TESTS ALL_OTHER_TESTS +declare -a SELECTED_TESTS CLI_TESTS API_TESTS OPERATORS_TESTS ALWAYS_TESTS PROVIDERS_TESTS \ + CORE_TESTS WWW_TESTS ALL_TESTS ALL_PRESELECTED_TESTS ALL_OTHER_TESTS function find_all_other_tests() { local all_tests_dirs @@ -1120,6 +1120,7 @@ else API_TESTS=("tests/api_experimental" "tests/api_connexion" "tests/api_internal") PROVIDERS_TESTS=("tests/providers") ALWAYS_TESTS=("tests/always") + OPERATORS_TESTS=("tests/operators") CORE_TESTS=( "tests/core" "tests/executors" @@ -1142,6 +1143,7 @@ else "${PROVIDERS_TESTS[@]}" "${CORE_TESTS[@]}" "${ALWAYS_TESTS[@]}" + "${OPERATORS_TESTS[@]}" "${WWW_TESTS[@]}" "${SYSTEM_TESTS[@]}" ) @@ -1163,6 +1165,8 @@ else SELECTED_TESTS=("${CORE_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "Always" ]]; then SELECTED_TESTS=("${ALWAYS_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "Operators" ]]; then + SELECTED_TESTS=("${OPERATORS_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "WWW" ]]; then SELECTED_TESTS=("${WWW_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then @@ -1226,8 +1230,8 @@ else exit 1 fi fi -readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ - ALL_TESTS ALL_PRESELECTED_TESTS +readonly SELECTED_TESTS CLI_TESTS API_TESTS OPERATORS_TESTS ALWAYS_TESTS PROVIDERS_TESTS \ + CORE_TESTS WWW_TESTS ALL_TESTS ALL_PRESELECTED_TESTS if [[ ${TEST_TYPE:=""} == "Long" ]]; then EXTRA_PYTEST_ARGS+=( @@ -1256,7 +1260,6 @@ echo "Running tests ${SELECTED_TESTS[*]}" echo ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${SELECTED_TESTS[@]}") - if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then "${IN_CONTAINER_DIR}/run_system_tests.sh" "${ARGS[@]}" else diff --git a/dev/breeze/SELECTIVE_CHECKS.md b/dev/breeze/SELECTIVE_CHECKS.md index d6e525485a97a..0806466a81c7a 100644 --- a/dev/breeze/SELECTIVE_CHECKS.md +++ b/dev/breeze/SELECTIVE_CHECKS.md @@ -52,11 +52,13 @@ We have the following unit test types that can be selectively disabled/enabled b content of the incoming PR: * Always - those are tests that should be always executed (always folder) +* Operators - tests for the operators (operators folder) * Core - for the core Airflow functionality (core folder) * API - Tests for the Airflow API (api and api_connexion folders) * CLI - Tests for the Airflow CLI (cli folder) * WWW - Tests for the Airflow webserver (www folder) * Providers - Tests for all Providers of Airflow (providers folder) +* Other - all other tests remaining after the above tests are selected We also have several special kinds of tests that are not separated by packages, but they are marked with pytest markers. They can be found in any of those packages and they can be selected by the appropriate @@ -95,8 +97,8 @@ The logic implements the following rules: * if there are any changes to "common" provider code not belonging to any provider (usually system tests or tests), then tests for all Providers are run * The specific unit test type is enabled only if changed files match the expected patterns for each type - (`API`, `CLI`, `WWW`, `Providers`). The `Always` test type is added always if any unit tests are run. - `Providers` tests are removed if current branch is different than `main` + (`API`, `CLI`, `WWW`, `Providers`, `Operators`). The `Always` test type is added always if any unit + tests are run. `Providers` tests are removed if current branch is different than `main` * If there are no files left in sources after matching the test types and Kubernetes files, then apparently some Core/Other files have been changed. This automatically adds all test types to execute. This is done because changes in core might impact all the other test types. diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index b5d09c944bc69..026ce7482c964 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -128,7 +128,7 @@ def docker_compose_tests( TEST_PROGRESS_REGEXP = r"tests/.*|.*=====.*" -PERCENT_TEST_PROGRESS_REGEXP = r"^tests/.*\[[ \d%]*\].*" +PERCENT_TEST_PROGRESS_REGEXP = r"^tests/.*\[[ \d%]*\].*|^\..*\[[ \d%]*\].*" def _run_test( diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 388c4ee0515e7..187cf5ace82f4 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -114,6 +114,7 @@ class SelectiveUnitTestTypes(Enum): CLI = "CLI" CORE = "Core" OTHER = "Other" + OPERATORS = "Operators" PROVIDERS = "Providers" WWW = "WWW" diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 8251f3f4de733..cb79df373791b 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -187,6 +187,10 @@ def __hash__(self): r"^airflow/cli", r"^tests/cli", ], + SelectiveUnitTestTypes.OPERATORS: [ + r"^airflow/operators", + r"^tests/operators", + ], SelectiveUnitTestTypes.PROVIDERS: [ r"^airflow/providers/", r"^tests/system/providers/", @@ -584,6 +588,9 @@ def _get_test_types_to_run(self) -> list[str]: matched_files.update( self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.CLI) ) + matched_files.update( + self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.OPERATORS) + ) matched_files.update( self._select_test_type_if_matching(candidate_test_types, SelectiveUnitTestTypes.API) ) @@ -670,7 +677,7 @@ def parallel_test_types_list_as_string(self) -> str | None: self._extract_long_provider_tests(current_test_types) # this should be hard-coded as we want to have very specific sequence of tests - sorting_order = ["Core", "Providers[-amazon,google]", "Other", "Providers[amazon]", "WWW"] + sorting_order = ["Operators", "Core", "Providers[-amazon,google]", "Providers[amazon]", "WWW"] sort_key = {item: i for i, item in enumerate(sorting_order)} # Put the test types in the order we want them to run return " ".join(sorted(current_test_types, key=lambda x: (sort_key.get(x, len(sorting_order)), x))) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 2868c203cbc99..0eb61aa5e0919 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -91,6 +91,26 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Only API tests and DOCS should run", ) ), + ( + pytest.param( + ("airflow/operators/file.py",), + { + "affected-providers-list-as-string": None, + "all-python-versions": "['3.8']", + "all-python-versions-list-as-string": "3.8", + "python-versions": "['3.8']", + "python-versions-list-as-string": "3.8", + "image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "run-amazon-tests": "false", + "docs-build": "true", + "upgrade-to-newer-dependencies": "false", + "parallel-test-types-list-as-string": "Operators Always", + }, + id="Only Operator tests and DOCS should run", + ) + ), ( pytest.param( ( @@ -277,9 +297,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " - "Other Providers[amazon] WWW " - "API Always CLI Providers[google]", + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " + "Providers[amazon] WWW " + "API Always CLI Other Providers[google]", }, id="Everything should run - including all providers and upgrading to " "newer requirements as setup.py changed and all Python versions", @@ -300,9 +320,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "upgrade-to-newer-dependencies": "true", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " - "Other Providers[amazon] WWW " - "API Always CLI Providers[google]", + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " + "Providers[amazon] WWW " + "API Always CLI Other Providers[google]", }, id="Everything should run and upgrading to newer requirements as dependencies change", ) @@ -409,9 +429,9 @@ def test_expected_output_pull_request_main( "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " - "Other Providers[amazon] WWW " - "API Always CLI Providers[google]", + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " + "Providers[amazon] WWW " + "API Always CLI Other Providers[google]", }, id="Everything should run including all providers when full tests are needed", ) @@ -436,9 +456,9 @@ def test_expected_output_pull_request_main( "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " - "Other Providers[amazon] WWW " - "API Always CLI Providers[google]", + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " + "Providers[amazon] WWW " + "API Always CLI Other Providers[google]", }, id="Everything should run including full providers when full " "tests are needed even with different label set as well", @@ -461,9 +481,9 @@ def test_expected_output_pull_request_main( "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " - "Other Providers[amazon] WWW " - "API Always CLI Providers[google]", + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " + "Providers[amazon] WWW " + "API Always CLI Other Providers[google]", }, id="Everything should run including full providers when" "full tests are needed even if no files are changed", @@ -488,7 +508,7 @@ def test_expected_output_pull_request_main( "full-tests-needed": "true", "skip-provider-tests": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Core Other WWW API Always CLI", + "parallel-test-types-list-as-string": "Operators Core WWW API Always CLI Other", }, id="Everything should run except Providers when full tests are needed for non-main branch", ) @@ -600,7 +620,7 @@ def test_expected_output_full_tests_needed( "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", - "parallel-test-types-list-as-string": "Core Other WWW API Always CLI", + "parallel-test-types-list-as-string": "Operators Core WWW API Always CLI Other", }, id="All tests except Providers should run if core file changed in non-main branch", ), @@ -708,6 +728,7 @@ def test_expected_output_pull_request_v2_3( pytest.param( ( "airflow/cli/file.py", + "airflow/operators/file.py", "airflow/www/file.py", "airflow/api/file.py", ), @@ -723,9 +744,9 @@ def test_expected_output_pull_request_v2_3( "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", - "parallel-test-types-list-as-string": "WWW API Always CLI", + "parallel-test-types-list-as-string": "Operators WWW API Always CLI", }, - id="No providers tests should run if only CLI/API/WWW file changed", + id="No providers tests should run if only CLI/API/Operators/WWW file changed", ), pytest.param( ("airflow/models/test.py",), @@ -741,9 +762,9 @@ def test_expected_output_pull_request_v2_3( "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "false", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " "Providers[amazon] WWW " - "API Always CLI Providers[google]", + "API Always CLI Other Providers[google]", }, id="Tests for all providers should run if model file changed", ), @@ -761,11 +782,11 @@ def test_expected_output_pull_request_v2_3( "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "false", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " "Providers[amazon] WWW " - "API Always CLI Providers[google]", + "API Always CLI Other Providers[google]", }, - id="Tests for all providers should run if any other than API/WWW/CLI file changed.", + id="Tests for all providers should run if any other than API/WWW/CLI/Operators file changed.", ), ], ) @@ -800,9 +821,9 @@ def test_expected_output_pull_request_target( "docs-build": "true", "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "true", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " "Providers[amazon] WWW " - "API Always CLI Providers[google]", + "API Always CLI Other Providers[google]", }, id="All tests run on push even if unimportant file changed", ), @@ -820,7 +841,7 @@ def test_expected_output_pull_request_target( "docs-build": "true", "docs-filter-list-as-string": "--package-filter apache-airflow --package-filter docker-stack", "upgrade-to-newer-dependencies": "true", - "parallel-test-types-list-as-string": "Core Other WWW API Always CLI", + "parallel-test-types-list-as-string": "Operators Core WWW API Always CLI Other", }, id="All tests except Providers and Helm run on push" " even if unimportant file changed in non-main branch", @@ -839,9 +860,9 @@ def test_expected_output_pull_request_target( "docs-build": "true", "docs-filter-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "true", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] Other " + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " "Providers[amazon] WWW " - "API Always CLI Providers[google]", + "API Always CLI Other Providers[google]", }, id="All tests run on push if core file changed", ), @@ -892,9 +913,9 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): "upgrade-to-newer-dependencies": "true" if github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE] else "false", - "parallel-test-types-list-as-string": "Core Providers[-amazon,google] " - "Other Providers[amazon] WWW " - "API Always CLI Providers[google]", + "parallel-test-types-list-as-string": "Operators Core Providers[-amazon,google] " + "Providers[amazon] WWW " + "API Always CLI Other Providers[google]", }, str(stderr), ) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 18bd1a776e473..9fc6763fca433 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -68,5 +68,5 @@ static-checks:19926b8fcea5784b28d4a0d99865363c testing:docker-compose-tests:fd154a058082fcfda12eb877a9a89338 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 testing:integration-tests:f57fb275b9733a6226601f8095ad4de0 -testing:tests:1cf0f4580ecd2a2321b341b1dd411bee -testing:31d3b4f2385aed509adee68201906ed4 +testing:tests:ce35463d1c67f3416eba77d3201c90ac +testing:9c9f35945852ff48da2f5aba6447463e diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index f89febadd3d35..23fc3adb760aa 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -240,8 +240,8 @@ │--test-typeType of test to run. With Providers, you can specify tests of which providers should be    │ │run: `Providers[airbyte,http]` or excluded from the full test suite:                       │ │`Providers[-amazon,google]`                                                                │ -│(All | API | Always | CLI | Core | Other | Providers | WWW | PlainAsserts | Postgres |     │ -│MySQL | Quarantine)                                                                        │ +│(All | API | Always | CLI | Core | Operators | Other | Providers | WWW | PlainAsserts |    │ +│Postgres | MySQL | Quarantine)                                                             │ │--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ │(INTEGER RANGE)                                                                  │ │[default: 60; x>=0]                                                              │ @@ -263,8 +263,8 @@ │--parallelismMaximum number of processes to use while running the operation in parallel.│ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ -│--parallel-test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Other Providers WWW PlainAsserts] │ +│--parallel-test-typesSpace separated list of test types used for testing in parallel.         (TEXT)│ +│[default: API Always CLI Core Operators Other Providers WWW PlainAsserts]│ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 23ad3c5328493..036c8ef633e4e 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -439,8 +439,8 @@ if [[ ${REMOVE_ARM_PACKAGES:="false"} == "true" ]]; then python "${IN_CONTAINER_DIR}/remove_arm_packages.py" fi -declare -a SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ - ALL_TESTS ALL_PRESELECTED_TESTS ALL_OTHER_TESTS +declare -a SELECTED_TESTS CLI_TESTS API_TESTS OPERATORS_TESTS ALWAYS_TESTS PROVIDERS_TESTS \ + CORE_TESTS WWW_TESTS ALL_TESTS ALL_PRESELECTED_TESTS ALL_OTHER_TESTS # Finds all directories that are not on the list of tests # - so that we do not skip any in the future if new directories are added @@ -471,6 +471,7 @@ else API_TESTS=("tests/api_experimental" "tests/api_connexion" "tests/api_internal") PROVIDERS_TESTS=("tests/providers") ALWAYS_TESTS=("tests/always") + OPERATORS_TESTS=("tests/operators") CORE_TESTS=( "tests/core" "tests/executors" @@ -493,6 +494,7 @@ else "${PROVIDERS_TESTS[@]}" "${CORE_TESTS[@]}" "${ALWAYS_TESTS[@]}" + "${OPERATORS_TESTS[@]}" "${WWW_TESTS[@]}" "${SYSTEM_TESTS[@]}" ) @@ -514,6 +516,8 @@ else SELECTED_TESTS=("${CORE_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "Always" ]]; then SELECTED_TESTS=("${ALWAYS_TESTS[@]}") + elif [[ ${TEST_TYPE:=""} == "Operators" ]]; then + SELECTED_TESTS=("${OPERATORS_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "WWW" ]]; then SELECTED_TESTS=("${WWW_TESTS[@]}") elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then @@ -577,8 +581,8 @@ else exit 1 fi fi -readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ - ALL_TESTS ALL_PRESELECTED_TESTS +readonly SELECTED_TESTS CLI_TESTS API_TESTS OPERATORS_TESTS ALWAYS_TESTS PROVIDERS_TESTS \ + CORE_TESTS WWW_TESTS ALL_TESTS ALL_PRESELECTED_TESTS if [[ ${TEST_TYPE:=""} == "Long" ]]; then EXTRA_PYTEST_ARGS+=( @@ -607,7 +611,6 @@ echo "Running tests ${SELECTED_TESTS[*]}" echo ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${SELECTED_TESTS[@]}") - if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then "${IN_CONTAINER_DIR}/run_system_tests.sh" "${ARGS[@]}" else