From 520e2f9773ad078e18559148786d1642b8866081 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 10:47:00 +0530 Subject: [PATCH 01/13] trigger openlineage test when asset files changes --- .../airflow_breeze/utils/selective_checks.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index f29678dadcfdc..c845c27d2d341 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -45,6 +45,7 @@ DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, KIND_VERSION, + OTHER_PROVIDERS_INTEGRATIONS, RUNS_ON_PUBLIC_RUNNER, RUNS_ON_SELF_HOSTED_ASF_RUNNER, RUNS_ON_SELF_HOSTED_RUNNER, @@ -119,6 +120,7 @@ class FileGroupForCi(Enum): ALL_PROVIDER_YAML_FILES = "all_provider_yaml_files" ALL_DOCS_PYTHON_FILES = "all_docs_python_files" TESTS_UTILS_FILES = "test_utils_files" + ASSET_FILES = "asset_files" class AllProvidersSentinel: @@ -253,6 +255,14 @@ def __hash__(self): r"^task_sdk/src/airflow/sdk/.*\.py$", r"^task_sdk/tests/.*\.py$", ], + FileGroupForCi.ASSET_FILES: [ + r"^airflow/assets/", + r"^airflow/models/assets/", + r"^task_sdk/src/airflow/sdk/definitions/asset/", + r"^airflow/datasets/", + r"^airflow/dag_processing/collection.py", + r"airflow/timetables/assets.py", + ], } ) @@ -860,13 +870,24 @@ def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = all_providers_source_files = self._matching_files( FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES ) - if len(all_providers_source_files) == 0 and not self.needs_api_tests: + assets_source_files = self._matching_files( + FileGroupForCi.ASSET_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES + ) + + if ( + len(all_providers_source_files) == 0 + and len(assets_source_files) == 0 + and not self.needs_api_tests + ): # IF API tests are needed, that will trigger extra provider checks return [] else: - affected_providers = self._find_all_providers_affected( - include_docs=False, - ) + if len(all_providers_source_files) != 0: + affected_providers = self._find_all_providers_affected( + include_docs=False, + ) + elif len(assets_source_files) != 0: + affected_providers = OTHER_PROVIDERS_INTEGRATIONS candidate_test_types: set[str] = set() if affected_providers and not isinstance(affected_providers, AllProvidersSentinel): if split_to_individual_providers: From 75f2e4cb9c31a3da3d6fad9990da37113dc83ecd Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 13:41:46 +0530 Subject: [PATCH 02/13] fixing breaking test and adding test for only ol test --- .../airflow_breeze/utils/selective_checks.py | 26 +- dev/breeze/tests/test_selective_checks.py | 353 ++++++++++-------- 2 files changed, 208 insertions(+), 171 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index c845c27d2d341..f6d713c038bba 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -45,7 +45,7 @@ DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, KIND_VERSION, - OTHER_PROVIDERS_INTEGRATIONS, + OPENLINEAGE_INTEGRATION, RUNS_ON_PUBLIC_RUNNER, RUNS_ON_SELF_HOSTED_ASF_RUNNER, RUNS_ON_SELF_HOSTED_RUNNER, @@ -706,6 +706,10 @@ def needs_javascript_scans(self) -> bool: def needs_api_tests(self) -> bool: return self._should_be_run(FileGroupForCi.API_TEST_FILES) + @cached_property + def needs_ol_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.ASSET_FILES) + @cached_property def needs_api_codegen(self) -> bool: return self._should_be_run(FileGroupForCi.API_CODEGEN_FILES) @@ -882,12 +886,9 @@ def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = # IF API tests are needed, that will trigger extra provider checks return [] else: - if len(all_providers_source_files) != 0: - affected_providers = self._find_all_providers_affected( - include_docs=False, - ) - elif len(assets_source_files) != 0: - affected_providers = OTHER_PROVIDERS_INTEGRATIONS + affected_providers = self._find_all_providers_affected( + include_docs=False, + ) candidate_test_types: set[str] = set() if affected_providers and not isinstance(affected_providers, AllProvidersSentinel): if split_to_individual_providers: @@ -1461,6 +1462,8 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro all_providers.add(provider) if self.needs_api_tests: all_providers.add("fab") + if self.needs_ol_tests: + all_providers.add(OPENLINEAGE_INTEGRATION) if all_providers_affected: return ALL_PROVIDERS_SENTINEL if suspended_providers: @@ -1494,10 +1497,11 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro ) if not all_providers: return None - for provider in list(all_providers): - all_providers.update( - get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True) - ) + if not all_providers == {OPENLINEAGE_INTEGRATION}: + for provider in list(all_providers): + all_providers.update( + get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True) + ) return sorted(all_providers) def _is_canary_run(self): diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 749b4e1fa6bdf..18f913558b3ed 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1491,97 +1491,198 @@ def test_expected_output_push( @pytest.mark.parametrize( "files, expected_outputs,", [ - pytest.param( - ("INTHEWILD.md",), - { - "selected-providers-list-as-string": None, - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "false", - "skip-providers-tests": "true", - "test-groups": "[]", - "docs-build": "false", - "docs-list-as-string": None, - "upgrade-to-newer-dependencies": "false", - "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - "core-test-types-list-as-string": None, - "needs-mypy": "false", - "mypy-checks": "[]", - }, - id="Nothing should run if only non-important files changed", - ), - pytest.param( - ("tests/system/any_file.py",), - { - "selected-providers-list-as-string": None, - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "true", - "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - "ts-compile-format-lint-ui,ts-compile-format-lint-www", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": "Always", - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", - }, - id="Only Always and docs build should run if only system tests changed", - ), + # pytest.param( + # ("INTHEWILD.md",), + # { + # "selected-providers-list-as-string": None, + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "false", + # "skip-providers-tests": "true", + # "test-groups": "[]", + # "docs-build": "false", + # "docs-list-as-string": None, + # "upgrade-to-newer-dependencies": "false", + # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "core-test-types-list-as-string": None, + # "needs-mypy": "false", + # "mypy-checks": "[]", + # }, + # id="Nothing should run if only non-important files changed", + # ), + # pytest.param( + # ("tests/system/any_file.py",), + # { + # "selected-providers-list-as-string": None, + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "true", + # "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": "Always", + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="Only Always and docs build should run if only system tests changed", + # ), + # pytest.param( + # ( + # "airflow/cli/test.py", + # "chart/aaaa.txt", + # "providers/tests/google/file.py", + # ), + # { + # "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " + # "cncf.kubernetes common.compat common.sql " + # "facebook google hashicorp microsoft.azure microsoft.mssql mysql " + # "openlineage oracle postgres presto salesforce samba sftp ssh trino", + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "true", + # "needs-helm-tests": "true", + # "run-tests": "true", + # "skip-providers-tests": "false", + # "test-groups": "['core', 'providers']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " + # "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " + # "microsoft.mssql mysql openlineage oracle postgres " + # "presto salesforce samba sftp ssh trino", + # "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "true", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": "Always CLI", + # "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," + # "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," + # "salesforce,samba,sftp,ssh,trino] Providers[google]", + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow', 'mypy-providers']", + # }, + # id="CLI tests and Google-related provider tests should run if cli/chart files changed but " + # "prod image should be build too and k8s tests too", + # ), + # pytest.param( + # ( + # "airflow/cli/file.py", + # "airflow/operators/file.py", + # "airflow/www/file.py", + # "airflow/api/file.py", + # ), + # { + # "selected-providers-list-as-string": "common.compat fab", + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "false", + # "test-groups": "['core', 'providers']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow common.compat fab", + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "false", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": "API Always CLI Operators WWW", + # "providers-test-types-list-as-string": "Providers[common.compat,fab]", + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", + # ), + # pytest.param( + # ("airflow/models/test.py",), + # { + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow", + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "false", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="Tests for all airflow core types except providers should run if model file changed", + # ), + # pytest.param( + # ("airflow/file.py",), + # { + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow", + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "false", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="Tests for all airflow core types except providers should run if " + # "any other than API/WWW/CLI/Operators file changed.", + # ), + # pytest.param( + # ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), + # { + # "selected-providers-list-as-string": None, + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "false", + # "docs-list-as-string": None, + # "upgrade-to-newer-dependencies": "false", + # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", + # "core-test-types-list-as-string": None, + # "needs-mypy": "false", + # "mypy-checks": "[]", + # }, + # id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", + # ), pytest.param( ( - "airflow/cli/test.py", - "chart/aaaa.txt", - "providers/tests/google/file.py", + "airflow/assets/", + "airflow/models/assets/", + "task_sdk/src/airflow/sdk/definitions/asset/", + "airflow/datasets/", + "airflow/dag_processing/collection.py", + "airflow/timetables/assets.py", ), { - "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " - "cncf.kubernetes common.compat common.sql " - "facebook google hashicorp microsoft.azure microsoft.mssql mysql " - "openlineage oracle postgres presto salesforce samba sftp ssh trino", - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "true", - "needs-helm-tests": "true", - "run-tests": "true", - "skip-providers-tests": "false", - "test-groups": "['core', 'providers']", - "docs-build": "true", - "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " - "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " - "microsoft.mssql mysql openlineage oracle postgres " - "presto salesforce samba sftp ssh trino", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - "run-kubernetes-tests": "true", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": "Always CLI", - "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," - "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," - "salesforce,samba,sftp,ssh,trino] Providers[google]", - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow', 'mypy-providers']", - }, - id="CLI tests and Google-related provider tests should run if cli/chart files changed but " - "prod image should be build too and k8s tests too", - ), - pytest.param( - ( - "airflow/cli/file.py", - "airflow/operators/file.py", - "airflow/www/file.py", - "airflow/api/file.py", - ), - { - "selected-providers-list-as-string": "common.compat fab", + "selected-providers-list-as-string": "openlineage", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "ci-image-build": "true", @@ -1591,86 +1692,17 @@ def test_expected_output_push( "skip-providers-tests": "false", "test-groups": "['core', 'providers']", "docs-build": "true", - "docs-list-as-string": "apache-airflow common.compat fab", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - "ts-compile-format-lint-ui,ts-compile-format-lint-www", - "run-kubernetes-tests": "false", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": "API Always CLI Operators WWW", - "providers-test-types-list-as-string": "Providers[common.compat,fab]", - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", - }, - id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", - ), - pytest.param( - ("airflow/models/test.py",), - { - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "true", - "docs-list-as-string": "apache-airflow", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - "ts-compile-format-lint-ui,ts-compile-format-lint-www", - "run-kubernetes-tests": "false", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", - }, - id="Tests for all airflow core types except providers should run if model file changed", - ), - pytest.param( - ("airflow/file.py",), - { - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "true", - "docs-list-as-string": "apache-airflow", + "docs-list-as-string": "apache-airflow openlineage", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + "core-test-types-list-as-string": "API Always CLI Core Operators Other Serialization WWW", + "providers-test-types-list-as-string": "Providers[openlineage]", "needs-mypy": "true", "mypy-checks": "['mypy-airflow']", }, - id="Tests for all airflow core types except providers should run if " - "any other than API/WWW/CLI/Operators file changed.", - ), - pytest.param( - ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), - { - "selected-providers-list-as-string": None, - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "false", - "docs-list-as-string": None, - "upgrade-to-newer-dependencies": "false", - "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", - "core-test-types-list-as-string": None, - "needs-mypy": "false", - "mypy-checks": "[]", - }, - id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", + id="No providers tests except openlineage should run if only Asset file changed", ), ], ) @@ -1685,6 +1717,7 @@ def test_expected_output_pull_request_target( pr_labels=(), default_branch="main", ) + print(f"stderr is {str(stderr)}") assert_outputs_are_printed(expected_outputs, str(stderr)) From 9a73b3e8644be30d2c4ec4059d5273f925dd75a0 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 13:46:39 +0530 Subject: [PATCH 03/13] uncomment incorrectly commented tests --- dev/breeze/tests/test_selective_checks.py | 362 +++++++++++----------- 1 file changed, 181 insertions(+), 181 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 18f913558b3ed..cf1c7a47c154f 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1491,187 +1491,187 @@ def test_expected_output_push( @pytest.mark.parametrize( "files, expected_outputs,", [ - # pytest.param( - # ("INTHEWILD.md",), - # { - # "selected-providers-list-as-string": None, - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "false", - # "skip-providers-tests": "true", - # "test-groups": "[]", - # "docs-build": "false", - # "docs-list-as-string": None, - # "upgrade-to-newer-dependencies": "false", - # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "core-test-types-list-as-string": None, - # "needs-mypy": "false", - # "mypy-checks": "[]", - # }, - # id="Nothing should run if only non-important files changed", - # ), - # pytest.param( - # ("tests/system/any_file.py",), - # { - # "selected-providers-list-as-string": None, - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "true", - # "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": "Always", - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="Only Always and docs build should run if only system tests changed", - # ), - # pytest.param( - # ( - # "airflow/cli/test.py", - # "chart/aaaa.txt", - # "providers/tests/google/file.py", - # ), - # { - # "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " - # "cncf.kubernetes common.compat common.sql " - # "facebook google hashicorp microsoft.azure microsoft.mssql mysql " - # "openlineage oracle postgres presto salesforce samba sftp ssh trino", - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "true", - # "needs-helm-tests": "true", - # "run-tests": "true", - # "skip-providers-tests": "false", - # "test-groups": "['core', 'providers']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " - # "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " - # "microsoft.mssql mysql openlineage oracle postgres " - # "presto salesforce samba sftp ssh trino", - # "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "true", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": "Always CLI", - # "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," - # "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," - # "salesforce,samba,sftp,ssh,trino] Providers[google]", - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow', 'mypy-providers']", - # }, - # id="CLI tests and Google-related provider tests should run if cli/chart files changed but " - # "prod image should be build too and k8s tests too", - # ), - # pytest.param( - # ( - # "airflow/cli/file.py", - # "airflow/operators/file.py", - # "airflow/www/file.py", - # "airflow/api/file.py", - # ), - # { - # "selected-providers-list-as-string": "common.compat fab", - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "false", - # "test-groups": "['core', 'providers']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow common.compat fab", - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "false", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": "API Always CLI Operators WWW", - # "providers-test-types-list-as-string": "Providers[common.compat,fab]", - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", - # ), - # pytest.param( - # ("airflow/models/test.py",), - # { - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow", - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "false", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="Tests for all airflow core types except providers should run if model file changed", - # ), - # pytest.param( - # ("airflow/file.py",), - # { - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow", - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "false", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="Tests for all airflow core types except providers should run if " - # "any other than API/WWW/CLI/Operators file changed.", - # ), - # pytest.param( - # ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), - # { - # "selected-providers-list-as-string": None, - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "false", - # "docs-list-as-string": None, - # "upgrade-to-newer-dependencies": "false", - # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", - # "core-test-types-list-as-string": None, - # "needs-mypy": "false", - # "mypy-checks": "[]", - # }, - # id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", - # ), + pytest.param( + ("INTHEWILD.md",), + { + "selected-providers-list-as-string": None, + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "false", + "skip-providers-tests": "true", + "test-groups": "[]", + "docs-build": "false", + "docs-list-as-string": None, + "upgrade-to-newer-dependencies": "false", + "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "core-test-types-list-as-string": None, + "needs-mypy": "false", + "mypy-checks": "[]", + }, + id="Nothing should run if only non-important files changed", + ), + pytest.param( + ("tests/system/any_file.py",), + { + "selected-providers-list-as-string": None, + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "true", + "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": "Always", + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="Only Always and docs build should run if only system tests changed", + ), + pytest.param( + ( + "airflow/cli/test.py", + "chart/aaaa.txt", + "providers/tests/google/file.py", + ), + { + "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " + "cncf.kubernetes common.compat common.sql " + "facebook google hashicorp microsoft.azure microsoft.mssql mysql " + "openlineage oracle postgres presto salesforce samba sftp ssh trino", + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "skip-providers-tests": "false", + "test-groups": "['core', 'providers']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " + "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " + "microsoft.mssql mysql openlineage oracle postgres " + "presto salesforce samba sftp ssh trino", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "true", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": "Always CLI", + "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," + "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," + "salesforce,samba,sftp,ssh,trino] Providers[google]", + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow', 'mypy-providers']", + }, + id="CLI tests and Google-related provider tests should run if cli/chart files changed but " + "prod image should be build too and k8s tests too", + ), + pytest.param( + ( + "airflow/cli/file.py", + "airflow/operators/file.py", + "airflow/www/file.py", + "airflow/api/file.py", + ), + { + "selected-providers-list-as-string": "common.compat fab", + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "false", + "test-groups": "['core', 'providers']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow common.compat fab", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": "API Always CLI Operators WWW", + "providers-test-types-list-as-string": "Providers[common.compat,fab]", + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", + ), + pytest.param( + ("airflow/models/test.py",), + { + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="Tests for all airflow core types except providers should run if model file changed", + ), + pytest.param( + ("airflow/file.py",), + { + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="Tests for all airflow core types except providers should run if " + "any other than API/WWW/CLI/Operators file changed.", + ), + pytest.param( + ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), + { + "selected-providers-list-as-string": None, + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "false", + "docs-list-as-string": None, + "upgrade-to-newer-dependencies": "false", + "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", + "core-test-types-list-as-string": None, + "needs-mypy": "false", + "mypy-checks": "[]", + }, + id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", + ), pytest.param( ( "airflow/assets/", From 858f50fcd047be2af98786a5bd00a7b43bfa5100 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 13:48:02 +0530 Subject: [PATCH 04/13] remove print typo --- dev/breeze/tests/test_selective_checks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index cf1c7a47c154f..24dbd3d22fdec 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1717,7 +1717,6 @@ def test_expected_output_pull_request_target( pr_labels=(), default_branch="main", ) - print(f"stderr is {str(stderr)}") assert_outputs_are_printed(expected_outputs, str(stderr)) From 39fcd12fe158d87d7f54742320697917b327d4cd Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 10:47:00 +0530 Subject: [PATCH 05/13] trigger openlineage test when asset files changes --- .../airflow_breeze/utils/selective_checks.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index f29678dadcfdc..c845c27d2d341 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -45,6 +45,7 @@ DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, KIND_VERSION, + OTHER_PROVIDERS_INTEGRATIONS, RUNS_ON_PUBLIC_RUNNER, RUNS_ON_SELF_HOSTED_ASF_RUNNER, RUNS_ON_SELF_HOSTED_RUNNER, @@ -119,6 +120,7 @@ class FileGroupForCi(Enum): ALL_PROVIDER_YAML_FILES = "all_provider_yaml_files" ALL_DOCS_PYTHON_FILES = "all_docs_python_files" TESTS_UTILS_FILES = "test_utils_files" + ASSET_FILES = "asset_files" class AllProvidersSentinel: @@ -253,6 +255,14 @@ def __hash__(self): r"^task_sdk/src/airflow/sdk/.*\.py$", r"^task_sdk/tests/.*\.py$", ], + FileGroupForCi.ASSET_FILES: [ + r"^airflow/assets/", + r"^airflow/models/assets/", + r"^task_sdk/src/airflow/sdk/definitions/asset/", + r"^airflow/datasets/", + r"^airflow/dag_processing/collection.py", + r"airflow/timetables/assets.py", + ], } ) @@ -860,13 +870,24 @@ def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = all_providers_source_files = self._matching_files( FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES ) - if len(all_providers_source_files) == 0 and not self.needs_api_tests: + assets_source_files = self._matching_files( + FileGroupForCi.ASSET_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES + ) + + if ( + len(all_providers_source_files) == 0 + and len(assets_source_files) == 0 + and not self.needs_api_tests + ): # IF API tests are needed, that will trigger extra provider checks return [] else: - affected_providers = self._find_all_providers_affected( - include_docs=False, - ) + if len(all_providers_source_files) != 0: + affected_providers = self._find_all_providers_affected( + include_docs=False, + ) + elif len(assets_source_files) != 0: + affected_providers = OTHER_PROVIDERS_INTEGRATIONS candidate_test_types: set[str] = set() if affected_providers and not isinstance(affected_providers, AllProvidersSentinel): if split_to_individual_providers: From d9050c113443f5bbaac95aeb863c39d7b0e18cac Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 13:41:46 +0530 Subject: [PATCH 06/13] fixing breaking test and adding test for only ol test --- .../airflow_breeze/utils/selective_checks.py | 26 +- dev/breeze/tests/test_selective_checks.py | 353 ++++++++++-------- 2 files changed, 208 insertions(+), 171 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index c845c27d2d341..f6d713c038bba 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -45,7 +45,7 @@ DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, KIND_VERSION, - OTHER_PROVIDERS_INTEGRATIONS, + OPENLINEAGE_INTEGRATION, RUNS_ON_PUBLIC_RUNNER, RUNS_ON_SELF_HOSTED_ASF_RUNNER, RUNS_ON_SELF_HOSTED_RUNNER, @@ -706,6 +706,10 @@ def needs_javascript_scans(self) -> bool: def needs_api_tests(self) -> bool: return self._should_be_run(FileGroupForCi.API_TEST_FILES) + @cached_property + def needs_ol_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.ASSET_FILES) + @cached_property def needs_api_codegen(self) -> bool: return self._should_be_run(FileGroupForCi.API_CODEGEN_FILES) @@ -882,12 +886,9 @@ def _get_providers_test_types_to_run(self, split_to_individual_providers: bool = # IF API tests are needed, that will trigger extra provider checks return [] else: - if len(all_providers_source_files) != 0: - affected_providers = self._find_all_providers_affected( - include_docs=False, - ) - elif len(assets_source_files) != 0: - affected_providers = OTHER_PROVIDERS_INTEGRATIONS + affected_providers = self._find_all_providers_affected( + include_docs=False, + ) candidate_test_types: set[str] = set() if affected_providers and not isinstance(affected_providers, AllProvidersSentinel): if split_to_individual_providers: @@ -1461,6 +1462,8 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro all_providers.add(provider) if self.needs_api_tests: all_providers.add("fab") + if self.needs_ol_tests: + all_providers.add(OPENLINEAGE_INTEGRATION) if all_providers_affected: return ALL_PROVIDERS_SENTINEL if suspended_providers: @@ -1494,10 +1497,11 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro ) if not all_providers: return None - for provider in list(all_providers): - all_providers.update( - get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True) - ) + if not all_providers == {OPENLINEAGE_INTEGRATION}: + for provider in list(all_providers): + all_providers.update( + get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True) + ) return sorted(all_providers) def _is_canary_run(self): diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 749b4e1fa6bdf..18f913558b3ed 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1491,97 +1491,198 @@ def test_expected_output_push( @pytest.mark.parametrize( "files, expected_outputs,", [ - pytest.param( - ("INTHEWILD.md",), - { - "selected-providers-list-as-string": None, - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "false", - "skip-providers-tests": "true", - "test-groups": "[]", - "docs-build": "false", - "docs-list-as-string": None, - "upgrade-to-newer-dependencies": "false", - "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - "core-test-types-list-as-string": None, - "needs-mypy": "false", - "mypy-checks": "[]", - }, - id="Nothing should run if only non-important files changed", - ), - pytest.param( - ("tests/system/any_file.py",), - { - "selected-providers-list-as-string": None, - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "true", - "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - "ts-compile-format-lint-ui,ts-compile-format-lint-www", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": "Always", - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", - }, - id="Only Always and docs build should run if only system tests changed", - ), + # pytest.param( + # ("INTHEWILD.md",), + # { + # "selected-providers-list-as-string": None, + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "false", + # "skip-providers-tests": "true", + # "test-groups": "[]", + # "docs-build": "false", + # "docs-list-as-string": None, + # "upgrade-to-newer-dependencies": "false", + # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "core-test-types-list-as-string": None, + # "needs-mypy": "false", + # "mypy-checks": "[]", + # }, + # id="Nothing should run if only non-important files changed", + # ), + # pytest.param( + # ("tests/system/any_file.py",), + # { + # "selected-providers-list-as-string": None, + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "true", + # "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": "Always", + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="Only Always and docs build should run if only system tests changed", + # ), + # pytest.param( + # ( + # "airflow/cli/test.py", + # "chart/aaaa.txt", + # "providers/tests/google/file.py", + # ), + # { + # "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " + # "cncf.kubernetes common.compat common.sql " + # "facebook google hashicorp microsoft.azure microsoft.mssql mysql " + # "openlineage oracle postgres presto salesforce samba sftp ssh trino", + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "true", + # "needs-helm-tests": "true", + # "run-tests": "true", + # "skip-providers-tests": "false", + # "test-groups": "['core', 'providers']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " + # "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " + # "microsoft.mssql mysql openlineage oracle postgres " + # "presto salesforce samba sftp ssh trino", + # "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "true", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": "Always CLI", + # "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," + # "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," + # "salesforce,samba,sftp,ssh,trino] Providers[google]", + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow', 'mypy-providers']", + # }, + # id="CLI tests and Google-related provider tests should run if cli/chart files changed but " + # "prod image should be build too and k8s tests too", + # ), + # pytest.param( + # ( + # "airflow/cli/file.py", + # "airflow/operators/file.py", + # "airflow/www/file.py", + # "airflow/api/file.py", + # ), + # { + # "selected-providers-list-as-string": "common.compat fab", + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "false", + # "test-groups": "['core', 'providers']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow common.compat fab", + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "false", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": "API Always CLI Operators WWW", + # "providers-test-types-list-as-string": "Providers[common.compat,fab]", + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", + # ), + # pytest.param( + # ("airflow/models/test.py",), + # { + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow", + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "false", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="Tests for all airflow core types except providers should run if model file changed", + # ), + # pytest.param( + # ("airflow/file.py",), + # { + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "prod-image-build": "false", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "true", + # "docs-list-as-string": "apache-airflow", + # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + # "ts-compile-format-lint-ui,ts-compile-format-lint-www", + # "run-kubernetes-tests": "false", + # "upgrade-to-newer-dependencies": "false", + # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + # "needs-mypy": "true", + # "mypy-checks": "['mypy-airflow']", + # }, + # id="Tests for all airflow core types except providers should run if " + # "any other than API/WWW/CLI/Operators file changed.", + # ), + # pytest.param( + # ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), + # { + # "selected-providers-list-as-string": None, + # "all-python-versions": "['3.9']", + # "all-python-versions-list-as-string": "3.9", + # "ci-image-build": "true", + # "needs-helm-tests": "false", + # "run-tests": "true", + # "skip-providers-tests": "true", + # "test-groups": "['core']", + # "docs-build": "false", + # "docs-list-as-string": None, + # "upgrade-to-newer-dependencies": "false", + # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", + # "core-test-types-list-as-string": None, + # "needs-mypy": "false", + # "mypy-checks": "[]", + # }, + # id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", + # ), pytest.param( ( - "airflow/cli/test.py", - "chart/aaaa.txt", - "providers/tests/google/file.py", + "airflow/assets/", + "airflow/models/assets/", + "task_sdk/src/airflow/sdk/definitions/asset/", + "airflow/datasets/", + "airflow/dag_processing/collection.py", + "airflow/timetables/assets.py", ), { - "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " - "cncf.kubernetes common.compat common.sql " - "facebook google hashicorp microsoft.azure microsoft.mssql mysql " - "openlineage oracle postgres presto salesforce samba sftp ssh trino", - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "true", - "needs-helm-tests": "true", - "run-tests": "true", - "skip-providers-tests": "false", - "test-groups": "['core', 'providers']", - "docs-build": "true", - "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " - "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " - "microsoft.mssql mysql openlineage oracle postgres " - "presto salesforce samba sftp ssh trino", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - "run-kubernetes-tests": "true", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": "Always CLI", - "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," - "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," - "salesforce,samba,sftp,ssh,trino] Providers[google]", - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow', 'mypy-providers']", - }, - id="CLI tests and Google-related provider tests should run if cli/chart files changed but " - "prod image should be build too and k8s tests too", - ), - pytest.param( - ( - "airflow/cli/file.py", - "airflow/operators/file.py", - "airflow/www/file.py", - "airflow/api/file.py", - ), - { - "selected-providers-list-as-string": "common.compat fab", + "selected-providers-list-as-string": "openlineage", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "ci-image-build": "true", @@ -1591,86 +1692,17 @@ def test_expected_output_push( "skip-providers-tests": "false", "test-groups": "['core', 'providers']", "docs-build": "true", - "docs-list-as-string": "apache-airflow common.compat fab", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - "ts-compile-format-lint-ui,ts-compile-format-lint-www", - "run-kubernetes-tests": "false", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": "API Always CLI Operators WWW", - "providers-test-types-list-as-string": "Providers[common.compat,fab]", - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", - }, - id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", - ), - pytest.param( - ("airflow/models/test.py",), - { - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "true", - "docs-list-as-string": "apache-airflow", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - "ts-compile-format-lint-ui,ts-compile-format-lint-www", - "run-kubernetes-tests": "false", - "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", - }, - id="Tests for all airflow core types except providers should run if model file changed", - ), - pytest.param( - ("airflow/file.py",), - { - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "prod-image-build": "false", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "true", - "docs-list-as-string": "apache-airflow", + "docs-list-as-string": "apache-airflow openlineage", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", - "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + "core-test-types-list-as-string": "API Always CLI Core Operators Other Serialization WWW", + "providers-test-types-list-as-string": "Providers[openlineage]", "needs-mypy": "true", "mypy-checks": "['mypy-airflow']", }, - id="Tests for all airflow core types except providers should run if " - "any other than API/WWW/CLI/Operators file changed.", - ), - pytest.param( - ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), - { - "selected-providers-list-as-string": None, - "all-python-versions": "['3.9']", - "all-python-versions-list-as-string": "3.9", - "ci-image-build": "true", - "needs-helm-tests": "false", - "run-tests": "true", - "skip-providers-tests": "true", - "test-groups": "['core']", - "docs-build": "false", - "docs-list-as-string": None, - "upgrade-to-newer-dependencies": "false", - "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", - "core-test-types-list-as-string": None, - "needs-mypy": "false", - "mypy-checks": "[]", - }, - id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", + id="No providers tests except openlineage should run if only Asset file changed", ), ], ) @@ -1685,6 +1717,7 @@ def test_expected_output_pull_request_target( pr_labels=(), default_branch="main", ) + print(f"stderr is {str(stderr)}") assert_outputs_are_printed(expected_outputs, str(stderr)) From ce61946644a10decd3a52f27af8e916ff7636a4f Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 13:46:39 +0530 Subject: [PATCH 07/13] uncomment incorrectly commented tests --- dev/breeze/tests/test_selective_checks.py | 362 +++++++++++----------- 1 file changed, 181 insertions(+), 181 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 18f913558b3ed..cf1c7a47c154f 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1491,187 +1491,187 @@ def test_expected_output_push( @pytest.mark.parametrize( "files, expected_outputs,", [ - # pytest.param( - # ("INTHEWILD.md",), - # { - # "selected-providers-list-as-string": None, - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "false", - # "skip-providers-tests": "true", - # "test-groups": "[]", - # "docs-build": "false", - # "docs-list-as-string": None, - # "upgrade-to-newer-dependencies": "false", - # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "core-test-types-list-as-string": None, - # "needs-mypy": "false", - # "mypy-checks": "[]", - # }, - # id="Nothing should run if only non-important files changed", - # ), - # pytest.param( - # ("tests/system/any_file.py",), - # { - # "selected-providers-list-as-string": None, - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "true", - # "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": "Always", - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="Only Always and docs build should run if only system tests changed", - # ), - # pytest.param( - # ( - # "airflow/cli/test.py", - # "chart/aaaa.txt", - # "providers/tests/google/file.py", - # ), - # { - # "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " - # "cncf.kubernetes common.compat common.sql " - # "facebook google hashicorp microsoft.azure microsoft.mssql mysql " - # "openlineage oracle postgres presto salesforce samba sftp ssh trino", - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "true", - # "needs-helm-tests": "true", - # "run-tests": "true", - # "skip-providers-tests": "false", - # "test-groups": "['core', 'providers']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " - # "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " - # "microsoft.mssql mysql openlineage oracle postgres " - # "presto salesforce samba sftp ssh trino", - # "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "true", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": "Always CLI", - # "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," - # "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," - # "salesforce,samba,sftp,ssh,trino] Providers[google]", - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow', 'mypy-providers']", - # }, - # id="CLI tests and Google-related provider tests should run if cli/chart files changed but " - # "prod image should be build too and k8s tests too", - # ), - # pytest.param( - # ( - # "airflow/cli/file.py", - # "airflow/operators/file.py", - # "airflow/www/file.py", - # "airflow/api/file.py", - # ), - # { - # "selected-providers-list-as-string": "common.compat fab", - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "false", - # "test-groups": "['core', 'providers']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow common.compat fab", - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "false", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": "API Always CLI Operators WWW", - # "providers-test-types-list-as-string": "Providers[common.compat,fab]", - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", - # ), - # pytest.param( - # ("airflow/models/test.py",), - # { - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow", - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "false", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="Tests for all airflow core types except providers should run if model file changed", - # ), - # pytest.param( - # ("airflow/file.py",), - # { - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "prod-image-build": "false", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "true", - # "docs-list-as-string": "apache-airflow", - # "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," - # "ts-compile-format-lint-ui,ts-compile-format-lint-www", - # "run-kubernetes-tests": "false", - # "upgrade-to-newer-dependencies": "false", - # "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, - # "needs-mypy": "true", - # "mypy-checks": "['mypy-airflow']", - # }, - # id="Tests for all airflow core types except providers should run if " - # "any other than API/WWW/CLI/Operators file changed.", - # ), - # pytest.param( - # ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), - # { - # "selected-providers-list-as-string": None, - # "all-python-versions": "['3.9']", - # "all-python-versions-list-as-string": "3.9", - # "ci-image-build": "true", - # "needs-helm-tests": "false", - # "run-tests": "true", - # "skip-providers-tests": "true", - # "test-groups": "['core']", - # "docs-build": "false", - # "docs-list-as-string": None, - # "upgrade-to-newer-dependencies": "false", - # "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - # "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", - # "core-test-types-list-as-string": None, - # "needs-mypy": "false", - # "mypy-checks": "[]", - # }, - # id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", - # ), + pytest.param( + ("INTHEWILD.md",), + { + "selected-providers-list-as-string": None, + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "false", + "skip-providers-tests": "true", + "test-groups": "[]", + "docs-build": "false", + "docs-list-as-string": None, + "upgrade-to-newer-dependencies": "false", + "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "core-test-types-list-as-string": None, + "needs-mypy": "false", + "mypy-checks": "[]", + }, + id="Nothing should run if only non-important files changed", + ), + pytest.param( + ("tests/system/any_file.py",), + { + "selected-providers-list-as-string": None, + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "true", + "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": "Always", + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="Only Always and docs build should run if only system tests changed", + ), + pytest.param( + ( + "airflow/cli/test.py", + "chart/aaaa.txt", + "providers/tests/google/file.py", + ), + { + "selected-providers-list-as-string": "amazon apache.beam apache.cassandra " + "cncf.kubernetes common.compat common.sql " + "facebook google hashicorp microsoft.azure microsoft.mssql mysql " + "openlineage oracle postgres presto salesforce samba sftp ssh trino", + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "true", + "needs-helm-tests": "true", + "run-tests": "true", + "skip-providers-tests": "false", + "test-groups": "['core', 'providers']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra " + "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " + "microsoft.mssql mysql openlineage oracle postgres " + "presto salesforce samba sftp ssh trino", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "true", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": "Always CLI", + "providers-test-types-list-as-string": "Providers[amazon] Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.compat,common.sql,facebook," + "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," + "salesforce,samba,sftp,ssh,trino] Providers[google]", + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow', 'mypy-providers']", + }, + id="CLI tests and Google-related provider tests should run if cli/chart files changed but " + "prod image should be build too and k8s tests too", + ), + pytest.param( + ( + "airflow/cli/file.py", + "airflow/operators/file.py", + "airflow/www/file.py", + "airflow/api/file.py", + ), + { + "selected-providers-list-as-string": "common.compat fab", + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "false", + "test-groups": "['core', 'providers']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow common.compat fab", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": "API Always CLI Operators WWW", + "providers-test-types-list-as-string": "Providers[common.compat,fab]", + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", + ), + pytest.param( + ("airflow/models/test.py",), + { + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="Tests for all airflow core types except providers should run if model file changed", + ), + pytest.param( + ("airflow/file.py",), + { + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "true", + "docs-list-as-string": "apache-airflow", + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "run-kubernetes-tests": "false", + "upgrade-to-newer-dependencies": "false", + "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, + "needs-mypy": "true", + "mypy-checks": "['mypy-airflow']", + }, + id="Tests for all airflow core types except providers should run if " + "any other than API/WWW/CLI/Operators file changed.", + ), + pytest.param( + ("airflow/api_fastapi/core_api/openapi/v1-generated.yaml",), + { + "selected-providers-list-as-string": None, + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "needs-helm-tests": "false", + "run-tests": "true", + "skip-providers-tests": "true", + "test-groups": "['core']", + "docs-build": "false", + "docs-list-as-string": None, + "upgrade-to-newer-dependencies": "false", + "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," + "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-www", + "core-test-types-list-as-string": None, + "needs-mypy": "false", + "mypy-checks": "[]", + }, + id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.", + ), pytest.param( ( "airflow/assets/", From cbc69b51ebe9fdefcbc54c021299fa9e1bcfe6e1 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Tue, 19 Nov 2024 13:48:02 +0530 Subject: [PATCH 08/13] remove print typo --- dev/breeze/tests/test_selective_checks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index cf1c7a47c154f..24dbd3d22fdec 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1717,7 +1717,6 @@ def test_expected_output_pull_request_target( pr_labels=(), default_branch="main", ) - print(f"stderr is {str(stderr)}") assert_outputs_are_printed(expected_outputs, str(stderr)) From 671f69b1b1c09b0e1a256699e251019fd0da8e82 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Wed, 20 Nov 2024 08:32:38 +0530 Subject: [PATCH 09/13] removing checks to ignore dependent test check --- .../src/airflow_breeze/utils/selective_checks.py | 13 ++++++------- dev/breeze/tests/test_selective_checks.py | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index f6d713c038bba..97e104ec28814 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -45,7 +45,6 @@ DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, KIND_VERSION, - OPENLINEAGE_INTEGRATION, RUNS_ON_PUBLIC_RUNNER, RUNS_ON_SELF_HOSTED_ASF_RUNNER, RUNS_ON_SELF_HOSTED_RUNNER, @@ -1463,7 +1462,7 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro if self.needs_api_tests: all_providers.add("fab") if self.needs_ol_tests: - all_providers.add(OPENLINEAGE_INTEGRATION) + all_providers.add("openlineage") if all_providers_affected: return ALL_PROVIDERS_SENTINEL if suspended_providers: @@ -1497,11 +1496,11 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro ) if not all_providers: return None - if not all_providers == {OPENLINEAGE_INTEGRATION}: - for provider in list(all_providers): - all_providers.update( - get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True) - ) + + for provider in list(all_providers): + all_providers.update( + get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True) + ) return sorted(all_providers) def _is_canary_run(self): diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 24dbd3d22fdec..7e580406777d6 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1682,7 +1682,7 @@ def test_expected_output_push( "airflow/timetables/assets.py", ), { - "selected-providers-list-as-string": "openlineage", + "selected-providers-list-as-string": "amazon common.compat common.io common.sql dbt.cloud ftp google mysql openlineage postgres sftp snowflake trino", "all-python-versions": "['3.9']", "all-python-versions-list-as-string": "3.9", "ci-image-build": "true", @@ -1692,13 +1692,13 @@ def test_expected_output_push( "skip-providers-tests": "false", "test-groups": "['core', 'providers']", "docs-build": "true", - "docs-list-as-string": "apache-airflow openlineage", + "docs-list-as-string": "apache-airflow amazon common.compat common.io common.sql dbt.cloud ftp google mysql openlineage postgres sftp snowflake trino", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-string": "API Always CLI Core Operators Other Serialization WWW", - "providers-test-types-list-as-string": "Providers[openlineage]", + "providers-test-types-list-as-string": "Providers[amazon] Providers[common.compat,common.io,common.sql,dbt.cloud,ftp,mysql,openlineage,postgres,sftp,snowflake,trino] Providers[google]", "needs-mypy": "true", "mypy-checks": "['mypy-airflow']", }, From 6e760feb232ff636aba34c60b240167809d7e48c Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Wed, 20 Nov 2024 08:47:41 +0530 Subject: [PATCH 10/13] update tests --- dev/breeze/tests/test_selective_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 7e580406777d6..5efb587beb397 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1702,7 +1702,7 @@ def test_expected_output_push( "needs-mypy": "true", "mypy-checks": "['mypy-airflow']", }, - id="No providers tests except openlineage should run if only Asset file changed", + id="Trigger openlineage and related providers tests when Assets files changed", ), ], ) From 91938e4c46887e89fe6d0eab4a23e2ef3001b7a1 Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Wed, 20 Nov 2024 19:36:08 +0530 Subject: [PATCH 11/13] remove not required asset files --- dev/breeze/src/airflow_breeze/utils/selective_checks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 97e104ec28814..c5ecef5526171 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -259,8 +259,6 @@ def __hash__(self): r"^airflow/models/assets/", r"^task_sdk/src/airflow/sdk/definitions/asset/", r"^airflow/datasets/", - r"^airflow/dag_processing/collection.py", - r"airflow/timetables/assets.py", ], } ) From 3f7195d4d6761df692ed3ffa7516ff733574958c Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Wed, 20 Nov 2024 19:57:59 +0530 Subject: [PATCH 12/13] remove not required asset files from tests --- dev/breeze/tests/test_selective_checks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 5efb587beb397..62e84b1a2e801 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1678,8 +1678,6 @@ def test_expected_output_push( "airflow/models/assets/", "task_sdk/src/airflow/sdk/definitions/asset/", "airflow/datasets/", - "airflow/dag_processing/collection.py", - "airflow/timetables/assets.py", ), { "selected-providers-list-as-string": "amazon common.compat common.io common.sql dbt.cloud ftp google mysql openlineage postgres sftp snowflake trino", From e40bd06ae128b568a2bf319faf1494762be3023a Mon Sep 17 00:00:00 2001 From: vatsrahul1001 Date: Wed, 20 Nov 2024 20:17:06 +0530 Subject: [PATCH 13/13] fix test failures --- dev/breeze/tests/test_selective_checks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 62e84b1a2e801..2f00013f89f08 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1691,14 +1691,14 @@ def test_expected_output_push( "test-groups": "['core', 'providers']", "docs-build": "true", "docs-list-as-string": "apache-airflow amazon common.compat common.io common.sql dbt.cloud ftp google mysql openlineage postgres sftp snowflake trino", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," + "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-string": "API Always CLI Core Operators Other Serialization WWW", "providers-test-types-list-as-string": "Providers[amazon] Providers[common.compat,common.io,common.sql,dbt.cloud,ftp,mysql,openlineage,postgres,sftp,snowflake,trino] Providers[google]", - "needs-mypy": "true", - "mypy-checks": "['mypy-airflow']", + "needs-mypy": "false", + "mypy-checks": "[]", }, id="Trigger openlineage and related providers tests when Assets files changed", ),