Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
520e2f9
trigger openlineage test when asset files changes
vatsrahul1001 Nov 19, 2024
75f2e4c
fixing breaking test and adding test for only ol test
vatsrahul1001 Nov 19, 2024
9a73b3e
uncomment incorrectly commented tests
vatsrahul1001 Nov 19, 2024
858f50f
remove print typo
vatsrahul1001 Nov 19, 2024
39fcd12
trigger openlineage test when asset files changes
vatsrahul1001 Nov 19, 2024
d9050c1
fixing breaking test and adding test for only ol test
vatsrahul1001 Nov 19, 2024
ce61946
uncomment incorrectly commented tests
vatsrahul1001 Nov 19, 2024
cbc69b5
remove print typo
vatsrahul1001 Nov 19, 2024
b4492d8
Merge branch 'trigger-openlineage-tests-on-asset-change' of github.co…
vatsrahul1001 Nov 20, 2024
671f69b
removing checks to ignore dependent test check
vatsrahul1001 Nov 20, 2024
b06cfd2
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 20, 2024
6e760fe
update tests
vatsrahul1001 Nov 20, 2024
9934919
Merge branch 'trigger-openlineage-tests-on-asset-change' of github.co…
vatsrahul1001 Nov 20, 2024
9afb9b8
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 20, 2024
f5be17a
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 20, 2024
f256620
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 20, 2024
91938e4
remove not required asset files
vatsrahul1001 Nov 20, 2024
3f7195d
remove not required asset files from tests
vatsrahul1001 Nov 20, 2024
e40bd06
fix test failures
vatsrahul1001 Nov 20, 2024
15ddb24
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 21, 2024
ca43d69
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 21, 2024
c66f998
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 21, 2024
fc5ec25
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 22, 2024
376541e
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 22, 2024
6c6647c
Merge branch 'main' into trigger-openlineage-tests-on-asset-change
vatsrahul1001 Nov 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion dev/breeze/src/airflow_breeze/utils/selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ class FileGroupForCi(Enum):
ALL_PROVIDER_YAML_FILES = "all_provider_yaml_files"
ALL_DOCS_PYTHON_FILES = "all_docs_python_files"
TESTS_UTILS_FILES = "test_utils_files"
ASSET_FILES = "asset_files"


class AllProvidersSentinel:
Expand Down Expand Up @@ -253,6 +254,12 @@ def __hash__(self):
r"^task_sdk/src/airflow/sdk/.*\.py$",
r"^task_sdk/tests/.*\.py$",
],
FileGroupForCi.ASSET_FILES: [
r"^airflow/assets/",
r"^airflow/models/assets/",
r"^task_sdk/src/airflow/sdk/definitions/asset/",
r"^airflow/datasets/",
],
}
)

Expand Down Expand Up @@ -696,6 +703,10 @@ def needs_javascript_scans(self) -> bool:
def needs_api_tests(self) -> bool:
return self._should_be_run(FileGroupForCi.API_TEST_FILES)

@cached_property
def needs_ol_tests(self) -> bool:
return self._should_be_run(FileGroupForCi.ASSET_FILES)

@cached_property
def needs_api_codegen(self) -> bool:
return self._should_be_run(FileGroupForCi.API_CODEGEN_FILES)
Expand Down Expand Up @@ -860,7 +871,15 @@ def _get_providers_test_types_to_run(self, split_to_individual_providers: bool =
all_providers_source_files = self._matching_files(
FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
)
if len(all_providers_source_files) == 0 and not self.needs_api_tests:
assets_source_files = self._matching_files(
FileGroupForCi.ASSET_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
)

if (
len(all_providers_source_files) == 0
and len(assets_source_files) == 0
and not self.needs_api_tests
):
# IF API tests are needed, that will trigger extra provider checks
return []
else:
Expand Down Expand Up @@ -1440,6 +1459,8 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro
all_providers.add(provider)
if self.needs_api_tests:
all_providers.add("fab")
if self.needs_ol_tests:
all_providers.add("openlineage")
if all_providers_affected:
return ALL_PROVIDERS_SENTINEL
if suspended_providers:
Expand Down Expand Up @@ -1473,6 +1494,7 @@ def _find_all_providers_affected(self, include_docs: bool) -> list[str] | AllPro
)
if not all_providers:
return None

for provider in list(all_providers):
all_providers.update(
get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True)
Expand Down
30 changes: 30 additions & 0 deletions dev/breeze/tests/test_selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1672,6 +1672,36 @@ def test_expected_output_push(
},
id="pre commit ts-compile-format-lint should not be ignored if openapi spec changed.",
),
pytest.param(
(
"airflow/assets/",
"airflow/models/assets/",
"task_sdk/src/airflow/sdk/definitions/asset/",
"airflow/datasets/",
),
{
"selected-providers-list-as-string": "amazon common.compat common.io common.sql dbt.cloud ftp google mysql openlineage postgres sftp snowflake trino",
"all-python-versions": "['3.9']",
"all-python-versions-list-as-string": "3.9",
"ci-image-build": "true",
"prod-image-build": "false",
"needs-helm-tests": "false",
"run-tests": "true",
"skip-providers-tests": "false",
"test-groups": "['core', 'providers']",
"docs-build": "true",
"docs-list-as-string": "apache-airflow amazon common.compat common.io common.sql dbt.cloud ftp google mysql openlineage postgres sftp snowflake trino",
"skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,"
"ts-compile-format-lint-ui,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-string": "API Always CLI Core Operators Other Serialization WWW",
"providers-test-types-list-as-string": "Providers[amazon] Providers[common.compat,common.io,common.sql,dbt.cloud,ftp,mysql,openlineage,postgres,sftp,snowflake,trino] Providers[google]",
"needs-mypy": "false",
"mypy-checks": "[]",
},
id="Trigger openlineage and related providers tests when Assets files changed",
),
],
)
def test_expected_output_pull_request_target(
Expand Down