Prevent __init__.py in providers from being modified

.pre-commit-config.yaml

@@ -352,6 +352,13 @@ repos:
         pass_filenames: true
         files: ^providers/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$
         additional_dependencies: [ 'rich>=12.4.4' ]
+      - id: update-providers-init-py
+        name: Update providers __init__.py files
+        entry: ./scripts/ci/pre_commit/update_providers_init.py
+        language: python
+        pass_filenames: true
+        files: ^providers/.*/__init__.py$|^providers/.*/provider.yaml$|^airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE.py.jinja2^
+        additional_dependencies: ['rich>=12.4.4','requests']
       - id: ruff
         name: Run 'ruff' for extremely fast Python linting
         description: "Run 'ruff' for extremely fast Python linting"

contributing-docs/08_static_code_checks.rst

@@ -384,6 +384,8 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | update-providers-dependencies                             | Update dependencies for provider packages              |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
+| update-providers-init-py                                  | Update providers __init__.py files                     |         |
++-----------------------------------------------------------+--------------------------------------------------------+---------+
 | update-reproducible-source-date-epoch                     | Update Source Date Epoch for reproducible builds       |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | update-spelling-wordlist-to-be-sorted                     | Sort spelling_wordlist.txt                             |         |

dev/breeze/doc/images/output_static-checks.txt

@@ -1 +1 @@
-53b7f32a93cb7dec849138d404c47f6c
+f9eb3d902f6df6f3bcf01be469d958e3

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -777,18 +777,19 @@ def prepare_provider_documentation(
                     with_breaking_changes=with_breaking_changes,
                     maybe_with_new_features=maybe_with_new_features,
                 )
-            with ci_group(
-                f"Updates changelog for last release of package '{provider_id}'",
-                skip_printing_title=only_min_version_update,
-            ):
-                update_changelog(
-                    package_id=provider_id,
-                    base_branch=base_branch,
-                    reapply_templates_only=reapply_templates_only,
-                    with_breaking_changes=with_breaking_changes,
-                    maybe_with_new_features=maybe_with_new_features,
-                    only_min_version_update=only_min_version_update,
-                )
+            if not only_min_version_update:
+                with ci_group(
+                    f"Updates changelog for last release of package '{provider_id}'",
+                    skip_printing_title=only_min_version_update,
+                ):
+                    update_changelog(
+                        package_id=provider_id,
+                        base_branch=base_branch,
+                        reapply_templates_only=reapply_templates_only,
+                        with_breaking_changes=with_breaking_changes,
+                        maybe_with_new_features=maybe_with_new_features,
+                        only_min_version_update=only_min_version_update,
+                    )
         except PrepareReleaseDocsNoChangesException:
             no_changes_packages.append(provider_id)
         except PrepareReleaseDocsChangesOnlyException:

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -146,6 +146,7 @@
     "update-migration-references",
     "update-openapi-spec-tags-to-be-sorted",
     "update-providers-dependencies",
+    "update-providers-init-py",
     "update-reproducible-source-date-epoch",
     "update-spelling-wordlist-to-be-sorted",
     "update-supported-versions",

dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py

@@ -1068,6 +1068,7 @@ def update_changelog(
     :param reapply_templates_only: only reapply templates, no changelog generation
     :param with_breaking_changes: whether there are any breaking changes
     :param maybe_with_new_features: whether there are any new features
+    :param only_min_version_update: whether to only update the min version
     """
     provider_details = get_provider_details(package_id)
     jinja_context = get_provider_documentation_jinja_context(

dev/breeze/src/airflow_breeze/utils/packages.py

@@ -554,7 +554,7 @@ def get_provider_details(provider_id: str) -> ProviderPackageDetails:
         provider_description=provider_info["description"],
         dependencies=provider_info["dependencies"],
         versions=provider_info["versions"],
-        excluded_python_versions=provider_info.get("excluded-python-versions") or [],
+        excluded_python_versions=provider_info.get("excluded-python-versions", []),
         plugins=plugins,
         removed=provider_info["state"] == "removed",
     )

providers/src/airflow/providers/common/compat/__init__.py

@@ -23,21 +23,17 @@
 #
 from __future__ import annotations
 
-from packaging.version import Version
+import packaging.version
 
-from airflow import __version__ as AIRFLOW_VERSION
+from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
 __version__ = "1.2.2"
 
-
-AIRFLOW_V_3_0_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("3.0.0")
-AIRFLOW_V_2_10_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.10.0")
-AIRFLOW_V_2_9_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.9.0")
-AIRFLOW_V_2_8_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.8.0")
-
-if not AIRFLOW_V_2_8_PLUS:
+if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
+    "2.8.0"
+):
     raise RuntimeError(
         f"The package `apache-airflow-providers-common-compat:{__version__}` needs Apache Airflow 2.8.0+"
     )

providers/src/airflow/providers/common/compat/assets/__init__.py

@@ -19,7 +19,7 @@
 
 from typing import TYPE_CHECKING
 
-from airflow.providers.common.compat import (
+from airflow.providers.common.compat.version_references import (
     AIRFLOW_V_2_8_PLUS,
     AIRFLOW_V_2_9_PLUS,
     AIRFLOW_V_2_10_PLUS,

providers/src/airflow/providers/common/compat/lineage/hook.py

@@ -16,7 +16,7 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.providers.common.compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS
+from airflow.providers.common.compat.version_references import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS
 
 
 def _get_asset_compat_hook_lineage_collector():

providers/src/airflow/providers/common/compat/version_references.py

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from packaging.version import Version
+
+from airflow import __version__ as AIRFLOW_VERSION
+
+AIRFLOW_V_3_0_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("3.0.0")
+AIRFLOW_V_2_10_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.10.0")
+AIRFLOW_V_2_9_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.9.0")
+AIRFLOW_V_2_8_PLUS = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.8.0")

providers/src/airflow/providers/fab/__init__.py

@@ -29,11 +29,11 @@
 
 __all__ = ["__version__"]
 
-__version__ = "1.5.1"
+__version__ = "2.0.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.9.0"
+    "3.0.0.dev0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-fab:{__version__}` needs Apache Airflow 2.9.0+"
+        f"The package `apache-airflow-providers-fab:{__version__}` needs Apache Airflow 3.0.0.dev0+"
     )

scripts/ci/pre_commit/update_providers_init.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
+from common_precommit_utils import console, initialize_breeze_precommit
+
+initialize_breeze_precommit(__name__, __file__)
+
+providers: set[str] = set()
+
+file_list = sys.argv[1:]
+console.print(f"[bright_blue]Determining providers to regenerate from: {file_list}\n")
+
+# get all folders from arguments
+for examined_file in file_list:
+    if not examined_file.startswith("providers/src"):
+        continue
+    console.print(f"[bright_blue]Looking at {examined_file} for provider.yaml")
+    # find the folder where provider.yaml is
+    for parent in Path(examined_file).parents:
+        console.print(f"[bright_blue]Checking {parent}")
+        if (parent / "provider.yaml").exists():
+            provider_folder = parent
+            break
+    else:
+        console.print(f"[yellow]\nCould not find `provider.yaml` in any parent of {examined_file}[/]")
+        continue
+    # find base for the provider sources
+    for parent in provider_folder.parents:
+        if parent.name == "providers":
+            base_folder = parent
+            console.print(f"[bright_blue]Found base folder {base_folder}")
+            break
+    else:
+        console.print(f"[red]\nCould not find base folder for {provider_folder}")
+        sys.exit(1)
+    provider_name = ".".join(provider_folder.relative_to(base_folder).as_posix().split("/"))
+    providers.add(provider_name)
+
+console.print(f"[bright_blue]Regenerating providers __init__ files for providers: {providers}[/]")
+
+if not providers:
+    console.print("[red]\nThe found providers list cannot be empty[/]")
+    sys.exit(1)
+
+res = subprocess.run(
+    [
+        "breeze",
+        "release-management",
+        "prepare-provider-documentation",
+        "--reapply-templates-only",
+        "--skip-git-fetch",
+        "--only-min-version-update",
+        *list(providers),
+    ],
+    check=False,
+)
+if res.returncode != 0:
+    console.print("[red]\nError while regenerating provider init files.")
+    sys.exit(res.returncode)