diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 779b3a46bba8b..848fca0e408a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1188,6 +1188,55 @@ jobs: uses: ./.github/actions/post_tests_failure if: failure() + tests-postgres-pre-release-pendulum: + timeout-minutes: 130 + name: > + PreReleasePendulum${{needs.build-info.outputs.default-postgres-version}}, + Py${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, wait-for-ci-images] + env: + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" + PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" + FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" + DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" + BACKEND: "postgres" + PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" + POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + USE_PENDULUM_PRERELEASE: "3.0.0b1" + JOB_ID: > + postgres-pre-release-pendulum-${{needs.build-info.outputs.default-python-version}}- + ${{needs.build-info.outputs.default-postgres-version}} + COVERAGE: "${{needs.build-info.outputs.run-coverage}}" + if: needs.build-info.outputs.run-tests == 'true' + steps: + - name: Cleanup repo + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: > + Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} + uses: ./.github/actions/prepare_breeze_and_image + - name: > + Tests: ${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} + run: breeze testing tests --run-in-parallel + - name: > + Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Pendulum" + uses: ./.github/actions/post_tests_success + if: success() + - name: > + Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Pendulum" + uses: ./.github/actions/post_tests_failure + if: failure() + tests-mysql: timeout-minutes: 130 name: > diff --git a/Dockerfile.ci b/Dockerfile.ci index 10355b5f046db..277b969cfd09b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1002,6 +1002,13 @@ if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" pip check fi +if [[ -n ${USE_PENDULUM_PRERELEASE:=} ]]; then + echo + echo "${COLOR_BLUE}Replace pendulum by pre-release version: ${USE_PENDULUM_PRERELEASE}${COLOR_RESET}" + echo + pip install --root-user-action ignore "pendulum==${USE_PENDULUM_PRERELEASE}" + pip check +fi set +u if [[ "${RUN_TESTS}" != "true" ]]; then diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 1fbed1489243e..75315a0740ffe 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -63,6 +63,7 @@ from airflow.utils.module_loading import import_string, qualname from airflow.utils.operator_resources import Resources from airflow.utils.task_group import MappedTaskGroup, TaskGroup +from airflow.utils.timezone import parse_timezone from airflow.utils.types import NOTSET, ArgNotSet if TYPE_CHECKING: @@ -165,9 +166,9 @@ def encode_timezone(var: Timezone) -> str | int: ) -def decode_timezone(var: str | int) -> Timezone: +def decode_timezone(var: str | int) -> Timezone | FixedTimezone: """Decode a previously serialized Pendulum Timezone.""" - return pendulum.tz.timezone(var) + return parse_timezone(var) def _get_registered_timetable(importable_string: str) -> type[Timetable] | None: @@ -605,7 +606,7 @@ def deserialize(cls, encoded_var: Any, use_pydantic_models=False) -> Any: raise TypeError(f"Invalid type {type_!s} in deserialization.") _deserialize_datetime = pendulum.from_timestamp - _deserialize_timezone = pendulum.tz.timezone + _deserialize_timezone = parse_timezone @classmethod def _deserialize_timedelta(cls, seconds: int) -> datetime.timedelta: diff --git a/airflow/serialization/serializers/datetime.py b/airflow/serialization/serializers/datetime.py index 49f0899a5918f..ea030a8afcba5 100644 --- a/airflow/serialization/serializers/datetime.py +++ b/airflow/serialization/serializers/datetime.py @@ -24,7 +24,7 @@ serialize as serialize_timezone, ) from airflow.utils.module_loading import qualname -from airflow.utils.timezone import convert_to_utc, is_naive +from airflow.utils.timezone import convert_to_utc, is_naive, parse_timezone if TYPE_CHECKING: import datetime @@ -65,23 +65,22 @@ def deserialize(classname: str, version: int, data: dict | str) -> datetime.date import datetime from pendulum import DateTime - from pendulum.tz import fixed_timezone, timezone tz: datetime.tzinfo | None = None if isinstance(data, dict) and TIMEZONE in data: if version == 1: # try to deserialize unsupported timezones timezone_mapping = { - "EDT": fixed_timezone(-4 * 3600), - "CDT": fixed_timezone(-5 * 3600), - "MDT": fixed_timezone(-6 * 3600), - "PDT": fixed_timezone(-7 * 3600), - "CEST": timezone("CET"), + "EDT": parse_timezone(-4 * 3600), + "CDT": parse_timezone(-5 * 3600), + "MDT": parse_timezone(-6 * 3600), + "PDT": parse_timezone(-7 * 3600), + "CEST": parse_timezone("CET"), } if data[TIMEZONE] in timezone_mapping: tz = timezone_mapping[data[TIMEZONE]] else: - tz = timezone(data[TIMEZONE]) + tz = parse_timezone(data[TIMEZONE]) else: tz = deserialize_timezone(data[TIMEZONE][1], data[TIMEZONE][2], data[TIMEZONE][0]) diff --git a/airflow/serialization/serializers/timezone.py b/airflow/serialization/serializers/timezone.py index 23901b9d444e8..0f580adef83f5 100644 --- a/airflow/serialization/serializers/timezone.py +++ b/airflow/serialization/serializers/timezone.py @@ -74,7 +74,7 @@ def serialize(o: object) -> tuple[U, str, int, bool]: def deserialize(classname: str, version: int, data: object) -> Any: - from pendulum.tz import fixed_timezone, timezone + from airflow.utils.timezone import parse_timezone if not isinstance(data, (str, int)): raise TypeError(f"{data} is not of type int or str but of {type(data)}") @@ -82,9 +82,6 @@ def deserialize(classname: str, version: int, data: object) -> Any: if version > __version__: raise TypeError(f"serialized {version} of {classname} > {__version__}") - if isinstance(data, int): - return fixed_timezone(data) - if "zoneinfo.ZoneInfo" in classname: try: from zoneinfo import ZoneInfo @@ -93,7 +90,7 @@ def deserialize(classname: str, version: int, data: object) -> Any: return ZoneInfo(data) - return timezone(data) + return parse_timezone(data) # ported from pendulum.tz.timezone._get_tzinfo_name diff --git a/airflow/settings.py b/airflow/settings.py index 23bf1cbfcbc8c..d438831a12923 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -40,6 +40,7 @@ from airflow.logging_config import configure_logging from airflow.utils.orm_event_handlers import setup_event_handlers from airflow.utils.state import State +from airflow.utils.timezone import parse_timezone, utc if TYPE_CHECKING: from sqlalchemy.engine import Engine @@ -54,9 +55,9 @@ if tz == "system": TIMEZONE = pendulum.tz.local_timezone() else: - TIMEZONE = pendulum.tz.timezone(tz) + TIMEZONE = parse_timezone(tz) except Exception: - TIMEZONE = pendulum.tz.timezone("UTC") + TIMEZONE = utc log.info("Configured default timezone %s", TIMEZONE) diff --git a/airflow/timetables/_cron.py b/airflow/timetables/_cron.py index f9b8efa465914..f9473ae2db2b8 100644 --- a/airflow/timetables/_cron.py +++ b/airflow/timetables/_cron.py @@ -22,14 +22,14 @@ from cron_descriptor import CasingTypeEnum, ExpressionDescriptor, FormatException, MissingFieldException from croniter import CroniterBadCronError, CroniterBadDateError, croniter -from pendulum.tz.timezone import Timezone from airflow.exceptions import AirflowTimetableInvalid from airflow.utils.dates import cron_presets -from airflow.utils.timezone import convert_to_utc, make_aware, make_naive +from airflow.utils.timezone import convert_to_utc, make_aware, make_naive, parse_timezone if TYPE_CHECKING: from pendulum import DateTime + from pendulum.tz.timezone import Timezone def _is_schedule_fixed(expression: str) -> bool: @@ -56,7 +56,7 @@ def __init__(self, cron: str, timezone: str | Timezone) -> None: self._expression = cron_presets.get(cron, cron) if isinstance(timezone, str): - timezone = Timezone(timezone) + timezone = parse_timezone(timezone) self._timezone = timezone try: diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index a042d4e9024de..fb241f482f511 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -24,7 +24,6 @@ import logging from typing import TYPE_CHECKING, Any, Generator, Iterable, overload -import pendulum from dateutil import relativedelta from sqlalchemy import TIMESTAMP, PickleType, and_, event, false, nullsfirst, or_, true, tuple_ from sqlalchemy.dialects import mssql, mysql @@ -34,7 +33,7 @@ from airflow import settings from airflow.configuration import conf from airflow.serialization.enums import Encoding -from airflow.utils.timezone import make_naive +from airflow.utils.timezone import make_naive, utc if TYPE_CHECKING: from kubernetes.client.models.v1_pod import V1Pod @@ -46,8 +45,6 @@ log = logging.getLogger(__name__) -utc = pendulum.tz.timezone("UTC") - class UtcDateTime(TypeDecorator): """ diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index 12c75bef5976b..61ad95f4b0647 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -23,9 +23,17 @@ import pendulum from dateutil.relativedelta import relativedelta from pendulum.datetime import DateTime +from pendulum.tz import fixed_timezone +from pendulum.tz.timezone import FixedTimezone, Timezone -# UTC time zone as a tzinfo instance. -utc = pendulum.tz.timezone("UTC") +from airflow.compat.functools import cache + +# UTC time zone as a Timezone instance (subclass of tzinfo) +# This type uses for compatibility between pendulum v2 and v3 +# - in pendulum 2.x ``pendulum.tz.timezone`` returns FixedTimezone +# - in pendulum 3.x ``pendulum.timezone`` returns Timezone +# Same is valid for pendulum.tz.UTC +utc = Timezone("UTC") def is_localized(value): @@ -273,3 +281,24 @@ def _format_part(key: str) -> str: if not joined: return "<1s" return joined + + +@cache +def parse_timezone(name: str | int) -> Timezone | FixedTimezone: + """ + Parse timezone and return one of the pendulum Timezone. + + Provide the same interface as ``pendulum.tz.timezone(name)`` + + .. note:: + This class for compatibility between pendulum 2 and 3. + In pendulum 3 ``pendulum.tz.timezone`` it is a module, which can't be used as parser + In pendulum 2 ``pendulum.timezone`` mypy failed on static check + + :meta: private + """ + if isinstance(name, int): + return fixed_timezone(name) + elif name.lower() == "utc": + return utc + return Timezone(name) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 5829cc471c19d..50cd39476e8b2 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -75,6 +75,7 @@ option_upgrade_boto, option_use_airflow_version, option_use_packages_from_dist, + option_use_pendulum_prerelease, option_verbose, ) from airflow_breeze.utils.console import get_console @@ -167,6 +168,7 @@ def run(self): @option_include_mypy_volume @option_upgrade_boto @option_downgrade_sqlalchemy +@option_use_pendulum_prerelease @option_verbose @option_dry_run @option_github_repository @@ -206,6 +208,7 @@ def shell( upgrade_boto: bool, downgrade_sqlalchemy: bool, standalone_dag_processor: bool, + use_pendulum_prerelease: str, ): """Enter breeze environment. this is the default command use when no other is selected.""" if get_verbose() or get_dry_run(): @@ -246,6 +249,7 @@ def shell( upgrade_boto=upgrade_boto, downgrade_sqlalchemy=downgrade_sqlalchemy, standalone_dag_processor=standalone_dag_processor, + use_pendulum_prerelease=use_pendulum_prerelease, ) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index e06adb5665371..3bfd2df5553ac 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -111,6 +111,7 @@ "options": [ "--upgrade-boto", "--downgrade-sqlalchemy", + "--use-pendulum-prerelease", ], }, ], diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 026ce7482c964..551f1cd04fc4e 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -55,6 +55,7 @@ option_skip_cleanup, option_upgrade_boto, option_use_airflow_version, + option_use_pendulum_prerelease, option_verbose, ) from airflow_breeze.utils.console import Output, get_console @@ -363,6 +364,7 @@ def run_tests_in_parallel( ) @option_upgrade_boto @option_downgrade_sqlalchemy +@option_use_pendulum_prerelease @click.option( "--collect-only", help="Collect tests only, do not run them.", @@ -407,6 +409,7 @@ def command_for_tests( extra_pytest_args: tuple, upgrade_boto: bool, downgrade_sqlalchemy: bool, + use_pendulum_prerelease: str, collect_only: bool, remove_arm_packages: bool, github_repository: str, @@ -428,6 +431,7 @@ def command_for_tests( test_type=test_type, upgrade_boto=upgrade_boto, downgrade_sqlalchemy=downgrade_sqlalchemy, + use_pendulum_prerelease=use_pendulum_prerelease, collect_only=collect_only, remove_arm_packages=remove_arm_packages, github_repository=github_repository, diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index b1913d9f058df..0105e53e35e9b 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -57,6 +57,7 @@ "--mount-sources", "--upgrade-boto", "--downgrade-sqlalchemy", + "--use-pendulum-prerelease", "--remove-arm-packages", "--skip-docker-compose-down", ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6ca36e58b5dfc..4ad1982f56186 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -121,6 +121,7 @@ class ShellParams: verbose: bool = False upgrade_boto: bool = False downgrade_sqlalchemy: bool = False + use_pendulum_prerelease: str = "" executor: str = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTORS celery_broker: str = DEFAULT_CELERY_BROKER celery_flower: bool = False diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 6165ae69dfb72..b08618946a761 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -54,6 +54,7 @@ CacheableDefault, DryRunOption, MySQLBackendVersionType, + PreReleaseVersion, UseAirflowVersionType, VerboseOption, ) @@ -642,3 +643,10 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, is_flag=True, envvar="DOWNGRADE_SQLALCHEMY", ) +option_use_pendulum_prerelease = click.option( + "--use-pendulum-prerelease", + help="Use pre-released version of pendulum.", + required=False, + type=PreReleaseVersion(), + envvar="USE_PENDULUM_PRERELEASE", +) diff --git a/dev/breeze/src/airflow_breeze/utils/custom_param_types.py b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py index f4983aa33f784..063d9d24770e3 100644 --- a/dev/breeze/src/airflow_breeze/utils/custom_param_types.py +++ b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py @@ -224,3 +224,20 @@ def convert(self, value, param, ctx): if re.match(r"^\d*\.\d*\.\d*\S*$", value): return value return super().convert(value, param, ctx) + + +class PreReleaseVersion(ParamType): + name = "PreReleaseVersion" + + def convert(self, value, param, ctx): + from packaging.version import Version + + if value: + try: + version = Version(value) + except ValueError: + self.fail(f"Unable to parse version {value!r}.", param, ctx) + else: + if not version.is_prerelease: + self.fail(f"Expected pre-release version, but got {value!r}.", param, ctx) + return super().convert(value, param, ctx) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 39bf0798720d4..f100595dce613 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -611,6 +611,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false") set_value_to_default_if_not_set(env, "DOWNGRADE_SQLALCHEMY", "false") set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false") + set_value_to_default_if_not_set(env, "USE_PENDULUM_PRERELEASE", "") set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false") set_value_to_default_if_not_set(env, "VERBOSE", "false") set_value_to_default_if_not_set(env, "VERBOSE_COMMANDS", "false") @@ -658,6 +659,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: "START_AIRFLOW": "start_airflow", "UPGRADE_BOTO": "upgrade_boto", "DOWNGRADE_SQLALCHEMY": "downgrade_sqlalchemy", + "USE_PENDULUM_PRERELEASE": "use_pendulum_prerelease", "USE_AIRFLOW_VERSION": "use_airflow_version", "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 4c1799c333a29..22738b237e69c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -62,11 +62,11 @@ setup:regenerate-command-images:a228071206e00b6004ede2e69b8f4a6c setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e setup:c569719a4fc7be726556a266e447c871 -shell:676e7d054d7ce079301018a98f406545 +shell:34ba9dbbfaf105befa2a21816ccd7f31 start-airflow:9a5668007f0acf1b954456c12d8b4051 static-checks:19926b8fcea5784b28d4a0d99865363c testing:docker-compose-tests:fd154a058082fcfda12eb877a9a89338 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 testing:integration-tests:f57fb275b9733a6226601f8095ad4de0 -testing:tests:ce35463d1c67f3416eba77d3201c90ac -testing:9c9f35945852ff48da2f5aba6447463e +testing:tests:0781727da542793f45709f1b1e57f406 +testing:a7b287b9a2226f2250eb8a2a55529f9d diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index 345b2b7dc7b76..9b960981da3ed 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -1,4 +1,4 @@ - +