From 9f11b9d656cfccbda9d9f86a6fae2899a675d698 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Wed, 4 Oct 2023 03:08:22 +0400 Subject: [PATCH 1/4] Allow to use pre-release version of `pendulum` in the breeze --- Dockerfile.ci | 7 +++++ .../commands/developer_commands.py | 4 +++ .../commands/developer_commands_config.py | 1 + .../commands/testing_commands.py | 4 +++ .../commands/testing_commands_config.py | 1 + .../src/airflow_breeze/params/shell_params.py | 1 + .../airflow_breeze/utils/common_options.py | 8 ++++++ .../utils/custom_param_types.py | 17 +++++++++++ .../utils/docker_command_utils.py | 2 ++ images/breeze/output-commands-hash.txt | 6 ++-- images/breeze/output_shell.svg | 28 +++++++++++-------- images/breeze/output_testing_tests.svg | 26 +++++++++-------- scripts/ci/docker-compose/_docker.env | 1 + scripts/ci/docker-compose/base.yml | 1 + scripts/ci/docker-compose/devcontainer.env | 1 + scripts/docker/entrypoint_ci.sh | 7 +++++ 16 files changed, 89 insertions(+), 26 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 10355b5f046db..277b969cfd09b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1002,6 +1002,13 @@ if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" pip check fi +if [[ -n ${USE_PENDULUM_PRERELEASE:=} ]]; then + echo + echo "${COLOR_BLUE}Replace pendulum by pre-release version: ${USE_PENDULUM_PRERELEASE}${COLOR_RESET}" + echo + pip install --root-user-action ignore "pendulum==${USE_PENDULUM_PRERELEASE}" + pip check +fi set +u if [[ "${RUN_TESTS}" != "true" ]]; then diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 5829cc471c19d..50cd39476e8b2 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -75,6 +75,7 @@ option_upgrade_boto, option_use_airflow_version, option_use_packages_from_dist, + option_use_pendulum_prerelease, option_verbose, ) from airflow_breeze.utils.console import get_console @@ -167,6 +168,7 @@ def run(self): @option_include_mypy_volume @option_upgrade_boto @option_downgrade_sqlalchemy +@option_use_pendulum_prerelease @option_verbose @option_dry_run @option_github_repository @@ -206,6 +208,7 @@ def shell( upgrade_boto: bool, downgrade_sqlalchemy: bool, standalone_dag_processor: bool, + use_pendulum_prerelease: str, ): """Enter breeze environment. this is the default command use when no other is selected.""" if get_verbose() or get_dry_run(): @@ -246,6 +249,7 @@ def shell( upgrade_boto=upgrade_boto, downgrade_sqlalchemy=downgrade_sqlalchemy, standalone_dag_processor=standalone_dag_processor, + use_pendulum_prerelease=use_pendulum_prerelease, ) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index e06adb5665371..3bfd2df5553ac 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -111,6 +111,7 @@ "options": [ "--upgrade-boto", "--downgrade-sqlalchemy", + "--use-pendulum-prerelease", ], }, ], diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 026ce7482c964..551f1cd04fc4e 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -55,6 +55,7 @@ option_skip_cleanup, option_upgrade_boto, option_use_airflow_version, + option_use_pendulum_prerelease, option_verbose, ) from airflow_breeze.utils.console import Output, get_console @@ -363,6 +364,7 @@ def run_tests_in_parallel( ) @option_upgrade_boto @option_downgrade_sqlalchemy +@option_use_pendulum_prerelease @click.option( "--collect-only", help="Collect tests only, do not run them.", @@ -407,6 +409,7 @@ def command_for_tests( extra_pytest_args: tuple, upgrade_boto: bool, downgrade_sqlalchemy: bool, + use_pendulum_prerelease: str, collect_only: bool, remove_arm_packages: bool, github_repository: str, @@ -428,6 +431,7 @@ def command_for_tests( test_type=test_type, upgrade_boto=upgrade_boto, downgrade_sqlalchemy=downgrade_sqlalchemy, + use_pendulum_prerelease=use_pendulum_prerelease, collect_only=collect_only, remove_arm_packages=remove_arm_packages, github_repository=github_repository, diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index b1913d9f058df..0105e53e35e9b 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -57,6 +57,7 @@ "--mount-sources", "--upgrade-boto", "--downgrade-sqlalchemy", + "--use-pendulum-prerelease", "--remove-arm-packages", "--skip-docker-compose-down", ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6ca36e58b5dfc..4ad1982f56186 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -121,6 +121,7 @@ class ShellParams: verbose: bool = False upgrade_boto: bool = False downgrade_sqlalchemy: bool = False + use_pendulum_prerelease: str = "" executor: str = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTORS celery_broker: str = DEFAULT_CELERY_BROKER celery_flower: bool = False diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 6165ae69dfb72..b08618946a761 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -54,6 +54,7 @@ CacheableDefault, DryRunOption, MySQLBackendVersionType, + PreReleaseVersion, UseAirflowVersionType, VerboseOption, ) @@ -642,3 +643,10 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, is_flag=True, envvar="DOWNGRADE_SQLALCHEMY", ) +option_use_pendulum_prerelease = click.option( + "--use-pendulum-prerelease", + help="Use pre-released version of pendulum.", + required=False, + type=PreReleaseVersion(), + envvar="USE_PENDULUM_PRERELEASE", +) diff --git a/dev/breeze/src/airflow_breeze/utils/custom_param_types.py b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py index f4983aa33f784..063d9d24770e3 100644 --- a/dev/breeze/src/airflow_breeze/utils/custom_param_types.py +++ b/dev/breeze/src/airflow_breeze/utils/custom_param_types.py @@ -224,3 +224,20 @@ def convert(self, value, param, ctx): if re.match(r"^\d*\.\d*\.\d*\S*$", value): return value return super().convert(value, param, ctx) + + +class PreReleaseVersion(ParamType): + name = "PreReleaseVersion" + + def convert(self, value, param, ctx): + from packaging.version import Version + + if value: + try: + version = Version(value) + except ValueError: + self.fail(f"Unable to parse version {value!r}.", param, ctx) + else: + if not version.is_prerelease: + self.fail(f"Expected pre-release version, but got {value!r}.", param, ctx) + return super().convert(value, param, ctx) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 39bf0798720d4..f100595dce613 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -611,6 +611,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false") set_value_to_default_if_not_set(env, "DOWNGRADE_SQLALCHEMY", "false") set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false") + set_value_to_default_if_not_set(env, "USE_PENDULUM_PRERELEASE", "") set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false") set_value_to_default_if_not_set(env, "VERBOSE", "false") set_value_to_default_if_not_set(env, "VERBOSE_COMMANDS", "false") @@ -658,6 +659,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: "START_AIRFLOW": "start_airflow", "UPGRADE_BOTO": "upgrade_boto", "DOWNGRADE_SQLALCHEMY": "downgrade_sqlalchemy", + "USE_PENDULUM_PRERELEASE": "use_pendulum_prerelease", "USE_AIRFLOW_VERSION": "use_airflow_version", "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 4c1799c333a29..22738b237e69c 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -62,11 +62,11 @@ setup:regenerate-command-images:a228071206e00b6004ede2e69b8f4a6c setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e setup:c569719a4fc7be726556a266e447c871 -shell:676e7d054d7ce079301018a98f406545 +shell:34ba9dbbfaf105befa2a21816ccd7f31 start-airflow:9a5668007f0acf1b954456c12d8b4051 static-checks:19926b8fcea5784b28d4a0d99865363c testing:docker-compose-tests:fd154a058082fcfda12eb877a9a89338 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 testing:integration-tests:f57fb275b9733a6226601f8095ad4de0 -testing:tests:ce35463d1c67f3416eba77d3201c90ac -testing:9c9f35945852ff48da2f5aba6447463e +testing:tests:0781727da542793f45709f1b1e57f406 +testing:a7b287b9a2226f2250eb8a2a55529f9d diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index 345b2b7dc7b76..9b960981da3ed 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -1,4 +1,4 @@ - +