From 364f2aa981fff478e92f250ecbdbd08b6ab53917 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 1 Apr 2022 22:28:35 +0200 Subject: [PATCH] Prepare Breeze2 for prime time :) This is a review and clean-up for all the parameters and commands for Breeze2 in order to prepare it for being used by the contribugors. There are various small fixes here and there, removal of duplicated code, refactoring and moving code around as well as cleanup and review all the parameters used for all implemented commands. The parameters, default values and their behaviours were updated to match "new" life of Breeze rather than old one. Some improvements are made to the autocomplete and click help messages printed. Full list of choices is always displayed, parameters are groups according to their target audience, and they were sorted according to importance and frequency of use. Various messages have been colourised according to their meaning - warnings as yellow, errors as red and informational messages as bright_blue. The `dry-run` option has been added to just show what would have been run without actually running some potentially "write" commands (read commands are still executed) so that you can easily verify and manually copy and execute the commands with option to modify them before. The `dry_run` and `verbose` options are now used for all commands. The "main" command now runs "shell" by default similarly as the original Breeze. All "shortcut" parameters have been standardized - i.e common options (verbose/dry run/help) have one and all common flags that are likely to be used often have an assigned shortcute. The "stop" and "cleanup" command have been added as they are necessary for average user to complete the regular usage cycle. Documentation for all the important methods have been updated. --- .github/workflows/build-images.yml | 2 +- .github/workflows/ci.yml | 2 +- .pre-commit-config.yaml | 2 +- BREEZE.rst | 21 - CONTRIBUTING.rst | 2 +- Dockerfile.ci | 50 +- .../www/ask_for_recompile_assets_if_needed.sh | 2 +- breeze | 12 - breeze-complete | 2 +- dev/breeze/setup.cfg | 7 +- .../src/airflow_breeze/branch_defaults.py | 21 + dev/breeze/src/airflow_breeze/breeze.py | 947 ++++++++++++++---- .../{prod => build_image}/__init__.py | 1 + .../{ => build_image}/ci/__init__.py | 1 + .../build_image/ci/build_ci_image.py | 122 +++ .../ci/build_ci_params.py} | 95 +- .../prod}/__init__.py | 1 + .../build_image/prod/build_prod_image.py | 173 ++++ .../prod/build_prod_params.py} | 141 +-- dev/breeze/src/airflow_breeze/cache.py | 112 --- .../src/airflow_breeze/ci/build_image.py | 124 --- dev/breeze/src/airflow_breeze/console.py | 21 - .../docs_generator/build_documentation.py | 39 - .../src/airflow_breeze/global_constants.py | 110 +- .../airflow_breeze/prod/build_prod_image.py | 200 ---- .../src/airflow_breeze/shell/__init__.py | 1 + .../src/airflow_breeze/shell/enter_shell.py | 313 +++--- .../{shell_builder.py => shell_params.py} | 110 +- .../src/airflow_breeze/utils/__init__.py | 1 + dev/breeze/src/airflow_breeze/utils/cache.py | 141 +++ .../doc_builder.py => utils/console.py} | 33 +- .../utils/docker_command_utils.py | 314 +++++- .../airflow_breeze/utils/host_info_utils.py | 15 +- .../airflow_breeze/utils/md5_build_check.py | 116 +++ .../src/airflow_breeze/utils/path_utils.py | 71 +- .../src/airflow_breeze/utils/registry.py | 56 ++ .../src/airflow_breeze/utils/run_utils.py | 293 ++++-- .../src/airflow_breeze/utils/visuals.py | 144 +++ .../src/airflow_breeze/visuals/__init__.py | 103 -- .../src/airflow_ci/find_newer_dependencies.py | 25 +- dev/breeze/src/airflow_ci/freespace.py | 35 +- dev/breeze/tests/test_build_image.py | 22 +- dev/breeze/tests/test_cache.py | 12 +- dev/breeze/tests/test_commands.py | 18 +- dev/breeze/tests/test_docker_command_utils.py | 25 +- .../tests/test_find_airflow_directory.py | 18 +- dev/breeze/tests/test_prod_image.py | 73 +- dev/retag_docker_images.py | 6 +- scripts/ci/docker-compose/_docker.env | 1 - scripts/ci/docker-compose/base.yml | 1 - scripts/ci/libraries/_build_images.sh | 14 +- scripts/docker/entrypoint_ci.sh | 25 +- scripts/in_container/_in_container_utils.sh | 12 +- scripts/in_container/check_environment.sh | 23 +- scripts/in_container/configure_environment.sh | 18 +- scripts/in_container/run_ci_tests.sh | 4 +- scripts/in_container/run_docs_build.sh | 2 +- scripts/in_container/run_init_script.sh | 5 - .../run_install_and_test_provider_packages.sh | 2 +- .../run_prepare_provider_documentation.sh | 4 +- scripts/in_container/run_system_tests.sh | 4 +- 61 files changed, 2501 insertions(+), 1769 deletions(-) rename dev/breeze/src/airflow_breeze/{prod => build_image}/__init__.py (96%) rename dev/breeze/src/airflow_breeze/{ => build_image}/ci/__init__.py (96%) create mode 100644 dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py rename dev/breeze/src/airflow_breeze/{ci/build_params.py => build_image/ci/build_ci_params.py} (59%) rename dev/breeze/src/airflow_breeze/{docs_generator => build_image/prod}/__init__.py (96%) create mode 100644 dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py rename dev/breeze/src/airflow_breeze/{prod/prod_params.py => build_image/prod/build_prod_params.py} (68%) delete mode 100644 dev/breeze/src/airflow_breeze/cache.py delete mode 100644 dev/breeze/src/airflow_breeze/ci/build_image.py delete mode 100644 dev/breeze/src/airflow_breeze/console.py delete mode 100644 dev/breeze/src/airflow_breeze/docs_generator/build_documentation.py delete mode 100644 dev/breeze/src/airflow_breeze/prod/build_prod_image.py rename dev/breeze/src/airflow_breeze/shell/{shell_builder.py => shell_params.py} (73%) create mode 100644 dev/breeze/src/airflow_breeze/utils/cache.py rename dev/breeze/src/airflow_breeze/{docs_generator/doc_builder.py => utils/console.py} (54%) create mode 100644 dev/breeze/src/airflow_breeze/utils/md5_build_check.py create mode 100644 dev/breeze/src/airflow_breeze/utils/registry.py create mode 100644 dev/breeze/src/airflow_breeze/utils/visuals.py delete mode 100644 dev/breeze/src/airflow_breeze/visuals/__init__.py diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 1d41133732097..81af3053462e8 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -229,7 +229,7 @@ jobs: - name: "Free space" run: airflow-freespace - name: "Build CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" - run: Breeze2 build-ci-image + run: Breeze2 build-image - name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: ./scripts/ci/images/ci_push_ci_images.sh - name: > diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ec3dab163b31..a532faa65c8f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -310,7 +310,7 @@ jobs: run: airflow-freespace if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Build CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" - run: Breeze2 build-ci-image + run: Breeze2 build-image if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: ./scripts/ci/images/ci_push_ci_images.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03f5076161004..6cbaec4d52617 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ default_stages: [commit, push] default_language_version: # force all unspecified python hooks to run python3 python: python3 -minimum_pre_commit_version: "1.20.0" +minimum_pre_commit_version: "2.0.0" repos: - repo: meta hooks: diff --git a/BREEZE.rst b/BREEZE.rst index 4b649db11d51d..00a7286e5093f 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1336,13 +1336,6 @@ This is the current syntax for `./breeze <./breeze>`_: --image-tag TAG Additional tag in the image. - --skip-installing-airflow-providers-from-sources - By default 'pip install' in Airflow 2.0 installs only the provider packages that - are needed by the extras. When you build image during the development (which is - default in Breeze) all providers are installed by default from sources. - You can disable it by adding this flag but then you have to install providers from - wheel packages via --use-packages-from-dist flag. - --disable-pypi-when-building Disable installing Airflow from pypi when building. If you use this flag and want to install Airflow, you have to install it from packages placed in @@ -2031,13 +2024,6 @@ This is the current syntax for `./breeze <./breeze>`_: --image-tag TAG Additional tag in the image. - --skip-installing-airflow-providers-from-sources - By default 'pip install' in Airflow 2.0 installs only the provider packages that - are needed by the extras. When you build image during the development (which is - default in Breeze) all providers are installed by default from sources. - You can disable it by adding this flag but then you have to install providers from - wheel packages via --use-packages-from-dist flag. - --disable-pypi-when-building Disable installing Airflow from pypi when building. If you use this flag and want to install Airflow, you have to install it from packages placed in @@ -2632,13 +2618,6 @@ This is the current syntax for `./breeze <./breeze>`_: --image-tag TAG Additional tag in the image. - --skip-installing-airflow-providers-from-sources - By default 'pip install' in Airflow 2.0 installs only the provider packages that - are needed by the extras. When you build image during the development (which is - default in Breeze) all providers are installed by default from sources. - You can disable it by adding this flag but then you have to install providers from - wheel packages via --use-packages-from-dist flag. - --disable-pypi-when-building Disable installing Airflow from pypi when building. If you use this flag and want to install Airflow, you have to install it from packages placed in diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2cb7e9db28984..f020066f6745b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -644,7 +644,7 @@ and not packaged together with the core, unless you set ``INSTALL_PROVIDERS_FROM variable to ``true``. In Breeze - which is a development environment, ``INSTALL_PROVIDERS_FROM_SOURCES`` variable is set to true, -but you can add ``--skip-installing-airflow-providers-from-sources`` flag to Breeze to skip installing providers when +but you can add ``--install-providers-from-sources=true`` flag to Breeze to skip installing providers when building the images. One watch-out - providers are still always installed (or rather available) if you install airflow from diff --git a/Dockerfile.ci b/Dockerfile.ci index d386f38ff1462..135b5a2a53aed 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -640,10 +640,11 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then echo - echo "Airflow home: ${AIRFLOW_HOME}" - echo "Airflow sources: ${AIRFLOW_SOURCES}" - echo "Airflow core SQL connection: ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}" - + echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}" + echo + echo " * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}" + echo " * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}" + echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}" echo RUN_TESTS=${RUN_TESTS:="false"} @@ -653,7 +654,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then if [[ ${USE_AIRFLOW_VERSION} == "" ]]; then export PYTHONPATH=${AIRFLOW_SOURCES} echo - echo "Using already installed airflow version" + echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}" echo if [[ -d "${AIRFLOW_SOURCES}/airflow/www/" ]]; then pushd "${AIRFLOW_SOURCES}/airflow/www/" >/dev/null @@ -667,38 +668,38 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then mkdir -p "${AIRFLOW_SOURCES}"/tmp/ elif [[ ${USE_AIRFLOW_VERSION} == "none" ]]; then echo - echo "Skip installing airflow - only install wheel/tar.gz packages that are present locally" + echo "${COLOR_BLUE}Skip installing airflow - only install wheel/tar.gz packages that are present locally.${COLOR_RESET}" echo uninstall_airflow_and_providers elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then echo - echo "Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." + echo "${COLOR_BLUE}Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}" echo uninstall_airflow_and_providers install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]" uninstall_providers elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then echo - echo "Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." + echo "${COLOR_BLUE}Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}" echo uninstall_airflow_and_providers install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]" uninstall_providers else echo - echo "Install airflow from PyPI without extras" + echo "${COLOR_BLUE}Install airflow from PyPI without extras" echo install_released_airflow_version "${USE_AIRFLOW_VERSION}" fi if [[ ${USE_PACKAGES_FROM_DIST=} == "true" ]]; then echo - echo "Install all packages from dist folder" + echo "${COLOR_BLUE}Install all packages from dist folder" if [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then echo "(except apache-airflow)" fi if [[ ${PACKAGE_FORMAT} == "both" ]]; then echo - echo "${COLOR_RED}ERROR:You can only specify 'wheel' or 'sdist' as PACKAGE_FORMAT not 'both'${COLOR_RESET}" + echo "${COLOR_RED}ERROR:You can only specify 'wheel' or 'sdist' as PACKAGE_FORMAT not 'both'.${COLOR_RESET}" echo exit 1 fi @@ -782,7 +783,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then cd "${AIRFLOW_SOURCES}" - if [[ ${START_AIRFLOW:="false"} == "true" ]]; then + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} # shellcheck source=scripts/in_container/bin/run_tmux @@ -998,10 +999,11 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then echo - echo "Airflow home: ${AIRFLOW_HOME}" - echo "Airflow sources: ${AIRFLOW_SOURCES}" - echo "Airflow core SQL connection: ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}" - + echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}" + echo + echo " * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}" + echo " * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}" + echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}" echo RUN_TESTS=${RUN_TESTS:="false"} @@ -1011,7 +1013,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then if [[ ${USE_AIRFLOW_VERSION} == "" ]]; then export PYTHONPATH=${AIRFLOW_SOURCES} echo - echo "Using already installed airflow version" + echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}" echo if [[ -d "${AIRFLOW_SOURCES}/airflow/www/" ]]; then pushd "${AIRFLOW_SOURCES}/airflow/www/" >/dev/null @@ -1025,38 +1027,38 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then mkdir -p "${AIRFLOW_SOURCES}"/tmp/ elif [[ ${USE_AIRFLOW_VERSION} == "none" ]]; then echo - echo "Skip installing airflow - only install wheel/tar.gz packages that are present locally" + echo "${COLOR_BLUE}Skip installing airflow - only install wheel/tar.gz packages that are present locally.${COLOR_RESET}" echo uninstall_airflow_and_providers elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then echo - echo "Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." + echo "${COLOR_BLUE}Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}" echo uninstall_airflow_and_providers install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]" uninstall_providers elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then echo - echo "Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." + echo "${COLOR_BLUE}Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}" echo uninstall_airflow_and_providers install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]" uninstall_providers else echo - echo "Install airflow from PyPI without extras" + echo "${COLOR_BLUE}Install airflow from PyPI without extras" echo install_released_airflow_version "${USE_AIRFLOW_VERSION}" fi if [[ ${USE_PACKAGES_FROM_DIST=} == "true" ]]; then echo - echo "Install all packages from dist folder" + echo "${COLOR_BLUE}Install all packages from dist folder" if [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then echo "(except apache-airflow)" fi if [[ ${PACKAGE_FORMAT} == "both" ]]; then echo - echo "${COLOR_RED}ERROR:You can only specify 'wheel' or 'sdist' as PACKAGE_FORMAT not 'both'${COLOR_RESET}" + echo "${COLOR_RED}ERROR:You can only specify 'wheel' or 'sdist' as PACKAGE_FORMAT not 'both'.${COLOR_RESET}" echo exit 1 fi @@ -1140,7 +1142,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then cd "${AIRFLOW_SOURCES}" - if [[ ${START_AIRFLOW:="false"} == "true" ]]; then + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} # shellcheck source=scripts/in_container/bin/run_tmux diff --git a/airflow/www/ask_for_recompile_assets_if_needed.sh b/airflow/www/ask_for_recompile_assets_if_needed.sh index d1a6f34cbded2..a50502e777d7a 100755 --- a/airflow/www/ask_for_recompile_assets_if_needed.sh +++ b/airflow/www/ask_for_recompile_assets_if_needed.sh @@ -30,7 +30,7 @@ NO_COLOR='\033[0m' md5sum=$(find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum) old_md5sum=$(cat "${MD5SUM_FILE}" 2>/dev/null || true) if [[ ${old_md5sum} != "${md5sum}" ]]; then - if [[ ${START_AIRFLOW:="false"} == "true" && ${USE_AIRFLOW_VERSION:=} == "" ]]; then + if [[ ( ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ) && ${USE_AIRFLOW_VERSION:=} == "" ]]; then echo echo -e "${YELLOW}Recompiling assets as they have changed and you need them for 'start_airflow' command${NO_COLOR}" echo diff --git a/breeze b/breeze index d04acd87a7f6d..27ef65cb8d43f 100755 --- a/breeze +++ b/breeze @@ -1047,11 +1047,6 @@ function breeze::parse_arguments() { echo "Extras : ${AIRFLOW_EXTRAS}" shift 2 ;; - --skip-installing-airflow-providers-from-sources) - export INSTALL_PROVIDERS_FROM_SOURCES="false" - echo "Install all Airflow Providers: false" - shift - ;; --additional-extras) export ADDITIONAL_AIRFLOW_EXTRAS="${2}" echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}" @@ -2700,13 +2695,6 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS} --image-tag TAG Additional tag in the image. ---skip-installing-airflow-providers-from-sources - By default 'pip install' in Airflow 2.0 installs only the provider packages that - are needed by the extras. When you build image during the development (which is - default in Breeze) all providers are installed by default from sources. - You can disable it by adding this flag but then you have to install providers from - wheel packages via --use-packages-from-dist flag. - --disable-pypi-when-building Disable installing Airflow from pypi when building. If you use this flag and want to install Airflow, you have to install it from packages placed in diff --git a/breeze-complete b/breeze-complete index b6577aae0606b..ec26c5c9d0053 100644 --- a/breeze-complete +++ b/breeze-complete @@ -194,7 +194,7 @@ postgres-version: mysql-version: mssql-version: version-suffix-for-pypi: version-suffix-for-svn: additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps: image-tag: disable-mysql-client-installation disable-mssql-client-installation constraints-location: disable-pip-cache install-from-docker-context-files -additional-extras: additional-python-deps: disable-pypi-when-building skip-installing-airflow-providers-from-sources +additional-extras: additional-python-deps: disable-pypi-when-building dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env: runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env: load-default-connections load-example-dags diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg index 3408a5523b394..0e5ab1b9cfa2c 100644 --- a/dev/breeze/setup.cfg +++ b/dev/breeze/setup.cfg @@ -54,14 +54,15 @@ package_dir= packages = find: install_requires = click + inputimeout pendulum + psutil pytest pytest-xdist + pyyaml + requests rich rich_click - requests - psutil - inputimeout [options.packages.find] where=src diff --git a/dev/breeze/src/airflow_breeze/branch_defaults.py b/dev/breeze/src/airflow_breeze/branch_defaults.py index 153981c0d22d2..f4903cd077e67 100644 --- a/dev/breeze/src/airflow_breeze/branch_defaults.py +++ b/dev/breeze/src/airflow_breeze/branch_defaults.py @@ -14,6 +14,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +Default configuration for this branch. Those two variables below +should be the only one that should be changed when we branch off +different airflow branch. + +This file is different in every branch (`main`, `vX_Y_test' of airflow) +The _stable branches have the same values as _test branches. + +Examples: + + main: + + AIRFLOW_BRANCH = "main" + DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-main" + + v2-2-test: + + AIRFLOW_BRANCH = "v2-2-test" + DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-2-2" + +""" AIRFLOW_BRANCH = "main" DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-main" diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index e6ea13fcd44ef..b0947475f5078 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -19,77 +19,414 @@ import shutil import subprocess import sys +from dataclasses import dataclass from pathlib import Path -from typing import Optional, Tuple +from typing import List, Optional, Tuple + +from airflow_breeze.shell.shell_params import ShellParams try: + # We handle ImportError so that click autocomplete works import rich_click as click + + click.rich_click.SHOW_METAVARS_COLUMN = False + click.rich_click.APPEND_METAVARS_HELP = True + click.rich_click.STYLE_ERRORS_SUGGESTION = "bright_blue italic" + click.rich_click.ERRORS_SUGGESTION = "\nTry running the '--help' flag for more information.\n" + click.rich_click.ERRORS_EPILOGUE = ( + "\nTo find out more, visit [bright_blue]https://github.com/apache/airflow/blob/main/BREEZE.rst[/]\n" + ) + click.rich_click.OPTION_GROUPS = { + "Breeze2": [ + { + "name": "Basic flags for the default (shell) command", + "options": [ + "--python", + "--backend", + "--use-airflow-version", + "--postgres-version", + "--mysql-version", + "--mssql-version", + "--forward-credentials", + "--db-reset", + ], + }, + { + "name": "Advanced flags for the default (shell) command", + "options": [ + "--force-build", + "--mount-sources", + "--integration", + ], + }, + ], + "Breeze2 shell": [ + { + "name": "Basic flags", + "options": [ + "--python", + "--backend", + "--use-airflow-version", + "--postgres-version", + "--mysql-version", + "--mssql-version", + "--forward-credentials", + "--db-reset", + ], + }, + { + "name": "Advanced flag for running", + "options": [ + "--force-build", + "--mount-sources", + "--integration", + ], + }, + ], + "Breeze2 start-airflow": [ + { + "name": "Basic flags", + "options": [ + "--python", + "--backend", + "--use-airflow-version", + "--postgres-version", + "--mysql-version", + "--mssql-version", + "--load-example-dags", + "--load-default-connections", + "--forward-credentials", + "--db-reset", + ], + }, + { + "name": "Advanced flag for running", + "options": [ + "--force-build", + "--mount-sources", + "--integration", + ], + }, + ], + "Breeze2 build-image": [ + { + "name": "Basic usage", + "options": [ + "--python", + "--upgrade-to-newer-dependencies", + "--debian-version", + "--image-tag", + "--docker-cache", + "--github-repository", + ], + }, + { + "name": "Advanced options (for power users)", + "options": [ + "--install-providers-from-sources", + "--additional-extras", + "--additional-dev-apt-deps", + "--additional-runtime-apt-deps", + "--additional-python-deps", + "--additional-dev-apt-command", + "--runtime-apt-command", + "--additional-dev-apt-env", + "--additional-runtime-apt-env", + "--additional-runtime-apt-command", + "--dev-apt-command", + "--dev-apt-deps", + "--runtime-apt-deps", + ], + }, + { + "name": "Preparing cache (for maintainers)", + "options": [ + "--platform", + "--prepare-buildx-cache", + ], + }, + ], + "Breeze2 build-prod-image": [ + { + "name": "Basic usage", + "options": [ + "--python", + "--install-airflow-version", + "--upgrade-to-newer-dependencies", + "--debian-version", + "--image-tag", + "--docker-cache", + "--github-repository", + ], + }, + { + "name": "Options for customizing images", + "options": [ + "--install-providers-from-sources", + "--extras", + "--additional-extras", + "--additional-dev-apt-deps", + "--additional-runtime-apt-deps", + "--additional-python-deps", + "--additional-dev-apt-command", + "--runtime-apt-command", + "--additional-dev-apt-env", + "--additional-runtime-apt-env", + "--additional-runtime-apt-command", + "--dev-apt-command", + "--dev-apt-deps", + "--runtime-apt-deps", + ], + }, + { + "name": "Customization options (for specific customization needs)", + "options": [ + "--install-from-docker-context-files", + "--cleanup-docker-context-files", + "--disable-mysql-client-installation", + "--disable-mssql-client-installation", + "--disable-postgres-client-installation", + "--disable-airflow-repo-cache", + "--disable-pypi", + "--install-airflow-reference", + "--installation-method", + ], + }, + { + "name": "Preparing cache (for maintainers)", + "options": [ + "--platform", + "--prepare-buildx-cache", + ], + }, + ], + "Breeze2 static-check": [ + { + "name": "Pre-commit flags", + "options": [ + "--type", + "--files", + "--all-files", + "--show-diff-on-failure", + "--last-commit", + ], + }, + ], + "Breeze2 build-docs": [ + { + "name": "Doc flags", + "options": [ + "--docs-only", + "--spellcheck-only", + "--package-filter", + ], + }, + ], + "Breeze2 stop": [ + { + "name": "Stop flags", + "options": [ + "--preserve-volumes", + ], + }, + ], + "Breeze2 setup-autocomplete": [ + { + "name": "Setup autocomplete flags", + "options": [ + "--force-setup", + ], + }, + ], + "Breeze2 config": [ + { + "name": "Config flags", + "options": [ + "--python", + "--backend", + "--cheatsheet", + "--asciiart", + ], + }, + ], + } + + click.rich_click.COMMAND_GROUPS = { + "Breeze2": [ + { + "name": "Developer tools", + "commands": [ + "shell", + "start-airflow", + "stop", + "build-image", + "build-prod-image", + "build-docs", + "static-check", + ], + }, + { + "name": "Configuration & maintenance", + "commands": ["cleanup", "setup-autocomplete", "config", "version"], + }, + ] + } + + except ImportError: - # We handle import errors so that click autocomplete works import click # type: ignore[no-redef] -from click import ClickException +from click import Context -from airflow_breeze.cache import delete_cache, touch_cache_file, write_to_cache_file -from airflow_breeze.ci.build_image import build_image -from airflow_breeze.ci.build_params import BuildParams -from airflow_breeze.console import console -from airflow_breeze.docs_generator import build_documentation -from airflow_breeze.docs_generator.doc_builder import DocBuilder +from airflow_breeze.build_image.ci.build_ci_image import build_image +from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams +from airflow_breeze.build_image.prod.build_prod_image import build_production_image from airflow_breeze.global_constants import ( ALLOWED_BACKENDS, + ALLOWED_BUILD_CACHE, ALLOWED_DEBIAN_VERSIONS, ALLOWED_EXECUTORS, - ALLOWED_INSTALL_AIRFLOW_VERSIONS, + ALLOWED_INSTALLATION_METHODS, ALLOWED_INTEGRATIONS, + ALLOWED_MOUNT_OPTIONS, ALLOWED_MSSQL_VERSIONS, ALLOWED_MYSQL_VERSIONS, + ALLOWED_PLATFORMS, ALLOWED_POSTGRES_VERSIONS, ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, - ALLOWED_USE_AIRFLOW_VERSIONS, + MOUNT_SELECTED, get_available_packages, ) from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST -from airflow_breeze.prod.build_prod_image import build_production_image -from airflow_breeze.shell.enter_shell import build_shell -from airflow_breeze.utils.docker_command_utils import check_docker_resources +from airflow_breeze.shell.enter_shell import enter_shell +from airflow_breeze.utils.cache import delete_cache, touch_cache_file, write_to_cache_file +from airflow_breeze.utils.console import console +from airflow_breeze.utils.docker_command_utils import ( + check_docker_resources, + construct_env_variables_docker_compose_command, + get_extra_docker_flags, +) from airflow_breeze.utils.path_utils import ( - __AIRFLOW_SOURCES_ROOT, + AIRFLOW_SOURCES_ROOT, + BUILD_CACHE_DIR, create_directories, find_airflow_sources_root, - get_airflow_sources_root, ) -from airflow_breeze.utils.run_utils import check_package_installed, run_command -from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE - -AIRFLOW_SOURCES_DIR = Path(__file__).resolve().parent.parent.parent.parent.parent.absolute() +from airflow_breeze.utils.run_utils import check_pre_commit_installed, run_command +from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE NAME = "Breeze2" VERSION = "0.0.1" - -@click.group() -def main(): - find_airflow_sources_root() - +find_airflow_sources_root() option_verbose = click.option( - "-v", "--verbose", is_flag=True, help="Print verbose information about performed steps", envvar='VERBOSE' + "-v", "--verbose", is_flag=True, help="Print verbose information about performed steps.", envvar='VERBOSE' ) -option_python_version = click.option( +option_dry_run = click.option( + "-D", + "--dry-run", + is_flag=True, + help="If dry-run is set, commands are only printed, not executed.", + envvar='DRY_RUN', +) + + +option_python = click.option( '-p', '--python', type=click.Choice(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS), - help='Choose your python version', + help='Python version to use.', envvar='PYTHON_MAJOR_MINOR_VERSION', ) option_backend = click.option( '-b', '--backend', + help="Database backend to use.", type=click.Choice(ALLOWED_BACKENDS), - help='Choose your backend database', +) + +option_integration = click.option( + '--integration', + help="Integration(s) to enable when running (can be more than one).", + type=click.Choice(ALLOWED_INTEGRATIONS), + multiple=True, +) + +option_postgres_version = click.option( + '-P', '--postgres-version', help="Version of Postgres.", type=click.Choice(ALLOWED_POSTGRES_VERSIONS) +) + +option_mysql_version = click.option( + '-M', '--mysql-version', help="Version of MySQL.", type=click.Choice(ALLOWED_MYSQL_VERSIONS) +) + +option_mssql_version = click.option( + '-S', '--mssql-version', help="Version of MsSQL.", type=click.Choice(ALLOWED_MSSQL_VERSIONS) +) + +option_executor = click.option( + '--executor', + help='Executor to use in a kubernetes cluster. Default is KubernetesExecutor.', + type=click.Choice(ALLOWED_EXECUTORS), +) + +option_forward_credentials = click.option( + '-f', '--forward-credentials', help="Forward local credentials to container when running.", is_flag=True +) + +option_use_airflow_version = click.option( + '-a', + '--use-airflow-version', + help="Use (reinstall) specified Airflow version after entering the container.", + envvar='USE_AIRFLOW_VERSION', +) + +option_mount_sources = click.option( + '--mount-sources', + type=click.Choice(ALLOWED_MOUNT_OPTIONS), + default=ALLOWED_MOUNT_OPTIONS[0], + help="Choose which local sources should be mounted (default = selected)", +) + +option_force_build = click.option('--force-build', help="Force image build before running.", is_flag=True) + +option_db_reset = click.option( + '-d', + '--db-reset', + help="Resets DB when entering the container.", + is_flag=True, + envvar='DB_RESET', +) + + +@click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']}) +@option_verbose +@option_dry_run +@option_python +@option_backend +@option_postgres_version +@option_mysql_version +@option_mssql_version +@option_forward_credentials +@option_force_build +@option_use_airflow_version +@option_mount_sources +@option_integration +@option_db_reset +@click.pass_context +def main(ctx: Context, **kwargs): + if not ctx.invoked_subcommand: + ctx.forward(shell, extra_args={}) + + +option_docker_cache = click.option( + '-c', + '--docker-cache', + help='Cache option for image used during the build.', + type=click.Choice(ALLOWED_BUILD_CACHE), ) option_github_repository = click.option( @@ -106,10 +443,15 @@ def main(): Breeze can automatically pull the commit SHA id specified Default: latest', ) -option_image_tag = click.option('--image-tag', help='Additional tag in the image.') +option_image_tag = click.option( + '-t', '--image-tag', help='Set tag for the image (additionally to default Airflow convention).' +) option_platform = click.option( - '--platform', help='Builds image for the platform specified.', envvar='PLATFORM' + '--platform', + help='Builds image for the platform specified.', + envvar='PLATFORM', + type=click.Choice(ALLOWED_PLATFORMS), ) option_debian_version = click.option( @@ -120,8 +462,9 @@ def main(): envvar='DEBIAN_VERSION', ) option_upgrade_to_newer_dependencies = click.option( + "-u", '--upgrade-to-newer-dependencies', - help='Upgrades PIP packages to latest versions available without looking at the constraints.', + help='If set to anything else than false, upgrades PIP packages to latest versions available.', envvar='UPGRADE_TO_NEWER_DEPENDENCIES', ) option_additional_extras = click.option( @@ -184,11 +527,43 @@ def main(): help='The basic apt runtime dependencies to use when building the images.', envvar='RUNTIME_APT_DEPS', ) -option_ci_flag = click.option( - '--ci', - help='Enabling this option will off the pip progress bar', + +option_skip_rebuild_check = click.option( + '-r', + '--skip-rebuild-check', + help="Skips checking if rebuild is needed", + is_flag=True, + envvar='SKIP_REBUILD_CHECK', +) + +option_prepare_buildx_cache = click.option( + '--prepare-buildx-cache', + help='Prepares build cache rather than build images locally.', + is_flag=True, + envvar='PREPARE_BUILDX_CACHE', +) + +option_install_providers_from_sources = click.option( + '--install-providers-from-sources', + help="Install providers from sources when installing.", + is_flag=True, + envvar='INSTALL_PROVIDERS_FROM_SOURCES', +) + +option_load_example_dags = click.option( + '-e', + '--load-example-dags', + help="Enable configuration to load example DAGs when starting Airflow.", + is_flag=True, + envvar='LOAD_EXAMPLES', +) + +option_load_default_connection = click.option( + '-c', + '--load-default-connections', + help="Enable configuration to load default connections when starting Airflow.", is_flag=True, - envvar='CI', + envvar='LOAD_DEFAULT_CONNECTIONS', ) @@ -199,82 +574,136 @@ def version(): console.print(f"\n[green]{NAME} version: {VERSION}[/]\n") +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# Make sure that whatever you add here as an option is also +# Added in the "main" command above. The min command above +# Is used for a shorthand of shell and except the extra +# Args it should have the same parameters. +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +@main.command() @option_verbose -@main.command( - context_settings=dict( - ignore_unknown_options=True, - allow_extra_args=True, - ), -) -@option_python_version +@option_dry_run +@option_python @option_backend -@click.option('--integration', type=click.Choice(ALLOWED_INTEGRATIONS), multiple=True) -@click.option('-L', '--build-cache-local', is_flag=True) -@click.option('-U', '--build-cache-pulled', is_flag=True) -@click.option('-X', '--build-cache-disabled', is_flag=True) -@click.option('--postgres-version', type=click.Choice(ALLOWED_POSTGRES_VERSIONS)) -@click.option('--mysql-version', type=click.Choice(ALLOWED_MYSQL_VERSIONS)) -@click.option('--mssql-version', type=click.Choice(ALLOWED_MSSQL_VERSIONS)) -@click.option( - '--executor', - type=click.Choice(ALLOWED_EXECUTORS), - help='Executor to use in a kubernetes cluster. Default is KubernetesExecutor', -) -@click.option('-f', '--forward-credentials', is_flag=True) -@click.option('-l', '--skip-mounting-local-sources', is_flag=True) -@click.option('--use-airflow-version', type=click.Choice(ALLOWED_INSTALL_AIRFLOW_VERSIONS)) -@click.option('--use-packages-from-dist', is_flag=True) -@click.option('--force-build', is_flag=True) +@option_postgres_version +@option_mysql_version +@option_mssql_version +@option_forward_credentials +@option_force_build +@option_use_airflow_version +@option_mount_sources +@option_integration +@option_db_reset @click.argument('extra-args', nargs=-1, type=click.UNPROCESSED) def shell( verbose: bool, + dry_run: bool, python: str, backend: str, integration: Tuple[str], - build_cache_local: bool, - build_cache_pulled: bool, - build_cache_disabled: bool, postgres_version: str, mysql_version: str, mssql_version: str, - executor: str, forward_credentials: bool, - skip_mounting_local_sources: bool, + mount_sources: str, use_airflow_version: str, - use_packages_from_dist: bool, force_build: bool, + db_reset: bool, extra_args: Tuple, ): """Enters breeze.py environment. this is the default command use when no other is selected.""" if verbose: console.print("\n[green]Welcome to breeze.py[/]\n") - console.print(f"\n[green]Root of Airflow Sources = {__AIRFLOW_SOURCES_ROOT}[/]\n") - build_shell( - verbose, - python_version=python, + console.print(f"\n[green]Root of Airflow Sources = {AIRFLOW_SOURCES_ROOT}[/]\n") + enter_shell( + verbose=verbose, + dry_run=dry_run, + python=python, + backend=backend, + integration=integration, + postgres_version=postgres_version, + mysql_version=mysql_version, + mssql_version=mssql_version, + forward_credentials=str(forward_credentials), + mount_sources=mount_sources, + use_airflow_version=use_airflow_version, + force_build=force_build, + db_reset=db_reset, + extra_args=extra_args, + ) + + +@option_verbose +@main.command(name='start-airflow') +@option_dry_run +@option_python +@option_backend +@option_postgres_version +@option_load_example_dags +@option_load_default_connection +@option_mysql_version +@option_mssql_version +@option_forward_credentials +@option_force_build +@option_use_airflow_version +@option_mount_sources +@option_integration +@option_db_reset +@click.argument('extra-args', nargs=-1, type=click.UNPROCESSED) +def start_airflow( + verbose: bool, + dry_run: bool, + python: str, + backend: str, + integration: Tuple[str], + postgres_version: str, + load_example_dags: bool, + load_default_connections: bool, + mysql_version: str, + mssql_version: str, + forward_credentials: bool, + mount_sources: str, + use_airflow_version: str, + force_build: bool, + db_reset: bool, + extra_args: Tuple, +): + """Enters breeze.py environment and starts all Airflow components in the tmux session.""" + enter_shell( + verbose=verbose, + dry_run=dry_run, + python=python, backend=backend, integration=integration, - build_cache_local=build_cache_local, - build_cache_disabled=build_cache_disabled, - build_cache_pulled=build_cache_pulled, postgres_version=postgres_version, + load_default_connections=load_default_connections, + load_example_dags=load_example_dags, mysql_version=mysql_version, mssql_version=mssql_version, - executor=executor, forward_credentials=str(forward_credentials), - skip_mounting_local_sources=skip_mounting_local_sources, + mount_sources=mount_sources, use_airflow_version=use_airflow_version, - use_packages_from_dist=use_packages_from_dist, force_build=force_build, + db_reset=db_reset, + start_airflow=True, extra_args=extra_args, ) +@main.command(name='build-image') @option_verbose -@main.command(name='build-ci-image') +@option_dry_run +@option_python +@option_upgrade_to_newer_dependencies +@option_platform +@option_debian_version +@option_github_repository +@option_docker_cache +@option_image_tag +@option_prepare_buildx_cache +@option_install_providers_from_sources @option_additional_extras -@option_python_version @option_additional_dev_apt_deps @option_additional_runtime_apt_deps @option_additional_python_deps @@ -287,17 +716,12 @@ def shell( @option_dev_apt_deps @option_runtime_apt_command @option_runtime_apt_deps -@option_github_repository -@click.option('--build-cache', help='Cache option') -@option_platform -@option_debian_version -@click.option('--prepare-buildx-cache', is_flag=True) -@option_ci_flag -@option_upgrade_to_newer_dependencies def build_ci_image( verbose: bool, + dry_run: bool, additional_extras: Optional[str], python: str, + image_tag: Optional[str], additional_dev_apt_deps: Optional[str], additional_runtime_apt_deps: Optional[str], additional_python_deps: Optional[str], @@ -307,28 +731,29 @@ def build_ci_image( additional_runtime_apt_env: Optional[str], dev_apt_command: Optional[str], dev_apt_deps: Optional[str], + install_providers_from_sources: bool, runtime_apt_command: Optional[str], runtime_apt_deps: Optional[str], github_repository: Optional[str], - build_cache: Optional[str], + docker_cache: Optional[str], platform: Optional[str], debian_version: Optional[str], prepare_buildx_cache: bool, - ci: bool, upgrade_to_newer_dependencies: str = "false", ): - """Builds docker CI image without entering the container.""" + """Builds docker CI image.""" if verbose: console.print( - f"\n[blue]Building image of airflow from {__AIRFLOW_SOURCES_ROOT} " + f"\n[bright_blue]Building image of airflow from {AIRFLOW_SOURCES_ROOT} " f"python version: {python}[/]\n" ) - create_directories() build_image( - verbose, + verbose=verbose, + dry_run=dry_run, additional_extras=additional_extras, - python_version=python, + python=python, + image_tag=image_tag, additional_dev_apt_deps=additional_dev_apt_deps, additional_runtime_apt_deps=additional_runtime_apt_deps, additional_python_deps=additional_python_deps, @@ -336,42 +761,65 @@ def build_ci_image( additional_dev_apt_command=additional_dev_apt_command, additional_dev_apt_env=additional_dev_apt_env, additional_runtime_apt_env=additional_runtime_apt_env, + install_providers_from_sources=install_providers_from_sources, dev_apt_command=dev_apt_command, dev_apt_deps=dev_apt_deps, runtime_apt_command=runtime_apt_command, runtime_apt_deps=runtime_apt_deps, github_repository=github_repository, - docker_cache=build_cache, + docker_cache=docker_cache, platform=platform, debian_version=debian_version, prepare_buildx_cache=prepare_buildx_cache, - ci=ci, upgrade_to_newer_dependencies=upgrade_to_newer_dependencies, ) @option_verbose +@option_dry_run @main.command(name='build-prod-image') +@option_python +@option_upgrade_to_newer_dependencies +@option_platform +@option_debian_version +@option_github_repository +@option_docker_cache +@option_image_tag +@option_prepare_buildx_cache +@click.option( + '--installation-method', + help="Whether to install airflow from sources ('.') or PyPI ('apache-airflow')", + type=click.Choice(ALLOWED_INSTALLATION_METHODS), +) +@option_install_providers_from_sources +@click.option( + '--install-from-docker-context-files', + help='Install wheels from local docker-context-files when building image', + is_flag=True, +) @click.option( - '--cleanup-docker-context-files', help='Preserves data volumes when stopping airflow.', is_flag=True -) -@click.option('--disable-mysql-client-installation', is_flag=True) -@click.option('--disable-mssql-client-installation', is_flag=True) -@click.option('--disable-postgres-client-installation', is_flag=True) -@click.option('--disable-pip-cache', is_flag=True) -@click.option('-t', '--install-airflow-reference') -@click.option('-a', '--install-airflow-version', type=click.Choice(ALLOWED_INSTALL_AIRFLOW_VERSIONS)) -@click.option('-r', '--skip-rebuild-check', is_flag=True) -@click.option('-L', '--build-cache-local', is_flag=True) -@click.option('-U', '--build-cache-pulled', is_flag=True) -@click.option('-X', '--build-cache-disabled', is_flag=True) + '--cleanup-docker-context-files', + help='Cleans up docker context files before running build.', + is_flag=True, +) +@click.option('--extras', help="Extras to install by default") +@click.option('--disable-mysql-client-installation', help="Do not install MySQL client", is_flag=True) +@click.option('--disable-mssql-client-installation', help="Do not install MsSQl client", is_flag=True) +@click.option('--disable-postgres-client-installation', help="Do not install Postgres client", is_flag=True) +@click.option( + '--disable-airflow-repo-cache', help="Disable cache from Airflow repository during building", is_flag=True +) +@click.option('--disable-pypi', help="Disable pypi during building", is_flag=True) +@click.option( + '--install-airflow-reference', + help="Install airflow using specified reference (tag/branch) from GitHub", +) +@click.option('-a', '--install-airflow-version', help="Install specified version of airflow") @option_additional_extras -@option_python_version @option_additional_dev_apt_deps @option_additional_runtime_apt_deps @option_additional_python_deps @option_additional_dev_apt_command -@option_runtime_apt_command @option_additional_dev_apt_env @option_additional_runtime_apt_env @option_additional_runtime_apt_command @@ -379,38 +827,21 @@ def build_ci_image( @option_dev_apt_deps @option_runtime_apt_command @option_runtime_apt_deps -@option_github_repository -@option_platform -@option_debian_version -@option_upgrade_to_newer_dependencies -@click.option('--prepare-buildx-cache', is_flag=True) -@click.option('--skip-installing-airflow-providers-from-sources', is_flag=True) -@click.option('--disable-pypi-when-building', is_flag=True) -@click.option('-E', '--extras') -@click.option('--installation-method', type=click.Choice(ALLOWED_USE_AIRFLOW_VERSIONS)) -@click.option( - '--install-from-docker-context-files', - help='Install wheels from local docker-context-files when building image', - is_flag=True, -) -@option_image_tag -@click.option('--github-token', envvar='GITHUB_TOKEN') -@option_ci_flag def build_prod_image( verbose: bool, + dry_run: bool, cleanup_docker_context_files: bool, disable_mysql_client_installation: bool, disable_mssql_client_installation: bool, disable_postgres_client_installation: bool, - disable_pip_cache: bool, + disable_airflow_repo_cache: bool, + disable_pypi: bool, install_airflow_reference: Optional[str], install_airflow_version: Optional[str], - skip_rebuild_check: bool, - build_cache_local: bool, - build_cache_pulled: bool, - build_cache_disabled: bool, + docker_cache: str, additional_extras: Optional[str], python: str, + image_tag: Optional[str], additional_dev_apt_deps: Optional[str], additional_runtime_apt_deps: Optional[str], additional_python_deps: Optional[str], @@ -426,37 +857,32 @@ def build_prod_image( platform: Optional[str], debian_version: Optional[str], prepare_buildx_cache: bool, - skip_installing_airflow_providers_from_sources: bool, - disable_pypi_when_building: bool, + install_providers_from_sources: bool, extras: Optional[str], installation_method: Optional[str], install_from_docker_context_files: bool, - image_tag: Optional[str], - github_token: Optional[str], - ci: bool, upgrade_to_newer_dependencies: str = "false", ): - """Builds docker Production image without entering the container.""" + """Builds docker Production image.""" if verbose: - console.print("\n[blue]Building image[/]\n") + console.print("\n[bright_blue]Building image[/]\n") if prepare_buildx_cache: - build_cache_pulled = True + docker_cache = "pulled" cleanup_docker_context_files = True build_production_image( verbose, + dry_run, cleanup_docker_context_files=cleanup_docker_context_files, disable_mysql_client_installation=disable_mysql_client_installation, disable_mssql_client_installation=disable_mssql_client_installation, disable_postgres_client_installation=disable_postgres_client_installation, - disable_pip_cache=disable_pip_cache, + disable_airflow_repo_cache=disable_airflow_repo_cache, + disable_pypi=disable_pypi, install_airflow_reference=install_airflow_reference, install_airflow_version=install_airflow_version, - skip_rebuild_check=skip_rebuild_check, - build_cache_local=build_cache_local, - build_cache_pulled=build_cache_pulled, - build_cache_disabled=build_cache_disabled, + docker_cache=docker_cache, additional_extras=additional_extras, - python_version=python, + python=python, additional_dev_apt_deps=additional_dev_apt_deps, additional_runtime_apt_deps=additional_runtime_apt_deps, additional_python_deps=additional_python_deps, @@ -473,27 +899,14 @@ def build_prod_image( debian_version=debian_version, upgrade_to_newer_dependencies=upgrade_to_newer_dependencies, prepare_buildx_cache=prepare_buildx_cache, - skip_installing_airflow_providers_from_sources=skip_installing_airflow_providers_from_sources, - disable_pypi_when_building=disable_pypi_when_building, + install_providers_from_sources=install_providers_from_sources, extras=extras, installation_method=installation_method, install_docker_context_files=install_from_docker_context_files, image_tag=image_tag, - github_token=github_token, - ci=ci, ) -@option_verbose -@main.command(name='start-airflow') -def start_airflow(verbose: bool): - """Enters breeze.py environment and set up the tmux session""" - if verbose: - console.print("\n[green]Welcome to breeze.py[/]\n") - console.print(ASCIIART, style=ASCIIART_STYLE) - raise ClickException("\nPlease implement entering breeze.py\n") - - BREEZE_COMMENT = "Added by Updated Airflow Breeze autocomplete setup" START_LINE = f"# START: {BREEZE_COMMENT}\n" END_LINE = f"# END: {BREEZE_COMMENT}\n" @@ -519,7 +932,7 @@ def backup(script_path_file: Path): shutil.copy(str(script_path_file), str(script_path_file) + ".bak") -def write_to_shell(command_to_execute: str, script_path: str, force_setup: bool) -> bool: +def write_to_shell(command_to_execute: str, dry_run: bool, script_path: str, force_setup: bool) -> bool: skip_check = False script_path_file = Path(script_path) if not script_path_file.exists(): @@ -528,7 +941,7 @@ def write_to_shell(command_to_execute: str, script_path: str, force_setup: bool) if BREEZE_COMMENT in script_path_file.read_text(): if not force_setup: console.print( - "\n[yellow]Autocompletion is already setup. Skipping. " + "\n[bright_yellow]Autocompletion is already setup. Skipping. " "You can force autocomplete installation by adding --force-setup[/]\n" ) return False @@ -537,26 +950,36 @@ def write_to_shell(command_to_execute: str, script_path: str, force_setup: bool) remove_autogenerated_code(script_path) console.print(f"\nModifying the {script_path} file!\n") console.print(f"\nCopy of the file is held in {script_path}.bak !\n") - backup(script_path_file) + if not dry_run: + backup(script_path_file) text = script_path_file.read_text() - script_path_file.write_text( - text + ("\n" if not text.endswith("\n") else "") + START_LINE + command_to_execute + "\n" + END_LINE - ) + if not dry_run: + script_path_file.write_text( + text + + ("\n" if not text.endswith("\n") else "") + + START_LINE + + command_to_execute + + "\n" + + END_LINE + ) + else: + console.print(f"[bright_blue]The autocomplete script would be added to {script_path}[/]") console.print( - "\n[yellow]Please exit and re-enter your shell or run:[/]\n\n" f" `source {script_path}`\n" + f"\n[bright_yellow]Please exit and re-enter your shell or run:[/]\n\n `source {script_path}`\n" ) return True @option_verbose +@option_dry_run @click.option( '-f', '--force-setup', is_flag=True, - help='Force autocomplete setup even if already setup before (overrides the setup).', + help='Force autocomplete setup even' 'if already setup before (overrides the setup).', ) @main.command(name='setup-autocomplete') -def setup_autocomplete(verbose: bool, force_setup: bool): +def setup_autocomplete(verbose: bool, dry_run: bool, force_setup: bool): """ Enables autocompletion of Breeze2 commands. """ @@ -569,29 +992,28 @@ def setup_autocomplete(verbose: bool, force_setup: bool): sys.exit(1) console.print(f"Installing {detected_shell} completion for local user") autocomplete_path = ( - Path(AIRFLOW_SOURCES_DIR) / "dev" / "breeze" / "autocomplete" / f"{NAME}-complete-{detected_shell}.sh" + AIRFLOW_SOURCES_ROOT / "dev" / "breeze" / "autocomplete" / f"{NAME}-complete-{detected_shell}.sh" ) console.print(f"[bright_blue]Activation command script is available here: {autocomplete_path}[/]\n") console.print( - f"[yellow]We need to add above script to your {detected_shell} profile and " + f"[bright_yellow]We need to add above script to your {detected_shell} profile and " "install 'click' package in your default python installation destination.[/]\n" ) - updated = False if click.confirm("Should we proceed ?"): if detected_shell == 'bash': script_path = str(Path('~').expanduser() / '.bash_completion') command_to_execute = f"source {autocomplete_path}" - updated = write_to_shell(command_to_execute, script_path, force_setup) + updated = write_to_shell(command_to_execute, dry_run, script_path, force_setup) elif detected_shell == 'zsh': script_path = str(Path('~').expanduser() / '.zshrc') command_to_execute = f"source {autocomplete_path}" - updated = write_to_shell(command_to_execute, script_path, force_setup) + updated = write_to_shell(command_to_execute, dry_run, script_path, force_setup) elif detected_shell == 'fish': # Include steps for fish shell script_path = str(Path('~').expanduser() / f'.config/fish/completions/{NAME}.fish') if os.path.exists(script_path) and not force_setup: console.print( - "\n[yellow]Autocompletion is already setup. Skipping. " + "\n[bright_yellow]Autocompletion is already setup. Skipping. " "You can force autocomplete installation by adding --force-setup[/]\n" ) else: @@ -606,9 +1028,9 @@ def setup_autocomplete(verbose: bool, force_setup: bool): subprocess.check_output(['powershell', '-NoProfile', 'echo $profile']).decode("utf-8").strip() ) command_to_execute = f". {autocomplete_path}" - write_to_shell(command_to_execute, script_path, force_setup) + write_to_shell(command_to_execute, dry_run, script_path, force_setup) if updated: - run_command(['pip', 'install', '--upgrade', 'click'], verbose=True, check=False) + run_command(['pip', 'install', '--upgrade', 'click'], verbose=True, dry_run=dry_run, check=False) else: console.print( "\nPlease follow the https://click.palletsprojects.com/en/8.1.x/shell-completion/ " @@ -617,23 +1039,23 @@ def setup_autocomplete(verbose: bool, force_setup: bool): @main.command(name='config') -@option_python_version +@option_python @option_backend -@click.option('--cheatsheet/--no-cheatsheet', default=None) -@click.option('--asciiart/--no-asciiart', default=None) +@click.option('-C/-c', '--cheatsheet/--no-cheatsheet', help="Enable/disable cheatsheet", default=None) +@click.option('-A/-a', '--asciiart/--no-asciiart', help="Enable/disable ASCIIart", default=None) def change_config(python, backend, cheatsheet, asciiart): """ - Toggles on/off cheatsheet, asciiart + Toggles on/off cheatsheet, asciiart. Sets default Python and backend. """ if asciiart: - console.print('[blue] ASCIIART enabled') + console.print('[bright_blue] ASCIIART enabled') delete_cache('suppress_asciiart') elif asciiart is not None: touch_cache_file('suppress_asciiart') else: pass if cheatsheet: - console.print('[blue] Cheatsheet enabled') + console.print('[bright_blue] Cheatsheet enabled') delete_cache('suppress_cheatsheet') elif cheatsheet is not None: touch_cache_file('suppress_cheatsheet') @@ -641,47 +1063,97 @@ def change_config(python, backend, cheatsheet, asciiart): pass if python is not None: write_to_cache_file('PYTHON_MAJOR_MINOR_VERSION', python) - console.print(f'[blue]Python cached_value {python}') + console.print(f'[bright_blue]Python cached_value {python}') if backend is not None: write_to_cache_file('BACKEND', backend) - console.print(f'[blue]Backend cached_value {backend}') + console.print(f'[bright_blue]Backend cached_value {backend}') + + +@dataclass +class DocParams: + package_filter: Tuple[str] + docs_only: bool + spellcheck_only: bool + + @property + def args_doc_builder(self) -> List[str]: + doc_args = [] + if self.docs_only: + doc_args.append("--docs-only") + if self.spellcheck_only: + doc_args.append("--spellcheck-only") + if self.package_filter and len(self.package_filter) > 0: + for single_filter in self.package_filter: + doc_args.extend(["--package-filter", single_filter]) + return doc_args -@option_verbose @main.command(name='build-docs') -@click.option('--docs-only', is_flag=True) -@click.option('--spellcheck-only', is_flag=True) -@click.option('--package-filter', type=click.Choice(get_available_packages()), multiple=True) -def build_docs(verbose: bool, docs_only: bool, spellcheck_only: bool, package_filter: Tuple[str]): +@option_verbose +@option_dry_run +@click.option('-d', '--docs-only', help="Only build documentation", is_flag=True) +@click.option('-s', '--spellcheck-only', help="Only run spell checking", is_flag=True) +@click.option( + '-p', + '--package-filter', + help="List of packages to consider", + type=click.Choice(get_available_packages()), + multiple=True, +) +def build_docs( + verbose: bool, dry_run: bool, docs_only: bool, spellcheck_only: bool, package_filter: Tuple[str] +): """ - Builds documentation in the container + Builds documentation in the container. + + * figures out CI image name + * checks if there are enough resources + * converts parameters into a DocParams class """ - params = BuildParams() - airflow_sources = str(get_airflow_sources_root()) - ci_image_name = params.airflow_ci_image_name - check_docker_resources(verbose, airflow_sources, ci_image_name) - doc_builder = DocBuilder( - package_filter=package_filter, docs_only=docs_only, spellcheck_only=spellcheck_only + params = BuildCiParams() + ci_image_name = params.airflow_image_name + check_docker_resources(verbose, ci_image_name) + doc_builder = DocParams( + package_filter=package_filter, + docs_only=docs_only, + spellcheck_only=spellcheck_only, ) - build_documentation.build(verbose, airflow_sources, ci_image_name, doc_builder) + extra_docker_flags = get_extra_docker_flags(MOUNT_SELECTED) + cmd = [] + cmd.extend(["docker", "run"]) + cmd.extend(extra_docker_flags) + cmd.extend(["-t", "-e", "GITHUB_ACTIONS="]) + cmd.extend(["--entrypoint", "/usr/local/bin/dumb-init", "--pull", "never"]) + cmd.extend([ci_image_name, "--", "/opt/airflow/scripts/in_container/run_docs_build.sh"]) + cmd.extend(doc_builder.args_doc_builder) + run_command(cmd, verbose=verbose, dry_run=dry_run, text=True) -@option_verbose @main.command( name="static-check", + help="Run static checks.", context_settings=dict( ignore_unknown_options=True, allow_extra_args=True, ), ) -@click.option('--all-files', is_flag=True) -@click.option('--show-diff-on-failure', is_flag=True) -@click.option('--last-commit', is_flag=True) -@click.option('-t', '--type', type=click.Choice(PRE_COMMIT_LIST), multiple=True) -@click.option('--files', is_flag=True) +@click.option( + '-t', + '--type', + help="Type(s) of the static checks to run", + type=click.Choice(PRE_COMMIT_LIST), + multiple=True, +) +@click.option('-a', '--all-files', help="Run checks on all files", is_flag=True) +@click.option('-f', '--files', help="List of files to run the checks on", multiple=True) +@click.option('-s', '--show-diff-on-failure', help="Show diff for files modified by the checks", is_flag=True) +@click.option('-c', '--last-commit', help="Run check for all files in all commits", is_flag=True) +@option_verbose +@option_dry_run @click.argument('precommit_args', nargs=-1, type=click.UNPROCESSED) def static_check( verbose: bool, + dry_run: bool, all_files: bool, show_diff_on_failure: bool, last_commit: bool, @@ -689,7 +1161,7 @@ def static_check( files: bool, precommit_args: Tuple, ): - if check_package_installed('pre_commit'): + if check_pre_commit_installed(verbose=verbose): command_to_execute = ['pre-commit', 'run'] for single_check in type: command_to_execute.append(single_check) @@ -701,9 +1173,76 @@ def static_check( command_to_execute.extend(["--from-ref", "HEAD^", "--to-ref", "HEAD"]) if files: command_to_execute.append("--files") + if verbose: + command_to_execute.append("--verbose") if precommit_args: command_to_execute.extend(precommit_args) - run_command(command_to_execute, suppress_raise_exception=True, suppress_console_print=True, text=True) + run_command( + command_to_execute, + verbose=verbose, + dry_run=dry_run, + check=False, + no_output_dump_on_exception=True, + text=True, + ) + + +@main.command(name="stop", help="Stops running breeze environment.") +@option_verbose +@option_dry_run +@click.option( + "-p", + "--preserve-volumes", + help="By default the stop command removes volumes with data. " "Specifying the flag will preserve them.", + is_flag=True, +) +def stop(verbose: bool, dry_run: bool, preserve_volumes: bool): + command_to_execute = ['docker-compose', 'down', "--remove-orphans"] + if not preserve_volumes: + command_to_execute.append("--volumes") + shell_params = ShellParams({}) + env_variables = construct_env_variables_docker_compose_command(shell_params) + run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables) + + +@main.command(name="cleanup", help="Removes the cache of parameters, images and cleans up docker cache.") +@option_verbose +@option_dry_run +def cleanup(verbose: bool, dry_run: bool): + console.print("\n[bright_yellow]Removing cache of parameters, images, and cleans up docker cache[/]") + if click.confirm("Are you sure?"): + docker_images_command_to_execute = [ + 'docker', + 'images', + '--filter', + 'label=org.apache.airflow.image', + '--format', + '{{.Repository}}:{{.Tag}}', + ] + process = run_command( + docker_images_command_to_execute, verbose=verbose, text=True, capture_output=True + ) + images = process.stdout.splitlines() if process and process.stdout else [] + if images: + console.print("[light_blue]Removing images:[/]") + for image in images: + console.print(f"[light_blue] * {image}[/]") + console.print() + docker_rmi_command_to_execute = [ + 'docker', + 'rmi', + '--force', + ] + docker_rmi_command_to_execute.extend(images) + run_command(docker_rmi_command_to_execute, verbose=verbose, dry_run=dry_run, check=False) + else: + console.print("[light_blue]No images to remote[/]\n") + system_prune_command_to_execute = ['docker', 'system', 'prune'] + console.print("Pruning docker images") + run_command(system_prune_command_to_execute, verbose=verbose, dry_run=dry_run, check=False) + console.print(f"Removing build cache dir ${BUILD_CACHE_DIR}") + if not dry_run: + shutil.rmtree(BUILD_CACHE_DIR, ignore_errors=True) if __name__ == '__main__': diff --git a/dev/breeze/src/airflow_breeze/prod/__init__.py b/dev/breeze/src/airflow_breeze/build_image/__init__.py similarity index 96% rename from dev/breeze/src/airflow_breeze/prod/__init__.py rename to dev/breeze/src/airflow_breeze/build_image/__init__.py index 13a83393a9124..7b70ea1ee8c1c 100644 --- a/dev/breeze/src/airflow_breeze/prod/__init__.py +++ b/dev/breeze/src/airflow_breeze/build_image/__init__.py @@ -14,3 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Build CI and PROD images.""" diff --git a/dev/breeze/src/airflow_breeze/ci/__init__.py b/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py similarity index 96% rename from dev/breeze/src/airflow_breeze/ci/__init__.py rename to dev/breeze/src/airflow_breeze/build_image/ci/__init__.py index 13a83393a9124..7a633ffa0b475 100644 --- a/dev/breeze/src/airflow_breeze/ci/__init__.py +++ b/dev/breeze/src/airflow_breeze/build_image/ci/__init__.py @@ -14,3 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Building CI image.""" diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py new file mode 100644 index 0000000000000..e7805170c3c8e --- /dev/null +++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Command to build CI image.""" +from typing import Dict + +from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams +from airflow_breeze.utils.cache import synchronize_parameters_with_cache, touch_cache_file +from airflow_breeze.utils.console import console +from airflow_breeze.utils.docker_command_utils import construct_build_docker_command +from airflow_breeze.utils.md5_build_check import calculate_md5_checksum_for_files +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, BUILD_CACHE_DIR +from airflow_breeze.utils.registry import login_to_docker_registry +from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command + +REQUIRED_CI_IMAGE_ARGS = [ + "python_base_image", + "airflow_version", + "airflow_branch", + "airflow_extras", + "airflow_pre_cached_pip_packages", + "additional_airflow_extras", + "additional_python_deps", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "upgrade_to_newer_dependencies", + "constraints_github_repository", + "airflow_constraints_reference", + "airflow_constraints", + "airflow_image_repository", + "airflow_image_date_created", + "build_id", +] + +OPTIONAL_CI_IMAGE_ARGS = [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", +] + + +def get_ci_image_build_params(parameters_passed: Dict[str, str]) -> BuildCiParams: + """ + Converts parameters received as dict into BuildCiParams. In case cacheable + parameters are missing, it reads the last used value for that parameter + from the cache and if it is not found, it uses default value for that parameter. + + This method updates cached based on parameters passed via Dict. + + :param parameters_passed: parameters to use when constructing BuildCiParams + """ + ci_image_params = BuildCiParams(**parameters_passed) + synchronize_parameters_with_cache(ci_image_params, parameters_passed) + return ci_image_params + + +def build_image(verbose: bool, dry_run: bool, **kwargs) -> None: + """ + Builds CI image: + + * fixes group permissions for files (to improve caching when umask is 002) + * converts all the parameters received via kwargs into BuildCIParams (including cache) + * prints info about the image to build + * logs int to docker registry on CI if build cache is being executed + * removes "tag" for previously build image so that inline cache uses only remote image + * constructs docker-compose command to run based on parameters passed + * run the build command + * update cached information that the build completed and saves checksums of all files + for quick future check if the build is needed + + :param verbose: print commands when running + :param dry_run: do not execute "write" commands - just print what would happen + :param kwargs: arguments passed from the command + """ + fix_group_permissions() + parameters_passed = filter_out_none(**kwargs) + ci_image_params = get_ci_image_build_params(parameters_passed) + ci_image_params.print_info() + run_command( + ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_image_name], + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + text=True, + check=False, + ) + cmd = construct_build_docker_command( + image_params=ci_image_params, + verbose=verbose, + required_args=REQUIRED_CI_IMAGE_ARGS, + optional_args=OPTIONAL_CI_IMAGE_ARGS, + production_image=False, + ) + if ci_image_params.prepare_buildx_cache: + login_to_docker_registry(ci_image_params) + console.print(f"\n[blue]Building CI Image for Python {ci_image_params.python}\n") + run_command(cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True) + if not dry_run: + ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch + ci_image_cache_dir.mkdir(parents=True, exist_ok=True) + touch_cache_file(f"built_{ci_image_params.python}", root_dir=ci_image_cache_dir) + calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir, update=True) + else: + console.print("[blue]Not updating build cache because we are in `dry_run` mode.[/]") diff --git a/dev/breeze/src/airflow_breeze/ci/build_params.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py similarity index 59% rename from dev/breeze/src/airflow_breeze/ci/build_params.py rename to dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py index 94e462695a677..ac7901e4aabc6 100644 --- a/dev/breeze/src/airflow_breeze/ci/build_params.py +++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py @@ -14,37 +14,36 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Parameters for Build CI Image.""" import os -import sys from dataclasses import dataclass from datetime import datetime +from pathlib import Path from typing import List, Optional from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH -from airflow_breeze.console import console from airflow_breeze.global_constants import get_airflow_version -from airflow_breeze.utils.docker_command_utils import check_if_buildx_plugin_available -from airflow_breeze.utils.run_utils import run_command +from airflow_breeze.utils.console import console +from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR @dataclass -class BuildParams: - # To construct ci_image_name +class BuildCiParams: + """ + CI build parameters. Those parameters are used to determine command issued to build CI image. + """ + upgrade_to_newer_dependencies: str = "false" - python_version: str = "3.7" + python: str = "3.7" airflow_branch: str = AIRFLOW_BRANCH build_id: int = 0 - # To construct docker cache ci directive docker_cache: str = "pulled" airflow_extras: str = "devel_ci" + install_providers_from_sources: bool = False additional_airflow_extras: str = "" additional_python_deps: str = "" - # To construct ci_image_name - tag: str = "latest" - # To construct airflow_image_repository github_repository: str = "apache/airflow" constraints_github_repository: str = "apache/airflow" - # Not sure if defaultConstraintsBranch and airflow_constraints_reference are different default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints: str = "constraints-source-providers" airflow_constraints_reference: Optional[str] = "constraints-main" @@ -52,6 +51,7 @@ class BuildParams: airflow_pre_cached_pip_packages: str = "true" dev_apt_command: str = "" dev_apt_deps: str = "" + image_tag: str = "" additional_dev_apt_command: str = "" additional_dev_apt_deps: str = "" additional_dev_apt_env: str = "" @@ -63,24 +63,24 @@ class BuildParams: platform: str = f"linux/{os.uname().machine}" debian_version: str = "bullseye" prepare_buildx_cache: bool = False - ci: bool = False + skip_rebuild_check: bool = False @property - def airflow_image_name(self): + def airflow_base_image_name(self): image = f'ghcr.io/{self.github_repository.lower()}' return image @property - def airflow_ci_image_name(self): + def airflow_image_name(self): """Construct CI image link""" - image = f'{self.airflow_image_name}/{self.airflow_branch}/ci/python{self.python_version}' + image = f'{self.airflow_base_image_name}/{self.airflow_branch}/ci/python{self.python}' return image @property def airflow_ci_image_name_with_tag(self): """Construct CI image link""" - image = f'{self.airflow_image_name}/{self.airflow_branch}/ci/python{self.python_version}' - return image if not self.tag else image + f":{self.tag}" + image = f'{self.airflow_base_image_name}/{self.airflow_branch}/ci/python{self.python}' + return image if not self.image_tag else image + f":{self.image_tag}" @property def airflow_image_repository(self): @@ -89,86 +89,53 @@ def airflow_image_repository(self): @property def python_base_image(self): """Construct Python Base Image""" - return f'python:{self.python_version}-slim-{self.debian_version}' + return f'python:{self.python}-slim-{self.debian_version}' @property def airflow_ci_local_manifest_image(self): """Construct CI Local Manifest Image""" - return f'local-airflow-ci-manifest/{self.airflow_branch}/python{self.python_version}' + return f'local-airflow-ci-manifest/{self.airflow_branch}/python{self.python}' @property def airflow_ci_remote_manifest_image(self): """Construct CI Remote Manifest Image""" - return f'{self.airflow_ci_image_name}/{self.airflow_branch}/ci-manifest//python:{self.python_version}' + return f'{self.airflow_image_name}/{self.airflow_branch}/ci-manifest//python:{self.python}' @property def airflow_image_date_created(self): now = datetime.now() return now.strftime("%Y-%m-%dT%H:%M:%SZ") - @property - def commit_sha(self): - output = run_command(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True) - return output.stdout.strip() - @property def airflow_version(self): return get_airflow_version() - def check_buildx_plugin_build_command(self): - build_command_param = [] - is_buildx_available = check_if_buildx_plugin_available(True) - if is_buildx_available: - if self.prepare_buildx_cache: - build_command_param.extend( - ["buildx", "build", "--builder", "airflow_cache", "--progress=tty"] - ) - cmd = ['docker', 'buildx', 'inspect', 'airflow_cache'] - output = run_command(cmd, verbose=True, text=True) - if output.returncode != 0: - next_cmd = ['docker', 'buildx', 'create', '--name', 'airflow_cache'] - run_command(next_cmd, verbose=True, text=True) - else: - build_command_param.extend(["buildx", "build", "--builder", "default", "--progress=tty"]) - else: - if self.prepare_buildx_cache: - console.print( - '\n[red] Buildx cli plugin is not available and you need it to prepare buildx cache. \n' - ) - console.print( - '[red] Please install it following https://docs.docker.com/buildx/working-with-buildx/ \n' - ) - sys.exit() - build_command_param.append("build") - return build_command_param - @property - def docker_cache_ci_directive(self) -> List: + def docker_cache_ci_directive(self) -> List[str]: docker_cache_ci_directive = [] if self.docker_cache == "pulled": - docker_cache_ci_directive.append(f"--cache-from={self.airflow_ci_image_name}") + docker_cache_ci_directive.append(f"--cache-from={self.airflow_image_name}") elif self.docker_cache == "disabled": docker_cache_ci_directive.append("--no-cache") else: docker_cache_ci_directive = [] - if self.prepare_buildx_cache: docker_cache_ci_directive.extend(["--cache-to=type=inline,mode=max", "--push"]) return docker_cache_ci_directive @property - def extra_docker_ci_flags(self) -> List[str]: + def extra_docker_build_flags(self) -> List[str]: extra_ci_flags = [] - if self.ci: - extra_ci_flags.extend( - [ - "--build-arg", - "PIP_PROGRESS_BAR=off", - ] - ) if self.airflow_constraints_location is not None and len(self.airflow_constraints_location) > 0: extra_ci_flags.extend( ["--build-arg", f"AIRFLOW_CONSTRAINTS_LOCATION={self.airflow_constraints_location}"] ) return extra_ci_flags + + @property + def md5sum_cache_dir(self) -> Path: + return Path(BUILD_CACHE_DIR, self.airflow_branch, self.python, "CI") + + def print_info(self): + console.print(f"CI Image: {self.airflow_version} Python: {self.python}.") diff --git a/dev/breeze/src/airflow_breeze/docs_generator/__init__.py b/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py similarity index 96% rename from dev/breeze/src/airflow_breeze/docs_generator/__init__.py rename to dev/breeze/src/airflow_breeze/build_image/prod/__init__.py index 13a83393a9124..141dd81f2169e 100644 --- a/dev/breeze/src/airflow_breeze/docs_generator/__init__.py +++ b/dev/breeze/src/airflow_breeze/build_image/prod/__init__.py @@ -14,3 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Building PROD Image.""" diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py new file mode 100644 index 0000000000000..2974d00d46e49 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Command to build PROD image.""" +import contextlib +import sys +from typing import Dict + +from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams +from airflow_breeze.utils.cache import synchronize_parameters_with_cache +from airflow_breeze.utils.console import console +from airflow_breeze.utils.docker_command_utils import construct_build_docker_command +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR +from airflow_breeze.utils.registry import login_to_docker_registry +from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command + +REQUIRED_PROD_IMAGE_ARGS = [ + "python_base_image", + "install_mysql_client", + "install_mssql_client", + "install_postgres_client", + "airflow_version", + "airflow_branch", + "airflow_extras", + "airflow_pre_cached_pip_packages", + "docker_context_files", + "additional_airflow_extras", + "additional_python_deps", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "upgrade_to_newer_dependencies", + "constraints_github_repository", + "airflow_constraints", + "airflow_image_repository", + "airflow_image_date_created", + "build_id", + "airflow_image_readme_url", + "install_providers_from_sources", + "install_from_pypi", + "install_from_docker_context_files", +] + +OPTIONAL_PROD_IMAGE_ARGS = [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", +] + + +def clean_docker_context_files(): + """ + Cleans up docker context files folder - leaving only README.md there. + """ + with contextlib.suppress(FileNotFoundError): + context_files_to_delete = DOCKER_CONTEXT_DIR.glob('**/*') + for file_to_delete in context_files_to_delete: + if file_to_delete.name != 'README.md': + file_to_delete.unlink() + + +def check_docker_context_files(install_from_docker_context_files: bool): + """ + Sanity check - if we want to install from docker-context-files we expect some packages there but if + we don't - we don't expect them, and they might invalidate Docker cache. + + This method exits with an error if what we see is unexpected for given operation. + + :param install_from_docker_context_files: whether we want to install from docker-context-files + """ + context_file = DOCKER_CONTEXT_DIR.glob('**/*') + number_of_context_files = len( + [context for context in context_file if context.is_file() and context.name != 'README.md'] + ) + if number_of_context_files == 0: + if install_from_docker_context_files: + console.print('[bright_yellow]\nERROR! You want to install packages from docker-context-files') + console.print('[bright_yellow]\n but there are no packages to install in this folder.') + sys.exit(1) + else: + if not install_from_docker_context_files: + console.print( + '[bright_yellow]\n ERROR! There are some extra files in docker-context-files except README.md' + ) + console.print('[bright_yellow]\nAnd you did not choose --install-from-docker-context-files flag') + console.print( + '[bright_yellow]\nThis might result in unnecessary cache invalidation and long build times' + ) + console.print( + '[bright_yellow]\nExiting now \ + - please restart the command with --cleanup-docker-context-files switch' + ) + sys.exit(1) + + +def get_prod_image_build_params(parameters_passed: Dict[str, str]) -> BuildProdParams: + """ + Converts parameters received as dict into BuildProdParams. In case cacheable + parameters are missing, it reads the last used value for that parameter + from the cache and if it is not found, it uses default value for that parameter. + + This method updates cached based on parameters passed via Dict. + + :param parameters_passed: parameters to use when constructing BuildCiParams + """ + prod_image_params = BuildProdParams(**parameters_passed) + synchronize_parameters_with_cache(prod_image_params, parameters_passed) + return prod_image_params + + +def build_production_image(verbose: bool, dry_run: bool, **kwargs): + """ + Builds PROD image: + + * fixes group permissions for files (to improve caching when umask is 002) + * converts all the parameters received via kwargs into BuildProdParams (including cache) + * prints info about the image to build + * removes docker-context-files if requested + * performs sanity check if the files are present in docker-context-files if expected + * logs int to docker registry on CI if build cache is being executed + * removes "tag" for previously build image so that inline cache uses only remote image + * constructs docker-compose command to run based on parameters passed + * run the build command + * update cached information that the build completed and saves checksums of all files + for quick future check if the build is needed + + :param verbose: print commands when running + :param dry_run: do not execute "write" commands - just print what would happen + :param kwargs: arguments passed from the command + """ + fix_group_permissions() + parameters_passed = filter_out_none(**kwargs) + prod_image_params = get_prod_image_build_params(parameters_passed) + prod_image_params.print_info() + if prod_image_params.cleanup_docker_context_files: + clean_docker_context_files() + check_docker_context_files(prod_image_params.install_docker_context_files) + if prod_image_params.prepare_buildx_cache: + login_to_docker_registry(prod_image_params) + run_command( + ["docker", "rmi", "--no-prune", "--force", prod_image_params.airflow_image_name], + verbose=verbose, + dry_run=dry_run, + cwd=AIRFLOW_SOURCES_ROOT, + text=True, + check=False, + ) + console.print(f"\n[blue]Building PROD Image for Python {prod_image_params.python}\n") + cmd = construct_build_docker_command( + image_params=prod_image_params, + verbose=verbose, + required_args=REQUIRED_PROD_IMAGE_ARGS, + optional_args=OPTIONAL_PROD_IMAGE_ARGS, + production_image=True, + ) + run_command(cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True) diff --git a/dev/breeze/src/airflow_breeze/prod/prod_params.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py similarity index 68% rename from dev/breeze/src/airflow_breeze/prod/prod_params.py rename to dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py index 9e5155c615f62..74667be5f0e63 100644 --- a/dev/breeze/src/airflow_breeze/prod/prod_params.py +++ b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Parameters to build PROD image.""" import os import re import sys @@ -22,7 +23,6 @@ from typing import List from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH -from airflow_breeze.console import console from airflow_breeze.global_constants import ( AIRFLOW_SOURCES_FROM, AIRFLOW_SOURCES_TO, @@ -31,33 +31,33 @@ get_airflow_extras, get_airflow_version, ) -from airflow_breeze.utils.docker_command_utils import check_if_buildx_plugin_available -from airflow_breeze.utils.run_utils import is_multi_platform, run_command +from airflow_breeze.utils.console import console +from airflow_breeze.utils.run_utils import commit_sha @dataclass -class ProdParams: - build_cache_local: bool - build_cache_pulled: bool - build_cache_disabled: bool - skip_rebuild_check: bool - disable_mysql_client_installation: bool - disable_mssql_client_installation: bool - disable_postgres_client_installation: bool - install_docker_context_files: bool - disable_pypi_when_building: bool - disable_pip_cache: bool - skip_installing_airflow_providers_from_sources: bool - cleanup_docker_context_files: bool - prepare_buildx_cache: bool +class BuildProdParams: + """ + PROD build parameters. Those parameters are used to determine command issued to build PROD image. + """ + + docker_cache: str + disable_mysql_client_installation: bool = False + disable_mssql_client_installation: bool = False + disable_postgres_client_installation: bool = False + install_docker_context_files: bool = False + disable_airflow_repo_cache: bool = False + install_providers_from_sources: bool = True + cleanup_docker_context_files: bool = False + prepare_buildx_cache: bool = False + disable_pypi: bool = False upgrade_to_newer_dependencies: str = "false" airflow_version: str = get_airflow_version() - python_version: str = "3.7" + python: str = "3.7" airflow_branch_for_pypi_preloading: str = AIRFLOW_BRANCH install_airflow_reference: str = "" install_airflow_version: str = "" default_constraints_branch = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH - ci: bool = False build_id: int = 0 airflow_constraints: str = "constraints-source-providers" github_repository: str = "apache/airflow" @@ -89,14 +89,14 @@ def airflow_branch(self) -> str: return self.airflow_branch_for_pypi_preloading @property - def airflow_image_name(self): + def airflow_base_image_name(self): image = f'ghcr.io/{self.github_repository.lower()}' return image @property - def airflow_prod_image_name(self): + def airflow_image_name(self): """Construct PROD image link""" - image = f'{self.airflow_image_name}/{self.airflow_branch}/prod/python{self.python_version}' + image = f'{self.airflow_base_image_name}/{self.airflow_branch}/prod/python{self.python}' return image @property @@ -104,11 +104,6 @@ def the_image_type(self) -> str: the_image_type = 'PROD' return the_image_type - @property - def image_description(self) -> str: - image_description = 'Airflow production' - return image_description - @property def args_for_remote_install(self) -> List: build_args = [] @@ -124,8 +119,6 @@ def args_for_remote_install(self) -> List: "AIRFLOW_SOURCES_TO=/empty", ] ) - if self.ci: - build_args.extend(["--build-arg", "PIP_PROGRESS_BAR=off"]) if len(self.airflow_constraints_reference) > 0: build_args.extend( ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}"] @@ -210,21 +203,11 @@ def extra_docker_build_flags(self) -> List[str]: return extra_build_flags @property - def docker_cache(self) -> str: - if self.build_cache_local: - docker_cache = "local" - elif self.build_cache_disabled: - docker_cache = "disabled" - else: - docker_cache = "pulled" - return docker_cache - - @property - def docker_cache_prod_directive(self) -> List: + def docker_cache_prod_directive(self) -> List[str]: docker_cache_prod_directive = [] if self.docker_cache == "pulled": - docker_cache_prod_directive.append(f"--cache-from={self.airflow_prod_image_name}") + docker_cache_prod_directive.append(f"--cache-from={self.airflow_image_name}") elif self.docker_cache == "disabled": docker_cache_prod_directive.append("--no-cache") else: @@ -232,49 +215,13 @@ def docker_cache_prod_directive(self) -> List: if self.prepare_buildx_cache: docker_cache_prod_directive.extend(["--cache-to=type=inline,mode=max", "--push"]) - if is_multi_platform(self.platform): - console.print("\nSkip loading docker image on multi-platform build") - else: - docker_cache_prod_directive.extend(["--load"]) return docker_cache_prod_directive - def check_buildx_plugin_build_command(self): - build_command_param = [] - is_buildx_available = check_if_buildx_plugin_available(True) - if is_buildx_available: - if self.prepare_buildx_cache: - build_command_param.extend( - ["buildx", "build", "--builder", "airflow_cache", "--progress=tty"] - ) - cmd = ['docker', 'buildx', 'inspect', 'airflow_cache'] - output = run_command(cmd, verbose=True, text=True) - if output.returncode != 0: - next_cmd = ['docker', 'buildx', 'create', '--name', 'airflow_cache'] - run_command(next_cmd, verbose=True, text=True) - else: - build_command_param.extend(["buildx", "build", "--builder", "default", "--progress=tty"]) - else: - if self.prepare_buildx_cache: - console.print( - '\n[red] Buildx cli plugin is not available and you need it to prepare buildx cache. \n' - ) - console.print( - '[red] Please install it following https://docs.docker.com/buildx/working-with-buildx/ \n' - ) - sys.exit() - build_command_param.append("build") - return build_command_param - @property def python_base_image(self): """Construct Python Base Image""" # ghcr.io/apache/airflow/main/python:3.8-slim-bullseye - return f'python:{self.python_version}-slim-{self.debian_version}' - - @property - def commit_sha(self): - output = run_command(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True) - return output.stdout.strip() + return f'python:{self.python}-slim-{self.debian_version}' @property def airflow_image_repository(self): @@ -287,41 +234,22 @@ def airflow_image_date_created(self): @property def airflow_image_readme_url(self): - return ( - f"https://raw.githubusercontent.com/apache/airflow/{self.commit_sha}/docs/docker-stack/README.md" - ) + return f"https://raw.githubusercontent.com/apache/airflow/{commit_sha()}/docs/docker-stack/README.md" def print_info(self): - console.print( - f"Airflow {self.airflow_version} Python: {self.python_version}.\ - Image description: {self.image_description}" - ) - - @property - def skip_building_prod_image(self) -> bool: - skip_build = False - if self.skip_rebuild_check: - skip_build = True - return skip_build - - @property - def check_image_for_rebuild(self) -> bool: - check_image = True - if self.skip_rebuild_check: - check_image = False - return check_image + console.print(f"CI Image: {self.airflow_version} Python: {self.python}.") @property def install_from_pypi(self) -> str: install_from_pypi = 'true' - if self.disable_pypi_when_building: + if self.disable_pypi: install_from_pypi = 'false' return install_from_pypi @property def airflow_pre_cached_pip_packages(self) -> str: airflow_pre_cached_pip = 'true' - if self.disable_pypi_when_building or self.disable_pip_cache: + if self.disable_pypi or self.disable_airflow_repo_cache: airflow_pre_cached_pip = 'false' return airflow_pre_cached_pip @@ -346,17 +274,6 @@ def install_postgres_client(self) -> str: install_postgres = 'false' return install_postgres - @property - def install_providers_from_sources(self) -> str: - install_providers_source = 'true' - if ( - self.skip_installing_airflow_providers_from_sources - or len(self.install_airflow_reference) > 0 - or len(self.install_airflow_version) > 0 - ): - install_providers_source = 'false' - return install_providers_source - @property def install_from_docker_context_files(self) -> str: install_from_docker_context_files = 'false' diff --git a/dev/breeze/src/airflow_breeze/cache.py b/dev/breeze/src/airflow_breeze/cache.py deleted file mode 100644 index c9ae31ee27f15..0000000000000 --- a/dev/breeze/src/airflow_breeze/cache.py +++ /dev/null @@ -1,112 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import sys -from pathlib import Path -from typing import Any, List, Optional, Tuple - -from airflow_breeze import global_constants -from airflow_breeze.console import console -from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR - - -def check_if_cache_exists(param_name: str) -> bool: - return (Path(BUILD_CACHE_DIR) / f".{param_name}").exists() - - -def read_from_cache_file(param_name: str) -> Optional[str]: - cache_exists = check_if_cache_exists(param_name) - if cache_exists: - return (Path(BUILD_CACHE_DIR) / f".{param_name}").read_text().strip() - else: - return None - - -def touch_cache_file(param_name: str, root_dir: Path = BUILD_CACHE_DIR): - (Path(root_dir) / f".{param_name}").touch() - - -def write_to_cache_file(param_name: str, param_value: str, check_allowed_values: bool = True) -> None: - allowed = False - allowed_values = None - if check_allowed_values: - allowed, allowed_values = check_if_values_allowed(param_name, param_value) - if allowed or not check_allowed_values: - print('BUILD CACHE DIR:', BUILD_CACHE_DIR) - cache_file = Path(BUILD_CACHE_DIR, f".{param_name}").open("w+") - cache_file.write(param_value) - else: - console.print(f'[cyan]You have sent the {param_value} for {param_name}') - console.print(f'[cyan]Allowed value for the {param_name} are {allowed_values}') - console.print('[cyan]Provide one of the supported params. Write to cache dir failed') - sys.exit() - - -def check_cache_and_write_if_not_cached( - param_name: str, default_param_value: str -) -> Tuple[bool, Optional[str]]: - is_cached = False - cached_value = read_from_cache_file(param_name) - if cached_value is None: - write_to_cache_file(param_name, default_param_value) - cached_value = default_param_value - else: - allowed, allowed_values = check_if_values_allowed(param_name, cached_value) - if allowed: - is_cached = True - else: - write_to_cache_file(param_name, default_param_value) - cached_value = default_param_value - return is_cached, cached_value - - -def check_if_values_allowed(param_name: str, param_value: str) -> Tuple[bool, List[Any]]: - allowed = False - allowed_values: List[Any] = [] - allowed_values = getattr(global_constants, f'ALLOWED_{param_name.upper()}S') - if param_value in allowed_values: - allowed = True - return allowed, allowed_values - - -def delete_cache(param_name: str) -> bool: - deleted = False - if check_if_cache_exists(param_name): - (Path(BUILD_CACHE_DIR) / f".{param_name}").unlink() - deleted = True - return deleted - - -def update_md5checksum_in_cache(file_content: str, cache_file_name: Path) -> bool: - modified = False - if cache_file_name.exists(): - old_md5_checksum_content = Path(cache_file_name).read_text() - if old_md5_checksum_content.strip() != file_content.strip(): - Path(cache_file_name).write_text(file_content) - modified = True - else: - Path(cache_file_name).write_text(file_content) - modified = True - return modified - - -def write_env_in_cache(env_variables) -> Path: - shell_path = Path(BUILD_CACHE_DIR, "shell_command.env") - with open(shell_path, 'w') as shell_env_file: - for env_variable in env_variables: - shell_env_file.write(env_variable + '\n') - return shell_path diff --git a/dev/breeze/src/airflow_breeze/ci/build_image.py b/dev/breeze/src/airflow_breeze/ci/build_image.py deleted file mode 100644 index 818cdf29eca62..0000000000000 --- a/dev/breeze/src/airflow_breeze/ci/build_image.py +++ /dev/null @@ -1,124 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from pathlib import Path -from typing import Dict, List - -from airflow_breeze.cache import check_cache_and_write_if_not_cached, touch_cache_file, write_to_cache_file -from airflow_breeze.ci.build_params import BuildParams -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE, BUILD_CACHE_DIR -from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command - -PARAMS_CI_IMAGE = [ - "python_base_image", - "airflow_version", - "airflow_branch", - "airflow_extras", - "airflow_pre_cached_pip_packages", - "additional_airflow_extras", - "additional_python_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "upgrade_to_newer_dependencies", - "constraints_github_repository", - "airflow_constraints_reference", - "airflow_constraints", - "airflow_image_repository", - "airflow_image_date_created", - "build_id", - "commit_sha", -] - -PARAMS_TO_VERIFY_CI_IMAGE = [ - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", -] - - -def construct_arguments_docker_command(ci_image: BuildParams) -> List[str]: - args_command = [] - for param in PARAMS_CI_IMAGE: - args_command.append("--build-arg") - args_command.append(param.upper() + "=" + str(getattr(ci_image, param))) - for verify_param in PARAMS_TO_VERIFY_CI_IMAGE: - param_value = str(getattr(ci_image, verify_param)) - if len(param_value) > 0: - args_command.append("--build-arg") - args_command.append(verify_param.upper() + "=" + param_value) - docker_cache = ci_image.docker_cache_ci_directive - if len(docker_cache) > 0: - args_command.extend(ci_image.docker_cache_ci_directive) - return args_command - - -def construct_docker_command(ci_image: BuildParams) -> List[str]: - arguments = construct_arguments_docker_command(ci_image) - build_command = ci_image.check_buildx_plugin_build_command() - build_flags = ci_image.extra_docker_ci_flags - final_command = [] - final_command.extend(["docker"]) - final_command.extend(build_command) - final_command.extend(build_flags) - final_command.extend(["--pull"]) - final_command.extend(arguments) - final_command.extend(["-t", ci_image.airflow_ci_image_name, "--target", "main", "."]) - final_command.extend(["-f", 'Dockerfile.ci']) - final_command.extend(["--platform", ci_image.platform]) - return final_command - - -def build_image(verbose, **kwargs): - fix_group_permissions() - parameters_passed = filter_out_none(**kwargs) - ci_image_params = get_image_build_params(parameters_passed) - ci_image_cache_dir = Path(BUILD_CACHE_DIR, ci_image_params.airflow_branch) - ci_image_cache_dir.mkdir(parents=True, exist_ok=True) - touch_cache_file( - f"built_{ci_image_params.python_version}", - root_dir=ci_image_cache_dir, - ) - run_command( - ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_ci_image_name], - verbose=verbose, - cwd=AIRFLOW_SOURCE, - text=True, - suppress_raise_exception=True, - ) - cmd = construct_docker_command(ci_image_params) - run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True) - - -def get_image_build_params(parameters_passed: Dict[str, str]): - cacheable_parameters = {"python_version": 'PYTHON_MAJOR_MINOR_VERSION'} - ci_image_params = BuildParams(**parameters_passed) - for parameter, cache_key in cacheable_parameters.items(): - value_from_parameter = parameters_passed.get(parameter) - if value_from_parameter: - write_to_cache_file(cache_key, value_from_parameter, check_allowed_values=True) - setattr(ci_image_params, parameter, value_from_parameter) - else: - is_cached, value = check_cache_and_write_if_not_cached( - cache_key, getattr(ci_image_params, parameter) - ) - if is_cached: - setattr(ci_image_params, parameter, value) - return ci_image_params diff --git a/dev/breeze/src/airflow_breeze/console.py b/dev/breeze/src/airflow_breeze/console.py deleted file mode 100644 index 175fb7014ea83..0000000000000 --- a/dev/breeze/src/airflow_breeze/console.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from rich.console import Console -from rich.theme import Theme - -custom_theme = Theme({"info": "blue", "warning": "magenta", "error": "red"}) -console = Console(force_terminal=True, color_system="standard", width=180, theme=custom_theme) diff --git a/dev/breeze/src/airflow_breeze/docs_generator/build_documentation.py b/dev/breeze/src/airflow_breeze/docs_generator/build_documentation.py deleted file mode 100644 index 91f6f469b68b3..0000000000000 --- a/dev/breeze/src/airflow_breeze/docs_generator/build_documentation.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from airflow_breeze.docs_generator.doc_builder import DocBuilder -from airflow_breeze.global_constants import MOUNT_ALL_LOCAL_SOURCES, MOUNT_SELECTED_LOCAL_SOURCES -from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags -from airflow_breeze.utils.run_utils import run_command - - -def build( - verbose: bool, - airflow_sources: str, - airflow_ci_image_name: str, - doc_builder: DocBuilder, -): - extra_docker_flags = get_extra_docker_flags( - MOUNT_ALL_LOCAL_SOURCES, MOUNT_SELECTED_LOCAL_SOURCES, airflow_sources - ) - cmd = [] - cmd.extend(["docker", "run"]) - cmd.extend(extra_docker_flags) - cmd.extend(["-t", "-e", "GITHUB_ACTIONS="]) - cmd.extend(["--entrypoint", "/usr/local/bin/dumb-init", "--pull", "never"]) - cmd.extend([airflow_ci_image_name, "--", "/opt/airflow/scripts/in_container/run_docs_build.sh"]) - cmd.extend(doc_builder.args_doc_builder) - run_command(cmd, verbose=verbose, text=True) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index da048049b1dcb..013a73a21db48 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -14,11 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +Global constants that are used by all other Breeze components. +""" import os -from pathlib import Path from typing import List -from airflow_breeze.utils.path_utils import get_airflow_sources_root +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT # Commented this out as we are using buildkit and this vars became irrelevant # FORCE_PULL_IMAGES = False @@ -34,88 +36,6 @@ # Checked before putting in build cache ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS = ['3.7', '3.8', '3.9', '3.10'] ALLOWED_BACKENDS = ['sqlite', 'mysql', 'postgres', 'mssql'] -ALLOWED_STATIC_CHECKS = [ - "all", - "airflow-config-yaml", - "airflow-providers-available", - "airflow-provider-yaml-files-ok", - "base-operator", - "black", - "blacken-docs", - "boring-cyborg", - "build-providers-dependencies", - "chart-schema-lint", - "capitalized-breeze", - "changelog-duplicates", - "check-apache-license", - "check-builtin-literals", - "check-executables-have-shebangs", - "check-extras-order", - "check-hooks-apply", - "check-integrations", - "check-merge-conflict", - "check-xml", - "check-system-tests", - "daysago-import-check", - "debug-statements", - "detect-private-key", - "doctoc", - "dont-use-safe-filter", - "end-of-file-fixer", - "fix-encoding-pragma", - "flake8", - "flynt", - "codespell", - "forbid-tabs", - "helm-lint", - "identity", - "incorrect-use-of-LoggingMixin", - "insert-license", - "isort", - "json-schema", - "language-matters", - "lint-dockerfile", - "lint-openapi", - "markdownlint", - "mermaid", - "mixed-line-ending", - "mypy", - "mypy-helm", - "no-providers-in-core-examples", - "no-relative-imports", - "pre-commit-descriptions", - "pre-commit-hook-names", - "pretty-format-json", - "provide-create-sessions", - "providers-changelogs", - "providers-init-file", - "providers-subpackages-init-file", - "provider-yamls", - "pydevd", - "pydocstyle", - "python-no-log-warn", - "pyupgrade", - "restrict-start_date", - "rst-backticks", - "setup-order", - "setup-extra-packages", - "shellcheck", - "sort-in-the-wild", - "sort-spelling-wordlist", - "stylelint", - "trailing-whitespace", - "ui-lint", - "update-breeze-file", - "update-extras", - "update-local-yml-file", - "update-setup-cfg-file", - "update-versions", - "verify-db-migrations-documented", - "version-sync", - "www-lint", - "yamllint", - "yesqa", -] ALLOWED_INTEGRATIONS = [ 'cassandra', 'kerberos', @@ -134,8 +54,13 @@ ALLOWED_HELM_VERSIONS = ['v3.6.3'] ALLOWED_EXECUTORS = ['KubernetesExecutor', 'CeleryExecutor', 'LocalExecutor', 'CeleryKubernetesExecutor'] ALLOWED_KIND_OPERATIONS = ['start', 'stop', 'restart', 'status', 'deploy', 'test', 'shell', 'k9s'] -ALLOWED_INSTALL_AIRFLOW_VERSIONS = ['2.0.2', '2.0.1', '2.0.0', 'wheel', 'sdist'] ALLOWED_GENERATE_CONSTRAINTS_MODES = ['source-providers', 'pypi-providers', 'no-providers'] + +MOUNT_SELECTED = "selected" +MOUNT_ALL = "all" +MOUNT_NONE = "none" + +ALLOWED_MOUNT_OPTIONS = [MOUNT_SELECTED, MOUNT_ALL, MOUNT_NONE] ALLOWED_POSTGRES_VERSIONS = ['10', '11', '12', '13'] ALLOWED_MYSQL_VERSIONS = ['5.7', '8'] ALLOWED_MSSQL_VERSIONS = ['2017-latest', '2019-latest'] @@ -155,8 +80,10 @@ 'Quarantined', ] ALLOWED_PACKAGE_FORMATS = ['both', 'sdist', 'wheel'] -ALLOWED_USE_AIRFLOW_VERSIONS = ['.', 'apache-airflow'] +ALLOWED_INSTALLATION_METHODS = ['.', 'apache-airflow'] ALLOWED_DEBIAN_VERSIONS = ['buster', 'bullseye'] +ALLOWED_BUILD_CACHE = ["pulled", "local", "disabled"] +ALLOWED_PLATFORMS = ["linux/amd64", "linux/arm64", "linux/amd64,linux/arm64"] PARAM_NAME_DESCRIPTION = { "BACKEND": "backend", @@ -193,7 +120,7 @@ def get_available_packages() -> List[str]: - docs_path_content = Path(get_airflow_sources_root(), 'docs').glob('*/') + docs_path_content = (AIRFLOW_SOURCES_ROOT / 'docs').glob('*/') available_packages = [x.name for x in docs_path_content if x.is_dir()] return list(set(available_packages) - set(EXCLUDE_DOCS_PACKAGE_FOLDER)) @@ -235,7 +162,7 @@ def get_available_packages() -> List[str]: def get_airflow_version(): - airflow_setup_file = Path(get_airflow_sources_root()) / 'setup.py' + airflow_setup_file = AIRFLOW_SOURCES_ROOT / 'setup.py' with open(airflow_setup_file) as setup_file: for line in setup_file.readlines(): if "version =" in line: @@ -243,7 +170,7 @@ def get_airflow_version(): def get_airflow_extras(): - airflow_dockerfile = Path(get_airflow_sources_root()) / 'Dockerfile' + airflow_dockerfile = AIRFLOW_SOURCES_ROOT / 'Dockerfile' with open(airflow_dockerfile) as dockerfile: for line in dockerfile.readlines(): if "ARG AIRFLOW_EXTRAS=" in line: @@ -284,13 +211,8 @@ def get_airflow_extras(): 'airflow/ui/yarn.lock', ] -# Initialize mount variables -MOUNT_SELECTED_LOCAL_SOURCES = True -MOUNT_ALL_LOCAL_SOURCES = False - ENABLED_SYSTEMS = "" - CURRENT_KUBERNETES_MODES = ['image'] CURRENT_KUBERNETES_VERSIONS = ['v1.23.4', 'v1.22.7', 'v1.21.10', 'v1.20.15'] CURRENT_KIND_VERSIONS = ['v0.12.0'] diff --git a/dev/breeze/src/airflow_breeze/prod/build_prod_image.py b/dev/breeze/src/airflow_breeze/prod/build_prod_image.py deleted file mode 100644 index 1573b547c43a2..0000000000000 --- a/dev/breeze/src/airflow_breeze/prod/build_prod_image.py +++ /dev/null @@ -1,200 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import contextlib -import sys -from typing import Dict, List - -from airflow_breeze.cache import check_cache_and_write_if_not_cached, write_to_cache_file -from airflow_breeze.console import console -from airflow_breeze.prod.prod_params import ProdParams -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE, DOCKER_CONTEXT_DIR -from airflow_breeze.utils.run_utils import filter_out_none, run_command - -PARAMS_PROD_IMAGE = [ - "python_base_image", - "install_mysql_client", - "install_mssql_client", - "install_postgres_client", - "airflow_version", - "airflow_branch", - "airflow_extras", - "airflow_pre_cached_pip_packages", - "docker_context_files", - "additional_airflow_extras", - "additional_python_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "upgrade_to_newer_dependencies", - "constraints_github_repository", - "airflow_constraints", - "airflow_image_repository", - "airflow_image_date_created", - "build_id", - "commit_sha", - "airflow_image_readme_url", - "install_providers_from_sources", - "install_from_pypi", - "install_from_docker_context_files", -] - -PARAMS_TO_VERIFY_PROD_IMAGE = [ - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", -] - - -def construct_arguments_docker_command(prod_image: ProdParams) -> List[str]: - args_command = [] - for param in PARAMS_PROD_IMAGE: - args_command.append("--build-arg") - args_command.append(param.upper() + "=" + str(getattr(prod_image, param))) - for verify_param in PARAMS_TO_VERIFY_PROD_IMAGE: - param_value = str(getattr(prod_image, verify_param)) - if len(param_value) > 0: - args_command.append("--build-arg") - args_command.append(verify_param.upper() + "=" + param_value) - docker_cache = prod_image.docker_cache_prod_directive - if len(docker_cache) > 0: - args_command.extend(prod_image.docker_cache_prod_directive) - return args_command - - -def construct_docker_command(prod_image: ProdParams) -> List[str]: - arguments = construct_arguments_docker_command(prod_image) - build_command = prod_image.check_buildx_plugin_build_command() - build_flags = prod_image.extra_docker_build_flags - final_command = [] - final_command.extend(["docker"]) - final_command.extend(build_command) - final_command.extend(build_flags) - final_command.extend(["--pull"]) - final_command.extend(arguments) - final_command.extend(["-t", prod_image.airflow_prod_image_name, "--target", "main", "."]) - final_command.extend(["-f", 'Dockerfile']) - final_command.extend(["--platform", prod_image.platform]) - return final_command - - -def login_to_docker_registry(build_params: ProdParams): - if build_params.ci == "true": - if len(build_params.github_token) == 0: - console.print("\n[blue]Skip logging in to GitHub Registry. No Token available!") - elif build_params.airflow_login_to_github_registry != "true": - console.print( - "\n[blue]Skip logging in to GitHub Registry.\ - AIRFLOW_LOGIN_TO_GITHUB_REGISTRY is set as false" - ) - elif len(build_params.github_token) > 0: - run_command(['docker', 'logout', 'ghcr.io'], verbose=True, text=True) - run_command( - [ - 'docker', - 'login', - '--username', - build_params.github_username, - '--password-stdin', - 'ghcr.io', - ], - verbose=True, - text=True, - input=build_params.github_token, - ) - else: - console.print('\n[blue]Skip Login to GitHub Container Registry as token is missing') - - -def clean_docker_context_files(): - with contextlib.suppress(FileNotFoundError): - context_files_to_delete = DOCKER_CONTEXT_DIR.glob('**/*') - for file_to_delete in context_files_to_delete: - if file_to_delete.name != 'README.md': - file_to_delete.unlink() - - -def check_docker_context_files(install_from_docker_context_files: bool): - context_file = DOCKER_CONTEXT_DIR.glob('**/*') - number_of_context_files = len( - [context for context in context_file if context.is_file() and context.name != 'README.md'] - ) - if number_of_context_files == 0: - if install_from_docker_context_files: - console.print('[bright_yellow]\nERROR! You want to install packages from docker-context-files') - console.print('[bright_yellow]\n but there are no packages to install in this folder.') - sys.exit() - else: - if not install_from_docker_context_files: - console.print( - '[bright_yellow]\n ERROR! There are some extra files in docker-context-files except README.md' - ) - console.print('[bright_yellow]\nAnd you did not choose --install-from-docker-context-files flag') - console.print( - '[bright_yellow]\nThis might result in unnecessary cache invalidation and long build times' - ) - console.print( - '[bright_yellow]\nExiting now \ - - please restart the command with --cleanup-docker-context-files switch' - ) - sys.exit() - - -def build_production_image(verbose, **kwargs): - parameters_passed = filter_out_none(**kwargs) - prod_params = get_image_build_params(parameters_passed) - prod_params.print_info() - if prod_params.cleanup_docker_context_files: - clean_docker_context_files() - check_docker_context_files(prod_params.install_docker_context_files) - if prod_params.skip_building_prod_image: - console.print('[bright_yellow]\nSkip building production image. Assume the one we have is good!') - console.print('bright_yellow]\nYou must run Breeze2 build-prod-image before for all python versions!') - if prod_params.prepare_buildx_cache: - login_to_docker_registry(prod_params) - - cmd = construct_docker_command(prod_params) - run_command( - ["docker", "rmi", "--no-prune", "--force", prod_params.airflow_prod_image_name], - verbose=verbose, - cwd=AIRFLOW_SOURCE, - text=True, - suppress_raise_exception=True, - ) - run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True) - if prod_params.prepare_buildx_cache: - run_command(['docker', 'push', prod_params.airflow_prod_image_name], verbose=True, text=True) - - -def get_image_build_params(parameters_passed: Dict[str, str]): - cacheable_parameters = {"python_version": 'PYTHON_MAJOR_MINOR_VERSION'} - prod_image_params = ProdParams(**parameters_passed) - for parameter, cache_key in cacheable_parameters.items(): - value_from_parameter = parameters_passed.get(parameter) - if value_from_parameter: - write_to_cache_file(cache_key, value_from_parameter, check_allowed_values=True) - setattr(prod_image_params, parameter, value_from_parameter) - else: - is_cached, value = check_cache_and_write_if_not_cached( - cache_key, getattr(prod_image_params, parameter) - ) - if is_cached: - setattr(prod_image_params, parameter, value) - return prod_image_params diff --git a/dev/breeze/src/airflow_breeze/shell/__init__.py b/dev/breeze/src/airflow_breeze/shell/__init__.py index 13a83393a9124..209c3a09f90c9 100644 --- a/dev/breeze/src/airflow_breeze/shell/__init__.py +++ b/dev/breeze/src/airflow_breeze/shell/__init__.py @@ -14,3 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Entering Shell""" diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py index 783ca6348db36..4fc710ebad3ff 100644 --- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py +++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py @@ -14,225 +14,194 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Command to enter container shell for Breeze.""" import sys from pathlib import Path from typing import Dict import click -from inputimeout import TimeoutOccurred, inputimeout from airflow_breeze import global_constants -from airflow_breeze.cache import ( - check_cache_and_write_if_not_cached, +from airflow_breeze.build_image.ci.build_ci_image import build_image +from airflow_breeze.shell.shell_params import ShellParams +from airflow_breeze.utils.cache import ( + check_cached_value_is_allowed, read_from_cache_file, write_to_cache_file, ) -from airflow_breeze.ci.build_image import build_image -from airflow_breeze.console import console -from airflow_breeze.global_constants import ( - FLOWER_HOST_PORT, - MSSQL_HOST_PORT, - MSSQL_VERSION, - MYSQL_HOST_PORT, - MYSQL_VERSION, - POSTGRES_HOST_PORT, - POSTGRES_VERSION, - REDIS_HOST_PORT, - SSH_PORT, - WEBSERVER_HOST_PORT, -) -from airflow_breeze.shell.shell_builder import ShellBuilder +from airflow_breeze.utils.console import console from airflow_breeze.utils.docker_command_utils import ( + SOURCE_OF_DEFAULT_VALUES_FOR_VARIABLES, + VARIABLES_IN_CACHE, check_docker_compose_version, check_docker_resources, check_docker_version, + construct_env_variables_docker_compose_command, ) +from airflow_breeze.utils.md5_build_check import md5sum_check_if_build_is_needed from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR -from airflow_breeze.utils.run_utils import ( - filter_out_none, - get_latest_sha, - instruct_build_image, - instruct_for_setup, - is_repo_rebased, - md5sum_check_if_build_is_needed, - run_command, -) -from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE - -PARAMS_TO_ENTER_SHELL = { - "HOST_USER_ID": "host_user_id", - "HOST_GROUP_ID": "host_group_id", - "COMPOSE_FILE": "compose_files", - "PYTHON_MAJOR_MINOR_VERSION": "python_version", - "BACKEND": "backend", - "AIRFLOW_VERSION": "airflow_version", - "INSTALL_AIRFLOW_VERSION": "install_airflow_version", - "AIRFLOW_SOURCES": "airflow_sources", - "AIRFLOW_CI_IMAGE": "airflow_ci_image_name", - "AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag", - "AIRFLOW_PROD_IMAGE": "airflow_prod_image_name", - "AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes", - "SQLITE_URL": "sqlite_url", - "USE_AIRFLOW_VERSION": "use_airflow_version", - "SKIP_TWINE_CHECK": "skip_twine_check", - "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", - "EXECUTOR": "executor", - "START_AIRFLOW": "start_airflow", - "ENABLED_INTEGRATIONS": "enabled_integrations", - "GITHUB_ACTIONS": "github_actions", - "ISSUE_ID": "issue_id", - "NUM_RUNS": "num_runs", - "VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn", - "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", -} - -PARAMS_FOR_SHELL_CONSTANTS = { - "SSH_PORT": SSH_PORT, - "WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT, - "FLOWER_HOST_PORT": FLOWER_HOST_PORT, - "REDIS_HOST_PORT": REDIS_HOST_PORT, - "MYSQL_HOST_PORT": MYSQL_HOST_PORT, - "MYSQL_VERSION": MYSQL_VERSION, - "MSSQL_HOST_PORT": MSSQL_HOST_PORT, - "MSSQL_VERSION": MSSQL_VERSION, - "POSTGRES_HOST_PORT": POSTGRES_HOST_PORT, - "POSTGRES_VERSION": POSTGRES_VERSION, -} - -PARAMS_IN_CACHE = { - 'python_version': 'PYTHON_MAJOR_MINOR_VERSION', - 'backend': 'BACKEND', - 'executor': 'EXECUTOR', - 'postgres_version': 'POSTGRES_VERSION', - 'mysql_version': 'MYSQL_VERSION', - 'mssql_version': 'MSSQL_VERSION', -} - -DEFAULT_VALUES_FOR_PARAM = { - 'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION', - 'backend': 'DEFAULT_BACKEND', - 'executor': 'DEFAULT_EXECUTOR', - 'postgres_version': 'POSTGRES_VERSION', - 'mysql_version': 'MYSQL_VERSION', - 'mssql_version': 'MSSQL_VERSION', -} - - -def construct_env_variables_docker_compose_command(shell_params: ShellBuilder) -> Dict[str, str]: - env_variables: Dict[str, str] = {} - for param_name in PARAMS_TO_ENTER_SHELL: - param_value = PARAMS_TO_ENTER_SHELL[param_name] - env_variables[param_name] = str(getattr(shell_params, param_value)) - for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS: - constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name] - env_variables[constant_param_name] = str(constant_param_value) - return env_variables - - -def build_image_if_needed_steps(verbose: bool, shell_params: ShellBuilder): +from airflow_breeze.utils.run_utils import filter_out_none, instruct_build_image, is_repo_rebased, run_command +from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE + + +def build_image_if_needed_steps(verbose: bool, dry_run: bool, shell_params: ShellParams) -> None: + """ + Check if image build is needed based on what files have been modified since last build. + + * If build is needed, the user is asked for confirmation + * If the branch is not rebased it warns the user to rebase (to make sure latest remote cache is useful) + * Builds Image/Skips/Quits depending on the answer + + :param verbose: print commands when running + :param dry_run: do not execute "write" commands - just print what would happen + :param shell_params: parameters for the build + """ + # We import those locally so that click autocomplete works + from inputimeout import TimeoutOccurred, inputimeout + build_needed = md5sum_check_if_build_is_needed(shell_params.md5sum_cache_dir, shell_params.the_image_type) - if build_needed: - try: - user_status = inputimeout( - prompt='\nDo you want to build image?Press y/n/q in 5 seconds\n', - timeout=5, - ) - if user_status == 'y': - latest_sha = get_latest_sha(shell_params.github_repository, shell_params.airflow_branch) - if is_repo_rebased(latest_sha): + if not build_needed: + return + try: + user_status = inputimeout( + prompt='\nDo you want to build image?Press y/n/q in 5 seconds\n', + timeout=5, + ) + if user_status in ['y', 'yes', 'Y', 'Yes', 'YES']: + if is_repo_rebased(shell_params.github_repository, shell_params.airflow_branch): + build_image( + verbose, + dry_run=dry_run, + python=shell_params.python, + upgrade_to_newer_dependencies="false", + ) + else: + console.print( + "\n[bright_yellow]This might take a lot of time, w" + "e think you should rebase first.[/]\n" + ) + if click.confirm("But if you really, really want - you can do it"): build_image( - verbose, - python_version=shell_params.python_version, + verbose=verbose, + dry_run=dry_run, + python=shell_params.python, upgrade_to_newer_dependencies="false", ) else: - if click.confirm( - "\nThis might take a lot of time, we think you should rebase first. \ - But if you really, really want - you can do it\n" - ): - build_image( - verbose, - python_version=shell_params.python_version, - upgrade_to_newer_dependencies="false", - ) - else: - console.print( - '\nPlease rebase your code before continuing.\ - Check this link to know more \ - https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#id15\n' - ) - console.print('Exiting the process') - sys.exit() - elif user_status == 'n': - instruct_build_image(shell_params.the_image_type, shell_params.python_version) - elif user_status == 'q': - console.print('\nQuitting the process') - sys.exit() - else: - console.print('\nYou have given a wrong choice:', user_status, ' Quitting the process') - sys.exit() - except TimeoutOccurred: - console.print('\nTimeout. Considering your response as No\n') - instruct_build_image(shell_params.the_image_type, shell_params.python_version) - except Exception: - console.print('\nTerminating the process') + console.print( + "[bright_blue]Please rebase your code before continuing.[/]\n" + "Check this link to know more " + "https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst#id15\n" + ) + console.print('[red]Exiting the process[/]\n') + sys.exit(1) + elif user_status in ['n', 'no', 'N', 'No', 'NO']: + instruct_build_image(shell_params.python) + elif user_status in ['q', 'quit', 'Q', 'Quit', 'QUIT']: + console.print('\n[bright_yellow]Quitting the process[/]\n') sys.exit() - - -def build_image_checks(verbose: bool, shell_params: ShellBuilder): + else: + console.print( + f'\n[red]You have given a wrong choice: {user_status}.' f' Quitting the process[/]\n' + ) + sys.exit() + except TimeoutOccurred: + console.print('\nTimeout. Considering your response as No\n') + instruct_build_image(shell_params.python) + except Exception as e: + console.print(f'\nTerminating the process on {e}') + sys.exit(1) + + +def run_shell_with_build_image_checks(verbose: bool, dry_run: bool, shell_params: ShellParams): + """ + Executes shell command built from params passed, checking if build is not needed. + * checks if there are enough resources to run shell + * checks if image was built at least once (if not - forces the build) + * if not forces, checks if build is needed and asks the user if so + * builds the image if needed + * prints information about the build + * constructs docker compose command to enter shell + * executes it + + :param verbose: print commands when running + :param dry_run: do not execute "write" commands - just print what would happen + :param shell_params: parameters of the execution + """ + check_docker_resources(verbose, shell_params.airflow_image_name) build_ci_image_check_cache = Path( - BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python_version}" + BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python}" ) if build_ci_image_check_cache.exists(): - console.print(f'{shell_params.the_image_type} image already built locally.') + console.print(f'[bright_blue]{shell_params.the_image_type} image already built locally.[/]') else: - console.print(f'{shell_params.the_image_type} image not built locally') + console.print( + f'[bright_yellow]{shell_params.the_image_type} image not built locally. ' f'Forcing build.[/]' + ) + shell_params.force_build = True if not shell_params.force_build: - build_image_if_needed_steps(verbose, shell_params) + build_image_if_needed_steps(verbose, dry_run, shell_params) else: build_image( verbose, - python_version=shell_params.python_version, + dry_run=dry_run, + python=shell_params.python, upgrade_to_newer_dependencies="false", ) - - instruct_for_setup() - check_docker_resources(verbose, str(shell_params.airflow_sources), shell_params.airflow_ci_image_name) - cmd = ['docker-compose', 'run', '--service-ports', '--rm', 'airflow'] + shell_params.print_badge_info() + cmd = ['docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm', 'airflow'] cmd_added = shell_params.command_passed env_variables = construct_env_variables_docker_compose_command(shell_params) if cmd_added is not None: cmd.extend(['-c', cmd_added]) - if verbose: - shell_params.print_badge_info() - output = run_command(cmd, verbose=verbose, env=env_variables, text=True) - if verbose: - console.print(f"[blue]{output}[/]") - - -def get_cached_params(user_params) -> Dict: - updated_params = dict(user_params) - for param in PARAMS_IN_CACHE: - if param in user_params: - param_name = PARAMS_IN_CACHE[param] - user_param_value = user_params[param] + run_command(cmd, verbose=verbose, dry_run=dry_run, env=env_variables, text=True) + + +def synchronize_cached_params(parameters_passed_by_the_user: Dict[str, str]) -> Dict[str, str]: + """ + Synchronizes cached params with arguments passed via dictionary. + + It will read from cache parameters that are missing and writes back in case user + actually provided new values for those parameters. It synchronizes all cacheable parameters. + + :param parameters_passed_by_the_user: user args passed + :return: updated args + """ + updated_params = dict(parameters_passed_by_the_user) + for param in VARIABLES_IN_CACHE: + if param in parameters_passed_by_the_user: + param_name = VARIABLES_IN_CACHE[param] + user_param_value = parameters_passed_by_the_user[param] if user_param_value is not None: write_to_cache_file(param_name, user_param_value) else: - param_value = getattr(global_constants, DEFAULT_VALUES_FOR_PARAM[param]) - _, user_param_value = check_cache_and_write_if_not_cached(param_name, param_value) + param_value = getattr(global_constants, SOURCE_OF_DEFAULT_VALUES_FOR_VARIABLES[param]) + _, user_param_value = check_cached_value_is_allowed(param_name, param_value) updated_params[param] = user_param_value return updated_params -def build_shell(verbose, **kwargs): +def enter_shell(**kwargs): + """ + Executes entering shell using the parameters passed as kwargs: + + * checks if docker version is good + * checks if docker-compose version is good + * updates kwargs with cached parameters + * displays ASCIIART and CHEATSHEET unless disabled + * build ShellParams from the updated kwargs + * executes the command to drop the user to Breeze shell + + """ + verbose = kwargs['verbose'] + dry_run = kwargs['dry_run'] check_docker_version(verbose) check_docker_compose_version(verbose) - updated_kwargs = get_cached_params(kwargs) + updated_kwargs = synchronize_cached_params(kwargs) if read_from_cache_file('suppress_asciiart') is None: console.print(ASCIIART, style=ASCIIART_STYLE) if read_from_cache_file('suppress_cheatsheet') is None: console.print(CHEATSHEET, style=CHEATSHEET_STYLE) - enter_shell_params = ShellBuilder(**filter_out_none(**updated_kwargs)) - build_image_checks(verbose, enter_shell_params) + enter_shell_params = ShellParams(**filter_out_none(**updated_kwargs)) + run_shell_with_build_image_checks(verbose, dry_run, enter_shell_params) diff --git a/dev/breeze/src/airflow_breeze/shell/shell_builder.py b/dev/breeze/src/airflow_breeze/shell/shell_params.py similarity index 73% rename from dev/breeze/src/airflow_breeze/shell/shell_builder.py rename to dev/breeze/src/airflow_breeze/shell/shell_params.py index 051b2d3936eee..4975fa6b34f81 100644 --- a/dev/breeze/src/airflow_breeze/shell/shell_builder.py +++ b/dev/breeze/src/airflow_breeze/shell/shell_params.py @@ -14,49 +14,63 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +"""Breeze shell paameters.""" from dataclasses import dataclass from pathlib import Path from typing import Tuple from airflow_breeze.branch_defaults import AIRFLOW_BRANCH -from airflow_breeze.console import console -from airflow_breeze.global_constants import AVAILABLE_INTEGRATIONS, get_airflow_version +from airflow_breeze.global_constants import ( + ALLOWED_BACKENDS, + ALLOWED_MSSQL_VERSIONS, + ALLOWED_MYSQL_VERSIONS, + ALLOWED_POSTGRES_VERSIONS, + ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, + AVAILABLE_INTEGRATIONS, + MOUNT_ALL, + MOUNT_SELECTED, + get_airflow_version, +) +from airflow_breeze.utils.console import console from airflow_breeze.utils.host_info_utils import get_host_group_id, get_host_user_id, get_stat_bin -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE, BUILD_CACHE_DIR, SCRIPTS_CI_DIR +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, BUILD_CACHE_DIR, SCRIPTS_CI_DIR from airflow_breeze.utils.run_utils import get_filesystem_type, run_command @dataclass -class ShellBuilder: - python_version: str # check in cache - build_cache_local: bool - build_cache_pulled: bool - build_cache_disabled: bool - backend: str # check in cache - integration: Tuple[str] # check in cache - postgres_version: str # check in cache - mssql_version: str # check in cache - mysql_version: str # check in cache - force_build: bool - extra_args: Tuple +class ShellParams: + """ + Shell parameters. Those parameters are used to determine command issued to run shell command. + """ + + verbose: bool + extra_args: Tuple = () + force_build: bool = False + integration: Tuple[str, ...] = () + postgres_version: str = ALLOWED_POSTGRES_VERSIONS[0] + mssql_version: str = ALLOWED_MSSQL_VERSIONS[0] + mysql_version: str = ALLOWED_MYSQL_VERSIONS[0] + backend: str = ALLOWED_BACKENDS[0] + python: str = ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0] + dry_run: bool = False + load_example_dags: bool = False + load_default_connections: bool = False use_airflow_version: str = "" install_airflow_version: str = "" tag: str = "latest" github_repository: str = "apache/airflow" - skip_mounting_local_sources: bool = False - mount_all_local_sources: bool = False + mount_sources: str = MOUNT_SELECTED forward_credentials: str = "false" airflow_branch: str = AIRFLOW_BRANCH - executor: str = "KubernetesExecutor" # check in cache start_airflow: str = "false" skip_twine_check: str = "" - use_packages_from_dist: str = "false" github_actions: str = "" issue_id: str = "" num_runs: str = "" version_suffix_for_pypi: str = "" version_suffix_for_svn: str = "" + db_reset: bool = False + ci: bool = False @property def airflow_version(self): @@ -64,7 +78,7 @@ def airflow_version(self): @property def airflow_version_for_production_image(self): - cmd = ['docker', 'run', '--entrypoint', '/bin/bash', f'{self.airflow_prod_image_name}'] + cmd = ['docker', 'run', '--entrypoint', '/bin/bash', f'{self.airflow_image_name}'] cmd.extend(['-c', 'echo "${AIRFLOW_VERSION}"']) output = run_command(cmd, capture_output=True, text=True) return output.stdout.strip() @@ -78,51 +92,29 @@ def host_group_id(self): return get_host_group_id() @property - def airflow_image_name(self) -> str: + def airflow_base_image_name(self) -> str: image = f'ghcr.io/{self.github_repository.lower()}' return image @property - def airflow_ci_image_name(self) -> str: + def airflow_image_name(self) -> str: """Construct CI image link""" - image = f'{self.airflow_image_name}/{self.airflow_branch}/ci/python{self.python_version}' + image = f'{self.airflow_base_image_name}/{self.airflow_branch}/ci/python{self.python}' return image @property def airflow_ci_image_name_with_tag(self) -> str: - image = self.airflow_ci_image_name + image = self.airflow_image_name return image if not self.tag else image + f":{self.tag}" - @property - def airflow_prod_image_name(self) -> str: - image = f'{self.airflow_image_name}/{self.airflow_branch}/prod/python{self.python_version}' - return image - @property def airflow_image_kubernetes(self) -> str: - image = f'{self.airflow_image_name}/{self.airflow_branch}/kubernetes/python{self.python_version}' + image = f'{self.airflow_base_image_name}/{self.airflow_branch}/kubernetes/python{self.python}' return image @property def airflow_sources(self): - return AIRFLOW_SOURCE - - @property - def docker_cache(self) -> str: - if self.build_cache_local: - docker_cache = "local" - elif self.build_cache_disabled: - docker_cache = "disabled" - else: - docker_cache = "pulled" - return docker_cache - - @property - def mount_selected_local_sources(self) -> bool: - mount_selected_local_sources = True - if self.mount_all_local_sources or self.skip_mounting_local_sources: - mount_selected_local_sources = False - return mount_selected_local_sources + return AIRFLOW_SOURCES_ROOT @property def enabled_integrations(self) -> str: @@ -139,14 +131,9 @@ def the_image_type(self) -> str: the_image_type = 'CI' return the_image_type - @property - def image_description(self) -> str: - image_description = 'Airflow CI' - return image_description - @property def md5sum_cache_dir(self) -> Path: - cache_dir = Path(BUILD_CACHE_DIR, self.airflow_branch, self.python_version, self.the_image_type) + cache_dir = Path(BUILD_CACHE_DIR, self.airflow_branch, self.python, self.the_image_type) return cache_dir @property @@ -170,7 +157,7 @@ def print_badge_info(self): console.print(f'Branch Name: {self.airflow_branch}') console.print(f'Docker Image: {self.airflow_ci_image_name_with_tag}') console.print(f'Airflow source version:{self.airflow_version}') - console.print(f'Python Version: {self.python_version}') + console.print(f'Python Version: {self.python}') console.print(f'Backend: {self.backend} {self.backend_version}') console.print(f'Airflow used at runtime: {self.use_airflow_version}') @@ -202,10 +189,13 @@ def compose_files(self): [main_ci_docker_compose_file, backend_docker_compose_file, files_docker_compose_file] ) - if self.mount_selected_local_sources: - compose_ci_file.extend([local_docker_compose_file, backend_port_docker_compose_file]) - if self.mount_all_local_sources: - compose_ci_file.extend([local_all_sources_docker_compose_file, backend_port_docker_compose_file]) + if self.mount_sources == MOUNT_SELECTED: + compose_ci_file.extend([local_docker_compose_file]) + elif self.mount_sources == MOUNT_ALL: + compose_ci_file.extend([local_all_sources_docker_compose_file]) + else: # none + compose_ci_file.extend([remove_sources_docker_compose_file]) + compose_ci_file.extend([backend_port_docker_compose_file]) if self.forward_credentials: compose_ci_file.append(forward_credentials_docker_compose_file) if len(self.use_airflow_version) > 0: diff --git a/dev/breeze/src/airflow_breeze/utils/__init__.py b/dev/breeze/src/airflow_breeze/utils/__init__.py index 13a83393a9124..73561b8457810 100644 --- a/dev/breeze/src/airflow_breeze/utils/__init__.py +++ b/dev/breeze/src/airflow_breeze/utils/__init__.py @@ -14,3 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Utils used internally by Breeze.""" diff --git a/dev/breeze/src/airflow_breeze/utils/cache.py b/dev/breeze/src/airflow_breeze/utils/cache.py new file mode 100644 index 0000000000000..bc22516c41f67 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/cache.py @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Some of the arguments ("Python/Backend/Versions of the backend) are cached locally in +".build" folder so that the last used value is used in the subsequent run if not specified. + +This allows to not remember what was the last version of Python used, if you just want to enter +the shell with the same version as the "previous run". +""" + +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from airflow_breeze import global_constants +from airflow_breeze.utils.console import console +from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR + + +def check_if_cache_exists(param_name: str) -> bool: + return (Path(BUILD_CACHE_DIR) / f".{param_name}").exists() + + +def read_from_cache_file(param_name: str) -> Optional[str]: + cache_exists = check_if_cache_exists(param_name) + if cache_exists: + return (Path(BUILD_CACHE_DIR) / f".{param_name}").read_text().strip() + else: + return None + + +def touch_cache_file(param_name: str, root_dir: Path = BUILD_CACHE_DIR): + (Path(root_dir) / f".{param_name}").touch() + + +def write_to_cache_file(param_name: str, param_value: str, check_allowed_values: bool = True) -> None: + """ + Writs value to cache. If asked it can also check if the value is allowed for the parameter. and exit + in case the value is not allowed for that parameter instead of writing it. + :param param_name: name of the parameter + :param param_value: new value for the parameter + :param check_allowed_values: whether to fail if the parameter value is not allowed for that name. + """ + allowed = False + allowed_values = None + if check_allowed_values: + allowed, allowed_values = check_if_values_allowed(param_name, param_value) + if allowed or not check_allowed_values: + print('BUILD CACHE DIR:', BUILD_CACHE_DIR) + cache_path = Path(BUILD_CACHE_DIR, f".{param_name}") + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(param_value) + else: + console.print(f'[cyan]You have sent the {param_value} for {param_name}') + console.print(f'[cyan]Allowed value for the {param_name} are {allowed_values}') + console.print('[cyan]Provide one of the supported params. Write to cache dir failed') + sys.exit(1) + + +def check_cached_value_is_allowed(param_name: str, default_param_value: str) -> Tuple[bool, Optional[str]]: + """ + Checks if the cache is present and whether its value is valid according to current rules. + It could happen that the allowed values have been modified since the last time cached value was set, + so this check is crucial to check outdated values. + If the value is not set or in case the cached value stored is not currently allowed, + the default value is stored in the cache and returned instead. + + :param param_name: name of the parameter + :param default_param_value: default value of the parameter + :return: Tuple informing whether the value was read from cache and the parameter value that is + set in the cache after this method returns. + """ + is_from_cache = False + cached_value = read_from_cache_file(param_name) + if cached_value is None: + write_to_cache_file(param_name, default_param_value) + cached_value = default_param_value + else: + allowed, allowed_values = check_if_values_allowed(param_name, cached_value) + if allowed: + is_from_cache = True + else: + write_to_cache_file(param_name, default_param_value) + cached_value = default_param_value + return is_from_cache, cached_value + + +def check_if_values_allowed(param_name: str, param_value: str) -> Tuple[bool, List[Any]]: + """Checks if parameter value is allowed by looking at global constants.""" + allowed = False + allowed_values = getattr(global_constants, f'ALLOWED_{param_name.upper()}S') + if param_value in allowed_values: + allowed = True + return allowed, allowed_values + + +def delete_cache(param_name: str) -> bool: + """Deletes value from cache. Returns true if the delete operation happened (i.e. cache was present).""" + deleted = False + if check_if_cache_exists(param_name): + (Path(BUILD_CACHE_DIR) / f".{param_name}").unlink() + deleted = True + return deleted + + +def synchronize_parameters_with_cache( + image_params: Any, parameters_passed_via_command_line: Dict[str, str] +) -> None: + """ + Synchronizes cacheable parameters between executions. It reads values from cache and updates + them wen done. It is only done for parameters that are relevant for image build. + + :param image_params: parameters of the build + :param parameters_passed_via_command_line: parameters that were passed by command line + """ + cacheable_parameters = { + 'python': 'PYTHON_MAJOR_MINOR_VERSION', + } + for parameter, cache_key in cacheable_parameters.items(): + value_from_parameter = parameters_passed_via_command_line.get(parameter) + if value_from_parameter: + write_to_cache_file(cache_key, value_from_parameter, check_allowed_values=True) + setattr(image_params, parameter, value_from_parameter) + else: + is_cached, value = check_cached_value_is_allowed(cache_key, getattr(image_params, parameter)) + if is_cached: + setattr(image_params, parameter, value) diff --git a/dev/breeze/src/airflow_breeze/docs_generator/doc_builder.py b/dev/breeze/src/airflow_breeze/utils/console.py similarity index 54% rename from dev/breeze/src/airflow_breeze/docs_generator/doc_builder.py rename to dev/breeze/src/airflow_breeze/utils/console.py index 4759c2c2d8f8b..ab7679f0f0a09 100644 --- a/dev/breeze/src/airflow_breeze/docs_generator/doc_builder.py +++ b/dev/breeze/src/airflow_breeze/utils/console.py @@ -14,25 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +Console used by all processes. We are forcing colors and terminal output as Breeze is supposed +to be only run in CI or real development terminal - in both cases we want to have colors on. +""" +try: + from rich.console import Console + from rich.theme import Theme -from dataclasses import dataclass -from typing import List, Tuple + custom_theme = Theme({"info": "blue", "warning": "magenta", "error": "red"}) + console = Console(force_terminal=True, color_system="standard", width=180, theme=custom_theme) - -@dataclass -class DocBuilder: - package_filter: Tuple[str] - docs_only: bool - spellcheck_only: bool - - @property - def args_doc_builder(self) -> List[str]: - doc_args = [] - if self.docs_only: - doc_args.append("--docs-only") - if self.spellcheck_only: - doc_args.append("--spellcheck-only") - if self.package_filter and len(self.package_filter) > 0: - for single_filter in self.package_filter: - doc_args.extend(["--package-filter", single_filter]) - return doc_args +except ImportError: + # We handle the ImportError so that autocomplete works with just click installed + custom_theme = None # type: ignore[assignment] + console = None # type: ignore[assignment] diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 33fcf72a42afe..70e5ebb302db2 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -14,20 +14,44 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Various utils to prepare docker and docker compose commands.""" +import os import re import subprocess -from typing import List +from typing import Dict, List, Union -from packaging import version +from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams +from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams +from airflow_breeze.shell.shell_params import ShellParams +from airflow_breeze.utils.host_info_utils import get_host_os +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT -from airflow_breeze.console import console +try: + from packaging import version +except ImportError: + # We handle the ImportError so that autocomplete works with just click installed + version = None # type: ignore[assignment] + +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( + FLOWER_HOST_PORT, MIN_DOCKER_COMPOSE_VERSION, MIN_DOCKER_VERSION, - MOUNT_ALL_LOCAL_SOURCES, - MOUNT_SELECTED_LOCAL_SOURCES, + MOUNT_ALL, + MOUNT_NONE, + MOUNT_SELECTED, + MSSQL_HOST_PORT, + MSSQL_VERSION, + MYSQL_HOST_PORT, + MYSQL_VERSION, + POSTGRES_HOST_PORT, + POSTGRES_VERSION, + REDIS_HOST_PORT, + SSH_PORT, + WEBSERVER_HOST_PORT, ) -from airflow_breeze.utils.run_utils import run_command +from airflow_breeze.utils.console import console +from airflow_breeze.utils.run_utils import commit_sha, prepare_build_command, run_command NECESSARY_HOST_VOLUMES = [ "/.bash_aliases:/root/.bash_aliases:cached", @@ -52,7 +76,7 @@ "/pyproject.toml:/opt/airflow/pyproject.toml:cached", "/pytest.ini:/opt/airflow/pytest.ini:cached", "/scripts:/opt/airflow/scripts:cached", - "/scripts/in_container/entrypoint_ci.sh:/entrypoint:cached", + "/scripts/docker/entrypoint_ci.sh:/entrypoint:cached", "/setup.cfg:/opt/airflow/setup.cfg:cached", "/setup.py:/opt/airflow/setup.py:cached", "/tests:/opt/airflow/tests:cached", @@ -63,46 +87,61 @@ ] -def get_extra_docker_flags(all: bool, selected: bool, airflow_sources: str) -> List: - # get_extra_docker_flags(False, str(airflow_source)) - # add verbosity - EXTRA_DOCKER_FLAGS = [] - if all: - EXTRA_DOCKER_FLAGS.extend(["-v", f"{airflow_sources}:/opt/airflow/:cached"]) - elif selected: +def get_extra_docker_flags(mount_sources: str) -> List[str]: + """ + Returns extra docker flags based on the type of mounting we want to do for sources. + :param mount_sources: type of mounting we want to have + :return: extra flag as list of strings + """ + extra_docker_flags = [] + if mount_sources == MOUNT_ALL: + extra_docker_flags.extend(["-v", f"{AIRFLOW_SOURCES_ROOT}:/opt/airflow/:cached"]) + elif mount_sources == MOUNT_SELECTED: for flag in NECESSARY_HOST_VOLUMES: - EXTRA_DOCKER_FLAGS.extend(["-v", airflow_sources + flag]) - else: - console.print('Skip mounting host volumes to Docker') - EXTRA_DOCKER_FLAGS.extend(["-v", f"{airflow_sources}/files:/files"]) - EXTRA_DOCKER_FLAGS.extend(["-v", f"{airflow_sources}/dist:/dist"]) - EXTRA_DOCKER_FLAGS.extend(["--rm"]) - EXTRA_DOCKER_FLAGS.extend(["--env-file", f"{airflow_sources}/scripts/ci/docker-compose/_docker.env"]) - return EXTRA_DOCKER_FLAGS + extra_docker_flags.extend(["-v", str(AIRFLOW_SOURCES_ROOT) + flag]) + else: # none + console.print('[bright_blue]Skip mounting host volumes to Docker[/]') + extra_docker_flags.extend(["-v", f"{AIRFLOW_SOURCES_ROOT}/files:/files"]) + extra_docker_flags.extend(["-v", f"{AIRFLOW_SOURCES_ROOT}/dist:/dist"]) + extra_docker_flags.extend(["--rm"]) + extra_docker_flags.extend(["--env-file", f"{AIRFLOW_SOURCES_ROOT}/scripts/ci/docker-compose/_docker.env"]) + return extra_docker_flags -def check_docker_resources(verbose: bool, airflow_sources: str, airflow_ci_image_name: str): - extra_docker_flags = get_extra_docker_flags( - MOUNT_ALL_LOCAL_SOURCES, MOUNT_SELECTED_LOCAL_SOURCES, airflow_sources - ) +def check_docker_resources(verbose: bool, airflow_image_name: str): + """ + Check if we have enough resources to run docker. This is done via running script embedded in our image. + :param verbose: print commands when running + :param airflow_image_name: name of the airflow image to use. + """ + extra_docker_flags = get_extra_docker_flags(MOUNT_NONE) cmd = [] cmd.extend(["docker", "run", "-t"]) cmd.extend(extra_docker_flags) - cmd.extend(["--entrypoint", "/bin/bash", airflow_ci_image_name]) + cmd.extend(["--entrypoint", "/bin/bash", airflow_image_name]) cmd.extend(["-c", "python /opt/airflow/scripts/in_container/run_resource_check.py"]) run_command(cmd, verbose=verbose, text=True) def check_docker_permission(verbose) -> bool: + """ + Checks if we have permission to write to docker socket. By default, on Linux you need to add your user + to docker group and some new users do not realize that. We help those users if we have + permission to run docker commands. + + :param verbose: print commands when running + :return: True if permission is denied. + """ permission_denied = False docker_permission_command = ["docker", "info"] try: _ = run_command( docker_permission_command, verbose=verbose, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, + check=True, ) except subprocess.CalledProcessError as ex: permission_denied = True @@ -120,6 +159,13 @@ def compare_version(current_version: str, min_version: str) -> bool: def check_docker_version(verbose: bool): + """ + Checks if the docker compose version is as expected (including some specific modifications done by + some vendors such as Microsoft (they might have modified version of docker-compose/docker in their + cloud. In case docker compose version is wrong we continue but print warning for the user. + + :param verbose: print commands when running + """ permission_denied = check_docker_permission(verbose) if not permission_denied: docker_version_command = ['docker', 'version', '--format', '{{.Client.Version}}'] @@ -127,7 +173,7 @@ def check_docker_version(verbose: bool): docker_version_output = run_command( docker_version_command, verbose=verbose, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) @@ -150,12 +196,19 @@ def check_docker_version(verbose: bool): def check_docker_compose_version(verbose: bool): + """ + Checks if the docker compose version is as expected (including some specific modifications done by + some vendors such as Microsoft (they might have modified version of docker-compose/docker in their + cloud. In case docker compose version is wrong we continue but print warning for the user. + + :param verbose: print commands when running + """ version_pattern = re.compile(r'(\d+)\.(\d+)\.(\d+)') docker_compose_version_command = ["docker-compose", "--version"] docker_compose_version_output = run_command( docker_compose_version_command, verbose=verbose, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) @@ -183,16 +236,193 @@ def check_docker_compose_version(verbose: bool): ) -def check_if_buildx_plugin_available(verbose: bool) -> bool: - is_buildx_available = False - check_buildx = ['docker', 'buildx', 'version'] - docker_buildx_version_output = run_command( - check_buildx, - verbose=verbose, - suppress_console_print=True, - capture_output=True, - text=True, +def construct_arguments_for_build_docker_command( + image_params: Union[BuildCiParams, BuildProdParams], required_args: List[str], optional_args: List[str] +) -> List[str]: + """ + Constructs docker compose command arguments list based on parameters passed + :param image_params: parameters of the image + :param required_args: build argument that are required + :param optional_args: build arguments that are optional (should not be used if missing or empty) + :return: list of `--build-arg` commands to use for the parameters passed + """ + args_command = [] + for param in required_args: + args_command.append("--build-arg") + args_command.append(param.upper() + "=" + str(getattr(image_params, param))) + for verify_param in optional_args: + param_value = str(getattr(image_params, verify_param)) + if len(param_value) > 0: + args_command.append("--build-arg") + args_command.append(verify_param.upper() + "=" + param_value) + args_command.extend(image_params.docker_cache_ci_directive) + return args_command + + +def construct_build_docker_command( + image_params: Union[BuildProdParams, BuildCiParams], + verbose: bool, + required_args: List[str], + optional_args: List[str], + production_image: bool, +) -> List[str]: + """ + Constructs docker compose command based on the parameters passed. + :param image_params: parameters of the image + :param verbose: print commands when running + :param required_args: build argument that are required + :param optional_args: build arguments that are optional (should not be used if missing or empty) + :param production_image: whether this is production image or ci image + :return: Command to run as list of string + """ + arguments = construct_arguments_for_build_docker_command( + image_params, required_args=required_args, optional_args=optional_args + ) + build_command = prepare_build_command( + prepare_buildx_cache=image_params.prepare_buildx_cache, verbose=verbose ) - if docker_buildx_version_output.returncode == 0 and docker_buildx_version_output.stdout != '': - is_buildx_available = True - return is_buildx_available + build_flags = image_params.extra_docker_build_flags + final_command = [] + final_command.extend(["docker"]) + final_command.extend(build_command) + final_command.extend(build_flags) + final_command.extend(["--pull"]) + final_command.extend(arguments) + final_command.extend(["-t", image_params.airflow_image_name, "--target", "main", "."]) + final_command.extend(["-f", 'Dockerfile' if production_image else 'Dockerfile.ci']) + final_command.extend(["--platform", image_params.platform]) + return final_command + + +def set_value_to_default_if_not_set(env: Dict[str, str], name: str, default: str): + if env.get(name) is None: + env[name] = os.environ.get(name, default) + + +def update_expected_environment_variables(env: Dict[str, str]) -> None: + """ + Updates default values for unset environment variables. + + :param env: environment variables to update with missing values if not set. + """ + set_value_to_default_if_not_set(env, 'BREEZE', "true") + set_value_to_default_if_not_set(env, 'CI', "false") + set_value_to_default_if_not_set(env, 'CI_BUILD_ID', "0") + set_value_to_default_if_not_set(env, 'CI_EVENT_TYPE', "pull_request") + set_value_to_default_if_not_set(env, 'CI_JOB_ID', "0") + set_value_to_default_if_not_set(env, 'CI_TARGET_BRANCH', AIRFLOW_BRANCH) + set_value_to_default_if_not_set(env, 'CI_TARGET_REPO', "apache/airflow") + set_value_to_default_if_not_set(env, 'COMMIT_SHA', commit_sha()) + set_value_to_default_if_not_set(env, 'DB_RESET', "false") + set_value_to_default_if_not_set(env, 'DEBIAN_VERSION', "bullseye") + set_value_to_default_if_not_set(env, 'DEFAULT_BRANCH', AIRFLOW_BRANCH) + set_value_to_default_if_not_set(env, 'DEFAULT_CONSTRAINTS_BRANCH', DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH) + set_value_to_default_if_not_set(env, 'ENABLED_SYSTEMS', "") + set_value_to_default_if_not_set(env, 'ENABLE_TEST_COVERAGE', "false") + set_value_to_default_if_not_set(env, 'GENERATE_CONSTRAINTS_MODE', "source-providers") + set_value_to_default_if_not_set(env, 'GITHUB_REGISTRY_PULL_IMAGE_TAG', "latest") + set_value_to_default_if_not_set(env, 'HOST_OS', get_host_os()) + set_value_to_default_if_not_set(env, 'INIT_SCRIPT_FILE', "init.sh") + set_value_to_default_if_not_set(env, 'INSTALL_PROVIDERS_FROM_SOURCES', "true") + set_value_to_default_if_not_set(env, 'INSTALL_PROVIDERS_FROM_SOURCES', "true") + set_value_to_default_if_not_set(env, 'LIST_OF_INTEGRATION_TESTS_TO_RUN', "") + set_value_to_default_if_not_set(env, 'LOAD_DEFAULT_CONNECTIONS', "false") + set_value_to_default_if_not_set(env, 'LOAD_EXAMPLES', "false") + set_value_to_default_if_not_set(env, 'PACKAGE_FORMAT', "wheel") + set_value_to_default_if_not_set(env, 'PRINT_INFO_FROM_SCRIPTS', "true") + set_value_to_default_if_not_set(env, 'PYTHONDONTWRITEBYTECODE', "true") + set_value_to_default_if_not_set(env, 'RUN_SYSTEM_TESTS', "false") + set_value_to_default_if_not_set(env, 'RUN_TESTS', "false") + set_value_to_default_if_not_set(env, 'SKIP_ENVIRONMENT_INITIALIZATION', "false") + set_value_to_default_if_not_set(env, 'SKIP_SSH_SETUP', "false") + set_value_to_default_if_not_set(env, 'TEST_TYPE', "") + set_value_to_default_if_not_set(env, 'UPGRADE_TO_NEWER_DEPENDENCIES', "false") + set_value_to_default_if_not_set(env, 'USE_PACKAGES_FROM_DIST', "false") + set_value_to_default_if_not_set(env, 'USE_PACKAGES_FROM_DIST', "false") + set_value_to_default_if_not_set(env, 'VERBOSE', "false") + set_value_to_default_if_not_set(env, 'VERBOSE_COMMANDS', "false") + set_value_to_default_if_not_set(env, 'WHEEL_VERSION', "0.36.2") + + +VARIABLES_TO_ENTER_DOCKER_COMPOSE = { + "AIRFLOW_CI_IMAGE": "airflow_image_name", + "AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag", + "AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes", + "AIRFLOW_PROD_IMAGE": "airflow_image_name", + "AIRFLOW_SOURCES": "airflow_sources", + "AIRFLOW_VERSION": "airflow_version", + "BACKEND": "backend", + "COMPOSE_FILE": "compose_files", + "DB_RESET": 'db_reset', + "ENABLED_INTEGRATIONS": "enabled_integrations", + "GITHUB_ACTIONS": "github_actions", + "HOST_GROUP_ID": "host_group_id", + "HOST_USER_ID": "host_user_id", + "INSTALL_AIRFLOW_VERSION": "install_airflow_version", + "ISSUE_ID": "issue_id", + "LOAD_EXAMPLES": "load_example_dags", + "LOAD_DEFAULT_CONNECTIONS": "load_default_connections", + "NUM_RUNS": "num_runs", + "PYTHON_MAJOR_MINOR_VERSION": "python", + "SKIP_TWINE_CHECK": "skip_twine_check", + "SQLITE_URL": "sqlite_url", + "START_AIRFLOW": "start_airflow", + "USE_AIRFLOW_VERSION": "use_airflow_version", + "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", + "VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn", +} + +VARIABLES_FOR_DOCKER_COMPOSE_CONSTANTS = { + "FLOWER_HOST_PORT": FLOWER_HOST_PORT, + "MSSQL_HOST_PORT": MSSQL_HOST_PORT, + "MSSQL_VERSION": MSSQL_VERSION, + "MYSQL_HOST_PORT": MYSQL_HOST_PORT, + "MYSQL_VERSION": MYSQL_VERSION, + "POSTGRES_HOST_PORT": POSTGRES_HOST_PORT, + "POSTGRES_VERSION": POSTGRES_VERSION, + "REDIS_HOST_PORT": REDIS_HOST_PORT, + "SSH_PORT": SSH_PORT, + "WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT, +} + +VARIABLES_IN_CACHE = { + 'backend': 'BACKEND', + 'mssql_version': 'MSSQL_VERSION', + 'mysql_version': 'MYSQL_VERSION', + 'postgres_version': 'POSTGRES_VERSION', + 'python': 'PYTHON_MAJOR_MINOR_VERSION', +} + +SOURCE_OF_DEFAULT_VALUES_FOR_VARIABLES = { + 'backend': 'DEFAULT_BACKEND', + 'mssql_version': 'MSSQL_VERSION', + 'mysql_version': 'MYSQL_VERSION', + 'postgres_version': 'POSTGRES_VERSION', + 'python': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION', +} + + +def construct_env_variables_docker_compose_command(shell_params: ShellParams) -> Dict[str, str]: + """ + Constructs environment variables needed by the docker-compose command, based on Shell parameters + passed to it. + + * It checks if appropriate params are defined for all the needed docker compose environment variables + * It sets the environment values from the parameters passed + * For the constant parameters that we do not have parameters for, we only override the constant values + if the env variable that we run with does not have it. + * Updates all other environment variables that docker-compose expects with default values if missing + + :param shell_params: shell parameters passed + :return: dictionary of env variables to set + """ + env_variables: Dict[str, str] = os.environ.copy() + for param_name in VARIABLES_TO_ENTER_DOCKER_COMPOSE: + param_value = VARIABLES_TO_ENTER_DOCKER_COMPOSE[param_name] + env_variables[param_name] = str(getattr(shell_params, param_value)) + for constant_param_name in VARIABLES_FOR_DOCKER_COMPOSE_CONSTANTS: + constant_param_value = VARIABLES_FOR_DOCKER_COMPOSE_CONSTANTS[constant_param_name] + if not env_variables.get(constant_param_value): + env_variables[constant_param_name] = str(constant_param_value) + update_expected_environment_variables(env_variables) + return env_variables diff --git a/dev/breeze/src/airflow_breeze/utils/host_info_utils.py b/dev/breeze/src/airflow_breeze/utils/host_info_utils.py index 07a91fe964173..3f34153de3382 100644 --- a/dev/breeze/src/airflow_breeze/utils/host_info_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/host_info_utils.py @@ -14,22 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +Returns information about Host that should be passed to the docker-compose. +""" import platform from airflow_breeze.utils.run_utils import run_command -# DIRECTORIES_TO_FIX=( -# "/files" -# "/root/.aws" -# "/root/.azure" -# "/root/.config/gcloud" -# "/root/.docker" -# "/opt/airflow/logs" -# "/opt/airflow/docs" -# "/opt/airflow/dags" -# "${AIRFLOW_SOURCE}" -# ) - def get_host_user_id(): host_user_id = '' diff --git a/dev/breeze/src/airflow_breeze/utils/md5_build_check.py b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py new file mode 100644 index 0000000000000..37191a1ef6a1a --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Utilities to check - with MD5 - whether files have been modified since the last successful build. +""" +import hashlib +from pathlib import Path +from typing import List, Tuple + +from airflow_breeze.global_constants import FILES_FOR_REBUILD_CHECK +from airflow_breeze.utils.console import console +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT + + +def check_md5checksum_in_cache_modified(file_hash: str, cache_path: Path, update: bool) -> bool: + """ + Check if the file hash is present in cache and it's content has been modified. Optionally updates + the hash. + + :param file_hash: hash of the current version of the file + :param cache_path: path where the hash is stored + :param update: whether to update hash if it is found different + :return: True if the hash file was missing or hash has changed. + """ + if cache_path.exists(): + old_md5_checksum_content = Path(cache_path).read_text() + if old_md5_checksum_content.strip() != file_hash.strip(): + if update: + save_md5_file(cache_path, file_hash) + return True + else: + if update: + save_md5_file(cache_path, file_hash) + return True + return False + + +def generate_md5(filename, file_size: int = 65536): + """Generates md5 hash for the file.""" + hash_md5 = hashlib.md5() + with open(filename, "rb") as f: + for file_chunk in iter(lambda: f.read(file_size), b""): + hash_md5.update(file_chunk) + return hash_md5.hexdigest() + + +def calculate_md5_checksum_for_files( + md5sum_cache_dir: Path, update: bool = False +) -> Tuple[List[str], List[str]]: + """ + Calculates checksums for all interesting files and stores the hashes in the md5sum_cache_dir. + Optionally modifies the hashes. + + :param md5sum_cache_dir: directory where to store cached information + :param update: whether to update the hashes + :return: Tuple of two lists: modified and not-modified files + """ + not_modified_files = [] + modified_files = [] + for calculate_md5_file in FILES_FOR_REBUILD_CHECK: + file_to_get_md5 = AIRFLOW_SOURCES_ROOT / calculate_md5_file + md5_checksum = generate_md5(file_to_get_md5) + sub_dir_name = file_to_get_md5.parts[-2] + actual_file_name = file_to_get_md5.parts[-1] + cache_file_name = Path(md5sum_cache_dir, sub_dir_name + '-' + actual_file_name + '.md5sum') + file_content = md5_checksum + ' ' + str(file_to_get_md5) + '\n' + is_modified = check_md5checksum_in_cache_modified(file_content, cache_file_name, update=update) + if is_modified: + modified_files.append(calculate_md5_file) + else: + not_modified_files.append(calculate_md5_file) + return modified_files, not_modified_files + + +def md5sum_check_if_build_is_needed(md5sum_cache_dir: Path, the_image_type: str) -> bool: + """ + Checks if build is needed based on whether important files were modified. + + :param md5sum_cache_dir: directory where cached md5 sums are stored + :param the_image_type: type of the image to check (PROD/CI) + :return: True if build is needed. + """ + build_needed = False + modified_files, not_modified_files = calculate_md5_checksum_for_files(md5sum_cache_dir, update=False) + if len(modified_files) > 0: + console.print( + '[bright_yellow]The following files are modified since last time image was built: [/]\n\n' + ) + for file in modified_files: + console.print(f" * [bright_blue]{file}[/]") + console.print(f'\n[bright_yellow]Likely {the_image_type} image needs rebuild[/]\n') + build_needed = True + else: + console.print( + f'Docker image build is not needed for {the_image_type} build as no important files are changed!' + ) + return build_needed + + +def save_md5_file(cache_path: Path, file_content: str) -> None: + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text(file_content) diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 81c08aa7bf140..6ce3f34def5bb 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -14,66 +14,83 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +Useful tools for various Paths used inside Airflow Sources. +""" + import os import tempfile from pathlib import Path from typing import Optional -from airflow_breeze.console import console - -__AIRFLOW_SOURCES_ROOT = Path.cwd() - -__AIRFLOW_CFG_FILE = "setup.cfg" - +from airflow_breeze.utils.console import console -def get_airflow_sources_root(): - return __AIRFLOW_SOURCES_ROOT +AIRFLOW_CFG_FILE = "setup.cfg" def search_upwards_for_airflow_sources_root(start_from: Path) -> Optional[Path]: root = Path(start_from.root) d = start_from while d != root: - attempt = d / __AIRFLOW_CFG_FILE + attempt = d / AIRFLOW_CFG_FILE if attempt.exists() and "name = apache-airflow\n" in attempt.read_text(): return attempt.parent d = d.parent return None -def find_airflow_sources_root(): +def find_airflow_sources_root() -> Path: + """ + Find the root of airflow sources. When Breeze is run from sources, it is easy, but this one also + has to handle the case when Breeze is installed via `pipx` so it searches upwards of the current + directory to find the right root of airflow directory. + + If not found, current directory is returned (this handles the case when Breeze is run from the local + directory. + + :return: Path for the found sources. + + """ + default_airflow_sources_root = Path.cwd() # Try to find airflow sources in current working dir airflow_sources_root = search_upwards_for_airflow_sources_root(Path.cwd()) if not airflow_sources_root: # Or if it fails, find it in parents of the directory where the ./breeze.py is. airflow_sources_root = search_upwards_for_airflow_sources_root(Path(__file__).resolve().parent) - global __AIRFLOW_SOURCES_ROOT if airflow_sources_root: - __AIRFLOW_SOURCES_ROOT = airflow_sources_root + os.chdir(airflow_sources_root) + return Path(airflow_sources_root) else: - console.print(f"\n[yellow]Could not find Airflow sources location. Assuming {__AIRFLOW_SOURCES_ROOT}") - os.chdir(__AIRFLOW_SOURCES_ROOT) + console.print( + f"\n[bright_yellow]Could not find Airflow sources location. " + f"Assuming {default_airflow_sources_root}" + ) + os.chdir(default_airflow_sources_root) + return Path(default_airflow_sources_root) + +AIRFLOW_SOURCES_ROOT = find_airflow_sources_root() -find_airflow_sources_root() -AIRFLOW_SOURCE = get_airflow_sources_root() -BUILD_CACHE_DIR = Path(AIRFLOW_SOURCE, '.build') -FILES_DIR = Path(AIRFLOW_SOURCE, 'files') -MSSQL_DATA_VOLUME = Path(BUILD_CACHE_DIR, 'tmp_mssql_volume') -MYPY_CACHE_DIR = Path(AIRFLOW_SOURCE, '.mypy_cache') -LOGS_DIR = Path(AIRFLOW_SOURCE, 'logs') -DIST_DIR = Path(AIRFLOW_SOURCE, 'dist') -SCRIPTS_CI_DIR = Path(AIRFLOW_SOURCE, 'scripts', 'ci') -DOCKER_CONTEXT_DIR = Path(AIRFLOW_SOURCE, 'docker-context-files') +BUILD_CACHE_DIR = AIRFLOW_SOURCES_ROOT / '.build' +FILES_DIR = AIRFLOW_SOURCES_ROOT / 'files' +MSSQL_DATA_VOLUME = AIRFLOW_SOURCES_ROOT / 'tmp_mssql_volume' +MYPY_CACHE_DIR = AIRFLOW_SOURCES_ROOT / '.mypy_cache' +LOGS_DIR = AIRFLOW_SOURCES_ROOT / 'logs' +DIST_DIR = AIRFLOW_SOURCES_ROOT / 'dist' +SCRIPTS_CI_DIR = AIRFLOW_SOURCES_ROOT / 'scripts' / 'ci' +DOCKER_CONTEXT_DIR = AIRFLOW_SOURCES_ROOT / 'docker-context-files' +CACHE_TMP_FILE_DIR = tempfile.TemporaryDirectory() +OUTPUT_LOG = Path(CACHE_TMP_FILE_DIR.name, 'out.log') -def create_directories(): +def create_directories() -> None: + """ + Creates all directories that are needed for Breeze to work. + """ BUILD_CACHE_DIR.mkdir(parents=True, exist_ok=True) FILES_DIR.mkdir(parents=True, exist_ok=True) MSSQL_DATA_VOLUME.mkdir(parents=True, exist_ok=True) MYPY_CACHE_DIR.mkdir(parents=True, exist_ok=True) LOGS_DIR.mkdir(parents=True, exist_ok=True) DIST_DIR.mkdir(parents=True, exist_ok=True) - CACHE_TMP_FILE_DIR = tempfile.TemporaryDirectory() - OUTPUT_LOG = Path(CACHE_TMP_FILE_DIR.name, 'out.log') OUTPUT_LOG.mkdir(parents=True, exist_ok=True) diff --git a/dev/breeze/src/airflow_breeze/utils/registry.py b/dev/breeze/src/airflow_breeze/utils/registry.py new file mode 100644 index 0000000000000..83323649b7f05 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/registry.py @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from typing import Any + +from airflow_breeze.utils.console import console +from airflow_breeze.utils.run_utils import run_command + + +def login_to_docker_registry(image_params: Any): + """ + In case of CI environment, we need to login to GitHub Registry if we want to prepare cache. + This method logs in using the params specified. + + :param image_params: parameters to use for Building prod image + """ + if os.environ.get("CI"): + if len(image_params.github_token) == 0: + console.print("\n[bright_blue]Skip logging in to GitHub Registry. No Token available!") + elif image_params.airflow_login_to_github_registry != "true": + console.print( + "\n[bright_blue]Skip logging in to GitHub Registry.\ + AIRFLOW_LOGIN_TO_GITHUB_REGISTRY is set as false" + ) + elif len(image_params.github_token) > 0: + run_command(['docker', 'logout', 'ghcr.io'], verbose=True, text=True) + run_command( + [ + 'docker', + 'login', + '--username', + image_params.github_username, + '--password-stdin', + 'ghcr.io', + ], + verbose=True, + text=True, + input=image_params.github_token, + ) + else: + console.print('\n[bright_blue]Skip Login to GitHub Container Registry as token is missing') diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 8553e3a8f28c1..da0ceeeb7eb94 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -14,23 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +"""Useful tools for running commands.""" import contextlib -import hashlib import os -import re import shlex import shutil import stat import subprocess -from copy import deepcopy +import sys +from functools import lru_cache from pathlib import Path from typing import Dict, List, Mapping, Optional -from airflow_breeze.cache import update_md5checksum_in_cache -from airflow_breeze.console import console -from airflow_breeze.global_constants import FILES_FOR_REBUILD_CHECK -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE +from airflow_breeze.utils.console import console +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT def run_command( @@ -38,67 +35,115 @@ def run_command( *, check: bool = True, verbose: bool = False, - suppress_raise_exception: bool = False, - suppress_console_print: bool = False, + dry_run: bool = False, + no_output_dump_on_exception: bool = False, env: Optional[Mapping[str, str]] = None, cwd: Optional[Path] = None, **kwargs, -): +) -> Optional[subprocess.CompletedProcess]: + """ + Runs command passed as list of strings with some extra functionality over POpen (kwargs from PoPen can + be used in this command even if not explicitly specified). + + It prints diagnostics when requested, also allows to "dry_run" the commands rather than actually + execute them. + + An important factor for having this command running tool is to be able (in verbose mode) to directly + copy&paste the verbose output and run the command manually - including all the environment variables + needed to run the command. + + :param cmd: command to run + :param check: whether to check status value and run exception (same as POpem) + :param verbose: print commands when running + :param dry_run: do not execute "the" command - just print what would happen + :param no_output_dump_on_exception: whether to suppress printing logs from output when command fails + :param env: mapping of environment variables to set for the run command + :param cwd: working directory to set for the command + :param kwargs: kwargs passed to POpen + """ workdir: str = str(cwd) if cwd else os.getcwd() - if verbose: + if verbose or dry_run: command_to_print = ' '.join(shlex.quote(c) for c in cmd) # if we pass environment variables to execute, then env_to_print = ' '.join(f'{key}="{val}"' for (key, val) in env.items()) if env else '' - console.print(f"\n[blue]Working directory {workdir} [/]\n") + if env_to_print: + env_to_print += ' ' + console.print(f"\n[bright_blue]Working directory {workdir} [/]\n") # Soft wrap allows to copy&paste and run resulting output as it has no hard EOL - console.print(f"\n[blue]{env_to_print} {command_to_print}[/]\n", soft_wrap=True) - + console.print(f"\n[bright_blue]{env_to_print}{command_to_print}[/]\n", soft_wrap=True) + if dry_run: + return None try: - # copy existing environment variables - cmd_env = deepcopy(os.environ) + cmd_env = os.environ.copy() if env: - # Add environment variables passed as parameters cmd_env.update(env) return subprocess.run(cmd, check=check, env=cmd_env, cwd=workdir, **kwargs) except subprocess.CalledProcessError as ex: - if not suppress_console_print: - console.print("========================= OUTPUT start ============================") - console.print(ex.stderr) - console.print(ex.stdout) - console.print("========================= OUTPUT end ============================") - if not suppress_raise_exception: + if not no_output_dump_on_exception: + if ex.stdout: + console.print("[blue]========================= OUTPUT start ============================[/]") + console.print(ex.stdout) + console.print("[blue]========================= OUTPUT end ==============================[/]") + if ex.stderr: + console.print("[red]========================= STDERR start ============================[/]") + console.print(ex.stderr) + console.print("[red]========================= STDERR end ==============================[/]") + if not check: raise + return None -def generate_md5(filename, file_size: int = 65536): - hash_md5 = hashlib.md5() - with open(filename, "rb") as f: - for file_chunk in iter(lambda: f.read(file_size), b""): - hash_md5.update(file_chunk) - return hash_md5.hexdigest() - - -def filter_out_none(**kwargs) -> Dict: - for key in list(kwargs): - if kwargs[key] is None: - kwargs.pop(key) - return kwargs +def check_pre_commit_installed(verbose: bool) -> bool: + """ + Check if pre-commit is installed in the right version. + :param verbose: print commands when running + :return: True is the pre-commit is installed in the right version. + """ + # Local import to make autocomplete work + import yaml + from pkg_resources import parse_version + pre_commit_config = yaml.safe_load((AIRFLOW_SOURCES_ROOT / ".pre-commit-config.yaml").read_text()) + min_pre_commit_version = pre_commit_config["minimum_pre_commit_version"] -def check_package_installed(package_name: str) -> bool: + pre_commit_name = "pre-commit" is_installed = False - if shutil.which('pre-commit') is not None: - is_installed = True - console.print(f"\n[blue]Package name {package_name} is installed to run static check test") - else: - console.print( - f"\n[red]Error: Package name {package_name} is not installed. \ - Please install using https://pre-commit.com/#install to continue[/]\n" + if shutil.which(pre_commit_name) is not None: + process = run_command( + [pre_commit_name, "--version"], verbose=verbose, check=True, capture_output=True, text=True ) + if process and process.stdout: + pre_commit_version = process.stdout.split(" ")[-1].strip() + if parse_version(pre_commit_version) >= parse_version(min_pre_commit_version): + console.print( + f"\n[green]Package {pre_commit_name} is installed. " + f"Good version {pre_commit_version} (>= {min_pre_commit_version})[/]\n" + ) + is_installed = True + else: + console.print( + f"\n[red]Package name {pre_commit_name} version is wrong. It should be" + f"aat least {min_pre_commit_version} and is {pre_commit_version}.[/]\n\n" + ) + else: + console.print( + "\n[bright_yellow]Could not determine version of pre-commit. " + "You might need to update it![/]\n" + ) + is_installed = True + else: + console.print(f"\n[red]Error: Package name {pre_commit_name} is not installed.[/]") + if not is_installed: + console.print("\nPlease install using https://pre-commit.com/#install to continue\n") return is_installed def get_filesystem_type(filepath): + """ + Determine the type of filesystem used - we might want to use different parameters if tmpfs is used. + :param filepath: path to check + :return: type of filesystem + """ # We import it locally so that click autocomplete works import psutil @@ -113,60 +158,21 @@ def get_filesystem_type(filepath): return root_type -def calculate_md5_checksum_for_files(md5sum_cache_dir: Path): - not_modified_files = [] - modified_files = [] - for calculate_md5_file in FILES_FOR_REBUILD_CHECK: - file_to_get_md5 = Path(AIRFLOW_SOURCE, calculate_md5_file) - md5_checksum = generate_md5(file_to_get_md5) - sub_dir_name = file_to_get_md5.parts[-2] - actual_file_name = file_to_get_md5.parts[-1] - cache_file_name = Path(md5sum_cache_dir, sub_dir_name + '-' + actual_file_name + '.md5sum') - file_content = md5_checksum + ' ' + str(file_to_get_md5) + '\n' - is_modified = update_md5checksum_in_cache(file_content, cache_file_name) - if is_modified: - modified_files.append(calculate_md5_file) - else: - not_modified_files.append(calculate_md5_file) - return modified_files, not_modified_files - - -def md5sum_check_if_build_is_needed(md5sum_cache_dir: Path, the_image_type: str) -> bool: - build_needed = False - modified_files, not_modified_files = calculate_md5_checksum_for_files(md5sum_cache_dir) - if len(modified_files) > 0: - console.print('The following files are modified: ', modified_files) - console.print(f'Likely {the_image_type} image needs rebuild') - build_needed = True - else: - console.print( - f'Docker image build is not needed for {the_image_type} build as no important files are changed!' - ) - return build_needed - - -def instruct_build_image(the_image_type: str, python_version: str): - console.print(f'\nThe {the_image_type} image for python version {python_version} may be outdated\n') +def instruct_build_image(python: str): + """Print instructions to the user that they should build the image""" + console.print(f'[bright_yellow]\nThe CI image for ' f'python version {python} may be outdated[/]\n') console.print('Please run this command at earliest convenience:\n') - if the_image_type == 'CI': - console.print(f'./Breeze2 build-ci-image --python {python_version}') - else: - console.print(f'./Breeze2 build-prod-image --python {python_version}') - console.print("\nIf you run it via pre-commit as individual hook, you can run 'pre-commit run build'.\n") - - -def instruct_for_setup(): - CMDNAME = 'Breeze2' - console.print(f"\nYou can setup autocomplete by running {CMDNAME} setup-autocomplete'") - console.print(" You can toggle ascii/cheatsheet by running:") - console.print(f" * {CMDNAME} toggle-suppress-cheatsheet") - console.print(f" * {CMDNAME} toggle-suppress-asciiart\n") + console.print(f' `./Breeze2 build-image --python {python}`\n') @contextlib.contextmanager def working_directory(source_path: Path): + """ # Equivalent of pushd and popd in bash script. # https://stackoverflow.com/a/42441759/3101838 + :param source_path: + :return: + """ prev_cwd = Path.cwd() os.chdir(source_path) try: @@ -176,6 +182,7 @@ def working_directory(source_path: Path): def change_file_permission(file_to_fix: Path): + """Update file permissions to not be group-writeable. Needed to solve cache invalidation problems.""" if file_to_fix.exists(): current = stat.S_IMODE(os.stat(file_to_fix).st_mode) new = current & ~stat.S_IWGRP & ~stat.S_IWOTH # Removes group/other write permission @@ -183,6 +190,7 @@ def change_file_permission(file_to_fix: Path): def change_directory_permission(directory_to_fix: Path): + """Update directory permissions to not be group-writeable. Needed to solve cache invalidation problems.""" if directory_to_fix.exists(): current = stat.S_IMODE(os.stat(directory_to_fix).st_mode) new = current & ~stat.S_IWGRP & ~stat.S_IWOTH # Removes group/other write permission @@ -192,9 +200,10 @@ def change_directory_permission(directory_to_fix: Path): os.chmod(directory_to_fix, new) -@working_directory(AIRFLOW_SOURCE) +@working_directory(AIRFLOW_SOURCES_ROOT) def fix_group_permissions(): - console.print("[blue]Fixing group permissions[/]") + """Fixes permissions of all the files and directories that have group-write access.""" + console.print("[bright_blue]Fixing group permissions[/]") files_to_fix_result = run_command(['git', 'ls-files', './'], capture_output=True, text=True) if files_to_fix_result.returncode == 0: files_to_fix = files_to_fix_result.stdout.strip().split('\n') @@ -209,29 +218,95 @@ def fix_group_permissions(): change_directory_permission(Path(directory_to_fix)) -def get_latest_sha(repo: str, branch: str): +def is_repo_rebased(repo: str, branch: str): + """Returns True if the local branch contains latest remote SHA (i.e. if it is rebased)""" # We import it locally so that click autocomplete works import requests gh_url = f"https://api.github.com/repos/{repo}/commits/{branch}" headers_dict = {"Accept": "application/vnd.github.VERSION.sha"} - resp = requests.get(gh_url, headers=headers_dict) - return resp.text - - -def is_repo_rebased(latest_sha: str): + latest_sha = requests.get(gh_url, headers=headers_dict).text.strip() rebased = False - output = run_command(['git', 'log', '--format=format:%H'], capture_output=True, text=True) - output = output.stdout.strip().splitlines() + process = run_command(['git', 'log', '--format=format:%H'], capture_output=True, text=True) + output = process.stdout.strip().splitlines() if process is not None else "missing" if latest_sha in output: rebased = True return rebased -def is_multi_platform(value: str) -> bool: - is_multi_platform = False - platform_pattern = re.compile('^[0-9a-zA-Z]+,[0-9a-zA-Z]+$') - platform_found = platform_pattern.search(value) - if platform_found is not None: - is_multi_platform = True - return is_multi_platform +def check_if_buildx_plugin_installed(verbose: bool) -> bool: + """ + Checks if buildx plugin is locally available. + :param verbose: print commands when running + :return True if the buildx plugin is installed. + """ + is_buildx_available = False + check_buildx = ['docker', 'buildx', 'version'] + docker_buildx_version_process = run_command( + check_buildx, + verbose=verbose, + no_output_dump_on_exception=True, + capture_output=True, + text=True, + ) + if ( + docker_buildx_version_process + and docker_buildx_version_process.returncode == 0 + and docker_buildx_version_process.stdout != '' + ): + is_buildx_available = True + return is_buildx_available + + +def prepare_build_command(prepare_buildx_cache: bool, verbose: bool) -> List[str]: + """ + Prepare build command for docker build. Depending on whether we have buildx plugin installed or not, + and whether we run cache preparation, there might be different results: + + * if buildx plugin is installed - `docker buildx` command is returned - using regular or cache builder + depending on whether we build regular image or cache + * if no buildx plugin is installed, and we do not prepare cache, regular docker `build` command is used. + * if no buildx plugin is installed, and we prepare cache - we fail. Cache can only be done with buildx + :param prepare_buildx_cache: whether we are preparing buildx cache. + :param verbose: print commands when running + :return: command to use as docker build command + """ + build_command_param = [] + is_buildx_available = check_if_buildx_plugin_installed(verbose=verbose) + if is_buildx_available: + if prepare_buildx_cache: + build_command_param.extend(["buildx", "build", "--builder", "airflow_cache", "--progress=tty"]) + cmd = ['docker', 'buildx', 'inspect', 'airflow_cache'] + process = run_command(cmd, verbose=True, text=True) + if process and process.returncode != 0: + next_cmd = ['docker', 'buildx', 'create', '--name', 'airflow_cache'] + run_command(next_cmd, verbose=True, text=True, check=False) + else: + build_command_param.extend(["buildx", "build", "--builder", "default", "--progress=tty"]) + else: + if prepare_buildx_cache: + console.print( + '\n[red] Buildx cli plugin is not available and you need it to prepare buildx cache. \n' + ) + console.print( + '[red] Please install it following https://docs.docker.com/buildx/working-with-buildx/ \n' + ) + sys.exit(1) + build_command_param.append("build") + return build_command_param + + +@lru_cache(maxsize=None) +def commit_sha(): + """Returns commit SHA of current repo. Cached for various usages.""" + return run_command( + ['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, check=False + ).stdout.strip() + + +def filter_out_none(**kwargs) -> Dict[str, str]: + """Filters out all None values from parameters passed.""" + for key in list(kwargs): + if kwargs[key] is None: + kwargs.pop(key) + return kwargs diff --git a/dev/breeze/src/airflow_breeze/utils/visuals.py b/dev/breeze/src/airflow_breeze/utils/visuals.py new file mode 100644 index 0000000000000..b5148a2c74b26 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/visuals.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Visuals displayed to the user when entering Breeze shell. +""" + +from airflow_breeze.global_constants import ( + FLOWER_HOST_PORT, + MSSQL_HOST_PORT, + MYSQL_HOST_PORT, + POSTGRES_HOST_PORT, + REDIS_HOST_PORT, + SSH_PORT, + WEBSERVER_HOST_PORT, +) +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT + +ASCIIART = """ + + + + + @&&&&&&@ + @&&&&&&&&&&&@ + &&&&&&&&&&&&&&&& + &&&&&&&&&& + &&&&&&& + &&&&&&& + @@@@@@@@@@@@@@@@ &&&&&& + @&&&&&&&&&&&&&&&&&&&&&&&&&& + &&&&&&&&&&&&&&&&&&&&&&&&&&&& + &&&&&&&&&&&& + &&&&&&&&& + &&&&&&&&&&&& + @@&&&&&&&&&&&&&&&@ + @&&&&&&&&&&&&&&&&&&&&&&&&&&&& &&&&&& + &&&&&&&&&&&&&&&&&&&&&&&&&&&& &&&&&& + &&&&&&&&&&&&&&&&&&&&&&&& &&&&&& + &&&&&& + &&&&&&& + @&&&&&&&& + @&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& + &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& + &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& + + + + @&&&@ && @&&&&&&&&&&& &&&&&&&&&&&& && &&&&&&&&&& &&& &&& &&& + &&& &&& && @&& &&& && && &&& &&&@ &&& &&&&& &&& + &&& &&& && @&&&&&&&&&&&& &&&&&&&&&&& && && &&& &&& &&& &&@ &&& + &&&&&&&&&&& && @&&&&&&&&& && && &&@ &&& &&@&& &&@&& + &&& &&& && @&& &&&@ && &&&&&&&&&&& &&&&&&&&&&&& &&&& &&&& + +&&&&&&&&&&&& &&&&&&&&&&&& &&&&&&&&&&&@ &&&&&&&&&&&& &&&&&&&&&&& &&&&&&&&&&& +&&& &&& && &&& && &&& &&&& && +&&&&&&&&&&&&@ &&&&&&&&&&&& &&&&&&&&&&& &&&&&&&&&&& &&&& &&&&&&&&&& +&&& && && &&&& && &&& &&&& && +&&&&&&&&&&&&& && &&&&@ &&&&&&&&&&&@ &&&&&&&&&&&& @&&&&&&&&&&& &&&&&&&&&&& + +""" +CHEATSHEET = f""" + + [bold][bright_blue]Airflow Breeze Cheatsheet[/][/] + + [bright_blue]* Installation[/] + + When you have multiple copies of Airflow, it's better if you use `./Breeze2` from those + repository as it will have the latest version of Breeze2 and it's dependencies. + + However if you only have one Airflow repository and you have `pipx` installed, you can use + `pipx` to install `Breeze2` command in your path (`Breeze2` command is run from this repository then) + + pipx install -e ./dev/breeze --force + + In case you use `pipx`, you might need to occasionally reinstall `Breeze2` with the `--force` flag + when dependencies change for it. You do not have to do it when you use it via `./Breeze2` + + [bright_blue]* Port forwarding:[/] + + Ports are forwarded to the running docker containers for webserver and database + * {SSH_PORT} -> forwarded to Airflow ssh server -> airflow:22 + * {WEBSERVER_HOST_PORT} -> forwarded to Airflow webserver -> airflow:8080 + * {FLOWER_HOST_PORT} -> forwarded to Flower dashboard -> airflow:5555 + * {POSTGRES_HOST_PORT} -> forwarded to Postgres database -> postgres:5432 + * {MYSQL_HOST_PORT} -> forwarded to MySQL database -> mysql:3306 + * {MSSQL_HOST_PORT} -> forwarded to MSSQL database -> mssql:1443 + * {REDIS_HOST_PORT} -> forwarded to Redis broker -> redis:6379 + + Direct links to those services that you can use from the host: + + * ssh connection for remote debugging: ssh -p {SSH_PORT} airflow@127.0.0.1 (password: airflow) + * Webserver: http://127.0.0.1:{WEBSERVER_HOST_PORT} + * Flower: http://127.0.0.1:{FLOWER_HOST_PORT} + * Postgres: jdbc:postgresql://127.0.0.1:{POSTGRES_HOST_PORT}/airflow?user=postgres&password=airflow + * Mysql: jdbc:mysql://127.0.0.1:{MYSQL_HOST_PORT}/airflow?user=root + * Redis: redis://127.0.0.1:{REDIS_HOST_PORT}/0 + + [bright_blue]* How can I add my stuff in Breeze:[/] + + * Your dags for webserver and scheduler are read from `/files/dags` directory + which is mounted from folder in Airflow sources: + * `{AIRFLOW_SOURCES_ROOT}/files/dags` + + * You can add `airflow-breeze-config` directory. Place it in + `{AIRFLOW_SOURCES_ROOT}/files/airflow-breeze-config` and: + * Add `variables.env` - to make breeze source the variables automatically for you + * Add `.tmux.conf` - to add extra initial configuration to `tmux` + * Add `init.sh` - this file will be sourced when you enter container, so you can add + any custom code there. + + * You can put any other files. You can add them in + `{AIRFLOW_SOURCES_ROOT}/files` folder + and they will be visible in `/files/` folder inside the container + + [bright_blue]* Other options[/] + + Check out `--help` for ./Breeze2 commands. It will show you other options, such as running + integration or starting complete Airflow using `start-airflow` command as well as ways + of cleaning up the installation. + + Make sure to run `setup-autocomplete` to get the commands and options auto-completable + in your shell. + + You can disable this cheatsheet by running: + + ./Breeze2 config --no-cheatsheet + +""" +CHEATSHEET_STYLE = "white" +ASCIIART_STYLE = "white" diff --git a/dev/breeze/src/airflow_breeze/visuals/__init__.py b/dev/breeze/src/airflow_breeze/visuals/__init__.py deleted file mode 100644 index f13a1af5c9717..0000000000000 --- a/dev/breeze/src/airflow_breeze/visuals/__init__.py +++ /dev/null @@ -1,103 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from airflow_breeze.global_constants import ( - FLOWER_HOST_PORT, - MSSQL_HOST_PORT, - MYSQL_HOST_PORT, - POSTGRES_HOST_PORT, - REDIS_HOST_PORT, - SSH_PORT, - WEBSERVER_HOST_PORT, -) -from airflow_breeze.utils.path_utils import get_airflow_sources_root - -ASCIIART = """ - - - - - @&&&&&&@ - @&&&&&&&&&&&@ - &&&&&&&&&&&&&&&& - &&&&&&&&&& - &&&&&&& - &&&&&&& - @@@@@@@@@@@@@@@@ &&&&&& - @&&&&&&&&&&&&&&&&&&&&&&&&&& - &&&&&&&&&&&&&&&&&&&&&&&&&&&& - &&&&&&&&&&&& - &&&&&&&&& - &&&&&&&&&&&& - @@&&&&&&&&&&&&&&&@ - @&&&&&&&&&&&&&&&&&&&&&&&&&&&& &&&&&& - &&&&&&&&&&&&&&&&&&&&&&&&&&&& &&&&&& - &&&&&&&&&&&&&&&&&&&&&&&& &&&&&& - &&&&&& - &&&&&&& - @&&&&&&&& - @&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& - &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& - &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& - - - - @&&&@ && @&&&&&&&&&&& &&&&&&&&&&&& && &&&&&&&&&& &&& &&& &&& - &&& &&& && @&& &&& && && &&& &&&@ &&& &&&&& &&& - &&& &&& && @&&&&&&&&&&&& &&&&&&&&&&& && && &&& &&& &&& &&@ &&& - &&&&&&&&&&& && @&&&&&&&&& && && &&@ &&& &&@&& &&@&& - &&& &&& && @&& &&&@ && &&&&&&&&&&& &&&&&&&&&&&& &&&& &&&& - -&&&&&&&&&&&& &&&&&&&&&&&& &&&&&&&&&&&@ &&&&&&&&&&&& &&&&&&&&&&& &&&&&&&&&&& -&&& &&& && &&& && &&& &&&& && -&&&&&&&&&&&&@ &&&&&&&&&&&& &&&&&&&&&&& &&&&&&&&&&& &&&& &&&&&&&&&& -&&& && && &&&& && &&& &&&& && -&&&&&&&&&&&&& && &&&&@ &&&&&&&&&&&@ &&&&&&&&&&&& @&&&&&&&&&&& &&&&&&&&&&& - -""" - -ASCIIART_STYLE = "white" - - -CHEATSHEET = f""" -Airflow Breeze CHEATSHEET -Adding breeze to your path: - When you exit the environment, you can add sources of Airflow to the path - you can - run breeze or the scripts above from any directory by calling 'breeze' commands directly - - \'{str(get_airflow_sources_root())}\' is exported into PATH - - Port forwarding: - Ports are forwarded to the running docker containers for webserver and database - * {SSH_PORT} -> forwarded to Airflow ssh server -> airflow:22 - * {WEBSERVER_HOST_PORT} -> forwarded to Airflow webserver -> airflow:8080 - * {FLOWER_HOST_PORT} -> forwarded to Flower dashboard -> airflow:5555 - * {POSTGRES_HOST_PORT} -> forwarded to Postgres database -> postgres:5432 - * {MYSQL_HOST_PORT} -> forwarded to MySQL database -> mysql:3306 - * {MSSQL_HOST_PORT} -> forwarded to MSSQL database -> mssql:1443 - * {REDIS_HOST_PORT} -> forwarded to Redis broker -> redis:6379 - Here are links to those services that you can use on host:" - * ssh connection for remote debugging: ssh -p {SSH_PORT} airflow@127.0.0.1 pw: airflow" - * Webserver: http://127.0.0.1:{WEBSERVER_HOST_PORT}" - * Flower: http://127.0.0.1:{FLOWER_HOST_PORT}" - * Postgres: jdbc:postgresql://127.0.0.1:{POSTGRES_HOST_PORT}/airflow?user=postgres&password=airflow" - * Mysql: jdbc:mysql://127.0.0.1:{MYSQL_HOST_PORT}/airflow?user=root" - * Redis: redis://127.0.0.1:{REDIS_HOST_PORT}/0" - -""" - -CHEATSHEET_STYLE = "white" diff --git a/dev/breeze/src/airflow_ci/find_newer_dependencies.py b/dev/breeze/src/airflow_ci/find_newer_dependencies.py index 7f6bbdc445cd0..7742ff291007e 100644 --- a/dev/breeze/src/airflow_ci/find_newer_dependencies.py +++ b/dev/breeze/src/airflow_ci/find_newer_dependencies.py @@ -14,7 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +Finds which newer dependencies were used to build that build and prints them for better diagnostics. +This is a common problem that currently `pip` does not produce "perfect" information about the errors, +and we sometimes need to guess what new dependency caused long backtracking. Once we know short +list of candidates, we can (following a manual process) pinpoint the actual culprit. + +This small tool is run in CI whenever the image build timed out - so that we can easier guess +which dependency caused the problem. + +The process to follow once you see the backtracking is described in: + +https://github.com/apache/airflow/blob/main/dev/TRACKING_BACKTRACKING_ISSUES.md +""" import json from datetime import timedelta from typing import Any, Dict, List, Tuple @@ -102,10 +115,10 @@ def main(constraints_branch: str, python: str, timezone: str, updated_on_or_afte hour=0, minute=0, second=0, microsecond=0 ) console.print( - "\n[yellow]Those are possible candidates that broke current " + "\n[bright_yellow]Those are possible candidates that broke current " "`pip` resolution mechanisms by falling back to long backtracking[/]\n" ) - console.print(f"\n[yellow]We are limiting to packages updated after {min_date} ({timezone})[/]\n") + console.print(f"\n[bright_yellow]We are limiting to packages updated after {min_date} ({timezone})[/]\n") with Progress(console=console) as progress: task = progress.add_task(f"Processing {count_packages} packages.", total=count_packages) for package_line in package_lines: @@ -122,14 +135,16 @@ def main(constraints_branch: str, python: str, timezone: str, updated_on_or_afte constrained_packages[package] = constraints_package_version progress.advance(task) progress.refresh() - console.print("\n[yellow]If you see long running builds with `pip` backtracking, you should follow[/]") console.print( - "[yellow]https://github.com/apache/airflow/blob/main/dev/TRACKING_BACKTRACKING_ISSUES.md[/]\n" + "\n[bright_yellow]If you see long running builds with `pip` backtracking, you should follow[/]" + ) + console.print( + "[bright_yellow]https://github.com/apache/airflow/blob/main/dev/TRACKING_BACKTRACKING_ISSUES.md[/]\n" ) constraint_string = "" for package, constrained_version in constrained_packages.items(): constraint_string += f' "{package}=={constrained_version}"' - console.print("[yellow]Use the following pip install command (see the doc above for details)\n") + console.print("[bright_yellow]Use the following pip install command (see the doc above for details)\n") console.print( 'pip install ".[devel_all]" --upgrade --upgrade-strategy eager ' '"dill<0.3.3" "certifi<2021.0.0" "google-ads<14.0.1"' + constraint_string, diff --git a/dev/breeze/src/airflow_ci/freespace.py b/dev/breeze/src/airflow_ci/freespace.py index bbec8c6d82328..cb24ac3a245f3 100755 --- a/dev/breeze/src/airflow_ci/freespace.py +++ b/dev/breeze/src/airflow_ci/freespace.py @@ -17,15 +17,14 @@ # specific language governing permissions and limitations # under the License. -"""freespace.py for clean environment before start CI""" +"""Cleans up the environment before starting CI.""" -import shlex -import subprocess -from typing import List import rich_click as click from rich.console import Console +from airflow_breeze.utils.run_utils import run_command + console = Console(force_terminal=True, color_system="standard", width=180) option_verbose = click.option( @@ -46,26 +45,14 @@ @option_verbose @option_dry_run def main(verbose, dry_run): - run_command(["sudo", "swapoff", "-a"], verbose, dry_run) - run_command(["sudo", "rm", "-f", "/swapfile"], verbose, dry_run) - run_command(["sudo", "apt-get", "clean"], verbose, dry_run, check=False) - run_command(["docker", "system", "prune", "--all", "--force", "--volumes"], verbose, dry_run) - run_command(["df", "-h"], verbose, dry_run) - run_command(["docker", "logout", "ghcr.io"], verbose, dry_run) - - -def run_command(cmd: List[str], verbose, dry_run, *, check: bool = True, **kwargs): - if verbose: - console.print(f"\n[green]$ {' '.join(shlex.quote(c) for c in cmd)}[/]\n") - if dry_run: - return - try: - subprocess.run(cmd, check=check, **kwargs) - except subprocess.CalledProcessError as ex: - print("========================= OUTPUT start ============================") - print(ex.stderr) - print(ex.stdout) - print("========================= OUTPUT end ============================") + run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run) + run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run) + run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False) + run_command( + ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run + ) + run_command(["df", "-h"], verbose=verbose, dry_run=dry_run) + run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run) if __name__ == '__main__': diff --git a/dev/breeze/tests/test_build_image.py b/dev/breeze/tests/test_build_image.py index f475657bd45ef..77c9c7759d751 100644 --- a/dev/breeze/tests/test_build_image.py +++ b/dev/breeze/tests/test_build_image.py @@ -18,30 +18,30 @@ import pytest -from airflow_breeze.ci.build_image import get_image_build_params +from airflow_breeze.build_image.ci.build_ci_image import get_ci_image_build_params @pytest.mark.parametrize( 'parameters, expected_build_params, cached_values, written_cache_version', [ - ({}, {"python_version": "3.7"}, {}, False), # default value no params - ({"python_version": "3.8"}, {"python_version": "3.8"}, {}, "3.8"), # default value override params - ({}, {"python_version": "3.8"}, {'PYTHON_MAJOR_MINOR_VERSION': "3.8"}, False), # value from cache + ({}, {"python": "3.7"}, {}, False), # default value no params + ({"python": "3.8"}, {"python": "3.8"}, {}, "3.8"), # default value override params + ({}, {"python": "3.8"}, {'PYTHON_MAJOR_MINOR_VERSION': "3.8"}, False), # value from cache ( - {"python_version": "3.9"}, - {"python_version": "3.9"}, + {"python": "3.9"}, + {"python": "3.9"}, {'PYTHON_MAJOR_MINOR_VERSION': "3.8"}, "3.9", ), # override cache with passed param ], ) def test_get_image_params(parameters, expected_build_params, cached_values, written_cache_version): - with patch('airflow_breeze.cache.read_from_cache_file') as read_from_cache_mock, patch( - 'airflow_breeze.cache.check_if_cache_exists' + with patch('airflow_breeze.utils.cache.read_from_cache_file') as read_from_cache_mock, patch( + 'airflow_breeze.utils.cache.check_if_cache_exists' ) as check_if_cache_exists_mock, patch( - 'airflow_breeze.ci.build_image.write_to_cache_file' + 'airflow_breeze.utils.cache.write_to_cache_file' ) as write_to_cache_file_mock, patch( - 'airflow_breeze.ci.build_image.check_cache_and_write_if_not_cached' + 'airflow_breeze.utils.cache.check_cached_value_is_allowed' ) as check_cache_and_write_mock: check_if_cache_exists_mock.return_value = True check_cache_and_write_mock.side_effect = lambda cache_key, default_value: ( @@ -49,7 +49,7 @@ def test_get_image_params(parameters, expected_build_params, cached_values, writ cached_values[cache_key] if cache_key in cached_values else default_value, ) read_from_cache_mock.side_effect = lambda param_name: cached_values.get(param_name) - build_parameters = get_image_build_params(parameters) + build_parameters = get_ci_image_build_params(parameters) for param, param_value in expected_build_params.items(): assert getattr(build_parameters, param) == param_value if written_cache_version: diff --git a/dev/breeze/tests/test_cache.py b/dev/breeze/tests/test_cache.py index 64bd6237cd51f..2e0931d2d5c01 100644 --- a/dev/breeze/tests/test_cache.py +++ b/dev/breeze/tests/test_cache.py @@ -20,7 +20,7 @@ import pytest -from airflow_breeze.cache import ( +from airflow_breeze.utils.cache import ( check_if_cache_exists, check_if_values_allowed, delete_cache, @@ -48,7 +48,7 @@ def test_allowed_values(parameter, value, result, exception): assert result == check_if_values_allowed(parameter, value) -@mock.patch("airflow_breeze.cache.Path") +@mock.patch("airflow_breeze.utils.cache.Path") def test_check_if_cache_exists(path): check_if_cache_exists("test_param") path.assert_called_once_with(AIRFLOW_SOURCES / ".build") @@ -72,8 +72,8 @@ def test_read_from_cache_file(param): assert param_value in param_list -@mock.patch('airflow_breeze.cache.Path') -@mock.patch('airflow_breeze.cache.check_if_cache_exists') +@mock.patch('airflow_breeze.utils.cache.Path') +@mock.patch('airflow_breeze.utils.cache.check_if_cache_exists') def test_delete_cache_exists(mock_check_if_cache_exists, mock_path): param = "MYSQL_VERSION" mock_check_if_cache_exists.return_value = True @@ -82,8 +82,8 @@ def test_delete_cache_exists(mock_check_if_cache_exists, mock_path): assert cache_deleted -@mock.patch('airflow_breeze.cache.Path') -@mock.patch('airflow_breeze.cache.check_if_cache_exists') +@mock.patch('airflow_breeze.utils.cache.Path') +@mock.patch('airflow_breeze.utils.cache.check_if_cache_exists') def test_delete_cache_not_exists(mock_check_if_cache_exists, mock_path): param = "TEST_PARAM" mock_check_if_cache_exists.return_value = False diff --git a/dev/breeze/tests/test_commands.py b/dev/breeze/tests/test_commands.py index 9d29251ea6aac..611675269943c 100644 --- a/dev/breeze/tests/test_commands.py +++ b/dev/breeze/tests/test_commands.py @@ -14,10 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +from airflow_breeze.global_constants import MOUNT_ALL, MOUNT_NONE, MOUNT_SELECTED from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags -from airflow_breeze.utils.path_utils import get_airflow_sources_root -from airflow_breeze.visuals import ASCIIART +from airflow_breeze.utils.visuals import ASCIIART def test_visuals(): @@ -25,13 +24,6 @@ def test_visuals(): def test_get_extra_docker_flags(): - airflow_sources = get_airflow_sources_root() - all = True - selected = False - assert len(get_extra_docker_flags(all, selected, str(airflow_sources))) < 10 - all = False - selected = True - assert len(get_extra_docker_flags(all, selected, str(airflow_sources))) > 60 - all = False - selected = False - assert len(get_extra_docker_flags(all, selected, str(airflow_sources))) < 8 + assert len(get_extra_docker_flags(MOUNT_ALL)) < 10 + assert len(get_extra_docker_flags(MOUNT_SELECTED)) > 60 + assert len(get_extra_docker_flags(MOUNT_NONE)) < 8 diff --git a/dev/breeze/tests/test_docker_command_utils.py b/dev/breeze/tests/test_docker_command_utils.py index 53df745501909..598e760fd585b 100644 --- a/dev/breeze/tests/test_docker_command_utils.py +++ b/dev/breeze/tests/test_docker_command_utils.py @@ -26,11 +26,18 @@ def test_check_docker_version_unknown(mock_console, mock_run_command): check_docker_version(verbose=True) expected_run_command_calls = [ - call(['docker', 'info'], verbose=True, suppress_console_print=True, capture_output=True, text=True), + call( + ['docker', 'info'], + verbose=True, + check=True, + no_output_dump_on_exception=True, + capture_output=True, + text=True, + ), call( ['docker', 'version', '--format', '{{.Client.Version}}'], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ), @@ -55,7 +62,7 @@ def test_check_docker_version_too_low(mock_console, mock_run_command, mock_check mock_run_command.assert_called_with( ['docker', 'version', '--format', '{{.Client.Version}}'], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) @@ -76,7 +83,7 @@ def test_check_docker_version_ok(mock_console, mock_run_command, mock_check_dock mock_run_command.assert_called_with( ['docker', 'version', '--format', '{{.Client.Version}}'], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) @@ -95,7 +102,7 @@ def test_check_docker_version_higher(mock_console, mock_run_command, mock_check_ mock_run_command.assert_called_with( ['docker', 'version', '--format', '{{.Client.Version}}'], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) @@ -110,7 +117,7 @@ def test_check_docker_compose_version_unknown(mock_console, mock_run_command): call( ["docker-compose", "--version"], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ), @@ -131,7 +138,7 @@ def test_check_docker_compose_version_low(mock_console, mock_run_command): mock_run_command.assert_called_with( ["docker-compose", "--version"], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) @@ -157,7 +164,7 @@ def test_check_docker_compose_version_ok(mock_console, mock_run_command): mock_run_command.assert_called_with( ["docker-compose", "--version"], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) @@ -173,7 +180,7 @@ def test_check_docker_compose_version_higher(mock_console, mock_run_command): mock_run_command.assert_called_with( ["docker-compose", "--version"], verbose=True, - suppress_console_print=True, + no_output_dump_on_exception=True, capture_output=True, text=True, ) diff --git a/dev/breeze/tests/test_find_airflow_directory.py b/dev/breeze/tests/test_find_airflow_directory.py index 90ac439895fb1..894c545875299 100644 --- a/dev/breeze/tests/test_find_airflow_directory.py +++ b/dev/breeze/tests/test_find_airflow_directory.py @@ -19,7 +19,7 @@ from pathlib import Path from unittest import mock -from airflow_breeze.utils.path_utils import find_airflow_sources_root, get_airflow_sources_root +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, find_airflow_sources_root ACTUAL_AIRFLOW_SOURCES = Path(__file__).parent.parent.parent.parent ROOT_PATH = Path(Path(__file__).root) @@ -28,7 +28,7 @@ def test_find_airflow_root_upwards_from_cwd(capsys): os.chdir(Path(__file__).parent) find_airflow_sources_root() - assert ACTUAL_AIRFLOW_SOURCES == get_airflow_sources_root() + assert ACTUAL_AIRFLOW_SOURCES == AIRFLOW_SOURCES_ROOT output = str(capsys.readouterr().out) assert output == '' @@ -36,16 +36,16 @@ def test_find_airflow_root_upwards_from_cwd(capsys): def test_find_airflow_root_upwards_from_file(capsys): os.chdir(Path(__file__).root) find_airflow_sources_root() - assert ACTUAL_AIRFLOW_SOURCES == get_airflow_sources_root() + assert ACTUAL_AIRFLOW_SOURCES == AIRFLOW_SOURCES_ROOT output = str(capsys.readouterr().out) assert output == '' -@mock.patch('airflow_breeze.utils.path_utils.__AIRFLOW_SOURCES_ROOT', ROOT_PATH) -@mock.patch('airflow_breeze.utils.path_utils.__AIRFLOW_CFG_FILE', "bad_name.cfg") -def test_fallback_find_airflow_root(capsys): - os.chdir(ROOT_PATH) - find_airflow_sources_root() - assert ROOT_PATH == get_airflow_sources_root() +@mock.patch('airflow_breeze.utils.path_utils.AIRFLOW_CFG_FILE', "bad_name.cfg") +@mock.patch('airflow_breeze.utils.path_utils.Path.cwd') +def test_fallback_find_airflow_root(mock_cwd, capsys): + mock_cwd.return_value = ROOT_PATH + sources = find_airflow_sources_root() + assert sources == ROOT_PATH output = str(capsys.readouterr().out) assert "Could not find Airflow sources" in output diff --git a/dev/breeze/tests/test_prod_image.py b/dev/breeze/tests/test_prod_image.py index 09507709f40a6..1c4b40690163e 100644 --- a/dev/breeze/tests/test_prod_image.py +++ b/dev/breeze/tests/test_prod_image.py @@ -19,69 +19,76 @@ import pytest -from airflow_breeze.prod.build_prod_image import get_image_build_params +from airflow_breeze.build_image.prod.build_prod_image import get_prod_image_build_params -default_params = { - 'build_cache_local': False, - 'build_cache_pulled': False, - 'build_cache_disabled': False, - 'skip_rebuild_check': False, +default_params: Dict[str, Union[str, bool]] = { + 'docker_cache': "pulled", 'disable_mysql_client_installation': False, 'disable_mssql_client_installation': False, 'disable_postgres_client_installation': False, 'install_docker_context_files': False, - 'disable_pypi_when_building': False, - 'disable_pip_cache': False, - 'upgrade_to_newer_dependencies': False, - 'skip_installing_airflow_providers_from_sources': False, + 'disable_airflow_repo_cache': False, + 'upgrade_to_newer_dependencies': "false", + 'install_providers_from_sources': False, 'cleanup_docker_context_files': False, 'prepare_buildx_cache': False, } -params_python8 = {**default_params, "python_version": "3.8"} # type: Dict[str, Union[str, bool]] +params_python8 = {**default_params, "python": "3.8"} # type: Dict[str, Union[str, bool]] -params_python9 = {**default_params, "python_version": "3.9"} # type: Dict[str, Union[str, bool]] +params_python9 = {**default_params, "python": "3.9"} # type: Dict[str, Union[str, bool]] @pytest.mark.parametrize( - 'parameters, expected_build_params, cached_values, written_cache_version', + 'description, parameters, expected_build_params, cached_values, written_cache_version, check_if_allowed', [ - (default_params, {"python_version": "3.7"}, {}, False), # default value no params - (params_python8, {"python_version": "3.8"}, {}, "3.8"), # default value override params + ("default value no cache", default_params, {"python": "3.7"}, {}, "3.7", False), + ("passed value different no cache", params_python8, {"python": "3.8"}, {}, "3.8", True), ( - default_params, - {"python_version": "3.8"}, + "passed value same as cache", + params_python8, + {"python": "3.8"}, {'PYTHON_MAJOR_MINOR_VERSION': "3.8"}, - False, - ), # value from cache + "3.8", + True, + ), ( + "passed value different than cache", params_python9, - {"python_version": "3.9"}, + {"python": "3.9"}, {'PYTHON_MAJOR_MINOR_VERSION': "3.8"}, "3.9", - ), # override cache with passed param + True, + ), ], ) -def test_get_image_params(parameters, expected_build_params, cached_values, written_cache_version): - with patch('airflow_breeze.cache.read_from_cache_file') as read_from_cache_mock, patch( - 'airflow_breeze.cache.check_if_cache_exists' +def test_get_image_params( + description, parameters, expected_build_params, cached_values, written_cache_version, check_if_allowed +): + with patch('airflow_breeze.utils.cache.read_from_cache_file') as read_from_cache_mock, patch( + 'airflow_breeze.utils.cache.check_if_cache_exists' ) as check_if_cache_exists_mock, patch( - 'airflow_breeze.prod.build_prod_image.write_to_cache_file' + 'airflow_breeze.utils.cache.write_to_cache_file' ) as write_to_cache_file_mock, patch( - 'airflow_breeze.prod.build_prod_image.check_cache_and_write_if_not_cached' - ) as check_cache_and_write_mock: + 'airflow_breeze.utils.cache.read_from_cache_file' + ) as read_from_cache_file: check_if_cache_exists_mock.return_value = True - check_cache_and_write_mock.side_effect = lambda cache_key, default_value: ( + read_from_cache_file.side_effect = lambda cache_key: ( cache_key in cached_values, - cached_values[cache_key] if cache_key in cached_values else default_value, + cached_values[cache_key] if cache_key in cached_values else None, ) read_from_cache_mock.side_effect = lambda param_name: cached_values.get(param_name) - build_parameters = get_image_build_params(parameters) + build_parameters = get_prod_image_build_params(parameters) for param, param_value in expected_build_params.items(): assert getattr(build_parameters, param) == param_value if written_cache_version: - write_to_cache_file_mock.assert_called_once_with( - "PYTHON_MAJOR_MINOR_VERSION", written_cache_version, check_allowed_values=True - ) + if check_if_allowed: + write_to_cache_file_mock.assert_called_once_with( + "PYTHON_MAJOR_MINOR_VERSION", written_cache_version, check_allowed_values=True + ) + else: + write_to_cache_file_mock.assert_called_once_with( + "PYTHON_MAJOR_MINOR_VERSION", written_cache_version + ) else: write_to_cache_file_mock.assert_not_called() diff --git a/dev/retag_docker_images.py b/dev/retag_docker_images.py index 765f6d8560ab8..6d4cf34611889 100755 --- a/dev/retag_docker_images.py +++ b/dev/retag_docker_images.py @@ -52,13 +52,13 @@ def pull_push_all_images( target_branch: str, target_repo: str, ): - for python_version in PYTHON_VERSIONS: + for python in PYTHON_VERSIONS: for image in images: source_image = image.format( - prefix=source_prefix, branch=source_branch, repo=source_repo, python_version=python_version + prefix=source_prefix, branch=source_branch, repo=source_repo, python=python ) target_image = image.format( - prefix=target_prefix, branch=target_branch, repo=target_repo, python_version=python_version + prefix=target_prefix, branch=target_branch, repo=target_repo, python=python ) print(f"Copying image: {source_image} -> {target_image}") subprocess.run(["docker", "pull", source_image], check=True) diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index b9271c063adbd..6e94776dfdfa6 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. AIRFLOW_CI_IMAGE -AIRFLOW_EXTRAS BACKEND BREEZE CI diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 3833e8807a35e..22e2e51bf0f23 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -29,7 +29,6 @@ services: # We need all those env variables here because docker-compose-v2 does not really work well # With env files and there are many problems with it: - AIRFLOW_CI_IMAGE=${AIRFLOW_CI_IMAGE} - - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS} - BACKEND=${BACKEND} - BREEZE=${BREEZE} - CI=${CI} diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 4c43360869db6..20a22c9207a79 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -69,11 +69,6 @@ function build_images::add_build_args_for_remote_install() { "--build-arg" "AIRFLOW_SOURCES_FROM=Dockerfile" "--build-arg" "AIRFLOW_SOURCES_TO=/Dockerfile" ) - if [[ ${CI} == "true" ]]; then - EXTRA_DOCKER_PROD_BUILD_FLAGS+=( - "--build-arg" "PIP_PROGRESS_BAR=off" - ) - fi if [[ -n "${AIRFLOW_CONSTRAINTS_REFERENCE}" ]]; then EXTRA_DOCKER_PROD_BUILD_FLAGS+=( "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE}" @@ -241,12 +236,10 @@ function build_images::confirm_image_rebuild() { echo "${COLOR_RED}ERROR: The ${THE_IMAGE_TYPE} needs to be rebuilt - it is outdated. ${COLOR_RESET}" echo """ - Make sure you build the image by running: + ${COLOR_YELLOW}Make sure you build the image by running:${COLOR_RESET} ./breeze --python ${PYTHON_MAJOR_MINOR_VERSION} build-image - If you run it via pre-commit as individual hook, you can run 'pre-commit run build'. - """ exit 1 else @@ -489,11 +482,6 @@ function build_images::build_ci_image() { ) fi local extra_docker_ci_flags=() - if [[ ${CI} == "true" ]]; then - EXTRA_DOCKER_PROD_BUILD_FLAGS+=( - "--build-arg" "PIP_PROGRESS_BAR=off" - ) - fi if [[ -n "${AIRFLOW_CONSTRAINTS_LOCATION}" ]]; then extra_docker_ci_flags+=( "--build-arg" "AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION}" diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 1cc866046542c..8b053d9e9cc04 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -55,10 +55,11 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then echo - echo "Airflow home: ${AIRFLOW_HOME}" - echo "Airflow sources: ${AIRFLOW_SOURCES}" - echo "Airflow core SQL connection: ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}" - + echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}" + echo + echo " * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}" + echo " * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}" + echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}" echo RUN_TESTS=${RUN_TESTS:="false"} @@ -68,7 +69,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then if [[ ${USE_AIRFLOW_VERSION} == "" ]]; then export PYTHONPATH=${AIRFLOW_SOURCES} echo - echo "Using already installed airflow version" + echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}" echo if [[ -d "${AIRFLOW_SOURCES}/airflow/www/" ]]; then pushd "${AIRFLOW_SOURCES}/airflow/www/" >/dev/null @@ -82,38 +83,38 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then mkdir -p "${AIRFLOW_SOURCES}"/tmp/ elif [[ ${USE_AIRFLOW_VERSION} == "none" ]]; then echo - echo "Skip installing airflow - only install wheel/tar.gz packages that are present locally" + echo "${COLOR_BLUE}Skip installing airflow - only install wheel/tar.gz packages that are present locally.${COLOR_RESET}" echo uninstall_airflow_and_providers elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then echo - echo "Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." + echo "${COLOR_BLUE}Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}" echo uninstall_airflow_and_providers install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]" uninstall_providers elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then echo - echo "Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers." + echo "${COLOR_BLUE}Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}" echo uninstall_airflow_and_providers install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]" uninstall_providers else echo - echo "Install airflow from PyPI without extras" + echo "${COLOR_BLUE}Install airflow from PyPI without extras" echo install_released_airflow_version "${USE_AIRFLOW_VERSION}" fi if [[ ${USE_PACKAGES_FROM_DIST=} == "true" ]]; then echo - echo "Install all packages from dist folder" + echo "${COLOR_BLUE}Install all packages from dist folder" if [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then echo "(except apache-airflow)" fi if [[ ${PACKAGE_FORMAT} == "both" ]]; then echo - echo "${COLOR_RED}ERROR:You can only specify 'wheel' or 'sdist' as PACKAGE_FORMAT not 'both'${COLOR_RESET}" + echo "${COLOR_RED}ERROR:You can only specify 'wheel' or 'sdist' as PACKAGE_FORMAT not 'both'.${COLOR_RESET}" echo exit 1 fi @@ -197,7 +198,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then cd "${AIRFLOW_SOURCES}" - if [[ ${START_AIRFLOW:="false"} == "true" ]]; then + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} # shellcheck source=scripts/in_container/bin/run_tmux diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 82b26090a4131..26d1f45200790 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -54,7 +54,7 @@ function assert_in_container() { } function in_container_script_start() { - if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then + if [[ ${VERBOSE_COMMANDS:="false"} == "true" || ${VERBOSE_COMMANDS} == "True" ]]; then set -x fi } @@ -63,14 +63,14 @@ function in_container_script_end() { #shellcheck disable=2181 EXIT_CODE=$? if [[ ${EXIT_CODE} != 0 ]]; then - if [[ "${PRINT_INFO_FROM_SCRIPTS="true"}" == "true" ]]; then + if [[ "${PRINT_INFO_FROM_SCRIPTS="true"}" == "true" || "${PRINT_INFO_FROM_SCRIPTS}" == "True" ]]; then echo "########################################################################################################################" echo "${COLOR_BLUE} [IN CONTAINER] EXITING ${0} WITH EXIT CODE ${EXIT_CODE} ${COLOR_RESET}" echo "########################################################################################################################" fi fi - if [[ ${VERBOSE_COMMANDS} == "true" ]]; then + if [[ ${VERBOSE_COMMANDS:="false"} == "true" || ${VERBOSE_COMMANDS} == "True" ]]; then set +x fi } @@ -322,7 +322,7 @@ function setup_provider_packages() { export PACKAGE_PREFIX_UPPERCASE="" export PACKAGE_PREFIX_LOWERCASE="" export PACKAGE_PREFIX_HYPHEN="" - if [[ ${VERBOSE} == "true" ]]; then + if [[ ${VERBOSE:="false"} == "true" || ${VERBOSE} == "True" ]]; then OPTIONAL_VERBOSE_FLAG+=("--verbose") fi readonly PACKAGE_TYPE @@ -429,7 +429,7 @@ function get_providers_to_act_on() { # Starts group for GitHub Actions - makes logs much more readable function group_start { - if [[ ${GITHUB_ACTIONS=} == "true" ]]; then + if [[ ${GITHUB_ACTIONS:="false"} == "true" || ${GITHUB_ACTIONS} == "True" ]]; then echo "::group::${1}" else echo @@ -440,7 +440,7 @@ function group_start { # Ends group for GitHub Actions function group_end { - if [[ ${GITHUB_ACTIONS=} == "true" ]]; then + if [[ ${GITHUB_ACTIONS:="false"} == "true" || ${GITHUB_ACTIONS} == "True" ]]; then echo -e "\033[0m" # Disable any colors set in the group echo "::endgroup::" fi diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index d525fb106c786..2ed2a4261047e 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -84,7 +84,7 @@ function check_integration { local env_var_name env_var_name=INTEGRATION_${integration_name^^} - if [[ ${!env_var_name:=} != "true" ]]; then + if [[ ${!env_var_name:=} != "true" || ${!env_var_name} != "True" ]]; then if [[ ! ${DISABLED_INTEGRATIONS} == *" ${integration_name}"* ]]; then DISABLED_INTEGRATIONS="${DISABLED_INTEGRATIONS} ${integration_name}" fi @@ -112,7 +112,7 @@ function check_db_backend { } function resetdb_if_requested() { - if [[ ${DB_RESET:="false"} == "true" ]]; then + if [[ ${DB_RESET:="false"} == "true" || ${DB_RESET} == "True" ]]; then echo echo "Resetting the DB" echo @@ -125,7 +125,7 @@ function resetdb_if_requested() { } function startairflow_if_requested() { - if [[ ${START_AIRFLOW:="false"} == "true" ]]; then + if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then echo echo "Starting Airflow" echo @@ -143,13 +143,14 @@ function startairflow_if_requested() { return $? } -echo "===============================================================================================" -echo " Checking integrations and backends" -echo "===============================================================================================" +echo +echo "${COLOR_BLUE}Checking integrations and backends.${COLOR_RESET}" +echo + if [[ -n ${BACKEND=} ]]; then check_db_backend 50 - echo "-----------------------------------------------------------------------------------------------" fi +echo check_integration "Kerberos" "kerberos" "run_nc kdc-server-example-com 88" 50 check_integration "MongoDB" "mongo" "run_nc mongo 27017" 50 check_integration "Redis" "redis" "run_nc redis 6379" 50 @@ -168,8 +169,6 @@ CMD="curl --max-time 1 -X GET 'http://pinot:8000/health' -H 'accept: text/plain' check_integration "Pinot (Broker API)" "pinot" "${CMD}" 50 check_integration "RabbitMQ" "rabbitmq" "run_nc rabbitmq 5672" 50 -echo "-----------------------------------------------------------------------------------------------" - if [[ ${EXIT_CODE} != 0 ]]; then echo echo "Error: some of the CI environment failed to initialize!" @@ -182,10 +181,8 @@ fi resetdb_if_requested startairflow_if_requested -if [[ -n ${DISABLED_INTEGRATIONS=} ]]; then - echo - echo "Disabled integrations:${DISABLED_INTEGRATIONS}" +if [[ -n ${DISABLED_INTEGRATIONS=} && (${VERBOSE=} == "true" || ${VERBOSE} == "True") ]]; then echo - echo "Enable them via --integration flags (you can use 'all' for all)" + echo "${COLOR_BLUE}Those integrations are disabled: ${DISABLED_INTEGRATIONS}" echo fi diff --git a/scripts/in_container/configure_environment.sh b/scripts/in_container/configure_environment.sh index cfd0c04f49c04..d9638b3219cac 100644 --- a/scripts/in_container/configure_environment.sh +++ b/scripts/in_container/configure_environment.sh @@ -25,12 +25,8 @@ if [[ -d "${FILES_DIR}" ]]; then export AIRFLOW__CORE__DAGS_FOLDER="/files/dags" mkdir -pv "${AIRFLOW__CORE__DAGS_FOLDER}" sudo chown "${HOST_USER_ID}":"${HOST_GROUP_ID}" "${AIRFLOW__CORE__DAGS_FOLDER}" - echo "Your dags for webserver and scheduler are read from ${AIRFLOW__CORE__DAGS_FOLDER} directory" - echo "which is mounted from your /files/dags folder" - echo else export AIRFLOW__CORE__DAGS_FOLDER="${AIRFLOW_HOME}/dags" - echo "Your dags for webserver and scheduler are read from ${AIRFLOW__CORE__DAGS_FOLDER} directory" fi @@ -38,16 +34,11 @@ if [[ -d "${AIRFLOW_BREEZE_CONFIG_DIR}" && \ -f "${AIRFLOW_BREEZE_CONFIG_DIR}/${VARIABLES_ENV_FILE}" ]]; then pushd "${AIRFLOW_BREEZE_CONFIG_DIR}" >/dev/null 2>&1 || exit 1 echo - echo "Sourcing environment variables from ${VARIABLES_ENV_FILE} in ${AIRFLOW_BREEZE_CONFIG_DIR}" + echo "${COLOR_BLUE}Sourcing environment variables from ${VARIABLES_ENV_FILE} in ${AIRFLOW_BREEZE_CONFIG_DIR}${COLOR_RESET}" echo # shellcheck disable=1090 source "${VARIABLES_ENV_FILE}" popd >/dev/null 2>&1 || exit 1 -else - echo - echo "You can add ${AIRFLOW_BREEZE_CONFIG_DIR} directory and place ${VARIABLES_ENV_FILE}" - echo "In it to make breeze source the variables automatically for you" - echo fi @@ -55,14 +46,9 @@ if [[ -d "${AIRFLOW_BREEZE_CONFIG_DIR}" && \ -f "${AIRFLOW_BREEZE_CONFIG_DIR}/${TMUX_CONF_FILE}" ]]; then pushd "${AIRFLOW_BREEZE_CONFIG_DIR}" >/dev/null 2>&1 || exit 1 echo - echo "Using ${TMUX_CONF_FILE} from ${AIRFLOW_BREEZE_CONFIG_DIR}" + echo "${COLOR_BLUE}Using ${TMUX_CONF_FILE} from ${AIRFLOW_BREEZE_CONFIG_DIR}${COLOR_RESET}" echo # shellcheck disable=1090 ln -sf "${AIRFLOW_BREEZE_CONFIG_DIR}/${TMUX_CONF_FILE}" ~ popd >/dev/null 2>&1 || exit 1 -else - echo - echo "You can add ${AIRFLOW_BREEZE_CONFIG_DIR} directory and place ${TMUX_CONF_FILE}" - echo "in it to make breeze use your local ${TMUX_CONF_FILE} for tmux" - echo fi diff --git a/scripts/in_container/run_ci_tests.sh b/scripts/in_container/run_ci_tests.sh index 52a2242eba172..efab241fb7711 100755 --- a/scripts/in_container/run_ci_tests.sh +++ b/scripts/in_container/run_ci_tests.sh @@ -39,7 +39,7 @@ if [[ ${RES} == "139" ]]; then fi set +x -if [[ "${RES}" == "0" && ${CI:="false"} == "true" ]]; then +if [[ "${RES}" == "0" && ( ${CI:="false"} == "true" || ${CI} == "True" ) ]]; then echo "All tests successful" cp .coverage /files fi @@ -61,7 +61,7 @@ if [[ ${TEST_TYPE:=} == "Quarantined" ]]; then fi fi -if [[ ${CI:=} == "true" ]]; then +if [[ ${CI:="false"} == "true" || ${CI} == "True" ]]; then if [[ ${RES} != "0" ]]; then echo echo "Dumping logs on error" diff --git a/scripts/in_container/run_docs_build.sh b/scripts/in_container/run_docs_build.sh index bcf94b9c8007e..0331557dc15e9 100755 --- a/scripts/in_container/run_docs_build.sh +++ b/scripts/in_container/run_docs_build.sh @@ -20,7 +20,7 @@ sudo -E "${AIRFLOW_SOURCES}/docs/build_docs.py" "${@}" -if [[ ${CI:="false"} == "true" && -d "${AIRFLOW_SOURCES}/docs/_build/docs/" ]]; then +if [[ ( ${CI:="false"} == "true" || ${CI} == "True" ) && -d "${AIRFLOW_SOURCES}/docs/_build/docs/" ]]; then rm -rf "/files/documentation" cp -r "${AIRFLOW_SOURCES}/docs/_build" "/files/documentation" fi diff --git a/scripts/in_container/run_init_script.sh b/scripts/in_container/run_init_script.sh index 0bd685cd9a067..48f7716f7cf02 100755 --- a/scripts/in_container/run_init_script.sh +++ b/scripts/in_container/run_init_script.sh @@ -37,9 +37,4 @@ if [[ -d "${AIRFLOW_BREEZE_CONFIG_DIR}" && \ # shellcheck disable=1090 source "${INIT_SCRIPT_FILE}" popd >/dev/null 2>&1 || exit 1 -else - echo - echo "You can add ${AIRFLOW_BREEZE_CONFIG_DIR} directory and place ${INIT_SCRIPT_FILE}" - echo "In it to make breeze source an initialization script automatically for you" - echo fi diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh index 2addc616ec07c..a5c6904beb18b 100755 --- a/scripts/in_container/run_install_and_test_provider_packages.sh +++ b/scripts/in_container/run_install_and_test_provider_packages.sh @@ -280,7 +280,7 @@ setup_provider_packages verify_parameters install_airflow_as_specified -if [[ ${SKIP_TWINE_CHECK=""} != "true" ]]; then +if [[ ${SKIP_TWINE_CHECK=""} != "true" && ${SKIP_TWINE_CHECK=""} != "True" ]]; then # Airflow 2.1.0 installs importlib_metadata version that does not work well with twine # So we should skip twine check in this case twine_check_provider_packages diff --git a/scripts/in_container/run_prepare_provider_documentation.sh b/scripts/in_container/run_prepare_provider_documentation.sh index 68e06dfe9624c..a8745ff16ecf7 100755 --- a/scripts/in_container/run_prepare_provider_documentation.sh +++ b/scripts/in_container/run_prepare_provider_documentation.sh @@ -111,7 +111,7 @@ function run_prepare_documentation() { echo "${COLOR_RED}There were errors when preparing documentation. Exiting! ${COLOR_RESET}" exit 1 else - if [[ ${GENERATE_PROVIDERS_ISSUE=} == "true" ]]; then + if [[ ${GENERATE_PROVIDERS_ISSUE=} == "true" || ${GENERATE_PROVIDERS_ISSUE} == "True" ]]; then echo python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content "${prepared_documentation[@]}" echo @@ -157,7 +157,7 @@ if [[ $# != "0" && ${1} =~ ^[0-9][0-9][0-9][0-9]\.[0-9][0-9]\.[0-9][0-9]$ ]]; th fi OPTIONAL_NON_INTERACTIVE_FLAG=() -if [[ ${NON_INTERACTIVE=} == "true" ]]; then +if [[ ${NON_INTERACTIVE=} == "true" || ${NON_INTERACTIVE} == "True" ]]; then OPTIONAL_NON_INTERACTIVE_FLAG+=("--non-interactive") fi diff --git a/scripts/in_container/run_system_tests.sh b/scripts/in_container/run_system_tests.sh index 2b1181c025749..f77a58f6ff33b 100755 --- a/scripts/in_container/run_system_tests.sh +++ b/scripts/in_container/run_system_tests.sh @@ -44,11 +44,11 @@ pytest "${PYTEST_ARGS[@]}" RES=$? set +x -if [[ "${RES}" == "0" && ${GITHUB_ACTIONS} == "true" ]]; then +if [[ "${RES}" == "0" && ( ${GITHUB_ACTIONS=} == "true" || ${GITHUB_ACTIONS} == "True" ) ]]; then echo "All tests successful" fi -if [[ ${GITHUB_ACTIONS} == "true" ]]; then +if [[ ${GITHUB_ACTIONS=} == "true" || ${GITHUB_ACTIONS} == "True" ]]; then dump_airflow_logs fi