From 61d1540364fc8b4cf08e22e980191df5cbedb31c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 12 Apr 2020 13:26:15 +0200 Subject: [PATCH 1/3] Help for breeze commands contain relevant flags. --- BREEZE.rst | 452 +++++++++++++++++++++++++++++++++++++++++++---------- breeze | 345 ++++++++++++++++++++++++++-------------- 2 files changed, 596 insertions(+), 201 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 4e66b3c7e0381..f777b7e838006 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -655,8 +655,8 @@ This is the current syntax for `./breeze <./breeze>`_: generate-requirements Generates pinned requirements for pip dependencies initialize-local-virtualenv Initializes local virtualenv setup-autocomplete Sets up autocomplete for breeze - stop Stops the docker-compose evironment - restart Stops the docker-compose evironment including DB cleanup + stop Stops the docker-compose environment + restart Stops the docker-compose environment including DB cleanup toggle-suppress-cheatsheet Toggles on/off cheatsheet toggle-suppress-asciiart Toggles on/off asciiart @@ -680,6 +680,9 @@ This is the current syntax for `./breeze <./breeze>`_: #################################################################################################### + + Detailed usage for command: shell + breeze [FLAGS] shell -- This is default subcommand if no subcommand is used. @@ -696,7 +699,17 @@ This is the current syntax for `./breeze <./breeze>`_: and setup. This file is automatically sourced when you enter the container. Database and webserver ports are forwarded to appropriate database/webserver so that you can connect to it from your host environment. - **************************************************************************************************** + + Flags: + + Run 'breeze flags' to see all available applicable flags. + + + #################################################################################################### + + + Detailed usage for command: build-docs + breeze [FLAGS] build-docs -- Builds airflow documentation. The documentation is build inside docker container - to @@ -704,27 +717,134 @@ This is the current syntax for `./breeze <./breeze>`_: the host to the container so that latest sources are used. The folders where documentation is generated ('docs/build') are also mounted to the container - this way results of the documentation build is available in the host. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: build-image + breeze [FLAGS] build-image -- Builds docker image (CI or production) without entering the container. You can pass additional options to this command, such as '--force-build-image', '--force-pull-image' '--python' '--use-local-cache'' in order to modify build behaviour. You can also pass '--production-image' flag to build production image rather than CI image. - **************************************************************************************************** + + Flags: + + -p, --python + Python version used for the image. This is always major/minor version. + One of: + + 3.6 3.7 + + -a, --install-airflow-version + If specified, installs airflow directly from PIP released version. This happens at + image building time in production image and at container entering time for CI image. One of: + + 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test + + -t, --install-airflow-reference + If specified, installs airflow directly from reference in GitHub. This happens at + image building time in production image and at container entering time for CI image. + + -I, --production-image + Use production image for entering the environment and builds (not for tests). + + -F, --force-build-images + Forces building of the local docker images. The images are rebuilt + automatically for the first time or when changes are detected in + package-related files, but you can force it using this flag. + + -P, --force-pull-images + Forces pulling of images from DockerHub before building to populate cache. The + images are pulled by default only for the first time you run the + environment, later the locally build images are used as cache. + + -E, --extras + Extras to pass to build images The default are different for CI and production images: + + CI image: + devel_ci + + Production image: + async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack, + ssh,statsd,virtualenv + + -C, --force-clean-images + Force build images with cache disabled. This will remove the pulled or build images + and start building images from scratch. This might take a long time. + + -L, --use-local-cache + Uses local cache to build images. No pulled images will be used, but results of local + builds in the Docker cache are used instead. + + -u, --push-images + After building - uploads the images to DockerHub + It is useful in case you use your own DockerHub user to store images and you want + to build them locally. Note that you need to use 'docker login' before you upload images. + + -D, --dockerhub-user + DockerHub user used to pull, push and build images. Default: apache. + + -H, --dockerhub-repo + DockerHub repository used to pull, push, build images. Default: airflow. + + -v, --verbose + Show verbose information about executed commands (enabled by default for running test). + } + Note that you can further increase verbosity and see all the commands executed by breeze + by running 'export VERBOSE_COMMANDS="true"' before running breeze. + + + #################################################################################################### + + + Detailed usage for command: cleanup-image + breeze [FLAGS] cleanup-image -- Removes the breeze-related images created in your local docker image cache. This will not reclaim space in docker cache. You need to 'docker system prune' (optionally with --all) to reclaim that space. - **************************************************************************************************** + + Flags: + + -p, --python + Python version used for the image. This is always major/minor version. + One of: + + 3.6 3.7 + + -I, --production-image + Use production image for entering the environment and builds (not for tests). + + -v, --verbose + Show verbose information about executed commands (enabled by default for running test). + } + Note that you can further increase verbosity and see all the commands executed by breeze + by running 'export VERBOSE_COMMANDS="true"' before running breeze. + + + #################################################################################################### + + + Detailed usage for command: exec + breeze [FLAGS] exec -- Execs into interactive shell to an already running container. The container mus be started already by breeze shell command. If you are not familiar with tmux, this is the best way to run multiple processes in the same container at the same time for example scheduler, webserver, workers, database console and interactive terminal. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: generate-requirements + breeze [FLAGS] generate-requirements -- Generates pinned requirements from setup.py. Those requirements are generated in requirements @@ -732,7 +852,27 @@ This is the current syntax for `./breeze <./breeze>`_: CI builds as well as run repeatable production image builds. You can use those requirements to predictably install released airflow versions. You should run it always after you update setup.py. - **************************************************************************************************** + + Flags: + + -p, --python + Python version used for the image. This is always major/minor version. + One of: + + 3.6 3.7 + + -v, --verbose + Show verbose information about executed commands (enabled by default for running test). + } + Note that you can further increase verbosity and see all the commands executed by breeze + by running 'export VERBOSE_COMMANDS="true"' before running breeze. + + + #################################################################################################### + + + Detailed usage for command: initialize-local-virtualenv + breeze [FLAGS] initialize-local-virtualenv -- Initializes locally created virtualenv installing all dependencies of Airflow @@ -740,33 +880,81 @@ This is the current syntax for `./breeze <./breeze>`_: This local virtualenv can be used to aid autocompletion and IDE support as well as run unit tests directly from the IDE. You need to have virtualenv activated before running this command. - **************************************************************************************************** + + Flags: + + -p, --python + Python version used for the image. This is always major/minor version. + One of: + + 3.6 3.7 + + + #################################################################################################### + + + Detailed usage for command: setup-autocomplete + breeze [FLAGS] setup-autocomplete -- Sets up autocomplete for breeze commands. Once you do it you need to re-enter the bash shell and when typing breeze command will provide autocomplete for parameters and values. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: stop + breeze [FLAGS] stop -- Brings down running docker compose environment. When you start the environment, the docker containers will continue running so that startup time is shorter. But they take quite a lot of memory and CPU. This command stops all running containers from the environment. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: restart + breeze [FLAGS] restart -- Restarts running docker compose environment. When you restart the environment, the docker containers will be restarted. That includes cleaning up the databases. This is especially useful if you switch between different versions of airflow. - **************************************************************************************************** + + Flags: + + Run 'breeze flags' to see all available applicable flags. + + + #################################################################################################### + + + Detailed usage for command: toggle-suppress-cheatsheet + breeze [FLAGS] toggle-suppress-cheatsheet -- Toggles on/off cheatsheet displayed before starting bash shell. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: toggle-suppress-asciiart + breeze [FLAGS] toggle-suppress-asciiart -- Toggles on/off asciiart displayed before starting bash shell. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: docker-compose + breeze [FLAGS] docker-compose -- Run docker-compose command instead of entering the environment. Use 'help' as command @@ -774,7 +962,45 @@ This is the current syntax for `./breeze <./breeze>`_: as additional options passed to docker-compose. For example 'breeze docker-compose pull -- --ignore-pull-failures' - **************************************************************************************************** + + Flags: + + -p, --python + Python version used for the image. This is always major/minor version. + One of: + + 3.6 3.7 + + -b, --backend + Backend to use for tests - it determines which database is used. + One of: + + sqlite mysql postgres + + Default: sqlite + + --postgres-version + Postgres version used. One of: + + 9.6 10 + + --mysql-version + Mysql version used. One of: + + 5.7 8 + + -v, --verbose + Show verbose information about executed commands (enabled by default for running test). + } + Note that you can further increase verbosity and see all the commands executed by breeze + by running 'export VERBOSE_COMMANDS="true"' before running breeze. + + + #################################################################################################### + + + Detailed usage for command: execute-command + breeze [FLAGS] execute-command -- Run chosen command instead of entering the environment. The command is run using @@ -784,7 +1010,45 @@ This is the current syntax for `./breeze <./breeze>`_: 'breeze execute-command "ls -la"' or 'breeze execute-command ls -- --la' - **************************************************************************************************** + + Flags: + + -p, --python + Python version used for the image. This is always major/minor version. + One of: + + 3.6 3.7 + + -b, --backend + Backend to use for tests - it determines which database is used. + One of: + + sqlite mysql postgres + + Default: sqlite + + --postgres-version + Postgres version used. One of: + + 9.6 10 + + --mysql-version + Mysql version used. One of: + + 5.7 8 + + -v, --verbose + Show verbose information about executed commands (enabled by default for running test). + } + Note that you can further increase verbosity and see all the commands executed by breeze + by running 'export VERBOSE_COMMANDS="true"' before running breeze. + + + #################################################################################################### + + + Detailed usage for command: static-check + breeze [FLAGS] static-check -- Run selected static checks for currently changed files. You should specify static check that @@ -804,7 +1068,13 @@ This is the current syntax for `./breeze <./breeze>`_: You can see all the options by adding --help EXTRA_ARG: 'breeze static-check mypy -- --help' - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: static-check-all-files + breeze [FLAGS] static-check-all-files -- Run selected static checks for all applicable files. You should specify static check that @@ -824,7 +1094,13 @@ This is the current syntax for `./breeze <./breeze>`_: You can see all the options by adding --help EXTRA_ARG: 'breeze static-check-all-files mypy -- --help' - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: test-target + breeze [FLAGS] test-target -- Run the specified unit test target. There might be multiple @@ -832,32 +1108,51 @@ This is the current syntax for `./breeze <./breeze>`_: as additional options passed to pytest. For example: 'breeze test-target tests/test_core.py -- --logging-level=DEBUG' - **************************************************************************************************** + + Flags: + + Run 'breeze flags' to see all available applicable flags. + + + #################################################################################################### + + + Detailed usage for command: flags + breeze [FLAGS] flags -- Explains in detail all the flags that can be used with breeze. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: help + breeze [FLAGS] help -- Shows this help message. - **************************************************************************************************** + + + #################################################################################################### + + + Detailed usage for command: help-all + breeze [FLAGS] help-all -- Shows detailed help for all commands and flags. - **************************************************************************************************** - #################################################################################################### - Flags #################################################################################################### - **************************************************************************************************** - List of flags supported by breeze: + #################################################################################################### + + Summary of all flags supported by Breeze: **************************************************************************************************** Choose Airflow variant - **************************************************************************************************** -p, --python Python version used for the image. This is always major/minor version. @@ -865,6 +1160,9 @@ This is the current syntax for `./breeze <./breeze>`_: 3.6 3.7 + **************************************************************************************************** + Choose backend to run for airflow + -b, --backend Backend to use for tests - it determines which database is used. One of: @@ -873,8 +1171,27 @@ This is the current syntax for `./breeze <./breeze>`_: Default: sqlite + --postgres-version + Postgres version used. One of: + + 9.6 10 + + --mysql-version + Mysql version used. One of: + + 5.7 8 + + **************************************************************************************************** + Enable production image + + -I, --production-image + Use production image for entering the environment and builds (not for tests). + + **************************************************************************************************** + Additional actions executed while entering breeze + -d, --db-reset - Resets the database at entry to the envvironment. It will drop all the tables + Resets the database at entry to the environment. It will drop all the tables and data and recreate the DB from scratch even if 'restart' command was not used. Combined with 'restart' command it enters the environment in the state that is ready to start airflow webserver/scheduler/worker. Without the switch, the database @@ -883,20 +1200,16 @@ This is the current syntax for `./breeze <./breeze>`_: -i, --integration Integration to start during tests - it determines which integrations are started for integration tests. There can be more than one integration started, or all to + } start all integrations. Selected integrations are not saved for future execution. One of: cassandra kerberos mongo openldap presto rabbitmq redis all - -I, --production-image - Use production image for entering the environment and builds (not for tests). - **************************************************************************************************** Manage Kind kubernetes cluster (optional) - **************************************************************************************************** - - Acion for the cluster : only one of the --kind-cluster-* flags can be used at a time: + Action for the cluster : only one of the --kind-cluster-* flags can be used at a time: -s, --kind-cluster-start Starts kind Kubernetes cluster after entering the environment. The cluster is started using @@ -936,44 +1249,13 @@ This is the current syntax for `./breeze <./breeze>`_: **************************************************************************************************** Manage mounting local files - **************************************************************************************************** -l, --skip-mounting-local-sources Skips mounting local volume with sources - you get exactly what is in the docker image rather than your current local sources of airflow. - **************************************************************************************************** - Install Airflow if different than current - **************************************************************************************************** - - -a, --install-airflow-version - If specified, installs airflow directly from PIP released version. One of: - - 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test - - -t, --install-airflow-reference - Only for production image - if specified, installs airflow directly from reference in GitHub - - - **************************************************************************************************** - Database versions - **************************************************************************************************** - - --postgres-version - Postgres version used. One of: - - 9.6 10 - - - --mysql-version - Mysql version used. One of: - - 5.7 8 - - **************************************************************************************************** Assume answers to questions - **************************************************************************************************** -y, --assume-yes Assume 'yes' answer to all questions. @@ -985,25 +1267,27 @@ This is the current syntax for `./breeze <./breeze>`_: Assume 'quit' answer to all questions. **************************************************************************************************** - Credentials + Choose different airflow version to install or run + + -a, --install-airflow-version + If specified, installs airflow directly from PIP released version. This happens at + image building time in production image and at container entering time for CI image. One of: + + 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test + + -t, --install-airflow-reference + If specified, installs airflow directly from reference in GitHub. This happens at + image building time in production image and at container entering time for CI image. + **************************************************************************************************** + Credentials -f, --forward-credentials Forwards host credentials to docker container. Use with care as it will make your credentials available to everything you install in Docker. **************************************************************************************************** - Increase verbosity of the script - **************************************************************************************************** - - -v, --verbose - Show verbose information about executed commands (enabled by default for running test). - Note that you can further increase verbosity and see all the commands executed by breeze - by running 'export VERBOSE_COMMANDS="true"' before running breeze. - - **************************************************************************************************** - Flags for building the docker images - **************************************************************************************************** + Flags for building Docker images (both CI and production) -F, --force-build-images Forces building of the local docker images. The images are rebuilt @@ -1034,18 +1318,13 @@ This is the current syntax for `./breeze <./breeze>`_: builds in the Docker cache are used instead. **************************************************************************************************** - Flags for pushing the docker images - **************************************************************************************************** + Flags for pushing Docker images (both CI and production) -u, --push-images After building - uploads the images to DockerHub It is useful in case you use your own DockerHub user to store images and you want to build them locally. Note that you need to use 'docker login' before you upload images. - **************************************************************************************************** - User and repo used to login to github registry - **************************************************************************************************** - -D, --dockerhub-user DockerHub user used to pull, push and build images. Default: apache. @@ -1053,6 +1332,13 @@ This is the current syntax for `./breeze <./breeze>`_: DockerHub repository used to pull, push, build images. Default: airflow. **************************************************************************************************** + Increase verbosity of the scripts + + -v, --verbose + Show verbose information about executed commands (enabled by default for running test). + } + Note that you can further increase verbosity and see all the commands executed by breeze + by running 'export VERBOSE_COMMANDS="true"' before running breeze. .. END BREEZE HELP MARKER diff --git a/breeze b/breeze index 5ff7febbce241..cba0b6d88f768 100755 --- a/breeze +++ b/breeze @@ -52,7 +52,7 @@ function setup_default_breeze_variables() { mkdir -pv "${FILES_DIR}" # Note - we do not use __script_init.sh here because it can only be used from within - # the CI directory and we need to overrride PYTHON_MAJOR_MINOR_VERSION based on what we store + # the CI directory and we need to override PYTHON_MAJOR_MINOR_VERSION based on what we store # in the .build directory # Beginning of the initialisation here @@ -86,7 +86,7 @@ function setup_default_breeze_variables() { # Skips mounting local Airflow sources MOUNT_LOCAL_SOURCES="true" - # Holds last subcommand used + # Holds last sub-command used LAST_SUBCOMMAND="" # Determines if help should be run (set to true by --help flag) @@ -124,7 +124,7 @@ function setup_default_breeze_variables() { export DB_RESET="false" # If install released airflow is set to specified version, then the source version of airflow - # is removed and the specified version of airflow is installed from pypi + # is removed and the specified version of airflow is installed from PyPi export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} # If install released airflow is set to specified reference (tag/branch), then the source version @@ -138,7 +138,7 @@ function setup_default_breeze_variables() { # Whether to force build without checking if it is needed export FORCE_BUILD_IMAGES=${FORCE_BUILD_IMAGES:="false"} - # Files determining whether asciiart/cheatsheet are suppressed + # Files determining whether ASCII-art/cheat-sheet are suppressed SUPPRESS_CHEATSHEET_FILE="${MY_DIR}/.suppress_cheatsheet" SUPPRESS_ASCIIART_FILE="${MY_DIR}/.suppress_asciiart" @@ -164,7 +164,7 @@ function initialize_virtualenv() { echo >&2 echo >&2 "ERROR: Initializing local virtualenv only works when you have virtualenv activated" echo >&2 - echo >&2 "Please enter your local virtualenv before (for example using 'workon') " + echo >&2 "Please enter your local virtualenv before (for example using 'pyenv activate' or 'workon') " echo >&2 exit 1 else @@ -187,11 +187,11 @@ function initialize_virtualenv() { if [[ ${RES} != "0" ]]; then echo "#######################################################################" echo " You had some troubles installing the venv !!!!!" - echo " Try runnning the command below and rerun virtualenv installation" + echo " Try running the command below and rerun virtualenv installation" echo SYSTEM=$(uname -s) if [[ ${SYSTEM} == "Darwin" ]]; then - echo " brew install sqlite mysql postgresql" + echo " brew install sqlite mysql postgresql openssl" else echo " sudo apt install build-essentials python3.6-dev python3.7-dev python-dev openssl \\" echo " sqlite sqlite-dev default-libmysqlclient-dev libmysqld-dev postgresql" @@ -544,7 +544,7 @@ function prepare_command_files() { '"' "false" "${COMPOSE_PROD_FILE}" "${AIRFLOW_PROD_IMAGE}" } -do_help_all() { +function do_help_all() { echo print_line usage @@ -558,16 +558,10 @@ do_help_all() { for SUBCOMMAND in ${ALL_BREEZE_COMMANDS} do detailed_usage "${SUBCOMMAND}" - print_star_line + print_line + echo done echo - print_line - echo - echo - echo "Flags" - echo - print_line - echo flags } @@ -754,7 +748,7 @@ function parse_arguments() { echo shift 2 ;; -f|--forward-credentials) - echo "Fowarding credentials. Be careful as your credentials ar available in the container!" + echo "Forwarding credentials. Be careful as your credentials ar available in the container!" echo export FORWARD_CREDENTIALS="true" shift 1 ;; @@ -777,7 +771,7 @@ function parse_arguments() { ;; esac done - # Parse commaands + # Parse commands if [[ "$#" -ne 0 ]]; then case "${1}" in shell) @@ -844,7 +838,7 @@ function parse_arguments() { echo "One of [${_BREEZE_ALLOWED_STATIC_CHECKS:=}]." echo echo "For example:" - echo "${CMDNAME} static-check mypy" + echo "${CMDNAME} static-check MyPy" exit 1 fi export PYTHON_MAJOR_MINOR_VERSION=${STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION} @@ -859,7 +853,7 @@ function parse_arguments() { echo "One of [${_BREEZE_ALLOWED_STATIC_CHECKS:=}]." echo echo "For example:" - echo "${CMDNAME} static-check-all-files mypy" + echo "${CMDNAME} static-check-all-files MyPy" exit 1 fi COMMAND_TO_RUN="perform_static_checks" @@ -916,7 +910,6 @@ function parse_arguments() { exit 0 ;; help) usage - flag_footer exit 0 ;; help-all) do_help_all @@ -940,14 +933,13 @@ function parse_arguments() { flag_footer else detailed_usage "${LAST_SUBCOMMAND}" - flag_footer fi exit 0 fi REMAINING_ARGS+=("$@") } -prepare_formatted_versions() { +function prepare_formatted_versions() { INDENT=15 LIST_PREFIX=$(printf "%-${INDENT}s" " ") WIDTH=$((SCREEN_WIDTH - INDENT)) @@ -975,7 +967,7 @@ prepare_formatted_versions() { tr ',' ' ' | fold -w "${WIDTH}" -s | sed "s/ /,/g; s/^/${LIST_PREFIX}/") } -prepare_usage() { +function prepare_usage() { # Note that MacOS uses Bash 3.* and we cannot use associative arrays export USAGE_SHELL="[Default] Enters interactive shell in the container" export USAGE_EXEC="Execs into running breeze container in new terminal" @@ -988,8 +980,8 @@ prepare_usage() { export USAGE_GENERATE_REQUIREMENTS="Generates pinned requirements for pip dependencies" export USAGE_INITIALIZE_LOCAL_VIRTUALENV="Initializes local virtualenv" export USAGE_SETUP_AUTOCOMPLETE="Sets up autocomplete for breeze" - export USAGE_STOP="Stops the docker-compose evironment" - export USAGE_RESTART="Stops the docker-compose evironment including DB cleanup" + export USAGE_STOP="Stops the docker-compose environment" + export USAGE_RESTART="Stops the docker-compose environment including DB cleanup" export USAGE_STATIC_CHECK="Performs selected static check for changed files" export USAGE_STATIC_CHECK_ALL_FILES="Performs selected static check for all files" export USAGE_TOGGLE_SUPPRESS_CHEATSHEET="Toggles on/off cheatsheet" @@ -999,7 +991,8 @@ prepare_usage() { export USAGE_HELP_ALL="Shows detailed help for all commands and flags" - export DETAILED_USAGE_SHELL=" + # shellcheck disable=SC2089 + DETAILED_USAGE_SHELL=" This is default subcommand if no subcommand is used. Enters interactive shell where you can run all tests, start airflow webserver, scheduler, @@ -1014,7 +1007,12 @@ prepare_usage() { and setup. This file is automatically sourced when you enter the container. Database and webserver ports are forwarded to appropriate database/webserver so that you can connect to it from your host environment. + +Flags: +$(flag_footer) " + # shellcheck disable=SC2090 + export DETAILED_USAGE_SHELL export DETAILED_USAGE_EXEC=" Execs into interactive shell to an already running container. The container mus be started already by breeze shell command. If you are not familiar with tmux, this is the best @@ -1028,25 +1026,53 @@ prepare_usage() { is generated ('docs/build') are also mounted to the container - this way results of the documentation build is available in the host. " - export DETAILED_USAGE_BUILD_IMAGE=" + # shellcheck disable=SC2089 + DETAILED_USAGE_BUILD_IMAGE=" Builds docker image (CI or production) without entering the container. You can pass additional options to this command, such as '--force-build-image', '--force-pull-image' '--python' '--use-local-cache'' in order to modify build behaviour. You can also pass '--production-image' flag to build production image rather than CI image. + +Flags: +$(flag_airflow_variants) +$(flag_choose_different_airflow_version) +$(flag_production_image) +$(flag_build_docker_images) +$(flag_push_docker_images) +$(flag_verbosity) " - export DETAILED_USAGE_CLEANUP_IMAGE=" + # shellcheck disable=SC2090 + export DETAILED_USAGE_BUILD_IMAGE + # shellcheck disable=SC2089 + DETAILED_USAGE_CLEANUP_IMAGE=" Removes the breeze-related images created in your local docker image cache. This will not reclaim space in docker cache. You need to 'docker system prune' (optionally with --all) to reclaim that space. + +Flags: +$(flag_airflow_variants) +$(flag_production_image) +$(flag_verbosity) " - export DETAILED_USAGE_DOCKER_COMPOSE=" + # shellcheck disable=SC2090 + export DETAILED_USAGE_CLEANUP_IMAGE + # shellcheck disable=SC2089 + DETAILED_USAGE_DOCKER_COMPOSE=" Run docker-compose command instead of entering the environment. Use 'help' as command to see available commands. The passed after -- are treated as additional options passed to docker-compose. For example '${CMDNAME} docker-compose pull -- --ignore-pull-failures' + +Flags: +$(flag_airflow_variants) +$(flag_backend_variants) +$(flag_verbosity) " - export DETAILED_USAGE_EXECUTE_COMMAND=" + # shellcheck disable=SC2090 + export DETAILED_USAGE_DOCKER_COMPOSE + # shellcheck disable=SC2089 + DETAILED_USAGE_EXECUTE_COMMAND=" Run chosen command instead of entering the environment. The command is run using 'bash -c \"\" if you need to pass arguments to your command, you need to pass them together with command surrounded with \" or '. Alternatively you can @@ -1054,24 +1080,42 @@ prepare_usage() { '${CMDNAME} execute-command \"ls -la\"' or '${CMDNAME} execute-command ls -- --la' + +Flags: +$(flag_airflow_variants) +$(flag_backend_variants) +$(flag_verbosity) " + # shellcheck disable=SC2090 + export DETAILED_USAGE_EXECUTE_COMMAND export DETAILED_USAGE_FLAGS=" Explains in detail all the flags that can be used with breeze. " - export DETAILED_USAGE_GENERATE_REQUIREMENTS=" + DETAILED_USAGE_GENERATE_REQUIREMENTS=" Generates pinned requirements from setup.py. Those requirements are generated in requirements directory - separately for different python version. Those requirements are used to run CI builds as well as run repeatable production image builds. You can use those requirements to predictably install released airflow versions. You should run it always after you update setup.py. + +Flags: +$(flag_airflow_variants) +$(flag_verbosity) " - export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV=" + # shellcheck disable=SC2090 + export DETAILED_USAGE_GENERATE_REQUIREMENTS + DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV=" Initializes locally created virtualenv installing all dependencies of Airflow taking into account the frozen requirements from requirements folder. This local virtualenv can be used to aid autocompletion and IDE support as well as run unit tests directly from the IDE. You need to have virtualenv activated before running this command. + +Flags: +$(flag_airflow_variants) " + # shellcheck disable=SC2090 + export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV export DETAILED_USAGE_SETUP_AUTOCOMPLETE=" Sets up autocomplete for breeze commands. Once you do it you need to re-enter the bash shell and when typing breeze command will provide autocomplete for @@ -1082,11 +1126,15 @@ prepare_usage() { containers will continue running so that startup time is shorter. But they take quite a lot of memory and CPU. This command stops all running containers from the environment. " - export DETAILED_USAGE_RESTART=" + DETAILED_USAGE_RESTART=" Restarts running docker compose environment. When you restart the environment, the docker containers will be restarted. That includes cleaning up the databases. This is especially useful if you switch between different versions of airflow. + +Flags: +$(flag_footer) " + export DETAILED_USAGE_RESTART export DETAILED_USAGE_STATIC_CHECK=" Run selected static checks for currently changed files. You should specify static check that you would like to run or 'all' to run all checks. One of: @@ -1119,13 +1167,19 @@ ${FORMATTED_STATIC_CHECKS} '${CMDNAME} static-check-all-files mypy -- --help' " - export DETAILED_USAGE_TEST_TARGET=" + # shellcheck disable=SC2089 + DETAILED_USAGE_TEST_TARGET=" Run the specified unit test target. There might be multiple targets specified separated with comas. The passed after -- are treated as additional options passed to pytest. For example: '${CMDNAME} test-target tests/test_core.py -- --logging-level=DEBUG' + +Flags: +$(flag_footer) " + # shellcheck disable=SC2090 + export DETAILED_USAGE_TEST_TARGET export DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET=" Toggles on/off cheatsheet displayed before starting bash shell. " @@ -1140,7 +1194,7 @@ ${FORMATTED_STATIC_CHECKS} " } -get_variable_from_lowercase_name() { +function get_variable_from_lowercase_name() { PREFIX="${1}" NAME="${2}" SUFFIX="$(echo "${NAME}" | tr "[:lower:]-" "[:upper:]_")" @@ -1148,16 +1202,16 @@ get_variable_from_lowercase_name() { echo "${!VARIABLE_NAME}" } -get_usage() { +function get_usage() { get_variable_from_lowercase_name USAGE "${1}" } -get_detailed_usage() { +function get_detailed_usage() { get_variable_from_lowercase_name DETAILED_USAGE "${1}" } -usage() { +function usage() { echo " Usage: ${CMDNAME} [FLAGS] [COMMAND] -- @@ -1188,35 +1242,36 @@ Help commands: echo } -detailed_usage() { - SUBCOMMAND=${1} +function detailed_usage() { + SUBCOMMAND=${1} echo " + +Detailed usage for command: ${SUBCOMMAND} + ${CMDNAME} [FLAGS] ${SUBCOMMAND} -- -$(get_detailed_usage "${SUBCOMMAND}")" +$(get_detailed_usage "${SUBCOMMAND}") + +" } -flag_footer() { - echo " -Run '${CMDNAME} flags' to see available flags +function flag_footer() { + echo " +Run '${CMDNAME} flags' to see all available applicable flags. " } -flags() { +function flag_airflow_variants() { echo " -$(print_star_line) - -List of flags supported by breeze: - -$(print_star_line) - Choose Airflow variant -$(print_star_line) - -p, --python Python version used for the image. This is always major/minor version. One of: ${FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS} +" +} +function flag_backend_variants() { + echo " -b, --backend Backend to use for tests - it determines which database is used. One of: @@ -1225,8 +1280,29 @@ ${FORMATTED_BACKENDS} Default: ${_BREEZE_DEFAULT_BACKEND:=} +--postgres-version + Postgres version used. One of: + +${FORMATTED_POSTGRES_VERSIONS} + +--mysql-version + Mysql version used. One of: + +${FORMATTED_MYSQL_VERSIONS} +" +} + +function flag_production_image() { + echo " +-I, --production-image + Use production image for entering the environment and builds (not for tests). +" +} + +function flag_breeze_actions() { + echo " -d, --db-reset - Resets the database at entry to the envvironment. It will drop all the tables + Resets the database at entry to the environment. It will drop all the tables and data and recreate the DB from scratch even if 'restart' command was not used. Combined with 'restart' command it enters the environment in the state that is ready to start airflow webserver/scheduler/worker. Without the switch, the database @@ -1235,20 +1311,17 @@ ${FORMATTED_BACKENDS} -i, --integration Integration to start during tests - it determines which integrations are started for integration tests. There can be more than one integration started, or all to + } start all integrations. Selected integrations are not saved for future execution. One of: ${FORMATTED_INTEGRATIONS} +" +} --I, --production-image - Use production image for entering the environment and builds (not for tests). - -$(print_star_line) - Manage Kind kubernetes cluster (optional) -$(print_star_line) - - -Acion for the cluster : only one of the --kind-cluster-* flags can be used at a time: +function flag_kubernetes_actions() { + echo " +Action for the cluster : only one of the --kind-cluster-* flags can be used at a time: -s, --kind-cluster-start Starts kind Kubernetes cluster after entering the environment. The cluster is started using @@ -1285,48 +1358,33 @@ ${FORMATTED_KUBERNETES_MODES} ${FORMATTED_KUBERNETES_VERSIONS} Default: ${_BREEZE_DEFAULT_KUBERNETES_VERSION:=} +" +} -$(print_star_line) - Manage mounting local files -$(print_star_line) - +function flag_local_file_mounting() { + echo " -l, --skip-mounting-local-sources Skips mounting local volume with sources - you get exactly what is in the docker image rather than your current local sources of airflow. +" +} -$(print_star_line) - Install Airflow if different than current -$(print_star_line) - +function flag_choose_different_airflow_version() { + echo " -a, --install-airflow-version - If specified, installs airflow directly from PIP released version. One of: + If specified, installs airflow directly from PIP released version. This happens at + image building time in production image and at container entering time for CI image. One of: ${FORMATTED_INSTALL_AIRFLOW_VERSIONS} -t, --install-airflow-reference - Only for production image - if specified, installs airflow directly from reference in GitHub - - -$(print_star_line) - Database versions -$(print_star_line) - ---postgres-version - Postgres version used. One of: - -${FORMATTED_POSTGRES_VERSIONS} - - ---mysql-version - Mysql version used. One of: - -${FORMATTED_MYSQL_VERSIONS} - - -$(print_star_line) - Assume answers to questions -$(print_star_line) + If specified, installs airflow directly from reference in GitHub. This happens at + image building time in production image and at container entering time for CI image. +" +} +function flag_assume_answers_to_questions() { + echo " -y, --assume-yes Assume 'yes' answer to all questions. @@ -1335,28 +1393,29 @@ $(print_star_line) -q, --assume-quit Assume 'quit' answer to all questions. +" +} -$(print_star_line) - Credentials -$(print_star_line) - +function flag_credentials() { + echo " -f, --forward-credentials Forwards host credentials to docker container. Use with care as it will make your credentials available to everything you install in Docker. +" +} -$(print_star_line) - Increase verbosity of the script -$(print_star_line) - +function flag_verbosity() { + echo " -v, --verbose Show verbose information about executed commands (enabled by default for running test). + } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS=\"true\"' before running breeze. +" +} -$(print_star_line) - Flags for building the docker images -$(print_star_line) - +function flag_build_docker_images() { + echo " -F, --force-build-images Forces building of the local docker images. The images are rebuilt automatically for the first time or when changes are detected in @@ -1383,27 +1442,77 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS} -L, --use-local-cache Uses local cache to build images. No pulled images will be used, but results of local builds in the Docker cache are used instead. +" +} -$(print_star_line) - Flags for pushing the docker images -$(print_star_line) - +function flag_push_docker_images() { + echo " -u, --push-images After building - uploads the images to DockerHub It is useful in case you use your own DockerHub user to store images and you want to build them locally. Note that you need to use 'docker login' before you upload images. -$(print_star_line) - User and repo used to login to github registry -$(print_star_line) - -D, --dockerhub-user DockerHub user used to pull, push and build images. Default: ${_BREEZE_DEFAULT_DOCKERHUB_USER:=}. -H, --dockerhub-repo DockerHub repository used to pull, push, build images. Default: ${_BREEZE_DEFAULT_DOCKERHUB_REPO:=}. +" +} + +function flags() { + echo " +$(print_line) + +Summary of all flags supported by Breeze: + +$(print_star_line) + Choose Airflow variant +$(flag_airflow_variants) + +$(print_star_line) + Choose backend to run for airflow +$(flag_backend_variants) + +$(print_star_line) + Enable production image +$(flag_production_image) + +$(print_star_line) + Additional actions executed while entering breeze +$(flag_breeze_actions) + +$(print_star_line) + Manage Kind kubernetes cluster (optional) +$(flag_kubernetes_actions) + +$(print_star_line) + Manage mounting local files +$(flag_local_file_mounting) + +$(print_star_line) + Assume answers to questions +$(flag_assume_answers_to_questions) + +$(print_star_line) + Choose different airflow version to install or run +$(flag_choose_different_airflow_version) + +$(print_star_line) + Credentials +$(flag_credentials) + +$(print_star_line) + Flags for building Docker images (both CI and production) +$(flag_build_docker_images) + +$(print_star_line) + Flags for pushing Docker images (both CI and production) +$(flag_push_docker_images) $(print_star_line) + Increase verbosity of the scripts +$(flag_verbosity) " } @@ -1482,7 +1591,7 @@ function touch_local_files { fix_local_file ".bash_aliases" fix_local_file ".inputrc" # When kind cluster is created, the folder keeps authentication information - # across sessiosn + # across sessions mkdir -pv "${MY_DIR}/.kube" } @@ -1506,7 +1615,7 @@ function print_cheatsheet() { echo " * Run command in CI environment : ${BUILD_CACHE_DIR}/${LAST_DC_RUN_CI_FILE} "\ "[command with args] [bash options]" echo " * Run tests in CI environment : ${BUILD_CACHE_DIR}/${LAST_DC_TEST_CI_FILE} "\ - "[test target] [nosetest options]" + "[test target] [pytest options]" echo " * Run docker-compose CI command : ${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE} "\ "[docker compose command] [docker-compose options]" echo " * Run docker-compose production cmd : ${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE} "\ @@ -1670,7 +1779,7 @@ function run_breeze_command { fi ;; run_exec) - # Unfortunately docker-compose exec does not support execing into containers started with run :( + # Unfortunately docker-compose exec does not support exec'ing into containers started with run :( set +e if [[ ${PRODUCTION_IMAGE} == "true" ]]; then DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}" From 7079f13515040672bd5623c18de9b7719ae99008 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 12 Apr 2020 18:51:30 +0200 Subject: [PATCH 2/3] fixup! Help for breeze commands contain relevant flags. --- BREEZE.rst | 4 ++-- breeze-complete | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index f777b7e838006..1655873be8f77 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -743,7 +743,7 @@ This is the current syntax for `./breeze <./breeze>`_: If specified, installs airflow directly from PIP released version. This happens at image building time in production image and at container entering time for CI image. One of: - 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test + 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test -t, --install-airflow-reference If specified, installs airflow directly from reference in GitHub. This happens at @@ -1273,7 +1273,7 @@ This is the current syntax for `./breeze <./breeze>`_: If specified, installs airflow directly from PIP released version. This happens at image building time in production image and at container entering time for CI image. One of: - 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test + 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test -t, --install-airflow-reference If specified, installs airflow directly from reference in GitHub. This happens at diff --git a/breeze-complete b/breeze-complete index f4195a290cfa9..6d74928ba8650 100644 --- a/breeze-complete +++ b/breeze-complete @@ -26,6 +26,7 @@ _BREEZE_ALLOWED_MYSQL_VERSIONS="5.7 8" _BREEZE_ALLOWED_POSTGRES_VERSIONS="9.6 10" _BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS=$(cat <<-EOF +1.10.10 1.10.9 1.10.8 1.10.7 From 41349c68d4e5a66ef87152d44a8ce2b2c2c5c594 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 12 Apr 2020 19:00:02 +0200 Subject: [PATCH 3/3] fixup! fixup! Help for breeze commands contain relevant flags. --- BREEZE.rst | 46 ++++++++++++++++++---------------------- breeze | 61 +++++++++++++++++++++++++++--------------------------- 2 files changed, 50 insertions(+), 57 deletions(-) diff --git a/BREEZE.rst b/BREEZE.rst index 1655873be8f77..55a2beedb1c4c 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -687,7 +687,7 @@ This is the current syntax for `./breeze <./breeze>`_: This is default subcommand if no subcommand is used. - Enters interactive shell where you can run all tests, start airflow webserver, scheduler, + Enters interactive shell where you can run all tests, start Airflow webserver, scheduler, workers, interact with the database, run DAGs etc. It is the default command if no command is selected. The shell is executed in the container and in case integrations are chosen, the integrations will be started as separated docker containers - under the docker-compose @@ -702,7 +702,7 @@ This is the current syntax for `./breeze <./breeze>`_: Flags: - Run 'breeze flags' to see all available applicable flags. + Run 'breeze flags' to see all applicable flags. #################################################################################################### @@ -712,7 +712,7 @@ This is the current syntax for `./breeze <./breeze>`_: breeze [FLAGS] build-docs -- - Builds airflow documentation. The documentation is build inside docker container - to + Builds Airflow documentation. The documentation is build inside docker container - to maintain the same build environment for everyone. Appropriate sources are mapped from the host to the container so that latest sources are used. The folders where documentation is generated ('docs/build') are also mounted to the container - this way results of @@ -740,13 +740,13 @@ This is the current syntax for `./breeze <./breeze>`_: 3.6 3.7 -a, --install-airflow-version - If specified, installs airflow directly from PIP released version. This happens at + If specified, installs Airflow directly from PIP released version. This happens at image building time in production image and at container entering time for CI image. One of: 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test -t, --install-airflow-reference - If specified, installs airflow directly from reference in GitHub. This happens at + If specified, installs Airflow directly from reference in GitHub. This happens at image building time in production image and at container entering time for CI image. -I, --production-image @@ -793,7 +793,6 @@ This is the current syntax for `./breeze <./breeze>`_: -v, --verbose Show verbose information about executed commands (enabled by default for running test). - } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS="true"' before running breeze. @@ -822,7 +821,6 @@ This is the current syntax for `./breeze <./breeze>`_: -v, --verbose Show verbose information about executed commands (enabled by default for running test). - } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS="true"' before running breeze. @@ -850,7 +848,7 @@ This is the current syntax for `./breeze <./breeze>`_: Generates pinned requirements from setup.py. Those requirements are generated in requirements directory - separately for different python version. Those requirements are used to run CI builds as well as run repeatable production image builds. You can use those requirements - to predictably install released airflow versions. You should run it always after you update + to predictably install released Airflow versions. You should run it always after you update setup.py. Flags: @@ -863,7 +861,6 @@ This is the current syntax for `./breeze <./breeze>`_: -v, --verbose Show verbose information about executed commands (enabled by default for running test). - } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS="true"' before running breeze. @@ -923,11 +920,11 @@ This is the current syntax for `./breeze <./breeze>`_: Restarts running docker compose environment. When you restart the environment, the docker containers will be restarted. That includes cleaning up the databases. This is - especially useful if you switch between different versions of airflow. + especially useful if you switch between different versions of Airflow. Flags: - Run 'breeze flags' to see all available applicable flags. + Run 'breeze flags' to see all applicable flags. #################################################################################################### @@ -991,7 +988,6 @@ This is the current syntax for `./breeze <./breeze>`_: -v, --verbose Show verbose information about executed commands (enabled by default for running test). - } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS="true"' before running breeze. @@ -1039,7 +1035,6 @@ This is the current syntax for `./breeze <./breeze>`_: -v, --verbose Show verbose information about executed commands (enabled by default for running test). - } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS="true"' before running breeze. @@ -1111,7 +1106,7 @@ This is the current syntax for `./breeze <./breeze>`_: Flags: - Run 'breeze flags' to see all available applicable flags. + Run 'breeze flags' to see all applicable flags. #################################################################################################### @@ -1161,7 +1156,7 @@ This is the current syntax for `./breeze <./breeze>`_: 3.6 3.7 **************************************************************************************************** - Choose backend to run for airflow + Choose backend to run for Airflow -b, --backend Backend to use for tests - it determines which database is used. @@ -1182,19 +1177,19 @@ This is the current syntax for `./breeze <./breeze>`_: 5.7 8 **************************************************************************************************** - Enable production image + Enable production image -I, --production-image Use production image for entering the environment and builds (not for tests). **************************************************************************************************** - Additional actions executed while entering breeze + Additional actions executed while entering breeze -d, --db-reset Resets the database at entry to the environment. It will drop all the tables and data and recreate the DB from scratch even if 'restart' command was not used. Combined with 'restart' command it enters the environment in the state that is - ready to start airflow webserver/scheduler/worker. Without the switch, the database + ready to start Airflow webserver/scheduler/worker. Without the switch, the database does not have any tables and you need to run reset db manually. -i, --integration @@ -1212,19 +1207,19 @@ This is the current syntax for `./breeze <./breeze>`_: Action for the cluster : only one of the --kind-cluster-* flags can be used at a time: -s, --kind-cluster-start - Starts kind Kubernetes cluster after entering the environment. The cluster is started using + Starts KinD Kubernetes cluster after entering the environment. The cluster is started using Kubernetes Mode selected and Kubernetes version specified via --kubernetes-mode and --kubernetes-version flags. -x, --kind-cluster-stop - Stops kind Kubernetes cluster if one has already been created. By default, if you do not + Stops KinD Kubernetes cluster if one has already been created. By default, if you do not stop environment, the Kubernetes cluster created for testing is continuously running and when you start Kubernetes testing again it will be reused. You can force deletion and recreation of such cluster with this flag. -r, --kind-cluster-recreate - Recreates kind Kubernetes cluster if one has already been created. By default, if you do + Recreates KinD Kubernetes cluster if one has already been created. By default, if you do not stop environment, the Kubernetes cluster created for testing is continuously running and when you start Kubernetes testing again it will be reused. You can force deletion and recreation of such cluster with this flag. @@ -1252,7 +1247,7 @@ This is the current syntax for `./breeze <./breeze>`_: -l, --skip-mounting-local-sources Skips mounting local volume with sources - you get exactly what is in the - docker image rather than your current local sources of airflow. + docker image rather than your current local sources of Airflow. **************************************************************************************************** Assume answers to questions @@ -1267,16 +1262,16 @@ This is the current syntax for `./breeze <./breeze>`_: Assume 'quit' answer to all questions. **************************************************************************************************** - Choose different airflow version to install or run + Choose different Airflow version to install or run -a, --install-airflow-version - If specified, installs airflow directly from PIP released version. This happens at + If specified, installs Airflow directly from PIP released version. This happens at image building time in production image and at container entering time for CI image. One of: 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2 master v1-10-test -t, --install-airflow-reference - If specified, installs airflow directly from reference in GitHub. This happens at + If specified, installs Airflow directly from reference in GitHub. This happens at image building time in production image and at container entering time for CI image. **************************************************************************************************** @@ -1336,7 +1331,6 @@ This is the current syntax for `./breeze <./breeze>`_: -v, --verbose Show verbose information about executed commands (enabled by default for running test). - } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS="true"' before running breeze. diff --git a/breeze b/breeze index cba0b6d88f768..0df6a7846f7e6 100755 --- a/breeze +++ b/breeze @@ -123,12 +123,12 @@ function setup_default_breeze_variables() { # Reset DB at entry export DB_RESET="false" - # If install released airflow is set to specified version, then the source version of airflow - # is removed and the specified version of airflow is installed from PyPi + # If it set is set to specified version, then the source version of Airflow + # is removed and the specified version of Airflow is installed from PyPi export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} - # If install released airflow is set to specified reference (tag/branch), then the source version - # of airflow is removed and the specified version of airflow is installed from GitHub + # If it is set to specified reference (tag/branch), then the source version + # of Airflow is removed and the specified version of Airflow is installed from GitHub export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} # Determine version of the Airflow from version.py @@ -373,7 +373,7 @@ EOF Kubernetes mode: ${KUBERNETES_MODE} Kubernetes version: ${KUBERNETES_VERSION} - Enable kind: ${ENABLE_KIND_CLUSTER} + Enable KinD: ${ENABLE_KIND_CLUSTER} Cluster operation: ${KIND_CLUSTER_OPERATION} EOF fi @@ -419,7 +419,7 @@ EOF Kubernetes mode: ${KUBERNETES_MODE} Kubernetes version: ${KUBERNETES_VERSION} - Enable kind: ${ENABLE_KIND_CLUSTER} + Enable KinD: ${ENABLE_KIND_CLUSTER} Cluster operation: ${KIND_CLUSTER_OPERATION} EOF fi @@ -696,21 +696,21 @@ function parse_arguments() { export RUNTIME=kubernetes export ENABLE_KIND_CLUSTER="true" export KIND_CLUSTER_OPERATION="start" - echo "Starting kubernetes cluster" + echo "Starting kubernetes KinD cluster" echo shift ;; -r|--kind-cluster-recreate) export RUNTIME=kubernetes export ENABLE_KIND_CLUSTER="true" export KIND_CLUSTER_OPERATION="recreate" - echo "Recreating kind cluster" + echo "Recreating KinD cluster" echo shift ;; -x|--kind-cluster-stop) export RUNTIME=kubernetes export ENABLE_KIND_CLUSTER="true" export KIND_CLUSTER_OPERATION="stop" - echo "Stop kind cluster" + echo "Stop KinD cluster" echo shift ;; -L|--use-local-cache) @@ -995,7 +995,7 @@ function prepare_usage() { DETAILED_USAGE_SHELL=" This is default subcommand if no subcommand is used. - Enters interactive shell where you can run all tests, start airflow webserver, scheduler, + Enters interactive shell where you can run all tests, start Airflow webserver, scheduler, workers, interact with the database, run DAGs etc. It is the default command if no command is selected. The shell is executed in the container and in case integrations are chosen, the integrations will be started as separated docker containers - under the docker-compose @@ -1020,7 +1020,7 @@ $(flag_footer) webserver, workers, database console and interactive terminal. " export DETAILED_USAGE_BUILD_DOCS=" - Builds airflow documentation. The documentation is build inside docker container - to + Builds Airflow documentation. The documentation is build inside docker container - to maintain the same build environment for everyone. Appropriate sources are mapped from the host to the container so that latest sources are used. The folders where documentation is generated ('docs/build') are also mounted to the container - this way results of @@ -1095,7 +1095,7 @@ $(flag_verbosity) Generates pinned requirements from setup.py. Those requirements are generated in requirements directory - separately for different python version. Those requirements are used to run CI builds as well as run repeatable production image builds. You can use those requirements - to predictably install released airflow versions. You should run it always after you update + to predictably install released Airflow versions. You should run it always after you update setup.py. Flags: @@ -1129,7 +1129,7 @@ $(flag_airflow_variants) DETAILED_USAGE_RESTART=" Restarts running docker compose environment. When you restart the environment, the docker containers will be restarted. That includes cleaning up the databases. This is - especially useful if you switch between different versions of airflow. + especially useful if you switch between different versions of Airflow. Flags: $(flag_footer) @@ -1256,7 +1256,7 @@ $(get_detailed_usage "${SUBCOMMAND}") function flag_footer() { echo " -Run '${CMDNAME} flags' to see all available applicable flags. +Run '${CMDNAME} flags' to see all applicable flags. " } @@ -1305,7 +1305,7 @@ function flag_breeze_actions() { Resets the database at entry to the environment. It will drop all the tables and data and recreate the DB from scratch even if 'restart' command was not used. Combined with 'restart' command it enters the environment in the state that is - ready to start airflow webserver/scheduler/worker. Without the switch, the database + ready to start Airflow webserver/scheduler/worker. Without the switch, the database does not have any tables and you need to run reset db manually. -i, --integration @@ -1324,19 +1324,19 @@ function flag_kubernetes_actions() { Action for the cluster : only one of the --kind-cluster-* flags can be used at a time: -s, --kind-cluster-start - Starts kind Kubernetes cluster after entering the environment. The cluster is started using + Starts KinD Kubernetes cluster after entering the environment. The cluster is started using Kubernetes Mode selected and Kubernetes version specified via --kubernetes-mode and --kubernetes-version flags. -x, --kind-cluster-stop - Stops kind Kubernetes cluster if one has already been created. By default, if you do not + Stops KinD Kubernetes cluster if one has already been created. By default, if you do not stop environment, the Kubernetes cluster created for testing is continuously running and when you start Kubernetes testing again it will be reused. You can force deletion and recreation of such cluster with this flag. -r, --kind-cluster-recreate - Recreates kind Kubernetes cluster if one has already been created. By default, if you do + Recreates KinD Kubernetes cluster if one has already been created. By default, if you do not stop environment, the Kubernetes cluster created for testing is continuously running and when you start Kubernetes testing again it will be reused. You can force deletion and recreation of such cluster with this flag. @@ -1365,20 +1365,20 @@ function flag_local_file_mounting() { echo " -l, --skip-mounting-local-sources Skips mounting local volume with sources - you get exactly what is in the - docker image rather than your current local sources of airflow. + docker image rather than your current local sources of Airflow. " } function flag_choose_different_airflow_version() { echo " -a, --install-airflow-version - If specified, installs airflow directly from PIP released version. This happens at + If specified, installs Airflow directly from PIP released version. This happens at image building time in production image and at container entering time for CI image. One of: ${FORMATTED_INSTALL_AIRFLOW_VERSIONS} -t, --install-airflow-reference - If specified, installs airflow directly from reference in GitHub. This happens at + If specified, installs Airflow directly from reference in GitHub. This happens at image building time in production image and at container entering time for CI image. " } @@ -1408,7 +1408,6 @@ function flag_verbosity() { echo " -v, --verbose Show verbose information about executed commands (enabled by default for running test). - } Note that you can further increase verbosity and see all the commands executed by breeze by running 'export VERBOSE_COMMANDS=\"true\"' before running breeze. " @@ -1471,15 +1470,15 @@ $(print_star_line) $(flag_airflow_variants) $(print_star_line) - Choose backend to run for airflow + Choose backend to run for Airflow $(flag_backend_variants) $(print_star_line) - Enable production image + Enable production image $(flag_production_image) $(print_star_line) - Additional actions executed while entering breeze + Additional actions executed while entering breeze $(flag_breeze_actions) $(print_star_line) @@ -1495,7 +1494,7 @@ $(print_star_line) $(flag_assume_answers_to_questions) $(print_star_line) - Choose different airflow version to install or run + Choose different Airflow version to install or run $(flag_choose_different_airflow_version) $(print_star_line) @@ -1590,7 +1589,7 @@ function touch_local_files { fix_local_file ".bash_history" fix_local_file ".bash_aliases" fix_local_file ".inputrc" - # When kind cluster is created, the folder keeps authentication information + # When KinD cluster is created, the folder keeps authentication information # across sessions mkdir -pv "${MY_DIR}/.kube" } @@ -1627,7 +1626,7 @@ function print_cheatsheet() { print_line echo echo " Adding breeze to your path:" - echo " When you exit the environment, you can add sources of airflow to the path - you can" + echo " When you exit the environment, you can add sources of Airflow to the path - you can" echo " run breeze or the scripts above from any directory by calling 'breeze' commands directly" echo echo " export PATH=\${PATH}:\"${MY_DIR}\"" @@ -1640,9 +1639,9 @@ function print_cheatsheet() { echo " Port forwarding:" echo echo " Ports are forwarded to the running docker containers for webserver and database" - echo " * ${WEBSERVER_HOST_PORT} -> forwarded to airflow webserver -> airflow:8080" - echo " * ${POSTGRES_HOST_PORT} -> forwarded to postgres database -> postgres:5432" - echo " * ${MYSQL_HOST_PORT} -> forwarded to mysql database -> mysql:3306" + echo " * ${WEBSERVER_HOST_PORT} -> forwarded to Airflow webserver -> airflow:8080" + echo " * ${POSTGRES_HOST_PORT} -> forwarded to Postgres database -> postgres:5432" + echo " * ${MYSQL_HOST_PORT} -> forwarded to MySQL database -> mysql:3306" echo echo " Here are links to those services that you can use on host:" echo " * Webserver: http://127.0.0.1:28080"