From e3c10f1d62fba2c04f5de347a899f7bd6161ada0 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Wed, 11 Nov 2020 17:14:40 +0100 Subject: [PATCH 1/6] [BEAM-10891] Standardized build environment using Docker --- dev-support/docker/Dockerfile | 110 +++++++++++++++ dev-support/docker/bashcolors.sh | 91 ++++++++++++ dev-support/docker/beam_env_checks.sh | 121 ++++++++++++++++ .../org/apache/beam/sdk/state/Timers.java | 2 +- start-build-env.sh | 132 ++++++++++++++++++ .../www/site/content/en/contribute/_index.md | 9 ++ 6 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 dev-support/docker/Dockerfile create mode 100755 dev-support/docker/bashcolors.sh create mode 100755 dev-support/docker/beam_env_checks.sh create mode 100755 start-build-env.sh diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile new file mode 100644 index 000000000000..b9896885b414 --- /dev/null +++ b/dev-support/docker/Dockerfile @@ -0,0 +1,110 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + + +# hadolint ignore=DL3008 +### +# Update and install common packages +### +RUN apt -q update \ + && apt install -y software-properties-common apt-utils apt-transport-https ca-certificates \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get -q install -y --no-install-recommends \ + bash-completion \ + build-essential \ + bzip2 \ + curl \ + docker.io \ + g++ \ + gcc \ + git \ + gnupg-agent \ + rsync \ + sudo \ + vim \ + locales \ + wget \ + time \ + openjdk-8-jdk \ + python3-setuptools \ + python3-pip \ + python3.6 \ + python3.7 \ + python3.8 \ + virtualenv \ + tox + +### +# Set the locale ( see https://stackoverflow.com/a/28406007/114196 ) +### +# TODO(BEAM-11327): Remove the need to run tests with UTF-8 encoding +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +### +# Set Python3.6 as default +### +RUN alias python=python3.6 + +### +# Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work +### +RUN pip3 install grpcio-tools mypy-protobuf + +### +# Install Go +### +ENV DOWNLOAD_GO_VERSION=1.12.17 +RUN wget https://golang.org/dl/go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz +ENV GOROOT /usr/local/go +ENV PATH $PATH:$GOROOT/bin + +### +# Add a welcome message and environment checks. +### +RUN mkdir /scripts +COPY beam_env_checks.sh /scripts/beam_env_checks.sh +COPY bashcolors.sh /scripts/bashcolors.sh +RUN chmod 755 /scripts /scripts/beam_env_checks.sh /scripts/bashcolors.sh + +# hadolint ignore=SC2016 +RUN echo '. /etc/bash_completion' >> /root/.bash_aliases +RUN echo '. /scripts/beam_env_checks.sh' >> /root/.bash_aliases diff --git a/dev-support/docker/bashcolors.sh b/dev-support/docker/bashcolors.sh new file mode 100755 index 000000000000..1e352fce66f7 --- /dev/null +++ b/dev-support/docker/bashcolors.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#https://wiki.archlinux.org/index.php/Color_Bash_Prompt +# Reset +export Color_Off='\e[0m' # Text Reset + +# Regular Colors +export Black='\e[0;30m' # Black +export Red='\e[0;31m' # Red +export Green='\e[0;32m' # Green +export Yellow='\e[0;33m' # Yellow +export Blue='\e[0;34m' # Blue +export Purple='\e[0;35m' # Purple +export Cyan='\e[0;36m' # Cyan +export White='\e[0;37m' # White + +# Bold +export BBlack='\e[1;30m' # Black +export BRed='\e[1;31m' # Red +export BGreen='\e[1;32m' # Green +export BYellow='\e[1;33m' # Yellow +export BBlue='\e[1;34m' # Blue +export BPurple='\e[1;35m' # Purple +export BCyan='\e[1;36m' # Cyan +export BWhite='\e[1;37m' # White + +# Underline +export UBlack='\e[4;30m' # Black +export URed='\e[4;31m' # Red +export UGreen='\e[4;32m' # Green +export UYellow='\e[4;33m' # Yellow +export UBlue='\e[4;34m' # Blue +export UPurple='\e[4;35m' # Purple +export UCyan='\e[4;36m' # Cyan +export UWhite='\e[4;37m' # White + +# Background +export On_Black='\e[40m' # Black +export On_Red='\e[41m' # Red +export On_Green='\e[42m' # Green +export On_Yellow='\e[43m' # Yellow +export On_Blue='\e[44m' # Blue +export On_Purple='\e[45m' # Purple +export On_Cyan='\e[46m' # Cyan +export On_White='\e[47m' # White + +# High Intensity +export IBlack='\e[0;90m' # Black +export IRed='\e[0;91m' # Red +export IGreen='\e[0;92m' # Green +export IYellow='\e[0;93m' # Yellow +export IBlue='\e[0;94m' # Blue +export IPurple='\e[0;95m' # Purple +export ICyan='\e[0;96m' # Cyan +export IWhite='\e[0;97m' # White + +# Bold High Intensity +export BIBlack='\e[1;90m' # Black +export BIRed='\e[1;91m' # Red +export BIGreen='\e[1;92m' # Green +export BIYellow='\e[1;93m' # Yellow +export BIBlue='\e[1;94m' # Blue +export BIPurple='\e[1;95m' # Purple +export BICyan='\e[1;96m' # Cyan +export BIWhite='\e[1;97m' # White + +# High Intensity backgrounds +export On_IBlack='\e[0;100m' # Black +export On_IRed='\e[0;101m' # Red +export On_IGreen='\e[0;102m' # Green +export On_IYellow='\e[0;103m' # Yellow +export On_IBlue='\e[0;104m' # Blue +export On_IPurple='\e[0;105m' # Purple +export On_ICyan='\e[0;106m' # Cyan +export On_IWhite='\e[0;107m' # White diff --git a/dev-support/docker/beam_env_checks.sh b/dev-support/docker/beam_env_checks.sh new file mode 100755 index 000000000000..1a50166766e6 --- /dev/null +++ b/dev-support/docker/beam_env_checks.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +# ------------------------------------------------------- +function showWelcome { +# http://patorjk.com/software/taag/#p=display&f=Doom&t=Beam%20Build%20Env. +cat << "Welcome-message" + +______ ______ _ _ _ _____ +| ___ \ | ___ \ (_) | | | | ___| +| |_/ / ___ __ _ _ __ ___ | |_/ /_ _ _| | __| | | |__ _ ____ __ +| ___ \/ _ \/ _` | '_ ` _ \ | ___ \ | | | | |/ _` | | __| '_ \ \ / / +| |_/ / __/ (_| | | | | | | | |_/ / |_| | | | (_| | | |__| | | \ V / +\____/ \___|\__,_|_| |_| |_| \____/ \__,_|_|_|\__,_| \____/_| |_|\_(_) + +This is the standard Beam Developer build environment. +This has all the right tools installed required to build +Apache Beam from source. + +Welcome-message +} + +# ------------------------------------------------------- + +function showAbort { + cat << "Abort-message" + + ___ _ _ _ + / _ \| | | | (_) +/ /_\ \ |__ ___ _ __| |_ _ _ __ __ _ +| _ | '_ \ / _ \| '__| __| | '_ \ / _\` | +| | | | |_) | (_) | | | |_| | | | | (_| | +\_| |_/_.__/ \___/|_| \__|_|_| |_|\__, | + __/ | + |___/ + +Abort-message +} + +# ------------------------------------------------------- + +function failIfUserIsRoot { + if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong. + then + cat </dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ ' diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java index 3da3bb04630e..bca398765d19 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java @@ -45,7 +45,7 @@ public interface Timers { /** Set a timer with outputTimestamp. */ void setTimer(Instant timestamp, Instant outputTimestamp, TimeDomain timeDomain); - /** Removes the timer set in this context for the {@code timestmap} and {@code timeDomain}. */ + /** Removes the timer set in this context for the {@code timestamp} and {@code timeDomain}. */ void deleteTimer(Instant timestamp, TimeDomain timeDomain); /** Returns the current processing time. */ diff --git a/start-build-env.sh b/start-build-env.sh new file mode 100755 index 000000000000..52cebe45455f --- /dev/null +++ b/start-build-env.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e # exit on error + +cd "$(dirname "$0")" # connect to root + +DOCKER_DIR=dev-support/docker +DOCKER_FILE="${DOCKER_DIR}/Dockerfile" + +CONTAINER_NAME=beam-dev-${USER}-$$ + +if [ ! -x "$(command -v docker)" ] ; then + echo "Unable to locate docker" + echo "This Beam Build environment is based upon docker so you must install it first." + exit 1; +fi + +docker build -t beam-build -f $DOCKER_FILE $DOCKER_DIR + +USER_NAME=${SUDO_USER:=$USER} +USER_ID=$(id -u "${USER_NAME}") + +if [ "$(uname -s)" = "Darwin" ]; then + GROUP_ID=100 +fi + +if [ "$(uname -s)" = "Linux" ]; then + GROUP_ID=$(id -g "${USER_NAME}") + # man docker-run + # When using SELinux, mounted directories may not be accessible + # to the container. To work around this, with Docker prior to 1.7 + # one needs to run the "chcon -Rt svirt_sandbox_file_t" command on + # the directories. With Docker 1.7 and later the z mount option + # does this automatically. + if command -v selinuxenabled >/dev/null && selinuxenabled; then + DCKR_VER=$(docker -v| + awk '$1 == "Docker" && $2 == "version" {split($3,ver,".");print ver[1]"."ver[2]}') + DCKR_MAJ=${DCKR_VER%.*} + DCKR_MIN=${DCKR_VER#*.} + if [ "${DCKR_MAJ}" -eq 1 ] && [ "${DCKR_MIN}" -ge 7 ] || + [ "${DCKR_MAJ}" -gt 1 ]; then + V_OPTS=:z + else + for d in "${PWD}" "${HOME}/.m2"; do + ctx=$(stat --printf='%C' "$d"|cut -d':' -f3) + if [ "$ctx" != svirt_sandbox_file_t ] && [ "$ctx" != container_file_t ]; then + printf 'INFO: SELinux is enabled.\n' + printf '\tMounted %s may not be accessible to the container.\n' "$d" + printf 'INFO: If so, on the host, run the following command:\n' + printf '\t# chcon -Rt svirt_sandbox_file_t %s\n' "$d" + fi + done + fi + fi +fi + +# Set the home directory in the Docker container. +DOCKER_HOME_DIR=${DOCKER_HOME_DIR:-/home/${USER_NAME}} + +DOCKER_GROUP_ID=$(getent group docker | cut -d':' -f3) + +docker build -t "beam-build-${USER_ID}" - < "/etc/sudoers.d/beam-build-${USER_ID}" +ENV HOME "${DOCKER_HOME_DIR}" +ENV GOPATH ${HOME}/beam/sdks/go/examples/.gogradle/project_gopath + +# This next command still runs as root causing the ~/.cache/go-build to be owned by root +RUN go get github.com/linkedin/goavro +RUN chown -R ${USER_NAME}:${GROUP_ID} ${HOME}/.cache +UserSpecificDocker + +echo "" +echo "Docker image build completed." +echo "==============================================================================================" +echo "" + +# If this env variable is empty, docker will be started +# in non interactive mode +DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"} + +DOCKER_SOCKET_MOUNT="" +if [ -S /var/run/docker.sock ]; +then + DOCKER_SOCKET_MOUNT="-v /var/run/docker.sock:/var/run/docker.sock${V_OPTS:-}" + echo "Enabling Docker support with the docker build environment." +else + echo "There is NO Docker support with the docker build environment." +fi + +COMMAND=( "$@" ) +if [ $# -eq 0 ]; +then + COMMAND=( "bash" ) +fi + +[ -d "${HOME}/.beam_docker_build_env/.gradle" ] || mkdir -p "${HOME}/.beam_docker_build_env/.gradle" + +# By mapping the .m2 directory you can do an mvn install from +# within the container and use the result on your normal +# system. And this also is a significant speedup in subsequent +# builds because the dependencies are downloaded only once. +docker run --rm=true ${DOCKER_INTERACTIVE_RUN} \ + --name "${CONTAINER_NAME}" \ + --network=host \ + -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \ + -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \ + -v "${HOME}/.beam_docker_build_env/.gradle:${DOCKER_HOME_DIR}/.gradle${V_OPTS:-}" \ + -v "${PWD}:${DOCKER_HOME_DIR}/beam${V_OPTS:-}" \ + -w "${DOCKER_HOME_DIR}/beam" \ + ${DOCKER_SOCKET_MOUNT} \ + -u "${USER_ID}" \ + "beam-build-${USER_ID}" "${COMMAND[@]}" diff --git a/website/www/site/content/en/contribute/_index.md b/website/www/site/content/en/contribute/_index.md index dd5e90eb8a8d..6bcafb0414f1 100644 --- a/website/www/site/content/en/contribute/_index.md +++ b/website/www/site/content/en/contribute/_index.md @@ -94,6 +94,15 @@ $ go get github.com/linkedin/goavro gLinux users should configure their machines for sudoless Docker. +Alternatively, you can use the Docker based local development environment to wrap your clone of the Beam repo +into a container meeting the requirements above. + +You can start this container using the [start-build-env.sh](https://github.com/apache/beam/blob/master/start-build-env.sh) +script which is part of the Beam repo: +``` +./start-build-env.sh +``` + ### Connect With the Beam community 1. Consider subscribing to the [dev@ mailing list](/community/contact-us/), especially From b93131be8876e6d3577a94af09a00da498029135 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Wed, 2 Dec 2020 13:16:37 +0100 Subject: [PATCH 2/6] Document additional python tools needed to build --- website/www/site/content/en/contribute/_index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/website/www/site/content/en/contribute/_index.md b/website/www/site/content/en/contribute/_index.md index 6bcafb0414f1..4a095b70ba38 100644 --- a/website/www/site/content/en/contribute/_index.md +++ b/website/www/site/content/en/contribute/_index.md @@ -83,6 +83,11 @@ sudo apt-get install \ docker-ce ``` +On some systems (like Ubuntu 20.04) these need to be installed also +``` +pip3 install grpcio-tools mypy-protobuf +``` + You also need to [install Go](https://golang.org/doc/install). Once Go is installed, install goavro: From e5b45b856523943a27daf5ac726025cfaaf1c887 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Wed, 2 Dec 2020 14:00:34 +0100 Subject: [PATCH 3/6] Add pyenv and pyenv-virtualenv to the docker image --- dev-support/docker/Dockerfile | 40 ++++++++++++++++++++++++++- dev-support/docker/beam_env_checks.sh | 5 ++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index b9896885b414..aa866a41f13f 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -47,6 +47,7 @@ RUN apt -q update \ bash-completion \ build-essential \ bzip2 \ + wget \ curl \ docker.io \ g++ \ @@ -66,7 +67,22 @@ RUN apt -q update \ python3.7 \ python3.8 \ virtualenv \ - tox + tox \ + # Needed for pyenv: + make \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + python-openssl ### # Set the locale ( see https://stackoverflow.com/a/28406007/114196 ) @@ -88,6 +104,28 @@ RUN alias python=python3.6 ### RUN pip3 install grpcio-tools mypy-protobuf +### +# Install pyenv and pyenv-virtualenv +### +ENV DOWNLOAD_PYENV_VERSION=v1.2.21 +ENV DOWNLOAD_PYENV_VIRTUALENV_VERSION=v1.1.5 + +RUN git clone https://github.com/pyenv/pyenv.git /opt/pyenv && \ + cd /opt/pyenv && \ + chmod a+rwX -R . && \ + git checkout "${DOWNLOAD_PYENV_VERSION}" + +RUN git clone https://github.com/pyenv/pyenv-virtualenv.git /opt/pyenv/plugins/pyenv-virtualenv && \ + cd /opt/pyenv/plugins/pyenv-virtualenv && \ + chmod a+rwX -R . && \ + git checkout "${DOWNLOAD_PYENV_VIRTUALENV_VERSION}" + +ENV PYENV_ROOT /opt/pyenv +ENV PATH $PYENV_ROOT/bin:$PATH +RUN pyenv install 3.6.12 +RUN pyenv install 3.7.9 +RUN pyenv install 3.8.6 + ### # Install Go ### diff --git a/dev-support/docker/beam_env_checks.sh b/dev-support/docker/beam_env_checks.sh index 1a50166766e6..3d604feea3f1 100755 --- a/dev-support/docker/beam_env_checks.sh +++ b/dev-support/docker/beam_env_checks.sh @@ -119,3 +119,8 @@ failIfUserIsRoot . "/scripts/bashcolors.sh" . "/usr/lib/git-core/git-sh-prompt" export PS1='\['${IBlue}${On_Black}'\] \u@\['${IWhite}${On_Red}'\][Beam Build Env.]\['${IBlue}${On_Black}'\]:\['${Cyan}${On_Black}'\]\w$(declare -F __git_ps1 &>/dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ ' + +# Configure pyenv and pyenv-virtualenv +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +. "/opt/pyenv/completions/pyenv.bash" \ No newline at end of file From 3c715c64d3f169f44c5aab3bb0dcb13b926f7396 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Wed, 2 Dec 2020 16:24:42 +0100 Subject: [PATCH 4/6] Attempt workaround wrong python distlib version --- dev-support/docker/Dockerfile | 14 ++++++++++---- sdks/python/build-requirements.txt | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index aa866a41f13f..7bc31d63a1f0 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -104,6 +104,11 @@ RUN alias python=python3.6 ### RUN pip3 install grpcio-tools mypy-protobuf +### +# Avoid https://github.com/pypa/virtualenv/issues/2006 +RUN pip3 install distlib==0.3.1 +### + ### # Install pyenv and pyenv-virtualenv ### @@ -112,19 +117,20 @@ ENV DOWNLOAD_PYENV_VIRTUALENV_VERSION=v1.1.5 RUN git clone https://github.com/pyenv/pyenv.git /opt/pyenv && \ cd /opt/pyenv && \ - chmod a+rwX -R . && \ - git checkout "${DOWNLOAD_PYENV_VERSION}" + git checkout "${DOWNLOAD_PYENV_VERSION}" && \ + chmod a+rwX -R . RUN git clone https://github.com/pyenv/pyenv-virtualenv.git /opt/pyenv/plugins/pyenv-virtualenv && \ cd /opt/pyenv/plugins/pyenv-virtualenv && \ - chmod a+rwX -R . && \ - git checkout "${DOWNLOAD_PYENV_VIRTUALENV_VERSION}" + git checkout "${DOWNLOAD_PYENV_VIRTUALENV_VERSION}" && \ + chmod a+rwX -R . ENV PYENV_ROOT /opt/pyenv ENV PATH $PYENV_ROOT/bin:$PATH RUN pyenv install 3.6.12 RUN pyenv install 3.7.9 RUN pyenv install 3.8.6 +RUN chmod a+rwX -R /opt/pyenv ### # Install Go diff --git a/sdks/python/build-requirements.txt b/sdks/python/build-requirements.txt index 1ecd6ec10a50..415ac340ab1d 100644 --- a/sdks/python/build-requirements.txt +++ b/sdks/python/build-requirements.txt @@ -17,3 +17,6 @@ grpcio-tools==1.30.0 future==0.18.2 mypy-protobuf==1.18 + +# Avoid https://github.com/pypa/virtualenv/issues/2006 +distlib==0.3.1 \ No newline at end of file From fb46a1423120b2c0ee6ea18b75834fad2c60c63d Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Fri, 4 Dec 2020 16:51:48 +0100 Subject: [PATCH 5/6] Install extra tools --- dev-support/docker/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 7bc31d63a1f0..046a0e4d8011 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -105,8 +105,9 @@ RUN alias python=python3.6 RUN pip3 install grpcio-tools mypy-protobuf ### -# Avoid https://github.com/pypa/virtualenv/issues/2006 -RUN pip3 install distlib==0.3.1 +# Install useful tools +# Install distlib to avoid https://github.com/pypa/virtualenv/issues/2006 +RUN pip3 install distlib==0.3.1 yapf==0.29.0 pytest ### ### From 503d140fee4a397aba74e41328ff1239dbfa16e1 Mon Sep 17 00:00:00 2001 From: Niels Basjes Date: Fri, 4 Dec 2020 22:52:03 +0100 Subject: [PATCH 6/6] Drop pyenv from the image --- dev-support/docker/Dockerfile | 40 +-------------------------- dev-support/docker/beam_env_checks.sh | 5 ---- 2 files changed, 1 insertion(+), 44 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 046a0e4d8011..92e5a8cdf845 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -67,22 +67,7 @@ RUN apt -q update \ python3.7 \ python3.8 \ virtualenv \ - tox \ - # Needed for pyenv: - make \ - libssl-dev \ - zlib1g-dev \ - libbz2-dev \ - libreadline-dev \ - libsqlite3-dev \ - llvm \ - libncurses5-dev \ - libncursesw5-dev \ - xz-utils \ - tk-dev \ - libffi-dev \ - liblzma-dev \ - python-openssl + tox ### # Set the locale ( see https://stackoverflow.com/a/28406007/114196 ) @@ -110,29 +95,6 @@ RUN pip3 install grpcio-tools mypy-protobuf RUN pip3 install distlib==0.3.1 yapf==0.29.0 pytest ### -### -# Install pyenv and pyenv-virtualenv -### -ENV DOWNLOAD_PYENV_VERSION=v1.2.21 -ENV DOWNLOAD_PYENV_VIRTUALENV_VERSION=v1.1.5 - -RUN git clone https://github.com/pyenv/pyenv.git /opt/pyenv && \ - cd /opt/pyenv && \ - git checkout "${DOWNLOAD_PYENV_VERSION}" && \ - chmod a+rwX -R . - -RUN git clone https://github.com/pyenv/pyenv-virtualenv.git /opt/pyenv/plugins/pyenv-virtualenv && \ - cd /opt/pyenv/plugins/pyenv-virtualenv && \ - git checkout "${DOWNLOAD_PYENV_VIRTUALENV_VERSION}" && \ - chmod a+rwX -R . - -ENV PYENV_ROOT /opt/pyenv -ENV PATH $PYENV_ROOT/bin:$PATH -RUN pyenv install 3.6.12 -RUN pyenv install 3.7.9 -RUN pyenv install 3.8.6 -RUN chmod a+rwX -R /opt/pyenv - ### # Install Go ### diff --git a/dev-support/docker/beam_env_checks.sh b/dev-support/docker/beam_env_checks.sh index 3d604feea3f1..1a50166766e6 100755 --- a/dev-support/docker/beam_env_checks.sh +++ b/dev-support/docker/beam_env_checks.sh @@ -119,8 +119,3 @@ failIfUserIsRoot . "/scripts/bashcolors.sh" . "/usr/lib/git-core/git-sh-prompt" export PS1='\['${IBlue}${On_Black}'\] \u@\['${IWhite}${On_Red}'\][Beam Build Env.]\['${IBlue}${On_Black}'\]:\['${Cyan}${On_Black}'\]\w$(declare -F __git_ps1 &>/dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ ' - -# Configure pyenv and pyenv-virtualenv -eval "$(pyenv init -)" -eval "$(pyenv virtualenv-init -)" -. "/opt/pyenv/completions/pyenv.bash" \ No newline at end of file