[BEAM-10891] Standardized developer build environment using Docker

dev-support/docker/Dockerfile

@@ -0,0 +1,117 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+
+# hadolint ignore=DL3008
+###
+# Update and install common packages
+###
+RUN apt -q update \
+   && apt install -y software-properties-common apt-utils apt-transport-https ca-certificates \
+   && add-apt-repository -y ppa:deadsnakes/ppa \
+   && apt-get -q install -y --no-install-recommends \
+        bash-completion \
+        build-essential \
+        bzip2 \
+        wget \
+        curl \
+        docker.io \
+        g++ \
+        gcc \
+        git \
+        gnupg-agent \
+        rsync \
+        sudo \
+        vim \
+        locales \
+        wget \
+        time \
+        openjdk-8-jdk \
+        python3-setuptools \
+        python3-pip \
+        python3.6 \
+        python3.7 \
+        python3.8 \
+        virtualenv \
+        tox
+
+###
+# Set the locale ( see https://stackoverflow.com/a/28406007/114196 )
+###
+# TODO(BEAM-11327): Remove the need to run tests with UTF-8 encoding
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+###
+# Set Python3.6 as default
+###
+RUN alias python=python3.6
+
+###
+# Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work
+###
+RUN pip3 install grpcio-tools mypy-protobuf
+
+###
+# Install useful tools
+# Install distlib to avoid https://github.com/pypa/virtualenv/issues/2006
+RUN pip3 install distlib==0.3.1 yapf==0.29.0 pytest
+###
+
+###
+# Install Go
+###
+ENV DOWNLOAD_GO_VERSION=1.12.17
+RUN wget https://golang.org/dl/go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz
+ENV GOROOT /usr/local/go
+ENV PATH $PATH:$GOROOT/bin
+
+###
+# Add a welcome message and environment checks.
+###
+RUN mkdir /scripts
+COPY beam_env_checks.sh /scripts/beam_env_checks.sh
+COPY bashcolors.sh      /scripts/bashcolors.sh
+RUN chmod 755 /scripts /scripts/beam_env_checks.sh /scripts/bashcolors.sh
+
+# hadolint ignore=SC2016
+RUN echo '. /etc/bash_completion'        >> /root/.bash_aliases
+RUN echo '. /scripts/beam_env_checks.sh' >> /root/.bash_aliases

dev-support/docker/bashcolors.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#https://wiki.archlinux.org/index.php/Color_Bash_Prompt
+# Reset
+export Color_Off='\e[0m'      # Text Reset
+
+# Regular Colors
+export Black='\e[0;30m'       # Black
+export Red='\e[0;31m'         # Red
+export Green='\e[0;32m'       # Green
+export Yellow='\e[0;33m'      # Yellow
+export Blue='\e[0;34m'        # Blue
+export Purple='\e[0;35m'      # Purple
+export Cyan='\e[0;36m'        # Cyan
+export White='\e[0;37m'       # White
+
+# Bold
+export BBlack='\e[1;30m'      # Black
+export BRed='\e[1;31m'        # Red
+export BGreen='\e[1;32m'      # Green
+export BYellow='\e[1;33m'     # Yellow
+export BBlue='\e[1;34m'       # Blue
+export BPurple='\e[1;35m'     # Purple
+export BCyan='\e[1;36m'       # Cyan
+export BWhite='\e[1;37m'      # White
+
+# Underline
+export UBlack='\e[4;30m'      # Black
+export URed='\e[4;31m'        # Red
+export UGreen='\e[4;32m'      # Green
+export UYellow='\e[4;33m'     # Yellow
+export UBlue='\e[4;34m'       # Blue
+export UPurple='\e[4;35m'     # Purple
+export UCyan='\e[4;36m'       # Cyan
+export UWhite='\e[4;37m'      # White
+
+# Background
+export On_Black='\e[40m'      # Black
+export On_Red='\e[41m'        # Red
+export On_Green='\e[42m'      # Green
+export On_Yellow='\e[43m'     # Yellow
+export On_Blue='\e[44m'       # Blue
+export On_Purple='\e[45m'     # Purple
+export On_Cyan='\e[46m'       # Cyan
+export On_White='\e[47m'      # White
+
+# High Intensity
+export IBlack='\e[0;90m'      # Black
+export IRed='\e[0;91m'        # Red
+export IGreen='\e[0;92m'      # Green
+export IYellow='\e[0;93m'     # Yellow
+export IBlue='\e[0;94m'       # Blue
+export IPurple='\e[0;95m'     # Purple
+export ICyan='\e[0;96m'       # Cyan
+export IWhite='\e[0;97m'      # White
+
+# Bold High Intensity
+export BIBlack='\e[1;90m'     # Black
+export BIRed='\e[1;91m'       # Red
+export BIGreen='\e[1;92m'     # Green
+export BIYellow='\e[1;93m'    # Yellow
+export BIBlue='\e[1;94m'      # Blue
+export BIPurple='\e[1;95m'    # Purple
+export BICyan='\e[1;96m'      # Cyan
+export BIWhite='\e[1;97m'     # White
+
+# High Intensity backgrounds
+export On_IBlack='\e[0;100m'  # Black
+export On_IRed='\e[0;101m'    # Red
+export On_IGreen='\e[0;102m'  # Green
+export On_IYellow='\e[0;103m' # Yellow
+export On_IBlue='\e[0;104m'   # Blue
+export On_IPurple='\e[0;105m' # Purple
+export On_ICyan='\e[0;106m'   # Cyan
+export On_IWhite='\e[0;107m'  # White

dev-support/docker/beam_env_checks.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SHELLDOC-IGNORE
+
+# -------------------------------------------------------
+function showWelcome {
+# http://patorjk.com/software/taag/#p=display&f=Doom&t=Beam%20Build%20Env.
+cat << "Welcome-message"
+
+______                       ______       _ _     _   _____
+| ___ \                      | ___ \     (_) |   | | |  ___|
+| |_/ / ___  __ _ _ __ ___   | |_/ /_   _ _| | __| | | |__ _ ____   __
+| ___ \/ _ \/ _` | '_ ` _ \  | ___ \ | | | | |/ _` | |  __| '_ \ \ / /
+| |_/ /  __/ (_| | | | | | | | |_/ / |_| | | | (_| | | |__| | | \ V /
+\____/ \___|\__,_|_| |_| |_| \____/ \__,_|_|_|\__,_| \____/_| |_|\_(_)
+
+This is the standard Beam Developer build environment.
+This has all the right tools installed required to build
+Apache Beam from source.
+
+Welcome-message
+}
+
+# -------------------------------------------------------
+
+function showAbort {
+  cat << "Abort-message"
+
+  ___  _                _   _
+ / _ \| |              | | (_)
+/ /_\ \ |__   ___  _ __| |_ _ _ __   __ _
+|  _  | '_ \ / _ \| '__| __| | '_ \ / _\` |
+| | | | |_) | (_) | |  | |_| | | | | (_| |
+\_| |_/_.__/ \___/|_|   \__|_|_| |_|\__, |
+                                     __/ |
+                                    |___/
+
+Abort-message
+}
+
+# -------------------------------------------------------
+
+function failIfUserIsRoot {
+    if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong.
+    then
+        cat <<End-of-message
+
+Apparently you are inside this docker container as the user root.
+Putting it simply:
+
+   This should not occur.
+
+Known possible causes of this are:
+1) Running this script as the root user ( Just don't )
+   Check https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user
+2) Running an old docker version ( upgrade to 1.4.1 or higher )
+
+End-of-message
+
+    showAbort
+
+    logout
+
+    fi
+}
+
+# -------------------------------------------------------
+
+function warnIfLowMemory {
+    MINIMAL_MEMORY=2046755
+    INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
+    if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then
+        cat <<End-of-message
+
+ _                    ___  ___
+| |                   |  \\/  |
+| |     _____      __ | .  . | ___ _ __ ___   ___  _ __ _   _
+| |    / _ \\ \\ /\\ / / | |\\/| |/ _ \\ '_ \` _ \\ / _ \\| '__| | | |
+| |___| (_) \\ V  V /  | |  | |  __/ | | | | | (_) | |  | |_| |
+\\_____/\\___/ \\_/\\_/   \\_|  |_/\\___|_| |_| |_|\\___/|_|   \\__, |
+                                                         __/ |
+                                                        |___/
+
+Your system is running on very little memory.
+This means it may work but it wil most likely be slower than needed.
+
+If you are running this via boot2docker you can simply increase
+the available memory to at least ${MINIMAL_MEMORY}KiB
+(you have ${INSTALLED_MEMORY}KiB )
+
+End-of-message
+    fi
+}
+
+# -------------------------------------------------------
+
+showWelcome
+warnIfLowMemory
+failIfUserIsRoot
+
+# -------------------------------------------------------
+
+. "/scripts/bashcolors.sh"
+. "/usr/lib/git-core/git-sh-prompt"
+export PS1='\['${IBlue}${On_Black}'\] \u@\['${IWhite}${On_Red}'\][Beam Build Env.]\['${IBlue}${On_Black}'\]:\['${Cyan}${On_Black}'\]\w$(declare -F __git_ps1 &>/dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ '

sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java

@@ -45,7 +45,7 @@ public interface Timers {
   /** Set a timer with outputTimestamp. */
   void setTimer(Instant timestamp, Instant outputTimestamp, TimeDomain timeDomain);
 
-  /** Removes the timer set in this context for the {@code timestmap} and {@code timeDomain}. */
+  /** Removes the timer set in this context for the {@code timestamp} and {@code timeDomain}. */
   void deleteTimer(Instant timestamp, TimeDomain timeDomain);
 
   /** Returns the current processing time. */

sdks/python/build-requirements.txt

@@ -17,3 +17,6 @@
 grpcio-tools==1.30.0
 future==0.18.2
 mypy-protobuf==1.18
+
+# Avoid https://github.com/pypa/virtualenv/issues/2006
+distlib==0.3.1