diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile new file mode 100644 index 000000000000..92e5a8cdf845 --- /dev/null +++ b/dev-support/docker/Dockerfile @@ -0,0 +1,117 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + + +# hadolint ignore=DL3008 +### +# Update and install common packages +### +RUN apt -q update \ + && apt install -y software-properties-common apt-utils apt-transport-https ca-certificates \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get -q install -y --no-install-recommends \ + bash-completion \ + build-essential \ + bzip2 \ + wget \ + curl \ + docker.io \ + g++ \ + gcc \ + git \ + gnupg-agent \ + rsync \ + sudo \ + vim \ + locales \ + wget \ + time \ + openjdk-8-jdk \ + python3-setuptools \ + python3-pip \ + python3.6 \ + python3.7 \ + python3.8 \ + virtualenv \ + tox + +### +# Set the locale ( see https://stackoverflow.com/a/28406007/114196 ) +### +# TODO(BEAM-11327): Remove the need to run tests with UTF-8 encoding +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +### +# Set Python3.6 as default +### +RUN alias python=python3.6 + +### +# Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work +### +RUN pip3 install grpcio-tools mypy-protobuf + +### +# Install useful tools +# Install distlib to avoid https://github.com/pypa/virtualenv/issues/2006 +RUN pip3 install distlib==0.3.1 yapf==0.29.0 pytest +### + +### +# Install Go +### +ENV DOWNLOAD_GO_VERSION=1.12.17 +RUN wget https://golang.org/dl/go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go${DOWNLOAD_GO_VERSION}.linux-amd64.tar.gz +ENV GOROOT /usr/local/go +ENV PATH $PATH:$GOROOT/bin + +### +# Add a welcome message and environment checks. +### +RUN mkdir /scripts +COPY beam_env_checks.sh /scripts/beam_env_checks.sh +COPY bashcolors.sh /scripts/bashcolors.sh +RUN chmod 755 /scripts /scripts/beam_env_checks.sh /scripts/bashcolors.sh + +# hadolint ignore=SC2016 +RUN echo '. /etc/bash_completion' >> /root/.bash_aliases +RUN echo '. /scripts/beam_env_checks.sh' >> /root/.bash_aliases diff --git a/dev-support/docker/bashcolors.sh b/dev-support/docker/bashcolors.sh new file mode 100755 index 000000000000..1e352fce66f7 --- /dev/null +++ b/dev-support/docker/bashcolors.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#https://wiki.archlinux.org/index.php/Color_Bash_Prompt +# Reset +export Color_Off='\e[0m' # Text Reset + +# Regular Colors +export Black='\e[0;30m' # Black +export Red='\e[0;31m' # Red +export Green='\e[0;32m' # Green +export Yellow='\e[0;33m' # Yellow +export Blue='\e[0;34m' # Blue +export Purple='\e[0;35m' # Purple +export Cyan='\e[0;36m' # Cyan +export White='\e[0;37m' # White + +# Bold +export BBlack='\e[1;30m' # Black +export BRed='\e[1;31m' # Red +export BGreen='\e[1;32m' # Green +export BYellow='\e[1;33m' # Yellow +export BBlue='\e[1;34m' # Blue +export BPurple='\e[1;35m' # Purple +export BCyan='\e[1;36m' # Cyan +export BWhite='\e[1;37m' # White + +# Underline +export UBlack='\e[4;30m' # Black +export URed='\e[4;31m' # Red +export UGreen='\e[4;32m' # Green +export UYellow='\e[4;33m' # Yellow +export UBlue='\e[4;34m' # Blue +export UPurple='\e[4;35m' # Purple +export UCyan='\e[4;36m' # Cyan +export UWhite='\e[4;37m' # White + +# Background +export On_Black='\e[40m' # Black +export On_Red='\e[41m' # Red +export On_Green='\e[42m' # Green +export On_Yellow='\e[43m' # Yellow +export On_Blue='\e[44m' # Blue +export On_Purple='\e[45m' # Purple +export On_Cyan='\e[46m' # Cyan +export On_White='\e[47m' # White + +# High Intensity +export IBlack='\e[0;90m' # Black +export IRed='\e[0;91m' # Red +export IGreen='\e[0;92m' # Green +export IYellow='\e[0;93m' # Yellow +export IBlue='\e[0;94m' # Blue +export IPurple='\e[0;95m' # Purple +export ICyan='\e[0;96m' # Cyan +export IWhite='\e[0;97m' # White + +# Bold High Intensity +export BIBlack='\e[1;90m' # Black +export BIRed='\e[1;91m' # Red +export BIGreen='\e[1;92m' # Green +export BIYellow='\e[1;93m' # Yellow +export BIBlue='\e[1;94m' # Blue +export BIPurple='\e[1;95m' # Purple +export BICyan='\e[1;96m' # Cyan +export BIWhite='\e[1;97m' # White + +# High Intensity backgrounds +export On_IBlack='\e[0;100m' # Black +export On_IRed='\e[0;101m' # Red +export On_IGreen='\e[0;102m' # Green +export On_IYellow='\e[0;103m' # Yellow +export On_IBlue='\e[0;104m' # Blue +export On_IPurple='\e[0;105m' # Purple +export On_ICyan='\e[0;106m' # Cyan +export On_IWhite='\e[0;107m' # White diff --git a/dev-support/docker/beam_env_checks.sh b/dev-support/docker/beam_env_checks.sh new file mode 100755 index 000000000000..1a50166766e6 --- /dev/null +++ b/dev-support/docker/beam_env_checks.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SHELLDOC-IGNORE + +# ------------------------------------------------------- +function showWelcome { +# http://patorjk.com/software/taag/#p=display&f=Doom&t=Beam%20Build%20Env. +cat << "Welcome-message" + +______ ______ _ _ _ _____ +| ___ \ | ___ \ (_) | | | | ___| +| |_/ / ___ __ _ _ __ ___ | |_/ /_ _ _| | __| | | |__ _ ____ __ +| ___ \/ _ \/ _` | '_ ` _ \ | ___ \ | | | | |/ _` | | __| '_ \ \ / / +| |_/ / __/ (_| | | | | | | | |_/ / |_| | | | (_| | | |__| | | \ V / +\____/ \___|\__,_|_| |_| |_| \____/ \__,_|_|_|\__,_| \____/_| |_|\_(_) + +This is the standard Beam Developer build environment. +This has all the right tools installed required to build +Apache Beam from source. + +Welcome-message +} + +# ------------------------------------------------------- + +function showAbort { + cat << "Abort-message" + + ___ _ _ _ + / _ \| | | | (_) +/ /_\ \ |__ ___ _ __| |_ _ _ __ __ _ +| _ | '_ \ / _ \| '__| __| | '_ \ / _\` | +| | | | |_) | (_) | | | |_| | | | | (_| | +\_| |_/_.__/ \___/|_| \__|_|_| |_|\__, | + __/ | + |___/ + +Abort-message +} + +# ------------------------------------------------------- + +function failIfUserIsRoot { + if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong. + then + cat </dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ ' diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java index 3da3bb04630e..bca398765d19 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/state/Timers.java @@ -45,7 +45,7 @@ public interface Timers { /** Set a timer with outputTimestamp. */ void setTimer(Instant timestamp, Instant outputTimestamp, TimeDomain timeDomain); - /** Removes the timer set in this context for the {@code timestmap} and {@code timeDomain}. */ + /** Removes the timer set in this context for the {@code timestamp} and {@code timeDomain}. */ void deleteTimer(Instant timestamp, TimeDomain timeDomain); /** Returns the current processing time. */ diff --git a/sdks/python/build-requirements.txt b/sdks/python/build-requirements.txt index 1ecd6ec10a50..415ac340ab1d 100644 --- a/sdks/python/build-requirements.txt +++ b/sdks/python/build-requirements.txt @@ -17,3 +17,6 @@ grpcio-tools==1.30.0 future==0.18.2 mypy-protobuf==1.18 + +# Avoid https://github.com/pypa/virtualenv/issues/2006 +distlib==0.3.1 \ No newline at end of file diff --git a/start-build-env.sh b/start-build-env.sh new file mode 100755 index 000000000000..52cebe45455f --- /dev/null +++ b/start-build-env.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e # exit on error + +cd "$(dirname "$0")" # connect to root + +DOCKER_DIR=dev-support/docker +DOCKER_FILE="${DOCKER_DIR}/Dockerfile" + +CONTAINER_NAME=beam-dev-${USER}-$$ + +if [ ! -x "$(command -v docker)" ] ; then + echo "Unable to locate docker" + echo "This Beam Build environment is based upon docker so you must install it first." + exit 1; +fi + +docker build -t beam-build -f $DOCKER_FILE $DOCKER_DIR + +USER_NAME=${SUDO_USER:=$USER} +USER_ID=$(id -u "${USER_NAME}") + +if [ "$(uname -s)" = "Darwin" ]; then + GROUP_ID=100 +fi + +if [ "$(uname -s)" = "Linux" ]; then + GROUP_ID=$(id -g "${USER_NAME}") + # man docker-run + # When using SELinux, mounted directories may not be accessible + # to the container. To work around this, with Docker prior to 1.7 + # one needs to run the "chcon -Rt svirt_sandbox_file_t" command on + # the directories. With Docker 1.7 and later the z mount option + # does this automatically. + if command -v selinuxenabled >/dev/null && selinuxenabled; then + DCKR_VER=$(docker -v| + awk '$1 == "Docker" && $2 == "version" {split($3,ver,".");print ver[1]"."ver[2]}') + DCKR_MAJ=${DCKR_VER%.*} + DCKR_MIN=${DCKR_VER#*.} + if [ "${DCKR_MAJ}" -eq 1 ] && [ "${DCKR_MIN}" -ge 7 ] || + [ "${DCKR_MAJ}" -gt 1 ]; then + V_OPTS=:z + else + for d in "${PWD}" "${HOME}/.m2"; do + ctx=$(stat --printf='%C' "$d"|cut -d':' -f3) + if [ "$ctx" != svirt_sandbox_file_t ] && [ "$ctx" != container_file_t ]; then + printf 'INFO: SELinux is enabled.\n' + printf '\tMounted %s may not be accessible to the container.\n' "$d" + printf 'INFO: If so, on the host, run the following command:\n' + printf '\t# chcon -Rt svirt_sandbox_file_t %s\n' "$d" + fi + done + fi + fi +fi + +# Set the home directory in the Docker container. +DOCKER_HOME_DIR=${DOCKER_HOME_DIR:-/home/${USER_NAME}} + +DOCKER_GROUP_ID=$(getent group docker | cut -d':' -f3) + +docker build -t "beam-build-${USER_ID}" - < "/etc/sudoers.d/beam-build-${USER_ID}" +ENV HOME "${DOCKER_HOME_DIR}" +ENV GOPATH ${HOME}/beam/sdks/go/examples/.gogradle/project_gopath + +# This next command still runs as root causing the ~/.cache/go-build to be owned by root +RUN go get github.com/linkedin/goavro +RUN chown -R ${USER_NAME}:${GROUP_ID} ${HOME}/.cache +UserSpecificDocker + +echo "" +echo "Docker image build completed." +echo "==============================================================================================" +echo "" + +# If this env variable is empty, docker will be started +# in non interactive mode +DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"} + +DOCKER_SOCKET_MOUNT="" +if [ -S /var/run/docker.sock ]; +then + DOCKER_SOCKET_MOUNT="-v /var/run/docker.sock:/var/run/docker.sock${V_OPTS:-}" + echo "Enabling Docker support with the docker build environment." +else + echo "There is NO Docker support with the docker build environment." +fi + +COMMAND=( "$@" ) +if [ $# -eq 0 ]; +then + COMMAND=( "bash" ) +fi + +[ -d "${HOME}/.beam_docker_build_env/.gradle" ] || mkdir -p "${HOME}/.beam_docker_build_env/.gradle" + +# By mapping the .m2 directory you can do an mvn install from +# within the container and use the result on your normal +# system. And this also is a significant speedup in subsequent +# builds because the dependencies are downloaded only once. +docker run --rm=true ${DOCKER_INTERACTIVE_RUN} \ + --name "${CONTAINER_NAME}" \ + --network=host \ + -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \ + -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \ + -v "${HOME}/.beam_docker_build_env/.gradle:${DOCKER_HOME_DIR}/.gradle${V_OPTS:-}" \ + -v "${PWD}:${DOCKER_HOME_DIR}/beam${V_OPTS:-}" \ + -w "${DOCKER_HOME_DIR}/beam" \ + ${DOCKER_SOCKET_MOUNT} \ + -u "${USER_ID}" \ + "beam-build-${USER_ID}" "${COMMAND[@]}" diff --git a/website/www/site/content/en/contribute/_index.md b/website/www/site/content/en/contribute/_index.md index dd5e90eb8a8d..4a095b70ba38 100644 --- a/website/www/site/content/en/contribute/_index.md +++ b/website/www/site/content/en/contribute/_index.md @@ -83,6 +83,11 @@ sudo apt-get install \ docker-ce ``` +On some systems (like Ubuntu 20.04) these need to be installed also +``` +pip3 install grpcio-tools mypy-protobuf +``` + You also need to [install Go](https://golang.org/doc/install). Once Go is installed, install goavro: @@ -94,6 +99,15 @@ $ go get github.com/linkedin/goavro gLinux users should configure their machines for sudoless Docker. +Alternatively, you can use the Docker based local development environment to wrap your clone of the Beam repo +into a container meeting the requirements above. + +You can start this container using the [start-build-env.sh](https://github.com/apache/beam/blob/master/start-build-env.sh) +script which is part of the Beam repo: +``` +./start-build-env.sh +``` + ### Connect With the Beam community 1. Consider subscribing to the [dev@ mailing list](/community/contact-us/), especially