Merge from Master

.gitattributes

@@ -32,4 +32,7 @@ Dockerfile text
 .gitignore export-ignore
 .gitattributes export-ignore
 /gradlew* export-ignore
-/gradle export-ignore
+**/gradle export-ignore
+
+# Website is not part of archive
+/website export-ignore

.github/autolabeler.yml

@@ -18,7 +18,7 @@
 # Please keep the entries sorted lexicographically in each category.
 
 # General
-build: ["assembly.xml", "build.gradle", "buildSrc/*", ".gitattributes", ".github/*", ".gitignore", "gradle/*", ".mailmap", "ownership/*", "release/*", "sdks/java/build-tools/*", "settings.gradle"]
+build: ["assembly.xml", "build.gradle.kts", "buildSrc/*", ".gitattributes", ".github/*", ".gitignore", "gradle/*", ".mailmap", "ownership/*", "release/*", "sdks/java/build-tools/*", "settings.gradle.kts"]
 docker: ["runners/flink/job-server-container/*", "runners/spark/job-server/container/*", "sdks/go/container/*", "sdks/java/container/*", "sdks/python/container/*"]
 examples: ["examples/*", "sdks/go/examples/*", "sdks/python/apache_beam/examples/*"]
 go: ["sdks/go/*"]

.github/workflows/build_wheels.yml

@@ -29,7 +29,7 @@ on:
     branches: ['master', 'release-*']
     tags: 'v*'
     paths: ['sdks/python/**', 'model/**', 'release/**']
-
+  workflow_dispatch:
 
 env:
   GCP_PATH: "gs://${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}/${GITHUB_REF##*/}/${GITHUB_SHA}-${GITHUB_RUN_ID}/"
@@ -59,6 +59,9 @@ jobs:
   build_source:
     runs-on: ubuntu-latest
     name: Build python source distribution
+    outputs:
+      is_rc: ${{ steps.is_rc.outputs.is_rc }}
+      rc_num: ${{ steps.get_rc_version.outputs.RC_NUM }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -71,6 +74,24 @@ jobs:
         run: python -m pip install -r build-requirements.txt
       - name: Install wheels
         run: python -m pip install wheel
+      - name: Get tag
+        id: get_tag
+        run: |
+          echo ::set-output name=TAG::${GITHUB_REF#refs/*/}
+      - name: Check whether an -RC tag was applied to the commit.
+        id: is_rc
+        run: |
+          echo ${{ steps.get_tag.outputs.TAG }} > temp
+          OUTPUT=$( if  grep -e '-RC.' -q temp; then echo 1; else echo 0; fi)
+          echo "::set-output name=is_rc::$OUTPUT"
+      - name: Get RELEASE_VERSION and RC_NUM
+        if: steps.is_rc.outputs.is_rc == 1
+        id: get_rc_version
+        run: |
+          RC_NUM=$(sed -n "s/^.*-RC\([0-9]*\)/\1/p" temp)
+          RELEASE_VERSION=$(sed -n "s/^v\(.*\)-RC[0-9]/\1/p" temp)
+          echo "::set-output name=RC_NUM::$RC_NUM"
+          echo "::set-output name=RELEASE_VERSION::$RELEASE_VERSION"
       - name: Build source
         working-directory: ./sdks/python
         run: python setup.py sdist --formats=zip
@@ -95,6 +116,50 @@ jobs:
         with:
           name: source_zip
           path: sdks/python/dist
+      - name: Clear dist
+        if: steps.is_rc.outputs.is_rc == 1
+        working-directory: ./sdks/python
+        run: |
+          rm -r ./dist
+          rm -rd apache-beam-source
+      - name: Rewrite SDK version to include RC number
+        if: steps.is_rc.outputs.is_rc == 1
+        working-directory: ./sdks/python
+        run: |
+          RELEASE_VERSION=${{ steps.get_rc_version.outputs.RELEASE_VERSION }}
+          RC_NUM=${{ steps.get_rc_version.outputs.RC_NUM }}
+          sed -i -e "s/${RELEASE_VERSION}/${RELEASE_VERSION}rc${RC_NUM}/g" apache_beam/version.py
+      - name: Build RC source
+        if: steps.is_rc.outputs.is_rc == 1
+        working-directory: ./sdks/python
+        run: python setup.py sdist --formats=zip
+      - name: Add RC checksums
+        if: steps.is_rc.outputs.is_rc == 1
+        working-directory: ./sdks/python/dist
+        run: |
+          file=$(ls | grep .zip | head -n 1)
+          sha512sum $file > ${file}.sha512
+      - name: Unzip RC source
+        if: steps.is_rc.outputs.is_rc == 1
+        working-directory: ./sdks/python
+        run: unzip dist/$(ls dist | grep .zip | head -n 1)
+      - name: Rename RC source directory
+        if: steps.is_rc.outputs.is_rc == 1
+        working-directory: ./sdks/python
+        run: mv $(ls | grep apache-beam) apache-beam-source-rc
+      - name: Upload RC source as artifact
+        if: steps.is_rc.outputs.is_rc == 1
+        uses: actions/upload-artifact@v2
+        with:
+          name: source_rc${{ steps.get_rc_version.outputs.RC_NUM }}
+          path: sdks/python/apache-beam-source-rc
+      - name: Upload compressed RC sources as artifacts
+        if: steps.is_rc.outputs.is_rc == 1
+        uses: actions/upload-artifact@v2
+        with:
+          name: source_zip_rc${{ steps.get_rc_version.outputs.RC_NUM }}
+          path: sdks/python/dist
+
 
   prepare_gcs:
     name: Prepare GCS
@@ -134,8 +199,10 @@ jobs:
         run: gsutil cp -r -a public-read source/* ${{ env.GCP_PATH }}
 
   build_wheels:
-    name: Build python wheels on ${{ matrix.os_python.os }}
+    name: Build python wheels on ${{matrix.arch}} for ${{ matrix.os_python.os }}
     needs: build_source
+    env:
+      CIBW_ARCHS_LINUX: ${{matrix.arch}}
     runs-on: ${{ matrix.os_python.os }}
     strategy:
       matrix:
@@ -144,18 +211,31 @@ jobs:
           {"os": "macos-latest", "python": "cp36-* cp37-* cp38-*"},
           {"os": "windows-latest", "python": "cp36-* cp37-* cp38-*"},
         ]
+        arch: [auto]
+        include:
+          - os_python: {"os": "ubuntu-latest", "python": "cp36-* cp37-* cp38-*"}
+            arch: aarch64
     steps:
     - name: Download python source distribution from artifacts
       uses: actions/download-artifact@v2
       with:
         name: source
         path: apache-beam-source
+    - name: Download Python SDK RC source distribution from artifacts
+      if: ${{ needs.build_source.outputs.is_rc == 1 }}
+      uses: actions/download-artifact@v2
+      with:
+        name: source_rc${{ needs.build_source.outputs.rc_num }}
+        path: apache-beam-source-rc
     - name: Install Python
       uses: actions/setup-python@v2
       with:
         python-version: 3.7
+    - uses: docker/setup-qemu-action@v1
+      if: ${{matrix.arch == 'aarch64'}}
+      name: Set up QEMU
     - name: Install cibuildwheel
-      run: pip install cibuildwheel==1.4.2
+      run: pip install cibuildwheel==1.11.0
     - name: Build wheel
       working-directory: apache-beam-source
       env:
@@ -178,6 +258,28 @@ jobs:
       with:
         name: wheelhouse-${{ matrix.os_python.os }}
         path: apache-beam-source/wheelhouse/
+    - name: Build RC wheels
+      if: ${{ needs.build_source.outputs.is_rc == 1 }}
+      working-directory: apache-beam-source-rc
+      env:
+        CIBW_BUILD: ${{ matrix.os_python.python }}
+        CIBW_BEFORE_BUILD: pip install cython
+      run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse
+      shell: bash
+    - name: Add RC checksums
+      if: ${{ needs.build_source.outputs.is_rc == 1 }}
+      working-directory: apache-beam-source-rc/wheelhouse/
+      run: |
+        for file in *.whl; do
+          sha512sum $file > ${file}.sha512
+        done
+      shell: bash
+    - name: Upload RC wheels as artifacts
+      if: ${{ needs.build_source.outputs.is_rc == 1 }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: wheelhouse-rc${{ needs.build_source.outputs.rc_num }}-${{ matrix.os_python.os }}
+        path: apache-beam-source-rc/wheelhouse/
 
   upload_wheels_to_gcs:
     name: Upload wheels to GCS bucket
@@ -249,14 +351,17 @@ jobs:
     if: github.repository_owner == 'apache' && github.event_name == 'schedule'
     steps:
       - name: Checkout code on master branch
-        uses: actions/checkout@master
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: Tag commit
         run: |
           BRANCH_NAME=${GITHUB_REF##*/}
           echo "Tagging ${BRANCH_NAME}"
           git tag -f nightly-${BRANCH_NAME} HEAD
       - name: Push tags
-        uses: ad-m/github-push-action@master
+        uses: ./.github/actions/github-push-action
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           tags: true

.github/workflows/cancel.yml

@@ -30,7 +30,12 @@ jobs:
     name: "Cancel duplicate workflow runs"
     runs-on: ubuntu-latest
     steps:
-      - uses: potiuk/cancel-workflow-runs@e9e87cb7738dbb999654aa90d69359d62c9e4eae #v3
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - uses: ./.github/actions/cancel-workflow-runs
         name: "Cancel duplicate workflow runs"
         with:
           cancelMode: duplicates

.github/workflows/java_tests.yml

@@ -33,7 +33,8 @@ on:
   pull_request:
     branches: ['master', 'release-*']
     tags: 'v*'
-    paths: ['sdks/java/**', 'model/**', 'runners/**', 'examples/java/**', 'examples/kotlin/**', 'release/**']
+    paths: ['sdks/java/**', 'model/**', 'runners/**', 'examples/java/**',
+            'examples/kotlin/**', 'release/**', 'buildSrc/**']
 
 
 jobs:
@@ -67,9 +68,12 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       # :sdks:java:core:test
       - name: Run :sdks:java:core:test
-        uses: eskatos/gradle-command-action@v1
+        uses: ./.github/actions/gradle-command-action
         with:
           arguments: :sdks:java:core:test
       - name: Upload test logs for :sdks:java:core:test
@@ -80,7 +84,7 @@ jobs:
           path: sdks/java/core/build/reports/tests/test
       # :sdks:java:harness:test
       - name: Run :sdks:java:harness:test
-        uses: eskatos/gradle-command-action@v1
+        uses: ./.github/actions/gradle-command-action
         with:
           arguments: :sdks:java:harness:test
         if: always()
@@ -92,7 +96,7 @@ jobs:
           path: sdks/java/harness/build/reports/tests/test
       # :runners:core-java:test
       - name: Run :runners:core-java:test
-        uses: eskatos/gradle-command-action@v1
+        uses: ./.github/actions/gradle-command-action
         with:
           arguments: :runners:core-java:test
         if: always()
@@ -113,8 +117,11 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: Run WordCount Unix
-        uses: eskatos/gradle-command-action@v1
+        uses: ./.github/actions/gradle-command-action
         with:
           arguments: -p examples/ integrationTest --tests org.apache.beam.examples.WordCountIT
             -DintegrationTestRunner=direct
@@ -143,15 +150,27 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
       - name: Authenticate on GCP
         uses: google-github-actions/setup-gcloud@master
         with:
           service_account_email: ${{ secrets.GCP_SA_EMAIL }}
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           export_default_credentials: true
+      - name: Set Java Version
+        uses: actions/setup-java@v1
+        with:
+          java-version: 8
+      - name: Remove default github maven configuration
+        # This step is a workaround to avoid a decryption issue of Beam's
+        # gradle-command-action plugin and github's provided maven
+        # settings.xml file
+        run: rm ~/.m2/settings.xml
       - name: Run WordCount
-        uses: eskatos/gradle-command-action@v1
+        uses: ./.github/actions/gradle-command-action
         with:
           arguments: -p examples/ integrationTest --tests org.apache.beam.examples.WordCountIT
             -DintegrationTestPipelineOptions=["--runner=DataflowRunner","--project=${{ secrets.GCP_PROJECT_ID }}","--tempRoot=gs://${{ secrets.GCP_TESTING_BUCKET }}/tmp/"]

.github/workflows/local_env_tests.yml

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# To learn more about GitHub Actions in Apache Beam check the CI.md
+
+name: Local environment tests
+
+on:
+  push:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+  pull_request:
+    branches: ['master', 'release-*']
+    tags: 'v*'
+    paths: ['dev-support/**', 'buildSrc/**', '**/build.gradle', 'sdks/python/setup.py', 'sdks/python/tox.ini']
+
+jobs:
+  run_local_env_install_ubuntu:
+    timeout-minutes: 25
+    name: "Ubuntu run local environment shell script"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Installing local env dependencies"
+        run: "sudo ./local-env-setup.sh"
+        id: local_env_install_ubuntu
+      - name: "Gradle check"
+        run: "./gradlew checkSetup"
+        id: local_env_install_gradle_check_ubuntu
+  run_local_env_install_mac:
+    timeout-minutes: 50
+    name: "Mac run local environment shell script"
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: "Installing local env dependencies"
+        run: "./local-env-setup.sh"
+        id: local_env_install_mac
+      - name: "Gradle check"
+        run: "./gradlew checkSetup"
+        id: local_env_install_gradle_check_mac