GA Migration Website Publish and Test

.github/workflows/job-postcommit-website-publish.yml

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Publishes generated website content into asf-site branch for hosting
+
+name: PostCommit Website Publish
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*', 'postcommit-website']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+permissions: write-all
+
+jobs:
+  website-publish:
+    name: PostCommit Website Publish
+    runs-on: [self-hosted, ubuntu-20.04]
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+
+      - name: Run :website:clean
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :website:clean
+
+      - name: Set git env var
+        run : |
+           ghEmail=actions@"$RUNNER_NAME".local
+           echo "ghEmail=${ghEmail}" >> $GITHUB_ENV
+
+      - name: Run :website:publishWebsite
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        env:
+          ghprbPullId: ${{github.event.pull_request.number}}
+          ghPullAuthorLogin: ${{github.actor}}
+          ghprbSourceBranch: ${{github.ref_name}}
+          ghBranch: ${{github.head_ref}}
+          repositoryUrl: ${{github.repository}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          gradle-command: :website:publishWebsiteDocker

.github/workflows/job-postcommit-website-test.yml

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to validate the Beam website, including external links.
+
+name: PostCommit Website Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 */6 * * *'
+  push:
+    branches: ['master', 'release-*', 'postcommit-website']
+    tags: 'v*'
+  pull_request_target:
+    branches: [ 'master', 'release-*']
+    tags: 'v*'
+permissions: read-all
+
+jobs:
+  website-test:
+    name: PostCommit Website Test
+    runs-on: [self-hosted, ubuntu-20.04]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
+
+      - name: Setup environment
+        uses: ./.github/actions/setup-self-hosted-action
+
+      - name: Run :website:testWebsite
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        env:
+          ghprbPullId: ${{github.event.pull_request.number}}
+          ghprbPullAuthorLogin: ${{github.actor}}
+          ghprbSourceBranch: ${{github.ref_name}}
+          ghBranch: ${{github.head_ref}}
+          repositoryUrl: ${{github.repository}}
+        with:
+          gradle-command: :website:testWebsite
+          arguments: "-PdisableExternal=false"

CI.md

@@ -138,9 +138,10 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog
 | [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No                    |
 
 ### PostCommit Workflows
-| Workflow                                                                           | Description             | Requires GCP Credentials |
-|------------------------------------------------------------------------------------|-------------------------|--------------------------|
-| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No                   |
+| Workflow                                                                                   | Description                                                          | Requires GCP Credentials |
+|--------------------------------------------------------------------------------------------|----------------------------------------------------------------------|--------------------------|
+| [job-postcommit-website-publish.yml](.github/workflows/job-postcommit-website-publish.yml) | Publishes generated website content into asf-site branch for hosting | No                       |
+| [job-postcommit-website-test.yml](.github/workflows/job-postcommit-website-test.yml)       | Test to validate the Beam website, including external links.         | No                       |
 
 ### GitHub Action Tips
 

runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java

@@ -182,8 +182,7 @@ private List<String> gcsCredentialArgs() {
     // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually.
     if (Files.exists(Paths.get(localGcloudConfig))) {
       return ImmutableList.of(
-          "--mount",
-          String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig));
+          "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig));
     } else {
       return ImmutableList.of();
     }

website/Dockerfile

@@ -69,4 +69,8 @@ RUN HUGOHOME="$(mktemp -d)" \
     && chmod +x /usr/local/bin/hugo \
     && rm -r "${HUGOHOME}"
 
+# TODO: Install SSH
+RUN apt-get update \
+    && apt-get install -y ssh-client
+
 WORKDIR /opt/

website/build.gradle

@@ -80,10 +80,10 @@ task createDockerContainer(type: Exec) {
     } else {
       // Otherwise: run as current user, so as to not require sudo to clean up
       // build/ directory.
-      extraOptions += " -u \$(id -u):\$(id -g)"
+      extraOptions += " -u 0:0"
     }
     commandLine '/bin/bash', '-c',
-      "docker create -v $project.rootDir:$dockerWorkDir $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'"
+      "docker create $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'"
   }
 }
 
@@ -115,9 +115,23 @@ task buildCodeSamples(type: Exec) {
               "${->startDockerContainer.containerId()}", 'yarn', 'build_code_samples'
 }
 
+task cloneRepository(type: Exec) {
+  def branch = System.getenv('ghBranch') ?  System.getenv('ghBranch'):
+          grgit ? grgit.branch.current().getName(): 'master'
+  def repositoryUrl = System.getenv('repositoryUrl') ? "https://github.com/${System.getenv('repositoryUrl')}" :
+                    "https://github.com/apache/beam"
+
+  commandLine 'docker', 'exec',
+          "${->startDockerContainer.containerId()}", "git", "clone", "-b", branch, repositoryUrl, "."
+}
+
+initGitSubmodules.mustRunAfter cloneRepository
+installDependencies.mustRunAfter cloneRepository
+buildCodeSamples.mustRunAfter cloneRepository
+
 task setupDockerContainer(type: Exec) {
   dependsOn startDockerContainer
-  finalizedBy initGitSubmodules, installDependencies, buildCodeSamples
+  finalizedBy cloneRepository, initGitSubmodules, installDependencies, buildCodeSamples
   ext.containerId = {
     return startDockerContainer.containerId()
   }
@@ -147,7 +161,7 @@ def createBuildTask = {
   BuildTaskConfiguration config = it as BuildTaskConfiguration
   task "build${config.name}Website" (type:Exec) {
     dependsOn setupDockerContainer
-    finalizedBy stopAndRemoveDockerContainer
+//     finalizedBy stopAndRemoveDockerContainer
 
     def configs = "$dockerSourceDir/site/config.toml"
     def baseUrlFlag = config.baseUrl ? "--baseURL /${config.baseUrl}" : ""
@@ -261,14 +275,55 @@ task preCommit {
   dependsOn testWebsite
 }
 
+task commitWebsiteDocker {
+    doLast {
+        def actor = System.getenv('ghPullAuthorLogin')
+        def email = System.getenv('ghEmail')
+
+
+       //TODO: Add validation that the build exists
+       //TODO: Erase testing remote
+        exec {
+            commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
+            """
+                git remote add website-publish git@github.com:roger-mike/beam.git && \
+                git fetch --force origin +asf-site:asf-site && \
+                git stash && \
+                git checkout asf-site && \
+                git rm -rq 'website/generated-content'
+            """
+        }
+
+        exec {
+            commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
+            """
+               mkdir website/generated-content
+               cp -r build/website/generated-apache-content/* website/generated-content
+            """
+        }
+
+        exec {
+            commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
+            """
+                git add 'website/generated-content'
+                git config user.name ${actor}
+                git config user.email ${email}
+            """
+        }
+
+
+
+    }
+}
+
 // Creates a new commit on asf-site branch
 task commitWebsite {
   doLast {
     assert grgit : "Cannot commit website outside of git repository"
-    assert file("${buildContentDir('apache')}/index.html").exists()
+//     assert file("${buildContentDir('apache')}/index.html").exists()
     // Generated javadoc and pydoc content is not built or stored in this repo.
-    assert !file("${buildContentDir('apache')}/documentation/sdks/javadoc").exists()
-    assert !file("${buildContentDir('apache')}/documentation/sdks/pydoc").exists()
+//     assert !file("${buildContentDir('apache')}/documentation/sdks/javadoc").exists()
+//     assert !file("${buildContentDir('apache')}/documentation/sdks/pydoc").exists()
 
     def git = grgit.open(currentDir: project.rootDir)
     // get the latest commit on master
@@ -327,18 +382,52 @@ task publishWebsite {
 
     def git = grgit.open(currentDir: project.rootDir)
     git.checkout(branch: 'asf-site')
+
     if (!commitedChanges) {
       println 'No changes to push'
       return
     }
 
+
     // Because git.push() fails to authenticate, run git push directly.
-    shell "git push ${gitboxUrl} asf-site"
+//     shell "git push ${gitboxUrl} asf-site"
+  }
+}
+
+task publishWebsiteDocker {
+  doLast {
+    def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss')
+    String message = "Publishing website ${currentDate} at commit \$LATEST_COMMIT"
+    def GITHUB_TOKEN = System.getenv("GITHUB_TOKEN")
+    def actor = System.getenv('ghPullAuthorLogin')
+    //TODO: Change repo to gitboxUrl
+    exec {
+        commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
+        """
+        export GITHUB_TOKEN=${GITHUB_TOKEN}
+        """
+    }
+
+    exec {
+            commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
+            """
+                export LATEST_COMMIT=\$(git log --format="%h" -n 1)
+                if [[ \$(git status --porcelain | wc -l) > 0 ]];
+                then
+                    echo "Creating commit for changes" && \
+                    git commit -am "${message}" && \
+                    echo "Pushing changes" && \
+                    git push https://${GITHUB_TOKEN}@github.com/${actor}/beam.git
+                else
+                    echo "No changes to commit/push"
+                fi
+            """
+        }
   }
 }
 
-commitWebsite.dependsOn buildApacheWebsite
-publishWebsite.dependsOn commitWebsite
+commitWebsiteDocker.dependsOn buildApacheWebsite
+publishWebsiteDocker.dependsOn commitWebsiteDocker
 
 /*
  * Stages a pull request on GCS