From 0344627f26743f880b975dfcbbd8365fb25ec753 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Mon, 19 Sep 2022 13:40:04 -0500 Subject: [PATCH 1/5] Migrated github action --- .../job-postcommit-website-publish.yml | 62 +++++++++++++++++++ .../workflows/job-postcommit-website-test.yml | 58 +++++++++++++++++ .../environment/DockerEnvironmentFactory.java | 3 +- website/build.gradle | 18 +++++- 4 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/job-postcommit-website-publish.yml create mode 100644 .github/workflows/job-postcommit-website-test.yml diff --git a/.github/workflows/job-postcommit-website-publish.yml b/.github/workflows/job-postcommit-website-publish.yml new file mode 100644 index 000000000000..f5fa8c3df63b --- /dev/null +++ b/.github/workflows/job-postcommit-website-publish.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Publishes generated website content into asf-site branch for hosting + +name: PostCommit Website Publish + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-website'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' +permissions: read-all + +jobs: + website-publish: + name: PostCommit Website Publish + runs-on: [self-hosted, ubuntu-20.04] + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :website:clean + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :website:clean + + - name: Run :website:publishWebsite + uses: ./.github/actions/gradle-command-self-hosted-action + env: + ghprbPullId: ${{github.event.pull_request.number}} + ghprbPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repositoryUrl}} + with: + gradle-command: :website:publishWebsite diff --git a/.github/workflows/job-postcommit-website-test.yml b/.github/workflows/job-postcommit-website-test.yml new file mode 100644 index 000000000000..6bd6ec3ed5d7 --- /dev/null +++ b/.github/workflows/job-postcommit-website-test.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test to validate the Beam website, including external links. + +name: PostCommit Website Test + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-website'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' +permissions: read-all + +jobs: + website-test: + name: PostCommit Website Test + runs-on: [self-hosted, ubuntu-20.04] + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + submodules: recursive + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :website:testWebsite + uses: ./.github/actions/gradle-command-self-hosted-action + env: + ghprbPullId: ${{github.event.pull_request.number}} + ghprbPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repositoryUrl}} + with: + gradle-command: :website:testWebsite + arguments: "-PdisableExternal=false" \ No newline at end of file diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a..edead73227d7 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,8 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "--mount", - String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } diff --git a/website/build.gradle b/website/build.gradle index 6aa88a2c7feb..a2f8af528bcf 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -80,10 +80,10 @@ task createDockerContainer(type: Exec) { } else { // Otherwise: run as current user, so as to not require sudo to clean up // build/ directory. - extraOptions += " -u \$(id -u):\$(id -g)" + extraOptions += " -u 0:0" } commandLine '/bin/bash', '-c', - "docker create -v $project.rootDir:$dockerWorkDir $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" + "docker create $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" } } @@ -115,9 +115,21 @@ task buildCodeSamples(type: Exec) { "${->startDockerContainer.containerId()}", 'yarn', 'build_code_samples' } +task cloneRepository(type: Exec) { + //TODO: Get repository and branch from GA + + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", "git", "clone", "-b", "postcommit-website", + "$getBranchRepo", "." +} + +initGitSubmodules.mustRunAfter cloneRepository +installDependencies.mustRunAfter cloneRepository +buildCodeSamples.mustRunAfter cloneRepository + task setupDockerContainer(type: Exec) { dependsOn startDockerContainer - finalizedBy initGitSubmodules, installDependencies, buildCodeSamples + finalizedBy cloneRepository, initGitSubmodules, installDependencies, buildCodeSamples ext.containerId = { return startDockerContainer.containerId() } From 6686a9e295416c94a904e34d2f37c8ed7f42df47 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Thu, 3 Nov 2022 11:01:35 -0600 Subject: [PATCH 2/5] CI.md updated --- .github/workflows/job-postcommit-website-publish.yml | 3 +-- .github/workflows/job-postcommit-website-test.yml | 4 +--- CI.md | 7 ++++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/job-postcommit-website-publish.yml b/.github/workflows/job-postcommit-website-publish.yml index f5fa8c3df63b..bbb1078693ff 100644 --- a/.github/workflows/job-postcommit-website-publish.yml +++ b/.github/workflows/job-postcommit-website-publish.yml @@ -40,7 +40,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action @@ -57,6 +56,6 @@ jobs: ghprbPullAuthorLogin: ${{github.actor}} ghprbSourceBranch: ${{github.ref_name}} ghBranch: ${{github.head_ref}} - repositoryUrl: ${{github.repositoryUrl}} + repositoryUrl: ${{github.repository}} with: gradle-command: :website:publishWebsite diff --git a/.github/workflows/job-postcommit-website-test.yml b/.github/workflows/job-postcommit-website-test.yml index 6bd6ec3ed5d7..b0a1245af687 100644 --- a/.github/workflows/job-postcommit-website-test.yml +++ b/.github/workflows/job-postcommit-website-test.yml @@ -33,14 +33,12 @@ jobs: website-test: name: PostCommit Website Test runs-on: [self-hosted, ubuntu-20.04] - timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action @@ -52,7 +50,7 @@ jobs: ghprbPullAuthorLogin: ${{github.actor}} ghprbSourceBranch: ${{github.ref_name}} ghBranch: ${{github.head_ref}} - repositoryUrl: ${{github.repositoryUrl}} + repositoryUrl: ${{github.repository}} with: gradle-command: :website:testWebsite arguments: "-PdisableExternal=false" \ No newline at end of file diff --git a/CI.md b/CI.md index ee911584f7f0..c81e9b9b453c 100644 --- a/CI.md +++ b/CI.md @@ -138,9 +138,10 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | ### PostCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|------------------------------------------------------------------------------------|-------------------------|--------------------------| -| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|--------------------------------------------------------------------------------------------|----------------------------------------------------------------------|--------------------------| +| [job-postcommit-website-publish.yml](.github/workflows/job-postcommit-website-publish.yml) | Publishes generated website content into asf-site branch for hosting | No | +| [job-postcommit-website-test.yml](.github/workflows/job-postcommit-website-test.yml) | Test to validate the Beam website, including external links. | No | ### GitHub Action Tips From 8397f86aca460e088efcac82a85ab5c30c2bc843 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Thu, 3 Nov 2022 16:19:50 -0600 Subject: [PATCH 3/5] Refactored publish task to work in Docker --- website/build.gradle | 72 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/website/build.gradle b/website/build.gradle index a2f8af528bcf..8520c86a65dc 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -116,11 +116,13 @@ task buildCodeSamples(type: Exec) { } task cloneRepository(type: Exec) { - //TODO: Get repository and branch from GA + def branch = System.getenv('ghBranch') ? System.getenv('ghBranch'): + grgit ? grgit.branch.current().getName(): 'master' + def repositoryUrl = System.getenv('repositoryUrl') ? "https://github.com/${System.getenv('repositoryUrl')}" : + "https://github.com/apache/beam" commandLine 'docker', 'exec', - "${->startDockerContainer.containerId()}", "git", "clone", "-b", "postcommit-website", - "$getBranchRepo", "." + "${->startDockerContainer.containerId()}", "git", "clone", "-b", branch, repositoryUrl, "." } initGitSubmodules.mustRunAfter cloneRepository @@ -159,7 +161,7 @@ def createBuildTask = { BuildTaskConfiguration config = it as BuildTaskConfiguration task "build${config.name}Website" (type:Exec) { dependsOn setupDockerContainer - finalizedBy stopAndRemoveDockerContainer +// finalizedBy stopAndRemoveDockerContainer def configs = "$dockerSourceDir/site/config.toml" def baseUrlFlag = config.baseUrl ? "--baseURL /${config.baseUrl}" : "" @@ -273,14 +275,64 @@ task preCommit { dependsOn testWebsite } +task commitWebsiteDocker { + doLast { + def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss') + String message = "Publishing website ${currentDate} at commit \$LATEST_COMMIT" + String author = System.getEnv('ghprbPullAuthorLogin') + + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + export LATEST_COMMIT=\$(git log --format="%h" -n 1) && \ + git fetch --force origin +asf-site:asf-site && \ + git stash && \ + git checkout asf-site && \ + git rm -rq 'website/generated-content' + """ + } + + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + mkdir website/generated-content + cp -r build/website/generated-apache-content/* website/generated-content + """ + } + + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + git add 'website/generated-content' + """ + } + + //Set user name and email to commit + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + if [ \$(git status --porcelain | wc -l) > 0 ]; + then + echo "Creating commit for changes" + git commit -am "${message}" + + else + echo "No changes to commit" + fi + """ + } + + } +} + // Creates a new commit on asf-site branch task commitWebsite { doLast { assert grgit : "Cannot commit website outside of git repository" - assert file("${buildContentDir('apache')}/index.html").exists() +// assert file("${buildContentDir('apache')}/index.html").exists() // Generated javadoc and pydoc content is not built or stored in this repo. - assert !file("${buildContentDir('apache')}/documentation/sdks/javadoc").exists() - assert !file("${buildContentDir('apache')}/documentation/sdks/pydoc").exists() +// assert !file("${buildContentDir('apache')}/documentation/sdks/javadoc").exists() +// assert !file("${buildContentDir('apache')}/documentation/sdks/pydoc").exists() def git = grgit.open(currentDir: project.rootDir) // get the latest commit on master @@ -345,12 +397,12 @@ task publishWebsite { } // Because git.push() fails to authenticate, run git push directly. - shell "git push ${gitboxUrl} asf-site" + //shell "git push ${gitboxUrl} asf-site" } } -commitWebsite.dependsOn buildApacheWebsite -publishWebsite.dependsOn commitWebsite +commitWebsiteDocker.dependsOn buildApacheWebsite +publishWebsite.dependsOn commitWebsiteDocker /* * Stages a pull request on GCS From cc7bb585dcc1b2f1e8433edd6f5cdf8f3beff450 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Tue, 29 Nov 2022 16:57:57 -0600 Subject: [PATCH 4/5] Publish task changed to work in docker --- .../job-postcommit-website-publish.yml | 1 + website/build.gradle | 33 ++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job-postcommit-website-publish.yml b/.github/workflows/job-postcommit-website-publish.yml index bbb1078693ff..5500e9079561 100644 --- a/.github/workflows/job-postcommit-website-publish.yml +++ b/.github/workflows/job-postcommit-website-publish.yml @@ -57,5 +57,6 @@ jobs: ghprbSourceBranch: ${{github.ref_name}} ghBranch: ${{github.head_ref}} repositoryUrl: ${{github.repository}} + ghEmail: ${{actions@"$RUNNER_NAME".local}} with: gradle-command: :website:publishWebsite diff --git a/website/build.gradle b/website/build.gradle index 8520c86a65dc..18798b428135 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -277,14 +277,16 @@ task preCommit { task commitWebsiteDocker { doLast { + def actor = System.getenv('ghprbPullAuthorLogin') + def email = System.getenv('ghEmail') def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss') String message = "Publishing website ${currentDate} at commit \$LATEST_COMMIT" - String author = System.getEnv('ghprbPullAuthorLogin') exec { commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """ export LATEST_COMMIT=\$(git log --format="%h" -n 1) && \ + git remote add website-publish git@github.com:roger-mike/beam.git git fetch --force origin +asf-site:asf-site && \ git stash && \ git checkout asf-site && \ @@ -304,18 +306,19 @@ task commitWebsiteDocker { commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """ git add 'website/generated-content' + git config user.name ${actor} + git config user.email ${email} """ } - //Set user name and email to commit exec { commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """ - if [ \$(git status --porcelain | wc -l) > 0 ]; + if [[ \$(git status --porcelain | wc -l) > 0 ]]; then echo "Creating commit for changes" + export ISCOMMITED=1 git commit -am "${message}" - else echo "No changes to commit" fi @@ -391,18 +394,38 @@ task publishWebsite { def git = grgit.open(currentDir: project.rootDir) git.checkout(branch: 'asf-site') + if (!commitedChanges) { println 'No changes to push' return } + // Because git.push() fails to authenticate, run git push directly. //shell "git push ${gitboxUrl} asf-site" } } +task publishWebsiteDocker { + doLast { + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + echo ${gitboxUrl} + if [[ \$ISCOMMITED -eq 1 ]]; + then + echo "Pushing changes" + git push website-publish asf-site + else + echo "No changes to push" + fi + """ + } + } +} + commitWebsiteDocker.dependsOn buildApacheWebsite -publishWebsite.dependsOn commitWebsiteDocker +publishWebsiteDocker.dependsOn commitWebsiteDocker /* * Stages a pull request on GCS From 6fbf9cac81e68904f211d9d30e9d46f557da7227 Mon Sep 17 00:00:00 2001 From: Miguel Hernandez Date: Fri, 9 Dec 2022 11:32:15 -0600 Subject: [PATCH 5/5] Executes publish in docker --- .../job-postcommit-website-publish.yml | 13 ++-- website/Dockerfile | 4 ++ website/build.gradle | 60 ++++++++++--------- 3 files changed, 44 insertions(+), 33 deletions(-) diff --git a/.github/workflows/job-postcommit-website-publish.yml b/.github/workflows/job-postcommit-website-publish.yml index 5500e9079561..7bf917f47e00 100644 --- a/.github/workflows/job-postcommit-website-publish.yml +++ b/.github/workflows/job-postcommit-website-publish.yml @@ -27,7 +27,7 @@ on: pull_request_target: branches: [ 'master', 'release-*'] tags: 'v*' -permissions: read-all +permissions: write-all jobs: website-publish: @@ -49,14 +49,19 @@ jobs: with: gradle-command: :website:clean + - name: Set git env var + run : | + ghEmail=actions@"$RUNNER_NAME".local + echo "ghEmail=${ghEmail}" >> $GITHUB_ENV + - name: Run :website:publishWebsite uses: ./.github/actions/gradle-command-self-hosted-action env: ghprbPullId: ${{github.event.pull_request.number}} - ghprbPullAuthorLogin: ${{github.actor}} + ghPullAuthorLogin: ${{github.actor}} ghprbSourceBranch: ${{github.ref_name}} ghBranch: ${{github.head_ref}} repositoryUrl: ${{github.repository}} - ghEmail: ${{actions@"$RUNNER_NAME".local}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} with: - gradle-command: :website:publishWebsite + gradle-command: :website:publishWebsiteDocker diff --git a/website/Dockerfile b/website/Dockerfile index 4c4847481045..1b1c32688d12 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -69,4 +69,8 @@ RUN HUGOHOME="$(mktemp -d)" \ && chmod +x /usr/local/bin/hugo \ && rm -r "${HUGOHOME}" +# TODO: Install SSH +RUN apt-get update \ + && apt-get install -y ssh-client + WORKDIR /opt/ diff --git a/website/build.gradle b/website/build.gradle index 18798b428135..c4d9f4bef3a2 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -277,16 +277,16 @@ task preCommit { task commitWebsiteDocker { doLast { - def actor = System.getenv('ghprbPullAuthorLogin') + def actor = System.getenv('ghPullAuthorLogin') def email = System.getenv('ghEmail') - def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss') - String message = "Publishing website ${currentDate} at commit \$LATEST_COMMIT" + + //TODO: Add validation that the build exists + //TODO: Erase testing remote exec { commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """ - export LATEST_COMMIT=\$(git log --format="%h" -n 1) && \ - git remote add website-publish git@github.com:roger-mike/beam.git + git remote add website-publish git@github.com:roger-mike/beam.git && \ git fetch --force origin +asf-site:asf-site && \ git stash && \ git checkout asf-site && \ @@ -311,19 +311,7 @@ task commitWebsiteDocker { """ } - exec { - commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', - """ - if [[ \$(git status --porcelain | wc -l) > 0 ]]; - then - echo "Creating commit for changes" - export ISCOMMITED=1 - git commit -am "${message}" - else - echo "No changes to commit" - fi - """ - } + } } @@ -402,25 +390,39 @@ task publishWebsite { // Because git.push() fails to authenticate, run git push directly. - //shell "git push ${gitboxUrl} asf-site" +// shell "git push ${gitboxUrl} asf-site" } } task publishWebsiteDocker { doLast { - exec { + def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss') + String message = "Publishing website ${currentDate} at commit \$LATEST_COMMIT" + def GITHUB_TOKEN = System.getenv("GITHUB_TOKEN") + def actor = System.getenv('ghPullAuthorLogin') + //TODO: Change repo to gitboxUrl + exec { commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', """ - echo ${gitboxUrl} - if [[ \$ISCOMMITED -eq 1 ]]; - then - echo "Pushing changes" - git push website-publish asf-site - else - echo "No changes to push" - fi + export GITHUB_TOKEN=${GITHUB_TOKEN} """ - } + } + + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + export LATEST_COMMIT=\$(git log --format="%h" -n 1) + if [[ \$(git status --porcelain | wc -l) > 0 ]]; + then + echo "Creating commit for changes" && \ + git commit -am "${message}" && \ + echo "Pushing changes" && \ + git push https://${GITHUB_TOKEN}@github.com/${actor}/beam.git + else + echo "No changes to commit/push" + fi + """ + } } }