diff --git a/.github/workflows/job-postcommit-website-publish.yml b/.github/workflows/job-postcommit-website-publish.yml new file mode 100644 index 000000000000..7bf917f47e00 --- /dev/null +++ b/.github/workflows/job-postcommit-website-publish.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Publishes generated website content into asf-site branch for hosting + +name: PostCommit Website Publish + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-website'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' +permissions: write-all + +jobs: + website-publish: + name: PostCommit Website Publish + runs-on: [self-hosted, ubuntu-20.04] + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :website:clean + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :website:clean + + - name: Set git env var + run : | + ghEmail=actions@"$RUNNER_NAME".local + echo "ghEmail=${ghEmail}" >> $GITHUB_ENV + + - name: Run :website:publishWebsite + uses: ./.github/actions/gradle-command-self-hosted-action + env: + ghprbPullId: ${{github.event.pull_request.number}} + ghPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repository}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + gradle-command: :website:publishWebsiteDocker diff --git a/.github/workflows/job-postcommit-website-test.yml b/.github/workflows/job-postcommit-website-test.yml new file mode 100644 index 000000000000..b0a1245af687 --- /dev/null +++ b/.github/workflows/job-postcommit-website-test.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test to validate the Beam website, including external links. + +name: PostCommit Website Test + +on: + workflow_dispatch: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*', 'postcommit-website'] + tags: 'v*' + pull_request_target: + branches: [ 'master', 'release-*'] + tags: 'v*' +permissions: read-all + +jobs: + website-test: + name: PostCommit Website Test + runs-on: [self-hosted, ubuntu-20.04] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + + - name: Run :website:testWebsite + uses: ./.github/actions/gradle-command-self-hosted-action + env: + ghprbPullId: ${{github.event.pull_request.number}} + ghprbPullAuthorLogin: ${{github.actor}} + ghprbSourceBranch: ${{github.ref_name}} + ghBranch: ${{github.head_ref}} + repositoryUrl: ${{github.repository}} + with: + gradle-command: :website:testWebsite + arguments: "-PdisableExternal=false" \ No newline at end of file diff --git a/CI.md b/CI.md index ee911584f7f0..c81e9b9b453c 100644 --- a/CI.md +++ b/CI.md @@ -138,9 +138,10 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | [job-precommit-placeholder.yml](.github/workflows/job-precommit-placeholder.yml) | Description placeholder | Yes/No | ### PostCommit Workflows -| Workflow | Description | Requires GCP Credentials | -|------------------------------------------------------------------------------------|-------------------------|--------------------------| -| [job-postcommit-placeholder.yml](.github/workflows/job-postcommit-placeholder.yml) | Description placeholder | Yes/No | +| Workflow | Description | Requires GCP Credentials | +|--------------------------------------------------------------------------------------------|----------------------------------------------------------------------|--------------------------| +| [job-postcommit-website-publish.yml](.github/workflows/job-postcommit-website-publish.yml) | Publishes generated website content into asf-site branch for hosting | No | +| [job-postcommit-website-test.yml](.github/workflows/job-postcommit-website-test.yml) | Test to validate the Beam website, including external links. | No | ### GitHub Action Tips diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java index ee816a944e5a..edead73227d7 100644 --- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java +++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactory.java @@ -182,8 +182,7 @@ private List gcsCredentialArgs() { // TODO(https://github.com/apache/beam/issues/19061): Allow this to be disabled manually. if (Files.exists(Paths.get(localGcloudConfig))) { return ImmutableList.of( - "--mount", - String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); + "-v", String.format("type=bind,src=%s,dst=%s", localGcloudConfig, dockerGcloudConfig)); } else { return ImmutableList.of(); } diff --git a/website/Dockerfile b/website/Dockerfile index 4c4847481045..1b1c32688d12 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -69,4 +69,8 @@ RUN HUGOHOME="$(mktemp -d)" \ && chmod +x /usr/local/bin/hugo \ && rm -r "${HUGOHOME}" +# TODO: Install SSH +RUN apt-get update \ + && apt-get install -y ssh-client + WORKDIR /opt/ diff --git a/website/build.gradle b/website/build.gradle index 6aa88a2c7feb..c4d9f4bef3a2 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -80,10 +80,10 @@ task createDockerContainer(type: Exec) { } else { // Otherwise: run as current user, so as to not require sudo to clean up // build/ directory. - extraOptions += " -u \$(id -u):\$(id -g)" + extraOptions += " -u 0:0" } commandLine '/bin/bash', '-c', - "docker create -v $project.rootDir:$dockerWorkDir $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" + "docker create $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'" } } @@ -115,9 +115,23 @@ task buildCodeSamples(type: Exec) { "${->startDockerContainer.containerId()}", 'yarn', 'build_code_samples' } +task cloneRepository(type: Exec) { + def branch = System.getenv('ghBranch') ? System.getenv('ghBranch'): + grgit ? grgit.branch.current().getName(): 'master' + def repositoryUrl = System.getenv('repositoryUrl') ? "https://github.com/${System.getenv('repositoryUrl')}" : + "https://github.com/apache/beam" + + commandLine 'docker', 'exec', + "${->startDockerContainer.containerId()}", "git", "clone", "-b", branch, repositoryUrl, "." +} + +initGitSubmodules.mustRunAfter cloneRepository +installDependencies.mustRunAfter cloneRepository +buildCodeSamples.mustRunAfter cloneRepository + task setupDockerContainer(type: Exec) { dependsOn startDockerContainer - finalizedBy initGitSubmodules, installDependencies, buildCodeSamples + finalizedBy cloneRepository, initGitSubmodules, installDependencies, buildCodeSamples ext.containerId = { return startDockerContainer.containerId() } @@ -147,7 +161,7 @@ def createBuildTask = { BuildTaskConfiguration config = it as BuildTaskConfiguration task "build${config.name}Website" (type:Exec) { dependsOn setupDockerContainer - finalizedBy stopAndRemoveDockerContainer +// finalizedBy stopAndRemoveDockerContainer def configs = "$dockerSourceDir/site/config.toml" def baseUrlFlag = config.baseUrl ? "--baseURL /${config.baseUrl}" : "" @@ -261,14 +275,55 @@ task preCommit { dependsOn testWebsite } +task commitWebsiteDocker { + doLast { + def actor = System.getenv('ghPullAuthorLogin') + def email = System.getenv('ghEmail') + + + //TODO: Add validation that the build exists + //TODO: Erase testing remote + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + git remote add website-publish git@github.com:roger-mike/beam.git && \ + git fetch --force origin +asf-site:asf-site && \ + git stash && \ + git checkout asf-site && \ + git rm -rq 'website/generated-content' + """ + } + + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + mkdir website/generated-content + cp -r build/website/generated-apache-content/* website/generated-content + """ + } + + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + git add 'website/generated-content' + git config user.name ${actor} + git config user.email ${email} + """ + } + + + + } +} + // Creates a new commit on asf-site branch task commitWebsite { doLast { assert grgit : "Cannot commit website outside of git repository" - assert file("${buildContentDir('apache')}/index.html").exists() +// assert file("${buildContentDir('apache')}/index.html").exists() // Generated javadoc and pydoc content is not built or stored in this repo. - assert !file("${buildContentDir('apache')}/documentation/sdks/javadoc").exists() - assert !file("${buildContentDir('apache')}/documentation/sdks/pydoc").exists() +// assert !file("${buildContentDir('apache')}/documentation/sdks/javadoc").exists() +// assert !file("${buildContentDir('apache')}/documentation/sdks/pydoc").exists() def git = grgit.open(currentDir: project.rootDir) // get the latest commit on master @@ -327,18 +382,52 @@ task publishWebsite { def git = grgit.open(currentDir: project.rootDir) git.checkout(branch: 'asf-site') + if (!commitedChanges) { println 'No changes to push' return } + // Because git.push() fails to authenticate, run git push directly. - shell "git push ${gitboxUrl} asf-site" +// shell "git push ${gitboxUrl} asf-site" + } +} + +task publishWebsiteDocker { + doLast { + def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss') + String message = "Publishing website ${currentDate} at commit \$LATEST_COMMIT" + def GITHUB_TOKEN = System.getenv("GITHUB_TOKEN") + def actor = System.getenv('ghPullAuthorLogin') + //TODO: Change repo to gitboxUrl + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + export GITHUB_TOKEN=${GITHUB_TOKEN} + """ + } + + exec { + commandLine 'docker', 'exec', "${->setupDockerContainer.containerId()}", '/bin/bash', '-c', + """ + export LATEST_COMMIT=\$(git log --format="%h" -n 1) + if [[ \$(git status --porcelain | wc -l) > 0 ]]; + then + echo "Creating commit for changes" && \ + git commit -am "${message}" && \ + echo "Pushing changes" && \ + git push https://${GITHUB_TOKEN}@github.com/${actor}/beam.git + else + echo "No changes to commit/push" + fi + """ + } } } -commitWebsite.dependsOn buildApacheWebsite -publishWebsite.dependsOn commitWebsite +commitWebsiteDocker.dependsOn buildApacheWebsite +publishWebsiteDocker.dependsOn commitWebsiteDocker /* * Stages a pull request on GCS