diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot-hermetic.yaml similarity index 100% rename from .github/.OwlBot.yaml rename to .github/.OwlBot-hermetic.yaml diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml new file mode 100644 index 000000000..a7e05557b --- /dev/null +++ b/.github/workflows/hermetic_library_generation.yaml @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Hermetic library generation upon generation config change through pull requests +on: + pull_request: + +jobs: + library_generation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Generate changed libraries + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash generation/hermetic_library_generation.sh \ + --target_branch ${{ github.base_ref }} \ + --current_branch ${{ github.head_ref }} \ + --image_tag $(cat generation_config.yaml | yq .gapic_generator_version) + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/update_googleapis_committish.yaml b/.github/workflows/update_googleapis_committish.yaml new file mode 100644 index 000000000..aac08d039 --- /dev/null +++ b/.github/workflows/update_googleapis_committish.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update googleapis commit +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-googleapis-committish: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update googleapis committish to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash generation/update_googleapis_committish.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh new file mode 100644 index 000000000..d7e55e3c6 --- /dev/null +++ b/generation/hermetic_library_generation.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to, when a pull request changes the generation +# configuration (generation_config.yaml by default): +# 1. Find whether the last commit in this pull request contains changes to +# the generation configuration and exit early if it doesn't have such a change +# since the generation result would be the same. +# 2. Compare generation configurations in the current branch (with which the +# pull request associated) and target branch (into which the pull request is +# merged); +# 3. Generate changed libraries using library_generation image; +# 4. Commit the changes to the pull request, if any. +# 5. Edit the PR body with generated pull request description, if applicable. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. docker + +# The parameters of this script is: +# 1. target_branch, the branch into which the pull request is merged. +# 2. current_branch, the branch with which the pull request is associated. +# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --target_branch) + target_branch="$2" + shift + ;; + --current_branch) + current_branch="$2" + shift + ;; + --image_tag) + image_tag="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${target_branch}" ]; then + echo "missing required argument --target_branch" + exit 1 +fi + +if [ -z "${current_branch}" ]; then + echo "missing required argument --current_branch" + exit 1 +fi + +if [ -z "${image_tag}" ]; then + echo "missing required argument --image_tag" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config=generation_config.yaml + echo "Use default generation config: ${generation_config}" +fi + +workspace_name="/workspace" +baseline_generation_config="baseline_generation_config.yaml" +message="chore: generate libraries at $(date)" + +git checkout "${target_branch}" +git checkout "${current_branch}" +# if the last commit doesn't contain changes to generation configuration, +# do not generate again as the result will be the same. +change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)" +if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then + echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true + exit 0 +fi +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" +config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) + +# run hermetic code generation docker image. +docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" + + +# commit the change to the pull request. +if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then + git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt +else + # The image leaves intermediate folders and files it works with. Here we remove them + rm -rdf output googleapis "${baseline_generation_config}" + git add --all -- ':!pr_description.txt' +fi +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "There is no generated code change with the generation config change ${config_diff}." + echo "Skip committing to the pull request." + exit 0 +fi + +echo "Configuration diff:" +echo "${config_diff}" +git commit -m "${message}" +git push +# set pr body if pr_description.txt is generated. +if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" +fi diff --git a/generation/update_googleapis_committish.sh b/generation/update_googleapis_committish.sh new file mode 100644 index 000000000..5dfcaba56 --- /dev/null +++ b/generation/update_googleapis_committish.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis committish to latest in generation +# configuration at the time of running and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: update googleapis committish at $(date)" + +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}" + +git add "${generation_config}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest googleapis commit is not changed." + echo "Skip committing to the pull request." + exit 0 +fi +git commit -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push +fi diff --git a/generation_config.yaml b/generation_config.yaml new file mode 100644 index 000000000..ca4ed34df --- /dev/null +++ b/generation_config.yaml @@ -0,0 +1,34 @@ +gapic_generator_version: 2.41.0 +protoc_version: '25.2' +googleapis_commitish: 573205a81b9b96ea6d4e0365373839b59f4b427e +libraries_bom_version: 26.38.0 +template_excludes: + - .kokoro/nightly/samples.cfg + - .kokoro/nightly/integration.cfg + - .kokoro/presubmit/samples.cfg + - .kokoro/presubmit/graalvm-native.cfg + - .kokoro/presubmit/graalvm-native-17.cfg + - codecov.yaml + - renovate.json + - .kokoro/build.sh + - .kokoro/requirements.in + - .kokoro/requirements.txt +libraries: + - api_shortname: "bigquery" + name_pretty: "Cloud BigQuery" + product_documentation: "https://cloud.google.com/bigquery" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-bigquery/latest/history" + api_description: "is a fully managed, NoOps, low cost data analytics service.\nData can be streamed into BigQuery at millions of rows per second to enable real-time analysis.\nWith BigQuery you can easily deploy Petabyte-scale Databases." + issue_tracker: "https://issuetracker.google.com/savedsearches/559654" + release_level: "stable" + language: "java" + repo: "googleapis/java-bigquery" + repo_short: "java-bigquery" + distribution_name: "com.google.cloud:google-cloud-bigquery" + codeowner_team: "@googleapis/api-bigquery" + api_id: "bigquery.googleapis.com" + library_type: "GAPIC_MANUAL" + requires_billing: true + recommended_package: "com.google.cloud.bigquery" + GAPICs: + - proto_path: google/api #we use a dummy path to produce a no-op