From 1f0af1afc7833f5812bc08f16b1d1660015f317e Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 8 Sep 2022 17:11:16 -0500 Subject: [PATCH 1/8] Migrate Precommit Java IOs from Jenkins to GA --- .../job-precommit-java-debezium-io-direct.yml | 54 +++++++++++++++++++ .../job-precommit-java-jdbc-io-direct.yml | 54 +++++++++++++++++++ .../job-precommit-java-jms-io-direct.yml | 49 +++++++++++++++++ .../job-precommit-java-kafka-io-direct.yml | 54 +++++++++++++++++++ .../job-precommit-java-kinesis-io-direct.yml | 54 +++++++++++++++++++ .../job-precommit-java-kudu-io-direct.yml | 49 +++++++++++++++++ .../job-precommit-java-mongodb-io-direct.yml | 49 +++++++++++++++++ .../job-precommit-java-neo4j-io-direct.yml | 54 +++++++++++++++++++ .test-infra/jenkins/README.md | 5 -- ...b_PreCommit_Java_Debezium_IO_Direct.groovy | 38 ------------- .../job_PreCommit_Java_JDBC_IO_Direct.groovy | 38 ------------- .../job_PreCommit_Java_Kafka_IO_Direct.groovy | 38 ------------- ...ob_PreCommit_Java_Kinesis_IO_Direct.groovy | 38 ------------- .../job_PreCommit_Java_Neo4j_IO_Direct.groovy | 38 ------------- CI.md | 50 +++++++++++++++++ 15 files changed, 467 insertions(+), 195 deletions(-) create mode 100644 .github/workflows/job-precommit-java-debezium-io-direct.yml create mode 100644 .github/workflows/job-precommit-java-jdbc-io-direct.yml create mode 100644 .github/workflows/job-precommit-java-jms-io-direct.yml create mode 100644 .github/workflows/job-precommit-java-kafka-io-direct.yml create mode 100644 .github/workflows/job-precommit-java-kinesis-io-direct.yml create mode 100644 .github/workflows/job-precommit-java-kudu-io-direct.yml create mode 100644 .github/workflows/job-precommit-java-mongodb-io-direct.yml create mode 100644 .github/workflows/job-precommit-java-neo4j-io-direct.yml delete mode 100644 .test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy delete mode 100644 .test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy diff --git a/.github/workflows/job-precommit-java-debezium-io-direct.yml b/.github/workflows/job-precommit-java-debezium-io-direct.yml new file mode 100644 index 000000000000..7942ae542e1b --- /dev/null +++ b/.github/workflows/job-precommit-java-debezium-io-direct.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Java SDK Debezium IO Unit Tests and Integration Tests with DirectRunner. + +name: Java Debezium IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/debezium/**'] + +jobs: + java-debezium-io: + name: Run Java Debezium IO Direct + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:debezium:integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:debezium:integrationTest + arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" + - name: Run :sdks:java:io:debezium:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:debezium:test diff --git a/.github/workflows/job-precommit-java-jdbc-io-direct.yml b/.github/workflows/job-precommit-java-jdbc-io-direct.yml new file mode 100644 index 000000000000..81a424d72620 --- /dev/null +++ b/.github/workflows/job-precommit-java-jdbc-io-direct.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Java SDK JDBC IO Unit Tests and Integration Tests with DirectRunner. + +name: Java JDBC IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/jdbc/**'] + +jobs: + java-jdbc-io: + name: Run Java JDBC IO Direct + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:jdbc:integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jdbc:integrationTest + arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" + - name: Run :sdks:java:io:jdbc:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jdbc:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-jms-io-direct.yml b/.github/workflows/job-precommit-java-jms-io-direct.yml new file mode 100644 index 000000000000..c98b5a6a56cd --- /dev/null +++ b/.github/workflows/job-precommit-java-jms-io-direct.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run JMS IO Unit Tests in Java SDK with DirectRunner + +name: Java JMS IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/jms/**'] + +jobs: + java-jms-io: + name: Run Java JMS IO Direct + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:jms:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:jms:test diff --git a/.github/workflows/job-precommit-java-kafka-io-direct.yml b/.github/workflows/job-precommit-java-kafka-io-direct.yml new file mode 100644 index 000000000000..06f7ace5ce42 --- /dev/null +++ b/.github/workflows/job-precommit-java-kafka-io-direct.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Java SDK Kafka IO Unit Tests and Integration Tests with DirectRunner. + +name: Java Kafka IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/kafka/**'] + +jobs: + java-kafka-io: + name: Run Java Kafka IO Direct + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:kafka:kafkaVersionsCompatibilityTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kafka:kafkaVersionsCompatibilityTest + arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" + - name: Run :sdks:java:io:kafka:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kafka:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-kinesis-io-direct.yml b/.github/workflows/job-precommit-java-kinesis-io-direct.yml new file mode 100644 index 000000000000..cb1dc586d510 --- /dev/null +++ b/.github/workflows/job-precommit-java-kinesis-io-direct.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Java SDK Kinesis IO Unit Tests and Integration Tests with DirectRunner. + +name: Java Kinesis IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/kinesis/**'] + +jobs: + java-kinesis-io: + name: Run Java Kinesis IO Direct + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:kinesis:integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kinesis:integrationTest + arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" + - name: Run :sdks:java:io:kinesis:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kinesis:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-kudu-io-direct.yml b/.github/workflows/job-precommit-java-kudu-io-direct.yml new file mode 100644 index 000000000000..f9aa6292279e --- /dev/null +++ b/.github/workflows/job-precommit-java-kudu-io-direct.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Kudu IO Unit Tests in Java SDK with DirectRunner + +name: Java Kudu IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/kudu/**'] + +jobs: + java-kudu-io: + name: Run Java Kudu IO Direct + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:kudu:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kudu:test diff --git a/.github/workflows/job-precommit-java-mongodb-io-direct.yml b/.github/workflows/job-precommit-java-mongodb-io-direct.yml new file mode 100644 index 000000000000..c19a241af1a7 --- /dev/null +++ b/.github/workflows/job-precommit-java-mongodb-io-direct.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run MongoDB IO Unit Tests in Java SDK with DirectRunner + +name: Java MongoDB IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/mongodb/**'] + +jobs: + java-mongodb-io: + name: Run Java MongoDB IO Direct + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:mongodb:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:mongodb:test diff --git a/.github/workflows/job-precommit-java-neo4j-io-direct.yml b/.github/workflows/job-precommit-java-neo4j-io-direct.yml new file mode 100644 index 000000000000..7e3acdb3d59d --- /dev/null +++ b/.github/workflows/job-precommit-java-neo4j-io-direct.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Java SDK Neo4j IO Unit Tests and Integration Tests with DirectRunner. + +name: Java Neo4j IO Direct + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/neo4j/**'] + +jobs: + java-neo4j-io: + name: Run Java Neo4j IO Direct + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:neo4j:integrationTest + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:neo4j:integrationTest + arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" + - name: Run :sdks:java:io:neo4j:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:neo4j:test diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index 2c252a4d55aa..0022c713dc3a 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -33,14 +33,9 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PreCommit_Java | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Phrase/) | `Run Java PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_Cron) | | beam_PreCommit_JavaPortabilityApi | [commit](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApi_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApi_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApi_Phrase/) | `Run JavaPortabilityApi PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApi_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApi_Cron) | | beam_PreCommit_JavaPortabilityApiJava11 | [commit](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApiJava11_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApiJava11_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApiJava11_Phrase/) | `Run JavaPortabilityApiJava11 PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApiJava11_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_JavaPortabilityApiJava11_Cron/) | -| beam_PreCommit_Java_Debezium_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Debezium_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Debezium_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Debezium_IO_Direct_Phrase/) | `Run Java_Debezium_IO_Direct PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_Debezium_IO_Direct_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_Debezium_IO_Direct_Cron/) | | beam_PreCommit_Java_Examples_Dataflow | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Phrase/) | `Run Java_Examples_Dataflow PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Cron) | | beam_PreCommit_Java_Examples_Dataflow_Java11 | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Java11_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Java11_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Java11_Phrase/) | `Run Java_Examples_Dataflow_Java11 PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Java11_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Java11_Cron/) | | beam_PreCommit_Java_GCP_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_GCP_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_GCP_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_GCP_IO_Direct_Phrase/) | `Run Java_GCP_IO_Direct PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_GCP_IO_Direct_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_GCP_IO_Direct_Cron/) | -| beam_PreCommit_Java_JDBC_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_JDBC_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_JDBC_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_JDBC_IO_Direct_Phrase/) | `Run Java_JDBC_IO_Direct PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_JDBC_IO_Direct_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_JDBC_IO_Direct_Cron/) | -| beam_PreCommit_Java_Kafka_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Phrase/) | `Run Java_Kafka_IO_Direct PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Cron/) | -| beam_PreCommit_Java_Kinesis_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Phrase/) | `Run Java_Kinesis_IO_Direct PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Cron/) | -| beam_PreCommit_Java_Neo4j_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Phrase/) | `Run Java_Neo4j_IO_Direct PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Cron/) | | beam_PreCommit_Portable_Python | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Phrase/) | `Run Portable_Python PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron) | | beam_PreCommit_PythonLint | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Phrase/) | `Run PythonLint PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron) | | beam_PreCommit_Python | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Phrase/) | `Run Python PreCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/badge/icon)](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron) | diff --git a/.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy deleted file mode 100644 index ec157c8a1a70..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Debezium_IO_Direct', - gradleTask: ':sdks:java:io:debezium:integrationTest', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true' - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^sdks/java/io/debezium/.*$', - ], - timeoutMins: 60, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy deleted file mode 100644 index ca7d69d5efc7..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_JDBC_IO_Direct', - gradleTask: ':sdks:java:io:jdbc:integrationTest', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true' - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^sdks/java/io/jdbc/.*$', - ], - timeoutMins: 60, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy deleted file mode 100644 index 6869b53ca5cc..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Kafka_IO_Direct', - gradleTask: ':sdks:java:io:kafka:kafkaVersionsCompatibilityTest', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true' - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^sdks/java/io/kafka/.*$', - ], - timeoutMins: 120, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy deleted file mode 100644 index 45861982cfd9..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Kinesis_IO_Direct', - gradleTask: ':sdks:java:io:kinesis:integrationTest', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true' - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^sdks/java/io/kinesis/.*$', - ], - timeoutMins: 60, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} diff --git a/.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy deleted file mode 100644 index b902c78899af..000000000000 --- a/.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import PrecommitJobBuilder - -PrecommitJobBuilder builder = new PrecommitJobBuilder( - scope: this, - nameBase: 'Java_Neo4j_IO_Direct', - gradleTask: ':sdks:java:io:neo4j:integrationTest', - gradleSwitches: [ - '-PdisableSpotlessCheck=true', - '-PdisableCheckStyle=true' - ], // spotless checked in separate pre-commit - triggerPathPatterns: [ - '^sdks/java/io/neo4j/.*$', - ], - timeoutMins: 60, - ) -builder.build { - publishers { - archiveJunit('**/build/test-results/**/*.xml') - } -} diff --git a/CI.md b/CI.md index ed75a4dd4433..375c430afb04 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,56 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PreCommit Workflows + +#### Java Debezium IO Direct - [job-precommit-java-debezium-io-direct.yml](.github/workflows/job-precommit-java-debezium-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-----------------------------|------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Debezium IO Direct | Runs Java SDK Debezium IO unit tests and integration tests with DirectRunner | Yes | Yes | Yes | No | + +#### Java JDBC IO Direct - [job-precommit-java-jdbc-io-direct.yml](.github/workflows/job-precommit-java-jdbc-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-------------------------|--------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java JDBC IO Direct | Runs Java SDK JDBC IO unit tests and integration tests with DirectRunner | Yes | Yes | Yes | No | + +#### Java JMS IO Direct - [job-precommit-java-jms-io-direct.yml](.github/workflows/job-precommit-java-jms-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------------|---------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java JMS IO Direct | Runs Java SDK JMS IO unit tests | Yes | Yes | Yes | No | + +#### Java Kafka IO Direct - [job-precommit-java-kafka-io-direct.yml](.github/workflows/job-precommit-java-kafka-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|--------------------------|---------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Kafka IO Direct | Runs Java SDK Kafka IO unit tests and integration tests with DirectRunner | Yes | Yes | Yes | No | + +#### Java Kinesis IO Direct - [job-precommit-java-kinesis-io-direct.yml](.github/workflows/job-precommit-java-kinesis-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|----------------------------|-----------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Kinesis IO Direct | Runs Java SDK Kinesis IO unit tests and integration tests with DirectRunner | Yes | Yes | Yes | No | + +#### Java Kudu IO Direct - [job-precommit-java-kudu-io-direct.yml](.github/workflows/job-precommit-java-kudu-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-------------------------|----------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Kudu IO Direct | Runs Java SDK Kudu IO unit tests | Yes | Yes | Yes | No | + +#### Java MongoDB IO Direct - [job-precommit-java-mongodb-io-direct.yml](.github/workflows/job-precommit-java-mongodb-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|----------------------------|-------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java MongoDB IO Direct | Runs Java SDK MongoDB IO unit tests | Yes | Yes | Yes | No | + +#### Java Neo4j IO Direct - [job-precommit-java-neo4j-io-direct.yml](.github/workflows/job-precommit-java-neo4j-io-direct.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|--------------------------|---------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Neo4j IO Direct | Runs Java SDK Neo4j IO unit tests and integration tests with DirectRunner | Yes | Yes | Yes | No | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 510129eb86eeddf9f7e0b7719cb5634161583e14 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 15 Sep 2022 14:20:06 -0500 Subject: [PATCH 2/8] Change pull_request to pull_request_target and use optional go and python installation in setup --- .../job-precommit-java-debezium-io-direct.yml | 17 ++++++++++++----- .../job-precommit-java-jdbc-io-direct.yml | 15 +++++++++++---- .../job-precommit-java-jms-io-direct.yml | 17 ++++++++++++----- .../job-precommit-java-kafka-io-direct.yml | 15 +++++++++++---- .../job-precommit-java-kinesis-io-direct.yml | 17 ++++++++++++----- .../job-precommit-java-kudu-io-direct.yml | 17 ++++++++++++----- .../job-precommit-java-mongodb-io-direct.yml | 17 ++++++++++++----- .../job-precommit-java-neo4j-io-direct.yml | 17 ++++++++++++----- 8 files changed, 94 insertions(+), 38 deletions(-) diff --git a/.github/workflows/job-precommit-java-debezium-io-direct.yml b/.github/workflows/job-precommit-java-debezium-io-direct.yml index 7942ae542e1b..7dd7a1802226 100644 --- a/.github/workflows/job-precommit-java-debezium-io-direct.yml +++ b/.github/workflows/job-precommit-java-debezium-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/debezium/**'] +permissions: read-all jobs: java-debezium-io: @@ -37,12 +38,18 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:debezium:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -51,4 +58,4 @@ jobs: - name: Run :sdks:java:io:debezium:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:debezium:test + gradle-command: :sdks:java:io:debezium:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-jdbc-io-direct.yml b/.github/workflows/job-precommit-java-jdbc-io-direct.yml index 81a424d72620..a6d08f24d93d 100644 --- a/.github/workflows/job-precommit-java-jdbc-io-direct.yml +++ b/.github/workflows/job-precommit-java-jdbc-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/jdbc/**'] +permissions: read-all jobs: java-jdbc-io: @@ -37,12 +38,18 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:jdbc:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-jms-io-direct.yml b/.github/workflows/job-precommit-java-jms-io-direct.yml index c98b5a6a56cd..e7230addfc15 100644 --- a/.github/workflows/job-precommit-java-jms-io-direct.yml +++ b/.github/workflows/job-precommit-java-jms-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/jms/**'] +permissions: read-all jobs: java-jms-io: @@ -37,13 +38,19 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:jms:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:jms:test + gradle-command: :sdks:java:io:jms:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-kafka-io-direct.yml b/.github/workflows/job-precommit-java-kafka-io-direct.yml index 06f7ace5ce42..f13ef2adfdf1 100644 --- a/.github/workflows/job-precommit-java-kafka-io-direct.yml +++ b/.github/workflows/job-precommit-java-kafka-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/kafka/**'] +permissions: read-all jobs: java-kafka-io: @@ -37,12 +38,18 @@ jobs: timeout-minutes: 120 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:kafka:kafkaVersionsCompatibilityTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-kinesis-io-direct.yml b/.github/workflows/job-precommit-java-kinesis-io-direct.yml index cb1dc586d510..d2c16f3b77bd 100644 --- a/.github/workflows/job-precommit-java-kinesis-io-direct.yml +++ b/.github/workflows/job-precommit-java-kinesis-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/kinesis/**'] +permissions: read-all jobs: java-kinesis-io: @@ -37,12 +38,18 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:kinesis:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -51,4 +58,4 @@ jobs: - name: Run :sdks:java:io:kinesis:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:kinesis:test \ No newline at end of file + gradle-command: :sdks:java:io:kinesis:test diff --git a/.github/workflows/job-precommit-java-kudu-io-direct.yml b/.github/workflows/job-precommit-java-kudu-io-direct.yml index f9aa6292279e..92e74473a93f 100644 --- a/.github/workflows/job-precommit-java-kudu-io-direct.yml +++ b/.github/workflows/job-precommit-java-kudu-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/kudu/**'] +permissions: read-all jobs: java-kudu-io: @@ -37,13 +38,19 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:kudu:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:kudu:test + gradle-command: :sdks:java:io:kudu:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-mongodb-io-direct.yml b/.github/workflows/job-precommit-java-mongodb-io-direct.yml index c19a241af1a7..63e4990d2992 100644 --- a/.github/workflows/job-precommit-java-mongodb-io-direct.yml +++ b/.github/workflows/job-precommit-java-mongodb-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/mongodb/**'] +permissions: read-all jobs: java-mongodb-io: @@ -37,13 +38,19 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:mongodb:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:mongodb:test + gradle-command: :sdks:java:io:mongodb:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-neo4j-io-direct.yml b/.github/workflows/job-precommit-java-neo4j-io-direct.yml index 7e3acdb3d59d..7231625e2b9d 100644 --- a/.github/workflows/job-precommit-java-neo4j-io-direct.yml +++ b/.github/workflows/job-precommit-java-neo4j-io-direct.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' - pull_request: + tags: ['v*'] + pull_request_target: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/neo4j/**'] +permissions: read-all jobs: java-neo4j-io: @@ -37,12 +38,18 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: + ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:neo4j:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -51,4 +58,4 @@ jobs: - name: Run :sdks:java:io:neo4j:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:neo4j:test + gradle-command: :sdks:java:io:neo4j:test \ No newline at end of file From 7b2cd1f758ae22a432577965cd3600b0ee10629f Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 23 Sep 2022 14:27:57 -0500 Subject: [PATCH 3/8] Change from self-hosted to [self-hosted, ubuntu-20.04] --- .github/workflows/job-precommit-java-debezium-io-direct.yml | 2 +- .github/workflows/job-precommit-java-jdbc-io-direct.yml | 2 +- .github/workflows/job-precommit-java-jms-io-direct.yml | 2 +- .github/workflows/job-precommit-java-kinesis-io-direct.yml | 2 +- .github/workflows/job-precommit-java-kudu-io-direct.yml | 2 +- .github/workflows/job-precommit-java-mongodb-io-direct.yml | 2 +- .github/workflows/job-precommit-java-neo4j-io-direct.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/job-precommit-java-debezium-io-direct.yml b/.github/workflows/job-precommit-java-debezium-io-direct.yml index 7dd7a1802226..2ab479587415 100644 --- a/.github/workflows/job-precommit-java-debezium-io-direct.yml +++ b/.github/workflows/job-precommit-java-debezium-io-direct.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: java-debezium-io: name: Run Java Debezium IO Direct - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 60 steps: - name: Checkout code diff --git a/.github/workflows/job-precommit-java-jdbc-io-direct.yml b/.github/workflows/job-precommit-java-jdbc-io-direct.yml index a6d08f24d93d..784dd64b88e9 100644 --- a/.github/workflows/job-precommit-java-jdbc-io-direct.yml +++ b/.github/workflows/job-precommit-java-jdbc-io-direct.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: java-jdbc-io: name: Run Java JDBC IO Direct - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 60 steps: - name: Checkout code diff --git a/.github/workflows/job-precommit-java-jms-io-direct.yml b/.github/workflows/job-precommit-java-jms-io-direct.yml index e7230addfc15..9add6f38c1f2 100644 --- a/.github/workflows/job-precommit-java-jms-io-direct.yml +++ b/.github/workflows/job-precommit-java-jms-io-direct.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: java-jms-io: name: Run Java JMS IO Direct - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 60 steps: - name: Checkout code diff --git a/.github/workflows/job-precommit-java-kinesis-io-direct.yml b/.github/workflows/job-precommit-java-kinesis-io-direct.yml index d2c16f3b77bd..820d21d271a5 100644 --- a/.github/workflows/job-precommit-java-kinesis-io-direct.yml +++ b/.github/workflows/job-precommit-java-kinesis-io-direct.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: java-kinesis-io: name: Run Java Kinesis IO Direct - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 60 steps: - name: Checkout code diff --git a/.github/workflows/job-precommit-java-kudu-io-direct.yml b/.github/workflows/job-precommit-java-kudu-io-direct.yml index 92e74473a93f..0721d19d07d2 100644 --- a/.github/workflows/job-precommit-java-kudu-io-direct.yml +++ b/.github/workflows/job-precommit-java-kudu-io-direct.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: java-kudu-io: name: Run Java Kudu IO Direct - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 60 steps: - name: Checkout code diff --git a/.github/workflows/job-precommit-java-mongodb-io-direct.yml b/.github/workflows/job-precommit-java-mongodb-io-direct.yml index 63e4990d2992..4fd960996421 100644 --- a/.github/workflows/job-precommit-java-mongodb-io-direct.yml +++ b/.github/workflows/job-precommit-java-mongodb-io-direct.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: java-mongodb-io: name: Run Java MongoDB IO Direct - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 60 steps: - name: Checkout code diff --git a/.github/workflows/job-precommit-java-neo4j-io-direct.yml b/.github/workflows/job-precommit-java-neo4j-io-direct.yml index 7231625e2b9d..c153bf44ed94 100644 --- a/.github/workflows/job-precommit-java-neo4j-io-direct.yml +++ b/.github/workflows/job-precommit-java-neo4j-io-direct.yml @@ -34,7 +34,7 @@ permissions: read-all jobs: java-neo4j-io: name: Run Java Neo4j IO Direct - runs-on: self-hosted + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 60 steps: - name: Checkout code From 92c289c8e996fe7aaefeec75eeef46a3fd0ad630 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Thu, 29 Sep 2022 17:13:06 -0500 Subject: [PATCH 4/8] Remove submodules:recursive --- .github/workflows/job-precommit-java-debezium-io-direct.yml | 1 - .github/workflows/job-precommit-java-jdbc-io-direct.yml | 1 - .github/workflows/job-precommit-java-jms-io-direct.yml | 1 - .github/workflows/job-precommit-java-kafka-io-direct.yml | 1 - .github/workflows/job-precommit-java-kinesis-io-direct.yml | 1 - .github/workflows/job-precommit-java-kudu-io-direct.yml | 1 - .github/workflows/job-precommit-java-mongodb-io-direct.yml | 1 - .github/workflows/job-precommit-java-neo4j-io-direct.yml | 1 - 8 files changed, 8 deletions(-) diff --git a/.github/workflows/job-precommit-java-debezium-io-direct.yml b/.github/workflows/job-precommit-java-debezium-io-direct.yml index b0b868b69215..4bd0df577135 100644 --- a/.github/workflows/job-precommit-java-debezium-io-direct.yml +++ b/.github/workflows/job-precommit-java-debezium-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-jdbc-io-direct.yml b/.github/workflows/job-precommit-java-jdbc-io-direct.yml index 9e505175aabf..8f6e41d64c67 100644 --- a/.github/workflows/job-precommit-java-jdbc-io-direct.yml +++ b/.github/workflows/job-precommit-java-jdbc-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-jms-io-direct.yml b/.github/workflows/job-precommit-java-jms-io-direct.yml index 1214dc295f5f..4b0acdc2f05e 100644 --- a/.github/workflows/job-precommit-java-jms-io-direct.yml +++ b/.github/workflows/job-precommit-java-jms-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-kafka-io-direct.yml b/.github/workflows/job-precommit-java-kafka-io-direct.yml index b85ed7205df7..e3b0e353f668 100644 --- a/.github/workflows/job-precommit-java-kafka-io-direct.yml +++ b/.github/workflows/job-precommit-java-kafka-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-kinesis-io-direct.yml b/.github/workflows/job-precommit-java-kinesis-io-direct.yml index 4af5d7d67666..9dd6c48de262 100644 --- a/.github/workflows/job-precommit-java-kinesis-io-direct.yml +++ b/.github/workflows/job-precommit-java-kinesis-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-kudu-io-direct.yml b/.github/workflows/job-precommit-java-kudu-io-direct.yml index a0645a394ccd..fda5bf070e76 100644 --- a/.github/workflows/job-precommit-java-kudu-io-direct.yml +++ b/.github/workflows/job-precommit-java-kudu-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-mongodb-io-direct.yml b/.github/workflows/job-precommit-java-mongodb-io-direct.yml index d5a821fb2816..b18125e66f19 100644 --- a/.github/workflows/job-precommit-java-mongodb-io-direct.yml +++ b/.github/workflows/job-precommit-java-mongodb-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-neo4j-io-direct.yml b/.github/workflows/job-precommit-java-neo4j-io-direct.yml index 42fd062048e0..bf33505523dc 100644 --- a/.github/workflows/job-precommit-java-neo4j-io-direct.yml +++ b/.github/workflows/job-precommit-java-neo4j-io-direct.yml @@ -43,7 +43,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false - submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action with: From 3cf21588402c1c47a055e6a4a0595cc6cc8cd1f4 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez <74670721+benWize@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:09:59 -0500 Subject: [PATCH 5/8] Update job-precommit-java-kafka-io-direct.yml --- .github/workflows/job-precommit-java-kafka-io-direct.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/job-precommit-java-kafka-io-direct.yml b/.github/workflows/job-precommit-java-kafka-io-direct.yml index e3b0e353f668..e847ad2b650e 100644 --- a/.github/workflows/job-precommit-java-kafka-io-direct.yml +++ b/.github/workflows/job-precommit-java-kafka-io-direct.yml @@ -35,7 +35,7 @@ permissions: read-all jobs: java-kafka-io: name: Run Java Kafka IO Direct - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04] timeout-minutes: 120 steps: - name: Checkout code @@ -58,4 +58,4 @@ jobs: - name: Run :sdks:java:io:kafka:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:kafka:test \ No newline at end of file + gradle-command: :sdks:java:io:kafka:test From ad900c50af4c6761f2694062b581842869014815 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez <74670721+benWize@users.noreply.github.com> Date: Mon, 5 Dec 2022 16:04:22 -0600 Subject: [PATCH 6/8] Release worflow choose rc commit test (#196) * Add workflow to choose a commit to be tagged as RC * Changes to make the description clear --- .github/workflows/choose_rc_commit.yml | 98 ++++++++++++++++++++++++++ CI.md | 6 ++ 2 files changed, 104 insertions(+) create mode 100644 .github/workflows/choose_rc_commit.yml diff --git a/.github/workflows/choose_rc_commit.yml b/.github/workflows/choose_rc_commit.yml new file mode 100644 index 000000000000..d4de9d454a5c --- /dev/null +++ b/.github/workflows/choose_rc_commit.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will choose a commit to be the basis of a release candidate +## and push a new tagged commit for that RC. + +# To learn more about GitHub Actions in Apache Beam check the CI.md + +name: Choose RC Commit +on: + workflow_dispatch: + inputs: + RELEASE: + description: Beam version of current release (e.g. 2.XX.0) + required: true + RC: + description: Integer RC version for the release (e.g. 3 for RC3) + required: true + COMMIT: + description: Selected commit to create the release (should be full commit sha) + required: true + PUSH_TAG: + description: Push tag (yes/no) + required: false + default: no + OVERWRITE: + description: Overwrite RC Tag (yes/no) + required: false + default: no + DEBUG: + description: Debug enabled (yes/no) + required: false + default: no + +jobs: + choose_rc_commit: + runs-on: [self-hosted, ubuntu-20.04] + env: + RC_TAG: v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }} + SCRIPT_DIR: ./release/src/main/scripts + DEBUG: "" + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + ref: release-${{ github.event.inputs.RELEASE }} + - name: Set git config + run: | + git config user.name $GITHUB_ACTOR + git config user.email actions@"$RUNNER_NAME".local + - name: Enable debugging + if: ${{github.event.inputs.DEBUG == 'yes'}} + run: | + echo "DEBUG=--debug" >> $GITHUB_ENV + - name: Set version and commit + run: | + bash "${SCRIPT_DIR}/set_version.sh" "${{ github.event.inputs.RELEASE }}" --release \ + --git-add $DEBUG + # suppress warning about detached HEAD: we want it detached so we do not edit the branch + git checkout --quiet ${{ github.event.inputs.COMMIT }} + git commit -m "Set version for ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" + - name: Overwrite tag + run: | + if git rev-parse "$RC_TAG" >/dev/null 2>&1; then + if [[ "${{ github.event.inputs.OVERWRITE }}" == yes ]]; then + git push origin ":refs/tags/$RC_TAG" + else + echo "Tag $RC_TAG already exists. Either delete it manually or run with --overwrite. Do not overwrite if an RC has been built and shared!" + exit 1 + fi + fi + - name: Tag for Go SDK + # Go Modules defined in sub directories need to have a prefixed tag + # in order to get the matching version. + # See BEAM-13119 for context. + run: git tag -a "sdks/$RC_TAG" -m "Go SDK $RC_TAG" HEAD + - name: Primary tag for the repo + run: git tag -a -m "$RC_TAG" "$RC_TAG" HEAD + - name: Push tag + if: ${{github.event.inputs.PUSH_TAG == 'yes'}} + run: | + git push --follow-tags origin "sdks/$RC_TAG" + git push --follow-tags origin "$RC_TAG" + diff --git a/CI.md b/CI.md index 2f000a3c76dc..2234af307ad4 100644 --- a/CI.md +++ b/CI.md @@ -127,6 +127,12 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog ### Release Preparation and Validation Workflows +#### Choose RC Commit - [choose_rc_commit.yml](.github/workflows/choose_rc_commit.yml) + +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------|-----------------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| +| Choose RC Commit | Chooses a commit to be the basis of a release candidate and pushes a new tagged commit for that RC. | No | No | No | No | + #### Cut Release Branch - [verify_release_build.yml](.github/workflows/cut_release_branch.yml) | Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | |-----------------------|------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| From 9cf8823bcdae9359d67abb216bb3b46962ab631c Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 9 Dec 2022 16:26:57 -0600 Subject: [PATCH 7/8] Add trigger paths and gradle builds --- .../job-precommit-java-debezium-io-direct.yml | 14 +++++++++++++- .../job-precommit-java-jdbc-io-direct.yml | 8 ++++++-- .../workflows/job-precommit-java-jms-io-direct.yml | 6 ++++-- .../job-precommit-java-kafka-io-direct.yml | 12 ++++++++++-- .../job-precommit-java-kinesis-io-direct.yml | 13 ++++++++++++- .../job-precommit-java-kudu-io-direct.yml | 7 +++++-- .../job-precommit-java-mongodb-io-direct.yml | 8 ++++++-- .../job-precommit-java-neo4j-io-direct.yml | 8 ++++++-- 8 files changed, 62 insertions(+), 14 deletions(-) diff --git a/.github/workflows/job-precommit-java-debezium-io-direct.yml b/.github/workflows/job-precommit-java-debezium-io-direct.yml index 4bd0df577135..69d60f170ab3 100644 --- a/.github/workflows/job-precommit-java-debezium-io-direct.yml +++ b/.github/workflows/job-precommit-java-debezium-io-direct.yml @@ -29,7 +29,11 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/debezium/**'] + paths: [ + 'runners/google-cloud-dataflow-java/**', + 'sdks/java/core/src/main/**', + 'sdks/java/io/common/**', + 'sdks/java/io/debezium/**'] permissions: read-all jobs: @@ -50,6 +54,14 @@ jobs: requires-py-38: false requires-py-39: false requires-go-18: false + - name: Run :sdks:java:io:debezium:build + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:debezium:build + - name: Run :sdks:java:io:debezium:expansion-service:build + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:debezium:expansion-service:build - name: Run :sdks:java:io:debezium:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-jdbc-io-direct.yml b/.github/workflows/job-precommit-java-jdbc-io-direct.yml index 8f6e41d64c67..0df733a9b80e 100644 --- a/.github/workflows/job-precommit-java-jdbc-io-direct.yml +++ b/.github/workflows/job-precommit-java-jdbc-io-direct.yml @@ -29,7 +29,11 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/jdbc/**'] + paths: [ + 'sdks/java/core/src/main/**', + 'sdks/java/testing/test-utils/**', + 'sdks/java/io/common/**', + 'sdks/java/io/jdbc/**'] permissions: read-all jobs: @@ -53,7 +57,7 @@ jobs: - name: Run :sdks:java:io:jdbc:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:jdbc:integrationTest + gradle-command: :sdks:java:io:jdbc:build :sdks:java:io:jdbc:integrationTest arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" - name: Run :sdks:java:io:jdbc:test uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/job-precommit-java-jms-io-direct.yml b/.github/workflows/job-precommit-java-jms-io-direct.yml index 4b0acdc2f05e..3ee009b8131d 100644 --- a/.github/workflows/job-precommit-java-jms-io-direct.yml +++ b/.github/workflows/job-precommit-java-jms-io-direct.yml @@ -29,7 +29,9 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/jms/**'] + paths: [ + 'sdks/java/core/src/main/**', + 'sdks/java/io/jms/**'] permissions: read-all jobs: @@ -53,4 +55,4 @@ jobs: - name: Run :sdks:java:io:jms:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:jms:test \ No newline at end of file + gradle-command: :sdks:java:io:jms:build :sdks:java:io:jms:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-kafka-io-direct.yml b/.github/workflows/job-precommit-java-kafka-io-direct.yml index e3b0e353f668..a45886f777e4 100644 --- a/.github/workflows/job-precommit-java-kafka-io-direct.yml +++ b/.github/workflows/job-precommit-java-kafka-io-direct.yml @@ -29,7 +29,15 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/kafka/**'] + paths: [ + 'runners/core-construction-java/**', + 'sdks/java/core/src/main/**', + 'sdks/java/testing/test-utils/**', + 'sdks/java/expansion-service/**', + 'sdks/java/io/common/**', + 'sdks/java/io/synthetic/**', + 'sdks/java/io/expansion-service/**', + 'sdks/java/io/kafka/**'] permissions: read-all jobs: @@ -53,7 +61,7 @@ jobs: - name: Run :sdks:java:io:kafka:kafkaVersionsCompatibilityTest uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:kafka:kafkaVersionsCompatibilityTest + gradle-command: :sdks:java:io:kafka:build :sdks:java:io:kafka:kafkaVersionsCompatibilityTest arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" - name: Run :sdks:java:io:kafka:test uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/job-precommit-java-kinesis-io-direct.yml b/.github/workflows/job-precommit-java-kinesis-io-direct.yml index 9dd6c48de262..f17660962dcd 100644 --- a/.github/workflows/job-precommit-java-kinesis-io-direct.yml +++ b/.github/workflows/job-precommit-java-kinesis-io-direct.yml @@ -29,7 +29,10 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/kinesis/**'] + paths: [ + 'sdks/java/core/src/main/**', + 'sdks/java/io/common/**', + 'sdks/java/io/kinesis/**'] permissions: read-all jobs: @@ -50,6 +53,14 @@ jobs: requires-py-38: false requires-py-39: false requires-go-18: false + - name: Run :sdks:java:io:kinesis:build + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kinesis:build + - name: Run :sdks:java:io:kinesis:expansion-service:build + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kinesis:expansion-service:build - name: Run :sdks:java:io:kinesis:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-kudu-io-direct.yml b/.github/workflows/job-precommit-java-kudu-io-direct.yml index fda5bf070e76..a4fe0bafaff4 100644 --- a/.github/workflows/job-precommit-java-kudu-io-direct.yml +++ b/.github/workflows/job-precommit-java-kudu-io-direct.yml @@ -29,7 +29,10 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/kudu/**'] + paths: [ + 'sdks/java/core/src/main/**', + 'sdks/java/io/common/**', + 'sdks/java/io/kudu/**'] permissions: read-all jobs: @@ -53,4 +56,4 @@ jobs: - name: Run :sdks:java:io:kudu:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:kudu:test \ No newline at end of file + gradle-command: :sdks:java:io:kudu:build :sdks:java:io:kudu:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-mongodb-io-direct.yml b/.github/workflows/job-precommit-java-mongodb-io-direct.yml index b18125e66f19..5cc4f50b3ebd 100644 --- a/.github/workflows/job-precommit-java-mongodb-io-direct.yml +++ b/.github/workflows/job-precommit-java-mongodb-io-direct.yml @@ -29,7 +29,11 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/mongodb/**'] + paths: [ + 'sdks/java/core/src/main/**', + 'sdks/java/testing/test-utils/**', + 'sdks/java/io/common/**', + 'sdks/java/io/mongodb/**'] permissions: read-all jobs: @@ -53,4 +57,4 @@ jobs: - name: Run :sdks:java:io:mongodb:test uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:mongodb:test \ No newline at end of file + gradle-command: :sdks:java:io:mongodb:build :sdks:java:io:mongodb:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-neo4j-io-direct.yml b/.github/workflows/job-precommit-java-neo4j-io-direct.yml index bf33505523dc..1b41d2cc2a26 100644 --- a/.github/workflows/job-precommit-java-neo4j-io-direct.yml +++ b/.github/workflows/job-precommit-java-neo4j-io-direct.yml @@ -29,7 +29,11 @@ on: pull_request_target: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/neo4j/**'] + paths: [ + 'sdks/java/core/src/main/**', + 'sdks/java/testing/test-utils/**', + 'sdks/java/io/common/**', + 'sdks/java/io/neo4j/**'] permissions: read-all jobs: @@ -53,7 +57,7 @@ jobs: - name: Run :sdks:java:io:neo4j:integrationTest uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :sdks:java:io:neo4j:integrationTest + gradle-command: :sdks:java:io:neo4j:build :sdks:java:io:neo4j:integrationTest arguments: "-PdisableSpotlessCheck=true -PdisableCheckStyle=true" - name: Run :sdks:java:io:neo4j:test uses: ./.github/actions/gradle-command-self-hosted-action From d1980c59455b101c0a20fe0209ce844e5a8761e0 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Fri, 9 Dec 2022 16:40:48 -0600 Subject: [PATCH 8/8] Remove file --- .github/workflows/choose_rc_commit.yml | 98 -------------------------- CI.md | 6 -- 2 files changed, 104 deletions(-) delete mode 100644 .github/workflows/choose_rc_commit.yml diff --git a/.github/workflows/choose_rc_commit.yml b/.github/workflows/choose_rc_commit.yml deleted file mode 100644 index d4de9d454a5c..000000000000 --- a/.github/workflows/choose_rc_commit.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This workflow will choose a commit to be the basis of a release candidate -## and push a new tagged commit for that RC. - -# To learn more about GitHub Actions in Apache Beam check the CI.md - -name: Choose RC Commit -on: - workflow_dispatch: - inputs: - RELEASE: - description: Beam version of current release (e.g. 2.XX.0) - required: true - RC: - description: Integer RC version for the release (e.g. 3 for RC3) - required: true - COMMIT: - description: Selected commit to create the release (should be full commit sha) - required: true - PUSH_TAG: - description: Push tag (yes/no) - required: false - default: no - OVERWRITE: - description: Overwrite RC Tag (yes/no) - required: false - default: no - DEBUG: - description: Debug enabled (yes/no) - required: false - default: no - -jobs: - choose_rc_commit: - runs-on: [self-hosted, ubuntu-20.04] - env: - RC_TAG: v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }} - SCRIPT_DIR: ./release/src/main/scripts - DEBUG: "" - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - ref: release-${{ github.event.inputs.RELEASE }} - - name: Set git config - run: | - git config user.name $GITHUB_ACTOR - git config user.email actions@"$RUNNER_NAME".local - - name: Enable debugging - if: ${{github.event.inputs.DEBUG == 'yes'}} - run: | - echo "DEBUG=--debug" >> $GITHUB_ENV - - name: Set version and commit - run: | - bash "${SCRIPT_DIR}/set_version.sh" "${{ github.event.inputs.RELEASE }}" --release \ - --git-add $DEBUG - # suppress warning about detached HEAD: we want it detached so we do not edit the branch - git checkout --quiet ${{ github.event.inputs.COMMIT }} - git commit -m "Set version for ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" - - name: Overwrite tag - run: | - if git rev-parse "$RC_TAG" >/dev/null 2>&1; then - if [[ "${{ github.event.inputs.OVERWRITE }}" == yes ]]; then - git push origin ":refs/tags/$RC_TAG" - else - echo "Tag $RC_TAG already exists. Either delete it manually or run with --overwrite. Do not overwrite if an RC has been built and shared!" - exit 1 - fi - fi - - name: Tag for Go SDK - # Go Modules defined in sub directories need to have a prefixed tag - # in order to get the matching version. - # See BEAM-13119 for context. - run: git tag -a "sdks/$RC_TAG" -m "Go SDK $RC_TAG" HEAD - - name: Primary tag for the repo - run: git tag -a -m "$RC_TAG" "$RC_TAG" HEAD - - name: Push tag - if: ${{github.event.inputs.PUSH_TAG == 'yes'}} - run: | - git push --follow-tags origin "sdks/$RC_TAG" - git push --follow-tags origin "$RC_TAG" - diff --git a/CI.md b/CI.md index 3fd3ad058058..95f9a506c5dc 100644 --- a/CI.md +++ b/CI.md @@ -127,12 +127,6 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog ### Release Preparation and Validation Workflows -#### Choose RC Commit - [choose_rc_commit.yml](.github/workflows/choose_rc_commit.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | -|------------------|-----------------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------| -| Choose RC Commit | Chooses a commit to be the basis of a release candidate and pushes a new tagged commit for that RC. | No | No | No | No | - #### Cut Release Branch - [verify_release_build.yml](.github/workflows/cut_release_branch.yml) | Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | |-----------------------|------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------|