From 88c5ba3e8649fd0f1661113c4ac6447fcb836d50 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Mon, 12 Sep 2022 11:44:03 -0500 Subject: [PATCH 1/3] Migrate and Sharding Java Precommit IOs --- .../workflows/job-precommit-java-mqtt-io.yml | 49 +++++++++++++++++++ .../job-precommit-java-parquet-io.yml | 49 +++++++++++++++++++ .../job-precommit-java-pulsar-io.yml | 49 +++++++++++++++++++ .../job-precommit-java-rabbitmq-io.yml | 49 +++++++++++++++++++ .../workflows/job-precommit-java-redis-io.yml | 49 +++++++++++++++++++ CI.md | 27 ++++++++++ 6 files changed, 272 insertions(+) create mode 100644 .github/workflows/job-precommit-java-mqtt-io.yml create mode 100644 .github/workflows/job-precommit-java-parquet-io.yml create mode 100644 .github/workflows/job-precommit-java-pulsar-io.yml create mode 100644 .github/workflows/job-precommit-java-rabbitmq-io.yml create mode 100644 .github/workflows/job-precommit-java-redis-io.yml diff --git a/.github/workflows/job-precommit-java-mqtt-io.yml b/.github/workflows/job-precommit-java-mqtt-io.yml new file mode 100644 index 000000000000..018b5adf829f --- /dev/null +++ b/.github/workflows/job-precommit-java-mqtt-io.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run MQTT IO Unit Tests in Java SDK + +name: Java MQTT IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/mqtt/**'] + +jobs: + java-mqtt-io: + name: Run Java MQTT IO + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:mqtt:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:mqtt:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-parquet-io.yml b/.github/workflows/job-precommit-java-parquet-io.yml new file mode 100644 index 000000000000..faa48334af97 --- /dev/null +++ b/.github/workflows/job-precommit-java-parquet-io.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Parquet IO Unit Tests in Java SDK + +name: Java Parquet IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/parquet/**'] + +jobs: + java-parquet-io: + name: Run Java Parquet IO + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:parquet:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:parquet:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-pulsar-io.yml b/.github/workflows/job-precommit-java-pulsar-io.yml new file mode 100644 index 000000000000..50f880f4ed28 --- /dev/null +++ b/.github/workflows/job-precommit-java-pulsar-io.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Pulsar IO Unit Tests in Java SDK + +name: Java Pulsar IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/pulsar/**'] + +jobs: + java-pulsar-io: + name: Run Java Pulsar IO + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:pulsar:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:pulsar:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-rabbitmq-io.yml b/.github/workflows/job-precommit-java-rabbitmq-io.yml new file mode 100644 index 000000000000..50f10d278e68 --- /dev/null +++ b/.github/workflows/job-precommit-java-rabbitmq-io.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run RabbitMQ IO Unit Tests in Java SDK + +name: Java RabbitMQ IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/rabbitmq/**'] + +jobs: + java-rabbitmq-io: + name: Run Java RabbitMQ IO + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:rabbitmq:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:rabbitmq:test \ No newline at end of file diff --git a/.github/workflows/job-precommit-java-redis-io.yml b/.github/workflows/job-precommit-java-redis-io.yml new file mode 100644 index 000000000000..58fa1246237a --- /dev/null +++ b/.github/workflows/job-precommit-java-redis-io.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This workflow will run Redis IO Unit Tests in Java SDK + +name: Java Redis IO + +on: + schedule: + - cron: '0 */6 * * *' + push: + branches: ['master', 'release-*'] + tags: 'v*' + pull_request: + branches: ['master', 'release-*'] + tags: 'v*' + paths: ['sdks/java/io/redis/**'] + +jobs: + java-redis-io: + name: Run Java Redis IO + runs-on: self-hosted + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Setup environment + uses: ./.github/actions/setup-self-hosted-action + - name: Run :sdks:java:io:redis:test + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:redis:test \ No newline at end of file diff --git a/CI.md b/CI.md index ed75a4dd4433..65da4eac8aab 100644 --- a/CI.md +++ b/CI.md @@ -125,6 +125,33 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog | Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - | | Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes | +### PreCommit Workflows + +#### Java MQTT IO - [job-precommit-java-mqtt-io.yml](.github/workflows/job-precommit-java-mqtt-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|------------------|------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java MQTT IO | Runs Java MQTT IO Unit Tests | Yes | Yes | Yes | - | + +#### Java Parquet IO - [job-precommit-java-parquet-io.yml](.github/workflows/job-precommit-java-parquet-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|---------------------|---------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Parquet IO | Runs Java Parquet IO Unit Tests | Yes | Yes | Yes | - | + +#### Java Pulsar IO - [job-precommit-java-pulsar-io.yml](.github/workflows/job-precommit-java-pulsar-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|--------------------|--------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Pulsar IO | Runs Java Pulsar IO Unit Tests | Yes | Yes | Yes | - | + +#### Java RabbitMQ IO - [job-precommit-java-rabbitmq-io.yml](.github/workflows/job-precommit-java-rabbitmq-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|----------------------|----------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java RabbitMQ IO | Runs Java RabbitMQ IO Unit Tests | Yes | Yes | Yes | - | + +#### Java Redis IO - [job-precommit-java-redis-io.yml](.github/workflows/job-precommit-java-redis-io.yml) +| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials | +|-------------------|-------------------------------|------------------|-----------------------|---------------|--------------------------| +| Run Java Redis IO | Runs Java Redis IO Unit Tests | Yes | Yes | Yes | - | + ### GitHub Action Tips * If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request. From 4f3ef33c304cff5359e5d7f6a5becf5c9095fbe5 Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Mon, 19 Sep 2022 14:43:45 -0500 Subject: [PATCH 2/3] Add permissions: read-all and minor updates to workflows --- .github/workflows/job-precommit-java-mqtt-io.yml | 12 +++++++++--- .github/workflows/job-precommit-java-parquet-io.yml | 12 +++++++++--- .github/workflows/job-precommit-java-pulsar-io.yml | 12 +++++++++--- .github/workflows/job-precommit-java-rabbitmq-io.yml | 12 +++++++++--- .github/workflows/job-precommit-java-redis-io.yml | 12 +++++++++--- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/.github/workflows/job-precommit-java-mqtt-io.yml b/.github/workflows/job-precommit-java-mqtt-io.yml index 018b5adf829f..da0eb1c6e55f 100644 --- a/.github/workflows/job-precommit-java-mqtt-io.yml +++ b/.github/workflows/job-precommit-java-mqtt-io.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] pull_request: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/mqtt/**'] +permissions: read-all jobs: java-mqtt-io: @@ -37,12 +38,17 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:mqtt:test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-parquet-io.yml b/.github/workflows/job-precommit-java-parquet-io.yml index faa48334af97..be2772b8e725 100644 --- a/.github/workflows/job-precommit-java-parquet-io.yml +++ b/.github/workflows/job-precommit-java-parquet-io.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] pull_request: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/parquet/**'] +permissions: read-all jobs: java-parquet-io: @@ -37,12 +38,17 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:parquet:test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-pulsar-io.yml b/.github/workflows/job-precommit-java-pulsar-io.yml index 50f880f4ed28..27353328c0b6 100644 --- a/.github/workflows/job-precommit-java-pulsar-io.yml +++ b/.github/workflows/job-precommit-java-pulsar-io.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] pull_request: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/pulsar/**'] +permissions: read-all jobs: java-pulsar-io: @@ -37,12 +38,17 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:pulsar:test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-rabbitmq-io.yml b/.github/workflows/job-precommit-java-rabbitmq-io.yml index 50f10d278e68..dca68cc06594 100644 --- a/.github/workflows/job-precommit-java-rabbitmq-io.yml +++ b/.github/workflows/job-precommit-java-rabbitmq-io.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] pull_request: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/rabbitmq/**'] +permissions: read-all jobs: java-rabbitmq-io: @@ -37,12 +38,17 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:rabbitmq:test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/job-precommit-java-redis-io.yml b/.github/workflows/job-precommit-java-redis-io.yml index 58fa1246237a..887188996049 100644 --- a/.github/workflows/job-precommit-java-redis-io.yml +++ b/.github/workflows/job-precommit-java-redis-io.yml @@ -24,11 +24,12 @@ on: - cron: '0 */6 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] pull_request: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/io/redis/**'] +permissions: read-all jobs: java-redis-io: @@ -37,12 +38,17 @@ jobs: timeout-minutes: 60 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: Setup environment uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-37: false + requires-py-38: false + requires-py-39: false + requires-go-18: false - name: Run :sdks:java:io:redis:test uses: ./.github/actions/gradle-command-self-hosted-action with: From 5dfb8704753e43898ac53d1bb0b9328c6a51b16b Mon Sep 17 00:00:00 2001 From: Benjamin Gonzalez Date: Mon, 19 Sep 2022 14:44:50 -0500 Subject: [PATCH 3/3] Testing --- .github/workflows/job-precommit-java-mqtt-io.yml | 2 +- .github/workflows/job-precommit-java-parquet-io.yml | 2 +- .github/workflows/job-precommit-java-pulsar-io.yml | 2 +- .github/workflows/job-precommit-java-rabbitmq-io.yml | 2 +- .github/workflows/job-precommit-java-redis-io.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/job-precommit-java-mqtt-io.yml b/.github/workflows/job-precommit-java-mqtt-io.yml index da0eb1c6e55f..bda11e41bee7 100644 --- a/.github/workflows/job-precommit-java-mqtt-io.yml +++ b/.github/workflows/job-precommit-java-mqtt-io.yml @@ -28,7 +28,7 @@ on: pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/mqtt/**'] +# paths: ['sdks/java/io/mqtt/**'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-java-parquet-io.yml b/.github/workflows/job-precommit-java-parquet-io.yml index be2772b8e725..455b5c23729e 100644 --- a/.github/workflows/job-precommit-java-parquet-io.yml +++ b/.github/workflows/job-precommit-java-parquet-io.yml @@ -28,7 +28,7 @@ on: pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/parquet/**'] +# paths: ['sdks/java/io/parquet/**'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-java-pulsar-io.yml b/.github/workflows/job-precommit-java-pulsar-io.yml index 27353328c0b6..a00352794539 100644 --- a/.github/workflows/job-precommit-java-pulsar-io.yml +++ b/.github/workflows/job-precommit-java-pulsar-io.yml @@ -28,7 +28,7 @@ on: pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/pulsar/**'] +# paths: ['sdks/java/io/pulsar/**'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-java-rabbitmq-io.yml b/.github/workflows/job-precommit-java-rabbitmq-io.yml index dca68cc06594..371398386db9 100644 --- a/.github/workflows/job-precommit-java-rabbitmq-io.yml +++ b/.github/workflows/job-precommit-java-rabbitmq-io.yml @@ -28,7 +28,7 @@ on: pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/rabbitmq/**'] +# paths: ['sdks/java/io/rabbitmq/**'] permissions: read-all jobs: diff --git a/.github/workflows/job-precommit-java-redis-io.yml b/.github/workflows/job-precommit-java-redis-io.yml index 887188996049..b7b2c922ccb7 100644 --- a/.github/workflows/job-precommit-java-redis-io.yml +++ b/.github/workflows/job-precommit-java-redis-io.yml @@ -28,7 +28,7 @@ on: pull_request: branches: ['master', 'release-*'] tags: ['v*'] - paths: ['sdks/java/io/redis/**'] +# paths: ['sdks/java/io/redis/**'] permissions: read-all jobs: