refactor: Introduce OpenDAL Workflow Planner

.github/actions/setup/action.yaml

@@ -49,24 +49,26 @@ runs:
       shell: bash
       run: sudo apt-get install libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
 
-    - name: Setup protoc on linux
-      if: inputs.need-protoc == 'true' && runner.os == 'Linux'
-      shell: bash
-      run: sudo apt install -y protobuf-compiler
+    - name: Setup Protoc
+      if: inputs.need-protoc == 'true'
+      uses: arduino/setup-protoc@v2
+      with:
+          version: "23.4"
 
     - name: Cache nextest on linux
       id: cache-nextest
       uses: actions/cache@v3
-      if: inputs.need-nextest == 'true' && runner.os == 'Linux'
+      if: inputs.need-nextest == 'true'
       with:
-        path: ~/.cargo/bin/cargo-nextest
-        # TODO: remove the runner.os while bumping version
-        key: r0-${{runner.os}}-nextest-0.9.53
+        path: |
+          ~/.cargo/bin/cargo-nextest
+          ~/.cargo/bin/cargo-nextest.exe
+        key: r0-${{runner.os}}-nextest-0.9.59
 
     - name: Build nextest if not cached
       if: steps.cache-nextest.outputs.cache-hit != 'true' && inputs.need-nextest == 'true'
       shell: bash
-      run: cargo install cargo-nextest@0.9.53 --locked
+      run: cargo install cargo-nextest@0.9.59 --locked
 
     - name: Setup rust on linux
       if: runner.os == 'Linux' && inputs.need-rocksdb == 'true'

.github/actions/test-core/action.yaml

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Test Core
+description: 'Test Core with given setup and service'
+inputs:
+  setup:
+    description: "The setup action for test"
+  service:
+    description: "The service to test"
+  secrets:
+    description: "The secrets for test"
+    default: "{}"
+
+runs:
+  using: "composite"
+  steps:
+      - name: Setup
+        shell: bash
+        run: |
+            mkdir -p ./dynamic_test_core &&
+            cat <<EOF >./dynamic_test_core/action.yml
+            runs:
+              using: composite
+              steps:
+              - name: Setup Test Core
+                uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
+                with:
+                  secrets: ${{ toJson(inputs.secrets) }}
+              - name: Run Test Core
+                shell: bash
+                working-directory: core
+                run: cargo nextest run services_${{ inputs.service }} --archive-file=./core-test.tar.zst
+            EOF
+      - name: Run
+        id: run
+        uses: ./dynamic_test_core

.github/services/fs/local-fs/action.yml

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: local_fs
+description: "Setup local fs"
+
+runs:
+    using: "composite"
+    steps:
+        - name: Setup
+          shell: bash
+          run: |
+              echo "OPENDAL_FS_TEST=on" >> $GITHUB_ENV
+              echo "OPENDAL_FS_ROOT=${{ runner.temp }}/" >> $GITHUB_ENV

.github/services/s3/aws-s3/action.yml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: aws_s3
+description: "Setup AWS S3"
+
+inputs:
+  secrets:
+    description: "The secrets for test"
+    default: "{}"
+
+# TODO: we should migrate to 1password instead.
+runs:
+  using: "composite"
+  steps:
+    - name: Setup
+      shell: bash
+      run: |
+        echo "OPENDAL_S3_TEST=${{ fromJson(inputs.secrets).OPENDAL_S3_TEST }}" >> $GITHUB_ENV
+        echo "OPENDAL_S3_ROOT=${{ fromJson(inputs.secrets).OPENDAL_S3_ROOT }}" >> $GITHUB_ENV
+        echo "OPENDAL_S3_BUCKET=${{ fromJson(inputs.secrets).OPENDAL_S3_BUCKET }}" >> $GITHUB_ENV
+        echo "OPENDAL_S3_ENDPOINT=${{ fromJson(inputs.secrets).OPENDAL_S3_ENDPOINT }}" >> $GITHUB_ENV
+        echo "OPENDAL_S3_ACCESS_KEY_ID=${{ fromJson(inputs.secrets).OPENDAL_S3_ACCESS_KEY_ID }}" >> $GITHUB_ENV
+        echo "OPENDAL_S3_SECRET_ACCESS_KEY=${{ fromJson(inputs.secrets).OPENDAL_S3_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
+        echo "OPENDAL_S3_REGION=ap-northeast-1" >> $GITHUB_ENV

.github/workflows/core_test.yml

@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Core
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+      cases:
+        required: true
+        type: string
+      features:
+        required: true
+        type: string
+
+jobs:
+  build:
+    name: build
+    runs-on: ${{ inputs.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          need-nextest: true
+          need-protoc: true
+          need-rocksdb: true
+
+      # Required by hdfs
+      - name: Setup java env
+        uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: "11"
+
+      - name: Build
+        shell: bash
+        working-directory: core
+        run: |
+          cargo nextest archive --features ${{ inputs.features }} --archive-file ./core-test.tar.zst
+
+      - name: Upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.os }}-core-test
+          path: ./core/core-test.tar.zst
+
+  test:
+    name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
+    needs: [build]
+    runs-on: ${{ inputs.os }}
+    strategy:
+      matrix:
+        cases: ${{ fromJson(inputs.cases) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          need-nextest: true
+          need-protoc: true
+          need-rocksdb: true
+
+      - name: Download
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ inputs.os }}-core-test
+          path: ./core
+
+      - name: Test Core
+        uses: ./.github/actions/test-core
+        with:
+            setup: ${{ matrix.cases.setup }}
+            service: ${{ matrix.cases.service }}
+            secrets: ${{ toJson(secrets) }}

.github/workflows/service_test_s3.yml

@@ -39,28 +39,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  aws_s3:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup
-        with:
-          need-nextest: true
-      - name: Test
-        shell: bash
-        working-directory: core
-        run: cargo nextest run s3
-        env:
-          OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }}
-          OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }}
-          OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }}
-          OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }}
-          OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }}
-          OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }}
-          OPENDAL_S3_REGION: ap-northeast-1
-
   aws_s3_with_virtual_host:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork