diff --git a/.github/actions/test-core/action.yaml b/.github/actions/behavior_test_core/action.yaml similarity index 87% rename from .github/actions/test-core/action.yaml rename to .github/actions/behavior_test_core/action.yaml index 7e29b6b411f0..ca27b8d3f55d 100644 --- a/.github/actions/test-core/action.yaml +++ b/.github/actions/behavior_test_core/action.yaml @@ -22,9 +22,8 @@ inputs: description: "The setup action for test" service: description: "The service to test" - secrets: - description: "The secrets for test" - default: "{}" + feature: + description: "The feature to test" runs: using: "composite" @@ -39,12 +38,10 @@ runs: steps: - name: Setup Test Core uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }} - with: - secrets: ${{ toJson(inputs.secrets) }} - name: Run Test Core shell: bash working-directory: core - run: cargo nextest run behavior --archive-file=./core-test.tar.zst + run: cargo nextest run behavior --features ${{ inputs.feature }} env: OPENDAL_TEST: ${{ inputs.service }} EOF diff --git a/.github/scripts/behavior_test/__init__.py b/.github/scripts/behavior_test/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/.github/scripts/behavior_test/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/scripts/workflow_planner.py b/.github/scripts/behavior_test/plan.py similarity index 80% rename from scripts/workflow_planner.py rename to .github/scripts/behavior_test/plan.py index 534c68ae5dd3..9040d1a60238 100755 --- a/scripts/workflow_planner.py +++ b/.github/scripts/behavior_test/plan.py @@ -21,9 +21,16 @@ import os from pathlib import Path +# The path for current script. +SCRIPT_PATH = Path(__file__).parent.absolute() +# The path for `.github` dir. +GITHUB_DIR = SCRIPT_PATH.parent.parent +# The project dir for opendal. +PROJECT_DIR = GITHUB_DIR.parent + def get_provided_cases(): - root_dir = ".github/services" + root_dir = f"{GITHUB_DIR}/services" cases = [ { @@ -62,7 +69,9 @@ def calculate_core_cases(cases, changed_files): # If any of the core files changed, we will run all cases. if any( - p.startswith("core/src/") and not p.startswith("core/src/services") + p.startswith("core/") + and not p.startswith("core/src/services/") + and not p.endswith(".md") for p in changed_files ): return cases @@ -89,11 +98,13 @@ def plan(changed_files): jobs["core"] = [ { "os": "ubuntu-latest", - "features": ",".join( - set([f"services-{v['service']}" for v in core_cases]) - ), "cases": [ - {"setup": v["setup"], "service": v["service"]} for v in core_cases + { + "setup": v["setup"], + "service": v["service"], + "feature": "services-{}".format(v["service"].replace("_", "-")), + } + for v in core_cases ], }, ] @@ -103,12 +114,13 @@ def plan(changed_files): jobs["core"].append( { "os": "windows-latest", - "features": "services-fs", - "cases": [{"setup": "local-fs", "service": "fs"}], + "cases": [ + {"setup": "local_fs", "service": "fs", "feature": "services-fs"} + ], } ) - return json.dumps(jobs) + return jobs # For quick test: @@ -117,4 +129,4 @@ def plan(changed_files): if __name__ == "__main__": changed_files = sys.argv[1:] result = plan(changed_files) - print(result) + print(json.dumps(result)) diff --git a/.github/scripts/behavior_test/test_plan.py b/.github/scripts/behavior_test/test_plan.py new file mode 100644 index 000000000000..090a56d14788 --- /dev/null +++ b/.github/scripts/behavior_test/test_plan.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +from plan import plan + + +class BehaviorTestPlan(unittest.TestCase): + def test_empty(self): + result = plan([]) + self.assertEqual(result, {}) + + def test_core_cargo_toml(self): + result = plan(["core/Cargo.toml"]) + self.assertTrue(result["components"]["core"]) + + def test_core_services_fs(self): + result = plan(["core/src/services/fs/mod.rs"]) + self.assertTrue(result["components"]["core"]) + self.assertTrue(len(result["core"]) > 0) + + cases = [v["service"] for v in result["core"][0]["cases"]] + # Should not contain fs + self.assertTrue("fs" in cases) + # Should not contain s3 + self.assertFalse("s3" in cases) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/services/fs/local-fs/action.yml b/.github/services/fs/local_fs/action.yml similarity index 95% rename from .github/services/fs/local-fs/action.yml rename to .github/services/fs/local_fs/action.yml index 72d54608c061..7b36dc3a9a40 100644 --- a/.github/services/fs/local-fs/action.yml +++ b/.github/services/fs/local_fs/action.yml @@ -16,7 +16,7 @@ # under the License. name: local_fs -description: "Setup local fs" +description: 'Behavior test for local fs' runs: using: "composite" diff --git a/.github/services/s3/aws_s3/action.yml b/.github/services/s3/aws_s3/action.yml new file mode 100644 index 000000000000..ce95ec8632e1 --- /dev/null +++ b/.github/services/s3/aws_s3/action.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: aws_s3 +description: 'Behavior test for AWS S3. This service is sponsored by @datafuse_labs.' + +runs: + using: "composite" + steps: + - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_S3_ROOT: op://services/s3/root + OPENDAL_S3_BUCKET: op://services/s3/bucket + OPENDAL_S3_ENDPOINT: op://services/s3/endpoint + OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id + OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key + OPENDAL_S3_REGION: op://services/s3/region diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/aws_s3_with_sse_c/action.yml new file mode 100644 index 000000000000..321b290d6a09 --- /dev/null +++ b/.github/services/s3/aws_s3_with_sse_c/action.yml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: aws_s3_with_sse_c +description: 'Behavior test for AWS S3 with SSE-C. This service is sponsored by @datafuse_labs.' + +runs: + using: "composite" + steps: + - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_S3_ROOT: op://services/s3/root + OPENDAL_S3_BUCKET: op://services/s3/bucket + OPENDAL_S3_ENDPOINT: op://services/s3/endpoint + OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id + OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key + OPENDAL_S3_REGION: op://services/s3/region + + - name: Add extra settings + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256 + OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA= + OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw== + EOF diff --git a/.github/services/s3/aws-s3/action.yml b/.github/services/s3/aws_s3_with_virtual_host/action.yml similarity index 53% rename from .github/services/s3/aws-s3/action.yml rename to .github/services/s3/aws_s3_with_virtual_host/action.yml index b560b443416a..730068d1fecb 100644 --- a/.github/services/s3/aws-s3/action.yml +++ b/.github/services/s3/aws_s3_with_virtual_host/action.yml @@ -15,24 +15,25 @@ # specific language governing permissions and limitations # under the License. -name: aws_s3 -description: "Setup AWS S3" +name: aws_s3_with_virtual_host +description: 'Behavior test for AWS S3 with virtual host. This service is sponsored by @datafuse_labs.' -inputs: - secrets: - description: "The secrets for test" - default: "{}" - -# TODO: we should migrate to 1password instead. runs: using: "composite" steps: - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_S3_ROOT: op://services/s3/root + OPENDAL_S3_BUCKET: op://services/s3/bucket + OPENDAL_S3_ENDPOINT: op://services/s3/endpoint + OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id + OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key + OPENDAL_S3_REGION: op://services/s3/region + + - name: Add extra settings shell: bash run: | - echo "OPENDAL_S3_ROOT=${{ fromJson(inputs.secrets).OPENDAL_S3_ROOT }}" >> $GITHUB_ENV - echo "OPENDAL_S3_BUCKET=${{ fromJson(inputs.secrets).OPENDAL_S3_BUCKET }}" >> $GITHUB_ENV - echo "OPENDAL_S3_ENDPOINT=${{ fromJson(inputs.secrets).OPENDAL_S3_ENDPOINT }}" >> $GITHUB_ENV - echo "OPENDAL_S3_ACCESS_KEY_ID=${{ fromJson(inputs.secrets).OPENDAL_S3_ACCESS_KEY_ID }}" >> $GITHUB_ENV - echo "OPENDAL_S3_SECRET_ACCESS_KEY=${{ fromJson(inputs.secrets).OPENDAL_S3_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV - echo "OPENDAL_S3_REGION=ap-northeast-1" >> $GITHUB_ENV + echo "OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE=on" >> $GITHUB_ENV diff --git a/.github/services/s3/minio_s3/action.yml b/.github/services/s3/minio_s3/action.yml new file mode 100644 index 000000000000..ed237bb37c54 --- /dev/null +++ b/.github/services/s3/minio_s3/action.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: minio_s3 +description: 'Behavior test for Minio S3.' + +runs: + using: "composite" + steps: + - name: Setup MinIO Server + shell: bash + working-directory: fixtures/s3 + run: docker-compose -f docker-compose-minio.yml up -d + - name: Setup test bucket + shell: bash + env: + AWS_ACCESS_KEY_ID: "minioadmin" + AWS_SECRET_ACCESS_KEY: "minioadmin" + AWS_EC2_METADATA_DISABLED: "true" + run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test + - name: Setup + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_BUCKET=test + OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000 + OPENDAL_S3_ACCESS_KEY_ID=minioadmin + OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin + OPENDAL_S3_REGION=us-east-1 + EOF diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/s3/minio_s3_with_anonymous/action.yml new file mode 100644 index 000000000000..407289d9c3c2 --- /dev/null +++ b/.github/services/s3/minio_s3_with_anonymous/action.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: minio_s3_with_anonymous +description: 'Behavior test for Minio S3 with anonymous access.' + +runs: + using: "composite" + steps: + - name: Setup MinIO Server + shell: bash + working-directory: fixtures/s3 + run: | + docker-compose -f docker-compose-minio.yml up -d + - name: Setup test bucket + shell: bash + env: + AWS_ACCESS_KEY_ID: "minioadmin" + AWS_SECRET_ACCESS_KEY: "minioadmin" + AWS_EC2_METADATA_DISABLED: "true" + run: | + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test + + curl -O https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc + ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin + ./mc anonymous set public local/test + - name: Setup + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_BUCKET=test + OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000 + OPENDAL_S3_REGION=us-east-1 + OPENDAL_S3_ALLOW_ANONYMOUS=on + EOF diff --git a/.github/services/s3/r2/action.yml b/.github/services/s3/r2/action.yml new file mode 100644 index 000000000000..38b85141f6c6 --- /dev/null +++ b/.github/services/s3/r2/action.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: r2 +description: 'Behavior test for Cloudflare R2. This service is sponsored by @Xuanwo.' + +runs: + using: "composite" + steps: + - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_S3_BUCKET: op://services/r2/bucket + OPENDAL_S3_ENDPOINT: op://services/r2/endpoint + OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id + OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key + + # OPENDAL_S3_BATCH_MAX_OPERATIONS is the R2's limitation + # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information + - name: Add extra settings + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_S3_REGION=auto + OPENDAL_S3_BATCH_MAX_OPERATIONS=700 + EOF diff --git a/.github/workflows/test_planner.yml b/.github/workflows/behavior_test.yml similarity index 88% rename from .github/workflows/test_planner.yml rename to .github/workflows/behavior_test.yml index 29eabd6f998e..a4b2b240d22e 100644 --- a/.github/workflows/test_planner.yml +++ b/.github/workflows/behavior_test.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: Test +name: Behavior Test on: push: @@ -62,10 +62,9 @@ jobs: has_secrets="true" fi - git fetch origin main:main - files_changed=$(git diff --name-only main ${{ github.event.pull_request.head.sha }}) + files_changed=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}) echo "Files changed:" - echo "$FILES_CHANGED" + echo "$files_changed" fi # Export variables @@ -73,12 +72,12 @@ jobs: export GITHUB_IS_PUSH=$is_push # Run the workflow planner script - PLAN=$(./scripts/workflow_planner.py $files_changed) + PLAN=$(./.github/scripts/behavior_test/plan.py $files_changed) echo "Plan:" echo "$PLAN" | jq . echo "plan=$PLAN" >> $GITHUB_OUTPUT - core_test: + test_core: name: core / ${{ matrix.os }} needs: [plan] if: fromJson(needs.plan.outputs.plan).components.core @@ -86,8 +85,7 @@ jobs: strategy: matrix: include: ${{ fromJson(needs.plan.outputs.plan).core }} - uses: ./.github/workflows/core_test.yml + uses: ./.github/workflows/behavior_test_core.yml with: os: ${{ matrix.os }} cases: ${{ toJson(matrix.cases) }} - features: ${{ matrix.features }} diff --git a/.github/workflows/core_test.yml b/.github/workflows/behavior_test_core.yml similarity index 58% rename from .github/workflows/core_test.yml rename to .github/workflows/behavior_test_core.yml index 9cd213d4e817..344fe62d743c 100644 --- a/.github/workflows/core_test.yml +++ b/.github/workflows/behavior_test_core.yml @@ -26,47 +26,10 @@ on: cases: required: true type: string - features: - required: true - type: string jobs: - build: - name: build - runs-on: ${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - need-protoc: true - need-rocksdb: true - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Required by hdfs - - name: Setup java env - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: "11" - - - name: Build - shell: bash - working-directory: core - run: | - cargo nextest archive --features ${{ inputs.features }} --archive-file ./core-test.tar.zst - - - name: Upload - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.os }}-core-test - path: ./core/core-test.tar.zst - test: name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }} - needs: [build] runs-on: ${{ inputs.os }} strategy: matrix: @@ -79,16 +42,19 @@ jobs: need-nextest: true need-protoc: true need-rocksdb: true + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Download - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.os }}-core-test - path: ./core + # TODO: 1password is only supported on linux + # + # Waiting for https://github.com/1Password/load-secrets-action/issues/46 + - name: Setup service account token for 1password + if: runner.os == 'Linux' + shell: bash + run: echo "OP_SERVICE_ACCOUNT_TOKEN=${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" >> $GITHUB_ENV - name: Test Core - uses: ./.github/actions/test-core + uses: ./.github/actions/behavior_test_core with: setup: ${{ matrix.cases.setup }} service: ${{ matrix.cases.service }} - secrets: ${{ toJson(secrets) }} + feature: ${{ matrix.cases.feature }} diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index 296ba03dfbcc..27d493bc7275 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -39,54 +39,6 @@ concurrency: cancel-in-progress: true jobs: - aws_s3_with_virtual_host: - runs-on: ubuntu-latest - if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} - OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }} - OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }} - OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }} - OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }} - OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE: on - OPENDAL_S3_REGION: ap-northeast-1 - - aws_s3_with_sse_c: - runs-on: ubuntu-latest - if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} - OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }} - OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }} - OPENDAL_S3_ACCESS_KEY_ID: ${{ secrets.OPENDAL_S3_ACCESS_KEY_ID }} - OPENDAL_S3_SECRET_ACCESS_KEY: ${{ secrets.OPENDAL_S3_SECRET_ACCESS_KEY }} - OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: AES256 - OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY: MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA= - OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5: zZ5FnqcIqUjVwvWmyog4zw== - OPENDAL_S3_REGION: ap-northeast-1 - aws_s3_with_assume_role: runs-on: ubuntu-latest if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork @@ -119,106 +71,6 @@ jobs: OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing OPENDAL_S3_REGION: ap-northeast-1 - minio_s3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup MinIO Server - shell: bash - working-directory: fixtures/s3 - run: docker-compose -f docker-compose-minio.yml up -d - - name: Setup test bucket - env: - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_BUCKET: test - OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" - OPENDAL_S3_ACCESS_KEY_ID: minioadmin - OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin - OPENDAL_S3_REGION: us-east-1 - - anonymous_minio_s3: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup MinIO Server - shell: bash - working-directory: fixtures/s3 - run: | - docker-compose -f docker-compose-minio.yml up -d - - name: Setup test bucket - env: - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - run: | - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - curl -O https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - ./mc alias set local http://127.0.0.1:9000/ minioadmin minioadmin - ./mc anonymous set public local/test - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_BUCKET: test - OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" - OPENDAL_S3_ALLOW_ANONYMOUS: on - OPENDAL_S3_REGION: us-east-1 - - r2: - runs-on: ubuntu-latest - if: (github.event_name == 'push' && github.repository == 'apache/incubator-opendal') || !github.event.pull_request.head.repo.fork - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - with: - need-nextest: true - - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v1 - with: - export-env: true - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_S3_BUCKET: op://services/r2/bucket - OPENDAL_S3_ENDPOINT: op://services/r2/endpoint - OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id - OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key - - - name: Test - shell: bash - working-directory: core - run: cargo nextest run behavior - env: - OPENDAL_TEST: s3 - OPENDAL_S3_REGION: auto - # This is the R2's limitation - # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information - OPENDAL_S3_BATCH_MAX_OPERATIONS: 700 - java: runs-on: ubuntu-latest steps: @@ -238,7 +90,7 @@ jobs: - name: Test shell: bash working-directory: bindings/java - run: ./mvnw test -Dtest="behavior.*Test" + run: ./mvnw test -Dtest="behavior.*Test" env: OPENDAL_TEST: s3 OPENDAL_S3_BUCKET: test diff --git a/.gitignore b/.gitignore index d5fd6da84020..dc9a92312a98 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ perf.* # Yarn Integrity file .yarn-integrity + +# Python cache files +**/__pycache__/