From dc155e218916368ab0cee47794d7033f00f5e72c Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Fri, 20 Feb 2026 15:07:12 +0900 Subject: [PATCH] ci: add s3 test case Signed-off-by: Kentaro Hayashi --- .github/workflows/apt.matrix.json | 12 ++++++--- .github/workflows/apt.yml | 27 +++++++++++++++++++- .github/workflows/yum.yml | 27 +++++++++++++++++++- fluent-package/apt/commonvar.sh | 15 +++++++++++ fluent-package/apt/systemd-test/s3.sh | 29 +++++++++++++++++++++ fluent-package/test-tools/s3.conf | 31 +++++++++++++++++++++++ fluent-package/yum/systemd-test/common.sh | 18 +++++++++++++ fluent-package/yum/systemd-test/s3.sh | 27 ++++++++++++++++++++ 8 files changed, 180 insertions(+), 6 deletions(-) create mode 100755 fluent-package/apt/systemd-test/s3.sh create mode 100644 fluent-package/test-tools/s3.conf create mode 100755 fluent-package/yum/systemd-test/s3.sh diff --git a/.github/workflows/apt.matrix.json b/.github/workflows/apt.matrix.json index e547d92b8..8bb25f13a 100644 --- a/.github/workflows/apt.matrix.json +++ b/.github/workflows/apt.matrix.json @@ -3,22 +3,26 @@ { "label": "Debian bookworm amd64", "rake-job": "debian-bookworm", - "test-docker-image": "debian:bookworm" + "test-docker-image": "debian:bookworm", + "test-incus-image": "images:debian/12" }, { "label": "Debian trixie amd64", "rake-job": "debian-trixie", - "test-docker-image": "debian:trixie" + "test-docker-image": "debian:trixie", + "test-incus-image": "images:debian/13" }, { "label": "Ubuntu Jammy amd64", "rake-job": "ubuntu-jammy", - "test-docker-image": "ubuntu:jammy" + "test-docker-image": "ubuntu:jammy", + "test-incus-image": "images:ubuntu/22.04" }, { "label": "Ubuntu Noble amd64", "rake-job": "ubuntu-noble", - "test-docker-image": "ubuntu:noble" + "test-docker-image": "ubuntu:noble", + "test-incus-image": "images:ubuntu/24.04" } ] } diff --git a/.github/workflows/apt.yml b/.github/workflows/apt.yml index d1955da9f..8190008aa 100644 --- a/.github/workflows/apt.yml +++ b/.github/workflows/apt.yml @@ -196,7 +196,7 @@ jobs: /fluentd/fluent-package/apt/binstubs-test.sh test: name: Test ${{ matrix.label }} ${{ matrix.test }} - needs: [check_package_size, installation_test, piuparts_test, serverspec_test, confluent_test, binstubs_test] + needs: [check_package_size, installation_test, piuparts_test, serverspec_test, binstubs_test] runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -320,3 +320,28 @@ jobs: --volume ${PWD}:/fluentd:ro \ ${{ matrix.test-docker-image }} \ /fluentd/fluent-package/apt/confluent-test.sh + + s3_test: + name: S3 Test (${{ matrix.label }} s3.sh + needs: [define-matrix, test] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.define-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: packages-${{ matrix.rake-job }} + - name: Setup Incus + run: ./fluent-package/test-tools/setup-incus.sh + - name: Setup localstack on host + run: | + docker run -d --name localstack -p 4566:4566 \ + -e AWS_ACCESS_KEY_ID=localstack-test -e AWS_SECRET_ACCESS_KEY=localstack-test -e AWS_DEFAULT_REGION=ap-northeast-1 localstack/localstack:4.13.1 + - name: Copy s3.conf + run: cp fluent-package/test-tools/s3.conf fluent-package/apt/systemd-test + - name: Ensure localstack availability from host + run: sleep 10 && curl http://127.0.0.1:4566/_localstack/health + - name: Run Test ${{matrix.test}} on ${{ matrix.test-incus-image }} + run: fluent-package/apt/systemd-test/test.sh ${{ matrix.test-incus-image }} s3.sh diff --git a/.github/workflows/yum.yml b/.github/workflows/yum.yml index 29c38d455..e84f45700 100644 --- a/.github/workflows/yum.yml +++ b/.github/workflows/yum.yml @@ -255,7 +255,7 @@ jobs: v2test: name: Test ${{ matrix.label }} ${{ matrix.test }} (CGroup V2) - needs: [define-matrix, check_package_size, installation_test, serverspec_test, confluent_test, binstubs_test] + needs: [define-matrix, check_package_size, installation_test, serverspec_test, binstubs_test] runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -413,3 +413,28 @@ jobs: run: ./fluent-package/test-tools/setup-incus.sh - name: Run Test ${{matrix.test}} on ${{ matrix.test-incus-image }} run: fluent-lts-release/yum/systemd-test/test.sh ${{ matrix.test-incus-image }} ${{ matrix.test }} + + s3_test: + name: S3 Test (${{ matrix.label }} s3.sh + needs: [define-matrix, v2test] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.define-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: packages-${{ matrix.rake-job }} + - name: Setup Incus + run: ./fluent-package/test-tools/setup-incus.sh + - name: Setup localstack on host + run: | + docker run -d --name localstack -p 4566:4566 \ + -e AWS_ACCESS_KEY_ID=localstack-test -e AWS_SECRET_ACCESS_KEY=localstack-test -e AWS_DEFAULT_REGION=ap-northeast-1 localstack/localstack:4.13.1 + - name: Copy s3.conf + run: cp fluent-package/test-tools/s3.conf fluent-package/yum/systemd-test + - name: Ensure localstack availability from host + run: sleep 10 && curl http://127.0.0.1:4566/_localstack/health + - name: Run Test s3.sh on ${{ matrix.test-incus-image }} + run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-incus-image }} s3.sh diff --git a/fluent-package/apt/commonvar.sh b/fluent-package/apt/commonvar.sh index 3d5ff4792..91fe70ff1 100644 --- a/fluent-package/apt/commonvar.sh +++ b/fluent-package/apt/commonvar.sh @@ -45,3 +45,18 @@ function test_suppressed_needrestart() esac fi } + +function install_aws_cli() +{ + case $architecture in + amd64) + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + ;; + arm64) + curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" + ;; + esac + sudo apt install -y unzip + unzip awscliv2.zip + sudo ./aws/install +} diff --git a/fluent-package/apt/systemd-test/s3.sh b/fluent-package/apt/systemd-test/s3.sh new file mode 100755 index 000000000..4a58aee91 --- /dev/null +++ b/fluent-package/apt/systemd-test/s3.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/../commonvar.sh + +# Install the current +sudo apt install -V -y \ + /host/${distribution}/pool/${code_name}/${channel}/*/*/fluent-package_*_${architecture}.deb +systemctl stop --no-pager fluentd + +install_aws_cli + +# Overwrite with s3.conf +GATEWAY=$(ip route | grep default | cut -d' ' -f3) +sed -e "s/127.0.0.1/${GATEWAY}/" /host/fluent-package/apt/systemd-test/s3.conf | sudo tee /etc/fluent/fluentd.conf + +# Check container => host localstack connectivity +curl http://${GATEWAY}:4566/_localstack/health + +sudo systemctl enable --now fluentd +systemctl status --no-pager fluentd + +# wait loading sample data +sleep 20 + +# Check existence of .json in localstack-bucket +count=$(AWS_ACCESS_KEY_ID=localstack-test AWS_SECRET_ACCESS_KEY=localstack-test aws --endpoint-url=http://${GATEWAY}:4566 s3 ls localstack-bucket | grep -c json) +test $count -ge 1 diff --git a/fluent-package/test-tools/s3.conf b/fluent-package/test-tools/s3.conf new file mode 100644 index 000000000..69ac38b3a --- /dev/null +++ b/fluent-package/test-tools/s3.conf @@ -0,0 +1,31 @@ + + @type sample + tag test + + + + @type s3 + + aws_key_id localstack-test + aws_sec_key localstack-test + s3_region ap-northeast-1 + s3_bucket localstack-bucket + s3_endpoint http://127.0.0.1:4566 + ssl_verify_peer false + force_path_style true + path logs-%Y-%m-%d + s3_object_key_format %{path}-%{index}.json + auto_create_bucket true + + + @type file + timekey 10s + timekey_wait 5s + path /tmp/test/s3 + flush_at_shutdown true + flush_interval 0.1 + + + @type json + + diff --git a/fluent-package/yum/systemd-test/common.sh b/fluent-package/yum/systemd-test/common.sh index a8f1b0c9c..4022ca1c6 100755 --- a/fluent-package/yum/systemd-test/common.sh +++ b/fluent-package/yum/systemd-test/common.sh @@ -134,3 +134,21 @@ function install_v6_lts() ;; esac } + +function install_aws_cli() +{ + ARCH=$(rpm --eval '%{_arch}') + case $ARCH in + x86_64) + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + ;; + arm*) + curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" + ;; + *) + ;; + esac + sudo $DNF install -y unzip + unzip awscliv2.zip + sudo ./aws/install +} diff --git a/fluent-package/yum/systemd-test/s3.sh b/fluent-package/yum/systemd-test/s3.sh new file mode 100755 index 000000000..669ebc26b --- /dev/null +++ b/fluent-package/yum/systemd-test/s3.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -exu + +. $(dirname $0)/common.sh + +install_current +install_aws_cli + +(! systemctl status --no-pager fluentd) + +# Overwrite with s3.conf +GATEWAY=$(ip route | grep default | cut -d' ' -f3) +sed -e "s/127.0.0.1/${GATEWAY}/" /host/fluent-package/yum/systemd-test/s3.conf | sudo tee /etc/fluent/fluentd.conf + +# Check container => host localstack connectivity +curl http://${GATEWAY}:4566/_localstack/health + +sudo systemctl enable --now fluentd +systemctl status --no-pager fluentd + +# wait loading sample data +sleep 20 + +# Check existence of .json in localstack-bucket +count=$(AWS_ACCESS_KEY_ID=localstack-test AWS_SECRET_ACCESS_KEY=localstack-test aws --endpoint-url=http://${GATEWAY}:4566 s3 ls localstack-bucket | grep -c json) +test $count -ge 1