From 4b9e3f4a3d18f01045fb6a2e2580ba5e087102c8 Mon Sep 17 00:00:00 2001 From: Gaurav Ghildiyal Date: Mon, 23 Jun 2025 19:34:53 -0700 Subject: [PATCH 1/4] Add _artifacts/ directory to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 44dcbfae..a192412d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ # Test binary, built with `go test -c` *.test +# Directory containing output logs generatedy by running e2e tests. +_artifacts/ + # Output of the go coverage tool, specifically when used with LiteIDE *.out From 678cc7ef29cb03efcd8c733c92e27d7d89e9b1b3 Mon Sep 17 00:00:00 2001 From: Gaurav Ghildiyal Date: Mon, 23 Jun 2025 21:07:14 -0700 Subject: [PATCH 2/4] Use bats asserts for proper logging of got and want outputs --- tests/README.md | 8 +++- tests/e2e.bats | 110 +++++++++++++++++++++++++----------------------- 2 files changed, 64 insertions(+), 54 deletions(-) diff --git a/tests/README.md b/tests/README.md index 0111184f..2f3d63dc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,4 +1,4 @@ -# Integration tests +# Running integration tests 1. Install `bats` https://bats-core.readthedocs.io/en/stable/installation.html @@ -7,6 +7,12 @@ 3. Run `bats tests/` +# Best practices for writing integration tests + +* For clear and debuggable test failures, prefer using a suitable helper + assertion from https://github.com/bats-core/bats-assert. These functions + automatically show "got" (actual) and "want" (expected) outputs, making it + easier to write tests and pinpoint issues. ## eBPF programs diff --git a/tests/e2e.bats b/tests/e2e.bats index e924818d..09e5fbd5 100644 --- a/tests/e2e.bats +++ b/tests/e2e.bats @@ -1,5 +1,8 @@ #!/usr/bin/env bats +load 'test_helper/bats-support/load' +load 'test_helper/bats-assert/load' + @test "dummy interface with IP addresses ResourceClaim" { docker exec "$CLUSTER_NAME"-worker bash -c "ip link add dummy0 type dummy" docker exec "$CLUSTER_NAME"-worker bash -c "ip link set up dev dummy0" @@ -8,11 +11,11 @@ kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim.yaml kubectl wait --timeout=30s --for=condition=ready pods -l app=pod run kubectl exec pod1 -- ip addr show eth99 - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.13"* ]] + assert_success + assert_output --partial "169.254.169.13" run kubectl get resourceclaims dummy-interface-static-ip -o=jsonpath='{.status.devices[0].networkData.ips[*]}' - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.13"* ]] + assert_success + assert_output --partial "169.254.169.13" kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim.yaml @@ -27,11 +30,11 @@ kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim.yaml kubectl wait --timeout=30s --for=condition=ready pods -l app=pod run kubectl exec pod1 -- ip addr show eth99 - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.13"* ]] + assert_success + assert_output --partial "169.254.169.13" run kubectl get resourceclaims dummy-interface-static-ip -o=jsonpath='{.status.devices[0].networkData.ips[*]}' - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.13"* ]] + assert_success + assert_output --partial "169.254.169.13" kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim.yaml @@ -45,12 +48,12 @@ kubectl wait --timeout=30s --for=condition=ready pods -l app=MyApp POD_NAME=$(kubectl get pods -l app=MyApp -o name) run kubectl exec $POD_NAME -- ip addr show dummy1 - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.14"* ]] + assert_success + assert_output --partial "169.254.169.14" # TODO list the specific resourceclaim and the networkdata run kubectl get resourceclaims -o yaml - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.14"* ]] + assert_success + assert_output --partial "169.254.169.14" kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaimtemplate.yaml } @@ -63,16 +66,16 @@ kubectl apply -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_route.yaml kubectl wait --timeout=30s --for=condition=ready pods -l app=pod run kubectl exec pod3 -- ip addr show eth99 - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.13"* ]] + assert_success + assert_output --partial "169.254.169.13" run kubectl exec pod3 -- ip route show - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.0/24 via 169.254.169.1"* ]] + assert_success + assert_output --partial "169.254.169.0/24 via 169.254.169.1" run kubectl get resourceclaims dummy-interface-static-ip-route -o=jsonpath='{.status.devices[0].networkData.ips[*]}' - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.1"* ]] + assert_success + assert_output --partial "169.254.169.1" kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_route.yaml @@ -87,7 +90,7 @@ --restart=Never \ --command \ -- sh -c "curl --silent localhost:9177/metrics | grep process_start_time_seconds >/dev/null && echo ok || echo fail") - test "$output" = "ok" + assert_equal "$output" "ok" } @@ -103,17 +106,17 @@ # Validate mtu and hardware address run kubectl exec pod-advanced-cfg -- ip addr show dranet0 - [ "$status" -eq 0 ] - [[ "$output" == *"169.254.169.14/24"* ]] - [[ "$output" == *"mtu 4321"* ]] - [[ "$output" == *"00:11:22:33:44:55"* ]] + assert_success + assert_output --partial "169.254.169.14/24" + assert_output --partial "mtu 4321" + assert_output --partial "00:11:22:33:44:55" # Validate ethtool settings inside the pod for interface dranet0 run kubectl exec pod-advanced-cfg -- ash -c "apk add ethtool && ethtool -k dranet0" - [ "$status" -eq 0 ] - [[ "$output" == *"tcp-segmentation-offload: off"* ]] - [[ "$output" == *"generic-receive-offload: off"* ]] - [[ "$output" == *"large-receive-offload: off"* ]] + assert_success + assert_output --partial "tcp-segmentation-offload: off" + assert_output --partial "generic-receive-offload: off" + assert_output --partial "large-receive-offload: off" # Cleanup the resources for this test kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_advanced.yaml @@ -130,18 +133,19 @@ kubectl wait --for=condition=ready pod/pod-bigtcp-test --timeout=300s run kubectl exec pod-bigtcp-test -- ip -d link show dranet1 - [ "$status" -eq 0 ] - [[ "$output" == *"mtu 8896"* ]] - [[ "$output" == *"gso_max_size 65536"* ]] - [[ "$output" == *"gro_max_size 65536"* ]] - [[ "$output" == *"gso_ipv4_max_size 65536"* ]] - [[ "$output" == *"gro_ipv4_max_size 65536"* ]] + assert_success + + assert_output --partial "mtu 8896" + assert_output --partial "gso_max_size 65536" + assert_output --partial "gro_max_size 65536" + assert_output --partial "gso_ipv4_max_size 65536" + assert_output --partial "gro_ipv4_max_size 65536" run kubectl exec pod-bigtcp-test -- ash -c "apk add ethtool && ethtool -k dranet1" - [ "$status" -eq 0 ] - [[ "$output" == *"tcp-segmentation-offload: on"* ]] - [[ "$output" == *"generic-receive-offload: on"* ]] - [[ "$output" == *"large-receive-offload: off"* ]] + assert_success + assert_output --partial "tcp-segmentation-offload: on" + assert_output --partial "generic-receive-offload: on" + assert_output --partial "large-receive-offload: off" # Cleanup the resources for this test kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_bigtcp.yaml @@ -158,8 +162,8 @@ docker exec "$CLUSTER_NAME"-worker2 bash -c "tc filter add dev dummy5 ingress bpf direct-action obj dummy_bpf.o sec classifier" run docker exec "$CLUSTER_NAME"-worker2 bash -c "tc filter show dev dummy5 ingress" - [ "$status" -eq 0 ] - [[ "$output" == *"dummy_bpf.o:[classifier] direct-action"* ]] + assert_success + assert_output --partial "dummy_bpf.o:[classifier] direct-action" for attempt in {1..4}; do run kubectl get resourceslices --field-selector spec.nodeName="$CLUSTER_NAME"-worker2 -o jsonpath='{.items[0].spec.devices[?(@.name=="dummy5")].attributes.dra\.net\/ebpf.bool}' @@ -170,13 +174,13 @@ sleep 5 fi done - [ "$status" -eq 0 ] - [[ "$output" == "true" ]] + assert_success + assert_output "true" # Validate bpfName attribute run kubectl get resourceslices --field-selector spec.nodeName="$CLUSTER_NAME"-worker2 -o jsonpath='{.items[0].spec.devices[?(@.name=="dummy5")].attributes.dra\.net\/tcFilterNames.string}' - [ "$status" -eq 0 ] - [[ "$output" == "dummy_bpf.o:[classifier]" ]] + assert_success + assert_output "dummy_bpf.o:[classifier]" } # This reuses previous test @@ -188,21 +192,21 @@ docker exec "$CLUSTER_NAME"-worker2 bash -c "./bpftool net attach tcx_ingress pinned /sys/fs/bpf/dummy_prog_tcx dev dummy5" run docker exec "$CLUSTER_NAME"-worker2 bash -c "./bpftool net show dev dummy5" - [ "$status" -eq 0 ] - [[ "$output" == *"tcx/ingress handle_ingress prog_id"* ]] + assert_success + assert_output --partial "tcx/ingress handle_ingress prog_id" # Wait for the interface to be discovered sleep 5 # Validate bpf attribute is true run kubectl get resourceslices --field-selector spec.nodeName="$CLUSTER_NAME"-worker2 -o jsonpath='{.items[0].spec.devices[?(@.name=="dummy5")].attributes.dra\.net\/ebpf.bool}' - [ "$status" -eq 0 ] - [[ "$output" == "true" ]] + assert_success + assert_output "true" # Validate bpfName attribute run kubectl get resourceslices --field-selector spec.nodeName="$CLUSTER_NAME"-worker2 -o jsonpath='{.items[0].spec.devices[?(@.name=="dummy5")].attributes.dra\.net\/tcxProgramNames.string}' - [ "$status" -eq 0 ] - [[ "$output" == "handle_ingress" ]] + assert_success + assert_output "handle_ingress" } # This reuses previous test @@ -212,12 +216,12 @@ kubectl wait --for=condition=ready pod/pod-ebpf --timeout=300s run kubectl exec pod-ebpf -- ash -c "curl --connect-timeout 5 --retry 3 -L https://github.com/libbpf/bpftool/releases/download/v7.5.0/bpftool-v7.5.0-amd64.tar.gz | tar -xz && chmod +x bpftool" - [ "$status" -eq 0 ] + assert_success run kubectl exec pod-ebpf -- ash -c "./bpftool net show dev dummy5" - [ "$status" -eq 0 ] - [[ "$output" != *"tcx/ingress handle_ingress prog_id"* ]] - [[ "$output" != *"dummy_bpf.o:[classifier]"* ]] + assert_success + refute_output --partial "tcx/ingress handle_ingress prog_id" + refute_output --partial "dummy_bpf.o:[classifier]" kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_disable_ebpf.yaml kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml From 57c6d16e15806ec14a29e85199b57bd9f9f3dade Mon Sep 17 00:00:00 2001 From: Gaurav Ghildiyal Date: Mon, 23 Jun 2025 21:51:57 -0700 Subject: [PATCH 3/4] Specify bat libraries path in github action and document tests/README.md --- .github/workflows/bats.yml | 4 ++++ tests/README.md | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml index 55992ebb..446c025d 100644 --- a/.github/workflows/bats.yml +++ b/.github/workflows/bats.yml @@ -24,9 +24,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Setup Bats and bats libs id: setup-bats uses: bats-core/bats-action@3.0.1 + with: + support-path: ${{ github.workspace }}/tests/test_helper/bats-support + assert-path: "${{ github.workspace }}/tests/test_helper/bats-assert" - name: Bats tests shell: bash env: diff --git a/tests/README.md b/tests/README.md index 2f3d63dc..e763fd9f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,7 +5,9 @@ 2. Install `kind` https://kind.sigs.k8s.io/ -3. Run `bats tests/` +3. Ensure git submodules have been initialized: `git submodule update --init --recursive` + +4. Run `bats tests/` # Best practices for writing integration tests From b92f6f729ebe3f81a22174f5fee416af854a506d Mon Sep 17 00:00:00 2001 From: Gaurav Ghildiyal Date: Tue, 24 Jun 2025 01:13:20 -0700 Subject: [PATCH 4/4] Add bats-support and bats-assert git submodules --- .gitmodules | 6 ++++++ tests/test_helper/bats-assert | 1 + tests/test_helper/bats-support | 1 + 3 files changed, 8 insertions(+) create mode 100644 .gitmodules create mode 160000 tests/test_helper/bats-assert create mode 160000 tests/test_helper/bats-support diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..0d8292be --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "tests/test_helper/bats-assert"] + path = tests/test_helper/bats-assert + url = https://github.com/ztombol/bats-assert +[submodule "tests/test_helper/bats-support"] + path = tests/test_helper/bats-support + url = https://github.com/ztombol/bats-support diff --git a/tests/test_helper/bats-assert b/tests/test_helper/bats-assert new file mode 160000 index 00000000..9f88b420 --- /dev/null +++ b/tests/test_helper/bats-assert @@ -0,0 +1 @@ +Subproject commit 9f88b4207da750093baabc4e3f41bf68f0dd3630 diff --git a/tests/test_helper/bats-support b/tests/test_helper/bats-support new file mode 160000 index 00000000..004e7076 --- /dev/null +++ b/tests/test_helper/bats-support @@ -0,0 +1 @@ +Subproject commit 004e707638eedd62e0481e8cdc9223ad471f12ee