diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml index ce82a454..55992ebb 100644 --- a/.github/workflows/bats.yml +++ b/.github/workflows/bats.yml @@ -32,7 +32,7 @@ jobs: env: BATS_LIB_PATH: ${{ steps.setup-bats.outputs.lib-path }} TERM: xterm - run: bats -o _artifacts tests/ + run: bats -o _artifacts --print-output-on-failure tests/ - name: Upload logs if: always() diff --git a/go.mod b/go.mod index 3507b75a..9fb45b0a 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( cloud.google.com/go/compute/metadata v0.7.0 cloud.google.com/go/container v1.43.0 github.com/Mellanox/rdmamap v1.1.0 + github.com/cilium/ebpf v0.18.0 github.com/containerd/nri v0.9.0 github.com/google/cel-go v0.25.0 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index 3723aae8..ae1addd8 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.18.0 h1:OsSwqS4y+gQHxaKgg2U/+Fev834kdnsQbtzRnbVC6Gs= +github.com/cilium/ebpf v0.18.0/go.mod h1:vmsAT73y4lW2b4peE+qcOqw6MxvWQdC+LiU5gd/xyo4= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.9.0 h1:jribDJs/oQ95vLO4Yn19HKFYriZGWKiG6nKWjl9Y/x4= @@ -48,6 +50,8 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -83,6 +87,9 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink v1.3.5 h1:hVlNQNRlLDGZz31gBPicsG7Q53rnlsz1l1Ix/9XlpVA= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/pkg/inventory/db.go b/pkg/inventory/db.go index ea63ca2d..064246b6 100644 --- a/pkg/inventory/db.go +++ b/pkg/inventory/db.go @@ -251,6 +251,22 @@ func (db *DB) netdevToDRAdev(link netlink.Link) (*resourceapi.Device, error) { device.Basic.Attributes["dra.net/alias"] = resourceapi.DeviceAttribute{StringValue: &linkAttrs.Alias} device.Basic.Attributes["dra.net/type"] = resourceapi.DeviceAttribute{StringValue: &linkType} + // Get eBPF properties from the interface using the legacy tc hooks + isEbpf := false + filterNames, ok := getTcFilters(link) + if ok { + isEbpf = true + device.Basic.Attributes["dra.net/tcFilterNames"] = resourceapi.DeviceAttribute{StringValue: ptr.To(strings.Join(filterNames, ","))} + } + + // Get eBPF properties from the interface using the tcx hooks + programNames, ok := getTcxFilters(link) + if ok { + isEbpf = true + device.Basic.Attributes["dra.net/tcxProgramNames"] = resourceapi.DeviceAttribute{StringValue: ptr.To(strings.Join(programNames, ","))} + } + device.Basic.Attributes["dra.net/ebpf"] = resourceapi.DeviceAttribute{BoolValue: &isEbpf} + isRDMA := rdmamap.IsRDmaDeviceForNetdevice(ifName) device.Basic.Attributes["dra.net/rdma"] = resourceapi.DeviceAttribute{BoolValue: &isRDMA} // from https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin/blob/ed1c14dd4c313c7dd9fe4730a60358fbeffbfdd4/pkg/netdevice/netDeviceProvider.go#L99 diff --git a/pkg/inventory/routes.go b/pkg/inventory/net.go similarity index 59% rename from pkg/inventory/routes.go rename to pkg/inventory/net.go index b622ac4c..f2f14d9a 100644 --- a/pkg/inventory/routes.go +++ b/pkg/inventory/net.go @@ -17,6 +17,8 @@ limitations under the License. package inventory import ( + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/link" "github.com/vishvananda/netlink" "k8s.io/apimachinery/pkg/util/sets" @@ -68,3 +70,51 @@ func getDefaultGwInterfaces() sets.Set[string] { klog.V(4).Infof("Found following interfaces for the default gateway: %v", interfaces.UnsortedList()) return interfaces } + +func getTcFilters(link netlink.Link) ([]string, bool) { + isTcEBPF := false + filterNames := sets.Set[string]{} + for _, parent := range []uint32{netlink.HANDLE_MIN_INGRESS, netlink.HANDLE_MIN_EGRESS} { + filters, err := netlink.FilterList(link, parent) + if err == nil { + for _, f := range filters { + if bpffFilter, ok := f.(*netlink.BpfFilter); ok { + isTcEBPF = true + filterNames.Insert(bpffFilter.Name) + } + } + } + } + return filterNames.UnsortedList(), isTcEBPF +} + +// see https://github.com/cilium/ebpf/issues/1117 +func getTcxFilters(device netlink.Link) ([]string, bool) { + isTcxEBPF := false + programNames := sets.Set[string]{} + for _, attach := range []ebpf.AttachType{ebpf.AttachTCXIngress, ebpf.AttachTCXEgress} { + result, err := link.QueryPrograms(link.QueryOptions{ + Target: int(device.Attrs().Index), + Attach: attach, + }) + if err != nil { + continue + } + + isTcxEBPF = true + for _, p := range result.Programs { + prog, err := ebpf.NewProgramFromID(p.ID) + if err != nil { + continue + } + defer prog.Close() + + pi, err := prog.Info() + if err != nil { + continue + } + programNames.Insert(pi.Name) + } + } + return programNames.UnsortedList(), isTcxEBPF +} diff --git a/site/content/docs/contributing/developer-guide.md b/site/content/docs/contributing/developer-guide.md index 31c27db9..f10e86f6 100644 --- a/site/content/docs/contributing/developer-guide.md +++ b/site/content/docs/contributing/developer-guide.md @@ -55,6 +55,37 @@ Waiting for daemon set "dranet" rollout to finish: 1 out of 5 new pods have been updated... ``` +## Accesing nodes + +When developing it may be also useful to log into the nodes to be able to debug problems. Just obtain the list of nodes with `kubectl get nodes -o wide` + +For Kind clusters use `docker exec -it bash` + +```sh +kubectl get nodes -o wide +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +gke-cluster-tpu-v6-default-pool-3f96de9d-hr87 Ready 8d v1.33.1-gke.1107000 10.202.0.21 34.162.194.173 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-cluster-tpu-v6-default-pool-955df71d-zd7b Ready 8d v1.33.1-gke.1107000 10.202.0.15 34.162.239.241 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-cluster-tpu-v6-default-pool-e1c69e67-k0jw Ready 8d v1.33.1-gke.1107000 10.202.0.20 34.162.209.25 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-kgdj Ready 8d v1.33.1-gke.1107000 10.202.0.26 34.162.163.69 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-prgf Ready 8d v1.33.1-gke.1107000 10.202.0.24 34.162.144.5 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-sjcp Ready 8d v1.33.1-gke.1107000 10.202.0.27 34.162.117.62 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-z8g1 Ready 8d v1.33.1-gke.1107000 10.202.0.25 34.162.239.83 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +``` + +For GKE or other clusters, you may have restrictions on ssh, so you can use + +```sh + kubectl debug -it node/gke-tpu-de8b9feb-kgdj --image busybox -- chroot /host +--profile=legacy is deprecated and will be removed in the future. It is recommended to explicitly specify a profile, for example "--profile=general". +Creating debugging pod node-debugger-gke-tpu-de8b9feb-kgdj-94xdx with container debugger on node gke-tpu-de8b9feb-kgdj. +If you don't see a command prompt, try pressing enter. +gke-tpu-de8b9feb-kgdj / # +``` + +If you want to upload some binary, per example `bpftrace` or `pwru` , you can use the streaming capabilities for that: + +``` ## Troubleshooting diff --git a/tests/dummy_bpf.c b/tests/dummy_bpf.c new file mode 100644 index 00000000..f1097aab --- /dev/null +++ b/tests/dummy_bpf.c @@ -0,0 +1,9 @@ +#include +#include + +__attribute__((section("classifier"), used)) +int handle_ingress(struct __sk_buff *skb) { + return TC_ACT_OK; +} + +char __license[] __attribute__((section("license"), used)) = "GPL"; \ No newline at end of file diff --git a/tests/dummy_bpf.o b/tests/dummy_bpf.o new file mode 100644 index 00000000..1f725b85 Binary files /dev/null and b/tests/dummy_bpf.o differ diff --git a/tests/e2e.bats b/tests/e2e.bats index 56613c73..bf5d9e4e 100644 --- a/tests/e2e.bats +++ b/tests/e2e.bats @@ -147,3 +147,31 @@ kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/resourceclaim_bigtcp.yaml kubectl delete -f "$BATS_TEST_DIRNAME"/../examples/deviceclass.yaml } + + +# Test case for validating ebpf attributes are exposed via resource slice. +# TODO use tcx hooks too +@test "validate bpf filter attributes" { + docker cp "$BATS_TEST_DIRNAME"/dummy_bpf.o "$CLUSTER_NAME"-worker2:/dummy_bpf.o + docker exec "$CLUSTER_NAME"-worker2 bash -c "ip link add dummy5 type dummy" + docker exec "$CLUSTER_NAME"-worker2 bash -c "ip link set up dev dummy5" + docker exec "$CLUSTER_NAME"-worker2 bash -c "tc qdisc add dev dummy5 clsact" + docker exec "$CLUSTER_NAME"-worker2 bash -c "tc filter add dev dummy5 ingress bpf direct-action obj dummy_bpf.o sec classifier" + + run docker exec "$CLUSTER_NAME"-worker2 bash -c "tc filter show dev dummy5 ingress" + [ "$status" -eq 0 ] + [[ "$output" == *"dummy_bpf.o:[classifier] direct-action"* ]] + + # Wait for the interface to be discovered + sleep 5 + + # Validate bpf attribute is true + run kubectl get resourceslices --field-selector spec.nodeName="$CLUSTER_NAME"-worker2 -o jsonpath='{.items[0].spec.devices[?(@.name=="dummy5")].attributes.dra\.net\/ebpf.bool}' + [ "$status" -eq 0 ] + [[ "$output" == "true" ]] + + # Validate bpfName attribute + run kubectl get resourceslices --field-selector spec.nodeName="$CLUSTER_NAME"-worker2 -o jsonpath='{.items[0].spec.devices[?(@.name=="dummy5")].attributes.dra\.net\/tcFilterNames.string}' + [ "$status" -eq 0 ] + [[ "$output" == "dummy_bpf.o:[classifier]" ]] +}