From 1eed7bf01d58e16a1d9bf4376a16e18cd25c7a44 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Wed, 11 Jun 2025 13:34:50 +0000 Subject: [PATCH] expose ebpf interface capabilities It will be useful for some platforms to filter based on the existince of ebpf programs on the interfaces. If the platforms also want to have more granularity on the filters, both the tc filters names and tcx program names are exposed, this can be useful for cilium environments, that attach the bpf programs with a well known prefix https: //github.com/cilium/cilium/blob/2732b5253beef350dd89216813d4e4da1180e8c7/pkg/datapath/loader/tcx.go#L139 Change-Id: I72d10dd04c364ca3d594c5fa2f1168fd2d179ba7 --- .github/workflows/bats.yml | 2 +- go.mod | 1 + go.sum | 7 +++ pkg/inventory/db.go | 16 ++++++ pkg/inventory/{routes.go => net.go} | 50 ++++++++++++++++++ .../docs/contributing/developer-guide.md | 31 +++++++++++ tests/dummy_bpf.c | 9 ++++ tests/dummy_bpf.o | Bin 0 -> 720 bytes tests/e2e.bats | 28 ++++++++++ 9 files changed, 143 insertions(+), 1 deletion(-) rename pkg/inventory/{routes.go => net.go} (59%) create mode 100644 tests/dummy_bpf.c create mode 100644 tests/dummy_bpf.o diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml index ce82a454..55992ebb 100644 --- a/.github/workflows/bats.yml +++ b/.github/workflows/bats.yml @@ -32,7 +32,7 @@ jobs: env: BATS_LIB_PATH: ${{ steps.setup-bats.outputs.lib-path }} TERM: xterm - run: bats -o _artifacts tests/ + run: bats -o _artifacts --print-output-on-failure tests/ - name: Upload logs if: always() diff --git a/go.mod b/go.mod index 3507b75a..9fb45b0a 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( cloud.google.com/go/compute/metadata v0.7.0 cloud.google.com/go/container v1.43.0 github.com/Mellanox/rdmamap v1.1.0 + github.com/cilium/ebpf v0.18.0 github.com/containerd/nri v0.9.0 github.com/google/cel-go v0.25.0 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index 3723aae8..ae1addd8 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.18.0 h1:OsSwqS4y+gQHxaKgg2U/+Fev834kdnsQbtzRnbVC6Gs= +github.com/cilium/ebpf v0.18.0/go.mod h1:vmsAT73y4lW2b4peE+qcOqw6MxvWQdC+LiU5gd/xyo4= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.9.0 h1:jribDJs/oQ95vLO4Yn19HKFYriZGWKiG6nKWjl9Y/x4= @@ -48,6 +50,8 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -83,6 +87,9 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink v1.3.5 h1:hVlNQNRlLDGZz31gBPicsG7Q53rnlsz1l1Ix/9XlpVA= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/pkg/inventory/db.go b/pkg/inventory/db.go index ea63ca2d..064246b6 100644 --- a/pkg/inventory/db.go +++ b/pkg/inventory/db.go @@ -251,6 +251,22 @@ func (db *DB) netdevToDRAdev(link netlink.Link) (*resourceapi.Device, error) { device.Basic.Attributes["dra.net/alias"] = resourceapi.DeviceAttribute{StringValue: &linkAttrs.Alias} device.Basic.Attributes["dra.net/type"] = resourceapi.DeviceAttribute{StringValue: &linkType} + // Get eBPF properties from the interface using the legacy tc hooks + isEbpf := false + filterNames, ok := getTcFilters(link) + if ok { + isEbpf = true + device.Basic.Attributes["dra.net/tcFilterNames"] = resourceapi.DeviceAttribute{StringValue: ptr.To(strings.Join(filterNames, ","))} + } + + // Get eBPF properties from the interface using the tcx hooks + programNames, ok := getTcxFilters(link) + if ok { + isEbpf = true + device.Basic.Attributes["dra.net/tcxProgramNames"] = resourceapi.DeviceAttribute{StringValue: ptr.To(strings.Join(programNames, ","))} + } + device.Basic.Attributes["dra.net/ebpf"] = resourceapi.DeviceAttribute{BoolValue: &isEbpf} + isRDMA := rdmamap.IsRDmaDeviceForNetdevice(ifName) device.Basic.Attributes["dra.net/rdma"] = resourceapi.DeviceAttribute{BoolValue: &isRDMA} // from https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin/blob/ed1c14dd4c313c7dd9fe4730a60358fbeffbfdd4/pkg/netdevice/netDeviceProvider.go#L99 diff --git a/pkg/inventory/routes.go b/pkg/inventory/net.go similarity index 59% rename from pkg/inventory/routes.go rename to pkg/inventory/net.go index b622ac4c..f2f14d9a 100644 --- a/pkg/inventory/routes.go +++ b/pkg/inventory/net.go @@ -17,6 +17,8 @@ limitations under the License. package inventory import ( + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/link" "github.com/vishvananda/netlink" "k8s.io/apimachinery/pkg/util/sets" @@ -68,3 +70,51 @@ func getDefaultGwInterfaces() sets.Set[string] { klog.V(4).Infof("Found following interfaces for the default gateway: %v", interfaces.UnsortedList()) return interfaces } + +func getTcFilters(link netlink.Link) ([]string, bool) { + isTcEBPF := false + filterNames := sets.Set[string]{} + for _, parent := range []uint32{netlink.HANDLE_MIN_INGRESS, netlink.HANDLE_MIN_EGRESS} { + filters, err := netlink.FilterList(link, parent) + if err == nil { + for _, f := range filters { + if bpffFilter, ok := f.(*netlink.BpfFilter); ok { + isTcEBPF = true + filterNames.Insert(bpffFilter.Name) + } + } + } + } + return filterNames.UnsortedList(), isTcEBPF +} + +// see https://github.com/cilium/ebpf/issues/1117 +func getTcxFilters(device netlink.Link) ([]string, bool) { + isTcxEBPF := false + programNames := sets.Set[string]{} + for _, attach := range []ebpf.AttachType{ebpf.AttachTCXIngress, ebpf.AttachTCXEgress} { + result, err := link.QueryPrograms(link.QueryOptions{ + Target: int(device.Attrs().Index), + Attach: attach, + }) + if err != nil { + continue + } + + isTcxEBPF = true + for _, p := range result.Programs { + prog, err := ebpf.NewProgramFromID(p.ID) + if err != nil { + continue + } + defer prog.Close() + + pi, err := prog.Info() + if err != nil { + continue + } + programNames.Insert(pi.Name) + } + } + return programNames.UnsortedList(), isTcxEBPF +} diff --git a/site/content/docs/contributing/developer-guide.md b/site/content/docs/contributing/developer-guide.md index 31c27db9..f10e86f6 100644 --- a/site/content/docs/contributing/developer-guide.md +++ b/site/content/docs/contributing/developer-guide.md @@ -55,6 +55,37 @@ Waiting for daemon set "dranet" rollout to finish: 1 out of 5 new pods have been updated... ``` +## Accesing nodes + +When developing it may be also useful to log into the nodes to be able to debug problems. Just obtain the list of nodes with `kubectl get nodes -o wide` + +For Kind clusters use `docker exec -it bash` + +```sh +kubectl get nodes -o wide +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +gke-cluster-tpu-v6-default-pool-3f96de9d-hr87 Ready 8d v1.33.1-gke.1107000 10.202.0.21 34.162.194.173 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-cluster-tpu-v6-default-pool-955df71d-zd7b Ready 8d v1.33.1-gke.1107000 10.202.0.15 34.162.239.241 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-cluster-tpu-v6-default-pool-e1c69e67-k0jw Ready 8d v1.33.1-gke.1107000 10.202.0.20 34.162.209.25 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-kgdj Ready 8d v1.33.1-gke.1107000 10.202.0.26 34.162.163.69 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-prgf Ready 8d v1.33.1-gke.1107000 10.202.0.24 34.162.144.5 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-sjcp Ready 8d v1.33.1-gke.1107000 10.202.0.27 34.162.117.62 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +gke-tpu-de8b9feb-z8g1 Ready 8d v1.33.1-gke.1107000 10.202.0.25 34.162.239.83 Container-Optimized OS from Google 6.6.87+ containerd://2.0.4 +``` + +For GKE or other clusters, you may have restrictions on ssh, so you can use + +```sh + kubectl debug -it node/gke-tpu-de8b9feb-kgdj --image busybox -- chroot /host +--profile=legacy is deprecated and will be removed in the future. It is recommended to explicitly specify a profile, for example "--profile=general". +Creating debugging pod node-debugger-gke-tpu-de8b9feb-kgdj-94xdx with container debugger on node gke-tpu-de8b9feb-kgdj. +If you don't see a command prompt, try pressing enter. +gke-tpu-de8b9feb-kgdj / # +``` + +If you want to upload some binary, per example `bpftrace` or `pwru` , you can use the streaming capabilities for that: + +``` ## Troubleshooting diff --git a/tests/dummy_bpf.c b/tests/dummy_bpf.c new file mode 100644 index 00000000..f1097aab --- /dev/null +++ b/tests/dummy_bpf.c @@ -0,0 +1,9 @@ +#include +#include + +__attribute__((section("classifier"), used)) +int handle_ingress(struct __sk_buff *skb) { + return TC_ACT_OK; +} + +char __license[] __attribute__((section("license"), used)) = "GPL"; \ No newline at end of file diff --git a/tests/dummy_bpf.o b/tests/dummy_bpf.o new file mode 100644 index 0000000000000000000000000000000000000000..1f725b85c8ebc26739a49b70693864eaac094a6d GIT binary patch literal 720 zcmbtQJ8r^25FOhnP!bA6n+q6G(z$VHj6@TOZe#2PR$3e6HBnr_C32QTxmBu^WNgh~ zw2Ek$=6&YPe&!fP&#vPDmjl1hDs}<-S7y-I0dz6>Ns$l2*&-tLKTOf?z|Yz4cF`Ds zcd`4##)ywN!Ntt=z%S){3EMbNwG=Af6w(+-bZm@TD_MZA^;;(5G%bwUfDl?GGB*;^ zLzY!ywO{)Q_@*q%c!gMHHQrqzp3wfHJ?@+6OKUakYONPtdk