diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index eadcb99583464f..a73f500c9be82f 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -349,6 +350,9 @@ struct ice_vsi { u16 num_xdp_txq; /* Used XDP queues */ u8 xdp_mapping_mode; /* ICE_MAP_MODE_[CONTIG|SCATTER] */ + bool xdp_metadata_support; /* true if VSI should support xdp meta */ + s32 btf_id; + /* setup back reference, to which aggregator node this VSI * corresponds to */ diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 0d6c143f665327..e81f2944b1ab80 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -2419,6 +2419,21 @@ static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); } +static void ice_xdp_rings_set_metadata(struct ice_vsi *vsi) +{ + int i; + + ice_for_each_rxq(vsi, i) { + vsi->rx_rings[i]->xdp_metadata_support = vsi->xdp_metadata_support; + vsi->rx_rings[i]->btf_id = vsi->btf_id; + } + + for (i = 0; i < vsi->num_xdp_txq; i++) { + vsi->xdp_rings[i]->xdp_metadata_support = vsi->xdp_metadata_support; + vsi->xdp_rings[i]->btf_id = vsi->btf_id; + } +} + /** * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP * @vsi: VSI to bring up Tx rings used by XDP @@ -2458,6 +2473,8 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) if (ice_xdp_alloc_setup_rings(vsi)) goto clear_xdp_rings; + ice_xdp_rings_set_metadata(vsi); + /* follow the logic from ice_vsi_map_rings_to_vectors */ ice_for_each_q_vector(vsi, v_idx) { struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; @@ -2613,7 +2630,7 @@ static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi) */ static int ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, - struct netlink_ext_ack *extack) + struct netlink_ext_ack *extack, u32 flags) { int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; bool if_running = netif_running(vsi->netdev); @@ -2633,6 +2650,11 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, } } + if (flags & XDP_FLAGS_USE_METADATA) { + vsi->xdp_metadata_support = true; + vsi->btf_id = btf_get_type_id(THIS_MODULE, "xdp_meta_generic", BTF_KIND_STRUCT); + } + if (!ice_is_xdp_ena_vsi(vsi) && prog) { vsi->num_xdp_txq = vsi->alloc_rxq; xdp_ring_err = ice_prepare_xdp_rings(vsi, prog); @@ -2686,7 +2708,7 @@ static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: - return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); + return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack, xdp->flags); case XDP_SETUP_XSK_POOL: return ice_xsk_pool_setup(vsi, xdp->xsk.pool, xdp->xsk.queue_id); diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index 6ee8e0032d52cb..f52ab7b2e3258f 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -1135,6 +1135,10 @@ int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) hard_start = page_address(rx_buf->page) + rx_buf->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); + + if (likely(rx_ring->xdp_metadata_support)) + ice_xdp_set_meta(&xdp, rx_desc, rx_ring->btf_id); + #if (PAGE_SIZE > 4096) /* At larger PAGE_SIZE, frame_sz depend on len size */ xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size); diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index 1e46e80f3d6f89..9419d4780acc1b 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -276,6 +276,7 @@ struct ice_ring { u16 q_handle; /* Queue handle per TC */ u8 ring_active:1; /* is ring online or not */ + u8 xdp_metadata_support:1; /* is xdp metadata support */ u16 count; /* Number of descriptors */ u16 reg_idx; /* HW register index of the ring */ @@ -301,6 +302,9 @@ struct ice_ring { /* CL3 - 3rd cacheline starts here */ struct xdp_rxq_info xdp_rxq; struct sk_buff *skb; + s32 btf_id; + + /* CLX - the below items are only accessed infrequently and should be * in their own cache line if possible */ diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h index 05ac3075290260..54c8acab1691b0 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h @@ -46,6 +46,26 @@ static inline void ice_xdp_ring_update_tail(struct ice_ring *xdp_ring) writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail); } +static inline void ice_xdp_set_meta(struct xdp_buff *xdp, union ice_32b_rx_flex_desc *desc, s32 btf_id) +{ + struct ice_32b_rx_flex_desc_nic *flex = (struct ice_32b_rx_flex_desc_nic *)desc; + struct xdp_meta_generic *md = xdp->data - sizeof(struct xdp_meta_generic); + + md->rxcvid = le16_to_cpu(flex->flex_ts.flex.vlan_id); + md->hash = le32_to_cpu(flex->rss_hash); + + md->flags = 0; + md->free_slot = 0; + md->csum_off = 0; + md->txcvid = 0; + md->csum = 0; + md->tstamp = 0; + + md->btf_id = btf_id; + + xdp->data_meta = md; +} + void ice_finalize_xdp_rx(struct ice_ring *rx_ring, unsigned int xdp_res); int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_ring *xdp_ring); int ice_xmit_xdp_ring(void *data, u16 size, struct ice_ring *xdp_ring); diff --git a/include/linux/btf.h b/include/linux/btf.h index 214fde93214b97..cd26d037a2dda5 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -94,6 +94,7 @@ u32 btf_obj_id(const struct btf *btf); bool btf_is_kernel(const struct btf *btf); bool btf_is_module(const struct btf *btf); struct module *btf_try_get_module(const struct btf *btf); +s32 btf_get_type_id(const struct module *mod, char *name, u32 kind); u32 btf_nr_types(const struct btf *btf); bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, const struct btf_member *m, diff --git a/include/net/xdp.h b/include/net/xdp.h index ad5b02dcb6f4c4..9c4d3c9ddc7de9 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -76,6 +76,24 @@ struct xdp_buff { u32 frame_sz; /* frame size to deduce data_hard_end/reserved tailroom*/ }; +struct xdp_meta_generic { + // Tx part + u32 flags; + u16 free_slot; + u16 csum_off; + u16 txcvid; + + // Rx part + u16 rxcvid; + u32 csum; + u32 hash; + u64 tstamp; + + // BTF ID + u32 btf_id; +} __packed __aligned(8); +static_assert(sizeof(struct xdp_meta_generic) == 32); + static __always_inline void xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) { diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 791f31dd0abee7..bee631b560c0cb 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -873,6 +873,7 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_BTF_VMLINUX_INFO, }; enum bpf_map_type { @@ -1396,6 +1397,11 @@ union bpf_attr { __aligned_u64 info; } info; + struct { /* anonymous struct used by BPF_BTF_VMLINUX_INFO */ + __u32 info_len; + __aligned_u64 info; + } info_vmlinux; + struct { /* anonymous struct used by BPF_PROG_QUERY command */ __u32 target_fd; /* container object to query */ __u32 attach_type; diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index eebd3894fe89a0..4326237c7ba1e8 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -1182,11 +1182,13 @@ enum { #define XDP_FLAGS_DRV_MODE (1U << 2) #define XDP_FLAGS_HW_MODE (1U << 3) #define XDP_FLAGS_REPLACE (1U << 4) +#define XDP_FLAGS_USE_METADATA (1U << 5) #define XDP_FLAGS_MODES (XDP_FLAGS_SKB_MODE | \ XDP_FLAGS_DRV_MODE | \ XDP_FLAGS_HW_MODE) #define XDP_FLAGS_MASK (XDP_FLAGS_UPDATE_IF_NOEXIST | \ - XDP_FLAGS_MODES | XDP_FLAGS_REPLACE) + XDP_FLAGS_MODES | XDP_FLAGS_REPLACE | \ + XDP_FLAGS_USE_METADATA) /* These are stored into IFLA_XDP_ATTACHED on dump. */ enum { diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index dfe61df4f974d1..10fd0405fec9da 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6152,6 +6152,43 @@ struct module *btf_try_get_module(const struct btf *btf) return res; } +struct btf *btf_get_from_module(const struct module *module) +{ + struct btf *res = NULL; +#ifdef CONFIG_DEBUG_INFO_BTF_MODULES + struct btf_module *btf_mod, *tmp; + + mutex_lock(&btf_module_mutex); + list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { + if (btf_mod->module != module) + continue; + + res = btf_mod->btf; + + break; + } + mutex_unlock(&btf_module_mutex); +#endif + + return res; +} + +s32 btf_get_type_id(const struct module *mod, char *name, u32 kind) +{ + struct btf *btf; + + if (mod) + btf = btf_get_from_module(mod); + else + btf = bpf_get_btf_vmlinux(); + + if (!btf) + return 0; + + return btf_find_by_name_kind(btf, name, kind); +} +EXPORT_SYMBOL_GPL(btf_get_type_id); + BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags) { struct btf *btf; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 585b2b77ccc4f1..7fe44cd178d498 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -215,6 +215,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, { struct xdp_rxq_info rxq; struct xdp_buff xdp; + volatile struct xdp_meta_generic xdp_meta_generic; int i, nframes = 0; xdp_set_return_frame_no_direct(); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 4e50c0bfdb7d38..a527fdd63be344 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4542,6 +4542,25 @@ static int bpf_prog_bind_map(union bpf_attr *attr) return ret; } +#define BPF_BTF_VMLINUX_INFO_LAST_FIELD info.info + +static int bpf_btf_get_vmlinux_info(const union bpf_attr *attr, + union bpf_attr __user *uattr) +{ + struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); + u32 info_len = attr->info.info_len; + int err; + + if (CHECK_ATTR(BPF_BTF_VMLINUX_INFO)) + return -EINVAL; + + err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len); + if (err) + return err; + + return btf_get_info_by_fd(bpf_get_btf_vmlinux(), attr, uattr); +} + static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) { union bpf_attr attr; @@ -4678,6 +4697,9 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) case BPF_PROG_BIND_MAP: err = bpf_prog_bind_map(&attr); break; + case BPF_BTF_VMLINUX_INFO: + err = bpf_btf_get_vmlinux_info(&attr, uattr.user); + break; default: err = -EINVAL; break; diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 4dc20be5fb96a6..9b5fc9f5e6c8d3 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -57,6 +57,7 @@ tprogs-y += xdp_redirect_map_multi tprogs-y += xdp_redirect_map tprogs-y += xdp_redirect tprogs-y += xdp_monitor +tprogs-y += xdp_meta # Libbpf dependencies LIBBPF = $(TOOLS_PATH)/lib/bpf/libbpf.a @@ -118,6 +119,7 @@ xdp_redirect_cpu-objs := xdp_redirect_cpu_user.o $(XDP_SAMPLE) xdp_redirect_map-objs := xdp_redirect_map_user.o $(XDP_SAMPLE) xdp_redirect-objs := xdp_redirect_user.o $(XDP_SAMPLE) xdp_monitor-objs := xdp_monitor_user.o $(XDP_SAMPLE) +xdp_meta-objs := xdp_meta_user.o # Tell kbuild to always build the programs always-y := $(tprogs-y) @@ -314,6 +316,7 @@ $(obj)/xdp_redirect_map_multi_user.o: $(obj)/xdp_redirect_map_multi.skel.h $(obj)/xdp_redirect_map_user.o: $(obj)/xdp_redirect_map.skel.h $(obj)/xdp_redirect_user.o: $(obj)/xdp_redirect.skel.h $(obj)/xdp_monitor_user.o: $(obj)/xdp_monitor.skel.h +$(obj)/xdp_meta_user.o: $(obj)/xdp_meta.skel.h $(obj)/tracex5_kern.o: $(obj)/syscall_nrs.h $(obj)/hbm_out_kern.o: $(src)/hbm.h $(src)/hbm_kern.h @@ -371,7 +374,8 @@ $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/x -c $(filter %.bpf.c,$^) -o $@ LINKED_SKELS := xdp_redirect_cpu.skel.h xdp_redirect_map_multi.skel.h \ - xdp_redirect_map.skel.h xdp_redirect.skel.h xdp_monitor.skel.h + xdp_redirect_map.skel.h xdp_redirect.skel.h xdp_monitor.skel.h \ + xdp_meta.skel.h clean-files += $(LINKED_SKELS) xdp_redirect_cpu.skel.h-deps := xdp_redirect_cpu.bpf.o xdp_sample.bpf.o @@ -379,9 +383,9 @@ xdp_redirect_map_multi.skel.h-deps := xdp_redirect_map_multi.bpf.o xdp_sample.bp xdp_redirect_map.skel.h-deps := xdp_redirect_map.bpf.o xdp_sample.bpf.o xdp_redirect.skel.h-deps := xdp_redirect.bpf.o xdp_sample.bpf.o xdp_monitor.skel.h-deps := xdp_monitor.bpf.o xdp_sample.bpf.o +xdp_meta.skel.h-deps := xdp_meta.bpf.o LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.bpf.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps))) - BPF_SRCS_LINKED := $(notdir $(wildcard $(src)/*.bpf.c)) BPF_OBJS_LINKED := $(patsubst %.bpf.c,$(obj)/%.bpf.o, $(BPF_SRCS_LINKED)) BPF_SKELS_LINKED := $(addprefix $(obj)/,$(LINKED_SKELS)) diff --git a/samples/bpf/xdp_meta.bpf.c b/samples/bpf/xdp_meta.bpf.c new file mode 100644 index 00000000000000..006ebd40119734 --- /dev/null +++ b/samples/bpf/xdp_meta.bpf.c @@ -0,0 +1,68 @@ +/* This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + */ +#include "vmlinux.h" + +#include +#include +#include + +struct ice_ring___min { + struct ice_ring *next; + void *desc; + struct device *dev; + struct net_device *netdev; + struct ice_vsi *vsi; + struct ice_q_vector *q_vector; + u8 *tail; +} __attribute__((preserve_access_index)); + +SEC("xdp") +int xdp_meta_prog(struct xdp_md *ctx) +{ + struct xdp_meta_generic *data_meta = + (void *)(long)ctx->data_meta; + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + struct ethhdr *eth = data; + u64 nh_off; + u64 btf_id_libbpf; + u32 btf_id_meta; + u64 btf_id_ring; + u16 rxcvid; + u32 hash; + long *value; + + nh_off = sizeof(*eth); + if (data + nh_off > data_end) + return XDP_DROP; + + + if (data_meta + 1 > data) + return XDP_DROP; + + btf_id_libbpf = bpf_core_type_id_kernel(struct xdp_meta_generic); + bpf_probe_read_kernel(&btf_id_meta, sizeof(btf_id_meta), (void*)data - 4); + + bpf_printk("id from libbpf %u (module BTF id: %u), id from hints metadata %u\n", + btf_id_libbpf & 0xFFFFFFFF, btf_id_libbpf >> 32, btf_id_meta); + + btf_id_ring = bpf_core_type_id_kernel(struct ice_ring___min); + bpf_printk("ring type id %u, ice BTF id %u\n", + btf_id_ring & 0xFFFFFFFF, btf_id_ring >> 32); + + if (btf_id_libbpf == btf_id_meta) + bpf_printk("Received meta is generic\n"); + else + bpf_printk("Received meta type is unknown\n"); + + + hash = BPF_CORE_READ(data_meta, hash); + rxcvid = BPF_CORE_READ(data_meta, rxcvid); + bpf_printk("Metadata. Hash: 0x%x, VID: %d\n", hash, rxcvid); + + return XDP_PASS; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/samples/bpf/xdp_meta_user.c b/samples/bpf/xdp_meta_user.c new file mode 100644 index 00000000000000..78fc31304bbc44 --- /dev/null +++ b/samples/bpf/xdp_meta_user.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xdp_meta.skel.h" + +#define DEBUGFS "/sys/kernel/debug/tracing/" + +static volatile bool xdp_meta_sample_running = true; + +static void xdp_meta_sample_stop(int signo) +{ + xdp_meta_sample_running = false; +} + +/* Had to change the standard read_trace_pipe from trace_helpers.h */ +static void xdp_meta_read_trace_pipe(void) +{ + int trace_fd; + + trace_fd = open(DEBUGFS "trace_pipe", O_RDONLY, 0); + if (trace_fd < 0){ + fprintf(stderr, "Could not open the trace_pipe\n"); + return; + } + + while (xdp_meta_sample_running) { + static char buf[4096]; + ssize_t sz; + + sz = read(trace_fd, buf, sizeof(buf) - 1); + if (sz > 0) { + buf[sz] = 0; + puts(buf); + } + } +} + +int main(int argc, char **argv) +{ + __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_USE_METADATA; + __u32 prog_id, prog_fd, running_prog_id; + struct xdp_meta *skel; + int ifindex, ret = 1; + struct sigaction handle_ctrl_c; + + if (argc == optind) { + return ret; + } + + ifindex = if_nametoindex(argv[optind]); + if (!ifindex) + ifindex = strtoul(argv[optind], NULL, 0); + if (!ifindex) { + fprintf(stderr, "Bad interface index or name\n"); + goto end; + } + + skel = xdp_meta__open(); + if (!skel) { + fprintf(stderr, "Failed to xdp_meta__open: %s\n", + strerror(errno)); + ret = 1; + goto end; + } + + ret = xdp_meta__load(skel); + if (ret < 0) { + fprintf(stderr, "Failed to xdp_meta__load: %s\n", strerror(errno)); + ret = 1; + goto end_destroy; + } + + ret = 1; + prog_fd = bpf_program__fd(skel->progs.xdp_meta_prog); + if (bpf_set_link_xdp_fd(ifindex, prog_fd, xdp_flags) < 0) { + fprintf(stderr, "Failed to set xdp link\n"); + goto end_destroy; + } + + if (bpf_get_link_xdp_id(ifindex, &prog_id, xdp_flags)) { + fprintf(stderr, "Failed to get XDP program id for ifindex\n"); + goto end_destroy; + } + + memset(&handle_ctrl_c, 0, sizeof(handle_ctrl_c)); + handle_ctrl_c.sa_handler = &xdp_meta_sample_stop; + sigaction(SIGINT, &handle_ctrl_c, NULL); + + xdp_meta_read_trace_pipe(); + + ret = 0; + + if(bpf_get_link_xdp_id(ifindex, &running_prog_id, xdp_flags) || running_prog_id != prog_id){ + fprintf(stderr, + "Failed to get the running XDP program id or another program is running. Exit without detaching.\n"); + goto end_destroy; + } + + fprintf(stderr, "Detaching the program...\n"); + bpf_set_link_xdp_fd(ifindex, -1, 0); + +end_destroy: + xdp_meta__destroy(skel); +end: + return ret; +} diff --git a/samples/bpf/xdp_redirect_map_multi.bpf.c b/samples/bpf/xdp_redirect_map_multi.bpf.c index 8f59d430cb64c8..bb0a5a3bfcf019 100644 --- a/samples/bpf/xdp_redirect_map_multi.bpf.c +++ b/samples/bpf/xdp_redirect_map_multi.bpf.c @@ -5,11 +5,6 @@ #include "xdp_sample.bpf.h" #include "xdp_sample_shared.h" -enum { - BPF_F_BROADCAST = (1ULL << 3), - BPF_F_EXCLUDE_INGRESS = (1ULL << 4), -}; - struct { __uint(type, BPF_MAP_TYPE_DEVMAP_HASH); __uint(key_size, sizeof(int)); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 791f31dd0abee7..ff2b8d059185de 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -873,6 +873,7 @@ enum bpf_cmd { BPF_ITER_CREATE, BPF_LINK_DETACH, BPF_PROG_BIND_MAP, + BPF_BTF_VMLINUX_INFO, }; enum bpf_map_type { diff --git a/tools/include/uapi/linux/if_link.h b/tools/include/uapi/linux/if_link.h index b3610fdd1feec3..595e238d489b71 100644 --- a/tools/include/uapi/linux/if_link.h +++ b/tools/include/uapi/linux/if_link.h @@ -969,11 +969,13 @@ enum { #define XDP_FLAGS_DRV_MODE (1U << 2) #define XDP_FLAGS_HW_MODE (1U << 3) #define XDP_FLAGS_REPLACE (1U << 4) +#define XDP_FLAGS_USE_METADATA (1U << 5) #define XDP_FLAGS_MODES (XDP_FLAGS_SKB_MODE | \ XDP_FLAGS_DRV_MODE | \ XDP_FLAGS_HW_MODE) #define XDP_FLAGS_MASK (XDP_FLAGS_UPDATE_IF_NOEXIST | \ - XDP_FLAGS_MODES | XDP_FLAGS_REPLACE) + XDP_FLAGS_MODES | XDP_FLAGS_REPLACE | \ + XDP_FLAGS_USE_METADATA) /* These are stored into IFLA_XDP_ATTACHED on dump. */ enum { diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 2401fad090c526..78ab4868edecea 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -1017,6 +1017,23 @@ int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_s return libbpf_err_errno(fd); } +int bpf_get_vmlinux_btf_info(void *info, __u32 *info_len) +{ + union bpf_attr attr; + int err; + + memset(&attr, 0, sizeof(attr)); + attr.info.info_len = *info_len; + attr.info.info = ptr_to_u64(info); + + err = sys_bpf(BPF_BTF_VMLINUX_INFO, &attr, sizeof(attr)); + + if (!err) + *info_len = attr.info.info_len; + + return libbpf_err_errno(err); +} + int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset, __u64 *probe_addr) diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 77dc24d58302dc..9549a9c7876382 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -430,6 +430,19 @@ const struct btf *btf__base_btf(const struct btf *btf) return btf->base_btf; } +__u32 btf_get_vmlinux_obj_id(void) +{ + struct bpf_btf_info btf_info; + unsigned int len = sizeof(btf_info); + int err = 0; + + memset(&btf_info, 0, sizeof(btf_info)); + err = bpf_get_vmlinux_btf_info(&btf_info, &len); + + if (err) return 0; + return btf_info.id; +} + /* internal helper returning non-const pointer to a type */ struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id) { @@ -1376,6 +1389,7 @@ struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) } btf = btf_new(ptr, btf_info.btf_size, base_btf); + btf->fd = btf_fd; exit_free: free(ptr); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 88d8825fc6f61f..e0c1128eb0ca51 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -4803,6 +4803,7 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand, size_t local_essent_len, const struct btf *targ_btf, const char *targ_btf_name, + __u32 targ_btf_id, int targ_start_id, struct bpf_core_cand_list *cands) { @@ -4843,6 +4844,7 @@ static int bpf_core_add_cands(struct bpf_core_cand *local_cand, cand->t = t; cand->name = targ_name; cand->id = i; + cand->btf_module_id = targ_btf_id; cands->cands = new_cands; cands->len++; @@ -4948,6 +4950,7 @@ bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 l struct bpf_core_cand local_cand = {}; struct bpf_core_cand_list *cands; const struct btf *main_btf; + __u32 main_btf_id; size_t local_essent_len; int err, i; @@ -4967,7 +4970,8 @@ bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 l /* Attempt to find target candidates in vmlinux BTF first */ main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; - err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); + main_btf_id = obj->btf_vmlinux_override ? 0 : btf_get_vmlinux_obj_id(); + err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", main_btf_id, 1, cands); if (err) goto err_out; @@ -4988,6 +4992,7 @@ bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 l err = bpf_core_add_cands(&local_cand, local_essent_len, obj->btf_modules[i].btf, obj->btf_modules[i].name, + obj->btf_modules[i].id, btf__get_nr_types(obj->btf_vmlinux) + 1, cands); if (err) diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 533b0211f40ac6..e44eeee6b12a48 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -287,6 +287,8 @@ int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, const char **prefix, int *kind); +__u32 btf_get_vmlinux_obj_id(void); +int bpf_get_vmlinux_btf_info(void *info, __u32 *info_len); struct btf_ext_info { /* diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 4016ed492d0c20..db7b72099369a6 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -763,6 +763,7 @@ struct bpf_core_relo_res __u32 orig_type_id; __u32 new_sz; __u32 new_type_id; + __u32 btf_obj_id; }; /* Calculate original and target relocation values, given local and target @@ -787,6 +788,7 @@ static int bpf_core_calc_relo(const char *prog_name, res->fail_memsz_adjust = false; res->orig_sz = res->new_sz = 0; res->orig_type_id = res->new_type_id = 0; + res->btf_obj_id = 0; if (core_relo_is_field_based(relo->kind)) { err = bpf_core_calc_field_relo(prog_name, relo, local_spec, @@ -1025,7 +1027,8 @@ static int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, } insn[0].imm = new_val; - insn[1].imm = 0; /* currently only 32-bit values are supported */ + insn[1].imm = relo->kind == BPF_TYPE_ID_TARGET ? + res->btf_obj_id : 0; pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n", prog_name, relo_idx, insn_idx, (unsigned long long)imm, new_val); @@ -1224,6 +1227,8 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, if (err) return err; + cand_res.btf_obj_id = cands->cands[i].btf_module_id; + if (j == 0) { targ_res = cand_res; targ_spec = cand_spec; diff --git a/tools/lib/bpf/relo_core.h b/tools/lib/bpf/relo_core.h index 3b9f8f18346c29..ee9441f19f4560 100644 --- a/tools/lib/bpf/relo_core.h +++ b/tools/lib/bpf/relo_core.h @@ -80,6 +80,7 @@ struct bpf_core_cand { const struct btf_type *t; const char *name; __u32 id; + __u32 btf_module_id; }; /* dynamically sized list of type IDs and its associated struct btf */