diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index b0b27bfcd7a289..85a283afdf9050 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -424,6 +424,9 @@ struct ice_vsi { struct ice_channel *ch; + u8 xdp_metadata_support:1; /* true if VSI should support xdp meta */ + struct xdp_meta_tail xdp_meta_tail; + /* setup back reference, to which aggregator node this VSI * corresponds to */ diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index b588d79956310d..f657975f5608f3 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -2607,6 +2607,21 @@ static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); } +static void ice_xdp_rings_set_metadata(const struct ice_vsi *vsi) +{ + int i; + + ice_for_each_rxq(vsi, i) { + vsi->rx_rings[i]->xdp_metadata_support = vsi->xdp_metadata_support; + vsi->rx_rings[i]->xdp_meta_tail = vsi->xdp_meta_tail; + } + + for (i = 0; i < vsi->num_xdp_txq; i++) { + vsi->xdp_rings[i]->xdp_metadata_support = vsi->xdp_metadata_support; + vsi->xdp_rings[i]->xdp_meta_tail = vsi->xdp_meta_tail; + } +} + /** * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP * @vsi: VSI to bring up Tx rings used by XDP @@ -2650,6 +2665,8 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) if (ice_xdp_alloc_setup_rings(vsi)) goto clear_xdp_rings; + ice_xdp_rings_set_metadata(vsi); + /* follow the logic from ice_vsi_map_rings_to_vectors */ ice_for_each_q_vector(vsi, v_idx) { struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; @@ -2842,7 +2859,7 @@ int ice_vsi_determine_xdp_res(struct ice_vsi *vsi) */ static int ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, - struct netlink_ext_ack *extack) + struct netlink_ext_ack *extack, u32 flags) { int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; bool if_running = netif_running(vsi->netdev); @@ -2862,6 +2879,18 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, } } + if (flags & XDP_FLAGS_USE_METADATA) { + vsi->xdp_metadata_support = true; + ret = xdp_meta_fill_id_magic(&vsi->xdp_meta_tail, THIS_MODULE, + "xdp_meta_generic"); + if (ret) { + NL_SET_ERR_MSG_MOD(extack, "Could not fill in xdp meta tail"); + vsi->xdp_metadata_support = false; + } + } else { + vsi->xdp_metadata_support = false; + } + if (!ice_is_xdp_ena_vsi(vsi) && prog) { xdp_ring_err = ice_vsi_determine_xdp_res(vsi); if (xdp_ring_err) { @@ -2924,7 +2953,7 @@ static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: - return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); + return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack, xdp->flags); case XDP_SETUP_XSK_POOL: return ice_xsk_pool_setup(vsi, xdp->xsk.pool, xdp->xsk.queue_id); diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index f9bf008471c9ea..a7348c5330817e 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -1184,6 +1184,10 @@ int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget) hard_start = page_address(rx_buf->page) + rx_buf->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); + + if (likely(rx_ring->xdp_metadata_support)) + ice_xdp_set_meta(&xdp, rx_desc, rx_ring->xdp_meta_tail); + #if (PAGE_SIZE > 4096) /* At larger PAGE_SIZE, frame_sz depend on len size */ xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size); diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index cead3eb149bd5e..921dbe5948c3c4 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -294,6 +294,10 @@ struct ice_rx_ring { struct xsk_buff_pool *xsk_pool; struct sk_buff *skb; dma_addr_t dma; /* physical address of ring */ + + u8 xdp_metadata_support:1; /* is xdp metadata supported */ + struct xdp_meta_tail xdp_meta_tail; + #define ICE_RX_FLAGS_RING_BUILD_SKB BIT(1) u64 cached_phctime; u8 dcb_tc; /* Traffic class of ring */ @@ -322,6 +326,7 @@ struct ice_tx_ring { u16 reg_idx; /* HW register index of the ring */ u16 count; /* Number of descriptors */ u16 q_index; /* Queue number of ring */ + /* stats structs */ struct ice_txq_stats tx_stats; /* CL3 - 3rd cacheline starts here */ @@ -335,6 +340,10 @@ struct ice_tx_ring { u32 txq_teid; /* Added Tx queue TEID */ /* CL4 - 4th cacheline starts here */ u16 xdp_tx_active; + + u8 xdp_metadata_support:1; /* is xdp metadata supported */ + struct xdp_meta_tail xdp_meta_tail; + #define ICE_TX_FLAGS_RING_XDP BIT(0) #define ICE_TX_FLAGS_RING_VLAN_L2TAG1 BIT(1) #define ICE_TX_FLAGS_RING_VLAN_L2TAG2 BIT(2) diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h index c7d2954dc9ea78..a085551bd7bfff 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h @@ -70,6 +70,32 @@ static inline void ice_xdp_ring_update_tail(struct ice_tx_ring *xdp_ring) writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail); } +static inline void ice_xdp_set_meta(struct xdp_buff *xdp, const union ice_32b_rx_flex_desc *desc, + const struct xdp_meta_tail tail) +{ + const struct ice_32b_rx_flex_desc_nic *flex = (struct ice_32b_rx_flex_desc_nic *)desc; + struct xdp_meta_generic *md = xdp->data - sizeof(struct xdp_meta_generic); + + /* Fields are already in little endian*/ + md->rx_vid = flex->flex_ts.flex.vlan_id; + md->rx_hash = flex->rss_hash; + + md->rx_flags = 0; + md->rx_qid = 0; + md->rx_csum = 0; + md->rx_tstamp = 0; + + md->tx_flags = 0; + md->tx_csum_off = 0; + md->tx_vid = 0; + + md->btf_id = tail.btf_id; + md->type_id = tail.type_id; + md->magic = tail.magic; + + xdp->data_meta = md; +} + void ice_finalize_xdp_rx(struct ice_tx_ring *xdp_ring, unsigned int xdp_res); int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring); int ice_xmit_xdp_ring(void *data, u16 size, struct ice_tx_ring *xdp_ring); diff --git a/include/linux/btf.h b/include/linux/btf.h index 36bc09b8e890d8..3684e9fa3b58a7 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -117,6 +117,7 @@ u32 btf_obj_id(const struct btf *btf); bool btf_is_kernel(const struct btf *btf); bool btf_is_module(const struct btf *btf); struct module *btf_try_get_module(const struct btf *btf); +struct btf *btf_get_module_btf(const struct module *module); u32 btf_nr_types(const struct btf *btf); bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, const struct btf_member *m, diff --git a/include/net/xdp.h b/include/net/xdp.h index 04c852c7a77fa2..4d994102813678 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -245,6 +245,15 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp); struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); +struct xdp_meta_tail { + __le32 btf_id; + __le32 type_id; + __le32 magic; +}; + +int xdp_meta_fill_id_magic(struct xdp_meta_tail *tail, const struct module *mod, + const char *type_name); + static inline void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) { @@ -395,7 +404,7 @@ xdp_data_meta_unsupported(const struct xdp_buff *xdp) static inline bool xdp_metalen_invalid(unsigned long metalen) { - return (metalen & (sizeof(__u32) - 1)) || (metalen > 32); + return (metalen & (sizeof(__u32) - 1)) || (metalen > 256); } struct xdp_attachment_info { diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index d14b10b85e51f1..57ee0bf5ae3c0c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -6634,4 +6634,58 @@ struct bpf_core_relo { enum bpf_core_relo_kind kind; }; +enum { + XDP_META_RX_QID_BIT = (0x1 << 9), + XDP_META_RX_TSTAMP_BIT = (0x1 << 8), + XDP_META_RX_VLAN_TYPE = (0x3 << 6), + XDP_META_RX_VLAN_NONE = 0x0, + XDP_META_RX_CVID = 0x1, + XDP_META_RX_SVID = 0x2, + XDP_META_RX_HASH_TYPE = (0x3 << 4), + XDP_META_RX_HASH_NONE = 0x0, + XDP_META_RX_HASH_L2 = 0x1, + XDP_META_RX_HASH_L3 = 0x2, + XDP_META_RX_HASH_L4 = 0x3, + XDP_META_RX_CSUM_LEVEL = (0x3 << 2), + XDP_META_RX_CSUM_STATUS = (0x3 << 0), + XDP_META_RX_CSUM_NONE = 0x0, + XDP_META_RX_CSUM_OK = 0x1, + XDP_META_RX_CSUM_COMP = 0x2, + XDP_META_TX_VLAN_TYPE = (0x3 << 1), + XDP_META_TX_CVID = 0x1, + XDP_META_TX_SVID = 0x2, + XDP_META_TX_CSUM_BIT = (0x1 << 0), + XDP_META_GENERIC_MAGIC = 0xe4a6327d, + XDP_META_ALIGN = 8, +}; + +struct xdp_meta_generic { + __u8 padding[4]; + + /* Add new fields here */ + + /* Ingress */ + __le32 rx_flags; + + __le16 rx_qid; + __le16 rx_vid; + __le32 rx_csum; + __le32 rx_hash; + __le64 rx_tstamp; + + /* Egress */ + + __le32 tx_flags; + + __le16 tx_csum_off; + __le16 tx_vid; + + /* Unique identifiers */ + __le32 btf_id; + __le32 type_id; + __le32 magic; +} +__attribute__((__packed__)) /* Might be composed directly by HW */ +__attribute__((aligned(XDP_META_ALIGN))); /* Resides before Eth header, padded/aligned */ + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index cc284c048e6923..e5a4d030ce746f 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -1275,11 +1275,13 @@ enum { #define XDP_FLAGS_DRV_MODE (1U << 2) #define XDP_FLAGS_HW_MODE (1U << 3) #define XDP_FLAGS_REPLACE (1U << 4) +#define XDP_FLAGS_USE_METADATA (1U << 5) #define XDP_FLAGS_MODES (XDP_FLAGS_SKB_MODE | \ XDP_FLAGS_DRV_MODE | \ XDP_FLAGS_HW_MODE) #define XDP_FLAGS_MASK (XDP_FLAGS_UPDATE_IF_NOEXIST | \ - XDP_FLAGS_MODES | XDP_FLAGS_REPLACE) + XDP_FLAGS_MODES | XDP_FLAGS_REPLACE | \ + XDP_FLAGS_USE_METADATA) /* These are stored into IFLA_XDP_ATTACHED on dump. */ enum { diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 24788ce564a086..b7f9d689a4c7a0 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6576,7 +6576,7 @@ struct module *btf_try_get_module(const struct btf *btf) /* Returns struct btf corresponding to the struct module. * This function can return NULL or ERR_PTR. */ -static struct btf *btf_get_module_btf(const struct module *module) +struct btf *btf_get_module_btf(const struct module *module) { #ifdef CONFIG_DEBUG_INFO_BTF_MODULES struct btf_module *btf_mod, *tmp; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 650e5d21f90d09..a43b6840a3691b 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -215,6 +215,7 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, { struct xdp_rxq_info rxq; struct xdp_buff xdp; + volatile struct xdp_meta_generic xdp_meta_generic; int i, nframes = 0; xdp_set_return_frame_no_direct(); diff --git a/net/core/xdp.c b/net/core/xdp.c index 24420209bf0e45..84851b3e8fa7a1 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -4,6 +4,7 @@ * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. */ #include +#include #include #include #include @@ -711,3 +712,39 @@ struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf) return nxdpf; } + +int xdp_meta_fill_id_magic(struct xdp_meta_tail *tail, const struct module *mod, + const char *type_name) +{ + struct btf *btf = NULL, *vmlinux; + int id, err = 0; + u32 vmlinux_len; + + vmlinux = bpf_get_btf_vmlinux(); + if (IS_ERR(vmlinux)) + return PTR_ERR(vmlinux); + + btf = btf_get_module_btf(mod); + if (IS_ERR(btf)) + return PTR_ERR(btf); + else if (!btf) + btf = vmlinux; + + id = btf_find_by_name_kind(btf, type_name, BTF_KIND_STRUCT); + if (id < 0) { + err = id; + goto put; + } + + tail->type_id = cpu_to_le32(id); + tail->magic = cpu_to_le32(XDP_META_GENERIC_MAGIC); + + vmlinux_len = btf_nr_types(vmlinux); + tail->btf_id = cpu_to_le32(btf_obj_id(id < vmlinux_len ? vmlinux : btf)); +put: + if (btf_is_module(btf)) + btf_put(btf); + + return err; +} +EXPORT_SYMBOL_GPL(xdp_meta_fill_id_magic); diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 38638845db9d74..3adad5f1d4effe 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -59,6 +59,7 @@ tprogs-y += xdp_redirect_map_multi tprogs-y += xdp_redirect_map tprogs-y += xdp_redirect tprogs-y += xdp_monitor +tprogs-y += xdp_meta # Libbpf dependencies LIBBPF_SRC = $(TOOLS_PATH)/lib/bpf @@ -124,6 +125,7 @@ xdp_redirect_cpu-objs := xdp_redirect_cpu_user.o $(XDP_SAMPLE) xdp_redirect_map-objs := xdp_redirect_map_user.o $(XDP_SAMPLE) xdp_redirect-objs := xdp_redirect_user.o $(XDP_SAMPLE) xdp_monitor-objs := xdp_monitor_user.o $(XDP_SAMPLE) +xdp_meta-objs := xdp_meta_user.o $(XDP_SAMPLE) # Tell kbuild to always build the programs always-y := $(tprogs-y) @@ -220,6 +222,7 @@ TPROGLDLIBS_xdp_redirect += -lm TPROGLDLIBS_xdp_redirect_cpu += -lm TPROGLDLIBS_xdp_redirect_map += -lm TPROGLDLIBS_xdp_redirect_map_multi += -lm +TPROGLDLIBS_xdp_meta += -lm TPROGLDLIBS_tracex4 += -lrt TPROGLDLIBS_trace_output += -lrt TPROGLDLIBS_map_perf_test += -lrt @@ -342,6 +345,7 @@ $(obj)/xdp_redirect_map_multi_user.o: $(obj)/xdp_redirect_map_multi.skel.h $(obj)/xdp_redirect_map_user.o: $(obj)/xdp_redirect_map.skel.h $(obj)/xdp_redirect_user.o: $(obj)/xdp_redirect.skel.h $(obj)/xdp_monitor_user.o: $(obj)/xdp_monitor.skel.h +$(obj)/xdp_meta_user.o: $(obj)/xdp_meta.skel.h $(obj)/tracex5_kern.o: $(obj)/syscall_nrs.h $(obj)/hbm_out_kern.o: $(src)/hbm.h $(src)/hbm_kern.h @@ -409,7 +413,8 @@ $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/x -c $(filter %.bpf.c,$^) -o $@ LINKED_SKELS := xdp_redirect_cpu.skel.h xdp_redirect_map_multi.skel.h \ - xdp_redirect_map.skel.h xdp_redirect.skel.h xdp_monitor.skel.h + xdp_redirect_map.skel.h xdp_redirect.skel.h xdp_monitor.skel.h \ + xdp_meta.skel.h clean-files += $(LINKED_SKELS) xdp_redirect_cpu.skel.h-deps := xdp_redirect_cpu.bpf.o xdp_sample.bpf.o @@ -417,6 +422,7 @@ xdp_redirect_map_multi.skel.h-deps := xdp_redirect_map_multi.bpf.o xdp_sample.bp xdp_redirect_map.skel.h-deps := xdp_redirect_map.bpf.o xdp_sample.bpf.o xdp_redirect.skel.h-deps := xdp_redirect.bpf.o xdp_sample.bpf.o xdp_monitor.skel.h-deps := xdp_monitor.bpf.o xdp_sample.bpf.o +xdp_meta.skel.h-deps := xdp_meta.bpf.o LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.bpf.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps))) diff --git a/samples/bpf/xdp_meta.bpf.c b/samples/bpf/xdp_meta.bpf.c new file mode 100644 index 00000000000000..de495fd84fa780 --- /dev/null +++ b/samples/bpf/xdp_meta.bpf.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Intel Corporation. + * + * Author: Larysa Zaremba + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + */ +#include "vmlinux.h" + +#include +#include +#include +#include + +# define __force __attribute__((force)) + +#define BITFIELD_GET(_mask, _reg) \ +({ \ + (typeof(_mask))(((_reg) & (_mask)) >> (__builtin_ffsll(_mask) - 1)); \ +}) + +#define BITFIELD_GET_META(_mask, _reg) BITFIELD_GET(_mask, __bpf_le32_to_cpu(_reg)) + +SEC("xdp") +int xdp_meta_prog(struct xdp_md *ctx) +{ + struct xdp_meta_generic *data_meta = (void *)(long)ctx->data_meta; + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + u16 vlan_type, hash_type, csum_level, csum_status; + u32 type_id_meta, btf_id_meta, magic_meta; + u64 btf_id_libbpf; + s64 offset; + s32 ret; + + if (data_meta + 1 > data) { + bpf_printk("Driver did not provide metadata: data_meta space is not sufficient for generic metadata, should be %ld, is %ld\n", + sizeof(struct xdp_meta_generic), (long)data - (long)data_meta); + return XDP_DROP; + } + + offset = (s64)data - (s64)(void *)data_meta - sizeof(struct xdp_meta_generic); + ret = bpf_xdp_adjust_meta(ctx, offset); + if (ret < 0) { + bpf_printk("Could not adjust meta, offset: %ld, error: %d\n", offset, ret); + return XDP_DROP; + } + + data_meta = (void *)(long)ctx->data_meta; + data_end = (void *)(long)ctx->data_end; + data = (void *)(long)ctx->data; + if (data_meta + 1 > data) { + bpf_printk("Meta was not properly adjusted: data_meta space is not sufficient for generic metadata, should be %ld, is %ld\n", + sizeof(struct xdp_meta_generic), (long)data - (long)data_meta); + return XDP_DROP; + } + + magic_meta = __bpf_le32_to_cpu(data_meta->magic); + if (magic_meta != XDP_META_GENERIC_MAGIC) { + bpf_printk("Meta des not contain generic hints, based on received magic: 0x%x\n", + magic_meta); + return XDP_DROP; + } + + btf_id_libbpf = bpf_core_type_id_kernel(struct xdp_meta_generic); + type_id_meta = __bpf_le32_to_cpu(data_meta->type_id); + btf_id_meta = __bpf_le32_to_cpu(data_meta->btf_id); + + bpf_printk("id from libbpf %u (module BTF id: %u), id from hints metadata %u (module BTF id: %u)\n", + btf_id_libbpf & 0xFFFFFFFF, btf_id_libbpf >> 32, type_id_meta, btf_id_meta); + + if (btf_id_libbpf == (((u64)btf_id_meta << 32) | type_id_meta)) + bpf_printk("Received meta is generic\n"); + else + bpf_printk("Received meta type is unknown\n"); + + if (BITFIELD_GET_META(XDP_META_RX_QID_BIT, data_meta->rx_flags)) + bpf_printk("RX queue ID: %d\n", __bpf_le32_to_cpu(data_meta->rx_qid)); + else + bpf_printk("RX queue ID not present\n"); + + if (BITFIELD_GET_META(XDP_META_RX_TSTAMP_BIT, data_meta->rx_flags)) + bpf_printk("RX timestamp: %d\n", __bpf_le32_to_cpu(data_meta->rx_tstamp)); + else + bpf_printk("RX timestamp not present\n"); + + vlan_type = BITFIELD_GET_META(XDP_META_RX_VLAN_TYPE, data_meta->rx_flags); + if (vlan_type) + bpf_printk("RX VLAN type: %s, VLAN ID: %d", + vlan_type == XDP_META_RX_CVID ? "customer" : "service", + __bpf_le32_to_cpu(data_meta->rx_vid)); + else + bpf_printk("No VLAN detected\n"); + + hash_type = BITFIELD_GET_META(XDP_META_RX_HASH_TYPE, data_meta->rx_flags); + if (hash_type) + bpf_printk("RX hash type: L%d, hash value: 0x%x\n", + hash_type + 1, __bpf_le32_to_cpu(data_meta->rx_hash)); + else + bpf_printk("RX hash not present\n"); + + csum_level = BITFIELD_GET_META(XDP_META_RX_CSUM_LEVEL, data_meta->rx_flags); + csum_status = BITFIELD_GET_META(XDP_META_RX_CSUM_STATUS, data_meta->rx_flags); + if (csum_status == XDP_META_RX_CSUM_COMP) + bpf_printk("L%d checksum is: 0x%x\n", csum_level, + __bpf_le32_to_cpu(data_meta->rx_csum)); + else if (csum_status == XDP_META_RX_CSUM_OK) + bpf_printk("L%d checksum was checked\n", csum_level); + else + bpf_printk("Checksum information was not provided\n"); + + return XDP_PASS; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/samples/bpf/xdp_meta_user.c b/samples/bpf/xdp_meta_user.c new file mode 100644 index 00000000000000..0b76aeb417d129 --- /dev/null +++ b/samples/bpf/xdp_meta_user.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Intel Corporation. + * + * Author: Larysa Zaremba + * + * xdp_meta_read_trace_pipe() code copied from xdp-project example + * (https://github.com/xdp-project/bpf-examples/tree/master/ktrace-CO-RE) + * by Jesper Dangaard Brouer + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + */ + +static const char *__doc__= + "Sample prints out generic hints (built by NIC driver) for every ingress packet.\n" + "Noflag option (-n) disables metadata XDP flag (for NIC driver testing).\n"; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xdp_sample_user.h" +#include "xdp_meta.skel.h" + +#define DEBUGFS "/sys/kernel/debug/tracing/" + +static volatile bool xdp_meta_sample_running = true; + +static void xdp_meta_sample_stop(int signo) +{ + xdp_meta_sample_running = false; +} + +static int mask = 0; + +static const struct option long_options[] = { + {"interface", required_argument, NULL, 'i' }, + {"force", no_argument, NULL, 'f'}, + {"noflag", no_argument, NULL, 'n'}, + {} +}; + +/* Had to change the standard read_trace_pipe from trace_helpers.h */ +static void xdp_meta_read_trace_pipe(void) +{ + int trace_fd; + + trace_fd = open(DEBUGFS "trace_pipe", O_RDONLY, 0); + if (trace_fd < 0) { + fprintf(stderr, "Could not open the trace_pipe\n"); + return; + } + + while (xdp_meta_sample_running) { + static char buf[4096]; + ssize_t sz; + + sz = read(trace_fd, buf, sizeof(buf) - 1); + if (sz > 0) { + buf[sz] = 0; + puts(buf); + } + } +} + +int main(int argc, char **argv) +{ + int ret = EXIT_FAIL_OPTION, opt; + struct sigaction handle_ctrl_c; + struct install_opts opts = { + .use_meta = true, + }; + struct xdp_meta *skel; + bool error = true; + + /* Parse commands line args */ + while ((opt = getopt_long(argc, argv, "i:nf", + long_options, NULL)) != -1) { + switch (opt) { + case 'i': + opts.ifindex = if_nametoindex(optarg); + if (!opts.ifindex) { + opts.ifindex = strtoul(optarg, NULL, 0); + } + if (!opts.ifindex) { + fprintf(stderr, "Bad interface index or name\n"); + goto arg_exit; + } + break; + case 'f': + opts.force = true; + break; + case 'n': + opts.use_meta = false; + break; +arg_exit: + default: + sample_usage(argv, long_options, __doc__, mask, error); + return ret; + } + } + + skel = xdp_meta__open(); + if (!skel) { + fprintf(stderr, "Failed to xdp_meta__open: %s\n", + strerror(errno)); + ret = 1; + goto end; + } + + ret = xdp_meta__load(skel); + if (ret < 0) { + fprintf(stderr, "Failed to xdp_meta__load: %s\n", strerror(errno)); + ret = 1; + goto end_destroy; + } + + ret = sample_install_xdp(skel->progs.xdp_meta_prog, &opts); + + memset(&handle_ctrl_c, 0, sizeof(handle_ctrl_c)); + handle_ctrl_c.sa_handler = &xdp_meta_sample_stop; + sigaction(SIGINT, &handle_ctrl_c, NULL); + + xdp_meta_read_trace_pipe(); + +end_destroy: + xdp_meta__destroy(skel); +end: + sample_exit(ret); + return ret; +} diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c index 5f74a70a902123..c2e68af56886f2 100644 --- a/samples/bpf/xdp_redirect_cpu_user.c +++ b/samples/bpf/xdp_redirect_cpu_user.c @@ -309,6 +309,9 @@ int main(int argc, char **argv) const char *mprog_filename = NULL, *mprog_name = NULL; struct xdp_redirect_cpu *skel; struct bpf_map_info info = {}; + struct install_opts opts = { + .ifindex = -1, + }; struct bpf_cpumap_val value; __u32 infosz = sizeof(info); int ret = EXIT_FAIL_OPTION; @@ -316,13 +319,10 @@ int main(int argc, char **argv) bool stress_mode = false; struct bpf_program *prog; const char *prog_name; - bool generic = false; - bool force = false; int added_cpus = 0; bool error = true; int longindex = 0; int add_cpu = -1; - int ifindex = -1; int *cpu, i, opt; __u32 qsize; int n_cpus; @@ -392,10 +392,10 @@ int main(int argc, char **argv) usage(argv, long_options, __doc__, mask, true, skel->obj); goto end_cpu; } - ifindex = if_nametoindex(optarg); - if (!ifindex) - ifindex = strtoul(optarg, NULL, 0); - if (!ifindex) { + opts.ifindex = if_nametoindex(optarg); + if (!opts.ifindex) + opts.ifindex = strtoul(optarg, NULL, 0); + if (!opts.ifindex) { fprintf(stderr, "Bad interface index or name (%d): %s\n", errno, strerror(errno)); usage(argv, long_options, __doc__, mask, true, skel->obj); @@ -409,7 +409,7 @@ int main(int argc, char **argv) interval = strtoul(optarg, NULL, 0); break; case 'S': - generic = true; + opts.generic = true; break; case 'x': stress_mode = true; @@ -457,7 +457,7 @@ int main(int argc, char **argv) qsize = strtoul(optarg, NULL, 0); break; case 'F': - force = true; + opts.force = true; break; case 'v': sample_switch_mode(); @@ -471,7 +471,7 @@ int main(int argc, char **argv) } ret = EXIT_FAIL_OPTION; - if (ifindex == -1) { + if (opts.ifindex == -1) { fprintf(stderr, "Required option --dev missing\n"); usage(argv, long_options, __doc__, mask, true, skel->obj); goto end_cpu; @@ -484,7 +484,7 @@ int main(int argc, char **argv) goto end_cpu; } - skel->rodata->from_match[0] = ifindex; + skel->rodata->from_match[0] = opts.ifindex; if (redir_interface) skel->rodata->to_match[0] = if_nametoindex(redir_interface); @@ -541,7 +541,7 @@ int main(int argc, char **argv) } ret = EXIT_FAIL_XDP; - if (sample_install_xdp(prog, ifindex, generic, force) < 0) + if (sample_install_xdp(prog, &opts) < 0) goto end_cpu; ret = sample_run(interval, stress_mode ? stress_cpumap : NULL, &value); diff --git a/samples/bpf/xdp_redirect_map_multi_user.c b/samples/bpf/xdp_redirect_map_multi_user.c index 315314716121df..3e7a96c4c35de9 100644 --- a/samples/bpf/xdp_redirect_map_multi_user.c +++ b/samples/bpf/xdp_redirect_map_multi_user.c @@ -81,13 +81,11 @@ int main(int argc, char **argv) struct xdp_redirect_map_multi *skel; struct bpf_program *ingress_prog; bool xdp_devmap_attached = false; + struct install_opts opts = { }; struct bpf_map *forward_map; int ret = EXIT_FAIL_OPTION; unsigned long interval = 2; char ifname[IF_NAMESIZE]; - unsigned int ifindex; - bool generic = false; - bool force = false; bool tried = false; bool error = true; int i, opt; @@ -96,13 +94,13 @@ int main(int argc, char **argv) long_options, NULL)) != -1) { switch (opt) { case 'S': - generic = true; + opts.generic = true; /* devmap_xmit tracepoint not available */ mask &= ~(SAMPLE_DEVMAP_XMIT_CNT | SAMPLE_DEVMAP_XMIT_CNT_MULTI); break; case 'F': - force = true; + opts.force = true; break; case 'X': xdp_devmap_attached = true; @@ -187,13 +185,13 @@ int main(int argc, char **argv) forward_map = skel->maps.forward_map_native; for (i = 0; ifaces[i] > 0; i++) { - ifindex = ifaces[i]; + opts.ifindex = ifaces[i]; ret = EXIT_FAIL_XDP; restart: /* bind prog_fd to each interface */ - if (sample_install_xdp(ingress_prog, ifindex, generic, force) < 0) { - if (generic && !tried) { + if (sample_install_xdp(ingress_prog, &opts) < 0) { + if (opts.generic && !tried) { fprintf(stderr, "Trying fallback to sizeof(int) as value_size for devmap in generic mode\n"); ingress_prog = skel->progs.xdp_redirect_map_general; @@ -207,10 +205,11 @@ int main(int argc, char **argv) /* Add all the interfaces to forward group and attach * egress devmap program if exist */ - devmap_val.ifindex = ifindex; + devmap_val.ifindex = opts.ifindex; if (xdp_devmap_attached) devmap_val.bpf_prog.fd = bpf_program__fd(skel->progs.xdp_devmap_prog); - ret = bpf_map_update_elem(bpf_map__fd(forward_map), &ifindex, &devmap_val, 0); + ret = bpf_map_update_elem(bpf_map__fd(forward_map), + &opts.ifindex, &devmap_val, 0); if (ret < 0) { fprintf(stderr, "Failed to update devmap value: %s\n", strerror(errno)); diff --git a/samples/bpf/xdp_redirect_map_user.c b/samples/bpf/xdp_redirect_map_user.c index b6e4fc849577cb..246c7259d8bc55 100644 --- a/samples/bpf/xdp_redirect_map_user.c +++ b/samples/bpf/xdp_redirect_map_user.c @@ -44,6 +44,7 @@ int main(int argc, char **argv) { struct bpf_devmap_val devmap_val = {}; bool xdp_devmap_attached = false; + struct install_opts opts = { }; struct xdp_redirect_map *skel; char str[2 * IF_NAMESIZE + 1]; char ifname_out[IF_NAMESIZE]; @@ -53,8 +54,6 @@ int main(int argc, char **argv) unsigned long interval = 2; int ret = EXIT_FAIL_OPTION; struct bpf_program *prog; - bool generic = false; - bool force = false; bool tried = false; bool error = true; int opt, key = 0; @@ -63,13 +62,13 @@ int main(int argc, char **argv) long_options, NULL)) != -1) { switch (opt) { case 'S': - generic = true; + opts.generic = true; /* devmap_xmit tracepoint not available */ mask &= ~(SAMPLE_DEVMAP_XMIT_CNT | SAMPLE_DEVMAP_XMIT_CNT_MULTI); break; case 'F': - force = true; + opts.force = true; break; case 'X': xdp_devmap_attached = true; @@ -157,13 +156,14 @@ int main(int argc, char **argv) prog = skel->progs.xdp_redirect_map_native; tx_port_map = skel->maps.tx_port_native; restart: - if (sample_install_xdp(prog, ifindex_in, generic, force) < 0) { + opts.ifindex = ifindex_in; + if (sample_install_xdp(prog, &opts) < 0) { /* First try with struct bpf_devmap_val as value for generic * mode, then fallback to sizeof(int) for older kernels. */ fprintf(stderr, "Trying fallback to sizeof(int) as value_size for devmap in generic mode\n"); - if (generic && !tried) { + if (opts.generic && !tried) { prog = skel->progs.xdp_redirect_map_general; tx_port_map = skel->maps.tx_port_general; tried = true; @@ -174,7 +174,8 @@ int main(int argc, char **argv) } /* Loading dummy XDP prog on out-device */ - sample_install_xdp(skel->progs.xdp_redirect_dummy_prog, ifindex_out, generic, force); + opts.ifindex = ifindex_out; + sample_install_xdp(skel->progs.xdp_redirect_dummy_prog, &opts); devmap_val.ifindex = ifindex_out; if (xdp_devmap_attached) diff --git a/samples/bpf/xdp_redirect_user.c b/samples/bpf/xdp_redirect_user.c index 7af5b07a752304..d026ab2fd624f4 100644 --- a/samples/bpf/xdp_redirect_user.c +++ b/samples/bpf/xdp_redirect_user.c @@ -43,26 +43,25 @@ static const struct option long_options[] = { int main(int argc, char **argv) { int ifindex_in, ifindex_out, opt; + struct install_opts opts = { }; char str[2 * IF_NAMESIZE + 1]; char ifname_out[IF_NAMESIZE]; char ifname_in[IF_NAMESIZE]; int ret = EXIT_FAIL_OPTION; unsigned long interval = 2; struct xdp_redirect *skel; - bool generic = false; - bool force = false; bool error = true; while ((opt = getopt_long(argc, argv, "hSFi:vs", long_options, NULL)) != -1) { switch (opt) { case 'S': - generic = true; + opts.generic = true; mask &= ~(SAMPLE_DEVMAP_XMIT_CNT | SAMPLE_DEVMAP_XMIT_CNT_MULTI); break; case 'F': - force = true; + opts.force = true; break; case 'i': interval = strtoul(optarg, NULL, 0); @@ -133,13 +132,13 @@ int main(int argc, char **argv) } ret = EXIT_FAIL_XDP; - if (sample_install_xdp(skel->progs.xdp_redirect_prog, ifindex_in, - generic, force) < 0) + opts.ifindex = ifindex_in; + if (sample_install_xdp(skel->progs.xdp_redirect_prog, &opts) < 0) goto end_destroy; /* Loading dummy XDP prog on out-device */ - sample_install_xdp(skel->progs.xdp_redirect_dummy_prog, ifindex_out, - generic, force); + opts.ifindex = ifindex_out; + sample_install_xdp(skel->progs.xdp_redirect_dummy_prog, &opts); ret = EXIT_FAIL; if (!if_indextoname(ifindex_in, ifname_in)) { diff --git a/samples/bpf/xdp_sample_user.c b/samples/bpf/xdp_sample_user.c index c4332d068b9199..929c17c7077d4d 100644 --- a/samples/bpf/xdp_sample_user.c +++ b/samples/bpf/xdp_sample_user.c @@ -1281,9 +1281,10 @@ static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags) return bpf_xdp_detach(ifindex, xdp_flags, NULL); } -int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic, - bool force) +int sample_install_xdp(struct bpf_program *xdp_prog, + const struct install_opts *opts) { + __u32 ifindex = opts->ifindex; int ret, xdp_flags = 0; __u32 prog_id = 0; @@ -1293,8 +1294,9 @@ int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic, return -ENOTSUP; } - xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0; - xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; + xdp_flags |= !opts->force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0; + xdp_flags |= opts->generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; + xdp_flags |= opts->use_meta ? XDP_FLAGS_USE_METADATA : 0; ret = bpf_xdp_attach(ifindex, bpf_program__fd(xdp_prog), xdp_flags, NULL); if (ret < 0) { ret = -errno; @@ -1302,7 +1304,8 @@ int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic, "Failed to install program \"%s\" on ifindex %d, mode = %s, " "force = %s: %s\n", bpf_program__name(xdp_prog), ifindex, - generic ? "skb" : "native", force ? "true" : "false", + opts->generic ? "skb" : "native", + opts->force ? "true" : "false", strerror(-ret)); return ret; } diff --git a/samples/bpf/xdp_sample_user.h b/samples/bpf/xdp_sample_user.h index f45051679977ed..5b22d650b5158f 100644 --- a/samples/bpf/xdp_sample_user.h +++ b/samples/bpf/xdp_sample_user.h @@ -30,14 +30,21 @@ enum stats_mask { #define EXIT_FAIL_BPF 4 #define EXIT_FAIL_MEM 5 +struct install_opts { + int ifindex; + __u32 force:1; + __u32 generic:1; + __u32 use_meta:1; +}; + int sample_setup_maps(struct bpf_map **maps); int __sample_init(int mask); void sample_exit(int status); int sample_run(int interval, void (*post_cb)(void *), void *ctx); void sample_switch_mode(void); -int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic, - bool force); +int sample_install_xdp(struct bpf_program *xdp_prog, + const struct install_opts *opts); void sample_usage(char *argv[], const struct option *long_options, const char *doc, int mask, bool error); diff --git a/tools/include/uapi/linux/if_link.h b/tools/include/uapi/linux/if_link.h index e1ba2d51b717b7..d13ce12c056c56 100644 --- a/tools/include/uapi/linux/if_link.h +++ b/tools/include/uapi/linux/if_link.h @@ -1185,11 +1185,13 @@ enum { #define XDP_FLAGS_DRV_MODE (1U << 2) #define XDP_FLAGS_HW_MODE (1U << 3) #define XDP_FLAGS_REPLACE (1U << 4) +#define XDP_FLAGS_USE_METADATA (1U << 5) #define XDP_FLAGS_MODES (XDP_FLAGS_SKB_MODE | \ XDP_FLAGS_DRV_MODE | \ XDP_FLAGS_HW_MODE) #define XDP_FLAGS_MASK (XDP_FLAGS_UPDATE_IF_NOEXIST | \ - XDP_FLAGS_MODES | XDP_FLAGS_REPLACE) + XDP_FLAGS_MODES | XDP_FLAGS_REPLACE | \ + XDP_FLAGS_USE_METADATA) /* These are stored into IFLA_XDP_ATTACHED on dump. */ enum { diff --git a/tools/lib/bpf/bpf_endian.h b/tools/lib/bpf/bpf_endian.h index ec9db4feca9f2f..7ed724dfd6b3f7 100644 --- a/tools/lib/bpf/bpf_endian.h +++ b/tools/lib/bpf/bpf_endian.h @@ -60,6 +60,10 @@ # define __bpf_cpu_to_be64(x) __builtin_bswap64(x) # define __bpf_constant_be64_to_cpu(x) ___bpf_swab64(x) # define __bpf_constant_cpu_to_be64(x) ___bpf_swab64(x) +# define __bpf_le32_to_cpu(x) (x) +# define __bpf_cpu_to_le32(x) (x) +# define __bpf_constant_le32_to_cpu(x) (x) +# define __bpf_constant_cpu_to_le32(x) (x) #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ # define __bpf_ntohs(x) (x) # define __bpf_htons(x) (x) @@ -73,6 +77,10 @@ # define __bpf_cpu_to_be64(x) (x) # define __bpf_constant_be64_to_cpu(x) (x) # define __bpf_constant_cpu_to_be64(x) (x) +# define __bpf_le32_to_cpu(x) __builtin_bswap32(x) +# define __bpf_cpu_to_le32(x) __builtin_bswap32(x) +# define __bpf_constant_le32_to_cpu(x) ___bpf_swab32(x) +# define __bpf_constant_cpu_to_le32(x) ___bpf_swab32(x) #else # error "Fix your compiler's __BYTE_ORDER__?!" #endif diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 1383e26c5d1f13..423f6875a0b7b2 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -121,6 +121,9 @@ struct btf { /* Pointer size (in bytes) for a target architecture of this BTF */ int ptr_sz; + + /* BTF object id, valid for vmlinux and module BTF */ + __u32 id; }; static inline __u64 ptr_to_u64(const void *ptr) @@ -453,6 +456,11 @@ const struct btf *btf__base_btf(const struct btf *btf) return btf->base_btf; } +__u32 btf_obj_id(const struct btf *btf) +{ + return btf->id; +} + /* internal helper returning non-const pointer to a type */ struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id) { @@ -793,6 +801,7 @@ static struct btf *btf_new_empty(struct btf *base_btf) btf->fd = -1; btf->ptr_sz = sizeof(void *); btf->swapped_endian = false; + btf->id = 0; if (base_btf) { btf->base_btf = base_btf; @@ -843,6 +852,7 @@ static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf) btf->start_id = 1; btf->start_str_off = 0; btf->fd = -1; + btf->id = 0; if (base_btf) { btf->base_btf = base_btf; @@ -1356,6 +1366,8 @@ struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf) } btf = btf_new(ptr, btf_info.btf_size, base_btf); + if (!IS_ERR_OR_NULL(btf)) + btf->id = btf_info.id; exit_free: free(ptr); @@ -4646,6 +4658,102 @@ static int btf_dedup_remap_types(struct btf_dedup *d) return 0; } +/** + * btf_load_next_with_info() - Get first BTF with id bigger than the input one. + * @start_id: Id to start the search from. + * @info: Buffer to put BTF info to, invalid content if call not succuessful. + * Name buffer and length fields must be initialized by the caller. + * It's recommended to prepare a buffer of BTF_NAME_BUFF_LEN length. + * @base_btf: Base BTF, can be NULL if load_vmlinux is true. + * @load_vmlinux: Look for the vmlinux BTF instead of a module BTF. + * + * Return: pointer to BTF loaded from kernel or an error pointer. + * FD must be closed after BTF is no longer needed. If load_vmlinux is true, + * FD can be closed and set to (-1) right away without preventing later usage. + */ +struct btf *btf_load_next_with_info(__u32 start_id, struct bpf_btf_info *info, + struct btf *base_btf, bool load_vmlinux) +{ + __u32 name_len = info->name_len; + __u32 len = sizeof(*info); + __u64 name = info->name; + __u32 id = start_id; + + while (true) { + struct btf *btf; + int err, fd; + + err = bpf_btf_get_next_id(id, &id); + if (err) { + err = -errno; + if (err != -ENOENT) + pr_warn("failed to iterate BTF objects: %d\n", err); + return ERR_PTR(err); + } + + fd = bpf_btf_get_fd_by_id(id); + if (fd < 0) { + if (errno == ENOENT) + continue; /* expected race: non-vmlinux BTF was unloaded */ + err = -errno; + pr_warn("failed to get BTF object #%d FD: %d\n", id, err); + return ERR_PTR(err); + } + + memset(info, 0, len); + info->name = name; + info->name_len = name_len; + + err = bpf_obj_get_info_by_fd(fd, info, &len); + if (err) { + err = -errno; + btf = ERR_PTR(err); + pr_warn("failed to get BTF object #%d info: %d\n", id, err); + goto err_out; + } + + /* filter BTFs */ + if (!info->kernel_btf || + !strncmp((const char *)name, "vmlinux", name_len) == !load_vmlinux) { + close(fd); + continue; + } + + btf = btf_get_from_fd(fd, base_btf); + err = libbpf_get_error(btf); + if (err) { + pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", + (const char *)name, id, err); + goto err_out; + } + + btf->fd = fd; + return btf; + +err_out: + close(fd); + return btf; + } +} + +static struct btf *btf_load_vmlinux_from_kernel(void) +{ + struct bpf_btf_info info; + struct btf *btf; + char name[BTF_NAME_BUFF_LEN] = ""; + + memset(&info, 0, sizeof(info)); + info.name = ptr_to_u64(name); + info.name_len = sizeof(name); + btf = btf_load_next_with_info(0, &info, NULL, true); + if (!libbpf_get_error(btf)) { + close(btf->fd); + btf__set_fd(btf, -1); + } + + return btf; +} + /* * Probe few well-known locations for vmlinux kernel image and try to load BTF * data out of it to use for target BTF. @@ -4672,6 +4780,14 @@ struct btf *btf__load_vmlinux_btf(void) struct btf *btf; int i, err; + btf = btf_load_vmlinux_from_kernel(); + err = libbpf_get_error(btf); + pr_debug("loading vmlinux BTF from kernel: %d\n", err); + if (!err) + return btf; + pr_warn("failed to load vmlinux BTF from kernel: %d, will look though filesystem, err\n", + err); + uname(&buf); for (i = 0; i < ARRAY_SIZE(locations); i++) { @@ -4685,14 +4801,14 @@ struct btf *btf__load_vmlinux_btf(void) else btf = btf__parse_elf(path, NULL); err = libbpf_get_error(btf); - pr_debug("loading kernel BTF '%s': %d\n", path, err); + pr_debug("loading vmlinux BTF '%s': %d\n", path, err); if (err) continue; return btf; } - pr_warn("failed to find valid kernel BTF\n"); + pr_warn("failed to find valid vmlinux BTF\n"); return libbpf_err_ptr(-ESRCH); } diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 809fe209cdcc0a..3b52253fc07bf7 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -5277,11 +5277,11 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, static int load_module_btfs(struct bpf_object *obj) { - struct bpf_btf_info info; struct module_btf *mod_btf; + struct bpf_btf_info info; struct btf *btf; - char name[64]; - __u32 id = 0, len; + char name[BTF_NAME_BUFF_LEN] = ""; + __u32 id = 0; int err, fd; if (obj->btf_modules_loaded) @@ -5298,50 +5298,22 @@ static int load_module_btfs(struct bpf_object *obj) return 0; while (true) { - err = bpf_btf_get_next_id(id, &id); - if (err && errno == ENOENT) - return 0; - if (err) { - err = -errno; - pr_warn("failed to iterate BTF objects: %d\n", err); - return err; - } - - fd = bpf_btf_get_fd_by_id(id); - if (fd < 0) { - if (errno == ENOENT) - continue; /* expected race: BTF was unloaded */ - err = -errno; - pr_warn("failed to get BTF object #%d FD: %d\n", id, err); - return err; - } - - len = sizeof(info); memset(&info, 0, sizeof(info)); info.name = ptr_to_u64(name); info.name_len = sizeof(name); - err = bpf_obj_get_info_by_fd(fd, &info, &len); - if (err) { - err = -errno; - pr_warn("failed to get BTF object #%d info: %d\n", id, err); - goto err_out; - } - - /* ignore non-module BTFs */ - if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { - close(fd); - continue; - } - - btf = btf_get_from_fd(fd, obj->btf_vmlinux); + btf = btf_load_next_with_info(id, &info, obj->btf_vmlinux, false); err = libbpf_get_error(btf); if (err) { - pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", - name, id, err); - goto err_out; + if (errno == ENOENT) + return 0; + return err; } + fd = btf__fd(btf); + btf__set_fd(btf, -1); + id = btf_obj_id(btf); + err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); if (err) diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index b6247dc7f8eb0f..53a0078ef4f9d2 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -345,6 +345,11 @@ int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 lo struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, const char **prefix, int *kind); +__u32 btf_obj_id(const struct btf *btf); + +#define BTF_NAME_BUFF_LEN 64 +struct btf *btf_load_next_with_info(__u32 start_id, struct bpf_btf_info *info, + struct btf *base_btf, bool load_vmlinux); struct btf_ext_info { /* diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index f946f23eab20b2..6dddbb711eed74 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -797,6 +797,7 @@ static int bpf_core_calc_relo(const char *prog_name, res->fail_memsz_adjust = false; res->orig_sz = res->new_sz = 0; res->orig_type_id = res->new_type_id = 0; + res->btf_obj_id = 0; if (core_relo_is_field_based(relo->kind)) { err = bpf_core_calc_field_relo(prog_name, relo, local_spec, @@ -847,6 +848,8 @@ static int bpf_core_calc_relo(const char *prog_name, } else if (core_relo_is_type_based(relo->kind)) { err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate); err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL); + if (!err && relo->kind == BPF_CORE_TYPE_ID_TARGET) + res->btf_obj_id = btf_obj_id(targ_spec->btf); } else if (core_relo_is_enumval_based(relo->kind)) { err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val); err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val); @@ -1035,7 +1038,8 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, } insn[0].imm = new_val; - insn[1].imm = 0; /* currently only 32-bit values are supported */ + insn[1].imm = relo->kind == BPF_CORE_TYPE_ID_TARGET ? + res->btf_obj_id : 0; pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n", prog_name, relo_idx, insn_idx, (unsigned long long)imm, new_val); diff --git a/tools/lib/bpf/relo_core.h b/tools/lib/bpf/relo_core.h index a28bf3711ce272..78f2510d013b41 100644 --- a/tools/lib/bpf/relo_core.h +++ b/tools/lib/bpf/relo_core.h @@ -66,6 +66,7 @@ struct bpf_core_relo_res { __u32 orig_type_id; __u32 new_sz; __u32 new_type_id; + __u32 btf_obj_id; }; int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index f28f75aa915496..7fe4449acf396f 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -2,6 +2,7 @@ #include #include "progs/core_reloc_types.h" #include "bpf_testmod/bpf_testmod.h" +#include "bpf/libbpf_internal.h" #include #include #include @@ -11,28 +12,37 @@ static int duration = 0; #define STRUCT_TO_CHAR_PTR(struct_name) (const char *)&(struct struct_name) -#define MODULES_CASE(name, pg_name, tp_name) { \ +#define MODULES_OUTPUT STRUCT_TO_CHAR_PTR(core_reloc_module_output) { \ + .read_ctx_sz = sizeof(struct bpf_testmod_test_read_ctx), \ + .read_ctx_exists = true, \ + .buf_exists = true, \ + .len_exists = true, \ + .off_exists = true, \ + .len = 123, \ + .off = 0, \ + .comm = "test_progs", \ + .comm_len = sizeof("test_progs"), \ +}, \ +.output_len = sizeof(struct core_reloc_module_output) + +#define MODULES_ID_OUTPUT STRUCT_TO_CHAR_PTR(core_reloc_mod_id_output) {\ + .comm = "test_progs", \ + .comm_len = sizeof("test_progs"), \ +}, \ +.output_len = sizeof(struct core_reloc_mod_id_output) + +#define MODULES_CASE(name, pg_name, tp_name, expected_out, setup_fn) { \ .case_name = name, \ .bpf_obj_file = "test_core_reloc_module.o", \ .btf_src_file = NULL, /* find in kernel module BTFs */ \ .input = "", \ .input_len = 0, \ - .output = STRUCT_TO_CHAR_PTR(core_reloc_module_output) { \ - .read_ctx_sz = sizeof(struct bpf_testmod_test_read_ctx),\ - .read_ctx_exists = true, \ - .buf_exists = true, \ - .len_exists = true, \ - .off_exists = true, \ - .len = 123, \ - .off = 0, \ - .comm = "test_progs", \ - .comm_len = sizeof("test_progs"), \ - }, \ - .output_len = sizeof(struct core_reloc_module_output), \ + .output = expected_out, \ .prog_name = pg_name, \ .raw_tp_name = tp_name, \ .trigger = __trigger_module_test_read, \ .needs_testmod = true, \ + .setup = setup_fn, \ } #define FLAVORS_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \ @@ -377,6 +387,37 @@ struct core_reloc_test_case { trigger_test_fn trigger; }; +static struct btf *find_module_btf(const char *name, struct btf *vmlinux_btf) +{ + struct bpf_btf_info info; + __u32 id = 0; + struct btf *btf; + char name_buff[64] = ""; + bool is_vmlinux = strncmp(name, "vmlinux", sizeof("vmlinux")) == 0; + + while (true) { + memset(&info, 0, sizeof(info)); + info.name = ptr_to_u64(name_buff); + info.name_len = sizeof(name_buff); + + btf = btf_load_next_with_info(id, &info, vmlinux_btf, is_vmlinux); + if (CHECK(!btf || IS_ERR(btf), "btf_load_next_with_info", + "can't get BTF and/or info: %s, vmlinux? %d \n", strerror(errno), (int)is_vmlinux)) + return ERR_PTR(-1); + + if (info.name_len < 1 || strncmp(name_buff, name, info.name_len) != 0) + goto skip; + + return btf; +skip: + id = btf_obj_id(btf); + btf__free(btf); + } + + PRINT_FAIL("could not find the bpf_testmod BTF\n"); + return ERR_PTR(-1); +} + static int find_btf_type(const struct btf *btf, const char *name, __u32 kind) { int id; @@ -388,6 +429,30 @@ static int find_btf_type(const struct btf *btf, const char *name, __u32 kind) return id; } +static int setup_module_id_case(struct core_reloc_test_case *test) +{ + struct core_reloc_mod_id_output *exp = (void *)test->output; + struct btf *vmlinux_btf, *testmod_btf; + int ret = -1; + + vmlinux_btf = find_module_btf("vmlinux", NULL); + testmod_btf = find_module_btf("bpf_testmod", vmlinux_btf); + + if (IS_ERR(testmod_btf) || !testmod_btf) + goto free_vmlinux; + + exp->testmod_btf_obj_id = btf_obj_id(testmod_btf); + exp->testmod_type_id = find_btf_type(testmod_btf, "bpf_testmod_test_read_ctx", + BTF_KIND_STRUCT); + if (exp->testmod_type_id >= 0) + ret = 0; + + btf__free(testmod_btf); +free_vmlinux: + btf__free(vmlinux_btf); + return ret; +} + static int setup_type_id_case_local(struct core_reloc_test_case *test) { struct core_reloc_type_id_output *exp = (void *)test->output; @@ -531,8 +596,14 @@ static const struct core_reloc_test_case test_cases[] = { }, /* validate we can find kernel module BTF types for relocs/attach */ - MODULES_CASE("module_probed", "test_core_module_probed", "bpf_testmod_test_read"), - MODULES_CASE("module_direct", "test_core_module_direct", NULL), + MODULES_CASE("module_probed", "test_core_module_probed", "bpf_testmod_test_read", + MODULES_OUTPUT, NULL), + MODULES_CASE("module_direct", "test_core_module_direct", NULL, + MODULES_OUTPUT, NULL), + MODULES_CASE("module_type_id_probed", "test_core_module_id_probed", "bpf_testmod_test_read", + MODULES_ID_OUTPUT, setup_module_id_case), + MODULES_CASE("module_type_id_direct", "test_core_module_id_direct", NULL, + MODULES_ID_OUTPUT, setup_module_id_case), /* validate BPF program can use multiple flavors to match against * single target BTF type @@ -687,8 +758,8 @@ static const struct core_reloc_test_case test_cases[] = { .ub7 = 96, .sb4 = -7, .sb20 = -0x76543, - .u32 = 0x80000000, - .s32 = -0x76543210, + .__u32 = 0x80000000, + .__s32 = -0x76543210, }), BITFIELDS_CASE(bitfields___bit_sz_change, { .ub1 = 6, @@ -696,8 +767,8 @@ static const struct core_reloc_test_case test_cases[] = { .ub7 = 1, .sb4 = -1, .sb20 = -0x17654321, - .u32 = 0xBEEF, - .s32 = -0x3FEDCBA987654321LL, + .__u32 = 0xBEEF, + .__s32 = -0x3FEDCBA987654321LL, }), BITFIELDS_CASE(bitfields___bitfield_vs_int, { .ub1 = 0xFEDCBA9876543210LL, @@ -705,8 +776,8 @@ static const struct core_reloc_test_case test_cases[] = { .ub7 = -0x7EDCBA987654321LL, .sb4 = -0x6123456789ABCDELL, .sb20 = 0xD00DLL, - .u32 = -0x76543, - .s32 = 0x0ADEADBEEFBADB0BLL, + .__u32 = -0x76543, + .__s32 = 0x0ADEADBEEFBADB0BLL, }), BITFIELDS_CASE(bitfields___just_big_enough, { .ub1 = 0xFLL, diff --git a/tools/testing/selftests/bpf/progs/core_reloc_types.h b/tools/testing/selftests/bpf/progs/core_reloc_types.h index c95c0cabe95127..f9df33f409c14a 100644 --- a/tools/testing/selftests/bpf/progs/core_reloc_types.h +++ b/tools/testing/selftests/bpf/progs/core_reloc_types.h @@ -32,6 +32,13 @@ struct core_reloc_module_output { int comm_len; }; +struct core_reloc_mod_id_output { + int testmod_btf_obj_id; + int testmod_type_id; + char comm[sizeof("test_progs")]; + int comm_len; +}; + /* * FLAVORS */ @@ -718,8 +725,8 @@ struct core_reloc_bitfields_output { int64_t ub7; int64_t sb4; int64_t sb20; - int64_t u32; - int64_t s32; + int64_t __u32; + int64_t __s32; }; struct core_reloc_bitfields { @@ -731,8 +738,8 @@ struct core_reloc_bitfields { int8_t sb4: 4; int32_t sb20: 20; /* non-bitfields */ - uint32_t u32; - int32_t s32; + uint32_t __u32; + int32_t __s32; }; /* different bit sizes (both up and down) */ @@ -745,8 +752,8 @@ struct core_reloc_bitfields___bit_sz_change { int8_t sb4: 1; /* 4 -> 1 */ int32_t sb20: 30; /* 20 -> 30 */ /* non-bitfields */ - uint16_t u32; /* 32 -> 16 */ - int64_t s32 __bpf_aligned; /* 32 -> 64 */ + uint16_t __u32; /* 32 -> 16 */ + int64_t __s32 __bpf_aligned; /* 32 -> 64 */ }; /* turn bitfield into non-bitfield and vice versa */ @@ -756,8 +763,8 @@ struct core_reloc_bitfields___bitfield_vs_int { int64_t ub7 __bpf_aligned; /* 7 -> 64 non-bitfield signed */ int64_t sb4 __bpf_aligned; /* 4 -> 64 non-bitfield signed */ uint64_t sb20 __bpf_aligned; /* 20 -> 16 non-bitfield unsigned */ - int32_t u32: 20; /* 32 non-bitfield -> 20 bitfield */ - uint64_t s32: 60 __bpf_aligned; /* 32 non-bitfield -> 60 bitfield */ + int32_t __u32: 20; /* 32 non-bitfield -> 20 bitfield */ + uint64_t __s32: 60 __bpf_aligned; /* 32 non-bitfield -> 60 bitfield */ }; struct core_reloc_bitfields___just_big_enough { @@ -766,8 +773,8 @@ struct core_reloc_bitfields___just_big_enough { uint32_t ub7; uint32_t sb4; uint32_t sb20; - uint32_t u32; - uint32_t s32; + uint32_t __u32; + uint32_t __s32; } __attribute__((packed)) ; struct core_reloc_bitfields___err_too_big_bitfield { @@ -776,8 +783,8 @@ struct core_reloc_bitfields___err_too_big_bitfield { uint32_t ub7; uint32_t sb4; uint32_t sb20; - uint32_t u32; - uint32_t s32; + uint32_t __u32; + uint32_t __s32; } __attribute__((packed)) ; /* diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_module.c b/tools/testing/selftests/bpf/progs/test_core_reloc_module.c index f59f175c7bafb6..4dab095aebb0f4 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_module.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_module.c @@ -35,6 +35,13 @@ struct core_reloc_module_output { int comm_len; }; +struct core_reloc_mod_id_output { + int testmod_btf_obj_id; + int testmod_type_id; + char comm[sizeof("test_progs")]; + int comm_len; +}; + SEC("raw_tp/bpf_testmod_test_read") int BPF_PROG(test_core_module_probed, struct task_struct *task, @@ -102,3 +109,45 @@ int BPF_PROG(test_core_module_direct, return 0; } + +SEC("raw_tp/bpf_testmod_test_read") +int BPF_PROG(test_core_module_id_probed, + struct task_struct *task, + struct bpf_testmod_test_read_ctx *read_ctx) +{ +#if __has_builtin(__builtin_preserve_enum_value) + struct core_reloc_mod_id_output *out = (void *)&data.out; + u64 full_btf_id; + + full_btf_id = bpf_core_type_id_kernel(struct bpf_testmod_test_read_ctx); + out->testmod_btf_obj_id = full_btf_id >> 32; + out->testmod_type_id = full_btf_id & 0xFFFFFFFF; + + out->comm_len = BPF_CORE_READ_STR_INTO(&out->comm, task, comm); +#else + data.skip = true; +#endif + + return 0; +} + +SEC("tp_btf/bpf_testmod_test_read") +int BPF_PROG(test_core_module_id_direct, + struct task_struct *task, + struct bpf_testmod_test_read_ctx *read_ctx) +{ +#if __has_builtin(__builtin_preserve_enum_value) + struct core_reloc_mod_id_output *out = (void *)&data.out; + u64 full_btf_id; + + full_btf_id = bpf_core_type_id_kernel(struct bpf_testmod_test_read_ctx); + out->testmod_btf_obj_id = full_btf_id >> 32; + out->testmod_type_id = full_btf_id & 0xFFFFFFFF; + + out->comm_len = BPF_CORE_READ_STR_INTO(&out->comm, task, comm); +#else + data.skip = true; +#endif + + return 0; +} \ No newline at end of file