The first draft of complete basic XDP series for IDPF upstream

drivers/net/ethernet/intel/idpf/idpf.h

@@ -16,6 +16,9 @@ struct idpf_vport_max_q;
 #include <linux/bitfield.h>
 #include <linux/sctp.h>
 #include <linux/ethtool.h>
+#include <linux/bpf_trace.h>
+#include <linux/filter.h>
+#include <linux/bpf.h>
 #include <net/gro.h>
 #include <linux/dim.h>
 
@@ -376,6 +379,13 @@ struct idpf_vport {
 	struct idpf_queue **txqs;
 	bool crc_enable;
 
+	int num_xdp_txq;
+	int num_xdp_rxq;
+	int num_xdp_complq;
+	int xdp_txq_offset;
+	int xdp_rxq_offset;
+	int xdp_complq_offset;
+
 	u16 num_rxq;
 	u16 num_bufq;
 	u32 rxq_desc_count;
@@ -423,6 +433,7 @@ struct idpf_vport {
  * @__IDPF_USER_FLAGS_NBITS: Must be last
  */
 enum idpf_user_flags {
+	__IDPF_PRIV_FLAGS_HDR_SPLIT = 0,
 	__IDPF_PROMISC_UC = 32,
 	__IDPF_PROMISC_MC,
 
@@ -466,6 +477,8 @@ struct idpf_vport_user_config_data {
 	u16 num_req_rx_qs;
 	u32 num_req_txq_desc;
 	u32 num_req_rxq_desc;
+	/* Duplicated in queue structure for performance reasons */
+	struct bpf_prog *xdp_prog;
 	DECLARE_BITMAP(user_flags, __IDPF_USER_FLAGS_NBITS);
 	struct list_head mac_filter_list;
 };
@@ -684,6 +697,18 @@ static inline int idpf_is_queue_model_split(u16 q_model)
 	return q_model == VIRTCHNL2_QUEUE_MODEL_SPLIT;
 }
 
+/**
+ * idpf_xdp_is_prog_ena - check if there is an XDP program on adapter
+ * @vport: vport to check
+ */
+static inline bool idpf_xdp_is_prog_ena(struct idpf_vport *vport)
+{
+	if (!vport->adapter)
+		return false;
+
+	return !!vport->adapter->vport_config[vport->idx]->user_config.xdp_prog;
+}
+
 #define idpf_is_cap_ena(adapter, field, flag) \
 	idpf_is_capability_ena(adapter, false, field, flag)
 #define idpf_is_cap_ena_all(adapter, field, flag) \
@@ -939,6 +964,7 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter, u32 op,
 int idpf_send_mb_msg(struct idpf_adapter *adapter, u32 op,
 		     u16 msg_size, u8 *msg);
 void idpf_set_ethtool_ops(struct net_device *netdev);
+void idpf_vport_set_hsplit(struct idpf_vport *vport, bool ena);
 int idpf_vport_alloc_max_qs(struct idpf_adapter *adapter,
 			    struct idpf_vport_max_q *max_q);
 void idpf_vport_dealloc_max_qs(struct idpf_adapter *adapter,

drivers/net/ethernet/intel/idpf/idpf_ethtool.c

@@ -900,6 +900,8 @@ static void idpf_get_ethtool_stats(struct net_device *netdev,
 
 			if (!txq)
 				idpf_add_empty_queue_stats(&data, qtype);
+			else if (test_bit(__IDPF_Q_XDP, txq->flags))
+				continue;
 			else
 				idpf_add_queue_stats(&data, txq);
 		}

drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h

@@ -184,13 +184,17 @@ struct idpf_base_tx_desc {
 	__le64 qw1; /* type_cmd_offset_bsz_l2tag1 */
 }; /* read used with buffer queues */
 
-struct idpf_splitq_tx_compl_desc {
+struct idpf_splitq_4b_tx_compl_desc {
 	/* qid=[10:0] comptype=[13:11] rsvd=[14] gen=[15] */
 	__le16 qid_comptype_gen;
 	union {
 		__le16 q_head; /* Queue head */
 		__le16 compl_tag; /* Completion tag */
 	} q_head_compl_tag;
+}; /* writeback used with completion queues */
+
+struct idpf_splitq_tx_compl_desc {
+	struct idpf_splitq_4b_tx_compl_desc common;
 	u8 ts[3];
 	u8 rsvd; /* Reserved */
 }; /* writeback used with completion queues */

drivers/net/ethernet/intel/idpf/idpf_lib.c

@@ -839,6 +839,7 @@ static int idpf_cfg_netdev(struct idpf_vport *vport)
 	netdev->features |= dflt_features;
 	netdev->hw_features |= dflt_features | offloads;
 	netdev->hw_enc_features |= dflt_features | offloads;
+	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
 	idpf_set_ethtool_ops(netdev);
 	SET_NETDEV_DEV(netdev, &adapter->pdev->dev);
 
@@ -1057,6 +1058,27 @@ static void idpf_vport_dealloc(struct idpf_vport *vport)
 	adapter->next_vport = idpf_get_free_slot(adapter);
 }
 
+/**
+ * idpf_vport_set_hsplit - enable or disable header split on a given vport
+ * @vport: virtual port
+ * @ena: flag controlling header split, On (true) or Off (false)
+ */
+void idpf_vport_set_hsplit(struct idpf_vport *vport, bool ena)
+{
+	struct idpf_vport_user_config_data *config_data;
+
+	config_data = &vport->adapter->vport_config[vport->idx]->user_config;
+	if (!ena) {
+		clear_bit(__IDPF_PRIV_FLAGS_HDR_SPLIT, config_data->user_flags);
+		return;
+	}
+
+	if (idpf_is_cap_ena_all(vport->adapter, IDPF_HSPLIT_CAPS,
+				IDPF_CAP_HSPLIT) &&
+	    idpf_is_queue_model_split(vport->rxq_model))
+		set_bit(__IDPF_PRIV_FLAGS_HDR_SPLIT, config_data->user_flags);
+}
+
 /**
  * idpf_vport_alloc - Allocates the next available struct vport in the adapter
  * @adapter: board private structure
@@ -1234,13 +1256,18 @@ static void idpf_restore_features(struct idpf_vport *vport)
  */
 static int idpf_set_real_num_queues(struct idpf_vport *vport)
 {
-	int err;
+	int num_txq, err;
 
 	err = netif_set_real_num_rx_queues(vport->netdev, vport->num_rxq);
 	if (err)
 		return err;
 
-	return netif_set_real_num_tx_queues(vport->netdev, vport->num_txq);
+	if (idpf_xdp_is_prog_ena(vport))
+		num_txq = vport->num_txq - vport->num_xdp_txq;
+	else
+		num_txq = vport->num_txq;
+
+	return netif_set_real_num_tx_queues(vport->netdev, num_txq);
 }
 
 /**
@@ -1353,6 +1380,15 @@ static int idpf_vport_open(struct idpf_vport *vport, bool alloc_res)
 
 	idpf_rx_init_buf_tail(vport);
 
+	if (idpf_xdp_is_prog_ena(vport)) {
+		err = idpf_xdp_rxq_info_init_all(vport);
+		if (err) {
+			dev_err(&adapter->pdev->dev, "Failed to initialize XDP info for vport %u, %d\n",
+				vport->vport_id, err);
+			goto intr_deinit;
+		}
+	}
+
 	err = idpf_send_config_queues_msg(vport);
 	if (err) {
 		dev_err(&adapter->pdev->dev, "Failed to configure queues for vport %u, %d\n",
@@ -2192,10 +2228,18 @@ static int idpf_change_mtu(struct net_device *netdev, int new_mtu)
 	idpf_vport_ctrl_lock(netdev);
 	vport = idpf_netdev_to_vport(netdev);
 
+	if (idpf_xdp_is_prog_ena(vport) && new_mtu > IDPF_XDP_MAX_MTU) {
+		netdev_err(netdev, "New MTU value is not valid. The maximum MTU value is %d.\n",
+			   IDPF_XDP_MAX_MTU);
+		err = -EINVAL;
+		goto unlock_exit;
+	}
+
 	netdev->mtu = new_mtu;
 
 	err = idpf_initiate_soft_reset(vport, IDPF_SR_MTU_CHANGE);
 
+unlock_exit:
 	idpf_vport_ctrl_unlock(netdev);
 
 	return err;
@@ -2262,6 +2306,198 @@ static netdev_features_t idpf_features_check(struct sk_buff *skb,
 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
 }
 
+/**
+ * idpf_copy_xdp_prog_to_qs - set pointers to xdp program for each Rx queue
+ * @vport: vport to setup XDP for
+ * @xdp_prog: XDP program that should be copied to all Rx queues
+ */
+static void
+idpf_copy_xdp_prog_to_qs(struct idpf_vport *vport, struct bpf_prog *xdp_prog)
+{
+	int i;
+
+	for (i = 0; i < vport->num_rxq_grp; i++) {
+		struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
+		struct idpf_queue *q;
+		u16 j, num_rxq;
+
+		if (idpf_is_queue_model_split(vport->rxq_model))
+			num_rxq = rx_qgrp->splitq.num_rxq_sets;
+		else
+			num_rxq = rx_qgrp->singleq.num_rxq;
+
+		for (j = 0; j < num_rxq; j++) {
+			if (idpf_is_queue_model_split(vport->rxq_model))
+				q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
+			else
+				q = rx_qgrp->singleq.rxqs[j];
+			WRITE_ONCE(q->xdp_prog, xdp_prog);
+		}
+
+		if (!idpf_is_queue_model_split(vport->rxq_model))
+			continue;
+
+		for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
+			q = &rx_qgrp->splitq.bufq_sets[j].bufq;
+			WRITE_ONCE(q->xdp_prog, xdp_prog);
+		}
+	}
+}
+
+/**
+ * idpf_xdp_reconfig_queues - reconfigure queues after the XDP setup
+ * @vport: vport to load or unload XDP for
+ */
+static int idpf_xdp_reconfig_queues(struct idpf_vport *vport)
+{
+	int err;
+
+	err = idpf_vport_adjust_qs(vport);
+	if (err) {
+		netdev_err(vport->netdev,
+			   "Could not adjust queue number for XDP\n");
+		return err;
+	}
+	idpf_vport_calc_num_q_desc(vport);
+
+	err = idpf_vport_queues_alloc(vport);
+	if (err) {
+		netdev_err(vport->netdev,
+			   "Could not allocate queues for XDP\n");
+		return err;
+	}
+
+	err = idpf_send_add_queues_msg(vport, vport->num_txq,
+				       vport->num_complq,
+				       vport->num_rxq, vport->num_bufq);
+	if (err) {
+		netdev_err(vport->netdev,
+			   "Could not add queues for XDP, VC message sent failed\n");
+		return err;
+	}
+
+	idpf_vport_alloc_vec_indexes(vport);
+
+	return 0;
+}
+
+/**
+ * idpf_assign_bpf_prog - Assign a given BPF program to vport
+ * @current_prog: pointer to XDP program in user config data
+ * @prog: BPF program to be assigned to vport
+ */
+static void idpf_assign_bpf_prog(struct bpf_prog **current_prog,
+				 struct bpf_prog *prog)
+{
+	struct bpf_prog *old_prog;
+
+	old_prog = xchg(current_prog, prog);
+	if (old_prog)
+		bpf_prog_put(old_prog);
+}
+
+/**
+ * idpf_xdp_setup_prog - Add or remove XDP eBPF program
+ * @vport: vport to setup XDP for
+ * @prog: XDP program
+ * @extack: netlink extended ack
+ */
+static int
+idpf_xdp_setup_prog(struct idpf_vport *vport, struct bpf_prog *prog,
+		    struct netlink_ext_ack *extack)
+{
+	struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
+	int frame_size = vport->netdev->mtu;
+	bool needs_reconfig, vport_is_up;
+	struct bpf_prog **current_prog;
+	u16 idx = vport->idx;
+	int err;
+
+	if (frame_size > IDPF_XDP_MAX_MTU ||
+	    frame_size > vport->bufq_size[0]) {
+		NL_SET_ERR_MSG_MOD(extack, "MTU too large for loading XDP");
+		return -EOPNOTSUPP;
+	}
+
+	vport_is_up = np->state == __IDPF_VPORT_UP;
+
+	current_prog = &vport->adapter->vport_config[idx]->user_config.xdp_prog;
+	needs_reconfig = !!(*current_prog) != !!prog;
+
+	if (!needs_reconfig) {
+		idpf_copy_xdp_prog_to_qs(vport, prog);
+		idpf_assign_bpf_prog(current_prog, prog);
+
+		return 0;
+	}
+
+	if (!vport_is_up) {
+		idpf_send_delete_queues_msg(vport);
+	} else {
+		set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags);
+		idpf_vport_stop(vport);
+	}
+
+	idpf_deinit_rss(vport);
+
+	if (!*current_prog && prog) {
+		netdev_warn(vport->netdev,
+			    "Setting up XDP disables header split\n");
+		idpf_vport_set_hsplit(vport, false);
+	} else {
+		idpf_vport_set_hsplit(vport, true);
+	}
+
+	idpf_assign_bpf_prog(current_prog, prog);
+
+	err = idpf_xdp_reconfig_queues(vport);
+	if (err) {
+		netdev_err(vport->netdev,
+			   "Could not reconfigure the queues after XDP setup\n");
+		return err;
+	}
+	if (prog)
+		xdp_features_set_redirect_target(vport->netdev, true);
+	else
+		xdp_features_clear_redirect_target(vport->netdev);
+
+	if (vport_is_up) {
+		err = idpf_vport_open(vport, false);
+		if (err) {
+			netdev_err(vport->netdev,
+				   "Could not re-open the vport after XDP setup\n");
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * idpf_xdp - implements XDP handler
+ * @netdev: netdevice
+ * @xdp: XDP command
+ */
+static int idpf_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
+{
+	struct idpf_vport *vport;
+	int err;
+
+	idpf_vport_ctrl_lock(netdev);
+	vport = idpf_netdev_to_vport(netdev);
+
+	switch (xdp->command) {
+	case XDP_SETUP_PROG:
+		err = idpf_xdp_setup_prog(vport, xdp->prog, xdp->extack);
+		break;
+	default:
+		err = -EINVAL;
+	}
+
+	idpf_vport_ctrl_unlock(netdev);
+	return err;
+}
+
 /**
  * idpf_set_mac - NDO callback to set port mac address
  * @netdev: network interface device structure
@@ -2362,6 +2598,8 @@ static const struct net_device_ops idpf_netdev_ops_splitq = {
 	.ndo_get_stats64 = idpf_get_stats64,
 	.ndo_set_features = idpf_set_features,
 	.ndo_tx_timeout = idpf_tx_timeout,
+	.ndo_bpf = idpf_xdp,
+	.ndo_xdp_xmit = idpf_xdp_xmit,
 };
 
 static const struct net_device_ops idpf_netdev_ops_singleq = {