From 7bc88941b77c70891663669e5d1f5f8f544c6a7f Mon Sep 17 00:00:00 2001 From: Dong Yibo Date: Sun, 27 Oct 2024 15:44:28 +0800 Subject: [PATCH] net: mucse: initial support for rnpvf driver from Mucse Technology This driver is to support mucse n10 virtual function Signed-off-by: Dong Yibo --- .../configs/deepin_arm64_desktop_defconfig | 4 + .../deepin_loongarch_desktop_defconfig | 4 + arch/x86/configs/deepin_x86_desktop_defconfig | 4 + drivers/net/ethernet/mucse/Kconfig | 37 + drivers/net/ethernet/mucse/Makefile | 1 + drivers/net/ethernet/mucse/rnpvf/Makefile | 15 + drivers/net/ethernet/mucse/rnpvf/defines.h | 367 + drivers/net/ethernet/mucse/rnpvf/ethtool.c | 781 ++ drivers/net/ethernet/mucse/rnpvf/mbx.c | 624 ++ drivers/net/ethernet/mucse/rnpvf/mbx.h | 119 + drivers/net/ethernet/mucse/rnpvf/regs.h | 141 + drivers/net/ethernet/mucse/rnpvf/rnpvf.h | 738 ++ drivers/net/ethernet/mucse/rnpvf/rnpvf_main.c | 6434 +++++++++++++++++ drivers/net/ethernet/mucse/rnpvf/sysfs.c | 21 + drivers/net/ethernet/mucse/rnpvf/vf.c | 849 +++ drivers/net/ethernet/mucse/rnpvf/vf.h | 203 + 16 files changed, 10342 insertions(+) create mode 100644 drivers/net/ethernet/mucse/rnpvf/Makefile create mode 100644 drivers/net/ethernet/mucse/rnpvf/defines.h create mode 100644 drivers/net/ethernet/mucse/rnpvf/ethtool.c create mode 100644 drivers/net/ethernet/mucse/rnpvf/mbx.c create mode 100644 drivers/net/ethernet/mucse/rnpvf/mbx.h create mode 100644 drivers/net/ethernet/mucse/rnpvf/regs.h create mode 100644 drivers/net/ethernet/mucse/rnpvf/rnpvf.h create mode 100644 drivers/net/ethernet/mucse/rnpvf/rnpvf_main.c create mode 100644 drivers/net/ethernet/mucse/rnpvf/sysfs.c create mode 100644 drivers/net/ethernet/mucse/rnpvf/vf.c create mode 100644 drivers/net/ethernet/mucse/rnpvf/vf.h diff --git a/arch/arm64/configs/deepin_arm64_desktop_defconfig b/arch/arm64/configs/deepin_arm64_desktop_defconfig index 39361b91b3493..c46969dbe09cb 100644 --- a/arch/arm64/configs/deepin_arm64_desktop_defconfig +++ b/arch/arm64/configs/deepin_arm64_desktop_defconfig @@ -1419,6 +1419,10 @@ CONFIG_MXGBE_FIX_MAC_PADDING=y # CONFIG_MXGBE_OPTM_WITH_LARGE is not set CONFIG_MXGBE_MSIX_COUNT=64 CONFIG_MXGBE_DCB=y +CONFIG_MXGBEVF=m +CONFIG_MXGBEVF_FIX_VF_QUEUE=y +CONFIG_MXGBEVF_FIX_MAC_PADDING=y +# CONFIG_MXGBEVF_OPTM_WITH_LARGE is not set CONFIG_JME=m CONFIG_ADIN1110=m CONFIG_LITEX_LITEETH=m diff --git a/arch/loongarch/configs/deepin_loongarch_desktop_defconfig b/arch/loongarch/configs/deepin_loongarch_desktop_defconfig index 029e742f9f953..423155c0ae4c1 100644 --- a/arch/loongarch/configs/deepin_loongarch_desktop_defconfig +++ b/arch/loongarch/configs/deepin_loongarch_desktop_defconfig @@ -1363,6 +1363,10 @@ CONFIG_MXGBE_FIX_MAC_PADDING=y # CONFIG_MXGBE_OPTM_WITH_LARGE is not set CONFIG_MXGBE_MSIX_COUNT=64 CONFIG_MXGBE_DCB=y +CONFIG_MXGBEVF=m +CONFIG_MXGBEVF_FIX_VF_QUEUE=y +CONFIG_MXGBEVF_FIX_MAC_PADDING=y +# CONFIG_MXGBEVF_OPTM_WITH_LARGE is not set CONFIG_JME=m CONFIG_ADIN1110=m CONFIG_LITEX_LITEETH=m diff --git a/arch/x86/configs/deepin_x86_desktop_defconfig b/arch/x86/configs/deepin_x86_desktop_defconfig index 1349c308395ec..d41829bb1b49d 100644 --- a/arch/x86/configs/deepin_x86_desktop_defconfig +++ b/arch/x86/configs/deepin_x86_desktop_defconfig @@ -1322,6 +1322,10 @@ CONFIG_MXGBE_FIX_MAC_PADDING=y # CONFIG_MXGBE_OPTM_WITH_LARGE is not set CONFIG_MXGBE_MSIX_COUNT=64 CONFIG_MXGBE_DCB=y +CONFIG_MXGBEVF=m +CONFIG_MXGBEVF_FIX_VF_QUEUE=y +CONFIG_MXGBEVF_FIX_MAC_PADDING=y +# CONFIG_MXGBEVF_OPTM_WITH_LARGE is not set CONFIG_JME=m CONFIG_ADIN1110=m CONFIG_MVMDIO=m diff --git a/drivers/net/ethernet/mucse/Kconfig b/drivers/net/ethernet/mucse/Kconfig index ef10f416ed565..ec5690a581ac6 100644 --- a/drivers/net/ethernet/mucse/Kconfig +++ b/drivers/net/ethernet/mucse/Kconfig @@ -119,5 +119,42 @@ config MXGBE_DCB If unsure, say N. +config MXGBEVF + tristate "Mucse(R) 1/10/25/40GbE PCI Express Virtual Function adapters support" + depends on PCI + help + This driver supports Mucse(R) 1/10/25/40GbE PCI Express Virtual Function + family of adapters. + + To compile this driver as a module, choose M here. The module + will be called rnpvf. + +config MXGBEVF_FIX_VF_QUEUE + bool "Fix VF Queue Used(vf)" + default y + depends on MXGBEVF + help + Say Y here if you want to fix vf queue order in the driver. + + If unsure, say N. + +config MXGBEVF_FIX_MAC_PADDING + bool "Close Mac Padding Function(pf)" + default y + depends on MXGBEVF + help + Say Y here if you want to fix close mac padding in the driver. + + If unsure, say N. + +config MXGBEVF_OPTM_WITH_LARGE + bool "Reduce Memory Cost In Large PAGE_SIZE(>8192)" + default n + depends on MXGBEVF + help + Say Y here if you want to reduce memory cost in large PAGE_SIZE. + + If unsure, say N. + endif # NET_VENDOR_MUCSE diff --git a/drivers/net/ethernet/mucse/Makefile b/drivers/net/ethernet/mucse/Makefile index 486184e693bec..c1357aab64c86 100644 --- a/drivers/net/ethernet/mucse/Makefile +++ b/drivers/net/ethernet/mucse/Makefile @@ -6,3 +6,4 @@ obj-$(CONFIG_MGBE) += rnpgbe/ obj-$(CONFIG_MGBEVF) += rnpgbevf/ obj-$(CONFIG_MXGBE) += rnp/ +obj-$(CONFIG_MXGBEVF) += rnpvf/ diff --git a/drivers/net/ethernet/mucse/rnpvf/Makefile b/drivers/net/ethernet/mucse/rnpvf/Makefile new file mode 100644 index 0000000000000..a9593401e4d7e --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2022 - 2024 Mucse Corporation +# +# Makefile for the Mucse(R) 10GbE-2ports PCI Express ethernet driver +# +# + +obj-$(CONFIG_MXGBEVF) += rnpvf.o +rnpvf-objs := \ + vf.o \ + mbx.o \ + ethtool.o \ + sysfs.o \ + rnpvf_main.o + diff --git a/drivers/net/ethernet/mucse/rnpvf/defines.h b/drivers/net/ethernet/mucse/rnpvf/defines.h new file mode 100644 index 0000000000000..33dc3deb02cc2 --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/defines.h @@ -0,0 +1,367 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#ifndef _RNPVF_DEFINES_H_ +#define _RNPVF_DEFINES_H_ + +#include +#include +/* Device IDs */ +#define RNP_DEV_ID_N10_PF0_VF 0x8001 +#define RNP_DEV_ID_N10_PF1_VF 0x8002 + +#define RNP_DEV_ID_N10_PF0_VF_N 0x1010 +#define RNP_DEV_ID_N10_PF1_VF_N 0x1011 + +#define RNP_VF_IRQ_CLEAR_MASK 7 +#define RNP_VF_MAX_TX_QUEUES 8 +#define RNP_VF_MAX_RX_QUEUES 8 + +/* DCB define */ +#define RNP_VF_MAX_TRAFFIC_CLASS 8 + +/* Link speed */ +typedef u32 rnp_link_speed; +#define RNP_LINK_SPEED_UNKNOWN 0 +#define RNP_LINK_SPEED_10_FULL BIT(2) +#define RNP_LINK_SPEED_100_FULL BIT(3) +#define RNP_LINK_SPEED_1GB_FULL BIT(4) +#define RNP_LINK_SPEED_10GB_FULL BIT(5) +#define RNP_LINK_SPEED_40GB_FULL BIT(6) +#define RNP_LINK_SPEED_25GB_FULL BIT(7) +#define RNP_LINK_SPEED_50GB_FULL BIT(8) +#define RNP_LINK_SPEED_100GB_FULL BIT(9) +#define RNP_LINK_SPEED_10_HALF BIT(10) +#define RNP_LINK_SPEED_100_HALF BIT(11) +#define RNP_LINK_SPEED_1GB_HALF BIT(12) +#define RNP_SFP_MODE_10G_LR BIT(13) +#define RNP_SFP_MODE_10G_SR BIT(14) +#define RNP_SFP_MODE_10G_LRM BIT(15) +#define RNP_SFP_MODE_1G_T BIT(16) +#define RNP_SFP_MODE_1G_KX BIT(17) +#define RNP_SFP_MODE_1G_SX BIT(18) +#define RNP_SFP_MODE_1G_LX BIT(19) +#define RNP_SFP_MODE_40G_SR4 BIT(20) +#define RNP_SFP_MODE_40G_CR4 BIT(21) +#define RNP_SFP_MODE_40G_LR4 BIT(22) +#define RNP_SFP_MODE_1G_CX BIT(23) + +/* Number of Transmit and Receive Descriptors must be a multiple of 8 */ +#define RNP_REQ_TX_DESCRIPTOR_MULTIPLE 8 +#define RNP_REQ_RX_DESCRIPTOR_MULTIPLE 8 +#define RNP_REQ_TX_BUFFER_GRANULARITY 1024 + +/* Interrupt Vector Allocation Registers */ +#define RNP_IVAR_ALLOC_VAL 0x80 /* Interrupt Allocation valid */ + +#define RNP_VF_INIT_TIMEOUT 200 /* Number of retries to clear RSTI */ + +/* Transmit Descriptor - Advanced */ +struct rnp_tx_desc { + union { + __le64 pkt_addr; /* Packet buffer address */ + struct { + __le32 adr_lo; + __le32 adr_hi; + }; + }; + __le16 blen; + union { + struct { + __le16 ip_len : 9; + __le16 mac_len : 7; + }; + __le16 mac_ip_len; + }; + __le16 vlan; +#define RNP_TXD_FLAGS_VLAN_PRIO_MASK 0xe000 +#define RNP_TX_FLAGS_VLAN_PRIO_SHIFT 13 +#define RNP_TX_FLAGS_VLAN_CFI_SHIFT 12 + + __le16 cmd; +#define RNP_TXD_VLAN_VALID (1 << 15) +#define RNP_TXD_SVLAN_TYPE (1 << 14) +#define RNP_TXD_VLAN_CTRL_NOP (0x00 << 13) +#define RNP_TXD_VLAN_CTRL_RM_VLAN (0x01 << 13) +#define RNP_TXD_VLAN_CTRL_INSERT_VLAN (0x02 << 13) +#define RNP_TXD_L4_CSUM (1 << 12) +#define RNP_TXD_IP_CSUM (1 << 11) +#define RNP_TXD_TUNNEL_MASK (0x3000000) +#define RNP_TXD_TUNNEL_VXLAN (0x01 << 8) +#define RNP_TXD_TUNNEL_NVGRE (0x02 << 8) +#define RNP_TXD_L4_TYPE_UDP (0x03 << 6) +#define RNP_TXD_L4_TYPE_TCP (0x01 << 6) +#define RNP_TXD_L4_TYPE_SCTP (0x02 << 6) +#define RNP_TXD_FLAG_IPv4 (0 << 5) +#define RNP_TXD_FLAG_IPv6 (1 << 5) +#define RNP_TXD_FLAG_TSO (1 << 4) +#define RNP_TXD_CMD_RS (1 << 2) +#define RNP_TXD_STAT_DD (1 << 1) +#define RNP_TXD_CMD_EOP (1 << 0) +} __packed; + +struct rnp_tx_ctx_desc { + __le16 mss_len; + u8 vfnum; + u8 l4_hdr_len; + u8 tunnel_hdr_len; + __le16 inner_vlan; + u8 vf_veb_flags; +#define VF_IGNORE_VLAN (1 << 1) /* bit 57 */ +#define VF_VEB_MARK (1 << 0) /* bit 56 */ + __le32 res; + __le32 resv_cmd; +#define RNP_TXD_FLAG_TO_RPU (1 << 15) +#define RNP_TXD_SMAC_CTRL_NOP (0x00 << 12) +#define RNP_TXD_SMAC_CTRL_REPLACE_MACADDR0 (0x02 << 12) +#define RNP_TXD_SMAC_CTRL_REPLACE_MACADDR1 (0bx06 << 12) +#define RNP_TXD_CTX_VLAN_CTRL_NOP (0x00 << 10) +#define RNP_TXD_CTX_VLAN_CTRL_RM_VLAN (0x01 << 10) +#define RNP_TXD_CTX_VLAN_CTRL_INSERT_VLAN (0x02 << 10) +#define RNP_TXD_MTI_CRC_PAD_CTRL (0x01000000) +#define RNP_TXD_CTX_CTRL_DESC (0x080000) +#define RNP_TXD_CTX_CMD_RS (1 << 2) +#define RNP_TXD_STAT_DD (1 << 1) +} __packed; + +/* Receive Descriptor - Advanced */ +union rnp_rx_desc { + struct { + union { + __le64 pkt_addr; /* Packet buffer address */ + struct { + __le32 addr_lo; + __le32 addr_hi; + }; + }; + u8 dumy[6]; + __le16 cmd; /* DD back */ +#define RNP_RXD_FLAG_RS (1 << 2) + }; + + struct { + __le32 rss_hash; + __le16 mark; + __le16 rev1; +#define RNP_RX_L3_TYPE_MASK (1 << 15) /* 1 is ipv4 */ +#define VEB_VF_PKG (1 << 0) /* bit 48 */ +#define VEB_VF_IGNORE_VLAN (1 << 1) /* bit 49 */ + __le16 len; + __le16 padding_len; + __le16 vlan; + __le16 cmd; +#define RNP_RXD_STAT_VLAN_VALID (1 << 15) +#define RNP_RXD_STAT_STAG (0x01 << 14) +#define RNP_RXD_STAT_TUNNEL_NVGRE (0x02 << 13) +#define RNP_RXD_STAT_TUNNEL_VXLAN (0x01 << 13) +#define RNP_RXD_STAT_ERR_MASK (0x1f << 8) +#define RNP_RXD_STAT_TUNNEL_MASK (0x03 << 13) +#define RNP_RXD_STAT_SCTP_MASK (0x04 << 8) +#define RNP_RXD_STAT_L4_MASK (0x02 << 8) +#define RNP_RXD_STAT_ERR_MASK_NOSCTP (0x1b << 8) +#define RNP_RXD_STAT_L4_SCTP (0x02 << 6) +#define RNP_RXD_STAT_L4_TCP (0x01 << 6) +#define RNP_RXD_STAT_L4_UDP (0x03 << 6) +#define RNP_RXD_STAT_IPV6 (1 << 5) +#define RNP_RXD_STAT_IPV4 (0 << 5) +#define RNP_RXD_STAT_PTP (1 << 4) +#define RNP_RXD_STAT_DD (1 << 1) +#define RNP_RXD_STAT_EOP (1 << 0) + } wb; +} __packed; + +/* Interrupt register bitmasks */ + +#define RNP_EITR_CNT_WDIS 0x80000000 +#define RNP_MAX_EITR 0x00000FF8 +#define RNP_MIN_EITR 8 + +/* Error Codes */ +#define RNP_ERR_INVALID_MAC_ADDR -1 +#define RNP_ERR_RESET_FAILED -2 +#define RNP_ERR_INVALID_ARGUMENT -3 + +#ifdef DEBUG +#define dbg(fmt, args...) \ + printk(KERN_DEBUG "[ %s:%d ] " fmt, __func__, __LINE__, ##args) +#else +#define dbg(fmt, args...) +#endif + +#define rnpvf_dbg(fmt, args...) printk(KERN_DEBUG fmt, ##args) +#define rnpvf_info(fmt, args...) \ + printk(KERN_DEBUG "rnpvf-info: " fmt, ##args) +#define rnpvf_warn(fmt, args...) \ + printk(KERN_DEBUG "rnpvf-warn: " fmt, ##args) +#define rnpvf_err(fmt, args...) printk(KERN_ERR "rnpvf-err : " fmt, ##args) + +#define DPRINTK(nlevel, klevel, fmt, args...) \ + ((NETIF_MSG_##nlevel & adapter->msg_enable) ? \ + (void)(netdev_printk(KERN_##klevel, adapter->netdev, \ + fmt, ##args)) : \ + NULL) + +#ifdef CONFIG_RNP_TX_DEBUG +static inline void buf_dump_line(const char *msg, int line, void *buf, + int len) +{ + int i, offset = 0; + int msg_len = 1024; + u8 msg_buf[1024]; + u8 *ptr = (u8 *)buf; + + offset += snprintf(msg_buf + offset, msg_len, + "=== %s #%d line:%d buf:%p==\n000: ", msg, len, + line, buf); + + for (i = 0; i < len; ++i) { + if ((i != 0) && (i % 16) == 0 && + (offset >= (1024 - 10 * 16))) { + printk("%s\n", msg_buf); + offset = 0; + } + + if ((i != 0) && (i % 16) == 0) { + offset += snprintf(msg_buf + offset, msg_len, + "\n%03x: ", i); + } + offset += snprintf(msg_buf + offset, msg_len, "%02x ", + ptr[i]); + } + + offset += snprintf(msg_buf + offset, msg_len, "\n"); + printk(KERN_DEBUG "%s\n", msg_buf); +} +#else +#define buf_dump_line(msg, line, buf, len) +#endif + +static inline void buf_dump(const char *msg, void *buf, int len) +{ + int i, offset = 0; + int msg_len = 1024; + u8 msg_buf[1024]; + u8 *ptr = (u8 *)buf; + + offset += snprintf(msg_buf + offset, msg_len, + "=== %s #%d ==\n000: ", msg, len); + + for (i = 0; i < len; ++i) { + if ((i != 0) && (i % 16) == 0 && + (offset >= (1024 - 10 * 16))) { + printk("%s\n", msg_buf); + offset = 0; + } + + if ((i != 0) && (i % 16) == 0) { + offset += snprintf(msg_buf + offset, msg_len, + "\n%03x: ", i); + } + offset += snprintf(msg_buf + offset, msg_len, "%02x ", + ptr[i]); + } + + offset += snprintf(msg_buf + offset, msg_len, "\n=== done ==\n"); + printk(KERN_DEBUG "%s\n", msg_buf); +} + +static inline void _rnp_skb_dump(const struct sk_buff *skb, bool full_pkt) +{ + static atomic_t can_dump_full = ATOMIC_INIT(5); + struct skb_shared_info *sh = skb_shinfo(skb); + struct net_device *dev = skb->dev; + struct sk_buff *list_skb; + bool has_mac, has_trans; + int headroom, tailroom; + int i, len, seg_len; + const char *level = KERN_WARNING; + + if (full_pkt) + full_pkt = atomic_dec_if_positive(&can_dump_full) >= 0; + + if (full_pkt) + len = skb->len; + else + len = min_t(int, skb->len, MAX_HEADER + 128); + + headroom = skb_headroom(skb); + tailroom = skb_tailroom(skb); + + has_mac = skb_mac_header_was_set(skb); + has_trans = skb_transport_header_was_set(skb); + + printk(KERN_DEBUG + "%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" + "mac=(%d,%d) net=(%d,%d) trans=%d\n" + "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" + "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" + "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", + level, skb->len, headroom, skb_headlen(skb), tailroom, + has_mac ? skb->mac_header : -1, + has_mac ? (skb->network_header - skb->mac_header) : -1, + skb->network_header, + has_trans ? skb_network_header_len(skb) : -1, + has_trans ? skb->transport_header : -1, sh->tx_flags, + sh->nr_frags, sh->gso_size, sh->gso_type, sh->gso_segs, + skb->csum, skb->ip_summed, skb->csum_complete_sw, + skb->csum_valid, skb->csum_level, skb->hash, skb->sw_hash, + skb->l4_hash, ntohs(skb->protocol), skb->pkt_type, + skb->skb_iif); + + if (dev) + printk(KERN_DEBUG "%sdev name=%s feat=0x%pNF\n", level, + dev->name, &dev->features); + + seg_len = min_t(int, skb_headlen(skb), len); + if (seg_len) + print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, + 16, 1, skb->data, seg_len, false); + len -= seg_len; + + for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + u32 p_len; + struct page *p; + u8 *vaddr; + + p = skb_frag_address(frag); + p_len = skb_frag_size(frag); + seg_len = min_t(int, p_len, len); + vaddr = kmap_atomic(p); + print_hex_dump(level, "skb frag: ", DUMP_PREFIX_OFFSET, + 16, 1, vaddr, seg_len, false); + kunmap_atomic(vaddr); + len -= seg_len; + if (!len) + break; + } + + if (full_pkt && skb_has_frag_list(skb)) { + printk(KERN_DEBUG "skb fraglist:\n"); + skb_walk_frags(skb, list_skb) + _rnp_skb_dump(list_skb, true); + } +} + +#define TRACE() printk(KERN_DEBUG "=[%s] %d == \n", __func__, __LINE__) + +#ifdef CONFIG_RNP_TX_DEBUG +#define desc_hex_dump(msg, buf, len) \ + print_hex_dump(KERN_WARNING, msg, DUMP_PREFIX_OFFSET, 16, 1, \ + (buf), (len), false) +#define rnpvf_skb_dump _rnp_skb_dump +#else +#define desc_hex_dump(msg, buf, len) +#define rnpvf_skb_dump(skb, full_pkt) +#endif + + +#ifdef CONFIG_RNP_RX_DEBUG +#define rx_debug_printk printk +#define rx_buf_dump buf_dump +#else +#define rx_debug_printk(fmt, args...) +#define rx_buf_dump(a, b, c) +#endif //CONFIG_RNP_RX_DEBUG + +#endif /* _RNPVF_DEFINES_H_ */ diff --git a/drivers/net/ethernet/mucse/rnpvf/ethtool.c b/drivers/net/ethernet/mucse/rnpvf/ethtool.c new file mode 100644 index 0000000000000..9bac619f03599 --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/ethtool.c @@ -0,0 +1,781 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rnpvf.h" + +#define RNP_ALL_RAR_ENTRIES 16 + +struct rnpvf_stats { + char stat_string[ETH_GSTRING_LEN]; + int sizeof_stat; + int stat_offset; + int base_stat_offset; + int saved_reset_offset; +}; + +#define RNPVF_NUM_RX_QUEUES netdev->real_num_rx_queues +#define RNPVF_NUM_TX_QUEUES netdev->real_num_tx_queues + +#define RNP_NETDEV_STAT(_net_stat) \ + { \ + .stat_string = #_net_stat, \ + .sizeof_stat = \ + sizeof_field(struct net_device_stats, _net_stat), \ + .stat_offset = \ + offsetof(struct net_device_stats, _net_stat) \ + } + +static const struct rnpvf_stats rnp_gstrings_net_stats[] = { + RNP_NETDEV_STAT(rx_packets), + RNP_NETDEV_STAT(tx_packets), + RNP_NETDEV_STAT(rx_bytes), + RNP_NETDEV_STAT(tx_bytes), + RNP_NETDEV_STAT(rx_errors), + RNP_NETDEV_STAT(tx_errors), + RNP_NETDEV_STAT(rx_dropped), + RNP_NETDEV_STAT(tx_dropped), + RNP_NETDEV_STAT(collisions), + RNP_NETDEV_STAT(rx_over_errors), + RNP_NETDEV_STAT(rx_crc_errors), + RNP_NETDEV_STAT(rx_frame_errors), + RNP_NETDEV_STAT(rx_fifo_errors), + RNP_NETDEV_STAT(rx_missed_errors), + RNP_NETDEV_STAT(tx_aborted_errors), + RNP_NETDEV_STAT(tx_carrier_errors), + RNP_NETDEV_STAT(tx_fifo_errors), + RNP_NETDEV_STAT(tx_heartbeat_errors), +}; + +#define RNPVF_GLOBAL_STATS_LEN ARRAY_SIZE(rnp_gstrings_net_stats) + +#define RNPVF_HW_STAT(_name, _stat) \ + { \ + .stat_string = _name, \ + .sizeof_stat = sizeof_field(struct rnpvf_adapter, _stat), \ + .stat_offset = offsetof(struct rnpvf_adapter, _stat) \ + } + +static struct rnpvf_stats rnpvf_hwstrings_stats[] = { + RNPVF_HW_STAT("vlan_add_cnt", hw_stats.vlan_add_cnt), + RNPVF_HW_STAT("vlan_strip_cnt", hw_stats.vlan_strip_cnt), + RNPVF_HW_STAT("rx_csum_offload_errors", hw_stats.csum_err), + RNPVF_HW_STAT("rx_csum_offload_good", hw_stats.csum_good), + RNPVF_HW_STAT("tx_spoof_dropped", hw_stats.spoof_dropped), +}; + +#define RNPVF_HWSTRINGS_STATS_LEN ARRAY_SIZE(rnpvf_hwstrings_stats) + +struct rnpvf_tx_queue_ring_stat { + u64 hw_head; + u64 hw_tail; + u64 sw_to_clean; +}; + +struct rnpvf_rx_queue_ring_stat { + u64 hw_head; + u64 hw_tail; + u64 sw_to_use; +}; + +#define RNP_QUEUE_STATS_LEN \ + (RNPVF_NUM_TX_QUEUES * \ + (sizeof(struct rnpvf_tx_queue_stats) / sizeof(u64) + \ + sizeof(struct rnpvf_queue_stats) / sizeof(u64) + \ + sizeof(struct rnpvf_tx_queue_ring_stat) / \ + sizeof(u64)) + \ + RNPVF_NUM_RX_QUEUES * \ + (sizeof(struct rnpvf_rx_queue_stats) / sizeof(u64) + \ + sizeof(struct rnpvf_queue_stats) / sizeof(u64) + \ + sizeof(struct rnpvf_rx_queue_ring_stat) / sizeof(u64))) + +#define RNPVF_STATS_LEN \ + (RNPVF_GLOBAL_STATS_LEN + RNP_QUEUE_STATS_LEN + \ + RNPVF_HWSTRINGS_STATS_LEN) + +static const char rnp_gstrings_test[][ETH_GSTRING_LEN] = { + "Register test (offline)", "Link test (on/offline)" +}; +#define RNPVF_TEST_LEN (sizeof(rnp_gstrings_test) / ETH_GSTRING_LEN) + +enum priv_bits { + padding_enable = 0, +}; + +static const char rnpvf_priv_flags_strings[][ETH_GSTRING_LEN] = { +#define RNPVF_FT_PADDING BIT(0) +#define RNPVF_FCS_ON BIT(1) + "ft_padding", "fcs" +}; +#define RNPVF_PRIV_FLAGS_STR_LEN ARRAY_SIZE(rnpvf_priv_flags_strings) + +#define ADVERTISED_MASK_10G \ + (SUPPORTED_10000baseT_Full | SUPPORTED_10000baseKX4_Full | \ + SUPPORTED_10000baseKR_Full) +static int rnpvf_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *cmd) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + bool autoneg = false; + bool link_up; + u32 supported, advertising = 0; + u32 link_speed = 0; + + ethtool_convert_link_mode_to_legacy_u32(&supported, + cmd->link_modes.supported); + + hw->mac.ops.check_link(hw, &link_speed, &link_up, false); + + switch (link_speed) { + case RNP_LINK_SPEED_1GB_FULL: + supported |= SUPPORTED_1000baseT_Full; + supported |= SUPPORTED_FIBRE; + advertising |= ADVERTISED_FIBRE | + ADVERTISED_1000baseKX_Full; + cmd->base.port = PORT_FIBRE; + break; + case RNP_LINK_SPEED_10GB_FULL: + supported |= SUPPORTED_10000baseT_Full; + supported |= SUPPORTED_FIBRE; + advertising |= ADVERTISED_FIBRE | + SUPPORTED_10000baseT_Full; + cmd->base.port = PORT_FIBRE; + break; + case RNP_LINK_SPEED_25GB_FULL: + supported |= SUPPORTED_40000baseKR4_Full; + supported |= SUPPORTED_FIBRE; + advertising |= ADVERTISED_FIBRE | + SUPPORTED_40000baseKR4_Full; + cmd->base.port = PORT_FIBRE; + break; + case RNP_LINK_SPEED_40GB_FULL: + supported |= SUPPORTED_40000baseCR4_Full | + SUPPORTED_40000baseSR4_Full | + SUPPORTED_40000baseLR4_Full; + supported |= SUPPORTED_FIBRE; + advertising |= ADVERTISED_FIBRE; + cmd->base.port = PORT_FIBRE; + break; + } + + if (autoneg) { + supported |= SUPPORTED_Autoneg; + advertising |= ADVERTISED_Autoneg; + cmd->base.autoneg = AUTONEG_ENABLE; + } else + cmd->base.autoneg = AUTONEG_DISABLE; + + /* set pause support */ + supported |= SUPPORTED_Pause; + + switch (hw->fc.current_mode) { + case rnp_fc_full: + advertising |= ADVERTISED_Pause; + break; + case rnp_fc_rx_pause: + advertising |= ADVERTISED_Pause | ADVERTISED_Asym_Pause; + break; + case rnp_fc_tx_pause: + advertising |= ADVERTISED_Asym_Pause; + break; + default: + advertising &= ~(ADVERTISED_Pause | ADVERTISED_Asym_Pause); + } + + if (link_up) { + switch (link_speed) { + case RNP_LINK_SPEED_40GB_FULL: + cmd->base.speed = SPEED_40000; + break; + case RNP_LINK_SPEED_25GB_FULL: + cmd->base.speed = SPEED_25000; + break; + case RNP_LINK_SPEED_10GB_FULL: + cmd->base.speed = SPEED_10000; + break; + case RNP_LINK_SPEED_1GB_FULL: + cmd->base.speed = SPEED_1000; + break; + case RNP_LINK_SPEED_100_FULL: + cmd->base.speed = SPEED_100; + break; + default: + break; + } + cmd->base.duplex = DUPLEX_FULL; + } else { + cmd->base.speed = SPEED_UNKNOWN; + cmd->base.duplex = DUPLEX_UNKNOWN; + } + + ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, + supported); + ethtool_convert_legacy_u32_to_link_mode( + cmd->link_modes.advertising, advertising); + return 0; +} + +static void rnpvf_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *drvinfo) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + + strlcpy(drvinfo->driver, rnpvf_driver_name, + sizeof(drvinfo->driver)); + strlcpy(drvinfo->version, rnpvf_driver_version, + sizeof(drvinfo->version)); + strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + sizeof(drvinfo->bus_info)); + if (hw->board_type == rnp_board_n10) { + snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), + "%d.%d.%d.%d", ((char *)&(hw->fw_version))[3], + ((char *)&(hw->fw_version))[2], + ((char *)&(hw->fw_version))[1], + ((char *)&(hw->fw_version))[0]); + } + drvinfo->n_priv_flags = RNPVF_PRIV_FLAGS_STR_LEN; +} + +void rnpvf_get_ringparam( + struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam __always_unused *ker, + struct netlink_ext_ack __always_unused *extack) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + ring->rx_max_pending = RNPVF_MAX_RXD; + ring->tx_max_pending = RNPVF_MAX_TXD; + ring->rx_pending = adapter->rx_ring_item_count; + ring->tx_pending = adapter->tx_ring_item_count; +} + +static void rnpvf_get_strings(struct net_device *netdev, u32 stringset, + u8 *data) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + char *p = (char *)data; + int i; + struct rnpvf_ring *ring; + u16 queue_idx; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < RNPVF_GLOBAL_STATS_LEN; i++) { + memcpy(p, rnp_gstrings_net_stats[i].stat_string, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < RNPVF_HWSTRINGS_STATS_LEN; i++) { + memcpy(p, rnpvf_hwstrings_stats[i].stat_string, + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + BUG_ON(RNPVF_NUM_TX_QUEUES != RNPVF_NUM_RX_QUEUES); + + for (i = 0; i < RNPVF_NUM_TX_QUEUES; i++) { + /* ==== tx ======== */ + ring = adapter->tx_ring[i]; + queue_idx = ring->rnpvf_queue_idx; + sprintf(p, "\n queue%u_tx_packets", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_bytes", i); + p += ETH_GSTRING_LEN; + + sprintf(p, "queue%u_tx_restart", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_busy", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_done_old", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_clean_desc", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_poll_count", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_irq_more", i); + p += ETH_GSTRING_LEN; + + sprintf(p, "queue%u_tx_hw_head", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_hw_tail", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_sw_next_to_clean", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_added_vlan_packets", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_irq_miss", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_next_to_clean", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_tx_equal_count", i); + p += ETH_GSTRING_LEN; + + /* ==== rx ======== */ + ring = adapter->rx_ring[i]; + queue_idx = ring->rnpvf_queue_idx; + sprintf(p, "\n queue%u_rx_packets", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_bytes", i); + p += ETH_GSTRING_LEN; + + sprintf(p, "queue%u_rx_driver_drop_packets", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_rsc", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_rsc_flush", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_non_eop_descs", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_alloc_page_failed", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_alloc_buff_failed", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_alloc_page", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_csum_err", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_csum_good", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_poll_again_count", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_poll_count", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_rm_vlan_packets", i); + p += ETH_GSTRING_LEN; + + sprintf(p, "queue%u_rx_hw_head", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_hw_tail", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_sw_next_to_use", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_irq_miss", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_next_to_clean", i); + p += ETH_GSTRING_LEN; + sprintf(p, "queue%u_rx_equal_count", i); + p += ETH_GSTRING_LEN; + } + break; + case ETH_SS_PRIV_FLAGS: + memcpy(data, rnpvf_priv_flags_strings, + RNPVF_PRIV_FLAGS_STR_LEN * ETH_GSTRING_LEN); + break; + } +} + +static int rnpvf_get_sset_count(struct net_device *netdev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return RNPVF_STATS_LEN; + case ETH_SS_PRIV_FLAGS: + return RNPVF_PRIV_FLAGS_STR_LEN; + default: + return -EOPNOTSUPP; + } +} + +static u32 rnpvf_get_priv_flags(struct net_device *netdev) +{ + struct rnpvf_adapter *adapter = + (struct rnpvf_adapter *)netdev_priv(netdev); + u32 priv_flags = 0; + + if (adapter->priv_flags & RNPVF_PRIV_FLAG_FT_PADDING) + priv_flags |= RNPVF_FT_PADDING; + if (adapter->priv_flags & RNPVF_PRIV_FLAG_FCS_ON) + priv_flags |= RNPVF_FCS_ON; + + return priv_flags; +} + +static int rnpvf_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *coal, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + coal->use_adaptive_tx_coalesce = adapter->adaptive_tx_coal; + coal->tx_coalesce_usecs = adapter->tx_usecs; + coal->tx_coalesce_usecs_irq = 0; + coal->tx_max_coalesced_frames = adapter->tx_frames; + coal->tx_max_coalesced_frames_irq = adapter->tx_work_limit; + + coal->use_adaptive_rx_coalesce = adapter->adaptive_rx_coal; + coal->rx_coalesce_usecs_irq = 0; + coal->rx_coalesce_usecs = adapter->rx_usecs; + coal->rx_max_coalesced_frames = adapter->rx_frames; + coal->rx_max_coalesced_frames_irq = adapter->napi_budge; + + /* this is not support */ + coal->pkt_rate_low = 0; + coal->pkt_rate_high = 0; + coal->rx_coalesce_usecs_low = 0; + coal->rx_max_coalesced_frames_low = 0; + coal->tx_coalesce_usecs_low = 0; + coal->tx_max_coalesced_frames_low = 0; + coal->rx_coalesce_usecs_high = 0; + coal->rx_max_coalesced_frames_high = 0; + coal->tx_coalesce_usecs_high = 0; + coal->tx_max_coalesced_frames_high = 0; + coal->rate_sample_interval = 0; + return 0; +} + +static int rnpvf_set_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + int reset = 0; + struct rnpvf_adapter *adapter = netdev_priv(netdev); + u32 value; + /* we don't support close tx and rx coalesce */ + if (!(ec->use_adaptive_tx_coalesce) || + !(ec->use_adaptive_rx_coalesce)) { + return -EINVAL; + } + + if ((ec->tx_max_coalesced_frames_irq < RNPVF_MIN_TX_WORK) || + (ec->tx_max_coalesced_frames_irq > RNPVF_MAX_TX_WORK)) + return -EINVAL; + + value = clamp_t(u32, ec->tx_max_coalesced_frames_irq, + RNPVF_MIN_TX_WORK, RNPVF_MAX_TX_WORK); + value = ALIGN(value, RNPVF_WORK_ALIGN); + + if (adapter->tx_work_limit != value) { + reset = 1; + adapter->tx_work_limit = value; + } + + if ((ec->tx_max_coalesced_frames < RNPVF_MIN_TX_FRAME) || + (ec->tx_max_coalesced_frames > RNPVF_MAX_TX_FRAME)) + return -EINVAL; + + value = clamp_t(u32, ec->tx_max_coalesced_frames, + RNPVF_MIN_TX_FRAME, RNPVF_MAX_TX_FRAME); + if (adapter->tx_frames != value) { + reset = 1; + adapter->tx_frames = value; + } + + if ((ec->tx_coalesce_usecs < RNPVF_MIN_TX_USEC) || + (ec->tx_coalesce_usecs > RNPVF_MAX_TX_USEC)) + return -EINVAL; + value = clamp_t(u32, ec->tx_coalesce_usecs, + RNPVF_MIN_TX_USEC, RNPVF_MAX_TX_USEC); + if (adapter->tx_usecs != value) { + reset = 1; + adapter->tx_usecs = value; + } + + if ((ec->rx_max_coalesced_frames_irq < RNPVF_MIN_RX_WORK) || + (ec->rx_max_coalesced_frames_irq > RNPVF_MAX_RX_WORK)) + return -EINVAL; + value = clamp_t(u32, ec->rx_max_coalesced_frames_irq, + RNPVF_MIN_RX_WORK, RNPVF_MAX_RX_WORK); + value = ALIGN(value, RNPVF_WORK_ALIGN); + + if (adapter->napi_budge != value) { + reset = 1; + adapter->napi_budge = value; + } + + if ((ec->rx_max_coalesced_frames < RNPVF_MIN_RX_FRAME) || + (ec->rx_max_coalesced_frames > RNPVF_MAX_RX_FRAME)) + return -EINVAL; + value = clamp_t(u32, ec->rx_max_coalesced_frames, + RNPVF_MIN_RX_FRAME, RNPVF_MAX_RX_FRAME); + if (adapter->rx_frames != value) { + reset = 1; + adapter->rx_frames = value; + } + + if ((ec->rx_coalesce_usecs < RNPVF_MIN_RX_USEC) || + (ec->rx_coalesce_usecs > RNPVF_MAX_RX_USEC)) + return -EINVAL; + value = clamp_t(u32, ec->rx_coalesce_usecs, + RNPVF_MIN_RX_USEC, RNPVF_MAX_RX_USEC); + + if (adapter->rx_usecs != value) { + reset = 1; + adapter->rx_usecs = value; + } + + /* other setup is not supported */ + if ((ec->pkt_rate_low) || (ec->pkt_rate_high) || + (ec->rx_coalesce_usecs_low) || + (ec->rx_max_coalesced_frames_low) || + (ec->tx_coalesce_usecs_low) || + (ec->tx_max_coalesced_frames_low) || + (ec->rx_coalesce_usecs_high) || + (ec->rx_max_coalesced_frames_high) || + (ec->tx_coalesce_usecs_high) || + (ec->tx_max_coalesced_frames_high) || + (ec->rate_sample_interval) || (ec->tx_coalesce_usecs_irq) || + (ec->rx_coalesce_usecs_irq)) + return -EINVAL; + + if (reset) { + if (netif_running(netdev)) + rnpvf_close(netdev); + remove_mbx_irq(adapter); + rnpvf_clear_interrupt_scheme(adapter); + rnpvf_init_interrupt_scheme(adapter); + register_mbx_irq(adapter); + if (netif_running(netdev)) + return rnpvf_open(netdev); + } + return 0; +} + +static void rnpvf_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct net_device_stats *net_stats = &netdev->stats; + struct rnpvf_ring *ring; + int i = 0, j; + char *p = NULL; + + rnpvf_update_stats(adapter); + + for (i = 0; i < RNPVF_GLOBAL_STATS_LEN; i++) { + p = (char *)net_stats + + rnp_gstrings_net_stats[i].stat_offset; + data[i] = (rnp_gstrings_net_stats[i].sizeof_stat == + sizeof(u64)) ? + *(u64 *)p : + *(u32 *)p; + } + for (j = 0; j < RNPVF_HWSTRINGS_STATS_LEN; j++, i++) { + p = (char *)adapter + rnpvf_hwstrings_stats[j].stat_offset; + data[i] = (rnpvf_hwstrings_stats[j].sizeof_stat == + sizeof(u64)) ? + *(u64 *)p : + *(u32 *)p; + } + + BUG_ON(RNPVF_NUM_TX_QUEUES != RNPVF_NUM_RX_QUEUES); + + for (j = 0; j < RNPVF_NUM_TX_QUEUES; j++) { + /* ===== tx-ring == */ + ring = adapter->tx_ring[j]; + + if (!ring) { + data[i++] = 0; + data[i++] = 0; + + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + + /* rnpvf_tx_queue_ring_stat */ + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + + /* ===== rx-ring == */ + data[i++] = 0; + data[i++] = 0; + + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + continue; + } + + data[i++] = ring->stats.packets; + data[i++] = ring->stats.bytes; + + data[i++] = ring->tx_stats.restart_queue; + data[i++] = ring->tx_stats.tx_busy; + data[i++] = ring->tx_stats.tx_done_old; + data[i++] = ring->tx_stats.clean_desc; + data[i++] = ring->tx_stats.poll_count; + data[i++] = ring->tx_stats.irq_more_count; + + /* rnpvf_tx_queue_ring_stat */ + data[i++] = ring_rd32(ring, RNP_DMA_REG_TX_DESC_BUF_HEAD); + data[i++] = ring_rd32(ring, RNP_DMA_REG_TX_DESC_BUF_TAIL); + data[i++] = ring->next_to_clean; + data[i++] = ring->tx_stats.vlan_add; + data[i++] = ring->tx_stats.tx_irq_miss; + if (ring->tx_stats.tx_next_to_clean == -1) + data[i++] = ring->count; + else + data[i++] = ring->tx_stats.tx_next_to_clean; + data[i++] = ring->tx_stats.tx_equal_count; + + /* ===== rx-ring == */ + ring = adapter->rx_ring[j]; + + if (!ring) { + /* ===== rx-ring == */ + data[i++] = 0; + data[i++] = 0; + + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + + data[i++] = 0; + data[i++] = 0; + data[i++] = 0; + continue; + } + + data[i++] = ring->stats.packets; + data[i++] = ring->stats.bytes; + + data[i++] = ring->rx_stats.driver_drop_packets; + data[i++] = ring->rx_stats.rsc_count; + data[i++] = ring->rx_stats.rsc_flush; + data[i++] = ring->rx_stats.non_eop_descs; + data[i++] = ring->rx_stats.alloc_rx_page_failed; + data[i++] = ring->rx_stats.alloc_rx_buff_failed; + data[i++] = ring->rx_stats.alloc_rx_page; + data[i++] = ring->rx_stats.csum_err; + data[i++] = ring->rx_stats.csum_good; + data[i++] = ring->rx_stats.poll_again_count; + data[i++] = ring->rx_stats.poll_count; + data[i++] = ring->rx_stats.vlan_remove; + data[i++] = ring_rd32(ring, RNP_DMA_REG_RX_DESC_BUF_HEAD); + data[i++] = ring_rd32(ring, RNP_DMA_REG_RX_DESC_BUF_TAIL); + data[i++] = ring->next_to_clean; + + data[i++] = ring->rx_stats.rx_irq_miss; + if (ring->rx_stats.rx_next_to_clean == -1) + data[i++] = ring->count; + else + data[i++] = ring->rx_stats.rx_next_to_clean; + data[i++] = ring->rx_stats.rx_equal_count; + } +} + +static void rnpvf_get_channels(struct net_device *dev, + struct ethtool_channels *ch) +{ + struct rnpvf_adapter *adapter = netdev_priv(dev); + + /* report maximum channels */ + ch->max_combined = min_t(int, adapter->hw.mac.max_tx_queues, + adapter->hw.mac.max_rx_queues); + + /* report info for other vector */ + ch->max_other = NON_Q_VECTORS; + ch->other_count = NON_Q_VECTORS; + + /* record RSS queues */ + ch->combined_count = adapter->dma_channels; +} + +static u32 rnpvf_get_msglevel(struct net_device *netdev) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + return adapter->msg_enable; +} + +static void rnpvf_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + + /* we don't support autoneg */ + pause->autoneg = 0; + + if (hw->fc.current_mode == rnp_fc_rx_pause) { + pause->rx_pause = 1; + } else if (hw->fc.current_mode == rnp_fc_tx_pause) { + pause->tx_pause = 1; + } else if (hw->fc.current_mode == rnp_fc_full) { + pause->rx_pause = 1; + pause->tx_pause = 1; + } +} + +static void rnpvf_set_msglevel(struct net_device *netdev, u32 data) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + adapter->msg_enable = data; +} + +static const struct ethtool_ops rnpvf_ethtool_ops = { + + .get_link_ksettings = rnpvf_get_link_ksettings, + .get_drvinfo = rnpvf_get_drvinfo, + .get_link = ethtool_op_get_link, + .get_ringparam = rnpvf_get_ringparam, + .get_strings = rnpvf_get_strings, + .get_pauseparam = rnpvf_get_pauseparam, + .get_msglevel = rnpvf_get_msglevel, + .set_msglevel = rnpvf_set_msglevel, + .get_sset_count = rnpvf_get_sset_count, + .get_priv_flags = rnpvf_get_priv_flags, + .get_ethtool_stats = rnpvf_get_ethtool_stats, + .get_coalesce = rnpvf_get_coalesce, + .set_coalesce = rnpvf_set_coalesce, + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_MAX_FRAMES_IRQ | + ETHTOOL_COALESCE_MAX_FRAMES, + .get_channels = rnpvf_get_channels, +}; + +void rnpvf_set_ethtool_ops(struct net_device *netdev) +{ + netdev->ethtool_ops = &rnpvf_ethtool_ops; +} diff --git a/drivers/net/ethernet/mucse/rnpvf/mbx.c b/drivers/net/ethernet/mucse/rnpvf/mbx.c new file mode 100644 index 0000000000000..d73ed4b4e2e8a --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/mbx.c @@ -0,0 +1,624 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#include "mbx.h" +#include "rnpvf.h" + +struct counter { + union { + struct { + unsigned short pf_req; + unsigned short pf_ack; + }; + struct { + unsigned short cpu_req; + unsigned short cpu_ack; + }; + }; + unsigned short vf_req; + unsigned short vf_ack; +} __packed; + +static s32 rnpvf_poll_for_msg(struct rnpvf_hw *hw, bool to_cm3); +static s32 rnpvf_poll_for_ack(struct rnpvf_hw *hw, bool to_cm3); + +#define PF2VF_MBOX_VEC(mbx, vf) (mbx->pf2vf_mbox_vec_base + 4 * (vf)) +#define CPU2VF_MBOX_VEC(mbx, vf) (mbx->cpu2vf_mbox_vec_base + 4 * (vf)) +#define PF_VF_SHM(mbx, vf) \ + ((mbx->pf_vf_shm_base) + \ + (64 * (vf))) +#define PF2VF_COUNTER(mbx, vf) (PF_VF_SHM(mbx, vf) + 0) +#define VF2PF_COUNTER(mbx, vf) (PF_VF_SHM(mbx, vf) + 4) +#define PF_VF_SHM_DATA(mbx, vf) (PF_VF_SHM(mbx, vf) + 8) +#define VF2PF_MBOX_CTRL(mbx, vf) ((mbx->vf2pf_mbox_ctrl_base) + (4 * (vf))) +#define CPU_VF_SHM(mbx, vf) (mbx->cpu_vf_shm_base + (64 * (vf))) +#define CPU2VF_COUNTER(mbx, vf) (CPU_VF_SHM(mbx, vf) + 0) +#define VF2CPU_COUNTER(mbx, vf) (CPU_VF_SHM(mbx, vf) + 4) +#define CPU_VF_SHM_DATA(mbx, vf) (CPU_VF_SHM(mbx, vf) + 8) +#define VF2CPU_MBOX_CTRL(mbx, vf) (mbx->vf2cpu_mbox_ctrl_base + 64 * (vf)) +#define CPU_VF_MBOX_MASK_LO(mbx, vf) \ + (mbx->cpu_vf_mbox_mask_lo_base + 64 * (vf)) +#define CPU_VF_MBOX_MASK_HI(mbx, vf) \ + (mbx->cpu_vf_mbox_mask_hi_base + 64 * (vf)) +#define MBOX_CTRL_REQ (1 << 0) +#define MBOX_CTRL_VF_HOLD_SHM (1 << 2) +#define MBOX_IRQ_EN 0 +#define MBOX_IRQ_DISABLE 1 + +/** + * rnpvf_read_posted_mbx - Wait for message notification and receive message + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully received a message notification and + * copied it into the receive buffer. + **/ +static s32 rnpvf_read_posted_mbx(struct rnpvf_hw *hw, u32 *msg, u16 size, + bool to_cm3) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + s32 ret_val = -RNP_ERR_MBX; + + if (!mbx->ops.read) + goto out; + + ret_val = rnpvf_poll_for_msg(hw, to_cm3); + + /* if ack received read message, otherwise we timed out */ + if (!ret_val) + ret_val = mbx->ops.read(hw, msg, size, to_cm3); +out: + return ret_val; +} + +/** + * rnpvf_write_posted_mbx - Write a message to the mailbox, wait for ack + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully copied message into the buffer and + * received an ack to that message within delay * timeout period + **/ +static s32 rnpvf_write_posted_mbx(struct rnpvf_hw *hw, u32 *msg, u16 size, + bool to_cm3) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + s32 ret_val = -RNP_ERR_MBX; + + /* exit if either we can't write or there isn't a defined timeout */ + if (!mbx->ops.write || !mbx->timeout) + goto out; + + /* send msg */ + ret_val = mbx->ops.write(hw, msg, size, to_cm3); + + /* if msg sent wait until we receive an ack */ + if (!ret_val) + ret_val = rnpvf_poll_for_ack(hw, to_cm3); +out: + return ret_val; +} + +static inline u16 rnpvf_mbx_get_req(struct rnpvf_hw *hw, int reg) +{ + mb(); + return mbx_rd32(hw, reg) & 0xffff; +} + +static inline u16 rnpvf_mbx_get_ack(struct rnpvf_hw *hw, int reg) +{ + mb(); + return (mbx_rd32(hw, reg) >> 16) & 0xffff; +} + +static inline void rnpvf_mbx_inc_vfreq(struct rnpvf_hw *hw, bool to_cm3) +{ + u16 req; + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + int reg = to_cm3 ? VF2CPU_COUNTER(mbx, vfnum) : + VF2PF_COUNTER(mbx, vfnum); + u32 v = mbx_rd32(hw, reg); + + req = (v & 0xffff); + req++; + v &= ~(0x0000ffff); + v |= req; + mb(); + mbx_wr32(hw, reg, v); + + /* update stats */ + hw->mbx.stats.msgs_tx++; +} + +static inline void rnpvf_mbx_inc_vfack(struct rnpvf_hw *hw, bool to_cm3) +{ + u16 ack; + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + int reg = to_cm3 ? VF2CPU_COUNTER(mbx, vfnum) : + VF2PF_COUNTER(mbx, vfnum); + u32 v = mbx_rd32(hw, reg); + + ack = (v >> 16) & 0xffff; + ack++; + v &= ~(0xffff0000); + v |= (ack << 16); + mb(); + mbx_wr32(hw, reg, v); + + /* update stats */ + hw->mbx.stats.msgs_rx++; +} + +/** + * rnpvf_check_for_msg_vf - checks to see if the PF has sent mail + * @hw: pointer to the HW structure + * + * returns 0 if the PF has set the Status bit or else ERR_MBX + **/ +static s32 rnpvf_check_for_msg_vf(struct rnpvf_hw *hw, bool to_cm3) +{ + s32 ret_val = RNP_ERR_MBX; + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + + if (to_cm3 == true) { + if (rnpvf_mbx_get_req(hw, CPU2VF_COUNTER(mbx, vfnum)) != + hw->mbx.cpu_req) { + ret_val = 0; + hw->mbx.stats.reqs++; + } + } else { + if (rnpvf_mbx_get_req(hw, PF2VF_COUNTER(mbx, vfnum)) != + hw->mbx.pf_req) { + ret_val = 0; + hw->mbx.stats.reqs++; + } + } + + return ret_val; +} + +/** + * rnpvf_poll_for_msg - Wait for message notification + * @hw: pointer to the HW structure + * + * returns 0 if it successfully received a message notification + **/ +static s32 rnpvf_poll_for_msg(struct rnpvf_hw *hw, bool to_cm3) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + int countdown = mbx->timeout; + + while (countdown && mbx->ops.check_for_msg(hw, to_cm3)) { + countdown--; + udelay(mbx->udelay); + } + + return countdown ? 0 : RNP_ERR_MBX; +} + +/** + * rnpvf_poll_for_ack - Wait for message acknowledgment + * @hw: pointer to the HW structure + * + * returns 0 if it successfully received a message acknowledgment + **/ +static s32 rnpvf_poll_for_ack(struct rnpvf_hw *hw, bool to_cm3) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + int countdown = mbx->timeout; + + while (countdown && mbx->ops.check_for_ack(hw, to_cm3)) { + countdown--; + udelay(mbx->udelay); + } + + /* if we failed, all future posted messages fail until reset */ + if (!countdown) { + mbx->timeout = 0; + dbg("%s timeout\n", __func__); + } + + return countdown ? 0 : RNP_ERR_MBX; +} + +/** + * rnpvf_check_for_rst_msg_vf - checks to see if the PF has ACK'd + * @hw: pointer to the HW structure + * + * returns 0 if the PF has set the ACK bit or else ERR_MBX + **/ +static s32 rnpvf_check_for_rst_msg_vf(struct rnpvf_hw *hw, bool to_cm3) +{ + struct rnpvf_adapter *adapter = hw->back; + struct rnp_mbx_info *mbx = &hw->mbx; + s32 ret_val = RNP_ERR_MBX; + u8 vfnum = VFNUM(mbx, hw->vfnum); + u32 DATA_REG = (to_cm3) ? CPU_VF_SHM_DATA(mbx, vfnum) : + PF_VF_SHM_DATA(mbx, vfnum); + u32 data; + int ret = 1; + + ret_val = rnpvf_check_for_msg_vf(hw, to_cm3); + if (!ret_val) { + data = mbx_rd32(hw, DATA_REG); + + data &= ~RNP_PF_VFNUM_MASK; + dbg("mbx %x\n", data); + /* add other mailbox setup */ + if (((data) & (~RNP_VT_MSGTYPE_CTS)) == + RNP_PF_CONTROL_PRING_MSG) { + } else if ((data) == RNP_PF_SET_FCS) { + data = mbx_rd32(hw, DATA_REG + 4); + if (data) { + adapter->priv_flags |= + RNPVF_PRIV_FLAG_FCS_ON; + adapter->netdev->features |= NETIF_F_RXFCS; + } else { + adapter->priv_flags &= + (~RNPVF_PRIV_FLAG_FCS_ON); + adapter->netdev->features &= + (~NETIF_F_RXFCS); + } + if ((adapter->priv_flags & + RNPVF_PRIV_FLAG_FCS_ON) && + (adapter->netdev->features & NETIF_F_RXCSUM)) + adapter->netdev->features &= + (~NETIF_F_RXCSUM); + else { + if (adapter->flags & + RNPVF_FLAG_RX_CHKSUM_ENABLED) + adapter->netdev->features |= + NETIF_F_RXCSUM; + else + adapter->netdev->features &= + (~NETIF_F_RXCSUM); + } + + } else if ((data) == RNP_PF_SET_PAUSE) { + hw->fc.current_mode = mbx_rd32(hw, DATA_REG + 4); + } else if ((data) == RNP_PF_SET_FT_PADDING) { + data = mbx_rd32(hw, DATA_REG + 4); + if (data) { + adapter->priv_flags |= + RNPVF_PRIV_FLAG_FT_PADDING; + } else { + adapter->priv_flags &= + (~RNPVF_PRIV_FLAG_FT_PADDING); + } + } else if ((data) == RNP_PF_SET_VLAN_FILTER) { + data = mbx_rd32(hw, DATA_REG + 4); + if (data) { + if (hw->feature_flags & + RNPVF_NET_FEATURE_VLAN_OFFLOAD) { + adapter->netdev->features |= + NETIF_F_HW_VLAN_CTAG_FILTER; + } + + if (hw->feature_flags & + RNPVF_NET_FEATURE_STAG_OFFLOAD) { + adapter->netdev->features |= + NETIF_F_HW_VLAN_STAG_FILTER; + } + + } else { + if (hw->feature_flags & + RNPVF_NET_FEATURE_VLAN_OFFLOAD) { + adapter->netdev->features &= + ~NETIF_F_HW_VLAN_CTAG_FILTER; + } + if (hw->feature_flags & + RNPVF_NET_FEATURE_STAG_OFFLOAD) { + adapter->netdev->features &= + ~NETIF_F_HW_VLAN_STAG_FILTER; + } + } + } else if ((data) == RNP_PF_SET_VLAN) { + struct rnp_mbx_info *mbx = &hw->mbx; + + data = mbx_rd32(hw, DATA_REG + 4); + /* pf set vlan for this vf */ + adapter->flags |= RNPVF_FLAG_PF_UPDATE_VLAN; + if (data) { + adapter->flags |= RNPVF_FLAG_PF_SET_VLAN; + adapter->vf_vlan = data; + /* we should record old value */ + /* we should open rx vlan offload */ + if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) + adapter->priv_flags |= RNPVF_FLAG_RX_VLAN_OFFLOAD; + else + adapter->priv_flags &= ~RNPVF_FLAG_RX_VLAN_OFFLOAD; + adapter->netdev->features |= + NETIF_F_HW_VLAN_CTAG_RX; + /* should close tx vlan offload */ + if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_TX) + adapter->priv_flags |= RNPVF_FLAG_TX_VLAN_OFFLOAD; + else + adapter->priv_flags &= ~RNPVF_FLAG_TX_VLAN_OFFLOAD; + adapter->netdev->features &= + ~NETIF_F_HW_VLAN_CTAG_TX; + } else { + adapter->flags &= + (~RNPVF_FLAG_PF_SET_VLAN); + adapter->vf_vlan = 0; + /* write back old value */ + if (adapter->priv_flags & RNPVF_FLAG_RX_VLAN_OFFLOAD) + adapter->netdev->features |= NETIF_F_HW_VLAN_CTAG_RX; + else + adapter->netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; + + if (adapter->priv_flags & RNPVF_FLAG_TX_VLAN_OFFLOAD) + adapter->netdev->features |= NETIF_F_HW_VLAN_CTAG_TX; + else + adapter->netdev->features &= ~NETIF_F_HW_VLAN_CTAG_TX; + } + hw->ops.set_veb_vlan(hw, data, + VFNUM(mbx, hw->vfnum)); + } else if ((data) == RNP_PF_SET_LINK) { + data = mbx_rd32(hw, DATA_REG + 4); + if (data & RNP_PF_LINK_UP) { + hw->link = true; + hw->speed = data & 0xffff; + } else { + hw->link = false; + hw->speed = 0; + } + } else if ((data) == RNP_PF_SET_MTU) { + data = mbx_rd32(hw, DATA_REG + 4); + hw->mtu = data; + adapter->flags |= RNPVF_FLAG_PF_UPDATE_MTU; + } else if ((data) == RNP_PF_SET_RESET) { + adapter->flags |= RNPVF_FLAG_PF_RESET; + } else if ((data) == RNP_PF_SET_MAC_SPOOF) { + data = mbx_rd32(hw, DATA_REG + 4); + if (data) + hw->pf_feature |= PF_MAC_SPOOF; + else + hw->pf_feature &= (~PF_MAC_SPOOF); + + } else { + return RNP_ERR_MBX; + } + } + + return ret; +} + +/** + * rnpvf_check_for_ack_vf - checks to see if the PF has ACK'd + * @hw: pointer to the HW structure + * + * returns 0 if the PF has set the ACK bit or else ERR_MBX + **/ +static s32 rnpvf_check_for_ack_vf(struct rnpvf_hw *hw, bool to_cm3) +{ + s32 ret_val = RNP_ERR_MBX; + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + + if (to_cm3 == true) { + if (rnpvf_mbx_get_ack(hw, CPU2VF_COUNTER(mbx, vfnum)) != + hw->mbx.cpu_ack) { + ret_val = 0; + hw->mbx.stats.acks++; + } + } else { + if (rnpvf_mbx_get_ack(hw, PF2VF_COUNTER(mbx, vfnum)) != + hw->mbx.pf_ack) { + ret_val = 0; + hw->mbx.stats.acks++; + } + } + + return ret_val; +} + +/** + * rnpvf_obtain_mbx_lock_vf - obtain mailbox lock + * @hw: pointer to the HW structure + * + * return 0 if we obtained the mailbox lock + **/ +static s32 rnpvf_obtain_mbx_lock_vf(struct rnpvf_hw *hw, bool to_cm3) +{ + int try_cnt = 2 * 1000; // 1s + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + struct rnpvf_adapter *adapter = hw->back; + u32 CTRL_REG = (to_cm3) ? VF2CPU_MBOX_CTRL(mbx, vfnum) : + VF2PF_MBOX_CTRL(mbx, vfnum); + + while (try_cnt-- > 0) { + /* Take ownership of the buffer */ + mbx_wr32(hw, CTRL_REG, MBOX_CTRL_VF_HOLD_SHM); + mb(); + /* reserve mailbox for vf use */ + if (mbx_rd32(hw, CTRL_REG) & MBOX_CTRL_VF_HOLD_SHM) + return 0; + udelay(500); + } + + printk("[rnpvf] %s: failed to get mbx-lock \n", adapter->name); + return RNP_ERR_MBX; +} + +/** + * rnpvf_write_mbx_vf - Write a message to the mailbox + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully copied message into the buffer + **/ +static s32 rnpvf_write_mbx_vf(struct rnpvf_hw *hw, u32 *msg, u16 size, + bool to_cm3) +{ + s32 ret_val; + struct rnp_mbx_info *mbx = &hw->mbx; + u32 i; + u8 vfnum = VFNUM(mbx, hw->vfnum); + u32 DATA_REG = (to_cm3) ? CPU_VF_SHM_DATA(mbx, vfnum) : + PF_VF_SHM_DATA(mbx, vfnum); + u32 CTRL_REG = (to_cm3) ? VF2CPU_MBOX_CTRL(mbx, vfnum) : + VF2PF_MBOX_CTRL(mbx, vfnum); + + /* lock the mailbox to prevent pf/vf race condition */ + ret_val = rnpvf_obtain_mbx_lock_vf(hw, to_cm3); + if (ret_val) { + printk("%s: get mbx wlock failed. ret:%d. req:0x%08x-0x%08x\n", + __func__, ret_val, msg[0], msg[1]); + goto out_no_write; + } + + /* add mailbox_id [27:21] */ +#define VF_NUM_OFFSET (21) + if (!to_cm3) + msg[0] |= ((hw->vfnum & 0x3f) << VF_NUM_OFFSET); + + /* copy the caller specified message to the mailbox memory buffer */ + for (i = 0; i < size; i++) + mbx_wr32(hw, DATA_REG + i * 4, msg[i]); + + /* update acks. used by rnpvf_check_for_ack_vf */ + if (to_cm3 == true) + hw->mbx.cpu_ack = + rnpvf_mbx_get_ack(hw, CPU2VF_COUNTER(mbx, vfnum)); + else + hw->mbx.pf_ack = + rnpvf_mbx_get_ack(hw, PF2VF_COUNTER(mbx, vfnum)); + rnpvf_mbx_inc_vfreq(hw, to_cm3); + + /* Drop VFU and interrupt the PF/CM3 to + * tell it a message has been sent + */ + mbx_wr32(hw, CTRL_REG, MBOX_CTRL_REQ); + +out_no_write: + return ret_val; +} + +/** + * rnpvf_read_mbx_vf - Reads a message from the inbox intended for vf + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully read message from buffer + **/ +static s32 rnpvf_read_mbx_vf(struct rnpvf_hw *hw, u32 *msg, u16 size, + bool to_cm3) +{ + s32 ret_val = 0; + struct rnp_mbx_info *mbx = &hw->mbx; + u32 i; + u8 vfnum = VFNUM(mbx, hw->vfnum); + u32 BUF_REG = (to_cm3) ? CPU_VF_SHM_DATA(mbx, vfnum) : + PF_VF_SHM_DATA(mbx, vfnum); + u32 CTRL_REG = (to_cm3) ? VF2CPU_MBOX_CTRL(mbx, vfnum) : + VF2PF_MBOX_CTRL(mbx, vfnum); + + /* lock the mailbox to prevent pf/vf race condition */ + ret_val = rnpvf_obtain_mbx_lock_vf(hw, to_cm3); + if (ret_val) + goto out_no_read; + + mb(); + /* copy the message from the mailbox memory buffer */ + for (i = 0; i < size; i++) + msg[i] = mbx_rd32(hw, BUF_REG + 4 * i); + + /* clear vf_num */ +#define RNP_VF_NUM_MASK (0x7f << 21) + msg[0] &= (~RNP_VF_NUM_MASK); + + /* update req. used by rnpvf_check_for_msg_vf */ + if (to_cm3 == true) + hw->mbx.cpu_req = + rnpvf_mbx_get_req(hw, CPU2VF_COUNTER(mbx, vfnum)); + else + hw->mbx.pf_req = + rnpvf_mbx_get_req(hw, PF2VF_COUNTER(mbx, vfnum)); + /* Acknowledge receipt and release mailbox, then we're done */ + rnpvf_mbx_inc_vfack(hw, to_cm3); + + /* free ownership of the buffer */ + mbx_wr32(hw, CTRL_REG, 0); + +out_no_read: + return ret_val; +} + +static void rnpvf_reset_mbx(struct rnpvf_hw *hw) +{ + u32 v; + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + + /* release vfu */ + mbx_wr32(hw, VF2CPU_MBOX_CTRL(mbx, vfnum), 0); + mbx_wr32(hw, VF2PF_MBOX_CTRL(mbx, vfnum), 0); + + /* fetch mbx counter values */ + v = mbx_rd32(hw, PF2VF_COUNTER(mbx, vfnum)); + hw->mbx.pf_req = v & 0xffff; + hw->mbx.pf_ack = (v >> 16) & 0xffff; + + v = mbx_rd32(hw, CPU2VF_COUNTER(mbx, vfnum)); + hw->mbx.cpu_req = v & 0xffff; + hw->mbx.cpu_ack = (v >> 16) & 0xffff; + + return; +} + +static s32 rnpvf_mbx_configure_vf(struct rnpvf_hw *hw, int nr_vec, + bool enable) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + int mbx_vec_reg, vfnum = VFNUM(mbx, hw->vfnum); + + mbx_vec_reg = PF2VF_MBOX_VEC(mbx, vfnum); + mbx_wr32(hw, mbx_vec_reg, nr_vec); + + return 0; +} + +/** + * rnpvf_init_mbx_params_vf - set initial values for vf mailbox + * @hw: pointer to the HW structure + * + * Initializes the hw->mbx struct to correct values for vf mailbox + */ +static s32 rnpvf_init_mbx_params_vf(struct rnpvf_hw *hw) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + + /* start mailbox as timed out and let the reset_hw call set the timeout + * value to begin communications + */ + mbx->timeout = 0; + mbx->udelay = RNP_VF_MBX_INIT_DELAY; + mbx->stats.msgs_tx = 0; + mbx->stats.msgs_rx = 0; + mbx->stats.reqs = 0; + mbx->stats.acks = 0; + mbx->stats.rsts = 0; + mbx->size = RNP_VFMAILBOX_SIZE; + rnpvf_reset_mbx(hw); + return 0; +} + +const struct rnp_mbx_operations rnpvf_mbx_ops = { + .init_params = rnpvf_init_mbx_params_vf, + .read = rnpvf_read_mbx_vf, + .write = rnpvf_write_mbx_vf, + .read_posted = rnpvf_read_posted_mbx, + .write_posted = rnpvf_write_posted_mbx, + .check_for_msg = rnpvf_check_for_msg_vf, + .check_for_ack = rnpvf_check_for_ack_vf, + .check_for_rst = rnpvf_check_for_rst_msg_vf, + .configure = rnpvf_mbx_configure_vf, +}; diff --git a/drivers/net/ethernet/mucse/rnpvf/mbx.h b/drivers/net/ethernet/mucse/rnpvf/mbx.h new file mode 100644 index 0000000000000..bb9eb55a9bc36 --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/mbx.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#ifndef _RNP_MBX_H_ +#define _RNP_MBX_H_ + +#include "vf.h" + +#define RNP_VFMAILBOX_SIZE 14 /* 16 32 bit words - 64 bytes */ +#define RNP_ERR_MBX -100 + +struct mbx_shm { + u32 stat; +#define MBX_ST_PF_ACK (1 << 0) +#define MBX_ST_PF_STS (1 << 1) +#define MBX_ST_PF_RST (1 << 2) + +#define MBX_ST_VF_ACK (1 << 3) +#define MBX_ST_VF_REQ (1 << 4) +#define MBX_ST_VF_RST (1 << 5) + +#define MBX_ST_CPU_ACK (1 << 6) +#define MBX_ST_CPU_REQ (1 << 7) + + u32 data[RNP_VFMAILBOX_SIZE]; +} __aligned(4); + +/* If it's a RNP_VF_* msg then it originates in the VF and is sent to the + * PF. The reverse is true if it is RNP_PF_*. + * Message ACK's are the value or'd with 0xF0000000 + */ +#define RNP_VT_MSGTYPE_ACK 0x80000000 +/* Messages below or'd with + * this are the ACK + */ +#define RNP_VT_MSGTYPE_NACK 0x40000000 +/* Messages below or'd with + * this are the NACK + */ +#define RNP_VT_MSGTYPE_CTS 0x20000000 +/* Indicates that VF is still + * clear to send requests + */ +#define RNP_VT_MSGINFO_SHIFT 14 +/* bits 23:16 are used for exra info for certain messages */ +#define RNP_VT_MSGINFO_MASK (0xFF << RNP_VT_MSGINFO_SHIFT) + +/* mailbox API, legacy requests */ +#define RNP_VF_RESET 0x01 /* VF requests reset */ +#define RNP_VF_SET_MAC_ADDR 0x02 /* VF requests PF to set MAC addr */ +#define RNP_VF_SET_MULTICAST 0x03 /* VF requests PF to set MC addr */ +#define RNP_VF_SET_VLAN 0x04 /* VF requests PF to set VLAN */ + +/* mailbox API, version 1.0 VF requests */ +#define RNP_VF_SET_LPE 0x05 /* VF requests PF to set VMOLR.LPE */ +#define RNP_VF_SET_MACVLAN 0x06 /* VF requests PF for unicast filter */ +#define RNP_VF_GET_MACVLAN 0x07 /* VF requests mac */ +#define RNP_VF_API_NEGOTIATE 0x08 /* negotiate API version */ + +/* mailbox API, version 1.1 VF requests */ +#define RNP_VF_GET_QUEUE 0x09 /* get queue configuration */ +#define RNP_VF_SET_VLAN_STRIP 0x0a /* VF requests PF to set VLAN STRIP */ +#define RNP_VF_REG_RD 0x0b /* vf read reg */ +#define RNP_VF_GET_MTU 0x0c /* vf read reg */ +#define RNP_VF_SET_MTU 0x0d /* vf read reg */ +#define RNP_VF_GET_FW 0x0e /* vf read reg */ +#define RNP_VF_RESET_PF 0x11 /* vf read reg */ +#define RNP_VF_SET_PROMISCE 0x16 + +#define RNP_PF_VFNUM_MASK (0x3f << 21) +#define RNP_PF_SET_FCS 0x10 /* PF set fcs status */ +#define RNP_PF_SET_PAUSE 0x11 /* PF set pause status */ +#define RNP_PF_SET_FT_PADDING 0x12 /* PF set ft padding status */ +#define RNP_PF_SET_VLAN_FILTER 0x13 +#define RNP_PF_SET_VLAN 0x14 +#define RNP_PF_SET_LINK 0x15 +#define RNP_PF_SET_MTU 0x16 +#define RNP_PF_SET_RESET 0x17 +#define RNP_PF_SET_MAC_SPOOF 0x18 +#define RNP_PF_LINK_UP (1 << 31) + +#define RNP_PF_REMOVE 0x0f +#define RNP_PF_GET_LINK 0x10 +/* GET_QUEUES return data indices within the mailbox */ +#define RNP_VF_TX_QUEUES 1 /* number of Tx queues supported */ +#define RNP_VF_RX_QUEUES 2 /* number of Rx queues supported */ +#define RNP_VF_TRANS_VLAN 3 /* Indication of port vlan */ +#define RNP_VF_DEF_QUEUE 4 /* Default queue offset */ + +/* length of permanent address message returned from PF */ +#define RNP_VF_PERMADDR_MSG_LEN 11 +/* word in permanent address message with the current multicast type */ +#define RNP_VF_MC_TYPE_WORD 3 +#define RNP_VF_DMA_VERSION_WORD 4 +#define RNP_VF_VLAN_WORD 5 +#define RNP_VF_PHY_TYPE_WORD 6 +#define RNP_VF_FW_VERSION_WORD 7 +#define RNP_VF_LINK_STATUS_WORD 8 +#define RNP_VF_AXI_MHZ 9 +#define RNP_VF_FEATURE 10 + +#define RNP_PF_CONTROL_PRING_MSG 0x0100 /* PF control message */ + +#define RNP_VF_MBX_INIT_TIMEOUT 2000 /* number of retries on mailbox */ +#define RNP_VF_MBX_INIT_DELAY 500 /* microseconds between retries */ + +/* forward declaration of the HW struct */ +struct rnpvf_hw; + +enum MBX_ID { + MBX_VF0 = 0, + MBX_VF1, + //... + MBX_VF63, + MBX_CM3CPU, + MBX_VFCNT +}; + +#endif /* _RNP_MBX_H_ */ diff --git a/drivers/net/ethernet/mucse/rnpvf/regs.h b/drivers/net/ethernet/mucse/rnpvf/regs.h new file mode 100644 index 0000000000000..4ed8eff2437a3 --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/regs.h @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#ifndef _RNPVF_REGS_H_ +#define _RNPVF_REGS_H_ + +enum NIC_MODE { + MODE_NIC_MODE_2PORT_40G = 0, + MODE_NIC_MODE_2PORT_10G = 1, + MODE_NIC_MODE_4PORT_10G = 2, + MODE_NIC_MODE_8PORT_10G = 3, +}; + +/* RNP-Ring Registers */ +#define RNP_DMA_RING_BASE 0x8000 +#define RNP_DMA_RX_DESC_TIMEOUT_TH 0x8000 +#define RNP_DMA_TX_DESC_FETCH_CTL 0x8004 +#define RNP_DMA_TX_FLOW_CTRL_TM 0x8008 +/* DMA-ENABLE-IRQ */ +#define RNP_RING_BASE_N10 (0x8000) +#define RNP_RING_BASE_N500 (0x1000) +#define RNP_RING_OFFSET(i) (0x100 * i) +#define RNP_DMA_RX_START (0x10) +#define RNP_DMA_RX_READY (0x14) +#define RNP_DMA_TX_START (0x18) +#define RNP_DMA_TX_READY (0x1c) +#define RNP_DMA_INT_STAT (0x20) +#define RNP_DMA_INT_MASK (0x24) +#define TX_INT_MASK (1 << 1) +#define RX_INT_MASK (1 << 0) +#define RNP_DMA_INT_CLR (0x28) +#define RNP_DMA_INT_TRIG (0x2c) +/* RX-Queue Registers */ +#define RNP_DMA_REG_RX_DESC_BUF_BASE_ADDR_HI (0x30) +#define RNP_DMA_REG_RX_DESC_BUF_BASE_ADDR_LO (0x34) +#define RNP_DMA_REG_RX_DESC_BUF_LEN (0x38) +#define RNP_DMA_REG_RX_DESC_BUF_HEAD (0x3c) +#define RNP_DMA_REG_RX_DESC_BUF_TAIL (0x40) +#define RNP_DMA_REG_RX_DESC_FETCH_CTRL (0x44) +#define RNP_DMA_REG_RX_INT_DELAY_TIMER (0x48) +#define RNP_DMA_REG_RX_INT_DELAY_PKTCNT (0x4c) +#define RNP_DMA_REG_RX_ARB_DEF_LVL (0x50) +#define PCI_DMA_REG_RX_DESC_TIMEOUT_TH (0x54) +#define PCI_DMA_REG_RX_SCATTER_LENGTH (0x58) +/* TX-Queue Registers */ +#define RNP_DMA_REG_TX_DESC_BUF_BASE_ADDR_HI (0x60) +#define RNP_DMA_REG_TX_DESC_BUF_BASE_ADDR_LO (0x64) +#define RNP_DMA_REG_TX_DESC_BUF_LEN (0x68) +#define RNP_DMA_REG_TX_DESC_BUF_HEAD (0x6c) +#define RNP_DMA_REG_TX_DESC_BUF_TAIL (0x70) +#define RNP_DMA_REG_TX_DESC_FETCH_CTRL (0x74) +#define RNP_DMA_REG_TX_INT_DELAY_TIMER (0x78) +#define RNP_DMA_REG_TX_INT_DELAY_PKTCNT (0x7c) +#define RNP_DMA_REG_TX_ARB_DEF_LVL (0x80) +#define RNP_DMA_REG_TX_FLOW_CTRL_TH (0x84) +#define RNP_DMA_REG_TX_FLOW_CTRL_TM (0x88) +/* VEB Registers */ +#define VEB_TBL_CNTS 64 +#define RNP_DMA_PORT_VBE_MAC_LO_TBL_N10(port, vf) \ + (0x80A0 + 4 * (port) + 0x100 * (vf)) +#define RNP_DMA_PORT_VBE_MAC_HI_TBL_N10(port, vf) \ + (0x80B0 + 4 * (port) + 0x100 * (vf)) +#define RNP_DMA_PORT_VEB_VID_TBL_N10(port, vf) \ + (0x80C0 + 4 * (port) + 0x100 * (vf)) +#define RNP_DMA_PORT_VEB_VF_RING_TBL_N10(port, vf) \ + (0x80D0 + 4 * (port) + \ + 0x100 * (vf)) +/* [0:7]:Ring_id,[8:15]:vf_num,vf_num[7]=1=vf valid */ +#define RNP_DMA_PORT_VBE_MAC_LO_TBL_N500 (0x10c0) +#define RNP_DMA_PORT_VBE_MAC_HI_TBL_N500 (0x10c4) +#define RNP_DMA_PORT_VEB_VID_TBL_N500 (0x10c8) +#define RNP_DMA_PORT_VEB_VF_RING_TBL_N500 (0x10cc) +#define RNP_DMA_STATS_DMA_TO_MAC (0x1a0) +#define RNP_DMA_STATS_DMA_TO_SWITCH (0x1a4) +#define RNP_DMA_STATS_MAC_TO_MAC (0x1b0) +#define RNP_DMA_STATS_SWITCH_TO_SWITCH (0x1a4) +#define RNP_DMA_STATS_MAC_TO_DMA (0x1a8) +#define RNP_DMA_STATS_SWITCH_TO_DMA (0x1ac) +/* ===== PF-VF Functions ==== */ +#define VF_NUM_REG 0xa3000 +#define VF_NUM_REG_N10 0x75f000 +#define VF_NUM_REG_N500 (0xe000) +/* 8bit: 7:vf_actiove 6:fun0/fun1 [5:0]:vf_num */ +#define VF_NUM(vfnum, fun) \ + ((1 << 7) | (((fun) & 0x1) << 6) | ((vfnum) & 0x3f)) +#define PF_NUM(fun) (((fun) & 0x1) << 6) +/* ==== Ring-MSIX Registers (MSI-X_module_design.docs) === */ +#define RING_VECTOR(n) (0x4000 + 0x04 * (n)) + +static inline unsigned int p_rnpvf_rd_reg(void *reg) +{ + unsigned int v = ioread32((void *)(reg)); + + printk(" rd-reg: %p ==> 0x%08x\n", reg, v); + return v; +} +#define p_rnpvf_wr_reg(reg, val) \ + do { \ + printk(" wr-reg: %p <== 0x%08x \t#%-4d %s\n", (reg), \ + (val), __LINE__, __FILE__); \ + iowrite32((val), (void *)(reg)); \ + } while (0) + +#ifdef IO_PRINT +#define rnpvf_rd_reg(reg) p_rnpvf_rd_reg(reg) +#define rnpvf_wr_reg(reg, val) p_rnpvf_wr_reg(reg, val) +#else +#define rnpvf_rd_reg(reg) readl((void *)(reg)) +#define rnpvf_wr_reg(reg, val) writel((val), (void *)(reg)) +#endif + +#ifdef CONFIG_RNP_MBX_DEBUG +#define mbx_rd32(hw, reg) p_rnpvf_rd_reg((hw)->hw_addr + (reg)) +#define mbx_wr32(hw, reg, val) p_rnpvf_wr_reg((hw)->hw_addr + (reg), (val)) +#else +#define mbx_rd32(hw, reg) rnpvf_rd_reg((hw)->hw_addr + (reg)) +#define mbx_wr32(hw, reg, val) rnpvf_wr_reg((hw)->hw_addr + (reg), (val)) +#endif + +#define rd32(hw, off) rnpvf_rd_reg((hw)->hw_addr + (off)) +#define wr32(hw, off, val) rnpvf_wr_reg((hw)->hw_addr + (off), (val)) + +#define ring_rd32(ring, off) rnpvf_rd_reg((ring)->ring_addr + (off)) +#define ring_wr32(ring, off, val) \ + rnpvf_wr_reg((ring)->ring_addr + (off), (val)) + +#define pwr32(hw, reg, val) \ + do { \ + printk(" wr-reg: %p <== 0x%08x \t#%-4d %s\n", \ + (hw)->hw_addr + (reg), (val), __LINE__, __FILE__); \ + iowrite32((val), (hw)->hw_addr + (reg)); \ + } while (0) + +/* ==== log helper === */ +#ifdef DEBUG +#define hw_dbg(hw, fmt, args...) printk("hw-dbg : " fmt, ##args) +#else +#define hw_dbg(hw, fmt, args...) +#endif + +#endif /* _RNPVF_REGS_H_ */ diff --git a/drivers/net/ethernet/mucse/rnpvf/rnpvf.h b/drivers/net/ethernet/mucse/rnpvf/rnpvf.h new file mode 100644 index 0000000000000..99ce052f06ace --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/rnpvf.h @@ -0,0 +1,738 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#ifndef _RNPVF_H_ +#define _RNPVF_H_ + +#include +#include +#include +#include +#include +#include + +#include "vf.h" + +#define RNPVF_ALLOC_PAGE_ORDER 0 +#define RNPVF_PAGE_BUFFER_NUMS(ring) \ + (((1 << RNPVF_ALLOC_PAGE_ORDER) * PAGE_SIZE) >> 11) + +#define RNPVF_RX_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) + +#if defined(CONFIG_MXGBEVF_FIX_VF_QUEUE) && !defined(FIX_VF_QUEUE) +#define FIX_VF_BUG +#endif + +#if defined(CONFIG_MXGBEVF_FIX_MAC_PADDING) && !defined(FIX_MAC_PADDING) +#define FIX_MAC_PADDIN +#endif + +#if defined(CONFIG_MXGBEVF_OPTM_WITH_LARGE) && !defined(OPTM_WITH_LARGE) +#define OPTM_WITH_LPAGE +#endif + +#if (PAGE_SIZE < 8192) +//error +#ifdef OPTM_WITH_LPAGE +//#error can't open OPTM_WITH_LPAGE with PAGE_SIZE small than 8192 +#undef OPTM_WITH_LPAGE +#endif +#endif + +struct rnpvf_queue_stats { + u64 packets; + u64 bytes; +}; + +struct rnpvf_tx_queue_stats { + u64 restart_queue; + u64 tx_busy; + u64 tx_done_old; + u64 clean_desc; + u64 poll_count; + u64 irq_more_count; + u64 vlan_add; + u64 tx_irq_miss; + u64 tx_next_to_clean; + u64 tx_equal_count; +}; + +struct rnpvf_rx_queue_stats { + u64 driver_drop_packets; + u64 rsc_count; + u64 rsc_flush; + u64 non_eop_descs; + u64 alloc_rx_page_failed; + u64 alloc_rx_buff_failed; + u64 alloc_rx_page; + u64 csum_err; + u64 csum_good; + u64 poll_again_count; + u64 poll_count; + u64 vlan_remove; + u64 rx_irq_miss; + u64 rx_next_to_clean; + u64 rx_equal_count; +}; + +/* wrapper around a pointer to a socket buffer, + * so a DMA handle can be stored along with the Buffers + */ +struct rnpvf_tx_buffer { + struct rnp_tx_desc *next_to_watch; + unsigned long time_stamp; + struct sk_buff *skb; + unsigned int bytecount; + unsigned short gso_segs; + bool gso_need_padding; + + __be16 protocol; + DEFINE_DMA_UNMAP_ADDR(dma); + DEFINE_DMA_UNMAP_LEN(len); + union { + u32 tx_flags; + struct { + u16 vlan; + u16 cmd_flags; + }; + }; + __le32 mac_ip_len; + /* for control desc */ + union { + u32 mss_len_vf_num; + struct { + __le16 mss_len; + u8 vf_num; + u8 l4_hdr_len; + }; + }; + union { + u32 inner_vlan_tunnel_len; + struct { + u8 tunnel_hdr_len; + u8 inner_vlan_l; + u8 inner_vlan_h; + u8 resv; + }; + }; + bool ctx_flag; +}; + +struct rnpvf_rx_buffer { + struct sk_buff *skb; + dma_addr_t dma; + struct page *page; +#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) + __u32 page_offset; +#else + __u16 page_offset; +#endif + __u16 pagecnt_bias; +}; + +enum rnpvf_ring_state_t { + __RNPVF_RX_3K_BUFFER, + __RNPVF_RX_BUILD_SKB_ENABLED, + __RNPVF_TX_FDIR_INIT_DONE, + __RNPVF_TX_XPS_INIT_DONE, + __RNPVF_TX_DETECT_HANG, + __RNPVF_HANG_CHECK_ARMED, + __RNPVF_RX_CSUM_UDP_ZERO_ERR, + __RNPVF_RX_FCOE, +}; + +#define ring_uses_build_skb(ring) \ + test_bit(__RNPVF_RX_BUILD_SKB_ENABLED, &(ring)->state) + +/* now tx max 4k for one desc */ +#define RNPVF_MAX_TXD_PWR 12 +#define RNPVF_MAX_DATA_PER_TXD (1 << RNPVF_MAX_TXD_PWR) +/* Tx Descriptors needed, worst case */ +#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), RNPVF_MAX_DATA_PER_TXD) +#define DESC_NEEDED (MAX_SKB_FRAGS + 4) + +struct rnpvf_ring { + struct rnpvf_ring *next; /* pointer to next ring in q_vector */ + struct rnpvf_q_vector *q_vector; /* backpointer to host q_vector */ + struct net_device *netdev; /* netdev ring belongs to */ + struct device *dev; /* device for DMA mapping */ + void *desc; /* descriptor ring memory */ + union { + struct rnpvf_tx_buffer *tx_buffer_info; + struct rnpvf_rx_buffer *rx_buffer_info; + }; + unsigned long last_rx_timestamp; + unsigned long state; + u8 __iomem *ring_addr; + u8 __iomem *hw_addr; + u8 __iomem *tail; + u8 __iomem *dma_int_stat; + u8 __iomem *dma_int_mask; + u8 __iomem *dma_int_clr; + dma_addr_t dma; /* phys. address of descriptor ring */ + unsigned int size; /* length in bytes */ + u32 ring_flags; +#define RNPVF_RING_FLAG_DELAY_SETUP_RX_LEN ((u32)(1 << 0)) +#define RNPVF_RING_FLAG_CHANGE_RX_LEN ((u32)(1 << 1)) +#define RNPVF_RING_FLAG_DO_RESET_RX_LEN ((u32)(1 << 2)) +#define RNPVF_RING_SKIP_TX_START ((u32)(1 << 3)) +#define RNPVF_RING_NO_TUNNEL_SUPPORT ((u32)(1 << 4)) +#define RNPVF_RING_SIZE_CHANGE_FIX ((u32)(1 << 5)) +#define RNPVF_RING_SCATER_SETUP ((u32)(1 << 6)) +#define RNPVF_RING_STAGS_SUPPORT ((u32)(1 << 7)) +#define RNPVF_RING_DOUBLE_VLAN_SUPPORT ((u32)(1 << 8)) +#define RNPVF_RING_VEB_MULTI_FIX ((u32)(1 << 9)) +#define RNPVF_RING_IRQ_MISS_FIX ((u32)(1 << 10)) +#define RNPVF_RING_CHKSM_FIX ((u32)(1 << 11)) + + u8 vfnum; + u8 rnpvf_msix_off; + + u16 count; /* amount of descriptors */ + + u8 queue_index; /* queue_index needed for multiqueue queue management */ + u8 rnpvf_queue_idx; + + u16 next_to_use; + u16 next_to_clean; + + u16 device_id; +#ifdef OPTM_WITH_LPAGE + u16 rx_page_buf_nums; + u32 rx_per_buf_mem; + struct sk_buff *skb; +#endif + union { + u16 next_to_alloc; + struct { + u8 atr_sample_rate; + u8 atr_count; + }; + }; + + u8 dcb_tc; + struct rnpvf_queue_stats stats; + struct u64_stats_sync syncp; + union { + struct rnpvf_tx_queue_stats tx_stats; + struct rnpvf_rx_queue_stats rx_stats; + }; +} ____cacheline_internodealigned_in_smp; + +#define RNPVF_ITR_ADAPTIVE_MIN_INC 2 +#define RNPVF_ITR_ADAPTIVE_MIN_USECS 5 +#define RNPVF_ITR_ADAPTIVE_MAX_USECS 800 +#define RNPVF_ITR_ADAPTIVE_LATENCY 0x400 +#define RNPVF_ITR_ADAPTIVE_BULK 0x00 +#define RNPVF_ITR_ADAPTIVE_MASK_USECS \ + (RNPVF_ITR_ADAPTIVE_LATENCY - RNPVF_ITR_ADAPTIVE_MIN_INC) + +/* How many Rx Buffers do we bundle into one write to the hardware ? */ +#define RNPVF_RX_BUFFER_WRITE 16 /* Must be power of 2 */ +#define RNPVF_VF_MAX_TX_QUEUES 32 +#define RNPVF_VF_MAX_RX_QUEUES 32 +#define MAX_RX_QUEUES RNPVF_VF_MAX_RX_QUEUES +#define MAX_TX_QUEUES RNPVF_VF_MAX_TX_QUEUES + +#ifndef RNPVF_PKT_TIMEOUT +#define RNPVF_PKT_TIMEOUT 30 +#endif + +#ifndef RNPVF_RX_PKT_POLL_BUDGET +#define RNPVF_RX_PKT_POLL_BUDGET 64 +#endif + +#ifndef RNPVF_TX_PKT_POLL_BUDGET +#define RNPVF_TX_PKT_POLL_BUDGET 0x30 +#endif + +#ifndef RNPVF_PKT_TIMEOUT_TX +#define RNPVF_PKT_TIMEOUT_TX 100 +#endif + +#define RNPVF_MIN_RX_WORK (32) +#define RNPVF_DEFAULT_RX_WORK (64) +#define RNPVF_MAX_RX_WORK (512) +#define RNPVF_WORK_ALIGN (2) +#define RNPVF_MIN_TX_FRAME (1) +#define RNPVF_MAX_TX_FRAME (256) +#define RNPVF_MIN_TX_USEC (30) +#define RNPVF_MAX_TX_USEC (10000) + +#define RNPVF_MIN_RX_FRAME (1) +#define RNPVF_MAX_RX_FRAME (256) +#define RNPVF_MIN_RX_USEC (10) +#define RNPVF_MAX_RX_USEC (10000) + +#define RNPVF_MIN_TX_WORK (32) +#define RNPVF_MAX_TX_WORK (512) +#define RNPVF_DEFAULT_TX_WORK 256 +#define RNPVF_DEFAULT_TXD 512 +#define RNPVF_DEFAULT_RXD 512 +#define RNPVF_MAX_TXD 4096 +#define RNPVF_MIN_TXD 256 +#define RNPVF_MAX_RXD 4096 +#define RNPVF_MIN_RXD 256 + +#ifndef TSRN10_RX_DEFAULT_BURST +#define TSRN10_RX_DEFAULT_BURST 16 +#endif + +#ifndef TSRN10_RX_DEFAULT_LINE +#define TSRN10_RX_DEFAULT_LINE 64 +#endif + +#define TSRN10_TX_DEFAULT_BURST 8 + +/* Supported Rx Buffer Sizes */ +#define RNPVF_RXBUFFER_256 256 /* Used for packet split */ +#define RNPVF_RXBUFFER_2K 2048 +#define RNPVF_RXBUFFER_1536 1536 +#define RNPVF_RXBUFFER_3K 3072 +#define RNPVF_RXBUFFER_4K 4096 +#define RNPVF_RXBUFFER_8K 8192 +#define RNPVF_RXBUFFER_10K 10240 + +#define RNPVF_RX_HDR_SIZE RNPVF_RXBUFFER_256 + +#define MAXIMUM_ETHERNET_VLAN_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN) + +#define RNPVF_TX_FLAGS_CSUM ((u32)(1)) +#define RNPVF_TX_FLAGS_VLAN ((u32)(1 << 1)) +#define RNPVF_TX_FLAGS_TSO ((u32)(1 << 2)) +#define RNPVF_TX_FLAGS_IPV4 ((u32)(1 << 3)) +#define RNPVF_TX_FLAGS_FCOE ((u32)(1 << 4)) +#define RNPVF_TX_FLAGS_FSO ((u32)(1 << 5)) +#define RNPVF_TX_FLAGS_VLAN_MASK 0xffff0000 +#define RNPVF_TX_FLAGS_VLAN_PRIO_MASK 0x0000e000 +#define RNPVF_TX_FLAGS_VLAN_SHIFT 16 + +#define RNPVF_GSO_PARTIAL_FEATURES \ + (NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM | \ + NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_UDP_TUNNEL_CSUM) + +struct rnpvf_ring_container { + struct rnpvf_ring *ring; /* pointer to linked list of rings */ + unsigned long next_update; /* jiffies value of last update */ + unsigned int total_bytes; /* total bytes processed this int */ + unsigned int total_packets; /* total packets processed this int */ + unsigned int total_packets_old; + u8 count; /* total number of rings in vector */ + u8 itr; /* current ITR setting for ring */ + u16 add_itr; +}; + +/* iterator for handling rings in ring container */ +#define rnpvf_for_each_ring(pos, head) \ + for (pos = (head).ring; pos != NULL; pos = pos->next) + +/* MAX_MSIX_Q_VECTORS of these are allocated, + * but we only use one per queue-specific vector. + */ +struct rnpvf_q_vector { + int old_rx_count; + int new_rx_count; + int large_times; + int small_times; + int too_small_times; + int middle_time; + struct rnpvf_adapter *adapter; + u16 v_idx; + /* index of q_vector within array, also used for + * finding the bit in EICR and friends that + * represents the vector for this rings + */ + struct rnpvf_ring_container rx, tx; + + struct napi_struct napi; + cpumask_t affinity_mask; + int numa_node; + u16 itr_rx; + u16 itr_tx; + struct rcu_head rcu; /* to avoid race with update stats on free */ + u32 vector_flags; +#define RNPVF_QVECTOR_FLAG_IRQ_MISS_CHECK ((u32)(1 << 0)) +#define RNPVF_QVECTOR_FLAG_ITR_FEATURE ((u32)(1 << 1)) +#define RNPVF_QVECTOR_FLAG_REDUCE_TX_IRQ_MISS ((u32)(1 << 2)) + + int irq_check_usecs; + struct hrtimer irq_miss_check_timer; + + char name[IFNAMSIZ + 9]; + + /* for dynamic allocation of rings associated with this q_vector */ + struct rnpvf_ring ring[0] ____cacheline_internodealigned_in_smp; +}; + +static inline __le16 rnpvf_test_staterr(union rnp_rx_desc *rx_desc, + const u16 stat_err_bits) +{ + return rx_desc->wb.cmd & cpu_to_le16(stat_err_bits); +} + +static inline __le16 rnpvf_get_stat(union rnp_rx_desc *rx_desc, + const u16 stat_mask) +{ + return rx_desc->wb.cmd & cpu_to_le16(stat_mask); +} + +static inline u16 rnpvf_desc_unused(struct rnpvf_ring *ring) +{ + u16 ntc = ring->next_to_clean; + u16 ntu = ring->next_to_use; + + return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1; +} + +/* + * microsecond values for various ITR rates shifted by 2 to fit itr register + * with the first 3 bits reserved 0 + */ +#define RNPVF_MIN_RSC_ITR 24 +#define RNPVF_100K_ITR 40 +#define RNPVF_20K_ITR 200 +#define RNPVF_10K_ITR 400 +#define RNPVF_8K_ITR 500 + +/* Helper macros to switch between ints/sec and what the register uses. + * And yes, it's the same math going both ways. The lowest value + * supported by all of the rnp hardware is 8. + */ +#define EITR_INTS_PER_SEC_TO_REG(_eitr) \ + ((_eitr) ? (1000000000 / ((_eitr) * 256)) : 8) +#define EITR_REG_TO_INTS_PER_SEC EITR_INTS_PER_SEC_TO_REG + +#define RNPVF_DESC_UNUSED(R) \ + ((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \ + (R)->next_to_clean - (R)->next_to_use - 1) + +#define RNPVF_RX_DESC(R, i) (&(((union rnp_rx_desc *)((R)->desc))[i])) +#define RNPVF_TX_DESC(R, i) (&(((struct rnp_tx_desc *)((R)->desc))[i])) +#define RNPVF_TX_CTXTDESC(R, i) \ + (&(((struct rnp_tx_ctx_desc *)((R)->desc))[i])) + +#define RNPVF_N10_MAX_JUMBO_FRAME_SIZE \ + 9590 /* Maximum Supported Size 9.5KB */ +#define RNPVF_N500_MAX_JUMBO_FRAME_SIZE \ + 9722 /* Maximum Supported Size 9.5KB */ +#define RNPVF_MIN_MTU 68 + +#define MAX_MSIX_VECTORS 4 +#define OTHER_VECTOR 1 +#define NON_Q_VECTORS (OTHER_VECTOR) + +#define MAX_MSIX_Q_VECTORS 2 + +#define MIN_MSIX_Q_VECTORS 1 +#define MIN_MSIX_COUNT (MIN_MSIX_Q_VECTORS + NON_Q_VECTORS) + +enum phy_type { + PHY_TYPE_NONE = 0, + PHY_TYPE_1G_BASE_KX, + PHY_TYPE_RGMII, + PHY_TYPE_10G_BASE_KR, + PHY_TYPE_25G_BASE_KR, + PHY_TYPE_40G_BASE_KR4, +}; + +struct rnpvf_hw { + void *back; + u8 __iomem *hw_addr; + u8 __iomem *hw_addr_bar0; + u8 __iomem *ring_msix_base; + u8 vfnum; +#define VF_NUM_MASK 0x3f + struct pci_dev *pdev; + + u16 device_id; + u16 vendor_id; + u16 subsystem_device_id; + u16 subsystem_vendor_id; + + enum rnp_board_type board_type; + + u32 dma_version; + u16 min_length; + u16 max_length; + + u16 queue_ring_base; + u32 tx_items_count; + u32 rx_items_count; + u16 mac_type; + u16 phy_type; + int mtu; + u16 link; + u16 speed; + + struct rnpvf_hw_operations ops; + struct rnp_mac_info mac; + struct rnp_fc_info fc; + struct rnp_mbx_info mbx; + + bool adapter_stopped; + u32 api_version; + int fw_version; + int usecstocount; +#define PF_FEATURE_VLAN_FILTER BIT(0) +#define PF_NCSI_EN BIT(1) +#define PF_MAC_SPOOF BIT(2) + u32 pf_feature; + + int mode; +#define RNPVF_NET_FEATURE_SG ((u32)(1 << 0)) +#define RNPVF_NET_FEATURE_TX_CHECKSUM ((u32)(1 << 1)) +#define RNPVF_NET_FEATURE_RX_CHECKSUM ((u32)(1 << 2)) +#define RNPVF_NET_FEATURE_TSO ((u32)(1 << 3)) +#define RNPVF_NET_FEATURE_TX_UDP_TUNNEL (1 << 4) +#define RNPVF_NET_FEATURE_VLAN_FILTER (1 << 5) +#define RNPVF_NET_FEATURE_VLAN_OFFLOAD (1 << 6) +#define RNPVF_NET_FEATURE_RX_NTUPLE_FILTER (1 << 7) +#define RNPVF_NET_FEATURE_TCAM (1 << 8) +#define RNPVF_NET_FEATURE_RX_HASH (1 << 9) +#define RNPVF_NET_FEATURE_RX_FCS (1 << 10) +#define RNPVF_NET_FEATURE_HW_TC (1 << 11) +#define RNPVF_NET_FEATURE_USO (1 << 12) +#define RNPVF_NET_FEATURE_STAG_FILTER (1 << 13) +#define RNPVF_NET_FEATURE_STAG_OFFLOAD (1 << 14) + + u32 feature_flags; +}; +#define VFNUM(mbx, num) ((num) & mbx->vf_num_mask) + +enum irq_mode_enum { + irq_mode_msix, + irq_mode_msi, + irq_mode_legency, +}; + +/* board specific private data structure */ +struct rnpvf_adapter { + unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; +#define GET_VFNUM_FROM_BAR0 BIT(0) + u16 status; + u16 vf_vlan; + struct timer_list watchdog_timer; + u16 bd_number; + struct work_struct reset_task; + bool promisc_mode; + + /* Interrupt Throttle Rate */ + u16 rx_itr_setting; + u16 tx_itr_setting; + + u16 rx_usecs; + u16 rx_frames; + u16 tx_usecs; + u16 tx_frames; + u32 pkt_rate_low; + u16 rx_usecs_low; + u32 pkt_rate_high; + u16 rx_usecs_high; + u32 sample_interval; + u32 adaptive_rx_coal; + u32 adaptive_tx_coal; + u32 auto_rx_coal; + u32 napi_budge; + u32 tx_work_limit; + /* TX */ + struct rnpvf_ring + *tx_ring[MAX_TX_QUEUES] ____cacheline_aligned_in_smp; + int tx_ring_item_count; + int num_q_vectors; + int num_tx_queues; + u64 restart_queue; + u64 hw_csum_tx_good; + u64 lsc_int; + u64 hw_tso_ctxt; + u64 hw_tso6_ctxt; + u32 tx_timeout_count; + + /* RX */ + struct rnpvf_ring *rx_ring[MAX_RX_QUEUES]; + int rx_ring_item_count; + int num_rx_queues; + u64 hw_csum_rx_error; + u64 hw_rx_no_dma_resources; + u64 hw_csum_rx_good; + u64 non_eop_descs; + + u32 alloc_rx_page_failed; + u32 alloc_rx_buff_failed; + + int vector_off; + int num_other_vectors; + int irq_mode; + struct rnpvf_q_vector *q_vector[MAX_MSIX_VECTORS]; + + int num_msix_vectors; + struct msix_entry *msix_entries; + + u32 dma_channels; + /* the real used dma ring channels */ + + /* Some features need tri-state capability, + * thus the additional *_CAPABLE flags. + */ + u32 flags; +#define RNPVF_FLAG_IN_WATCHDOG_TASK ((u32)(1)) +#define RNPVF_FLAG_IN_NETPOLL ((u32)(1 << 1)) +#define RNPVF_FLAG_PF_SET_VLAN ((u32)(1 << 2)) +#define RNPVF_FLAG_PF_UPDATE_MTU ((u32)(1 << 3)) +#define RNPVF_FLAG_PF_UPDATE_MAC ((u32)(1 << 4)) +#define RNPVF_FLAG_PF_UPDATE_VLAN ((u32)(1 << 5)) +#define RNPVF_FLAG_PF_RESET ((u32)(1 << 6)) +#define RNPVF_FLAG_PF_RESET_REQ ((u32)(1 << 7)) +#define RNPVF_FLAG_MSI_CAPABLE ((u32)(1 << 8)) +#define RNPVF_FLAG_MSI_ENABLED ((u32)(1 << 9)) +#define RNPVF_FLAG_MSIX_CAPABLE ((u32)(1 << 10)) +#define RNPVF_FLAG_MSIX_ENABLED ((u32)(1 << 11)) +#define RNPVF_FLAG_RX_CHKSUM_ENABLED ((u32)(1 << 12)) +#define RNPVF_FLAG_TX_VLAN_OFFLOAD ((u32)(1 << 13)) +#define RNPVF_FLAG_RX_VLAN_OFFLOAD ((u32)(1 << 14)) + + u32 priv_flags; +#define RNPVF_PRIV_FLAG_FT_PADDING BIT(0) +#define RNPVF_PRIV_FLAG_PADDING_DEBUG BIT(1) +#define RNPVF_PRIV_FLAG_FCS_ON BIT(2) +#define RNPVF_PRIV_FLAG_TX_PADDING BIT(3) + + /* OS defined structs */ + struct net_device *netdev; + struct pci_dev *pdev; + + /* structs defined in rnp_vf.h */ + struct rnpvf_hw hw; + u16 msg_enable; + struct rnpvf_hw_stats stats; + struct rnpvf_hw_stats_own hw_stats; + u64 zero_base; + /* Interrupt Throttle Rate */ + u32 eitr_param; + + unsigned long state; + u64 tx_busy; + u32 link_speed; + bool link_up; + + struct work_struct watchdog_task; + + u8 port; + + spinlock_t mbx_lock; + char name[60]; +}; + +enum rnpvf_state_t { + __RNPVF_TESTING, + __RNPVF_RESETTING, + __RNPVF_DOWN, + __RNPVF_REMOVE, + __RNPVF_MBX_POLLING, + __RNPVF_LINK_DOWN +}; + +struct rnpvf_cb { + union { /* Union defining head/tail partner */ + struct sk_buff *head; + struct sk_buff *tail; + }; + dma_addr_t dma; + u16 append_cnt; + bool page_released; +}; +#define RNPVF_CB(skb) ((struct rnpvf_cb *)(skb)->cb) + +#define RING2ADAPT(ring) netdev_priv((ring)->netdev) + +enum rnpvf_boards { + board_n10, + board_n500, +}; + +extern const struct rnpvf_info rnpvf_82599_vf_info; +extern const struct rnpvf_info rnpvf_X540_vf_info; +extern const struct rnp_mbx_operations rnpvf_mbx_ops; + +/* needed by ethtool.c */ +extern char rnpvf_driver_name[]; +extern const char rnpvf_driver_version[]; + +extern void rnpvf_up(struct rnpvf_adapter *adapter); +extern void rnpvf_down(struct rnpvf_adapter *adapter); +extern void rnpvf_reinit_locked(struct rnpvf_adapter *adapter); +extern void rnpvf_reset(struct rnpvf_adapter *adapter); +extern void rnpvf_set_ethtool_ops(struct net_device *netdev); +extern int rnpvf_setup_rx_resources(struct rnpvf_adapter *, + struct rnpvf_ring *); +extern int rnpvf_setup_tx_resources(struct rnpvf_adapter *, + struct rnpvf_ring *); +extern void rnpvf_free_rx_resources(struct rnpvf_adapter *, + struct rnpvf_ring *); +extern void rnpvf_free_tx_resources(struct rnpvf_adapter *, + struct rnpvf_ring *); +extern void rnpvf_update_stats(struct rnpvf_adapter *adapter); +extern int ethtool_ioctl(struct ifreq *ifr); +extern void remove_mbx_irq(struct rnpvf_adapter *adapter); +extern void rnpvf_clear_interrupt_scheme(struct rnpvf_adapter *adapter); +extern int register_mbx_irq(struct rnpvf_adapter *adapter); +extern int rnpvf_init_interrupt_scheme(struct rnpvf_adapter *adapter); +extern int rnpvf_close(struct net_device *netdev); +extern int rnpvf_open(struct net_device *netdev); + +extern void rnp_napi_add_all(struct rnpvf_adapter *adapter); +extern void rnp_napi_del_all(struct rnpvf_adapter *adapter); + +extern int rnpvf_sysfs_init(struct net_device *ndev); +extern void rnpvf_sysfs_exit(struct net_device *ndev); + +static inline int rnpvf_is_pf1(struct pci_dev *pdev) +{ + return ((pdev->devfn) ? 1 : 0); +} + +static inline struct netdev_queue * +txring_txq(const struct rnpvf_ring *ring) +{ + return netdev_get_tx_queue(ring->netdev, ring->queue_index); +} +/* + * FCoE requires that all Rx buffers be over 2200 bytes in length. Since + * this is twice the size of a half page we need to double the page order + * for FCoE enabled Rx queues. + */ +static inline unsigned int rnpvf_rx_bufsz(struct rnpvf_ring *ring) +{ + /* 1 rx-desc trans max half page(2048), for jumbo frame sg is needed */ + return RNPVF_RXBUFFER_1536; +} + +/* SG , 1 rx-desc use one page */ +static inline unsigned int rnpvf_rx_pg_order(struct rnpvf_ring *ring) +{ + return 0; +} +#define rnpvf_rx_pg_size(_ring) (PAGE_SIZE << rnpvf_rx_pg_order(_ring)) + +static inline u32 rnpvf_rx_desc_used_hw(struct rnpvf_hw *hw, + struct rnpvf_ring *rx_ring) +{ + u32 head = ring_rd32(rx_ring, RNP_DMA_REG_RX_DESC_BUF_HEAD); + u32 tail = ring_rd32(rx_ring, RNP_DMA_REG_RX_DESC_BUF_TAIL); + u16 count = rx_ring->count; + + return ((tail >= head) ? (count - tail + head) : (head - tail)); +} + +static inline u32 rnpvf_tx_desc_unused_hw(struct rnpvf_hw *hw, + struct rnpvf_ring *tx_ring) +{ + u32 head = ring_rd32(tx_ring, RNP_DMA_REG_TX_DESC_BUF_HEAD); + u32 tail = ring_rd32(tx_ring, RNP_DMA_REG_TX_DESC_BUF_TAIL); + u16 count = tx_ring->count; + + return ((tail > head) ? (count - tail + head) : (head - tail)); +} + +#define IS_VALID_VID(vid) ((vid) >= 0 && (vid) < 4096) + +#endif /* _RNPVF_H_ */ diff --git a/drivers/net/ethernet/mucse/rnpvf/rnpvf_main.c b/drivers/net/ethernet/mucse/rnpvf/rnpvf_main.c new file mode 100644 index 0000000000000..85a25bc5a2f7b --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/rnpvf_main.c @@ -0,0 +1,6434 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rnpvf.h" + +#ifdef FIX_VF_BUG +#define CONFIG_BAR4_PFVFNUM 0 +#else +#define CONFIG_BAR4_PFVFNUM 1 +#endif +char rnpvf_driver_name[] = "rnpvf"; +static const char rnpvf_driver_string[] = + "Mucse(R) 10/40G Gigabit PCI Express Virtual Function Network Driver"; + +#define DRV_VERSION "1.0.1-rc7" +const char rnpvf_driver_version[] = DRV_VERSION; +static const char rnpvf_copyright[] = + "Copyright (c) 2020 - 2024 Mucse Corporation."; + +extern const struct rnpvf_info rnp_n10_vf_info; + +static const struct rnpvf_info *rnpvf_info_tbl[] = { + [board_n10] = &rnp_n10_vf_info, +}; + +#define N10_BOARD board_n10 + +static unsigned int fix_eth_name; +module_param(fix_eth_name, uint, 0000); +MODULE_PARM_DESC(fix_eth_name, "set eth adapter name to rnpvfXX"); +static struct pci_device_id rnpvf_pci_tbl[] = { + { PCI_DEVICE(0x8848, 0x1080), .driver_data = N10_BOARD }, + { PCI_DEVICE(0x8848, 0x1084), .driver_data = N10_BOARD }, + { PCI_DEVICE(0x8848, 0x1081), .driver_data = N10_BOARD }, + { PCI_DEVICE(0x8848, 0x1083), .driver_data = N10_BOARD }, + { PCI_DEVICE(0x8848, 0x1C80), .driver_data = N10_BOARD }, + { PCI_DEVICE(0x8848, 0x1C81), .driver_data = N10_BOARD }, + { PCI_DEVICE(0x8848, 0x1C83), .driver_data = N10_BOARD }, + /* required last entry */ + { + 0, + }, +}; + +MODULE_DEVICE_TABLE(pci, rnpvf_pci_tbl); +MODULE_AUTHOR("Mucse Corporation, "); +MODULE_DESCRIPTION("Mucse(R) N10/N400 Virtual Function Driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_VERSION); + +#define DEFAULT_MSG_ENABLE \ + (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) +static int debug = -1; +module_param(debug, int, 0000); +MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); + +static int pci_using_hi_dma; + +/* forward decls */ +static void rnpvf_set_itr(struct rnpvf_q_vector *q_vector); +static void rnpvf_free_all_rx_resources(struct rnpvf_adapter *adapter); + +#define RNPVF_XDP_PASS 0 +#define RNPVF_XDP_CONSUMED 1 +#define RNPVF_XDP_TX 2 + +static void rnpvf_pull_tail(struct sk_buff *skb); +#ifdef OPTM_WITH_LPAGE +static bool rnpvf_alloc_mapped_page(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *bi, + union rnp_rx_desc *rx_desc, u16 bufsz, + u64 fun_id); + +static void rnpvf_put_rx_buffer(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer); +#else /* OPTM_WITH_LPAGE */ +static bool rnpvf_alloc_mapped_page(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *bi); +static void rnpvf_put_rx_buffer(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + struct sk_buff *skb); +#endif /* OPTM_WITH_LPAGE */ + +/** + * rnpvf_set_ivar - set IVAR registers - maps interrupt causes to vectors + * @adapter: pointer to adapter struct + * @direction: 0 for Rx, 1 for Tx, -1 for other causes + * @queue: queue to map the corresponding interrupt to + * @msix_vector: the vector to map to the corresponding queue + */ +static void rnpvf_set_ring_vector(struct rnpvf_adapter *adapter, + u8 rnpvf_queue, u8 rnpvf_msix_vector) +{ + struct rnpvf_hw *hw = &adapter->hw; + u32 data = 0; + + data = hw->vfnum << 24; + data |= (rnpvf_msix_vector << 8); + data |= (rnpvf_msix_vector << 0); + DPRINTK(IFUP, INFO, + "Set Ring-Vector queue:%d (reg:0x%x) <-- Rx-MSIX:%d, Tx-MSIX:%d\n", + rnpvf_queue, RING_VECTOR(rnpvf_queue), rnpvf_msix_vector, + rnpvf_msix_vector); + + rnpvf_wr_reg(hw->ring_msix_base + RING_VECTOR(rnpvf_queue), data); +} + +void rnpvf_unmap_and_free_tx_resource(struct rnpvf_ring *ring, + struct rnpvf_tx_buffer *tx_buffer) +{ + if (tx_buffer->skb) { + dev_kfree_skb_any(tx_buffer->skb); + if (dma_unmap_len(tx_buffer, len)) + dma_unmap_single(ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } else if (dma_unmap_len(tx_buffer, len)) { + dma_unmap_page(ring->dev, dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } + tx_buffer->next_to_watch = NULL; + tx_buffer->skb = NULL; + dma_unmap_len_set(tx_buffer, len, 0); + /* tx_buffer must be completely set up in the transmit path */ +} + +static void rnpvf_tx_timeout(struct net_device *netdev); + +/** + * rnpvf_clean_tx_irq - Reclaim resources after transmit completes + * @q_vector: board private structure + * @tx_ring: tx ring to clean + **/ +static bool rnpvf_clean_tx_irq(struct rnpvf_q_vector *q_vector, + struct rnpvf_ring *tx_ring) +{ + struct rnpvf_adapter *adapter = q_vector->adapter; + struct rnpvf_tx_buffer *tx_buffer; + struct rnp_tx_desc *tx_desc; + unsigned int total_bytes = 0, total_packets = 0; + unsigned int budget = adapter->tx_work_limit; + unsigned int i = tx_ring->next_to_clean; + + if (test_bit(__RNPVF_DOWN, &adapter->state)) + return true; + tx_ring->tx_stats.poll_count++; + tx_buffer = &tx_ring->tx_buffer_info[i]; + tx_desc = RNPVF_TX_DESC(tx_ring, i); + i -= tx_ring->count; + + do { + struct rnp_tx_desc *eop_desc = tx_buffer->next_to_watch; + + /* if next_to_watch is not set then there is no work pending */ + if (!eop_desc) + break; + + /* prevent any other reads prior to eop_desc */ + rmb(); + + /* if eop DD is not set pending work has not been completed */ + if (!(eop_desc->cmd & cpu_to_le16(RNP_TXD_STAT_DD))) + break; + + /* clear next_to_watch to prevent false hangs */ + tx_buffer->next_to_watch = NULL; + + /* update the statistics for this packet */ + total_bytes += tx_buffer->bytecount; + total_packets += tx_buffer->gso_segs; + + /* free the skb */ + dev_kfree_skb_any(tx_buffer->skb); + + /* unmap skb header data */ + dma_unmap_single(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + + /* clear tx_buffer data */ + tx_buffer->skb = NULL; + dma_unmap_len_set(tx_buffer, len, 0); + + /* unmap remaining buffers */ + while (tx_desc != eop_desc) { + tx_buffer++; + tx_desc++; + i++; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buffer = tx_ring->tx_buffer_info; + tx_desc = RNPVF_TX_DESC(tx_ring, 0); + } + + /* unmap any remaining paged data */ + if (dma_unmap_len(tx_buffer, len)) { + dma_unmap_page( + tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + dma_unmap_len_set(tx_buffer, len, 0); + } + } + + /* move us one more past the eop_desc for start of next pkt */ + tx_buffer++; + tx_desc++; + i++; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buffer = tx_ring->tx_buffer_info; + tx_desc = RNPVF_TX_DESC(tx_ring, 0); + } + + /* issue prefetch for next Tx descriptor */ + prefetch(tx_desc); + + /* update budget accounting */ + budget--; + } while (likely(budget)); + + i += tx_ring->count; + tx_ring->next_to_clean = i; + u64_stats_update_begin(&tx_ring->syncp); + tx_ring->stats.bytes += total_bytes; + tx_ring->stats.packets += total_packets; + u64_stats_update_end(&tx_ring->syncp); + q_vector->tx.total_bytes += total_bytes; + q_vector->tx.total_packets += total_packets; + + netdev_tx_completed_queue(txring_txq(tx_ring), total_packets, + total_bytes); + + if (!(q_vector->vector_flags & + RNPVF_QVECTOR_FLAG_REDUCE_TX_IRQ_MISS)) { +#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) + if (unlikely(total_packets && + netif_carrier_ok(tx_ring->netdev) && + (rnpvf_desc_unused(tx_ring) >= + TX_WAKE_THRESHOLD))) { + /* Make sure that anybody stopping the queue after this + * sees the new next_to_clean. + */ + smp_mb(); + if (__netif_subqueue_stopped( + tx_ring->netdev, + tx_ring->queue_index) && + !test_bit(__RNPVF_DOWN, &adapter->state)) { + netif_wake_subqueue(tx_ring->netdev, + tx_ring->queue_index); + ++tx_ring->tx_stats.restart_queue; + } + } + } + + return !!budget; +} + +static inline void rnpvf_rx_hash(struct rnpvf_ring *ring, + union rnp_rx_desc *rx_desc, + struct sk_buff *skb) +{ + int rss_type; + + if (!(ring->netdev->features & NETIF_F_RXHASH)) + return; + +#define RNPVF_RSS_TYPE_MASK 0xc0 + rss_type = rx_desc->wb.cmd & RNPVF_RSS_TYPE_MASK; + skb_set_hash(skb, le32_to_cpu(rx_desc->wb.rss_hash), + rss_type ? PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); +} + +/** + * rnpvf_rx_checksum - indicate in skb if hw indicated a good cksum + * @ring: structure containing ring specific data + * @rx_desc: current Rx descriptor being processed + * @skb: skb currently being received and modified + **/ +static inline void rnpvf_rx_checksum(struct rnpvf_ring *ring, + union rnp_rx_desc *rx_desc, + struct sk_buff *skb) +{ + bool encap_pkt = false; + + skb_checksum_none_assert(skb); + + /* Rx csum disabled */ + if (!(ring->netdev->features & NETIF_F_RXCSUM)) + return; + + /* vxlan packet handle ? */ + if (!(ring->ring_flags & RNPVF_RING_NO_TUNNEL_SUPPORT)) { + if (rnpvf_get_stat(rx_desc, RNP_RXD_STAT_TUNNEL_MASK) == + RNP_RXD_STAT_TUNNEL_VXLAN) { + encap_pkt = true; + skb->encapsulation = 1; + skb->ip_summed = CHECKSUM_NONE; + } + } + + /* if L3/L4 error:ignore errors from veb(other vf) */ + if (unlikely(rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_ERR_MASK))) { + ring->rx_stats.csum_err++; + return; + } + ring->rx_stats.csum_good++; + /* It must be a TCP or UDP packet with a valid checksum */ + skb->ip_summed = CHECKSUM_UNNECESSARY; + if (encap_pkt) { + /* If we checked the outer header let the stack know */ + skb->csum_level = 1; + } +} + +static inline void rnpvf_update_rx_tail(struct rnpvf_ring *rx_ring, + u32 val) +{ + rx_ring->next_to_use = val; + + /* update next to alloc since we have filled the ring */ + rx_ring->next_to_alloc = val; + /* + * Force memory writes to complete before letting h/w + * know there are new descriptors to fetch. (Only + * applicable for weak-ordered memory model archs, + * such as IA-64). + */ + wmb(); + rnpvf_wr_reg(rx_ring->tail, val); +} + +#ifndef OPTM_WITH_LPAGE +/** + * rnpvf_alloc_rx_buffers - Replace used receive buffers + * @rx_ring: ring to place buffers on + * @cleaned_count: number of buffers to replace + **/ +void rnpvf_alloc_rx_buffers(struct rnpvf_ring *rx_ring, u16 cleaned_count) +{ + union rnp_rx_desc *rx_desc; + struct rnpvf_rx_buffer *bi; + u16 i = rx_ring->next_to_use; + u64 fun_id = ((u64)(rx_ring->vfnum) << (32 + 24)); + u16 bufsz; + + /* nothing to do */ + if (!cleaned_count) + return; + + rx_desc = RNPVF_RX_DESC(rx_ring, i); + BUG_ON(rx_desc == NULL); + bi = &rx_ring->rx_buffer_info[i]; + BUG_ON(bi == NULL); + i -= rx_ring->count; + bufsz = rnpvf_rx_bufsz(rx_ring); + + do { + if (!rnpvf_alloc_mapped_page(rx_ring, bi)) + break; + + dma_sync_single_range_for_device(rx_ring->dev, bi->dma, + bi->page_offset, bufsz, + DMA_FROM_DEVICE); + + /* + * Refresh the desc even if buffer_addrs didn't change + * because each write-back erases this info. + */ + rx_desc->pkt_addr = + cpu_to_le64(bi->dma + bi->page_offset + fun_id); + /* clean dd */ + rx_desc->cmd = 0; + + rx_desc++; + bi++; + i++; + if (unlikely(!i)) { + rx_desc = RNPVF_RX_DESC(rx_ring, 0); + bi = rx_ring->rx_buffer_info; + i -= rx_ring->count; + } + + /* clear the hdr_addr for the next_to_use descriptor */ + cleaned_count--; + } while (cleaned_count); + + i += rx_ring->count; + + if (rx_ring->next_to_use != i) + rnpvf_update_rx_tail(rx_ring, i); +} +#endif + +/** + * rnpvf_reuse_rx_page - page flip buffer and store it back on the ring + * @rx_ring: rx descriptor ring to store buffers on + * @old_buff: donor buffer to have page reused + * + * Synchronizes page for reuse by the adapter + **/ +static void rnpvf_reuse_rx_page(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *old_buff) +{ + struct rnpvf_rx_buffer *new_buff; + u16 nta = rx_ring->next_to_alloc; + + new_buff = &rx_ring->rx_buffer_info[nta]; + + /* update, and store next to alloc */ + nta++; + rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; + + /* + * Transfer page from old buffer to new buffer. + * Move each member individually to avoid possible store + * forwarding stalls and unnecessary copy of skb. + */ + new_buff->dma = old_buff->dma; + new_buff->page = old_buff->page; + new_buff->page_offset = old_buff->page_offset; + new_buff->pagecnt_bias = old_buff->pagecnt_bias; +} + +static inline bool rnpvf_page_is_reserved(struct page *page) +{ + return (page_to_nid(page) != numa_mem_id()) || + page_is_pfmemalloc(page); +} + +static bool rnpvf_can_reuse_rx_page(struct rnpvf_rx_buffer *rx_buffer) +{ + unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; + struct page *page = rx_buffer->page; + +#ifdef OPTM_WITH_LPAGE + return false; +#endif + /* avoid re-using remote pages */ + if (unlikely(rnpvf_page_is_reserved(page))) + return false; + +#if (PAGE_SIZE < 8192) + /* if we are only owner of page we can reuse it */ + if (unlikely((page_ref_count(page) - pagecnt_bias) > 1)) + return false; +#else + /* + * The last offset is a bit aggressive in that we assume the + * worst case of FCoE being enabled and using a 3K buffer. + * However this should have minimal impact as the 1K extra is + * still less than one buffer in size. + */ +#define RNPVF_LAST_OFFSET \ + (SKB_WITH_OVERHEAD(PAGE_SIZE) - RNPVF_RXBUFFER_2K) + if (rx_buffer->page_offset > RNPVF_LAST_OFFSET) + return false; +#endif + + /* If we have drained the page fragment pool we need to update + * the pagecnt_bias and page count so that we fully restock the + * number of references the driver holds. + */ + if (unlikely(pagecnt_bias == 1)) { + page_ref_add(page, USHRT_MAX - 1); + rx_buffer->pagecnt_bias = USHRT_MAX; + } + + return true; +} + +#if (PAGE_SIZE < 8192) +#define RNPVF_MAX_2K_FRAME_BUILD_SKB (RNPVF_RXBUFFER_1536 - NET_IP_ALIGN) +#define RNPVF_2K_TOO_SMALL_WITH_PADDING \ + ((NET_SKB_PAD + RNPVF_RXBUFFER_1536) > \ + SKB_WITH_OVERHEAD(RNPVF_RXBUFFER_2K)) + +static inline int rnpvf_compute_pad(int rx_buf_len) +{ + int page_size, pad_size; + + page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); + pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len; + + return pad_size; +} + +static inline int rnpvf_skb_pad(void) +{ + int rx_buf_len; + + /* If a 2K buffer cannot handle a standard Ethernet frame then + * optimize padding for a 3K buffer instead of a 1.5K buffer. + * + * For a 3K buffer we need to add enough padding to allow for + * tailroom due to NET_IP_ALIGN possibly shifting us out of + * cache-line alignment. + */ + if (RNPVF_2K_TOO_SMALL_WITH_PADDING) + rx_buf_len = + RNPVF_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN); + else + rx_buf_len = RNPVF_RXBUFFER_1536; + + /* if needed make room for NET_IP_ALIGN */ + rx_buf_len -= NET_IP_ALIGN; + return rnpvf_compute_pad(rx_buf_len); +} + +#define RNPVF_SKB_PAD rnpvf_skb_pad() +#else /* PAGE_SIZE < 8192 */ +#define RNPVF_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +#endif + +/** + * rnp_clean_rx_ring - Free Rx Buffers per Queue + * @rx_ring: ring to free buffers from + **/ +static void rnpvf_clean_rx_ring(struct rnpvf_ring *rx_ring) +{ + u16 i = rx_ring->next_to_clean; + struct rnpvf_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i]; + + /* Free all the Rx ring sk_buffs */ + while (i != rx_ring->next_to_alloc) { + if (rx_buffer->skb) { + struct sk_buff *skb = rx_buffer->skb; + + dev_kfree_skb(skb); + rx_buffer->skb = NULL; + } + + /* Invalidate cache lines that may have been written to by + * device so that we avoid corrupting memory. + */ + dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, + rx_buffer->page_offset, + rnpvf_rx_bufsz(rx_ring), + DMA_FROM_DEVICE); + + /* free resources associated with mapping */ + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, + rnpvf_rx_pg_size(rx_ring), + DMA_FROM_DEVICE, + RNPVF_RX_DMA_ATTR); + + __page_frag_cache_drain(rx_buffer->page, + rx_buffer->pagecnt_bias); + /* now this page is not used */ + rx_buffer->page = NULL; + i++; + rx_buffer++; + if (i == rx_ring->count) { + i = 0; + rx_buffer = rx_ring->rx_buffer_info; + } + } + + rx_ring->next_to_alloc = 0; + rx_ring->next_to_clean = 0; + rx_ring->next_to_use = 0; +} + +static inline unsigned int rnpvf_rx_offset(struct rnpvf_ring *rx_ring) +{ + return ring_uses_build_skb(rx_ring) ? RNPVF_SKB_PAD : 0; +} + +#ifdef OPTM_WITH_LPAGE +static bool rnpvf_alloc_mapped_page(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *bi, + union rnp_rx_desc *rx_desc, u16 bufsz, + u64 fun_id) +{ + struct page *page = bi->page; + dma_addr_t dma; + + /* since we are recycling buffers we should seldom need to alloc */ + if (likely(page)) + return true; + + page = dev_alloc_pages(RNPVF_ALLOC_PAGE_ORDER); + if (unlikely(!page)) { + rx_ring->rx_stats.alloc_rx_page_failed++; + return false; + } + + bi->page_offset = rnpvf_rx_offset(rx_ring); + + /* map page for use */ + dma = dma_map_page_attrs(rx_ring->dev, page, bi->page_offset, + bufsz, DMA_FROM_DEVICE, + RNPVF_RX_DMA_ATTR); + + /* + * if mapping failed free memory back to system since + * there isn't much point in holding memory we can't use + */ + if (dma_mapping_error(rx_ring->dev, dma)) { + __free_pages(page, RNPVF_ALLOC_PAGE_ORDER); + printk("map failed\n"); + + rx_ring->rx_stats.alloc_rx_page_failed++; + return false; + } + bi->dma = dma; + bi->page = page; + bi->page_offset = rnpvf_rx_offset(rx_ring); + page_ref_add(page, USHRT_MAX - 1); + bi->pagecnt_bias = USHRT_MAX; + rx_ring->rx_stats.alloc_rx_page++; + + /* sync the buffer for use by the device */ + dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 0, bufsz, + DMA_FROM_DEVICE); + + /* + * Refresh the desc even if buffer_addrs didn't change + * because each write-back erases this info. + */ + rx_desc->pkt_addr = cpu_to_le64(bi->dma + fun_id); + + return true; +} + +static void rnpvf_put_rx_buffer(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer) +{ + if (rnpvf_can_reuse_rx_page(rx_buffer)) { + /* hand second half of page back to the ring */ + rnpvf_reuse_rx_page(rx_ring, rx_buffer); + } else { + /* we are not reusing the buffer so unmap it */ + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, + rnpvf_rx_bufsz(rx_ring), + DMA_FROM_DEVICE, + RNPVF_RX_DMA_ATTR); + __page_frag_cache_drain(rx_buffer->page, + rx_buffer->pagecnt_bias); + } + + /* clear contents of rx_buffer */ + rx_buffer->page = NULL; +} + +/** + * rnpvf_alloc_rx_buffers - Replace used receive buffers + * @rx_ring: ring to place buffers on + * @cleaned_count: number of buffers to replace + **/ +void rnpvf_alloc_rx_buffers(struct rnpvf_ring *rx_ring, u16 cleaned_count) +{ + union rnp_rx_desc *rx_desc; + struct rnpvf_rx_buffer *bi; + u16 i = rx_ring->next_to_use; + u64 fun_id = ((u64)(rx_ring->vfnum) << (32 + 24)); + u16 bufsz; + /* nothing to do */ + if (!cleaned_count) + return; + + rx_desc = RNPVF_RX_DESC(rx_ring, i); + + BUG_ON(rx_desc == NULL); + + bi = &rx_ring->rx_buffer_info[i]; + + BUG_ON(bi == NULL); + + i -= rx_ring->count; + bufsz = rnpvf_rx_bufsz(rx_ring); + + do { + int count = 1; + struct page *page; + + if (!rnpvf_alloc_mapped_page(rx_ring, bi, rx_desc, bufsz, + fun_id)) + break; + page = bi->page; + + rx_desc->cmd = 0; + + rx_desc++; + i++; + bi++; + + if (unlikely(!i)) { + rx_desc = RNPVF_RX_DESC(rx_ring, 0); + bi = rx_ring->rx_buffer_info; + i -= rx_ring->count; + } + + rx_desc->cmd = 0; + + cleaned_count--; + + while (count < rx_ring->rx_page_buf_nums && + cleaned_count) { + dma_addr_t dma; + + bi->page_offset = rx_ring->rx_per_buf_mem * count + + rnpvf_rx_offset(rx_ring); + /* map page for use */ + dma = dma_map_page_attrs(rx_ring->dev, page, + bi->page_offset, bufsz, + DMA_FROM_DEVICE, + RNPVF_RX_DMA_ATTR); + + if (dma_mapping_error(rx_ring->dev, dma)) { + printk("map second error\n"); + rx_ring->rx_stats.alloc_rx_page_failed++; + break; + } + + bi->dma = dma; + bi->page = page; + + page_ref_add(page, USHRT_MAX); + bi->pagecnt_bias = USHRT_MAX; + + /* sync the buffer for use by the device */ + dma_sync_single_range_for_device(rx_ring->dev, + bi->dma, 0, bufsz, + DMA_FROM_DEVICE); + + /* + * Refresh the desc even if buffer_addrs didn't change + * because each write-back erases this info. + */ + rx_desc->pkt_addr = cpu_to_le64(bi->dma + fun_id); + /* clean dd */ + rx_desc->cmd = 0; + + rx_desc++; + bi++; + i++; + if (unlikely(!i)) { + rx_desc = RNPVF_RX_DESC(rx_ring, 0); + bi = rx_ring->rx_buffer_info; + i -= rx_ring->count; + } + count++; + /* clear the hdr_addr for the next_to_use descriptor */ + cleaned_count--; + } + } while (cleaned_count); + + i += rx_ring->count; + + if (rx_ring->next_to_use != i) + rnpvf_update_rx_tail(rx_ring, i); +} + +#else + +static bool rnpvf_alloc_mapped_page(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *bi) +{ + struct page *page = bi->page; + dma_addr_t dma; + + /* since we are recycling buffers we should seldom need to alloc */ + if (likely(page)) + return true; + + page = dev_alloc_pages(rnpvf_rx_pg_order(rx_ring)); + if (unlikely(!page)) { + rx_ring->rx_stats.alloc_rx_page_failed++; + return false; + } + + /* map page for use */ + dma = dma_map_page_attrs(rx_ring->dev, page, 0, + rnpvf_rx_pg_size(rx_ring), + DMA_FROM_DEVICE, + RNPVF_RX_DMA_ATTR); + + /* + * if mapping failed free memory back to system since + * there isn't much point in holding memory we can't use + */ + if (dma_mapping_error(rx_ring->dev, dma)) { + __free_pages(page, rnpvf_rx_pg_order(rx_ring)); + printk("map failed\n"); + + rx_ring->rx_stats.alloc_rx_page_failed++; + return false; + } + bi->dma = dma; + bi->page = page; + bi->page_offset = rnpvf_rx_offset(rx_ring); + page_ref_add(page, USHRT_MAX - 1); + bi->pagecnt_bias = USHRT_MAX; + rx_ring->rx_stats.alloc_rx_page++; + + return true; +} + +static void rnpvf_put_rx_buffer(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + struct sk_buff *skb) +{ + if (rnpvf_can_reuse_rx_page(rx_buffer)) { + /* hand second half of page back to the ring */ + rnpvf_reuse_rx_page(rx_ring, rx_buffer); + } else { + /* we are not reusing the buffer so unmap it */ + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, + rnpvf_rx_pg_size(rx_ring), + DMA_FROM_DEVICE, + RNPVF_RX_DMA_ATTR); + __page_frag_cache_drain(rx_buffer->page, + rx_buffer->pagecnt_bias); + } + + /* clear contents of rx_buffer */ + rx_buffer->page = NULL; + rx_buffer->skb = NULL; +} + +#endif /* OPTM_WITH_LPAGE */ + +/* drop this packets if error */ +static bool rnpvf_check_csum_error(struct rnpvf_ring *rx_ring, + union rnp_rx_desc *rx_desc, + unsigned int size, + unsigned int *driver_drop_packets) +{ + bool err = false; + + struct net_device *netdev = rx_ring->netdev; + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + if ((netdev->features & NETIF_F_RXCSUM) && + (!(adapter->priv_flags & RNPVF_PRIV_FLAG_FCS_ON))) { + if (unlikely(rnpvf_test_staterr(rx_desc, + RNP_RXD_STAT_ERR_MASK))) { + /* push this packet to stack if in promisc mode */ + rx_ring->rx_stats.csum_err++; + + if ((!(netdev->flags & IFF_PROMISC) && + (!(netdev->features & NETIF_F_RXALL)))) { + if (rx_ring->ring_flags & + RNPVF_RING_CHKSM_FIX) { + err = true; + goto skip_fix; + } + if (unlikely(rnpvf_test_staterr( + rx_desc, + RNP_RXD_STAT_L4_MASK) && + (!(rx_desc->wb.rev1 & + RNP_RX_L3_TYPE_MASK)))) { + rx_ring->rx_stats.csum_err--; + goto skip_fix; + } + + if (unlikely(rnpvf_test_staterr( + rx_desc, + RNP_RXD_STAT_SCTP_MASK))) { + if (size > 60) { + err = true; + } else { + /* sctp less than 60 hw report err by mistake */ + rx_ring->rx_stats + .csum_err--; + } + } else { + err = true; + } + } + } + } + +skip_fix: + if (err) { + u32 ntc = rx_ring->next_to_clean + 1; + struct rnpvf_rx_buffer *rx_buffer; +#if (PAGE_SIZE < 8192) + unsigned int truesize = rnpvf_rx_pg_size(rx_ring) / 2; +#else + unsigned int truesize = + ring_uses_build_skb(rx_ring) ? + SKB_DATA_ALIGN(RNPVF_SKB_PAD + size) : + SKB_DATA_ALIGN(size); +#endif + + if (likely(rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_EOP))) + *driver_drop_packets = *driver_drop_packets + 1; + + /* we are reusing so sync this buffer for CPU use */ + rx_buffer = + &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, + rx_buffer->page_offset, size, + DMA_FROM_DEVICE); + + /* we should clean it since we used all info in it */ + rx_desc->wb.cmd = 0; + +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif +#ifdef OPTM_WITH_LPAGE + rnpvf_put_rx_buffer(rx_ring, rx_buffer); +#else + rnpvf_put_rx_buffer(rx_ring, rx_buffer, NULL); +#endif + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + } + return err; +} + +/** + * rnpvf_process_skb_fields - Populate skb header fields from Rx descriptor + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being populated + * + * This function checks the ring, descriptor, and packet information in + * order to populate the hash, checksum, VLAN, timestamp, protocol, and + * other fields within the skb. + **/ +static void rnpvf_process_skb_fields(struct rnpvf_ring *rx_ring, + union rnp_rx_desc *rx_desc, + struct sk_buff *skb) +{ + struct net_device *dev = rx_ring->netdev; + struct rnpvf_adapter *adapter = netdev_priv(dev); + struct rnpvf_hw *hw = &adapter->hw; + + rnpvf_rx_hash(rx_ring, rx_desc, skb); + rnpvf_rx_checksum(rx_ring, rx_desc, skb); + + /* if it is a ncsi card and pf set vlan, we should check vlan id here + * in this case rx vlan offload must off + */ + if ((hw->pf_feature & PF_NCSI_EN) && + (adapter->flags & RNPVF_FLAG_PF_SET_VLAN)) { + u16 vid_pf; + u8 header[ETH_ALEN + ETH_ALEN]; + u8 *data = skb->data; + + if (__vlan_get_tag(skb, &vid_pf)) + goto skip_vf_vlan; + + if (vid_pf == adapter->vf_vlan) { + memcpy(header, data, ETH_ALEN + ETH_ALEN); + memcpy(skb->data + 4, header, ETH_ALEN + ETH_ALEN); + skb->len -= 4; + skb->data += 4; + goto skip_vf_vlan; + } + } + + /* remove vlan if pf set a vlan */ + if (((dev->features & NETIF_F_HW_VLAN_CTAG_RX) + || (dev->features & NETIF_F_HW_VLAN_STAG_RX)) && + rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_VLAN_VALID) && + !(cpu_to_le16(rx_desc->wb.rev1) & VEB_VF_IGNORE_VLAN)) { + u16 vid = le16_to_cpu(rx_desc->wb.vlan); + + if ((adapter->vf_vlan) && (adapter->vf_vlan == vid)) + goto skip_vf_vlan; + + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), + vid); + rx_ring->rx_stats.vlan_remove++; + } +skip_vf_vlan: + skb_record_rx_queue(skb, rx_ring->queue_index); + + skb->protocol = eth_type_trans(skb, dev); +} + +static void rnpvf_rx_skb(struct rnpvf_q_vector *q_vector, + struct sk_buff *skb) +{ + struct rnpvf_adapter *adapter = q_vector->adapter; + + if (!(adapter->flags & RNPVF_FLAG_IN_NETPOLL)) + napi_gro_receive(&q_vector->napi, skb); + else + netif_rx(skb); +} + +static bool rnpvf_check_src_mac(struct sk_buff *skb, + struct net_device *netdev) +{ + char *data = (char *)skb->data; + bool ret = false; + struct netdev_hw_addr *ha; + + if (is_multicast_ether_addr(data)) { + if (0 == memcmp(data + netdev->addr_len, netdev->dev_addr, + netdev->addr_len)) { + dev_kfree_skb_any(skb); + ret = true; + } + /* if src mac equal own mac */ + netdev_for_each_uc_addr(ha, netdev) { + if (0 == memcmp(data + netdev->addr_len, ha->addr, + netdev->addr_len)) { + dev_kfree_skb_any(skb); + ret = true; + } + } + } + return ret; +} + +/** + * rnpvf_get_headlen - determine size of header for RSC/LRO/GRO/FCOE + * @data: pointer to the start of the headers + * @max_len: total length of section to find headers in + * + * This function is meant to determine the length of headers that will + * be recognized by hardware for LRO, GRO, and RSC offloads. The main + * motivation of doing this is to only perform one pull for IPv4 TCP + * packets so that we can do basic things like calculating the gso_size + * based on the average data per packet. + **/ +static unsigned int rnpvf_get_headlen(unsigned char *data, + unsigned int max_len) +{ + union { + unsigned char *network; + /* l2 headers */ + struct ethhdr *eth; + struct vlan_hdr *vlan; + /* l3 headers */ + struct iphdr *ipv4; + struct ipv6hdr *ipv6; + } hdr; + __be16 protocol; + u8 nexthdr = 0; /* default to not TCP */ + u8 hlen; + + /* this should never happen, but better safe than sorry */ + if (max_len < ETH_HLEN) + return max_len; + + /* initialize network frame pointer */ + hdr.network = data; + + /* set first protocol and move network header forward */ + protocol = hdr.eth->h_proto; + hdr.network += ETH_HLEN; + + /* handle any vlan tag if present */ + if (protocol == htons(ETH_P_8021Q)) { + if ((hdr.network - data) > (max_len - VLAN_HLEN)) + return max_len; + + protocol = hdr.vlan->h_vlan_encapsulated_proto; + hdr.network += VLAN_HLEN; + } + + /* handle L3 protocols */ + if (protocol == htons(ETH_P_IP)) { + if ((hdr.network - data) > + (max_len - sizeof(struct iphdr))) + return max_len; + + /* access ihl as a u8 to avoid unaligned access on ia64 */ + hlen = (hdr.network[0] & 0x0F) << 2; + + /* verify hlen meets minimum size requirements */ + if (hlen < sizeof(struct iphdr)) + return hdr.network - data; + + /* record next protocol if header is present */ + if (!(hdr.ipv4->frag_off & htons(IP_OFFSET))) + nexthdr = hdr.ipv4->protocol; + } else if (protocol == htons(ETH_P_IPV6)) { + if ((hdr.network - data) > + (max_len - sizeof(struct ipv6hdr))) + return max_len; + + /* record next protocol */ + nexthdr = hdr.ipv6->nexthdr; + hlen = sizeof(struct ipv6hdr); + } else { + return hdr.network - data; + } + + /* relocate pointer to start of L4 header */ + hdr.network += hlen; + + /* finally sort out TCP/UDP */ + if (nexthdr == IPPROTO_TCP) { + if ((hdr.network - data) > + (max_len - sizeof(struct tcphdr))) + return max_len; + + /* access doff as a u8 to avoid unaligned access on ia64 */ + hlen = (hdr.network[12] & 0xF0) >> 2; + + /* verify hlen meets minimum size requirements */ + if (hlen < sizeof(struct tcphdr)) + return hdr.network - data; + + hdr.network += hlen; + } else if (nexthdr == IPPROTO_UDP) { + if ((hdr.network - data) > + (max_len - sizeof(struct udphdr))) + return max_len; + + hdr.network += sizeof(struct udphdr); + } + + /* + * If everything has gone correctly hdr.network should be the + * data section of the packet and will be the end of the header. + * If not then it probably represents the end of the last recognized + * header. + */ + if ((hdr.network - data) < max_len) + return hdr.network - data; + else + return max_len; +} + +/** + * rnpvf_pull_tail - rnp specific version of skb_pull_tail + * @rx_ring: rx descriptor ring packet is being transacted on + * @skb: pointer to current skb being adjusted + * + * This function is an rnp specific version of __pskb_pull_tail. The + * main difference between this version and the original function is that + * this function can make several assumptions about the state of things + * that allow for significant optimizations versus the standard function. + * As a result we can do things like drop a frag and maintain an accurate + * truesize for the skb. + */ +static void rnpvf_pull_tail(struct sk_buff *skb) +{ + skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; + unsigned char *va; + unsigned int pull_len; + + /* + * it is valid to use page_address instead of kmap since we are + * working with pages allocated out of the lomem pool per + * alloc_page(GFP_ATOMIC) + */ + va = skb_frag_address(frag); + + /* + * we need the header to contain the greater of either ETH_HLEN or + * 60 bytes if the skb->len is less than 60 for skb_pad. + */ + pull_len = rnpvf_get_headlen(va, RNPVF_RX_HDR_SIZE); + + /* align pull length to size of long to optimize memcpy performance */ + skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); + + /* update all of the pointers */ + skb_frag_size_sub(frag, pull_len); + skb_frag_off_add(frag, pull_len); + skb->data_len -= pull_len; + skb->tail += pull_len; +} + +/** + * rnpvf_cleanup_headers - Correct corrupted or empty headers + * @rx_ring: rx descriptor ring packet is being transacted on + * @rx_desc: pointer to the EOP Rx descriptor + * @skb: pointer to current skb being fixed + * + * Check for corrupted packet headers caused by senders on the local L2 + * embedded NIC switch not setting up their Tx Descriptors right. These + * should be very rare. + * + * Also address the case where we are pulling data in on pages only + * and as such no data is present in the skb header. + * + * In addition if skb is not at least 60 bytes we need to pad it so that + * it is large enough to qualify as a valid Ethernet frame. + * + * Returns true if an error was encountered and skb was freed. + **/ +static bool rnpvf_cleanup_headers(struct rnpvf_ring *rx_ring, + union rnp_rx_desc *rx_desc, + struct sk_buff *skb) +{ +#ifdef OPTM_WITH_LPAGE +#else + /* XDP packets use error pointer so abort at this point */ + if (IS_ERR(skb)) + return true; +#endif + + /* place header in linear portion of buffer */ + if (!skb_headlen(skb)) + rnpvf_pull_tail(skb); + + if (eth_skb_pad(skb)) + return true; + if (!(rx_ring->ring_flags & RNPVF_RING_VEB_MULTI_FIX)) + return rnpvf_check_src_mac(skb, rx_ring->netdev); + else + return false; + + return false; +} + +/** + * rnpvf_add_rx_frag - Add contents of Rx buffer to sk_buff + * @rx_ring: rx descriptor ring to transact packets on + * @rx_buffer: buffer containing page to add + * @skb: sk_buff to place the data into + * @size: size of data + * + * This function will add the data contained in rx_buffer->page to the skb. + * This is done either through a direct copy if the data in the buffer is + * less than the skb header size, otherwise it will just attach the page as + * a frag to the skb. + * + * The function will then update the page offset if necessary and return + * true if the buffer can be reused by the adapter. + **/ +static void rnpvf_add_rx_frag(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + struct sk_buff *skb, unsigned int size) +{ +#if (PAGE_SIZE < 8192) + unsigned int truesize = rnpvf_rx_pg_size(rx_ring) / 2; +#else + unsigned int truesize = + ring_uses_build_skb(rx_ring) ? + SKB_DATA_ALIGN(RNPVF_SKB_PAD + size) : + SKB_DATA_ALIGN(size); +#endif + + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, + rx_buffer->page_offset, size, truesize); + +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif +} + +#ifdef OPTM_WITH_LPAGE +static struct sk_buff *rnpvf_build_skb(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + union rnp_rx_desc *rx_desc, + unsigned int size) +{ + void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; + unsigned int truesize = + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(size + RNPVF_SKB_PAD); + struct sk_buff *skb; + + /* prefetch first cache line of first page */ + prefetch(va); +#if L1_CACHE_BYTES < 128 + prefetch(va + L1_CACHE_BYTES); +#endif + + /* build an skb around the page buffer */ + skb = build_skb(va - RNPVF_SKB_PAD, truesize); + if (unlikely(!skb)) + return NULL; + + /* update pointers within the skb to store the data */ + skb_reserve(skb, RNPVF_SKB_PAD); + __skb_put(skb, size); + + return skb; +} + +static struct rnpvf_rx_buffer * +rnpvf_get_rx_buffer(struct rnpvf_ring *rx_ring, union rnp_rx_desc *rx_desc, + const unsigned int size) +{ + struct rnpvf_rx_buffer *rx_buffer; + + rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + prefetchw(rx_buffer->page); + + rx_buf_dump("rx buf", + page_address(rx_buffer->page) + rx_buffer->page_offset, + rx_desc->wb.len); + + /* we are reusing so sync this buffer for CPU use */ + dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, 0, + size, DMA_FROM_DEVICE); + /* skip_sync: */ + rx_buffer->pagecnt_bias--; + + return rx_buffer; +} + +/** + * rnpvf_is_non_eop - process handling of non-EOP buffers + * @rx_ring: Rx ring being processed + * @rx_desc: Rx descriptor for current buffer + * @skb: Current socket buffer containing buffer in progress + * + * This function updates next to clean. If the buffer is an EOP buffer + * this function exits returning false, otherwise it will place the + * sk_buff in the next buffer to be chained and return true indicating + * that this is in fact a non-EOP buffer. + **/ +static bool rnpvf_is_non_eop(struct rnpvf_ring *rx_ring, + union rnp_rx_desc *rx_desc) +{ + u32 ntc = rx_ring->next_to_clean + 1; + /* fetch, update, and store next to clean */ + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + + prefetch(RNPVF_RX_DESC(rx_ring, ntc)); + + /* if we are the last buffer then there is nothing else to do */ + if (likely(rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_EOP))) + return false; + /* place skb in next buffer to be received */ + /* we should clean it since we used all info in it */ + rx_desc->wb.cmd = 0; + + return true; +} + +static struct sk_buff * +rnpvf_construct_skb(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + union rnp_rx_desc *rx_desc, unsigned int size) +{ + void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; + unsigned int truesize = SKB_DATA_ALIGN(size); + unsigned int headlen; + struct sk_buff *skb; + + /* prefetch first cache line of first page */ + prefetch(va); +#if L1_CACHE_BYTES < 128 + prefetch(va + L1_CACHE_BYTES); +#endif + /* Note, we get here by enabling legacy-rx via: + * + * ethtool --set-priv-flags legacy-rx on + * + * In this mode, we currently get 0 extra XDP headroom as + * opposed to having legacy-rx off, where we process XDP + * packets going to stack via rnpvf_build_skb(). The latter + * provides us currently with 192 bytes of headroom. + * + * For rnp_construct_skb() mode it means that the + * xdp->data_meta will always point to xdp->data, since + * the helper cannot expand the head. Should this ever + * change in future for legacy-rx mode on, then lets also + * add xdp->data_meta handling here. + */ + + /* allocate a skb to store the frags */ + skb = napi_alloc_skb(&rx_ring->q_vector->napi, RNPVF_RX_HDR_SIZE); + if (unlikely(!skb)) + return NULL; + + prefetchw(skb->data); + + /* Determine available headroom for copy */ + headlen = size; + if (headlen > RNPVF_RX_HDR_SIZE) + headlen = rnpvf_get_headlen(va, RNPVF_RX_HDR_SIZE); + + /* align pull length to size of long to optimize memcpy performance */ + memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); + + /* update all of the pointers */ + size -= headlen; + + if (size) { + + skb_add_rx_frag(skb, 0, rx_buffer->page, + (va + headlen) - + page_address(rx_buffer->page), + size, truesize); + rx_buffer->page_offset += truesize; + } else { + rx_buffer->pagecnt_bias++; + } + + return skb; +} + +/** + * rnp_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf + * @q_vector: structure containing interrupt and ring information + * @rx_ring: rx descriptor ring to transact packets on + * @budget: Total limit on number of packets to process + * + * This function provides a "bounce buffer" approach to Rx interrupt + * processing. The advantage to this is that on systems that have + * expensive overhead for IOMMU access this provides a means of avoiding + * it by maintaining the mapping of the page to the system. + * + * Returns amount of work completed. + **/ +static int rnpvf_clean_rx_irq(struct rnpvf_q_vector *q_vector, + struct rnpvf_ring *rx_ring, int budget) +{ + unsigned int total_rx_bytes = 0, total_rx_packets = 0; + unsigned int err_packets = 0; + unsigned int driver_drop_packets = 0; + struct sk_buff *skb = rx_ring->skb; + struct rnpvf_adapter *adapter = q_vector->adapter; + u16 cleaned_count = rnpvf_desc_unused(rx_ring); + + while (likely(total_rx_packets < budget)) { + union rnp_rx_desc *rx_desc; + struct rnpvf_rx_buffer *rx_buffer; + unsigned int size; + + /* return some buffers to hardware, one at a time is too slow */ + if (cleaned_count >= RNPVF_RX_BUFFER_WRITE) { + rnpvf_alloc_rx_buffers(rx_ring, cleaned_count); + cleaned_count = 0; + } + rx_desc = RNPVF_RX_DESC(rx_ring, rx_ring->next_to_clean); + rx_buf_dump("rx-desc:", rx_desc, sizeof(*rx_desc)); + rx_debug_printk(" dd set: %s\n", + (rx_desc->wb.cmd & RNP_RXD_STAT_DD) ? + "Yes" : + "No"); + + if (!rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_DD)) + break; + + rx_debug_printk( + "queue:%d rx-desc:%d has-data len:%d next_to_clean %d\n", + rx_ring->rnp_queue_idx, rx_ring->next_to_clean, + rx_desc->wb.len, rx_ring->next_to_clean); + + /* handle padding */ + if ((adapter->priv_flags & RNPVF_PRIV_FLAG_FT_PADDING) && + (!(adapter->priv_flags & + RNPVF_PRIV_FLAG_PADDING_DEBUG))) { + if (likely(rnpvf_test_staterr(rx_desc, + RNP_RXD_STAT_EOP))) { + size = le16_to_cpu(rx_desc->wb.len) - + le16_to_cpu( + rx_desc->wb.padding_len); + } else { + size = le16_to_cpu(rx_desc->wb.len); + } + } else { + /* size should not zero */ + size = le16_to_cpu(rx_desc->wb.len); + } + + if (!size) + break; + + /* + * should check csum err + * maybe one packet use multiple descs + * no problems hw set all csum_err in multiple descs + * maybe BUG if the last sctp desc less than 60 + */ + if (rnpvf_check_csum_error(rx_ring, rx_desc, size, + &driver_drop_packets)) { + cleaned_count++; + err_packets++; + if (err_packets + total_rx_packets > budget) + break; + continue; + } + /* This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc until we know the + * descriptor has been written back + */ + dma_rmb(); + + rx_buffer = rnpvf_get_rx_buffer(rx_ring, rx_desc, size); + + if (skb) { + rnpvf_add_rx_frag(rx_ring, rx_buffer, skb, size); + } else if (ring_uses_build_skb(rx_ring)) { + skb = rnpvf_build_skb(rx_ring, rx_buffer, rx_desc, + size); + } else { + skb = rnpvf_construct_skb(rx_ring, rx_buffer, + rx_desc, size); + } + + /* exit if we failed to retrieve a buffer */ + if (!skb) { + rx_ring->rx_stats.alloc_rx_buff_failed++; + rx_buffer->pagecnt_bias++; + break; + } + + rnpvf_put_rx_buffer(rx_ring, rx_buffer); + cleaned_count++; + + /* place incomplete frames back on ring for completion */ + if (rnpvf_is_non_eop(rx_ring, rx_desc)) + continue; + + /* verify the packet layout is correct */ + if (rnpvf_cleanup_headers(rx_ring, rx_desc, skb)) { + /* we should clean it since we used all info in it */ + rx_desc->wb.cmd = 0; + skb = NULL; + continue; + } + + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + + /* populate checksum, timestamp, VLAN, and protocol */ + rnpvf_process_skb_fields(rx_ring, rx_desc, skb); + + /* we should clean it since we used all info in it */ + rx_desc->wb.cmd = 0; + + rnpvf_rx_skb(q_vector, skb); + skb = NULL; + + /* update budget accounting */ + total_rx_packets++; + } + + rx_ring->skb = skb; + + u64_stats_update_begin(&rx_ring->syncp); + rx_ring->stats.packets += total_rx_packets; + rx_ring->stats.bytes += total_rx_bytes; + rx_ring->rx_stats.driver_drop_packets += driver_drop_packets; + u64_stats_update_end(&rx_ring->syncp); + q_vector->rx.total_packets += total_rx_packets; + q_vector->rx.total_bytes += total_rx_bytes; + + if (total_rx_packets >= budget) + rx_ring->rx_stats.poll_again_count++; + + return total_rx_packets; +} + +#else + +/** + * rnpvf_is_non_eop - process handling of non-EOP buffers + * @rx_ring: Rx ring being processed + * @rx_desc: Rx descriptor for current buffer + * @skb: Current socket buffer containing buffer in progress + * + * This function updates next to clean. If the buffer is an EOP buffer + * this function exits returning false, otherwise it will place the + * sk_buff in the next buffer to be chained and return true indicating + * that this is in fact a non-EOP buffer. + **/ +static bool rnpvf_is_non_eop(struct rnpvf_ring *rx_ring, + union rnp_rx_desc *rx_desc, + struct sk_buff *skb) +{ + u32 ntc = rx_ring->next_to_clean + 1; + + /* fetch, update, and store next to clean */ + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + + prefetch(RNPVF_RX_DESC(rx_ring, ntc)); + + /* if we are the last buffer then there is nothing else to do */ + if (likely(rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_EOP))) + return false; + /* place skb in next buffer to be received */ + rx_ring->rx_buffer_info[ntc].skb = skb; + /* we should clean it since we used all info in it */ + rx_desc->wb.cmd = 0; + rx_ring->rx_stats.non_eop_descs++; + + return true; +} + +static struct sk_buff *rnpvf_build_skb(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + struct xdp_buff *xdp, + union rnp_rx_desc *rx_desc) +{ + unsigned int metasize = xdp->data - xdp->data_meta; + void *va = xdp->data_meta; +#if (PAGE_SIZE < 8192) + unsigned int truesize = rnpvf_rx_pg_size(rx_ring) / 2; +#else + unsigned int truesize = + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + SKB_DATA_ALIGN(xdp->data_end - xdp->data_hard_start); +#endif + struct sk_buff *skb; + + /* prefetch first cache line of first page */ + prefetch(va); +#if L1_CACHE_BYTES < 128 + prefetch(va + L1_CACHE_BYTES); +#endif + + /* build an skb around the page buffer */ + skb = build_skb(xdp->data_hard_start, truesize); + if (unlikely(!skb)) + return NULL; + + /* update pointers within the skb to store the data */ + skb_reserve(skb, xdp->data - xdp->data_hard_start); + __skb_put(skb, xdp->data_end - xdp->data); + if (metasize) + skb_metadata_set(skb, metasize); +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif + + return skb; +} + +static void rnpvf_rx_buffer_flip(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + unsigned int size) +{ +#if (PAGE_SIZE < 8192) + unsigned int truesize = rnpvf_rx_pg_size(rx_ring) / 2; + + rx_buffer->page_offset ^= truesize; +#else + unsigned int truesize = + ring_uses_build_skb(rx_ring) ? + SKB_DATA_ALIGN(RNPVF_SKB_PAD + size) : + SKB_DATA_ALIGN(size); + + rx_buffer->page_offset += truesize; +#endif +} + +static struct rnpvf_rx_buffer * +rnpvf_get_rx_buffer(struct rnpvf_ring *rx_ring, union rnp_rx_desc *rx_desc, + struct sk_buff **skb, const unsigned int size) +{ + struct rnpvf_rx_buffer *rx_buffer; + + rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + prefetchw(rx_buffer->page); + *skb = rx_buffer->skb; + + rx_buf_dump("rx buf", + page_address(rx_buffer->page) + rx_buffer->page_offset, + rx_desc->wb.len); + + /* we are reusing so sync this buffer for CPU use */ + dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, + rx_buffer->page_offset, size, + DMA_FROM_DEVICE); + /* skip_sync: */ + rx_buffer->pagecnt_bias--; + + return rx_buffer; +} + +static struct sk_buff * +rnpvf_construct_skb(struct rnpvf_ring *rx_ring, + struct rnpvf_rx_buffer *rx_buffer, + struct xdp_buff *xdp, union rnp_rx_desc *rx_desc) +{ + unsigned int size = xdp->data_end - xdp->data; +#if (PAGE_SIZE < 8192) + unsigned int truesize = rnpvf_rx_pg_size(rx_ring) / 2; +#else + unsigned int truesize = + SKB_DATA_ALIGN(xdp->data_end - xdp->data_hard_start); +#endif + struct sk_buff *skb; + + /* prefetch first cache line of first page */ + prefetch(xdp->data); +#if L1_CACHE_BYTES < 128 + prefetch(xdp->data + L1_CACHE_BYTES); +#endif + /* Note, we get here by enabling legacy-rx via: + * + * ethtool --set-priv-flags legacy-rx on + * + * In this mode, we currently get 0 extra XDP headroom as + * opposed to having legacy-rx off, where we process XDP + * packets going to stack via rnpvf_build_skb(). The latter + * provides us currently with 192 bytes of headroom. + * + * For rnp_construct_skb() mode it means that the + * xdp->data_meta will always point to xdp->data, since + * the helper cannot expand the head. Should this ever + * change in future for legacy-rx mode on, then lets also + * add xdp->data_meta handling here. + */ + + /* allocate a skb to store the frags */ + skb = napi_alloc_skb(&rx_ring->q_vector->napi, RNPVF_RX_HDR_SIZE); + if (unlikely(!skb)) + return NULL; + + prefetchw(skb->data); + + if (size > RNPVF_RX_HDR_SIZE) { + + skb_add_rx_frag(skb, 0, rx_buffer->page, + xdp->data - page_address(rx_buffer->page), + size, truesize); +#if (PAGE_SIZE < 8192) + rx_buffer->page_offset ^= truesize; +#else + rx_buffer->page_offset += truesize; +#endif + } else { + memcpy(__skb_put(skb, size), xdp->data, + ALIGN(size, sizeof(long))); + rx_buffer->pagecnt_bias++; + } + + return skb; +} + +/** + * rnp_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf + * @q_vector: structure containing interrupt and ring information + * @rx_ring: rx descriptor ring to transact packets on + * @budget: Total limit on number of packets to process + * + * This function provides a "bounce buffer" approach to Rx interrupt + * processing. The advantage to this is that on systems that have + * expensive overhead for IOMMU access this provides a means of avoiding + * it by maintaining the mapping of the page to the system. + * + * Returns amount of work completed. + **/ +static int rnpvf_clean_rx_irq(struct rnpvf_q_vector *q_vector, + struct rnpvf_ring *rx_ring, int budget) +{ + unsigned int total_rx_bytes = 0, total_rx_packets = 0; + unsigned int err_packets = 0; + unsigned int driver_drop_packets = 0; + struct rnpvf_adapter *adapter = q_vector->adapter; + u16 cleaned_count = rnpvf_desc_unused(rx_ring); + bool xdp_xmit = false; + struct xdp_buff xdp; + + xdp.data = NULL; + xdp.data_end = NULL; + + while (likely(total_rx_packets < budget)) { + union rnp_rx_desc *rx_desc; + struct rnpvf_rx_buffer *rx_buffer; + struct sk_buff *skb; + unsigned int size; + + /* return some buffers to hardware, one at a time is too slow */ + if (cleaned_count >= RNPVF_RX_BUFFER_WRITE) { + rnpvf_alloc_rx_buffers(rx_ring, cleaned_count); + cleaned_count = 0; + } + rx_desc = RNPVF_RX_DESC(rx_ring, rx_ring->next_to_clean); + + rx_buf_dump("rx-desc:", rx_desc, sizeof(*rx_desc)); + rx_debug_printk(" dd set: %s\n", + (rx_desc->wb.cmd & RNP_RXD_STAT_DD) ? + "Yes" : + "No"); + + if (!rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_DD)) + break; + + rx_debug_printk( + "queue:%d rx-desc:%d has-data len:%d next_to_clean %d\n", + rx_ring->rnpvf_queue_idx, rx_ring->next_to_clean, + rx_desc->wb.len, rx_ring->next_to_clean); + + /* handle padding */ + if ((adapter->priv_flags & RNPVF_PRIV_FLAG_FT_PADDING) && + (!(adapter->priv_flags & + RNPVF_PRIV_FLAG_PADDING_DEBUG))) { + if (likely(rnpvf_test_staterr(rx_desc, + RNP_RXD_STAT_EOP))) { + size = le16_to_cpu(rx_desc->wb.len) - + le16_to_cpu( + rx_desc->wb.padding_len); + } else { + size = le16_to_cpu(rx_desc->wb.len); + } + } else { + /* size should not zero */ + size = le16_to_cpu(rx_desc->wb.len); + } + + if (!size) + break; + + /* + * should check csum err + * maybe one packet use multiple descs + * no problems hw set all csum_err in multiple descs + * maybe BUG if the last sctp desc less than 60 + */ + if (rnpvf_check_csum_error(rx_ring, rx_desc, size, + &driver_drop_packets)) { + cleaned_count++; + err_packets++; + if (err_packets + total_rx_packets > budget) + break; + continue; + } + /* This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc until we know the + * descriptor has been written back + */ + dma_rmb(); + + rx_buffer = + rnpvf_get_rx_buffer(rx_ring, rx_desc, &skb, size); + + if (!skb) { + xdp.data = page_address(rx_buffer->page) + + rx_buffer->page_offset; + xdp.data_meta = xdp.data; + xdp.data_hard_start = + xdp.data - rnpvf_rx_offset(rx_ring); + xdp.data_end = xdp.data + size; + } + + if (IS_ERR(skb)) { + if (PTR_ERR(skb) == -RNPVF_XDP_TX) { + xdp_xmit = true; + rnpvf_rx_buffer_flip(rx_ring, rx_buffer, + size); + } else { + rx_buffer->pagecnt_bias++; + } + total_rx_packets++; + total_rx_bytes += size; + } else if (skb) { + rnpvf_add_rx_frag(rx_ring, rx_buffer, skb, size); + } else if (ring_uses_build_skb(rx_ring)) { + skb = rnpvf_build_skb(rx_ring, rx_buffer, &xdp, + rx_desc); + } else { + skb = rnpvf_construct_skb(rx_ring, rx_buffer, &xdp, + rx_desc); + } + + /* exit if we failed to retrieve a buffer */ + if (!skb) { + rx_ring->rx_stats.alloc_rx_buff_failed++; + rx_buffer->pagecnt_bias++; + break; + } + + rnpvf_put_rx_buffer(rx_ring, rx_buffer, skb); + cleaned_count++; + + /* place incomplete frames back on ring for completion */ + if (rnpvf_is_non_eop(rx_ring, rx_desc, skb)) + continue; + + /* verify the packet layout is correct */ + if (rnpvf_cleanup_headers(rx_ring, rx_desc, skb)) { + + continue; + } + + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + + /* populate checksum, timestamp, VLAN, and protocol */ + rnpvf_process_skb_fields(rx_ring, rx_desc, skb); + + /* we should clean it since we used all info in it */ + rx_desc->wb.cmd = 0; + + rnpvf_rx_skb(q_vector, skb); + + /* update budget accounting */ + total_rx_packets++; + } + + u64_stats_update_begin(&rx_ring->syncp); + rx_ring->stats.packets += total_rx_packets; + rx_ring->stats.bytes += total_rx_bytes; + rx_ring->rx_stats.driver_drop_packets += driver_drop_packets; + u64_stats_update_end(&rx_ring->syncp); + q_vector->rx.total_packets += total_rx_packets; + q_vector->rx.total_bytes += total_rx_bytes; + + if (total_rx_packets >= budget) + rx_ring->rx_stats.poll_again_count++; + return total_rx_packets; +} +#endif + +/** + * rnpvf_configure_msix - Configure MSI-X hardware + * @adapter: board private structure + * + * rnpvf_configure_msix sets up the hardware to properly generate MSI-X + * interrupts. + **/ +static void rnpvf_configure_msix(struct rnpvf_adapter *adapter) +{ + struct rnpvf_q_vector *q_vector; + int i; + + for (i = 0; i < adapter->num_q_vectors; i++) { + struct rnpvf_ring *ring; + + q_vector = adapter->q_vector[i]; + + rnpvf_for_each_ring(ring, q_vector->rx) { + rnpvf_set_ring_vector(adapter, + ring->rnpvf_msix_off, + q_vector->v_idx); + } + } +} + +enum latency_range { + lowest_latency = 0, + low_latency = 1, + bulk_latency = 2, + latency_invalid = 255 +}; + +static inline void rnpvf_irq_enable_queues(struct rnpvf_q_vector *q_vector) +{ + struct rnpvf_ring *ring; + + rnpvf_for_each_ring(ring, q_vector->rx) { + rnpvf_wr_reg(ring->dma_int_clr, RX_INT_MASK | TX_INT_MASK); + /* we need this */ + wmb(); + rnpvf_wr_reg(ring->dma_int_mask, + ~(RX_INT_MASK | TX_INT_MASK)); + } +} + +static inline void +rnpvf_irq_disable_queues(struct rnpvf_q_vector *q_vector) +{ + struct rnpvf_ring *ring; + + rnpvf_for_each_ring(ring, q_vector->tx) { + rnpvf_wr_reg(ring->dma_int_mask, + (RX_INT_MASK | TX_INT_MASK)); + } +} + +/** + * rnpvf_irq_enable - Enable default interrupt generation settings + * @adapter: board private structure + **/ +static inline void rnpvf_irq_enable(struct rnpvf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_q_vectors; i++) + rnpvf_irq_enable_queues(adapter->q_vector[i]); +} + +static irqreturn_t rnpvf_msix_other(int irq, void *data) +{ + struct rnpvf_adapter *adapter = data; + struct rnpvf_hw *hw = &adapter->hw; + + dbg("\n\n !!! %s irq-coming !!!\n", __func__); + + /* link is down by pf */ + if (test_bit(__RNPVF_MBX_POLLING, &adapter->state)) + goto NO_WORK_DONE; + if (!hw->mbx.ops.check_for_rst(hw, false)) { + if (test_bit(__RNPVF_REMOVE, &adapter->state)) { + printk("rnpvf is removed\n"); + } + } +NO_WORK_DONE: + + return IRQ_HANDLED; +} + +static void rnpvf_htimer_start(struct rnpvf_q_vector *q_vector) +{ + unsigned long ns = q_vector->irq_check_usecs * NSEC_PER_USEC / 2; + + hrtimer_start_range_ns(&q_vector->irq_miss_check_timer, + ns_to_ktime(ns), ns, HRTIMER_MODE_REL); +} + +static void rnpvf_htimer_stop(struct rnpvf_q_vector *q_vector) +{ + hrtimer_cancel(&q_vector->irq_miss_check_timer); +} + +static irqreturn_t rnpvf_intr(int irq, void *data) +{ + struct rnpvf_adapter *adapter = data; + struct rnpvf_q_vector *q_vector = adapter->q_vector[0]; + struct rnpvf_hw *hw = &adapter->hw; + + if (q_vector->vector_flags & RNPVF_QVECTOR_FLAG_IRQ_MISS_CHECK) + rnpvf_htimer_stop(q_vector); + + /* disabled interrupts (on this vector) for us */ + rnpvf_irq_disable_queues(q_vector); + + if (q_vector->rx.ring || q_vector->tx.ring) + napi_schedule_irqoff(&q_vector->napi); + + dbg("\n\n !!! %s irq-coming !!!\n", __func__); + + /* link is down by pf */ + if (test_bit(__RNPVF_MBX_POLLING, &adapter->state)) + goto WORK_DONE; + if (!hw->mbx.ops.check_for_rst(hw, false)) { + if (test_bit(__RNPVF_REMOVE, &adapter->state)) { + printk("rnpvf is removed\n"); + } + } +WORK_DONE: + return IRQ_HANDLED; +} + +static irqreturn_t rnpvf_msix_clean_rings(int irq, void *data) +{ + struct rnpvf_q_vector *q_vector = data; + + if (q_vector->vector_flags & RNPVF_QVECTOR_FLAG_IRQ_MISS_CHECK) + rnpvf_htimer_stop(q_vector); + /* disabled interrupts (on this vector) for us */ + rnpvf_irq_disable_queues(q_vector); + + if (q_vector->rx.ring || q_vector->tx.ring) + napi_schedule(&q_vector->napi); + + return IRQ_HANDLED; +} + +void update_rx_count(int cleaned, struct rnpvf_q_vector *q_vector) +{ + struct rnpvf_adapter *adapter = q_vector->adapter; + + if ((cleaned) && (cleaned != q_vector->new_rx_count)) { + if (cleaned < 5) { + q_vector->small_times = 0; + q_vector->large_times = 0; + q_vector->too_small_times++; + if (q_vector->too_small_times >= 2) { + q_vector->new_rx_count = 1; + } + } else if (cleaned < 30) { + q_vector->too_small_times = 0; + q_vector->middle_time++; + if (cleaned < q_vector->new_rx_count) { + q_vector->small_times = 0; + q_vector->new_rx_count -= + (1 << (q_vector->large_times++)); + if (q_vector->new_rx_count < 0) + q_vector->new_rx_count = 1; + } else { + q_vector->large_times = 0; + + if (cleaned > 30) { + if (q_vector->new_rx_count == + (cleaned - 4)) { + } else { + q_vector->new_rx_count += + (1 + << (q_vector->small_times++)); + } + if (q_vector->new_rx_count >= + cleaned) { + q_vector->new_rx_count = + cleaned - 4; + q_vector->small_times = 0; + } + + } else { + if (q_vector->new_rx_count == + (cleaned - 1)) { + } else { + q_vector->new_rx_count += + (1 + << (q_vector->small_times++)); + } + if (q_vector->new_rx_count >= + cleaned) { + q_vector->new_rx_count = + cleaned - 1; + q_vector->small_times = 0; + } + } + } + } else { + q_vector->too_small_times = 0; + q_vector->new_rx_count = + max_t(int, 64, adapter->rx_frames); + q_vector->small_times = 0; + q_vector->large_times = 0; + } + } +} + +static void rnpvf_check_restart_tx(struct rnpvf_q_vector *q_vector, + struct rnpvf_ring *tx_ring) +{ + struct rnpvf_adapter *adapter = q_vector->adapter; +#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) + if (likely(netif_carrier_ok(tx_ring->netdev) && + (rnpvf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { + /* Make sure that anybody stopping the queue after this + * sees the new next_to_clean. + */ + smp_mb(); + if (__netif_subqueue_stopped(tx_ring->netdev, + tx_ring->queue_index) && + !test_bit(__RNPVF_DOWN, &adapter->state)) { + netif_wake_subqueue(tx_ring->netdev, + tx_ring->queue_index); + ++tx_ring->tx_stats.restart_queue; + } + } +} + +/** + * rnpvf_poll - NAPI polling calback + * @napi: napi struct with our devices info in it + * @budget: amount of work driver is allowed to do this pass, in packets + * + * This function will clean more than one or more rings associated with a + * q_vector. + **/ +static int rnpvf_poll(struct napi_struct *napi, int budget) +{ + struct rnpvf_q_vector *q_vector = + container_of(napi, struct rnpvf_q_vector, napi); + struct rnpvf_adapter *adapter = q_vector->adapter; + struct rnpvf_ring *ring; + int per_ring_budget, work_done = 0; + bool clean_complete = true; + int cleaned_total = 0; + + rnpvf_for_each_ring(ring, q_vector->tx) + clean_complete &= !!rnpvf_clean_tx_irq(q_vector, ring); + + /* attempt to distribute budget to each queue fairly, but don't allow + * the budget to go below 1 because we'll exit polling + */ + if (q_vector->rx.count > 1) + per_ring_budget = max(budget / q_vector->rx.count, 1); + else + per_ring_budget = budget; + + rnpvf_for_each_ring(ring, q_vector->rx) { + int cleaned = 0; + + cleaned = rnpvf_clean_rx_irq(q_vector, ring, + per_ring_budget); + + work_done += cleaned; + cleaned_total += cleaned; + + if (cleaned >= per_ring_budget) + clean_complete = false; + } + + if (test_bit(__RNPVF_DOWN, &adapter->state)) + clean_complete = true; + + if (!(q_vector->vector_flags & RNPVF_QVECTOR_FLAG_ITR_FEATURE)) + update_rx_count(cleaned_total, q_vector); + + /* If all work not completed, return budget and keep polling */ + if (!clean_complete) + return budget; + + /* all work done, exit the polling mode */ + if (likely(napi_complete_done(napi, work_done))) { + /* try to do itr handle */ + if (q_vector->vector_flags & + RNPVF_QVECTOR_FLAG_ITR_FEATURE) + rnpvf_set_itr(q_vector); + + if (!test_bit(__RNPVF_DOWN, &adapter->state)) { + rnpvf_irq_enable_queues(q_vector); + smp_mb(); + /* we need this to ensure irq start before tx start */ + if (q_vector->vector_flags & + RNPVF_QVECTOR_FLAG_REDUCE_TX_IRQ_MISS) { + rnpvf_for_each_ring(ring, q_vector->tx) { + rnpvf_check_restart_tx(q_vector, + ring); + if (q_vector->new_rx_count != + q_vector->old_rx_count) { + ring_wr32( + ring, + RNP_DMA_REG_RX_INT_DELAY_PKTCNT, + q_vector->new_rx_count); + q_vector->old_rx_count = + q_vector->new_rx_count; + } + } + } + if (q_vector->vector_flags & + RNPVF_QVECTOR_FLAG_IRQ_MISS_CHECK) + rnpvf_htimer_start(q_vector); + } + } + + return 0; +} + +/** + * rnpvf_request_msix_irqs - Initialize MSI-X interrupts + * @adapter: board private structure + * + * rnpvf_request_msix_irqs allocates MSI-X vectors and requests + * interrupts from the kernel. + **/ +static int rnpvf_request_msix_irqs(struct rnpvf_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + int err; + int i = 0; + + DPRINTK(IFUP, INFO, "num_q_vectors:%d\n", adapter->num_q_vectors); + + for (i = 0; i < adapter->num_q_vectors; i++) { + struct rnpvf_q_vector *q_vector = adapter->q_vector[i]; + struct msix_entry *entry = + &adapter->msix_entries[i + adapter->vector_off]; + + if (q_vector->tx.ring && q_vector->rx.ring) { + snprintf(q_vector->name, + sizeof(q_vector->name) - 1, "%s-%s-%d-%d", + netdev->name, "TxRx", i, q_vector->v_idx); + } else { + WARN(!(q_vector->tx.ring && q_vector->rx.ring), + "%s vector%d tx rx is null, v_idx:%d\n", + netdev->name, i, q_vector->v_idx); + /* skip this unused q_vector */ + continue; + } + err = request_irq(entry->vector, &rnpvf_msix_clean_rings, + 0, q_vector->name, q_vector); + if (err) { + rnpvf_err( + "%s:request_irq failed for MSIX interrupt:%d " + "Error: %d\n", + netdev->name, entry->vector, err); + goto free_queue_irqs; + } + irq_set_affinity_hint(entry->vector, + &q_vector->affinity_mask); + } + + return 0; + +free_queue_irqs: + while (i) { + i--; + irq_set_affinity_hint( + adapter->msix_entries[i + adapter->vector_off] + .vector, + NULL); + free_irq(adapter->msix_entries[i + adapter->vector_off] + .vector, + adapter->q_vector[i]); + } + return err; +} + +static int rnpvf_free_msix_irqs(struct rnpvf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_q_vectors; i++) { + struct rnpvf_q_vector *q_vector = adapter->q_vector[i]; + struct msix_entry *entry = + &adapter->msix_entries[i + adapter->vector_off]; + + /* free only the irqs that were actually requested */ + if (!q_vector->rx.ring && !q_vector->tx.ring) + continue; + + /* clear the affinity_mask in the IRQ descriptor */ + irq_set_affinity_hint(entry->vector, NULL); + DPRINTK(IFDOWN, INFO, "free irq %s\n", q_vector->name); + free_irq(entry->vector, q_vector); + } + + return 0; +} + +/** + * rnpvf_update_itr - update the dynamic ITR value based on statistics + * @q_vector: structure containing interrupt and ring information + * @ring_container: structure containing ring performance data + * + * Stores a new ITR value based on packets and byte + * counts during the last interrupt. The advantage of per interrupt + * computation is faster updates and more accurate ITR for the current + * traffic pattern. Constants in this function were computed + * based on theoretical maximum wire speed and thresholds were set based + * on testing data as well as attempting to minimize response time + * while increasing bulk throughput. + **/ +static void rnpvf_update_itr(struct rnpvf_q_vector *q_vector, + struct rnpvf_ring_container *ring_container, + int type) +{ + unsigned int itr = RNPVF_ITR_ADAPTIVE_MIN_USECS | + RNPVF_ITR_ADAPTIVE_LATENCY; + unsigned int avg_wire_size, packets, bytes; + unsigned int packets_old; + unsigned long next_update = jiffies; + u32 old_itr; + u16 add_itr, add = 0; + if (type) + old_itr = q_vector->itr_rx; + else + old_itr = q_vector->itr_tx; + + /* If we don't have any rings just leave ourselves set for maximum + * possible latency so we take ourselves out of the equation. + */ + if (!ring_container->ring) + return; + + packets_old = ring_container->total_packets_old; + packets = ring_container->total_packets; + bytes = ring_container->total_bytes; + add_itr = ring_container->add_itr; + /* If Rx and there are 1 to 23 packets and bytes are less than + * 12112 assume insufficient data to use bulk rate limiting + * approach. Instead we will focus on simply trying to target + * receiving 8 times as much data in the next interrupt. + */ + + if (!packets) + return; + + if (packets && packets < 24 && bytes < 12112) { + itr = RNPVF_ITR_ADAPTIVE_LATENCY; + + avg_wire_size = (bytes + packets * 24); + avg_wire_size = + clamp_t(unsigned int, avg_wire_size, 128, 12800); + + goto adjust_for_speed; + } + + /* Less than 48 packets we can assume that our current interrupt delay + * is only slightly too low. As such we should increase it by a small + * fixed amount. + */ + if (packets < 48) { + if (add_itr) { + if (packets_old < packets) { + itr = (old_itr >> 2) + + RNPVF_ITR_ADAPTIVE_MIN_INC; + if (itr > RNPVF_ITR_ADAPTIVE_MAX_USECS) + itr = RNPVF_ITR_ADAPTIVE_MAX_USECS; + add = 1; + + if (packets < 8) + itr += RNPVF_ITR_ADAPTIVE_LATENCY; + else + itr += ring_container->itr & + RNPVF_ITR_ADAPTIVE_LATENCY; + + } else { + itr = (old_itr >> 2) - + RNPVF_ITR_ADAPTIVE_MIN_INC; + if (itr < RNPVF_ITR_ADAPTIVE_MIN_USECS) + itr = RNPVF_ITR_ADAPTIVE_MIN_USECS; + } + + } else { + add = 1; + itr = (old_itr >> 2) + RNPVF_ITR_ADAPTIVE_MIN_INC; + if (itr > RNPVF_ITR_ADAPTIVE_MAX_USECS) + itr = RNPVF_ITR_ADAPTIVE_MAX_USECS; + + /* If sample size is 0 - 7 we should probably switch + * to latency mode instead of trying to control + * things as though we are in bulk. + * + * Otherwise if the number of packets is less than 48 + * we should maintain whatever mode we are currently + * in. The range between 8 and 48 is the cross-over + * point between latency and bulk traffic. + */ + if (packets < 8) + itr += RNPVF_ITR_ADAPTIVE_LATENCY; + else + itr += ring_container->itr & + RNPVF_ITR_ADAPTIVE_LATENCY; + } + goto clear_counts; + } + + /* Between 48 and 96 is our "goldilocks" zone where we are working + * out "just right". Just report that our current ITR is good for us. + */ + if (packets < 96) { + itr = old_itr >> 2; + goto clear_counts; + } + /* If packet count is 96 or greater we are likely looking at a slight + * overrun of the delay we want. Try halving our delay to see if that + * will cut the number of packets in half per interrupt. + */ + if (packets < 256) { + itr = old_itr >> 3; + if (itr < RNPVF_ITR_ADAPTIVE_MIN_USECS) + itr = RNPVF_ITR_ADAPTIVE_MIN_USECS; + goto clear_counts; + } + + /* The paths below assume we are dealing with a bulk ITR since number + * of packets is 256 or greater. We are just going to have to compute + * a value and try to bring the count under control, though for smaller + * packet sizes there isn't much we can do as NAPI polling will likely + * be kicking in sooner rather than later. + */ + itr = RNPVF_ITR_ADAPTIVE_BULK; + + /* If packet counts are 256 or greater we can assume we have a gross + * overestimation of what the rate should be. Instead of trying to fine + * tune it just use the formula below to try and dial in an exact value + * give the current packet size of the frame. + */ + avg_wire_size = bytes / packets; + + /* The following is a crude approximation of: + * wmem_default / (size + overhead) = desired_pkts_per_int + * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate + * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value + * + * Assuming wmem_default is 212992 and overhead is 640 bytes per + * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the + * formula down to + * + * (170 * (size + 24)) / (size + 640) = ITR + * + * We first do some math on the packet size and then finally bitshift + * by 8 after rounding up. We also have to account for PCIe link speed + * difference as ITR scales based on this. + */ + if (avg_wire_size <= 60) { + /* Start at 50k ints/sec */ + avg_wire_size = 5120; + } else if (avg_wire_size <= 316) { + /* 50K ints/sec to 16K ints/sec */ + avg_wire_size *= 40; + avg_wire_size += 2720; + } else if (avg_wire_size <= 1084) { + /* 16K ints/sec to 9.2K ints/sec */ + avg_wire_size *= 15; + avg_wire_size += 11452; + } else if (avg_wire_size <= 1980) { + /* 9.2K ints/sec to 8K ints/sec */ + avg_wire_size *= 5; + avg_wire_size += 22420; + } else { + /* plateau at a limit of 8K ints/sec */ + avg_wire_size = 32256; + } + +adjust_for_speed: + /* Resultant value is 256 times larger than it needs to be. This + * gives us room to adjust the value as needed to either increase + * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc. + * + * Use addition as we have already recorded the new latency flag + * for the ITR value. + */ + switch (q_vector->adapter->link_speed) { + case RNP_LINK_SPEED_10GB_FULL: + case RNP_LINK_SPEED_100_FULL: + default: + itr += DIV_ROUND_UP(avg_wire_size, + RNPVF_ITR_ADAPTIVE_MIN_INC * 256) * + RNPVF_ITR_ADAPTIVE_MIN_INC; + break; + case RNP_LINK_SPEED_1GB_FULL: + case RNP_LINK_SPEED_10_FULL: + itr += DIV_ROUND_UP(avg_wire_size, + RNPVF_ITR_ADAPTIVE_MIN_INC * 64) * + RNPVF_ITR_ADAPTIVE_MIN_INC; + break; + } + + /* In the case of a latency specific workload only allow us to + * reduce the ITR by at most 2us. By doing this we should dial + * in so that our number of interrupts is no more than 2x the number + * of packets for the least busy workload. So for example in the case + * of a TCP worload the ack packets being received would set the + * the interrupt rate as they are a latency specific workload. + */ + if ((itr & RNPVF_ITR_ADAPTIVE_LATENCY) && + itr < ring_container->itr) + itr = ring_container->itr - RNPVF_ITR_ADAPTIVE_MIN_INC; + +clear_counts: + /* write back value */ + ring_container->itr = itr; + + /* next update should occur within next jiffy */ + ring_container->next_update = next_update + 1; + + ring_container->total_bytes = 0; + ring_container->total_packets_old = packets; + ring_container->add_itr = add; + ring_container->total_packets = 0; +} + +/** + * rnpvf_write_eitr - write EITR register in hardware specific way + * @q_vector: structure containing interrupt and ring information + * + * This function is made to be called by ethtool and by the driver + * when it needs to update EITR registers at runtime. Hardware + * specific quirks/differences are taken care of here. + */ +void rnpvf_write_eitr_rx(struct rnpvf_q_vector *q_vector) +{ + struct rnpvf_adapter *adapter = q_vector->adapter; + struct rnpvf_hw *hw = &adapter->hw; + u32 itr_reg = q_vector->itr_rx >> 2; + struct rnpvf_ring *ring; + + itr_reg = itr_reg * hw->usecstocount; + + rnpvf_for_each_ring(ring, q_vector->rx) + ring_wr32(ring, RNP_DMA_REG_RX_INT_DELAY_TIMER, itr_reg); +} + +static void rnpvf_set_itr(struct rnpvf_q_vector *q_vector) +{ + u32 new_itr_rx; + + rnpvf_update_itr(q_vector, &q_vector->rx, 1); + + new_itr_rx = q_vector->rx.itr; + /* Clear latency flag if set, shift into correct position */ + new_itr_rx &= RNPVF_ITR_ADAPTIVE_MASK_USECS; + /* in 2us unit */ + new_itr_rx <<= 2; + if (new_itr_rx != q_vector->itr_rx) { + /* save the algorithm value here */ + q_vector->itr_rx = new_itr_rx; + rnpvf_write_eitr_rx(q_vector); + } +} + +/** + * rnpvf_request_irq - initialize interrupts + * @adapter: board private structure + * + * Attempts to configure interrupts using the best available + * capabilities of the hardware and kernel. + **/ +static int rnpvf_request_irq(struct rnpvf_adapter *adapter) +{ + int err; + + if (adapter->flags & RNPVF_FLAG_MSIX_ENABLED) { + err = rnpvf_request_msix_irqs(adapter); + } else if (adapter->flags & RNPVF_FLAG_MSI_ENABLED) { + /* in this case one for all */ + err = request_irq(adapter->pdev->irq, rnpvf_intr, 0, + adapter->netdev->name, adapter); + } else { + err = request_irq(adapter->pdev->irq, rnpvf_intr, + IRQF_SHARED, adapter->netdev->name, + adapter); + } + if (err) + rnpvf_err("request_irq failed, Error %d\n", err); + + return err; +} + +static void rnpvf_free_irq(struct rnpvf_adapter *adapter) +{ + + if (adapter->flags & RNPVF_FLAG_MSIX_ENABLED) { + rnpvf_free_msix_irqs(adapter); + } else if (adapter->flags & RNPVF_FLAG_MSI_ENABLED) { + /* in this case one for all */ + free_irq(adapter->pdev->irq, adapter); + } else { + free_irq(adapter->pdev->irq, adapter); + } +} + +/** + * rnpvf_irq_disable - Mask off interrupt generation on the NIC + * @adapter: board private structure + **/ +static inline void rnpvf_irq_disable(struct rnpvf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_q_vectors; i++) { + rnpvf_irq_disable_queues(adapter->q_vector[i]); + if (adapter->flags & RNPVF_FLAG_MSIX_ENABLED) { + synchronize_irq( + adapter->msix_entries[i + + adapter->vector_off] + .vector); + } else { + synchronize_irq(adapter->pdev->irq); + } + } +} + +/** + * rnpvf_configure_tx_ring - Configure 8259x Tx ring after Reset + * @adapter: board private structure + * @ring: structure containing ring specific data + * + * Configure the Tx descriptor ring after a reset. + **/ +void rnpvf_configure_tx_ring(struct rnpvf_adapter *adapter, + struct rnpvf_ring *ring) +{ + struct rnpvf_hw *hw = &adapter->hw; + + /* disable queue to avoid issues while updating state */ + if (!(ring->ring_flags & RNPVF_RING_SKIP_TX_START)) + ring_wr32(ring, RNP_DMA_TX_START, 0); + + ring_wr32(ring, RNP_DMA_REG_TX_DESC_BUF_BASE_ADDR_LO, + (u32)ring->dma); + /* dma high address is used for vfnum */ + ring_wr32(ring, RNP_DMA_REG_TX_DESC_BUF_BASE_ADDR_HI, + (u32)(((u64)ring->dma) >> 32) | (hw->vfnum << 24)); + ring_wr32(ring, RNP_DMA_REG_TX_DESC_BUF_LEN, ring->count); + + ring->next_to_clean = + ring_rd32(ring, RNP_DMA_REG_TX_DESC_BUF_HEAD); + ring->next_to_use = ring->next_to_clean; + ring->tail = ring->ring_addr + RNP_DMA_REG_TX_DESC_BUF_TAIL; + rnpvf_wr_reg(ring->tail, ring->next_to_use); + + ring_wr32(ring, RNP_DMA_REG_TX_DESC_FETCH_CTRL, + (8 << 0) /* max_water_flow */ + | (TSRN10_TX_DEFAULT_BURST + << 16)); /* max-num_descs_peer_read */ + + ring_wr32(ring, RNP_DMA_REG_TX_INT_DELAY_TIMER, + adapter->tx_usecs * hw->usecstocount); + ring_wr32(ring, RNP_DMA_REG_TX_INT_DELAY_PKTCNT, + adapter->tx_frames); + + if (!(ring->ring_flags & RNPVF_RING_SKIP_TX_START)) { + /* n500 should wait tx_ready before open tx start */ + int timeout = 0; + u32 status = 0; + + do { + status = ring_rd32(ring, RNP_DMA_TX_READY); + usleep_range(100, 200); + timeout++; + rnpvf_dbg("wait %d tx ready to 1\n", + ring->rnpvf_queue_idx); + } while ((status != 1) && (timeout < 100)); + + if (timeout >= 100) + printk("wait tx ready timeout\n"); + ring_wr32(ring, RNP_DMA_TX_START, 1); + } +} + +/** + * rnpvf_configure_tx - Configure 82599 VF Transmit Unit after Reset + * @adapter: board private structure + * + * Configure the Tx unit of the MAC after a reset. + **/ +static void rnpvf_configure_tx(struct rnpvf_adapter *adapter) +{ + u32 i; + + /* Setup the HW Tx Head and Tail descriptor pointers */ + for (i = 0; i < (adapter->num_tx_queues); i++) + rnpvf_configure_tx_ring(adapter, adapter->tx_ring[i]); +} + +#define RNP_SRRCTL_BSIZEHDRSIZE_SHIFT 2 + +void rnpvf_disable_rx_queue(struct rnpvf_adapter *adapter, + struct rnpvf_ring *ring) +{ + ring_wr32(ring, RNP_DMA_RX_START, 0); +} + +void rnpvf_enable_rx_queue(struct rnpvf_adapter *adapter, + struct rnpvf_ring *ring) +{ + ring_wr32(ring, RNP_DMA_RX_START, 1); +} + +void rnpvf_configure_rx_ring(struct rnpvf_adapter *adapter, + struct rnpvf_ring *ring) +{ + struct rnpvf_hw *hw = &adapter->hw; + u64 desc_phy = ring->dma; + + /* disable queue to avoid issues while updating state */ + rnpvf_disable_rx_queue(adapter, ring); + + /* set descripts registers*/ + ring_wr32(ring, RNP_DMA_REG_RX_DESC_BUF_BASE_ADDR_LO, + (u32)desc_phy); + /* dma address high bits is used */ + ring_wr32(ring, RNP_DMA_REG_RX_DESC_BUF_BASE_ADDR_HI, + ((u32)(desc_phy >> 32)) | (hw->vfnum << 24)); + ring_wr32(ring, RNP_DMA_REG_RX_DESC_BUF_LEN, ring->count); + + ring->tail = ring->ring_addr + RNP_DMA_REG_RX_DESC_BUF_TAIL; + ring->next_to_clean = + ring_rd32(ring, RNP_DMA_REG_RX_DESC_BUF_HEAD); + ring->next_to_use = ring->next_to_clean; + +#define SCATER_SIZE (96) + if (ring->ring_flags & RNPVF_RING_SCATER_SETUP) { + ring_wr32(ring, PCI_DMA_REG_RX_SCATTER_LENGTH, + SCATER_SIZE); + } + + ring_wr32(ring, RNP_DMA_REG_RX_DESC_FETCH_CTRL, + 0 | (TSRN10_RX_DEFAULT_LINE << 0) /* rx-desc-flow */ + | (TSRN10_RX_DEFAULT_BURST + << 16) /* max-read-desc-cnt */ + ); + + if (ring->ring_flags & RNPVF_RING_IRQ_MISS_FIX) + ring_wr32(ring, RNP_DMA_INT_TRIG, + TX_INT_MASK | RX_INT_MASK); + + ring_wr32(ring, RNP_DMA_REG_RX_INT_DELAY_TIMER, + adapter->rx_usecs * hw->usecstocount); + ring_wr32(ring, RNP_DMA_REG_RX_INT_DELAY_PKTCNT, + adapter->rx_frames); + + rnpvf_alloc_rx_buffers(ring, rnpvf_desc_unused(ring)); +} + +static void rnpvf_set_rx_buffer_len(struct rnpvf_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN * 3; + struct rnpvf_ring *rx_ring; + int i; + + if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) + max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); + + for (i = 0; i < adapter->num_rx_queues; i++) { + rx_ring = adapter->rx_ring[i]; + clear_bit(__RNPVF_RX_3K_BUFFER, &rx_ring->state); + clear_bit(__RNPVF_RX_BUILD_SKB_ENABLED, &rx_ring->state); + + set_bit(__RNPVF_RX_BUILD_SKB_ENABLED, &rx_ring->state); + +#ifdef OPTM_WITH_LPAGE + rx_ring->rx_page_buf_nums = + RNPVF_PAGE_BUFFER_NUMS(rx_ring); + rx_ring->rx_per_buf_mem = RNPVF_RXBUFFER_2K; +#endif + + } +} + +/** + * rnpvf_configure_rx - Configure 82599 VF Receive Unit after Reset + * @adapter: board private structure + * + * Configure the Rx unit of the MAC after a reset. + **/ +static void rnpvf_configure_rx(struct rnpvf_adapter *adapter) +{ + int i; + + /* set_rx_buffer_len must be called before ring initialization */ + rnpvf_set_rx_buffer_len(adapter); + + /* + * Setup the HW Rx Head and Tail Descriptor Pointers and + * the Base and Length of the Rx Descriptor Ring + */ + for (i = 0; i < adapter->num_rx_queues; i++) + rnpvf_configure_rx_ring(adapter, adapter->rx_ring[i]); +} + +static int rnpvf_vlan_rx_add_vid(struct net_device *netdev, + __always_unused __be16 proto, u16 vid) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + struct rnp_mbx_info *mbx = &hw->mbx; + int err = 0; + + if ((vid) && (adapter->vf_vlan) && (vid != adapter->vf_vlan)) { + dev_err(&adapter->pdev->dev, + "only 1 vlan for vf or pf set vlan already\n"); + return 0; + } + if ((vid) && (!adapter->vf_vlan)) { + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + /* add VID to filter table */ + err = hw->mac.ops.set_vfta(hw, vid, 0, true); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + } + + /* translate error return types so error makes sense */ + if (err == RNP_ERR_MBX) + return -EIO; + + if (err == RNP_ERR_INVALID_ARGUMENT) + return -EACCES; + + set_bit(vid, adapter->active_vlans); + + if (vid) + hw->ops.set_veb_vlan(hw, vid, VFNUM(mbx, hw->vfnum)); + + return err; +} + +static int rnpvf_vlan_rx_kill_vid(struct net_device *netdev, + __always_unused __be16 proto, u16 vid) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + struct rnp_mbx_info *mbx = &hw->mbx; + int err = -EOPNOTSUPP; + + if (vid) { + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + /* remove VID from filter table */ + err = hw->mac.ops.set_vfta(hw, vid, 0, false); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + hw->ops.set_veb_vlan(hw, 0, VFNUM(mbx, hw->vfnum)); + } + + clear_bit(vid, adapter->active_vlans); + + return 0; +} + +/** + * rnpvf_vlan_strip_disable - helper to disable hw vlan stripping + * @adapter: driver data + */ +__maybe_unused static void +rnpvf_vlan_strip_disable(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + hw->mac.ops.set_vlan_strip(hw, false); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); +} + +/** + * rnpvf_vlan_strip_enable - helper to enable hw vlan stripping + * @adapter: driver data + */ +__maybe_unused static s32 +rnpvf_vlan_strip_enable(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + int err; + + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + err = hw->mac.ops.set_vlan_strip(hw, true); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + + return err; +} + +static void rnpvf_restore_vlan(struct rnpvf_adapter *adapter) +{ + u16 vid; + + rnpvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0); + for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) { + rnpvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), + vid); + } + +} + +static int rnpvf_write_uc_addr_list(struct net_device *netdev) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + int count = 0; + + if ((netdev_uc_count(netdev)) > 10) { + pr_err("Too many unicast filters - No Space\n"); + return -ENOSPC; + } + + if (!netdev_uc_empty(netdev)) { + struct netdev_hw_addr *ha; + + netdev_for_each_uc_addr(ha, netdev) { + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + udelay(200); + } + } else { + /* + * If the list is empty then send message to PF driver to + * clear all macvlans on this VF. + */ + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + hw->mac.ops.set_uc_addr(hw, 0, NULL); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + udelay(200); + } + + return count; +} + +/** + * rnpvf_set_rx_mode - Multicast and unicast set + * @netdev: network interface device structure + * + * The set_rx_method entry point is called whenever the multicast address + * list, unicast address list or the network interface flags are updated. + * This routine is responsible for configuring the hardware for proper + * multicast mode and configuring requested unicast filters. + **/ +static void rnpvf_set_rx_mode(struct net_device *netdev) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + + netdev_features_t features = netdev->features; + if ((netdev->flags & IFF_PROMISC) && (!adapter->promisc_mode)) { + adapter->promisc_mode = true; + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + /* reprogram multicast list */ + hw->mac.ops.set_promisc_mode(hw, true); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + + } + + if ((!(netdev->flags & IFF_PROMISC)) && (adapter->promisc_mode)) { + adapter->promisc_mode = false; + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + /* reprogram multicast list */ + hw->mac.ops.set_promisc_mode(hw, false); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + } + + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + /* reprogram multicast list */ + hw->mac.ops.update_mc_addr_list(hw, netdev); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + + rnpvf_write_uc_addr_list(netdev); + + if (features & NETIF_F_HW_VLAN_CTAG_RX) + rnpvf_vlan_strip_enable(adapter); + else + rnpvf_vlan_strip_disable(adapter); +} + +static void rnpvf_napi_enable_all(struct rnpvf_adapter *adapter) +{ + int q_idx; + + for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) + napi_enable(&adapter->q_vector[q_idx]->napi); +} + +static void rnpvf_napi_disable_all(struct rnpvf_adapter *adapter) +{ + int q_idx; + + for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) + napi_disable(&adapter->q_vector[q_idx]->napi); +} + +static void rnpvf_configure_veb(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + u32 ring; + u8 *mac; + + if (is_valid_ether_addr(hw->mac.addr)) + mac = hw->mac.addr; + else + mac = hw->mac.perm_addr; + + ring = adapter->rx_ring[0]->rnpvf_queue_idx; + ring |= ((0x80 | vfnum) << 8); + + hw->ops.set_veb_mac(hw, mac, vfnum, ring); +} + +static void rnpvf_configure(struct rnpvf_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + + rnpvf_set_rx_mode(netdev); + rnpvf_restore_vlan(adapter); + rnpvf_configure_tx(adapter); + rnpvf_configure_rx(adapter); + rnpvf_configure_veb(adapter); +} + +#define RNP_MAX_RX_DESC_POLL 10 + +static void rnpvf_save_reset_stats(struct rnpvf_adapter *adapter) +{ + /* Only save pre-reset stats if there are some */ + if (adapter->stats.vfgprc || adapter->stats.vfgptc) { + adapter->stats.saved_reset_vfgprc += + adapter->stats.vfgprc - adapter->stats.base_vfgprc; + adapter->stats.saved_reset_vfgptc += + adapter->stats.vfgptc - adapter->stats.base_vfgptc; + adapter->stats.saved_reset_vfgorc += + adapter->stats.vfgorc - adapter->stats.base_vfgorc; + adapter->stats.saved_reset_vfgotc += + adapter->stats.vfgotc - adapter->stats.base_vfgotc; + adapter->stats.saved_reset_vfmprc += + adapter->stats.vfmprc - adapter->stats.base_vfmprc; + } +} + +static void rnpvf_up_complete(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + int i; + + rnpvf_configure_msix(adapter); + + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + + if (is_valid_ether_addr(hw->mac.addr)) + hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); + else + hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); + + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + + rnpvf_napi_enable_all(adapter); + + /*clear any pending interrupts*/ + rnpvf_irq_enable(adapter); + + /* enable transmits */ + netif_tx_start_all_queues(adapter->netdev); + + rnpvf_save_reset_stats(adapter); + + hw->mac.get_link_status = 1; + mod_timer(&adapter->watchdog_timer, jiffies); + + clear_bit(__RNPVF_DOWN, &adapter->state); + for (i = 0; i < adapter->num_rx_queues; i++) + rnpvf_enable_rx_queue(adapter, adapter->rx_ring[i]); +} + +void rnpvf_reinit_locked(struct rnpvf_adapter *adapter) +{ + WARN_ON(in_interrupt()); + /* put off any impending NetWatchDogTimeout */ + + while (test_and_set_bit(__RNPVF_RESETTING, &adapter->state)) + usleep_range(1000, 2000); + + rnpvf_down(adapter); + rnpvf_reset(adapter); + rnpvf_up(adapter); + clear_bit(__RNPVF_RESETTING, &adapter->state); +} + +void rnpvf_up(struct rnpvf_adapter *adapter) +{ + + rnpvf_configure(adapter); + rnpvf_up_complete(adapter); +} + +void rnpvf_reset(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + struct net_device *netdev = adapter->netdev; + + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + if (hw->mac.ops.reset_hw(hw)) + hw_dbg(hw, "PF still resetting\n"); + else + hw->mac.ops.init_hw(hw); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + + if (is_valid_ether_addr(adapter->hw.mac.addr)) { + eth_hw_addr_set(netdev, adapter->hw.mac.addr); + memcpy(netdev->perm_addr, adapter->hw.mac.addr, + netdev->addr_len); + } +} + +/** + * rnpvf_clean_tx_ring - Free Tx Buffers + * @adapter: board private structure + * @tx_ring: ring to be cleaned + **/ +static void rnpvf_clean_tx_ring(struct rnpvf_adapter *adapter, + struct rnpvf_ring *tx_ring) +{ + struct rnpvf_tx_buffer *tx_buffer_info; + unsigned long size; + u16 i; + + BUG_ON(tx_ring == NULL); + + /* ring already cleared, nothing to do */ + if (!tx_ring->tx_buffer_info) + return; + + /* Free all the Tx ring sk_buffs */ + for (i = 0; i < tx_ring->count; i++) { + tx_buffer_info = &tx_ring->tx_buffer_info[i]; + rnpvf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); + } + + netdev_tx_reset_queue(txring_txq(tx_ring)); + + size = sizeof(struct rnpvf_tx_buffer) * tx_ring->count; + memset(tx_ring->tx_buffer_info, 0, size); + + /* Zero out the descriptor ring */ + memset(tx_ring->desc, 0, tx_ring->size); + + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; +} + +/** + * rnpvf_clean_all_rx_rings - Free Rx Buffers for all queues + * @adapter: board private structure + **/ +static void rnpvf_clean_all_rx_rings(struct rnpvf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_rx_queues; i++) + rnpvf_clean_rx_ring(adapter->rx_ring[i]); +} + +/** + * rnpvf_clean_all_tx_rings - Free Tx Buffers for all queues + * @adapter: board private structure + **/ +static void rnpvf_clean_all_tx_rings(struct rnpvf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) + rnpvf_clean_tx_ring(adapter, adapter->tx_ring[i]); +} + +void rnpvf_down(struct rnpvf_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + int i; + + /* signal that we are down to the interrupt handler */ + set_bit(__RNPVF_DOWN, &adapter->state); + set_bit(__RNPVF_LINK_DOWN, &adapter->state); + + /* disable all enabled rx queues */ + for (i = 0; i < adapter->num_rx_queues; i++) + rnpvf_disable_rx_queue(adapter, adapter->rx_ring[i]); + + usleep_range(1000, 2000); + + netif_tx_stop_all_queues(netdev); + + /* call carrier off first to avoid false dev_watchdog timeouts */ + netif_carrier_off(netdev); + + netif_tx_disable(netdev); + + rnpvf_irq_disable(adapter); + + rnpvf_napi_disable_all(adapter); + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct rnpvf_ring *tx_ring = adapter->tx_ring[i]; + + if (!(tx_ring->ring_flags & RNPVF_RING_SKIP_TX_START)) { + int head, tail; + int timeout = 0; + + head = ring_rd32(tx_ring, + RNP_DMA_REG_TX_DESC_BUF_HEAD); + tail = ring_rd32(tx_ring, + RNP_DMA_REG_TX_DESC_BUF_TAIL); + + while (head != tail) { + usleep_range(10000, 20000); + + head = ring_rd32( + tx_ring, + RNP_DMA_REG_TX_DESC_BUF_HEAD); + tail = ring_rd32( + tx_ring, + RNP_DMA_REG_TX_DESC_BUF_TAIL); + timeout++; + if (timeout >= 100) { + printk("vf wait tx done timeout\n"); + break; + } + } + } + } + + /* disable transmits in the hardware now that interrupts are off */ + for (i = 0; i < adapter->num_tx_queues; i++) { + struct rnpvf_ring *tx_ring = adapter->tx_ring[i]; + + if (!(tx_ring->ring_flags & RNPVF_RING_SKIP_TX_START)) + ring_wr32(tx_ring, RNP_DMA_TX_START, 0); + } + + netif_carrier_off(netdev); + rnpvf_clean_all_tx_rings(adapter); + rnpvf_clean_all_rx_rings(adapter); +} + +static netdev_features_t rnpvf_fix_features(struct net_device *netdev, + netdev_features_t features) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + /* If Rx checksum is disabled, then RSC/LRO should also be disabled */ + if (!(features & NETIF_F_RXCSUM)) { + features &= ~NETIF_F_LRO; + adapter->flags &= (~RNPVF_FLAG_RX_CHKSUM_ENABLED); + } else + adapter->flags |= RNPVF_FLAG_RX_CHKSUM_ENABLED; + + /* vf not support change vlan filter */ + if ((netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) != + (features & NETIF_F_HW_VLAN_CTAG_FILTER)) { + if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) + features |= NETIF_F_HW_VLAN_CTAG_FILTER; + else + features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; + } + if (adapter->flags & RNPVF_FLAG_PF_SET_VLAN) { + /* if in this mode , close tx/rx vlan offload */ + if (features & NETIF_F_HW_VLAN_CTAG_RX) + adapter->priv_flags |= RNPVF_FLAG_RX_VLAN_OFFLOAD; + else + adapter->priv_flags &= ~RNPVF_FLAG_RX_VLAN_OFFLOAD; + + features |= NETIF_F_HW_VLAN_CTAG_RX; + + if (features & NETIF_F_HW_VLAN_CTAG_TX) + adapter->priv_flags |= RNPVF_FLAG_TX_VLAN_OFFLOAD; + else + adapter->priv_flags &= ~RNPVF_FLAG_TX_VLAN_OFFLOAD; + + features &= ~NETIF_F_HW_VLAN_CTAG_TX; + } + + /* if pf set fcs on, we should close rx chksum */ + if (adapter->priv_flags & RNPVF_PRIV_FLAG_FCS_ON) + features &= ~NETIF_F_RXCSUM; + + return features; +} + +static int rnpvf_set_features(struct net_device *netdev, + netdev_features_t features) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + netdev_features_t changed = netdev->features ^ features; + bool need_reset = false; + int err = 0; + + netdev->features = features; + if (changed & NETIF_F_HW_VLAN_CTAG_RX) { + if (features & NETIF_F_HW_VLAN_CTAG_RX) { + if ((!rnpvf_vlan_strip_enable(adapter))) + features &= ~NETIF_F_HW_VLAN_CTAG_RX; + } else { + rnpvf_vlan_strip_disable(adapter); + } + } + + netdev->features = features; + + if (need_reset) + rnpvf_reset(adapter); + + return err; +} + +/** + * rnpvf_tx_timeout - Respond to a Tx Hang + * @netdev: network interface device structure + **/ +__maybe_unused static void rnpvf_tx_timeout(struct net_device *netdev) +{ +} + +/** + * rnpvf_sw_init - Initialize general software structures + * (struct rnpvf_adapter) + * @adapter: board private structure to initialize + * + * rnpvf_sw_init initializes the Adapter private data structure. + * Fields are initialized based on PCI device information and + * OS network device settings (MTU size). + **/ +static int rnpvf_sw_init(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + struct pci_dev *pdev = adapter->pdev; + struct net_device *netdev = adapter->netdev; + int err; + + /* PCI config space info */ + hw->pdev = pdev; + + hw->vendor_id = pdev->vendor; + hw->device_id = pdev->device; + hw->subsystem_vendor_id = pdev->subsystem_vendor; + hw->subsystem_device_id = pdev->subsystem_device; + hw->mbx.ops.init_params(hw); + + /*initialization default pause flow */ + hw->fc.requested_mode = rnp_fc_none; + hw->fc.current_mode = rnp_fc_none; + + /* now vf other irq handler is not regist */ + err = hw->mac.ops.reset_hw(hw); + if (err) { + dev_info( + &pdev->dev, + "PF still in reset state. Is the PF interface up?\n"); + hw->adapter_stopped = false; + hw->link = false; + hw->speed = 0; + hw->usecstocount = 500; + return err; + } else { + err = hw->mac.ops.init_hw(hw); + if (err) { + pr_err("init_shared_code failed: %d\n", err); + goto out; + } + err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); + if (err) + dev_info(&pdev->dev, + "Error reading MAC address\n"); + else if (is_zero_ether_addr(adapter->hw.mac.addr)) + dev_info( + &pdev->dev, + "MAC address not assigned by administrator.\n"); + eth_hw_addr_set(netdev, hw->mac.addr); + } + + if (!is_valid_ether_addr(netdev->dev_addr)) { + dev_info(&pdev->dev, "Assigning random MAC address\n"); + eth_hw_addr_random(netdev); + memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len); + } + /* get info from pf */ + err = hw->mac.ops.get_queues(hw); + if (err) { + dev_info(&pdev->dev, + "Get queue info error, use default one \n"); + hw->mac.max_tx_queues = MAX_TX_QUEUES; + hw->mac.max_rx_queues = MAX_RX_QUEUES; + hw->queue_ring_base = + (hw->vfnum & VF_NUM_MASK) * MAX_RX_QUEUES; + } + + dev_info(&pdev->dev, "queue_ring_base %d num %d\n", + hw->queue_ring_base, hw->mac.max_tx_queues); + err = hw->mac.ops.get_mtu(hw); + if (err) { + dev_info(&pdev->dev, "Get mtu error ,use default one\n"); + hw->mtu = 1500; + } + /* lock to protect mailbox accesses */ + spin_lock_init(&adapter->mbx_lock); + + /* set default ring sizes */ + adapter->tx_ring_item_count = hw->tx_items_count; + adapter->rx_ring_item_count = hw->rx_items_count; + adapter->dma_channels = + min_t(int, hw->mac.max_tx_queues, hw->mac.max_rx_queues); + DPRINTK(PROBE, INFO, "tx parameters %d, rx parameters %d\n", + adapter->tx_ring_item_count, adapter->rx_ring_item_count); + + /* set default tx/rx soft count */ + adapter->adaptive_rx_coal = 1; + adapter->adaptive_tx_coal = 1; + adapter->napi_budge = RNPVF_DEFAULT_RX_WORK; + adapter->tx_work_limit = RNPVF_DEFAULT_TX_WORK; + adapter->rx_usecs = RNPVF_PKT_TIMEOUT; + adapter->rx_frames = RNPVF_RX_PKT_POLL_BUDGET; + adapter->tx_usecs = RNPVF_PKT_TIMEOUT_TX; + adapter->tx_frames = RNPVF_TX_PKT_POLL_BUDGET; + + set_bit(__RNPVF_DOWN, &adapter->state); + return 0; + +out: + return err; +} + +static int rnpvf_acquire_msix_vectors(struct rnpvf_adapter *adapter, + int vectors) +{ + int err = 0; + int vector_threshold; + + /* We'll want at least 2 (vector_threshold): + * 1) TxQ[0] + RxQ[0] handler + * 2) Other (Link Status Change, etc.) + */ + vector_threshold = MIN_MSIX_COUNT; + + /* The more we get, the more we will assign to Tx/Rx Cleanup + * for the separate queues...where Rx Cleanup >= Tx Cleanup. + * Right now, we simply care about how many we'll get; we'll + * set them up later while requesting irq's. + */ + err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, + vectors, vectors); + if (err > 0) { + /* Success or a nasty failure. */ + vectors = err; + err = 0; + } + DPRINTK(PROBE, INFO, "err:%d, vectors:%d\n", err, vectors); + if (err < 0) { + dev_err(&adapter->pdev->dev, + "Unable to allocate MSI-X interrupts\n"); + kfree(adapter->msix_entries); + adapter->msix_entries = NULL; + } else { + /* + * Adjust for only the vectors we'll use, which is minimum + * of max_msix_q_vectors + NON_Q_VECTORS, or the number of + * vectors we were allocated. + */ + adapter->num_msix_vectors = vectors; + } + + return err; +} + +/** + * rnpvf_set_num_queues - Allocate queues for device, feature dependent + * @adapter: board private structure to initialize + * + * This is the top level queue allocation routine. The order here is very + * important, starting with the "most" number of features turned on at once, + * and ending with the smallest set of features. This way large combinations + * can be allocated if they're turned on, and smaller combinations are the + * fallthrough conditions. + * + **/ +static void rnpvf_set_num_queues(struct rnpvf_adapter *adapter) +{ + /* Start with base case */ + adapter->num_rx_queues = adapter->dma_channels; + adapter->num_tx_queues = adapter->dma_channels; +} + +/** + * rnpvf_set_interrupt_capability - set MSI-X or FAIL if not supported + * @adapter: board private structure to initialize + * + * Attempt to configure the interrupts using the best available + * capabilities of the hardware and the kernel. + **/ +static int rnpvf_set_interrupt_capability(struct rnpvf_adapter *adapter) +{ + int err = 0; + int vector, v_budget; + int irq_mode_back = adapter->irq_mode; + /* + * It's easy to be greedy for MSI-X vectors, but it really + * doesn't do us much good if we have a lot more vectors + * than CPU's. So let's be conservative and only ask for + * (roughly) the same number of vectors as there are CPU's. + * The default is to use pairs of vectors. + */ + v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); + v_budget = min_t(int, v_budget, num_online_cpus()); + v_budget += NON_Q_VECTORS; + v_budget = min_t(int, v_budget, MAX_MSIX_VECTORS); + + if (adapter->irq_mode == irq_mode_msix) { + /* A failure in MSI-X entry allocation isn't fatal, but it does + * mean we disable MSI-X capabilities of the adapter. */ + adapter->msix_entries = kcalloc( + v_budget, sizeof(struct msix_entry), GFP_KERNEL); + if (!adapter->msix_entries) { + err = -ENOMEM; + goto out; + } + + for (vector = 0; vector < v_budget; vector++) + adapter->msix_entries[vector].entry = vector; + + err = rnpvf_acquire_msix_vectors(adapter, v_budget); + if (!err) { + adapter->vector_off = NON_Q_VECTORS; + adapter->num_q_vectors = + adapter->num_msix_vectors - NON_Q_VECTORS; + DPRINTK(PROBE, INFO, + "adapter%d alloc vectors: cnt:%d [%d~%d] num_msix_vectors:%d\n", + adapter->bd_number, v_budget, + adapter->vector_off, + adapter->vector_off + v_budget - 1, + adapter->num_msix_vectors); + adapter->flags |= RNPVF_FLAG_MSIX_ENABLED; + goto out; + } + kfree(adapter->msix_entries); + + if (adapter->flags & RNPVF_FLAG_MSI_CAPABLE) { + adapter->irq_mode = irq_mode_msi; + pr_info("acquire msix failed, try to use msi\n"); + } + + } else { + pr_info("adapter not in msix mode\n"); + } + + if (adapter->irq_mode == irq_mode_msi) { + err = pci_enable_msi(adapter->pdev); + if (err) { + pr_info("Failed to allocate MSI interrupt, falling back to legacy. " + "Error"); + } else { + /* msi mode use only 1 irq */ + adapter->flags |= RNPVF_FLAG_MSI_ENABLED; + } + } + /* write back origin irq_mode */ + adapter->irq_mode = irq_mode_back; + /* legacy and msi only 1 vectors */ + adapter->num_q_vectors = 1; + +out: + return err; +} + +static void rnpvf_add_ring(struct rnpvf_ring *ring, + struct rnpvf_ring_container *head) +{ + ring->next = head->ring; + head->ring = ring; + head->count++; +} + +static enum hrtimer_restart irq_miss_check(struct hrtimer *hrtimer) +{ + struct rnpvf_q_vector *q_vector; + struct rnpvf_ring *ring; + struct rnp_tx_desc *eop_desc; + struct rnpvf_adapter *adapter; + struct rnpvf_hw *hw; + + int tx_next_to_clean; + int tx_next_to_use; + + struct rnpvf_tx_buffer *tx_buffer; + union rnp_rx_desc *rx_desc; + + q_vector = container_of(hrtimer, struct rnpvf_q_vector, + irq_miss_check_timer); + adapter = q_vector->adapter; + hw = &adapter->hw; + + /* If we're already down or resetting, just bail */ + if (test_bit(__RNPVF_DOWN, &adapter->state) || + test_bit(__RNPVF_RESETTING, &adapter->state)) + goto do_self_napi; + + rnpvf_irq_disable_queues(q_vector); + /* check tx irq miss */ + rnpvf_for_each_ring(ring, q_vector->tx) { + tx_next_to_clean = ring->next_to_clean; + tx_next_to_use = ring->next_to_use; + /* have work to do */ + if (tx_next_to_use != tx_next_to_clean) { + tx_buffer = + &ring->tx_buffer_info[tx_next_to_clean]; + eop_desc = tx_buffer->next_to_watch; + /* have tx done */ + if (eop_desc) { + if ((eop_desc->cmd & + cpu_to_le16(RNP_TXD_STAT_DD))) { + if (q_vector->new_rx_count != + q_vector->old_rx_count) { + ring_wr32( + ring, + RNP_DMA_REG_RX_INT_DELAY_PKTCNT, + q_vector->new_rx_count); + q_vector->old_rx_count = + q_vector->new_rx_count; + } + napi_schedule_irqoff( + &q_vector->napi); + goto do_self_napi; + } + } + } + } + + /* check rx irq */ + rnpvf_for_each_ring(ring, q_vector->rx) { + rx_desc = RNPVF_RX_DESC(ring, ring->next_to_clean); + + if (rx_desc) { + if (rnpvf_test_staterr(rx_desc, RNP_RXD_STAT_DD)) { + unsigned int size; + + size = le16_to_cpu(rx_desc->wb.len) - + le16_to_cpu( + rx_desc->wb.padding_len); + + if (size) { + if (q_vector->new_rx_count != + q_vector->old_rx_count) { + ring_wr32( + ring, + RNP_DMA_REG_RX_INT_DELAY_PKTCNT, + q_vector->new_rx_count); + q_vector->old_rx_count = + q_vector->new_rx_count; + } + napi_schedule_irqoff( + &q_vector->napi); + } else { + adapter->flags |= + RNPVF_FLAG_PF_RESET_REQ; + } + goto do_self_napi; + } + } + } + rnpvf_irq_enable_queues(q_vector); +do_self_napi: + return HRTIMER_NORESTART; +} + +static int rnpvf_alloc_q_vector(struct rnpvf_adapter *adapter, + int eth_queue_idx, int rnpvf_vector, + int rnpvf_queue, int r_count, int step) +{ + struct rnpvf_q_vector *q_vector; + struct rnpvf_ring *ring; + struct rnpvf_hw *hw = &adapter->hw; + int node = NUMA_NO_NODE; + int cpu = -1; + int ring_count, size; + int txr_count, rxr_count, idx; + int rxr_idx = rnpvf_queue, txr_idx = rnpvf_queue; + + DPRINTK(PROBE, INFO, + "eth_queue_idx:%d rnpvf_vector:%d(off:%d) ring:%d " + "ring_cnt:%d, step:%d\n", + eth_queue_idx, rnpvf_vector, adapter->vector_off, + rnpvf_queue, r_count, step); + + txr_count = rxr_count = r_count; + + ring_count = txr_count + rxr_count; + size = sizeof(struct rnpvf_q_vector) + + (sizeof(struct rnpvf_ring) * ring_count); + + if (cpu_online(rnpvf_vector)) { + cpu = rnpvf_vector; + node = cpu_to_node(cpu); + } + + /* allocate q_vector and rings */ + q_vector = kzalloc_node(size, GFP_KERNEL, node); + if (!q_vector) + q_vector = kzalloc(size, GFP_KERNEL); + if (!q_vector) + return -ENOMEM; + + /* setup affinity mask and node */ + if (cpu != -1) + cpumask_set_cpu(cpu, &q_vector->affinity_mask); + q_vector->numa_node = node; + + netif_napi_add_weight(adapter->netdev, &q_vector->napi, rnpvf_poll, + adapter->napi_budge); + /* tie q_vector and adapter together */ + adapter->q_vector[rnpvf_vector - adapter->vector_off] = q_vector; + q_vector->adapter = adapter; + q_vector->v_idx = rnpvf_vector; + + /* initialize pointer to rings */ + ring = q_vector->ring; + + for (idx = 0; idx < txr_count; idx++) { + /* assign generic ring traits */ + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; + + /* configure backlink on ring */ + ring->q_vector = q_vector; + + /* update q_vector Tx values */ + rnpvf_add_ring(ring, &q_vector->tx); + + /* apply Tx specific ring traits */ + ring->count = adapter->tx_ring_item_count; + ring->queue_index = eth_queue_idx + idx; + ring->rnpvf_queue_idx = txr_idx; + + if (hw->board_type == rnp_board_n10) { + ring->ring_flags |= RNPVF_RING_SKIP_TX_START; + ring->ring_addr = hw->hw_addr + RNP_RING_BASE_N10 + + RNP_RING_OFFSET(txr_idx); + ring->rnpvf_msix_off = txr_idx; + } + + ring->dma_int_stat = ring->ring_addr + RNP_DMA_INT_STAT; + ring->dma_int_mask = ring->dma_int_stat + 4; + ring->dma_int_clr = ring->dma_int_stat + 8; + ring->device_id = adapter->pdev->device; + ring->vfnum = hw->vfnum; + + /* assign ring to adapter */ + adapter->tx_ring[ring->queue_index] = ring; + dbg("adapter->tx_ringp[%d] <= %p\n", ring->queue_index, + ring); + + /* update count and index */ + txr_idx += step; + + DPRINTK(PROBE, INFO, + "vector[%d] <--RNP TxRing:%d, eth_queue:%d\n", + rnpvf_vector, ring->rnpvf_queue_idx, + ring->queue_index); + + /* push pointer to next ring */ + ring++; + } + + for (idx = 0; idx < rxr_count; idx++) { + /* assign generic ring traits */ + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; + + /* configure backlink on ring */ + ring->q_vector = q_vector; + + /* update q_vector Rx values */ + rnpvf_add_ring(ring, &q_vector->rx); + + /* apply Rx specific ring traits */ + ring->count = adapter->rx_ring_item_count; + ring->queue_index = eth_queue_idx + idx; + ring->rnpvf_queue_idx = rxr_idx; + + if (hw->board_type == rnp_board_n10) { + ring->ring_addr = hw->hw_addr + RNP_RING_BASE_N10 + + RNP_RING_OFFSET(rxr_idx); + ring->rnpvf_msix_off = rxr_idx; + } + ring->dma_int_stat = ring->ring_addr + RNP_DMA_INT_STAT; + ring->dma_int_mask = ring->dma_int_stat + 4; + ring->dma_int_clr = ring->dma_int_stat + 8; + ring->device_id = adapter->pdev->device; + ring->vfnum = hw->vfnum; + + /* assign ring to adapter */ + adapter->rx_ring[ring->queue_index] = ring; + DPRINTK(PROBE, INFO, + "vector[%d] <--RNP RxRing:%d, eth_queue:%d\n", + rnpvf_vector, ring->rnpvf_queue_idx, + ring->queue_index); + + /* update count and index */ + rxr_idx += step; + + /* push pointer to next ring */ + ring++; + } + + if (hw->board_type == rnp_board_n10) { + q_vector->vector_flags |= + RNPVF_QVECTOR_FLAG_IRQ_MISS_CHECK; + q_vector->vector_flags |= + RNPVF_QVECTOR_FLAG_REDUCE_TX_IRQ_MISS; + /* initialize timer */ + q_vector->irq_check_usecs = 1000; + hrtimer_init(&q_vector->irq_miss_check_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + q_vector->irq_miss_check_timer.function = + irq_miss_check; /* initialize NAPI */ + } + + return 0; +} + +static void rnpvf_free_q_vector(struct rnpvf_adapter *adapter, int v_idx) +{ + struct rnpvf_q_vector *q_vector; + struct rnpvf_ring *ring; + + dbg("v_idx:%d\n", v_idx); + + q_vector = adapter->q_vector[v_idx]; + + rnpvf_for_each_ring(ring, q_vector->tx) + adapter->tx_ring[ring->queue_index] = NULL; + + rnpvf_for_each_ring(ring, q_vector->rx) + adapter->rx_ring[ring->queue_index] = NULL; + + adapter->q_vector[v_idx] = NULL; + netif_napi_del(&q_vector->napi); + + if (q_vector->vector_flags & RNPVF_QVECTOR_FLAG_IRQ_MISS_CHECK) + rnpvf_htimer_stop(q_vector); + + /* + * rnpvf_get_stats64() might access the rings on this vector, + * we must wait a grace period before freeing it. + */ + kfree_rcu(q_vector, rcu); +} + +/** + * rnpvf_alloc_q_vectors - Allocate memory for interrupt vectors + * @adapter: board private structure to initialize + * + * We allocate one q_vector per queue interrupt. If allocation fails we + * return -ENOMEM. + **/ +static int rnpvf_alloc_q_vectors(struct rnpvf_adapter *adapter) +{ + int vector_idx = adapter->vector_off; + int ring_idx = adapter->hw.queue_ring_base; + int ring_remaing = + min_t(int, adapter->num_tx_queues, adapter->num_rx_queues); + int ring_step = 1; + int err, ring_cnt, + vector_remaing = adapter->num_msix_vectors - NON_Q_VECTORS; + int eth_queue_idx = 0; + + BUG_ON(ring_remaing == 0); + BUG_ON(vector_remaing == 0); + + for (; ring_remaing > 0 && vector_remaing > 0; vector_remaing--) { + ring_cnt = DIV_ROUND_UP(ring_remaing, vector_remaing); + + err = rnpvf_alloc_q_vector(adapter, eth_queue_idx, + vector_idx, ring_idx, ring_cnt, + ring_step); + if (err) + goto err_out; + + ring_idx += ring_step * ring_cnt; + ring_remaing -= ring_cnt; + vector_idx++; + eth_queue_idx += ring_cnt; + } + + return 0; + +err_out: + vector_idx -= adapter->vector_off; + while (vector_idx--) + rnpvf_free_q_vector(adapter, vector_idx); + return -ENOMEM; +} + +/** + * rnpvf_free_q_vectors - Free memory allocated for interrupt vectors + * @adapter: board private structure to initialize + * + * This function frees the memory allocated to the q_vectors. In addition if + * NAPI is enabled it will delete any references to the NAPI struct prior + * to freeing the q_vector. + **/ +static void rnpvf_free_q_vectors(struct rnpvf_adapter *adapter) +{ + int i, v_idx = adapter->num_q_vectors; + struct rnpvf_hw_stats_own *hw_stats = &adapter->hw_stats; + + adapter->num_rx_queues = 0; + adapter->num_tx_queues = 0; + adapter->num_q_vectors = 0; + + for (i = 0; i < v_idx; i++) + rnpvf_free_q_vector(adapter, i); + hw_stats->spoof_dropped = 0; +} + +/** + * rnpvf_reset_interrupt_capability - Reset MSIX setup + * @adapter: board private structure + * + **/ +static void rnpvf_reset_interrupt_capability(struct rnpvf_adapter *adapter) +{ + if (adapter->flags & RNPVF_FLAG_MSIX_ENABLED) { + pci_disable_msix(adapter->pdev); + kfree(adapter->msix_entries); + adapter->msix_entries = NULL; + } else if (adapter->flags & RNPVF_FLAG_MSI_ENABLED) { + pci_disable_msi(adapter->pdev); + } +} + +/** + * rnpvf_init_interrupt_scheme - Determine if MSIX is supported and init + * @adapter: board private structure to initialize + * + **/ +int rnpvf_init_interrupt_scheme(struct rnpvf_adapter *adapter) +{ + int err; + + /* Number of supported queues */ + rnpvf_set_num_queues(adapter); + + err = rnpvf_set_interrupt_capability(adapter); + if (err) { + hw_dbg(&adapter->hw, + "Unable to setup interrupt capabilities\n"); + goto err_set_interrupt; + } + + err = rnpvf_alloc_q_vectors(adapter); + if (err) { + hw_dbg(&adapter->hw, "Unable to allocate memory for queue " + "vectors\n"); + goto err_alloc_q_vectors; + } + + hw_dbg(&adapter->hw, + "Multiqueue %s: Rx Queue count = %u, " + "Tx Queue count = %u\n", + (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled", + adapter->num_rx_queues, adapter->num_tx_queues); + + set_bit(__RNPVF_DOWN, &adapter->state); + + return 0; +err_alloc_q_vectors: + rnpvf_reset_interrupt_capability(adapter); +err_set_interrupt: + return err; +} + +/** + * rnpvf_clear_interrupt_scheme - Clear the current interrupt scheme settings + * @adapter: board private structure to clear interrupt scheme on + * + * We go through and clear interrupt specific resources and reset the structure + * to pre-load conditions + **/ +void rnpvf_clear_interrupt_scheme(struct rnpvf_adapter *adapter) +{ + adapter->num_tx_queues = 0; + adapter->num_rx_queues = 0; + + rnpvf_free_q_vectors(adapter); + rnpvf_reset_interrupt_capability(adapter); +} + +#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ + { \ + u32 current_counter = RNP_READ_REG(hw, reg); \ + if (current_counter < last_counter) \ + counter += 0x100000000LL; \ + last_counter = current_counter; \ + counter &= 0xFFFFFFFF00000000LL; \ + counter |= current_counter; \ + } + +#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ + { \ + u64 current_counter_lsb = RNP_READ_REG(hw, reg_lsb); \ + u64 current_counter_msb = RNP_READ_REG(hw, reg_msb); \ + u64 current_counter = (current_counter_msb << 32) | \ + current_counter_lsb; \ + if (current_counter < last_counter) \ + counter += 0x1000000000LL; \ + last_counter = current_counter; \ + counter &= 0xFFFFFFF000000000LL; \ + counter |= current_counter; \ + } + +/** + * rnpvf_update_stats - Update the board statistics counters. + * @adapter: board private structure + **/ +void rnpvf_update_stats(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw_stats_own *hw_stats = &adapter->hw_stats; + int i; + struct net_device_stats *net_stats = &adapter->netdev->stats; + + net_stats->tx_packets = 0; + net_stats->tx_bytes = 0; + net_stats->rx_packets = 0; + net_stats->rx_bytes = 0; + net_stats->rx_dropped = 0; + net_stats->rx_errors = 0; + + hw_stats->vlan_add_cnt = 0; + hw_stats->vlan_strip_cnt = 0; + hw_stats->csum_err = 0; + hw_stats->csum_good = 0; + for (i = 0; i < adapter->num_q_vectors; i++) { + struct rnpvf_ring *ring; + struct rnpvf_q_vector *q_vector = adapter->q_vector[i]; + + rnpvf_for_each_ring(ring, q_vector->tx) { + hw_stats->vlan_add_cnt += ring->tx_stats.vlan_add; + net_stats->tx_packets += ring->stats.packets; + net_stats->tx_bytes += ring->stats.bytes; + } + + rnpvf_for_each_ring(ring, q_vector->rx) { + hw_stats->csum_err += ring->rx_stats.csum_err; + hw_stats->csum_good += ring->rx_stats.csum_good; + hw_stats->vlan_strip_cnt += + ring->rx_stats.vlan_remove; + net_stats->rx_packets += ring->stats.packets; + net_stats->rx_bytes += ring->stats.bytes; + net_stats->rx_errors += ring->rx_stats.csum_err; + } + } +} + +static void rnpvf_reset_pf_request(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + + if (!(adapter->flags & RNPVF_FLAG_PF_RESET_REQ)) + return; + + adapter->flags &= (~RNPVF_FLAG_PF_RESET_REQ); + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + hw->mac.ops.req_reset_pf(hw); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); +} + +static int rnpvf_reset_subtask(struct rnpvf_adapter *adapter) +{ + if (!(adapter->flags & RNPVF_FLAG_PF_RESET)) + return 0; + /* If we're already down or resetting, just bail */ + if (test_bit(__RNPVF_DOWN, &adapter->state) || + test_bit(__RNPVF_RESETTING, &adapter->state)) + return 0; + + adapter->tx_timeout_count++; + + rtnl_lock(); + rnpvf_reinit_locked(adapter); + rtnl_unlock(); + + adapter->flags &= (~RNPVF_FLAG_PF_RESET); + + return 1; +} + +/** + * rnpvf_watchdog - Timer Call-back + * @data: pointer to adapter cast into an unsigned long + **/ +static void rnpvf_watchdog(struct timer_list *t) +{ + struct rnpvf_adapter *adapter = + from_timer(adapter, t, watchdog_timer); + + /* + * Do the watchdog outside of interrupt context due to the lovely + * delays that some of the newer hardware requires + */ + + if (test_bit(__RNPVF_DOWN, &adapter->state)) + goto watchdog_short_circuit; + +watchdog_short_circuit: + if (!test_bit(__RNPVF_REMOVE, &adapter->state)) + schedule_work(&adapter->watchdog_task); +} + +static void rnpvf_check_hang_subtask(struct rnpvf_adapter *adapter) +{ + int i; + struct rnpvf_ring *tx_ring; + u64 tx_next_to_clean_old; + u64 tx_next_to_clean; + u64 tx_next_to_use; + struct rnpvf_ring *rx_ring; + u64 rx_next_to_clean_old; + u64 rx_next_to_clean; + union rnp_rx_desc *rx_desc; + + /* If we're down or resetting, just bail */ + if (test_bit(__RNPVF_DOWN, &adapter->state) || + test_bit(__RNPVF_RESETTING, &adapter->state)) + return; + + for (i = 0; i < adapter->num_tx_queues; i++) { + tx_ring = adapter->tx_ring[i]; + /* get the last next_to_clean */ + tx_next_to_clean_old = tx_ring->tx_stats.tx_next_to_clean; + tx_next_to_clean = tx_ring->next_to_clean; + tx_next_to_use = tx_ring->next_to_use; + + /* if we have tx desc to clean */ + if (tx_next_to_use != tx_next_to_clean) { + if (tx_next_to_clean == tx_next_to_clean_old) { + tx_ring->tx_stats.tx_equal_count++; + if (tx_ring->tx_stats.tx_equal_count > 2) { + /* maybe not so good */ + struct rnpvf_q_vector *q_vector = + tx_ring->q_vector; + + /* stats */ + if (q_vector->rx.ring || + q_vector->tx.ring) + napi_schedule_irqoff( + &q_vector->napi); + + tx_ring->tx_stats.tx_irq_miss++; + tx_ring->tx_stats.tx_equal_count = + 0; + } + } else { + tx_ring->tx_stats.tx_equal_count = 0; + } + /* update */ + /* record this next_to_clean */ + tx_ring->tx_stats.tx_next_to_clean = + tx_next_to_clean; + } else { + /* clean record to -1 */ + tx_ring->tx_stats.tx_next_to_clean = -1; + } + } + /* check if we lost rx irq */ + for (i = 0; i < adapter->num_rx_queues; i++) { + rx_ring = adapter->rx_ring[i]; + /* get the last next_to_clean */ + rx_next_to_clean_old = rx_ring->rx_stats.rx_next_to_clean; + /* get the now clean */ + rx_next_to_clean = rx_ring->next_to_clean; + + if (rx_next_to_clean == rx_next_to_clean_old) { + rx_ring->rx_stats.rx_equal_count++; + + if ((rx_ring->rx_stats.rx_equal_count > 2) && + (rx_ring->rx_stats.rx_equal_count < 5)) { + rx_desc = RNPVF_RX_DESC( + rx_ring, rx_ring->next_to_clean); + if (rnpvf_test_staterr(rx_desc, + RNP_RXD_STAT_DD)) { + struct rnpvf_q_vector *q_vector = + rx_ring->q_vector; + unsigned int size; + + size = le16_to_cpu( + rx_desc->wb.len) - + le16_to_cpu( + rx_desc->wb + .padding_len); + if (size) { + rx_ring->rx_stats + .rx_irq_miss++; + if (q_vector->rx.ring || + q_vector->tx.ring) + napi_schedule_irqoff( + &q_vector->napi); + } + } + } + if (rx_ring->rx_stats.rx_equal_count > 1000) + rx_ring->rx_stats.rx_equal_count = 0; + } else { + rx_ring->rx_stats.rx_equal_count = 0; + } + /* update new clean */ + rx_ring->rx_stats.rx_next_to_clean = rx_next_to_clean; + } +} + +/** + * rnpvf_watchdog_task - worker thread to bring link up + * @work: pointer to work_struct containing our data + **/ +static void rnpvf_watchdog_task(struct work_struct *work) +{ + struct rnpvf_adapter *adapter = + container_of(work, struct rnpvf_adapter, watchdog_task); + struct net_device *netdev = adapter->netdev; + struct rnpvf_hw *hw = &adapter->hw; + u32 link_speed = adapter->link_speed; + bool link_up = adapter->link_up; + s32 need_reset; + + adapter->flags |= RNPVF_FLAG_IN_WATCHDOG_TASK; + + rnpvf_reset_pf_request(adapter); + + if (rnpvf_reset_subtask(adapter)) { + adapter->flags &= ~RNPVF_FLAG_PF_UPDATE_MTU; + adapter->flags &= ~RNPVF_FLAG_PF_UPDATE_VLAN; + goto pf_has_reset; + } + + need_reset = + hw->mac.ops.check_link(hw, &link_speed, &link_up, false); + + if (need_reset) { + adapter->link_up = link_up; + adapter->link_speed = link_speed; + netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); + schedule_work(&adapter->reset_task); + goto pf_has_reset; + } + adapter->link_up = link_up; + adapter->link_speed = link_speed; + + /* if we ready down */ + if (test_bit(__RNPVF_DOWN, &adapter->state)) { + if (test_bit(__RNPVF_LINK_DOWN, &adapter->state)) { + clear_bit(__RNPVF_LINK_DOWN, &adapter->state); + dev_info(&adapter->pdev->dev, + "NIC Link is Down\n"); + } + goto skip_link_check; + } + + if (link_up) { + if (!netif_carrier_ok(netdev)) { + char *link_speed_string; + switch (link_speed) { + case RNP_LINK_SPEED_40GB_FULL: + link_speed_string = "40 Gbps"; + break; + case RNP_LINK_SPEED_25GB_FULL: + link_speed_string = "25 Gbps"; + break; + case RNP_LINK_SPEED_10GB_FULL: + link_speed_string = "10 Gbps"; + break; + case RNP_LINK_SPEED_1GB_FULL: + link_speed_string = "1 Gbps"; + break; + case RNP_LINK_SPEED_100_FULL: + link_speed_string = "100 Mbps"; + break; + default: + link_speed_string = "unknown speed"; + break; + } + dev_info(&adapter->pdev->dev, + "NIC Link is Up, %s\n", + link_speed_string); + netif_carrier_on(netdev); + netif_tx_wake_all_queues(netdev); + } + } else { + adapter->link_up = false; + adapter->link_speed = 0; + if (netif_carrier_ok(netdev)) { + dev_info(&adapter->pdev->dev, + "NIC Link is Down\n"); + netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); + } + } +skip_link_check: + if (adapter->flags & RNPVF_FLAG_PF_UPDATE_MTU) { + adapter->flags &= ~RNPVF_FLAG_PF_UPDATE_MTU; + if (netdev->mtu > hw->mtu) { + netdev->mtu = hw->mtu; + rtnl_lock(); + call_netdevice_notifiers(NETDEV_CHANGEMTU, + adapter->netdev); + rtnl_unlock(); + } + } + if (adapter->flags & RNPVF_FLAG_PF_UPDATE_VLAN) { + adapter->flags &= ~RNPVF_FLAG_PF_UPDATE_VLAN; + rnpvf_set_rx_mode(adapter->netdev); + } + + rnpvf_check_hang_subtask(adapter); + rnpvf_update_stats(adapter); + +pf_has_reset: + /* Reset the timer */ + mod_timer(&adapter->watchdog_timer, + round_jiffies(jiffies + (2 * HZ))); + + adapter->flags &= ~RNPVF_FLAG_IN_WATCHDOG_TASK; +} + +/** + * rnpvf_free_tx_resources - Free Tx Resources per Queue + * @adapter: board private structure + * @tx_ring: Tx descriptor ring for a specific queue + * + * Free all transmit software resources + **/ +void rnpvf_free_tx_resources(struct rnpvf_adapter *adapter, + struct rnpvf_ring *tx_ring) +{ + BUG_ON(tx_ring == NULL); + + rnpvf_clean_tx_ring(adapter, tx_ring); + + vfree(tx_ring->tx_buffer_info); + tx_ring->tx_buffer_info = NULL; + + /* if not set, then don't free */ + if (!tx_ring->desc) + return; + + dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, + tx_ring->dma); + + tx_ring->desc = NULL; +} + +/** + * rnpvf_free_all_tx_resources - Free Tx Resources for All Queues + * @adapter: board private structure + * + * Free all transmit software resources + **/ +static void rnpvf_free_all_tx_resources(struct rnpvf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) + rnpvf_free_tx_resources(adapter, adapter->tx_ring[i]); +} + +/** + * rnpvf_setup_tx_resources - allocate Tx resources (Descriptors) + * @adapter: board private structure + * @tx_ring: tx descriptor ring (for a specific queue) to setup + * + * Return 0 on success, negative on failure + **/ +int rnpvf_setup_tx_resources(struct rnpvf_adapter *adapter, + struct rnpvf_ring *tx_ring) +{ + struct device *dev = tx_ring->dev; + int orig_node = dev_to_node(dev); + int numa_node = NUMA_NO_NODE; + int size; + + size = sizeof(struct rnpvf_tx_buffer) * tx_ring->count; + + if (tx_ring->q_vector) + numa_node = tx_ring->q_vector->numa_node; + + dbg("%s size:%d count:%d\n", __func__, size, tx_ring->count); + tx_ring->tx_buffer_info = vzalloc_node(size, numa_node); + if (!tx_ring->tx_buffer_info) + tx_ring->tx_buffer_info = vzalloc(size); + if (!tx_ring->tx_buffer_info) + goto err_buffer; + + /* round up to nearest 4K */ + tx_ring->size = tx_ring->count * sizeof(struct rnp_tx_desc); + tx_ring->size = ALIGN(tx_ring->size, 4096); + + set_dev_node(dev, numa_node); + tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, + &tx_ring->dma, GFP_KERNEL); + set_dev_node(dev, orig_node); + if (!tx_ring->desc) + tx_ring->desc = dma_alloc_coherent( + dev, tx_ring->size, &tx_ring->dma, GFP_KERNEL); + if (!tx_ring->desc) + goto err; + memset(tx_ring->desc, 0, tx_ring->size); + + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; + + DPRINTK(IFUP, INFO, + "%d TxRing:%d, vector:%d ItemCounts:%d " + "desc:%p(0x%llx) node:%d\n", + tx_ring->queue_index, tx_ring->rnpvf_queue_idx, + tx_ring->q_vector->v_idx, tx_ring->count, tx_ring->desc, + tx_ring->dma, numa_node); + return 0; + +err: + rnpvf_err( + "%s [SetupTxResources] ERROR: #%d TxRing:%d, vector:%d ItemCounts:%d\n", + tx_ring->netdev->name, tx_ring->queue_index, + tx_ring->rnpvf_queue_idx, tx_ring->q_vector->v_idx, + tx_ring->count); + vfree(tx_ring->tx_buffer_info); +err_buffer: + tx_ring->tx_buffer_info = NULL; + dev_err(dev, + "Unable to allocate memory for the Tx descriptor ring\n"); + return -ENOMEM; +} + +/** + * rnpvf_setup_all_tx_resources - allocate all queues Tx resources + * @adapter: board private structure + * + * If this function returns with an error, then it's possible one or + * more of the rings is populated (while the rest are not). It is the + * callers duty to clean those orphaned rings. + * + * Return 0 on success, negative on failure + **/ +static int rnpvf_setup_all_tx_resources(struct rnpvf_adapter *adapter) +{ + int i, err = 0; + + dbg("adapter->num_tx_queues:%d, adapter->tx_ring[0]:%p\n", + adapter->num_tx_queues, adapter->tx_ring[0]); + + for (i = 0; i < adapter->num_tx_queues; i++) { + BUG_ON(adapter->tx_ring[i] == NULL); + err = rnpvf_setup_tx_resources(adapter, + adapter->tx_ring[i]); + if (!err) + continue; + hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", + i); + goto err_setup_tx; + } + + return 0; + +err_setup_tx: + /* rewind the index freeing the rings as we go */ + while (i--) + rnpvf_free_tx_resources(adapter, adapter->tx_ring[i]); + return err; +} + +/** + * rnpvf_setup_rx_resources - allocate Rx resources (Descriptors) + * @adapter: board private structure + * @rx_ring: rx descriptor ring (for a specific queue) to setup + * + * Returns 0 on success, negative on failure + **/ +int rnpvf_setup_rx_resources(struct rnpvf_adapter *adapter, + struct rnpvf_ring *rx_ring) +{ + struct device *dev = rx_ring->dev; + int orig_node = dev_to_node(dev); + int numa_node = -1; + int size; + + BUG_ON(rx_ring == NULL); + + size = sizeof(struct rnpvf_rx_buffer) * rx_ring->count; + + if (rx_ring->q_vector) + numa_node = rx_ring->q_vector->numa_node; + + rx_ring->rx_buffer_info = vzalloc_node(size, numa_node); + if (!rx_ring->rx_buffer_info) + rx_ring->rx_buffer_info = vzalloc(size); + if (!rx_ring->rx_buffer_info) + goto alloc_buffer; + + /* Round up to nearest 4K */ + rx_ring->size = rx_ring->count * sizeof(union rnp_rx_desc); + rx_ring->size = ALIGN(rx_ring->size, 4096); + + set_dev_node(dev, numa_node); + rx_ring->desc = dma_alloc_coherent(&adapter->pdev->dev, + rx_ring->size, &rx_ring->dma, + GFP_KERNEL); + set_dev_node(dev, orig_node); + if (!rx_ring->desc) { + vfree(rx_ring->rx_buffer_info); + rx_ring->rx_buffer_info = NULL; + goto alloc_failed; + } + + memset(rx_ring->desc, 0, rx_ring->size); + rx_ring->next_to_clean = 0; + rx_ring->next_to_use = 0; + + DPRINTK(IFUP, INFO, + "%d RxRing:%d, vector:%d ItemCounts:%d " + "desc:%p(0x%llx) node:%d\n", + rx_ring->queue_index, rx_ring->rnpvf_queue_idx, + rx_ring->q_vector->v_idx, rx_ring->count, rx_ring->desc, + rx_ring->dma, numa_node); + + return 0; +alloc_failed: + rnpvf_err( + "%s [SetupTxResources] ERROR: #%d RxRing:%d, vector:%d ItemCounts:%d\n", + rx_ring->netdev->name, rx_ring->queue_index, + rx_ring->rnpvf_queue_idx, rx_ring->q_vector->v_idx, + rx_ring->count); + vfree(rx_ring->tx_buffer_info); +alloc_buffer: + rx_ring->tx_buffer_info = NULL; + dev_err(dev, + "Unable to allocate memory for the Rx descriptor ring\n"); + + return -ENOMEM; +} + +/** + * rnpvf_setup_all_rx_resources - allocate all queues Rx resources + * @adapter: board private structure + * + * If this function returns with an error, then it's possible one or + * more of the rings is populated (while the rest are not). It is the + * callers duty to clean those orphaned rings. + * + * Return 0 on success, negative on failure + **/ +static int rnpvf_setup_all_rx_resources(struct rnpvf_adapter *adapter) +{ + int i, err = 0; + + for (i = 0; i < adapter->num_rx_queues; i++) { + BUG_ON(adapter->rx_ring[i] == NULL); + + err = rnpvf_setup_rx_resources(adapter, + adapter->rx_ring[i]); + if (!err) + continue; + hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", + i); + goto err_setup_rx; + } + + return 0; + +err_setup_rx: + /* rewind the index freeing the rings as we go */ + while (i--) + rnpvf_free_rx_resources(adapter, adapter->rx_ring[i]); + return err; +} + +/** + * rnpvf_free_rx_resources - Free Rx Resources + * @adapter: board private structure + * @rx_ring: ring to clean the resources from + * + * Free all receive software resources + **/ +void rnpvf_free_rx_resources(struct rnpvf_adapter *adapter, + struct rnpvf_ring *rx_ring) +{ + struct pci_dev *pdev = adapter->pdev; + + rnpvf_clean_rx_ring(rx_ring); + + vfree(rx_ring->rx_buffer_info); + rx_ring->rx_buffer_info = NULL; + + /* if not set, then don't free */ + if (!rx_ring->desc) + return; + + dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, + rx_ring->dma); + + rx_ring->desc = NULL; +} + +/** + * rnpvf_free_all_rx_resources - Free Rx Resources for All Queues + * @adapter: board private structure + * + * Free all receive software resources + **/ +static void rnpvf_free_all_rx_resources(struct rnpvf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_rx_queues; i++) + rnpvf_free_rx_resources(adapter, adapter->rx_ring[i]); +} + +/** + * rnpvf_change_mtu - Change the Maximum Transfer Unit + * @netdev: network interface device structure + * @new_mtu: new value for maximum frame size + * + * Returns 0 on success, negative on failure + **/ +static int rnpvf_change_mtu(struct net_device *netdev, int new_mtu) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + + if (new_mtu > hw->mtu) { + dev_info(&adapter->pdev->dev, + "PF limit vf mtu setup too large %d \n", hw->mtu); + return -EINVAL; + + } else { + hw_dbg(&adapter->hw, "changing MTU from %d to %d\n", + netdev->mtu, new_mtu); + /* must set new MTU before calling down or up */ + netdev->mtu = new_mtu; + } + + if (netif_running(netdev)) + rnpvf_reinit_locked(adapter); + + return 0; +} + +/** + * rnpvf_open - Called when a network interface is made active + * @netdev: network interface device structure + * + * Returns 0 on success, negative value on failure + * + * The open entry point is called when a network interface is made + * active by the system (IFF_UP). At this point all resources needed + * for transmit and receive operations are allocated, the interrupt + * handler is registered with the OS, the watchdog timer is started, + * and the stack is notified that the interface is ready. + **/ +int rnpvf_open(struct net_device *netdev) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + int err; + + DPRINTK(IFUP, INFO, "ifup\n"); + + /* A previous failure to open the device because of a lack of + * available MSIX vector resources may have reset the number + * of msix vectors variable to zero. The only way to recover + * is to unload/reload the driver and hope that the system has + * been able to recover some MSIX vector resources. + */ + if (!adapter->num_msix_vectors) + return -ENOMEM; + + /* disallow open during test */ + if (test_bit(__RNPVF_TESTING, &adapter->state)) + return -EBUSY; + + if (hw->adapter_stopped) { + rnpvf_reset(adapter); + /* if adapter is still stopped then PF isn't up and + * the vf can't start. */ + if (hw->adapter_stopped) { + err = RNP_ERR_MBX; + dev_err(&hw->pdev->dev, + "%s(%s):error: Unable to start - perhaps the PF Driver isn't " + "up yet\n", + adapter->name, netdev->name); + goto err_setup_reset; + } + } + + netif_carrier_off(netdev); + + /* allocate transmit descriptors */ + err = rnpvf_setup_all_tx_resources(adapter); + if (err) + goto err_setup_tx; + + /* allocate receive descriptors */ + err = rnpvf_setup_all_rx_resources(adapter); + if (err) + goto err_setup_rx; + + rnpvf_configure(adapter); + + /* clear any pending interrupts, may auto mask */ + err = rnpvf_request_irq(adapter); + if (err) + goto err_req_irq; + + /* Notify the stack of the actual queue counts. */ + err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); + if (err) + goto err_set_queues; + + err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); + if (err) + goto err_set_queues; + + rnpvf_up_complete(adapter); + + return 0; + +err_set_queues: + rnpvf_free_irq(adapter); +err_req_irq: + +err_setup_rx: + rnpvf_free_all_rx_resources(adapter); +err_setup_tx: + rnpvf_free_all_tx_resources(adapter); + +err_setup_reset: + + return err; +} + +/** + * rnpvf_close - Disables a network interface + * @netdev: network interface device structure + * + * Returns 0, this is not allowed to fail + * + * The close entry point is called when an interface is de-activated + * by the OS. The hardware is still under the drivers control, but + * needs to be disabled. A global MAC reset is issued to stop the + * hardware, and all transmit and receive resources are freed. + **/ +int rnpvf_close(struct net_device *netdev) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + DPRINTK(IFDOWN, INFO, "ifdown\n"); + + rnpvf_down(adapter); + rnpvf_free_irq(adapter); + + rnpvf_free_all_tx_resources(adapter); + rnpvf_free_all_rx_resources(adapter); + + return 0; +} + +void rnpvf_tx_ctxtdesc(struct rnpvf_ring *tx_ring, u16 mss_seg_len, + u8 l4_hdr_len, u8 tunnel_hdr_len, int ignore_vlan, + u32 type_tucmd, bool crc_pad) +{ + struct rnp_tx_ctx_desc *context_desc; + u16 i = tx_ring->next_to_use; + struct rnpvf_adapter *adapter = RING2ADAPT(tx_ring); + struct rnpvf_hw *hw = &adapter->hw; + struct rnp_mbx_info *mbx = &hw->mbx; + u8 vfnum = VFNUM(mbx, hw->vfnum); + + context_desc = RNPVF_TX_CTXTDESC(tx_ring, i); + + i++; + tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; + + /* set bits to identify this as an advanced context descriptor */ + type_tucmd |= RNP_TXD_CTX_CTRL_DESC; + + if (adapter->priv_flags & RNPVF_PRIV_FLAG_TX_PADDING) { + if (!crc_pad) + type_tucmd |= + RNP_TXD_MTI_CRC_PAD_CTRL; + } + + context_desc->mss_len = cpu_to_le16(mss_seg_len); + context_desc->vfnum = 0x80 | vfnum; + context_desc->l4_hdr_len = l4_hdr_len; + + if (ignore_vlan) + context_desc->vf_veb_flags |= VF_IGNORE_VLAN; + + context_desc->tunnel_hdr_len = tunnel_hdr_len; + context_desc->resv_cmd = cpu_to_le32(type_tucmd); + context_desc->res = 0; + buf_dump_line("ctx ", __LINE__, context_desc, + sizeof(*context_desc)); +} + +static int rnpvf_tso(struct rnpvf_ring *tx_ring, + struct rnpvf_tx_buffer *first, u8 *hdr_len) +{ + struct sk_buff *skb = first->skb; + struct net_device *netdev = tx_ring->netdev; + struct rnpvf_adapter *adapter = netdev_priv(netdev); + union { + struct iphdr *v4; + struct ipv6hdr *v6; + unsigned char *hdr; + } ip; + union { + struct tcphdr *tcp; + struct udphdr *udp; + unsigned char *hdr; + } l4; + u32 paylen, l4_offset; + int err; + u8 *inner_mac; + u16 gso_segs, gso_size; + u16 gso_need_pad; + + if (skb->ip_summed != CHECKSUM_PARTIAL) + return 0; + + if (!skb_is_gso(skb)) + return 0; + + err = skb_cow_head(skb, 0); + if (err < 0) + return err; + + inner_mac = skb->data; + ip.hdr = skb_network_header(skb); + l4.hdr = skb_transport_header(skb); + + /* initialize outer IP header fields */ + if (ip.v4->version == 4) { + /* IP header will have to cancel out any data that + * is not a part of the outer IP header + */ + ip.v4->check = 0x0000; + } else { + ip.v6->payload_len = 0; + } + if (skb_shinfo(skb)->gso_type & + (SKB_GSO_GRE | + SKB_GSO_GRE_CSUM | + SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM)) { + if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && + (skb_shinfo(skb)->gso_type & + SKB_GSO_UDP_TUNNEL_CSUM)) { + } + inner_mac = skb_inner_mac_header(skb); + first->tunnel_hdr_len = inner_mac - skb->data; + + if (skb_shinfo(skb)->gso_type & + (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM)) { + first->cmd_flags |= RNP_TXD_TUNNEL_VXLAN; + l4.udp->check = 0; + } else { + first->cmd_flags |= RNP_TXD_TUNNEL_NVGRE; + } + dbg("set outer l4.udp to 0\n"); + + /* reset pointers to inner headers */ + ip.hdr = skb_inner_network_header(skb); + l4.hdr = skb_inner_transport_header(skb); + } + if (ip.v4->version == 4) { + /* IP header will have to cancel out any data that + * is not a part of the outer IP header + */ + ip.v4->check = 0x0000; + + } else { + ip.v6->payload_len = 0; + /* set ipv6 type */ + first->cmd_flags |= (RNP_TXD_FLAG_IPv6); + } + + /* determine offset of inner transport header */ + l4_offset = l4.hdr - skb->data; + + paylen = skb->len - l4_offset; + dbg("before l4 checksum is %x\n", l4.tcp->check); + + if (skb->csum_offset == offsetof(struct tcphdr, check)) { + dbg("tcp before l4 checksum is %x\n", l4.tcp->check); + first->cmd_flags |= RNP_TXD_L4_TYPE_TCP; + /* compute length of segmentation header */ + *hdr_len = (l4.tcp->doff * 4) + l4_offset; + csum_replace_by_diff(&l4.tcp->check, + (__force __wsum)htonl(paylen)); + dbg("tcp l4 checksum is %x\n", l4.tcp->check); + l4.tcp->psh = 0; + } else { + dbg("paylen is %x\n", paylen); + first->cmd_flags |= RNP_TXD_L4_TYPE_UDP; + /* compute length of segmentation header */ + dbg("udp before l4 checksum is %x\n", l4.udp->check); + *hdr_len = sizeof(*l4.udp) + l4_offset; + csum_replace_by_diff(&l4.udp->check, + (__force __wsum)htonl(paylen)); + dbg("udp l4 checksum is %x\n", l4.udp->check); + } + + dbg("l4 checksum is %x\n", l4.tcp->check); + + first->mac_ip_len = l4.hdr - ip.hdr; + first->mac_ip_len |= (ip.hdr - inner_mac) << 9; + + /* compute header lengths */ + /* pull values out of skb_shinfo */ + gso_size = skb_shinfo(skb)->gso_size; + gso_segs = skb_shinfo(skb)->gso_segs; + + /* if we close padding check gso confition */ + if (adapter->priv_flags & RNPVF_PRIV_FLAG_TX_PADDING) { + gso_need_pad = (first->skb->len - *hdr_len) % gso_size; + if (gso_need_pad) { + if ((gso_need_pad + *hdr_len) <= 60) { + gso_need_pad = + 60 - (gso_need_pad + *hdr_len); + first->gso_need_padding = !!gso_need_pad; + } + } + } + + /* update gso size and bytecount with header size */ + /* to fix tx status */ + first->gso_segs = gso_segs; + first->bytecount += (first->gso_segs - 1) * *hdr_len; + first->mss_len_vf_num |= (gso_size | ((l4.tcp->doff * 4) << 24)); + first->cmd_flags |= RNP_TXD_FLAG_TSO | RNP_TXD_IP_CSUM | + RNP_TXD_L4_CSUM; + first->ctx_flag = true; + return 1; +} + +static int rnpvf_tx_csum(struct rnpvf_ring *tx_ring, + struct rnpvf_tx_buffer *first) +{ + struct sk_buff *skb = first->skb; + u8 l4_proto = 0; + u8 ip_len = 0; + u8 mac_len = 0; + u8 *inner_mac = skb->data; + u8 *exthdr; + __be16 frag_off; + union { + struct iphdr *v4; + struct ipv6hdr *v6; + unsigned char *hdr; + } ip; + union { + struct tcphdr *tcp; + struct udphdr *udp; + unsigned char *hdr; + } l4; + + if (skb->ip_summed != CHECKSUM_PARTIAL) { + return 0; + } + + ip.hdr = skb_network_header(skb); + l4.hdr = skb_transport_header(skb); + + inner_mac = skb->data; + + /* outer protocol */ + if (skb->encapsulation) { + /* define outer network header type */ + if (ip.v4->version == 4) { + l4_proto = ip.v4->protocol; + } else { + exthdr = ip.hdr + sizeof(*ip.v6); + l4_proto = ip.v6->nexthdr; + if (l4.hdr != exthdr) + ipv6_skip_exthdr(skb, exthdr - skb->data, + &l4_proto, &frag_off); + } + + /* define outer transport */ + switch (l4_proto) { + case IPPROTO_UDP: + l4.udp->check = 0; + first->cmd_flags |= RNP_TXD_TUNNEL_VXLAN; + + break; + case IPPROTO_GRE: + + first->cmd_flags |= RNP_TXD_TUNNEL_NVGRE; + /* There was a long-standing issue in GRE where GSO + * was not setting the outer transport header unless + * a GRE checksum was requested. This was fixed in + * the 4.6 version of the kernel. In the 4.7 kernel + * support for GRE over IPv6 was added to GSO. So we + * can assume this workaround for all IPv4 headers + * without impacting later versions of the GRE. + */ + if (ip.v4->version == 4) + l4.hdr = ip.hdr + (ip.v4->ihl * 4); + break; + default: + skb_checksum_help(skb); + return -1; + } + + /* switch IP header pointer from outer to inner header */ + ip.hdr = skb_inner_network_header(skb); + l4.hdr = skb_inner_transport_header(skb); + + inner_mac = skb_inner_mac_header(skb); + first->tunnel_hdr_len = inner_mac - skb->data; + first->ctx_flag = true; + dbg("tunnel length is %d\n", first->tunnel_hdr_len); + } + + mac_len = (ip.hdr - inner_mac); + dbg("inner checksum needed %d", skb_checksum_start_offset(skb)); + dbg("skb->encapsulation %d\n", skb->encapsulation); + ip_len = (l4.hdr - ip.hdr); + if (ip.v4->version == 4) { + l4_proto = ip.v4->protocol; + } else { + exthdr = ip.hdr + sizeof(*ip.v6); + l4_proto = ip.v6->nexthdr; + if (l4.hdr != exthdr) + ipv6_skip_exthdr(skb, exthdr - skb->data, + &l4_proto, &frag_off); + first->cmd_flags |= RNP_TXD_FLAG_IPv6; + } + /* Enable L4 checksum offloads */ + switch (l4_proto) { + case IPPROTO_TCP: + first->cmd_flags |= RNP_TXD_L4_TYPE_TCP | RNP_TXD_L4_CSUM; + break; + case IPPROTO_SCTP: + first->cmd_flags |= RNP_TXD_L4_TYPE_SCTP | RNP_TXD_L4_CSUM; + break; + case IPPROTO_UDP: + first->cmd_flags |= RNP_TXD_L4_TYPE_UDP | RNP_TXD_L4_CSUM; + break; + default: + skb_checksum_help(skb); + return 0; + } + if ((tx_ring->ring_flags & RNPVF_RING_NO_TUNNEL_SUPPORT) && + (first->ctx_flag)) { + /* if not support tunnel */ + first->cmd_flags &= (~RNP_TXD_TUNNEL_MASK); + mac_len += first->tunnel_hdr_len; + first->tunnel_hdr_len = 0; + first->ctx_flag = false; + } + + dbg("mac length is %d\n", mac_len); + dbg("ip length is %d\n", ip_len); + first->mac_ip_len = (mac_len << 9) | ip_len; + return 0; +} + +static void rnpvf_tx_map(struct rnpvf_ring *tx_ring, + struct rnpvf_tx_buffer *first, const u8 hdr_len) +{ + struct sk_buff *skb = first->skb; + struct rnpvf_tx_buffer *tx_buffer; + struct rnp_tx_desc *tx_desc; + skb_frag_t *frag; + dma_addr_t dma; + unsigned int data_len, size; + u16 vlan = first->vlan; + u16 cmd = first->cmd_flags; + u16 i = tx_ring->next_to_use; + u64 fun_id = ((u64)(tx_ring->vfnum) << (32 + 24)); + + tx_desc = RNPVF_TX_DESC(tx_ring, i); + tx_desc->blen = cpu_to_le16(skb->len - hdr_len); /* maybe no-use */ + tx_desc->vlan = cpu_to_le16(vlan); + tx_desc->cmd = cpu_to_le16(cmd); + tx_desc->mac_ip_len = first->mac_ip_len; + + size = skb_headlen(skb); + data_len = skb->data_len; + + dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); + + tx_buffer = first; + + for (frag = &skb_shinfo(skb)->frags[0];; frag++) { + if (dma_mapping_error(tx_ring->dev, dma)) + goto dma_error; + + /* record length, and DMA address */ + dma_unmap_len_set(tx_buffer, len, size); + dma_unmap_addr_set(tx_buffer, dma, dma); + + /* 1st desc */ + tx_desc->pkt_addr = cpu_to_le64(dma | fun_id); + + while (unlikely(size > RNPVF_MAX_DATA_PER_TXD)) { + tx_desc->cmd = cpu_to_le16(cmd); + tx_desc->blen = + cpu_to_le16(RNPVF_MAX_DATA_PER_TXD); + buf_dump_line("tx0 ", __LINE__, tx_desc, + sizeof(*tx_desc)); + i++; + tx_desc++; + if (i == tx_ring->count) { + tx_desc = RNPVF_TX_DESC(tx_ring, 0); + i = 0; + } + + dma += RNPVF_MAX_DATA_PER_TXD; + size -= RNPVF_MAX_DATA_PER_TXD; + + tx_desc->pkt_addr = cpu_to_le64(dma | fun_id); + } + + buf_dump_line("tx1 ", __LINE__, tx_desc, + sizeof(*tx_desc)); + if (likely(!data_len)) + break; + tx_desc->cmd = cpu_to_le16(cmd); + tx_desc->blen = cpu_to_le16(size); + buf_dump_line("tx2 ", __LINE__, tx_desc, + sizeof(*tx_desc)); + + /* ==== frag== */ + i++; + tx_desc++; + if (i == tx_ring->count) { + tx_desc = RNPVF_TX_DESC(tx_ring, 0); + i = 0; + } + tx_desc->cmd = RNP_TXD_CMD_RS; + tx_desc->mac_ip_len = 0; + + size = skb_frag_size(frag); + + data_len -= size; + + dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, + DMA_TO_DEVICE); + + tx_buffer = &tx_ring->tx_buffer_info[i]; + } + + /* write last descriptor with RS and EOP bits */ + tx_desc->cmd = cpu_to_le16(cmd | RNP_TXD_CMD_EOP | RNP_TXD_CMD_RS); + tx_desc->blen = cpu_to_le16(size); + buf_dump_line("tx3 ", __LINE__, tx_desc, sizeof(*tx_desc)); + netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); + + /* set the timestamp */ + first->time_stamp = jiffies; + + /* + * Force memory writes to complete before letting h/w know there + * are new descriptors to fetch. (Only applicable for weak-ordered + * memory model archs, such as IA-64). + * + * We also need this memory barrier to make certain all of the + * status bits have been updated before next_to_watch is written. + */ + wmb(); + + /* set next_to_watch value indicating a packet is present */ + first->next_to_watch = tx_desc; + + buf_dump_line("tx4 ", __LINE__, tx_desc, sizeof(*tx_desc)); + i++; + if (i == tx_ring->count) + i = 0; + + tx_ring->next_to_use = i; + + /* notify HW of packet */ + rnpvf_wr_reg(tx_ring->tail, i); + + return; +dma_error: + dev_err(tx_ring->dev, "TX DMA map failed\n"); + + /* clear dma mappings for failed tx_buffer_info map */ + for (;;) { + tx_buffer = &tx_ring->tx_buffer_info[i]; + rnpvf_unmap_and_free_tx_resource(tx_ring, tx_buffer); + if (tx_buffer == first) + break; + if (i == 0) + i = tx_ring->count; + i--; + } + + tx_ring->next_to_use = i; +} + +static int __rnpvf_maybe_stop_tx(struct rnpvf_ring *tx_ring, int size) +{ + struct rnpvf_adapter *adapter = netdev_priv(tx_ring->netdev); + + dbg("stop subqueue\n"); + netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); + /* Herbert's original patch had: + * smp_mb__after_netif_stop_queue(); + * but since that doesn't exist yet, just open code it. */ + smp_mb(); + + /* We need to check again in a case another CPU has just + * made room available. */ + if (likely(rnpvf_desc_unused(tx_ring) < size)) + return -EBUSY; + + /* A reprieve! - use start_queue because it doesn't call schedule */ + netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); + ++adapter->restart_queue; + return 0; +} + +void rnpvf_maybe_tx_ctxtdesc(struct rnpvf_ring *tx_ring, + struct rnpvf_tx_buffer *first, + int ignore_vlan, u32 type_tucmd) +{ + if (first->ctx_flag) { + rnpvf_tx_ctxtdesc(tx_ring, first->mss_len, + first->l4_hdr_len, first->tunnel_hdr_len, + ignore_vlan, type_tucmd, + first->gso_need_padding); + } +} + +static int rnpvf_maybe_stop_tx(struct rnpvf_ring *tx_ring, int size) +{ + if (likely(RNPVF_DESC_UNUSED(tx_ring) >= size)) + return 0; + return __rnpvf_maybe_stop_tx(tx_ring, size); +} + +static int rnpvf_check_spoof_mac(struct sk_buff *skb, + struct net_device *netdev, + struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + struct rnpvf_hw_stats_own *hw_stats = &adapter->hw_stats; + int ret; + u8 *data = skb->data; + + /* if not in mac spoof, do nothing */ + if (!(hw->pf_feature & PF_MAC_SPOOF)) + return 0; + + if (0 == memcmp(data + netdev->addr_len, netdev->dev_addr, + netdev->addr_len)) { + ret = 0; + } else { + hw_stats->spoof_dropped++; + ret = 1; + } + + + return ret; +} + +static void rnpvf_force_src_mac(struct sk_buff *skb, + struct net_device *netdev) +{ + u8 *data = skb->data; + bool ret = false; + struct netdev_hw_addr *ha; + + /* force all src mac to myself */ + if (is_multicast_ether_addr(data)) { + if (0 == memcmp(data + netdev->addr_len, netdev->dev_addr, + netdev->addr_len)) { + ret = true; + goto DONE; + } + netdev_for_each_uc_addr(ha, netdev) { + if (0 == memcmp(data + netdev->addr_len, ha->addr, + netdev->addr_len)) { + ret = true; + goto DONE; + } + } + /* if not src mac, force to src mac */ + if (!ret) + memcpy(data + netdev->addr_len, netdev->dev_addr, + netdev->addr_len); + } +DONE: + return; +} + +netdev_tx_t rnpvf_xmit_frame_ring(struct sk_buff *skb, + struct rnpvf_adapter *adapter, + struct rnpvf_ring *tx_ring, + bool tx_padding) +{ + struct rnpvf_tx_buffer *first; + int tso; + u16 cmd = RNP_TXD_CMD_RS; + u16 vlan = 0; + unsigned short f; + u16 count = TXD_USE_COUNT(skb_headlen(skb)); + __be16 protocol = skb->protocol; + u8 hdr_len = 0; + int ignore_vlan = 0; + + dbg("=== begin ====\n"); + + rnpvf_skb_dump(skb, true); + + dbg("skb:%p, skb->len:%d headlen:%d, data_len:%d, tx_ring->next_to_use:%d " + "count:%d\n", + skb, skb->len, skb_headlen(skb), skb->data_len, + tx_ring->next_to_use, tx_ring->count); + /* + * need: 1 descriptor per page * PAGE_SIZE/RNPVF_MAX_DATA_PER_TXD, + * + 1 desc for skb_headlen/RNPVF_MAX_DATA_PER_TXD, + * + 2 desc gap to keep tail from touching head, + * + 1 desc for context descriptor, + * otherwise try next time + */ + for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) { + skb_frag_t *frag_temp = &skb_shinfo(skb)->frags[f]; + count += TXD_USE_COUNT(skb_frag_size(frag_temp)); + dbg(" #%d frag: size:%d\n", f, + skb_shinfo(skb)->frags[f].size); + } + + if (rnpvf_maybe_stop_tx(tx_ring, count + 3)) { + tx_ring->tx_stats.tx_busy++; + return NETDEV_TX_BUSY; + } + dbg("xx %p\n", tx_ring->tx_buffer_info); + + /* patch force send src mac to this netdev->mac */ + if (!(tx_ring->ring_flags & RNPVF_RING_VEB_MULTI_FIX)) + rnpvf_force_src_mac(skb, tx_ring->netdev); + if (rnpvf_check_spoof_mac(skb, tx_ring->netdev, adapter)) { + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + /* record the location of the first descriptor for this packet */ + first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + first->skb = skb; + first->bytecount = skb->len; + first->gso_segs = 1; + first->mss_len_vf_num = 0; + first->inner_vlan_tunnel_len = 0; + + if (adapter->priv_flags & RNPVF_PRIV_FLAG_TX_PADDING) { + first->ctx_flag = true; + first->gso_need_padding = tx_padding; + } + + /* if we have a HW VLAN tag being added default to the HW one */ + + if (adapter->flags & RNPVF_FLAG_PF_SET_VLAN) { + vlan |= adapter->vf_vlan; + cmd |= RNP_TXD_VLAN_VALID | RNP_TXD_VLAN_CTRL_INSERT_VLAN; + + } else { + if (skb_vlan_tag_present(skb)) { + if (skb->vlan_proto != htons(ETH_P_8021Q)) { + /* veb only use ctags */ + vlan |= skb_vlan_tag_get(skb); + cmd |= RNP_TXD_SVLAN_TYPE | + RNP_TXD_VLAN_CTRL_INSERT_VLAN; + } else { + vlan |= skb_vlan_tag_get(skb); + cmd |= RNP_TXD_VLAN_VALID | + RNP_TXD_VLAN_CTRL_INSERT_VLAN; + } + tx_ring->tx_stats.vlan_add++; + } else if (protocol == __constant_htons(ETH_P_8021Q)) { + struct vlan_hdr *vhdr, _vhdr; + vhdr = skb_header_pointer(skb, ETH_HLEN, + sizeof(_vhdr), &_vhdr); + if (!vhdr) + goto out_drop; + + protocol = vhdr->h_vlan_encapsulated_proto; + vlan = ntohs(vhdr->h_vlan_TCI); + cmd |= RNP_TXD_VLAN_VALID | RNP_TXD_VLAN_CTRL_NOP; + ignore_vlan = 1; + } + } + + /* record initial flags and protocol */ + first->cmd_flags = cmd; + first->vlan = vlan; + first->protocol = protocol; + /* default len should not 0 (hw request) */ + first->mac_ip_len = 20; + first->tunnel_hdr_len = 0; + + tso = rnpvf_tso(tx_ring, first, &hdr_len); + if (tso < 0) { + goto out_drop; + } else if (!tso) { + rnpvf_tx_csum(tx_ring, first); + } + /* vf should always send ctx with vf_num*/ + first->ctx_flag = true; + /* add control desc */ + rnpvf_maybe_tx_ctxtdesc(tx_ring, first, ignore_vlan, 0); + + rnpvf_tx_map(tx_ring, first, hdr_len); + + rnpvf_maybe_stop_tx(tx_ring, DESC_NEEDED); + + dbg("=== end ====\n\n\n\n"); + return NETDEV_TX_OK; + +out_drop: + dev_kfree_skb_any(first->skb); + first->skb = NULL; + + return NETDEV_TX_OK; +} + +static bool check_sctp_no_padding(struct sk_buff *skb) +{ + bool no_padding = false; + u8 l4_proto = 0; + u8 *exthdr; + __be16 frag_off; + union { + struct iphdr *v4; + struct ipv6hdr *v6; + unsigned char *hdr; + } ip; + union { + struct tcphdr *tcp; + struct udphdr *udp; + unsigned char *hdr; + } l4; + + ip.hdr = skb_network_header(skb); + l4.hdr = skb_transport_header(skb); + + if (ip.v4->version == 4) { + l4_proto = ip.v4->protocol; + } else { + exthdr = ip.hdr + sizeof(*ip.v6); + l4_proto = ip.v6->nexthdr; + if (l4.hdr != exthdr) + ipv6_skip_exthdr(skb, exthdr - skb->data, + &l4_proto, &frag_off); + } + switch (l4_proto) { + case IPPROTO_SCTP: + no_padding = true; + break; + default: + + break; + } + return no_padding; +} + +static int rnpvf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_ring *tx_ring; + bool tx_padding = false; + + if (!netif_carrier_ok(netdev)) { + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + if (adapter->priv_flags & RNPVF_PRIV_FLAG_TX_PADDING) { + if (skb->len < 60) { + if (!check_sctp_no_padding(skb)) { + if (skb_put_padto(skb, 60)) + return NETDEV_TX_OK; + + } else { + tx_padding = true; + } + } + + } else { + if (skb_put_padto(skb, 17)) + return NETDEV_TX_OK; + } + + tx_ring = adapter->tx_ring[skb->queue_mapping]; + dbg("xmi:queue_mapping:%d ring:%p\n", skb->queue_mapping, tx_ring); + return rnpvf_xmit_frame_ring(skb, adapter, tx_ring, tx_padding); +} + +/** + * rnpvf_set_mac - Change the Ethernet Address of the NIC + * @netdev: network interface device structure + * @p: pointer to an address structure + * + * Returns 0 on success, negative on failure + **/ +static int rnpvf_set_mac(struct net_device *netdev, void *p) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + struct rnpvf_hw *hw = &adapter->hw; + struct sockaddr *addr = p; + s32 ret_val; + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + ret_val = hw->mac.ops.set_rar(hw, 0, addr->sa_data, 0); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + if (0 != ret_val) { + /* set mac failed */ + dev_err(&adapter->pdev->dev, "pf not allowed reset mac\n"); + return -EADDRNOTAVAIL; + + } else { + eth_hw_addr_set(netdev, addr->sa_data); + memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); + rnpvf_configure_veb(adapter); + } + + return 0; +} + +void remove_mbx_irq(struct rnpvf_adapter *adapter) +{ + u32 msgbuf[2]; + struct rnpvf_hw *hw = &adapter->hw; + + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + msgbuf[0] = RNP_PF_REMOVE; + adapter->hw.mbx.ops.write_posted(hw, msgbuf, 1, false); + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); + mdelay(10); + + /* mbx */ + if (adapter->flags & RNPVF_FLAG_MSIX_ENABLED) { + adapter->hw.mbx.ops.configure( + &adapter->hw, adapter->msix_entries[0].entry, + false); + free_irq(adapter->msix_entries[0].vector, adapter); + } +} + +static void rnp_get_link_status(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + u32 msgbuf[3]; + s32 ret_val = -1; + + spin_lock_bh(&adapter->mbx_lock); + set_bit(__RNPVF_MBX_POLLING, &adapter->state); + msgbuf[0] = RNP_PF_GET_LINK; + adapter->hw.mbx.ops.write_posted(hw, msgbuf, 1, false); + mdelay(2); + ret_val = adapter->hw.mbx.ops.read_posted(hw, msgbuf, 2, false); + if (ret_val == 0) { + if (msgbuf[1] & RNP_PF_LINK_UP) { + hw->link = true; + hw->speed = msgbuf[1] & 0xffff; + + } else { + hw->link = false; + hw->speed = 0; + } + } else { + printk("[rpnvf] error! mbx GET_LINK failed!\n"); + } + clear_bit(__RNPVF_MBX_POLLING, &adapter->state); + spin_unlock_bh(&adapter->mbx_lock); +} + +int register_mbx_irq(struct rnpvf_adapter *adapter) +{ + struct rnpvf_hw *hw = &adapter->hw; + struct net_device *netdev = adapter->netdev; + int err = 0; + + /* for mbx:vector0 */ + if (adapter->flags & RNPVF_FLAG_MSIX_ENABLED) { + err = request_irq(adapter->msix_entries[0].vector, + rnpvf_msix_other, 0, netdev->name, + adapter); + if (err) { + dev_err(&adapter->pdev->dev, + "request_irq for msix_other failed: %d\n", + err); + goto err_mbx; + } + hw->mbx.ops.configure(hw, adapter->msix_entries[0].entry, + true); + } + + rnp_get_link_status(adapter); +err_mbx: + return err; +} + +static int rnpvf_suspend(struct pci_dev *pdev, pm_message_t state) +{ + struct rnpvf_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; +#ifdef CONFIG_PM + int retval = 0; +#endif + + netif_device_detach(netdev); + + if (netif_running(netdev)) { + rtnl_lock(); + rnpvf_down(adapter); + rnpvf_free_irq(adapter); + rnpvf_free_all_tx_resources(adapter); + rnpvf_free_all_rx_resources(adapter); + rtnl_unlock(); + } + + remove_mbx_irq(adapter); + rnpvf_clear_interrupt_scheme(adapter); + +#ifdef CONFIG_PM + retval = pci_save_state(pdev); + if (retval) + return retval; + +#endif + pci_disable_device(pdev); + + return 0; +} + +#ifdef CONFIG_PM +static int rnpvf_resume(struct pci_dev *pdev) +{ + struct rnpvf_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; + u32 err; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + /* + * pci_restore_state clears dev->state_saved so call + * pci_save_state to restore it. + */ + pci_save_state(pdev); + + err = pcim_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, + "Cannot enable PCI device from suspend\n"); + return err; + } + pci_set_master(pdev); + + rtnl_lock(); + err = rnpvf_init_interrupt_scheme(adapter); + rtnl_unlock(); + register_mbx_irq(adapter); + + if (err) { + dev_err(&pdev->dev, "Cannot initialize interrupts\n"); + return err; + } + + rnpvf_reset(adapter); + + if (netif_running(netdev)) { + err = rnpvf_open(netdev); + if (err) + return err; + } + + netif_device_attach(netdev); + + return err; +} + +#endif /* CONFIG_PM */ +static void rnpvf_shutdown(struct pci_dev *pdev) +{ + rnpvf_suspend(pdev, PMSG_SUSPEND); +} + +static void rnpvf_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats) +{ + struct rnpvf_adapter *adapter = netdev_priv(netdev); + int i; + u64 ring_csum_err = 0; + u64 ring_csum_good = 0; + + rcu_read_lock(); + for (i = 0; i < adapter->num_rx_queues; i++) { + struct rnpvf_ring *ring = adapter->rx_ring[i]; + u64 bytes, packets; + unsigned int start; + + if (ring) { + do { + start = u64_stats_fetch_begin( + &ring->syncp); + packets = ring->stats.packets; + bytes = ring->stats.bytes; + ring_csum_err += ring->rx_stats.csum_err; + ring_csum_good += ring->rx_stats.csum_good; + } while (u64_stats_fetch_retry(&ring->syncp, + start)); + stats->rx_packets += packets; + stats->rx_bytes += bytes; + } + } + + for (i = 0; i < adapter->num_tx_queues; i++) { + struct rnpvf_ring *ring = adapter->tx_ring[i]; + u64 bytes, packets; + unsigned int start; + + if (ring) { + do { + start = u64_stats_fetch_begin( + &ring->syncp); + packets = ring->stats.packets; + bytes = ring->stats.bytes; + } while (u64_stats_fetch_retry(&ring->syncp, + start)); + stats->tx_packets += packets; + stats->tx_bytes += bytes; + } + } + rcu_read_unlock(); + /* following stats updated by rnp_watchdog_task() */ + stats->multicast = netdev->stats.multicast; + stats->rx_errors = netdev->stats.rx_errors; + stats->rx_length_errors = netdev->stats.rx_length_errors; + stats->rx_crc_errors = netdev->stats.rx_crc_errors; + stats->rx_missed_errors = netdev->stats.rx_missed_errors; + +} + +#define RNP_MAX_TUNNEL_HDR_LEN 80 +#define RNP_MAX_MAC_HDR_LEN 127 +#define RNP_MAX_NETWORK_HDR_LEN 511 + +static netdev_features_t rnpvf_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ + unsigned int network_hdr_len, mac_hdr_len; + + /* Make certain the headers can be described by a context descriptor */ + mac_hdr_len = skb_network_header(skb) - skb->data; + if (unlikely(mac_hdr_len > RNP_MAX_MAC_HDR_LEN)) + return features & ~(NETIF_F_HW_CSUM | NETIF_F_SCTP_CRC | + NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_TSO | + NETIF_F_TSO6); + + network_hdr_len = + skb_checksum_start(skb) - skb_network_header(skb); + if (unlikely(network_hdr_len > RNP_MAX_NETWORK_HDR_LEN)) + return features & ~(NETIF_F_HW_CSUM | NETIF_F_SCTP_CRC | + NETIF_F_TSO | NETIF_F_TSO6); + + /* We can only support IPV4 TSO in tunnels if we can mangle the + * inner IP ID field, so strip TSO if MANGLEID is not supported. + */ + if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) + features &= ~NETIF_F_TSO; + + return features; +} + +static const struct net_device_ops rnpvf_netdev_ops = { + .ndo_open = rnpvf_open, + .ndo_stop = rnpvf_close, + .ndo_start_xmit = rnpvf_xmit_frame, + .ndo_validate_addr = eth_validate_addr, + .ndo_get_stats64 = rnpvf_get_stats64, + .ndo_set_rx_mode = rnpvf_set_rx_mode, + .ndo_set_mac_address = rnpvf_set_mac, + .ndo_change_mtu = rnpvf_change_mtu, + .ndo_vlan_rx_add_vid = rnpvf_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = rnpvf_vlan_rx_kill_vid, + .ndo_features_check = rnpvf_features_check, + .ndo_set_features = rnpvf_set_features, + .ndo_fix_features = rnpvf_fix_features, +}; + +void rnpvf_assign_netdev_ops(struct net_device *dev) +{ + /* different hw can assign difference fun */ + dev->netdev_ops = &rnpvf_netdev_ops; + rnpvf_set_ethtool_ops(dev); + dev->watchdog_timeo = 5 * HZ; +} + +static u8 rnpvf_vfnum(struct rnpvf_hw *hw) +{ + u16 vf_num = -1; +#if CONFIG_BAR4_PFVFNUM + int ring, v; + u16 func = 0; + + func = ((hw->pdev->devfn & 0x1) ? 1 : 0); + for (ring = 0; ring < 128; ring += 2) { + v = rd32(hw, 0x8010 + 0x100 * ring); + if ((v & 0xFFFF) == hw->pdev->vendor) { + continue; + } else { + vf_num = (1 << 7) /* vf-active */ | + (func << 6) /* pf */ | + (ring / 2) /* vfnum */; + break; + } + } + return vf_num; +#else + u32 pfvfnum_reg; + + pfvfnum_reg = + (VF_NUM_REG_N10 & (pci_resource_len(hw->pdev, 0) - 1)); + vf_num = readl(hw->hw_addr_bar0 + pfvfnum_reg); +#define VF_NUM_MASK_TEMP (0xff0) +#define VF_NUM_OFF (4) + return ((vf_num & VF_NUM_MASK_TEMP) >> VF_NUM_OFF); +#endif +} + +static inline unsigned long rnpvf_tso_features(struct rnpvf_hw *hw) +{ + unsigned long features = 0; + + if (hw->feature_flags & RNPVF_NET_FEATURE_TSO) + features |= NETIF_F_TSO; + if (hw->feature_flags & RNPVF_NET_FEATURE_TSO) + features |= NETIF_F_TSO6; + features |= NETIF_F_GSO_PARTIAL; + if (hw->feature_flags & RNPVF_NET_FEATURE_TX_UDP_TUNNEL) + features |= RNPVF_GSO_PARTIAL_FEATURES; + + return features; +} + +static int rnpvf_add_adpater(struct pci_dev *pdev, + const struct rnpvf_info *ii, + struct rnpvf_adapter **padapter) +{ + int err = 0; + struct rnpvf_adapter *adapter = NULL; + struct net_device *netdev; + struct rnpvf_hw *hw; + unsigned int queues = MAX_TX_QUEUES; + static int pf0_cards_found; + static int pf1_cards_found; + + pr_info("==== add adapter queues:%d ====", queues); + + netdev = alloc_etherdev_mq(sizeof(struct rnpvf_adapter), queues); + if (!netdev) + return -ENOMEM; + + SET_NETDEV_DEV(netdev, &pdev->dev); + adapter = netdev_priv(netdev); + adapter->netdev = netdev; + adapter->pdev = pdev; + /* setup some status */ +#ifdef FIX_VF_BUG + adapter->status |= GET_VFNUM_FROM_BAR0; +#endif + + if (padapter) + *padapter = adapter; + pci_set_drvdata(pdev, adapter); + + hw = &adapter->hw; + hw->back = adapter; + hw->pdev = pdev; + hw->board_type = ii->board_type; + adapter->msg_enable = + netif_msg_init(debug, NETIF_MSG_DRV +#ifdef MSG_PROBE_ENABLE + | NETIF_MSG_PROBE +#endif +#ifdef MSG_IFUP_ENABLE + | NETIF_MSG_IFUP +#endif +#ifdef MSG_IFDOWN_ENABLE + | NETIF_MSG_IFDOWN +#endif + ); + + switch (ii->mac) { + case rnp_mac_2port_10G: + hw->mode = MODE_NIC_MODE_2PORT_10G; + break; + case rnp_mac_2port_40G: + hw->mode = MODE_NIC_MODE_2PORT_40G; + break; + case rnp_mac_4port_10G: + hw->mode = MODE_NIC_MODE_4PORT_10G; + break; + case rnp_mac_8port_10G: + hw->mode = MODE_NIC_MODE_8PORT_10G; + break; + default: + break; + } + + switch (hw->board_type) { + case rnp_board_n10: +#define RNP_N10_BAR 4 + hw->hw_addr = pcim_iomap(pdev, RNP_N10_BAR, 0); + if (!hw->hw_addr) { + err = -EIO; + goto err_ioremap; + } + dev_info(&pdev->dev, "[bar%d]:%p %llx len=%d MB\n", + RNP_N10_BAR, hw->hw_addr, + (unsigned long long)pci_resource_start( + pdev, RNP_N10_BAR), + (int)pci_resource_len(pdev, RNP_N10_BAR) / 1024 / + 1024); +#if CONFIG_BAR4_PFVFNUM +#else + hw->hw_addr_bar0 = pcim_iomap(pdev, 0, 0); + if (!hw->hw_addr_bar0) { + err = -EIO; + goto err_ioremap; + } +#endif + + hw->vfnum = rnpvf_vfnum(hw); + dev_info(&adapter->pdev->dev, "hw->vfnum is %x\n", + hw->vfnum); + hw->ring_msix_base = hw->hw_addr + 0xa0000; + + if (hw->vfnum & 0x40) { +#ifdef FIX_VF_BUG + /* in this mode offset hw_addr */ + hw->ring_msix_base += 0x200; + hw->hw_addr += 0x100000; +#endif + adapter->port = adapter->bd_number = + pf1_cards_found++; + if (pf1_cards_found == 1000) + pf1_cards_found = 0; + } else { + adapter->port = adapter->bd_number = + pf0_cards_found++; + if (pf0_cards_found == 1000) + pf0_cards_found = 0; + } + snprintf(adapter->name, sizeof(netdev->name), "%s%d%d", + rnpvf_driver_name, (hw->vfnum & 0x40) >> 6, + adapter->bd_number); + /* n10 only support msix */ + adapter->irq_mode = irq_mode_msix; + break; + } + + pr_info("%s %s: vfnum:0x%x\n", adapter->name, pci_name(pdev), + hw->vfnum); + + rnpvf_assign_netdev_ops(netdev); + strncpy(netdev->name, adapter->name, sizeof(netdev->name) - 1); + + /* Setup hw api */ + memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); + hw->mac.type = ii->mac; + + ii->get_invariants(hw); + + memcpy(&hw->mbx.ops, &rnpvf_mbx_ops, + sizeof(struct rnp_mbx_operations)); + + /* setup the private structure */ + err = rnpvf_sw_init(adapter); + if (err) + goto err_sw_init; + + /* The HW MAC address was set and/or determined in sw_init */ + if (!is_valid_ether_addr(netdev->dev_addr)) { + pr_err("invalid MAC address\n"); + err = -EIO; + goto err_sw_init; + } + /* MTU range: 68 - 9710 */ + netdev->min_mtu = hw->min_length; + netdev->max_mtu = hw->max_length - (ETH_HLEN + 2 * ETH_FCS_LEN); + + netdev->mtu = hw->mtu; + + if (hw->feature_flags & RNPVF_NET_FEATURE_SG) + netdev->features |= NETIF_F_SG; + if (hw->feature_flags & RNPVF_NET_FEATURE_TSO) + netdev->features |= NETIF_F_TSO | NETIF_F_TSO6; + if (hw->feature_flags & RNPVF_NET_FEATURE_RX_HASH) + netdev->features |= NETIF_F_RXHASH; + if (hw->feature_flags & RNPVF_NET_FEATURE_RX_CHECKSUM) { + netdev->features |= NETIF_F_RXCSUM; + adapter->flags |= RNPVF_FLAG_RX_CHKSUM_ENABLED; + } + if (hw->feature_flags & RNPVF_NET_FEATURE_TX_CHECKSUM) { + netdev->features |= NETIF_F_HW_CSUM | NETIF_F_SCTP_CRC; + } + if (hw->feature_flags & RNPVF_NET_FEATURE_USO) { + netdev->features |= NETIF_F_GSO_UDP_L4; + } + + netdev->features |= NETIF_F_HIGHDMA; + + if (hw->feature_flags & RNPVF_NET_FEATURE_TX_UDP_TUNNEL) { + netdev->gso_partial_features = RNPVF_GSO_PARTIAL_FEATURES; + netdev->features |= NETIF_F_GSO_PARTIAL | + RNPVF_GSO_PARTIAL_FEATURES; + } + + netdev->hw_features |= netdev->features; + + if (hw->feature_flags & RNPVF_NET_FEATURE_VLAN_FILTER) + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; + if (hw->pf_feature & PF_NCSI_EN) + hw->feature_flags &= (~RNPVF_NET_FEATURE_VLAN_OFFLOAD); + if (hw->feature_flags & RNPVF_NET_FEATURE_VLAN_OFFLOAD) { + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX; + } + + if (hw->feature_flags & RNPVF_NET_FEATURE_STAG_OFFLOAD) { + netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX | + NETIF_F_HW_VLAN_STAG_TX; + } + + netdev->hw_features |= NETIF_F_RXALL; + if (hw->feature_flags & RNPVF_NET_FEATURE_RX_NTUPLE_FILTER) + netdev->hw_features |= NETIF_F_NTUPLE; + if (hw->feature_flags & RNPVF_NET_FEATURE_RX_FCS) + netdev->hw_features |= NETIF_F_RXFCS; + + netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; + netdev->hw_enc_features |= netdev->vlan_features; + netdev->mpls_features |= NETIF_F_HW_CSUM; + + /* some fixed feature control by pf */ + if (hw->pf_feature & PF_FEATURE_VLAN_FILTER) + netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; + + if (hw->pf_feature & PF_NCSI_EN) + hw->feature_flags &= (~RNPVF_NET_FEATURE_VLAN_OFFLOAD); + + if (hw->feature_flags & RNPVF_NET_FEATURE_VLAN_OFFLOAD) { + netdev->features |= NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX; + } + if (hw->feature_flags & RNPVF_NET_FEATURE_STAG_OFFLOAD) { + netdev->features |= NETIF_F_HW_VLAN_STAG_RX | + NETIF_F_HW_VLAN_STAG_TX; + } + netdev->priv_flags |= IFF_UNICAST_FLT; + netdev->priv_flags |= IFF_SUPP_NOFCS; + netdev->priv_flags |= IFF_UNICAST_FLT; + netdev->priv_flags |= IFF_SUPP_NOFCS; + + timer_setup(&adapter->watchdog_timer, rnpvf_watchdog, 0); + INIT_WORK(&adapter->watchdog_task, rnpvf_watchdog_task); + + err = rnpvf_init_interrupt_scheme(adapter); + if (err) + goto err_sw_init; + + err = register_mbx_irq(adapter); + if (err) + goto err_register; + + if (fix_eth_name) { + strncpy(netdev->name, adapter->name, + sizeof(netdev->name) - 1); + } else { + strscpy(netdev->name, pci_name(pdev), + sizeof(netdev->name)); + strscpy(netdev->name, "eth%d", sizeof(netdev->name)); + } + err = register_netdev(netdev); + if (err) { + rnpvf_err("register_netdev failed!\n"); + dev_err(&pdev->dev, + "%s %s: vfnum:0x%x. register_netdev failed!\n", + adapter->name, pci_name(pdev), hw->vfnum); + goto err_register; + } + + /* carrier off reporting is important to ethtool even BEFORE open */ + netif_carrier_off(netdev); + rnpvf_sysfs_init(netdev); + + /* print the MAC address */ + hw_dbg(hw, "%pM\n", netdev->dev_addr); + + hw_dbg(hw, "Mucse(R) n10 Virtual Function\n"); + + return 0; +err_register: + remove_mbx_irq(adapter); + rnpvf_clear_interrupt_scheme(adapter); +err_sw_init: +err_ioremap: + free_netdev(netdev); + + dev_err(&pdev->dev, "%s failed. err:%d\n", __func__, err); + return err; +} + +static int rnpvf_rm_adpater(struct rnpvf_adapter *adapter) +{ + struct net_device *netdev; + + if (!adapter) + return -EINVAL; + + rnpvf_info("= remove adapter:%s =\n", adapter->name); + netdev = adapter->netdev; + + if (netdev) { + netif_carrier_off(netdev); + rnpvf_sysfs_exit(netdev); + } + + set_bit(__RNPVF_REMOVE, &adapter->state); + del_timer_sync(&adapter->watchdog_timer); + cancel_work_sync(&adapter->watchdog_task); + + if (netdev) { + if (netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(netdev); + } + + remove_mbx_irq(adapter); + rnpvf_clear_interrupt_scheme(adapter); + rnpvf_reset_interrupt_capability(adapter); + + free_netdev(netdev); + + rnpvf_info("remove %s complete\n", adapter->name); + + return 0; +} + +/** + * rnpvf_probe - Device Initialization Routine + * @pdev: PCI device information struct + * @ent: entry in rnpvf_pci_tbl + * + * Returns 0 on success, negative on failure + * + * rnpvf_probe initializes an adapter identified by a pci_dev structure. + * The OS initialization, configuring of the adapter private structure, + * and a hardware reset occur. + **/ +static int rnpvf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct rnpvf_adapter *adapter = NULL; + const struct rnpvf_info *ii = rnpvf_info_tbl[ent->driver_data]; + int err; + + err = pci_enable_device_mem(pdev); + if (err) + return err; + + if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(56)) && + !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(56))) { + pci_using_hi_dma = 1; + } else { + err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + err = dma_set_coherent_mask(&pdev->dev, + DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, + "No usable DMA " + "configuration, aborting\n"); + goto err_dma; + } + } + pci_using_hi_dma = 0; + } + + err = pci_request_mem_regions(pdev, rnpvf_driver_name); + if (err) { + dev_err(&pdev->dev, + "pci_request_selected_regions failed 0x%x\n", err); + goto err_pci_reg; + } + pci_set_master(pdev); + pci_save_state(pdev); + + err = rnpvf_add_adpater(pdev, ii, &adapter); + if (err) { + dev_err(&pdev->dev, "ERROR %s: %d\n", __func__, __LINE__); + goto err_regions; + } + + return 0; + +err_regions: + pci_release_mem_regions(pdev); +err_dma: +err_pci_reg: + return err; +} + +/** + * rnpvf_remove - Device Removal Routine + * @pdev: PCI device information struct + * + * rnpvf_remove is called by the PCI subsystem to alert the driver + * that it should release a PCI device. The could be caused by a + * Hot-Plug event, or because the driver is going to be removed from + * memory. + **/ +static void rnpvf_remove(struct pci_dev *pdev) +{ + struct rnpvf_adapter *adapter = pci_get_drvdata(pdev); + + rnpvf_rm_adpater(adapter); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +/** + * rnpvf_io_error_detected - called when PCI error is detected + * @pdev: Pointer to PCI device + * @state: The current pci connection state + * + * This function is called after a PCI bus error affecting + * this device has been detected. + */ +static pci_ers_result_t rnpvf_io_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + netif_device_detach(netdev); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + if (netif_running(netdev)) + rnpvf_down(adapter); + + pci_disable_device(pdev); + + /* Request a slot reset. */ + return PCI_ERS_RESULT_NEED_RESET; +} + +/** + * rnpvf_io_slot_reset - called after the pci bus has been reset. + * @pdev: Pointer to PCI device + * + * Restart the card from scratch, as if from a cold-boot. Implementation + * resembles the first-half of the rnpvf_resume routine. + */ +static pci_ers_result_t rnpvf_io_slot_reset(struct pci_dev *pdev) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + if (pci_enable_device_mem(pdev)) { + dev_err(&pdev->dev, + "Cannot re-enable PCI device after reset.\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_set_master(pdev); + + rnpvf_reset(adapter); + + return PCI_ERS_RESULT_RECOVERED; +} + +/** + * rnpvf_io_resume - called when traffic can start flowing again. + * @pdev: Pointer to PCI device + * + * This callback is called when the error recovery driver tells us that + * its OK to resume normal operation. Implementation resembles the + * second-half of the rnpvf_resume routine. + */ +static void rnpvf_io_resume(struct pci_dev *pdev) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct rnpvf_adapter *adapter = netdev_priv(netdev); + + if (netif_running(netdev)) + rnpvf_up(adapter); + + netif_device_attach(netdev); +} + +/* PCI Error Recovery (ERS) */ +static const struct pci_error_handlers rnpvf_err_handler = { + .error_detected = rnpvf_io_error_detected, + .slot_reset = rnpvf_io_slot_reset, + .resume = rnpvf_io_resume, +}; + +static struct pci_driver rnpvf_driver = { + .name = rnpvf_driver_name, + .id_table = rnpvf_pci_tbl, + .probe = rnpvf_probe, + .remove = rnpvf_remove, +#ifdef CONFIG_PM + /* Power Management Hooks */ + .suspend = rnpvf_suspend, + .resume = rnpvf_resume, +#endif + .shutdown = rnpvf_shutdown, + .err_handler = &rnpvf_err_handler, +}; + +/** + * rnpvf_init_module - Driver Registration Routine + * + * rnpvf_init_module is the first routine called when the driver is + * loaded. All it does is register with the PCI subsystem. + **/ +static int __init rnpvf_init_module(void) +{ + int ret; + + pr_info("%s - version %s\n", rnpvf_driver_string, + rnpvf_driver_version); + pr_info("%s\n", rnpvf_copyright); + + ret = pci_register_driver(&rnpvf_driver); + return ret; +} + +module_init(rnpvf_init_module); + +/** + * rnpvf_exit_module - Driver Exit Cleanup Routine + * + * rnpvf_exit_module is called just before the driver is removed + * from memory. + **/ +static void __exit rnpvf_exit_module(void) +{ + pci_unregister_driver(&rnpvf_driver); +} + +module_exit(rnpvf_exit_module); diff --git a/drivers/net/ethernet/mucse/rnpvf/sysfs.c b/drivers/net/ethernet/mucse/rnpvf/sysfs.c new file mode 100644 index 0000000000000..90a59af0de5da --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/sysfs.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include "rnpvf.h" + +#define to_net_device(n) container_of(n, struct net_device, dev) + +int rnpvf_sysfs_init(struct net_device *ndev) +{ + return 0; +} + +void rnpvf_sysfs_exit(struct net_device *ndev) +{ +} diff --git a/drivers/net/ethernet/mucse/rnpvf/vf.c b/drivers/net/ethernet/mucse/rnpvf/vf.c new file mode 100644 index 0000000000000..c332a19db84d7 --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/vf.c @@ -0,0 +1,849 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#include "vf.h" +#include "rnpvf.h" + +static int rnpvf_reset_pf(struct rnpvf_hw *hw) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[2]; + s32 ret_val; + + memset(msgbuf, 0, sizeof(msgbuf)); + msgbuf[0] = RNP_VF_RESET_PF; + + ret_val = mbx->ops.write_posted(hw, msgbuf, 2, false); + + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 2, false); + + return ret_val; +} + +static int rnpvf_get_mtu(struct rnpvf_hw *hw) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[2]; + s32 ret_val; + + memset(msgbuf, 0, sizeof(msgbuf)); + msgbuf[0] = RNP_VF_GET_MTU; + + ret_val = mbx->ops.write_posted(hw, msgbuf, 2, false); + + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 2, false); + + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + + /* if nacked the address was rejected, use "perm_addr" */ + if (!ret_val && + (msgbuf[0] == (RNP_VF_SET_MTU | RNP_VT_MSGTYPE_NACK))) + return -1; + hw->mtu = msgbuf[1]; + + return ret_val; +} + +static int rnpvf_set_mtu(struct rnpvf_hw *hw, int mtu) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[2]; + s32 ret_val; + + memset(msgbuf, 0, sizeof(msgbuf)); + msgbuf[0] = RNP_VF_SET_MTU; + msgbuf[1] = mtu; + + ret_val = mbx->ops.write_posted(hw, msgbuf, 2, false); + + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 2, false); + + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + + /* if nacked the address was rejected, use "perm_addr" */ + if (!ret_val && + (msgbuf[0] == (RNP_VF_SET_MTU | RNP_VT_MSGTYPE_NACK))) + return -1; + + return ret_val; +} + +static int rnpvf_read_eth_reg(struct rnpvf_hw *hw, int reg, u32 *value) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[2]; + int err; + + msgbuf[0] = RNP_VF_REG_RD; + msgbuf[1] = reg; + + err = mbx->ops.write_posted(hw, msgbuf, 2, false); + if (err) + goto mbx_err; + + err = mbx->ops.read_posted(hw, msgbuf, 2, false); + if (err) + goto mbx_err; + + /* remove extra bits from the message */ + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + msgbuf[0] &= ~(0xFF << RNP_VT_MSGINFO_SHIFT); + + if (msgbuf[0] != (RNP_VF_REG_RD | RNP_VT_MSGTYPE_ACK)) + err = RNP_ERR_INVALID_ARGUMENT; + + *value = msgbuf[1]; + +mbx_err: + return err; +} + +/** + * rnpvf_start_hw_vf - Prepare hardware for Tx/Rx + * @hw: pointer to hardware structure + * + * Starts the hardware by filling the bus info structure and media type, clears + * all on chip counters, initializes receive address registers, multicast + * table, VLAN filter table, calls routine to set up link and flow control + * settings, and leaves transmit and receive units disabled and uninitialized + **/ +static s32 rnpvf_start_hw_vf(struct rnpvf_hw *hw) +{ + /* Clear adapter stopped flag */ + hw->adapter_stopped = false; + + return 0; +} + +/** + * rnpvf_init_hw_vf - virtual function hardware initialization + * @hw: pointer to hardware structure + * + * Initialize the hardware by resetting the hardware and then starting + * the hardware + **/ +static s32 rnpvf_init_hw_vf(struct rnpvf_hw *hw) +{ + s32 status; + + status = hw->mac.ops.start_hw(hw); + + hw->mac.ops.get_mac_addr(hw, hw->mac.addr); + + return status; +} + +/** + * rnpvf_reset_hw_vf - Performs hardware reset + * @hw: pointer to hardware structure + * + * Resets the hardware by resetting the transmit and receive units, masks and + * clears all interrupts. + **/ +static s32 rnpvf_reset_hw_vf(struct rnpvf_hw *hw) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + struct rnpvf_adapter *adapter = hw->back; + // u32 timeout = RNP_VF_INIT_TIMEOUT; + s32 ret_val = RNP_ERR_INVALID_MAC_ADDR; + u32 msgbuf[RNP_VF_PERMADDR_MSG_LEN]; + u8 *addr = (u8 *)(&msgbuf[1]); + u32 vlan; + int try_cnt = 10; + + /* Call adapter stop to disable tx/rx and clear interrupts */ + hw->mac.ops.stop_adapter(hw); + + /* reset the api version */ + hw->api_version = 0; + + /* mailbox timeout can now become active */ + mbx->timeout = RNP_VF_MBX_INIT_TIMEOUT; + + while (try_cnt--) { + msgbuf[0] = RNP_VF_RESET; + mbx->ops.write_posted(hw, msgbuf, 1, false); + /* ack write back maybe too fast */ + mdelay(20); + + /* set our "perm_addr" based on info provided by PF */ + /* also set up the mc_filter_type which is piggy backed + * on the mac address in word 3 + */ + ret_val = mbx->ops.read_posted( + hw, msgbuf, RNP_VF_PERMADDR_MSG_LEN, false); + if (ret_val == 0) + break; + } + if (ret_val) + return ret_val; + + /* New versions of the PF may NACK the reset return message + * to indicate that no MAC address has yet been assigned for + * the VF. + */ + if (msgbuf[0] != (RNP_VF_RESET | RNP_VT_MSGTYPE_ACK) && + msgbuf[0] != (RNP_VF_RESET | RNP_VT_MSGTYPE_NACK)) + return RNP_ERR_INVALID_MAC_ADDR; + /* we get mac address from mailbox */ + + memcpy(hw->mac.perm_addr, addr, ETH_ALEN); + hw->mac.mc_filter_type = msgbuf[RNP_VF_MC_TYPE_WORD] & 0xff; + + /* ft padding */ + if ((msgbuf[RNP_VF_MC_TYPE_WORD] >> 8) & 0xff) + adapter->priv_flags |= RNPVF_PRIV_FLAG_FT_PADDING; + else + adapter->priv_flags = 0; + /* fc mode */ + hw->fc.current_mode = (msgbuf[RNP_VF_MC_TYPE_WORD] >> 16) & 0xff; + + /* phy status */ + hw->phy_type = (msgbuf[RNP_VF_PHY_TYPE_WORD] & 0xffff); + + hw->dma_version = hw->mac.dma_version = + msgbuf[RNP_VF_DMA_VERSION_WORD]; + + /* vlan status */ + vlan = msgbuf[RNP_VF_VLAN_WORD]; + if (vlan & 0xffff) { + adapter->vf_vlan = vlan & 0xffff; + adapter->flags |= RNPVF_FLAG_PF_SET_VLAN; + } + hw->ops.set_veb_vlan(hw, vlan, VFNUM(mbx, hw->vfnum)); + hw->fw_version = msgbuf[RNP_VF_FW_VERSION_WORD]; + + if (msgbuf[RNP_VF_LINK_STATUS_WORD] & RNP_PF_LINK_UP) { + hw->link = true; + hw->speed = msgbuf[RNP_VF_LINK_STATUS_WORD] & 0xffff; + + } else { + hw->link = false; + hw->speed = 0; + } + + hw->usecstocount = msgbuf[RNP_VF_AXI_MHZ]; + + DPRINTK(PROBE, INFO, "dma_versioin:%x vlan %d \n", + hw->mac.dma_version, adapter->vf_vlan); + DPRINTK(PROBE, INFO, "axi:%x\n", hw->usecstocount); + DPRINTK(PROBE, INFO, "firmware :%x\n", hw->fw_version); + DPRINTK(PROBE, INFO, "link speed :%x\n", hw->speed); + DPRINTK(PROBE, INFO, "link status :%s\n", + hw->link ? "up" : "down"); + hw->pf_feature = msgbuf[RNP_VF_FEATURE]; + + return 0; +} + +/** + * rnpvf_stop_hw_vf - Generic stop Tx/Rx units + * @hw: pointer to hardware structure + * + * Sets the adapter_stopped flag within rnpvf_hw struct. Clears interrupts, + * disables transmit and receive units. The adapter_stopped flag is used by + * the shared code and drivers to determine if the adapter is in a stopped + * state and should not touch the hardware. + **/ +static s32 rnpvf_stop_hw_vf(struct rnpvf_hw *hw) +{ + u32 number_of_queues; + u16 i; + struct rnpvf_adapter *adapter = hw->back; + struct rnpvf_ring *ring; + + /* + * Set the adapter_stopped flag so other driver functions stop touching + * the hardware + */ + hw->adapter_stopped = true; + + /* Disable the receive unit by stopped each queue */ + for (i = 0; i < adapter->num_rx_queues; i++) { + ring = adapter->rx_ring[i]; + ring_wr32(ring, RNP_DMA_RX_START, 0); + } + + /* Disable the transmit unit. Each queue must be disabled. */ + number_of_queues = hw->mac.max_tx_queues; + + return 0; +} + +/** + * rnpvf_mta_vector - Determines bit-vector in multicast table to set + * @hw: pointer to hardware structure + * @mc_addr: the multicast address + * + * Extracts the 12 bits, from a multicast address, to determine which + * bit-vector to set in the multicast table. The hardware uses 12 bits, from + * incoming rx multicast addresses, to determine the bit-vector to check in + * the MTA. Which of the 4 combination, of 12-bits, the hardware uses is set + * by the MO field of the MCSTCTRL. The MO field is set during initialization + * to mc_filter_type. + **/ +static s32 rnpvf_mta_vector(struct rnpvf_hw *hw, u8 *mc_addr) +{ + u32 vector = 0; + + switch (hw->mac.mc_filter_type) { + case 0: /* use bits [47:36] of the address */ + vector = ((mc_addr[4] << 8) | (((u16)mc_addr[5]))); + break; + case 1: /* use bits [46:35] of the address */ + vector = ((mc_addr[4] << 7) | (((u16)mc_addr[5]) >> 1)); + break; + case 2: /* use bits [45:34] of the address */ + vector = ((mc_addr[4] << 6) | (((u16)mc_addr[5]) >> 2)); + break; + case 3: /* use bits [43:32] of the address */ + vector = ((mc_addr[4]) << 4 | (((u16)mc_addr[5]) >> 4)); + break; + case 4: /* use bits [32:43] of the address */ + vector = ((mc_addr[0] << 8) | (((u16)mc_addr[1]))); + vector = (vector >> 4); + break; + case 5: /* use bits [32:43] of the address */ + vector = ((mc_addr[0] << 8) | (((u16)mc_addr[1]))); + vector = (vector >> 3); + break; + case 6: /* use bits [32:43] of the address */ + vector = ((mc_addr[0] << 8) | (((u16)mc_addr[1]))); + vector = (vector >> 2); + break; + case 7: /* use bits [32:43] of the address */ + vector = ((mc_addr[0] << 8) | (((u16)mc_addr[1]))); + break; + default: /* Invalid mc_filter_type */ + break; + } + + /* vector can only be 12-bits or boundary will be exceeded */ + vector &= 0xFFF; + return vector; +} + +/** + * rnpvf_get_mac_addr_vf - Read device MAC address + * @hw: pointer to the HW structure + * @mac_addr: pointer to storage for retrieved MAC address + **/ +static s32 rnpvf_get_mac_addr_vf(struct rnpvf_hw *hw, u8 *mac_addr) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[3]; + u8 *msg_addr = (u8 *)(&msgbuf[1]); + s32 ret_val = 0; + + memset(msgbuf, 0, sizeof(msgbuf)); + /* + * If index is one then this is the start of a new list and needs + * indication to the PF so it can do it's own list management. + * If it is zero then that tells the PF to just clear all of + * this VF's macvlans and there is no new list. + */ + msgbuf[0] |= RNP_VF_SET_MACVLAN; + ret_val = mbx->ops.write_posted(hw, msgbuf, 1, false); + + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 3, false); + + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + + if (!ret_val) + if (msgbuf[0] == + (RNP_VF_GET_MACVLAN | RNP_VT_MSGTYPE_NACK)) + ret_val = -ENOMEM; + + memcpy(mac_addr, msg_addr, 6); + + return 0; +} + +/** + * rnpvf_get_queues_vf - Read device MAC address + * @hw: pointer to the HW structure + * @mac_addr: pointer to storage for retrieved MAC address + **/ +static s32 rnpvf_get_queues_vf(struct rnpvf_hw *hw) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + s32 ret_val = 0; + u32 msgbuf[7]; + + memset(msgbuf, 0, sizeof(msgbuf)); + msgbuf[0] |= RNP_VF_GET_QUEUE; + + ret_val = mbx->ops.write_posted(hw, msgbuf, 1, false); + + mdelay(10); + + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 7, false); + + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + + if (!ret_val) + if (msgbuf[0] == (RNP_VF_GET_QUEUE | RNP_VT_MSGTYPE_NACK)) + ret_val = -ENOMEM; +#define MSG_TX_NUM_WORD 1 +#define MSG_RX_NUM_WORD 2 +#define MSG_RING_BASE_WORD 5 +#define MSG_RING_DEPTH 6 + + hw->queue_ring_base = msgbuf[MSG_RING_BASE_WORD]; + hw->mac.max_tx_queues = msgbuf[MSG_TX_NUM_WORD]; + hw->mac.max_rx_queues = msgbuf[MSG_RX_NUM_WORD]; + hw->tx_items_count = 0xffff & (msgbuf[MSG_RING_DEPTH] >> 16); + hw->rx_items_count = 0xffff & (msgbuf[MSG_RING_DEPTH] >> 0); + + return 0; +} + +static s32 rnpvf_set_uc_addr_vf(struct rnpvf_hw *hw, u32 index, u8 *addr) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[3]; + u8 *msg_addr = (u8 *)(&msgbuf[1]); + s32 ret_val = 0; + + memset(msgbuf, 0, sizeof(msgbuf)); + /* + * If index is one then this is the start of a new list and needs + * indication to the PF so it can do it's own list management. + * If it is zero then that tells the PF to just clear all of + * this VF's macvlans and there is no new list. + */ + msgbuf[0] |= index << RNP_VT_MSGINFO_SHIFT; + msgbuf[0] |= RNP_VF_SET_MACVLAN; + if (addr) + memcpy(msg_addr, addr, 6); + ret_val = mbx->ops.write_posted(hw, msgbuf, 3, false); + + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 3, false); + + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + + if (!ret_val) + if (msgbuf[0] == + (RNP_VF_SET_MACVLAN | RNP_VT_MSGTYPE_NACK)) + ret_val = -ENOMEM; + return ret_val; +} + +/** + * rnpvf_set_rar_vf - set device MAC address + * @hw: pointer to hardware structure + * @index: Receive address register to write + * @addr: Address to put into receive address register + * @vmdq: Unused in this implementation + **/ +static s32 rnpvf_set_rar_vf(struct rnpvf_hw *hw, u32 index, u8 *addr, + u32 vmdq) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[3]; + u8 *msg_addr = (u8 *)(&msgbuf[1]); + s32 ret_val; + + memset(msgbuf, 0, sizeof(msgbuf)); + msgbuf[0] = RNP_VF_SET_MAC_ADDR; + memcpy(msg_addr, addr, 6); + ret_val = mbx->ops.write_posted(hw, msgbuf, 3, false); + + if (!ret_val) + ret_val = mbx->ops.read_posted(hw, msgbuf, 3, false); + + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + + /* if nacked the address was rejected, use "perm_addr" */ + if (!ret_val && + (msgbuf[0] == (RNP_VF_SET_MAC_ADDR | RNP_VT_MSGTYPE_NACK))) { + rnpvf_get_mac_addr_vf(hw, hw->mac.addr); + return -1; + } + + return ret_val; +} + +static void rnpvf_write_msg_read_ack(struct rnpvf_hw *hw, u32 *msg, + u16 size) +{ + u32 retmsg[RNP_VFMAILBOX_SIZE]; + s32 retval; + struct rnp_mbx_info *mbx = &hw->mbx; + + retval = mbx->ops.write_posted(hw, msg, size, false); + if (!retval) + mbx->ops.read_posted(hw, retmsg, size, false); +} + +u8 *rnpvf_addr_list_itr(struct rnpvf_hw __maybe_unused *hw, + u8 **mc_addr_ptr) +{ + struct netdev_hw_addr *mc_ptr; + u8 *addr = *mc_addr_ptr; + + mc_ptr = container_of(addr, struct netdev_hw_addr, addr[0]); + if (mc_ptr->list.next) { + struct netdev_hw_addr *ha; + + ha = list_entry(mc_ptr->list.next, struct netdev_hw_addr, + list); + *mc_addr_ptr = ha->addr; + } else + *mc_addr_ptr = NULL; + + return addr; +} + +/** + * rnpvf_update_mc_addr_list_vf - Update Multicast addresses + * @hw: pointer to the HW structure + * @netdev: pointer to net device structure + * + * Updates the Multicast Table Array. + **/ +static s32 rnpvf_update_mc_addr_list_vf(struct rnpvf_hw *hw, + struct net_device *netdev) +{ + struct netdev_hw_addr *ha; + u32 msgbuf[RNP_VFMAILBOX_SIZE]; + u16 *vector_list = (u16 *)&msgbuf[1]; + u32 cnt, i; + int addr_count = 0; + u8 *addr_list = NULL; + + /* Each entry in the list uses 1 16 bit word. We have 30 + * 16 bit words available in our HW msg buffer (minus 1 for the + * msg type). That's 30 hash values if we pack 'em right. If + * there are more than 30 MC addresses to add then punt the + * extras for now and then add code to handle more than 30 later. + * It would be unusual for a server to request that many multi-cast + * addresses except for in large enterprise network environments. + */ + + cnt = netdev_mc_count(netdev); + if (cnt > 30) + cnt = 30; + msgbuf[0] = RNP_VF_SET_MULTICAST; + msgbuf[0] |= cnt << RNP_VT_MSGINFO_SHIFT; + + addr_count = netdev_mc_count(netdev); + + ha = list_first_entry(&netdev->mc.list, struct netdev_hw_addr, + list); + addr_list = ha->addr; + for (i = 0; i < addr_count; i++) { + vector_list[i] = rnpvf_mta_vector( + hw, rnpvf_addr_list_itr(hw, &addr_list)); + } + + rnpvf_write_msg_read_ack(hw, msgbuf, RNP_VFMAILBOX_SIZE); + + return 0; +} + +/** + * rnpvf_set_vfta_vf - Set/Unset vlan filter table address + * @hw: pointer to the HW structure + * @vlan: 12 bit VLAN ID + * @vind: unused by VF drivers + * @vlan_on: if true then set bit, else clear bit + **/ +static s32 rnpvf_set_vfta_vf(struct rnpvf_hw *hw, u32 vlan, u32 vind, + bool vlan_on) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[2]; + s32 err; + + msgbuf[0] = RNP_VF_SET_VLAN; + msgbuf[1] = vlan; + /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ + msgbuf[0] |= vlan_on << RNP_VT_MSGINFO_SHIFT; + + err = mbx->ops.write_posted(hw, msgbuf, 2, false); + if (err) { + printk("vlan write_posted failed\n"); + goto mbx_err; + } + + err = mbx->ops.read_posted(hw, msgbuf, 2, false); + if (err) { + printk("vlan read_posted failed\n"); + goto mbx_err; + } + + /* remove extra bits from the message */ + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + msgbuf[0] &= ~(0xFF << RNP_VT_MSGINFO_SHIFT); + + if (msgbuf[0] != (RNP_VF_SET_VLAN | RNP_VT_MSGTYPE_ACK)) + err = RNP_ERR_INVALID_ARGUMENT; + +mbx_err: + return err; +} + +static s32 rnpvf_set_vlan_strip(struct rnpvf_hw *hw, bool vlan_on) +{ + struct rnp_mbx_info *mbx = &hw->mbx; + struct rnpvf_adapter *adapter = (struct rnpvf_adapter *)hw->back; + u32 msgbuf[4]; + s32 err; + int i; + + if (adapter->num_rx_queues > 2) { + err = -EINVAL; + goto mbx_err; + } + + msgbuf[0] = RNP_VF_SET_VLAN_STRIP; + msgbuf[1] = (vlan_on << 31) | adapter->num_rx_queues; + + for (i = 0; i < adapter->num_rx_queues; i++) + msgbuf[2 + i] = adapter->rx_ring[i]->rnpvf_queue_idx; + + err = mbx->ops.write_posted(hw, msgbuf, 2 + adapter->num_rx_queues, + false); + if (err) + goto mbx_err; + + err = mbx->ops.read_posted(hw, msgbuf, 1, false); + if (err) + goto mbx_err; + + /* remove extra bits from the message */ + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + msgbuf[0] &= ~(0xFF << RNP_VT_MSGINFO_SHIFT); + + if (msgbuf[0] != (RNP_VF_SET_VLAN_STRIP | RNP_VT_MSGTYPE_ACK)) + err = RNP_ERR_INVALID_ARGUMENT; + +mbx_err: + return err; +} + +/** + * rnpvf_setup_mac_link_vf - Setup MAC link settings + * @hw: pointer to hardware structure + * @speed: Unused in this implementation + * @autoneg: Unused in this implementation + * @autoneg_wait_to_complete: Unused in this implementation + * + * Do nothing and return success. VF drivers are not allowed to change + * global settings. Maintained for driver compatibility. + **/ +static s32 rnpvf_setup_mac_link_vf(struct rnpvf_hw *hw, + rnp_link_speed speed, bool autoneg, + bool autoneg_wait_to_complete) +{ + return 0; +} + +/** + * rnpvf_check_mac_link_vf - Get link/speed status + * @hw: pointer to hardware structure + * @speed: pointer to link speed + * @link_up: true is link is up, false otherwise + * @autoneg_wait_to_complete: true when waiting for completion is needed + * + * Reads the links register to determine if link is up and the current speed + **/ +static s32 rnpvf_check_mac_link_vf(struct rnpvf_hw *hw, + rnp_link_speed *speed, bool *link_up, + bool autoneg_wait_to_complete) +{ + *speed = hw->speed; + *link_up = hw->link; + + return 0; +} + +/** + * rnpvf_rlpml_set_vf - Set the maximum receive packet length + * @hw: pointer to the HW structure + * @max_size: value to assign to max frame size + **/ +void rnpvf_rlpml_set_vf(struct rnpvf_hw *hw, u16 max_size) +{ + u32 msgbuf[2]; + + msgbuf[0] = RNP_VF_SET_LPE; + msgbuf[1] = max_size; + rnpvf_write_msg_read_ack(hw, msgbuf, 2); +} + +/** + * rnpvf_negotiate_api_version - Negotiate supported API version + * @hw: pointer to the HW structure + * @api: integer containing requested API version + **/ +int rnpvf_negotiate_api_version(struct rnpvf_hw *hw, int api) +{ + return 0; +} + +int rnpvf_get_queues(struct rnpvf_hw *hw, unsigned int *num_tcs, + unsigned int *default_tc) +{ + return -1; +} + +void rnpvf_set_veb_mac_n10(struct rnpvf_hw *hw, u8 *mac, u32 vfnum, + u32 ring) +{ + int port; + u32 maclow, machi; + + maclow = (mac[2] << 24) | (mac[3] << 16) | (mac[4] << 8) | mac[5]; + machi = (mac[0] << 8) | mac[1]; + for (port = 0; port < 4; port++) { + maclow = (mac[2] << 24) | (mac[3] << 16) | (mac[4] << 8) | + mac[5]; + machi = (mac[0] << 8) | mac[1]; + + wr32(hw, RNP_DMA_PORT_VBE_MAC_LO_TBL_N10(port, vfnum), + maclow); + wr32(hw, RNP_DMA_PORT_VBE_MAC_HI_TBL_N10(port, vfnum), + machi); + + wr32(hw, RNP_DMA_PORT_VEB_VF_RING_TBL_N10(port, vfnum), + ring); + } +} + +void rnpvf_set_vlan_n10(struct rnpvf_hw *hw, u16 vid, u32 vf_num) +{ + int port; + + for (port = 0; port < 4; port++) + wr32(hw, RNP_DMA_PORT_VEB_VID_TBL_N10(port, vf_num), vid); +} + +int rnpvf_set_promisc_mode(struct rnpvf_hw *hw, bool promisc) +{ + + struct rnp_mbx_info *mbx = &hw->mbx; + u32 msgbuf[2]; + s32 err; + + msgbuf[0] = RNP_VF_SET_PROMISCE; + if (promisc) + msgbuf[1] = 1; + else + msgbuf[1] = 0; + + err = mbx->ops.write_posted(hw, msgbuf, 2, false); + if (err) { + printk("promisc write_posted failed\n"); + goto mbx_err; + } + + err = mbx->ops.read_posted(hw, msgbuf, 2, false); + if (err) { + printk("promisc read_posted failed\n"); + goto mbx_err; + } + + /* remove extra bits from the message */ + msgbuf[0] &= ~RNP_VT_MSGTYPE_CTS; + msgbuf[0] &= ~(0xFF << RNP_VT_MSGINFO_SHIFT); + + if (msgbuf[0] != (RNP_VF_SET_PROMISCE | RNP_VT_MSGTYPE_ACK)) { + err = RNP_ERR_INVALID_ARGUMENT; + printk("set promisc failed\n"); + } + +mbx_err: + return err; + + + +} + +static const struct rnpvf_hw_operations rnpvf_hw_ops_n10 = { + .set_veb_mac = rnpvf_set_veb_mac_n10, + .set_veb_vlan = rnpvf_set_vlan_n10, +}; + +static s32 rnpvf_get_invariants_n10(struct rnpvf_hw *hw) +{ + struct rnp_mbx_info *mbx = &hw->mbx; +#ifdef FIX_MAC_PADDIN + struct rnpvf_adapter *adapter = (struct rnpvf_adapter *)hw->back; +#endif + + hw->feature_flags |= + RNPVF_NET_FEATURE_SG | RNPVF_NET_FEATURE_TX_CHECKSUM | + RNPVF_NET_FEATURE_RX_CHECKSUM | RNPVF_NET_FEATURE_TSO | + RNPVF_NET_FEATURE_TX_UDP_TUNNEL | + RNPVF_NET_FEATURE_VLAN_OFFLOAD | RNPVF_NET_FEATURE_RX_HASH; + + /* mbx setup */ + mbx->pf2vf_mbox_vec_base = 0xa5000; + mbx->vf2pf_mbox_vec_base = 0xa5100; + mbx->cpu2vf_mbox_vec_base = 0xa5200; + mbx->cpu2pf_mbox_vec = 0xa5300; + mbx->pf_vf_shm_base = 0xa6000; + mbx->cpu_vf_shm_base = 0xa8000; + mbx->vf2cpu_mbox_ctrl_base = 0xa9000; + mbx->cpu_vf_mbox_mask_lo_base = 0xa9200; + mbx->cpu_vf_mbox_mask_hi_base = 0xa9300; + mbx->mbx_mem_size = 64; + + mbx->vf2pf_mbox_ctrl_base = 0xa7000; + mbx->pf2vf_mbox_ctrl_base = 0xa7100; + mbx->pf_vf_mbox_mask_lo = 0xa7200; + mbx->pf_vf_mbox_mask_hi = 0xa7300; + + mbx->cpu_pf_shm_base = 0xaa000; + mbx->pf2cpu_mbox_ctrl = 0xaa100; + mbx->pf2cpu_mbox_mask = 0xaa300; + + mbx->vf_num_mask = 0x3f; + + hw->min_length = RNPVF_MIN_MTU; + hw->max_length = RNPVF_N10_MAX_JUMBO_FRAME_SIZE; + +#ifdef FIX_MAC_PADDIN + adapter->priv_flags |= RNPVF_PRIV_FLAG_TX_PADDING; +#endif + + memcpy(&hw->ops, &rnpvf_hw_ops_n10, sizeof(hw->ops)); + + return 0; +} + +static const struct rnp_mac_operations rnpvf_mac_ops = { + .init_hw = rnpvf_init_hw_vf, + .reset_hw = rnpvf_reset_hw_vf, + .start_hw = rnpvf_start_hw_vf, + .get_mac_addr = rnpvf_get_mac_addr_vf, + .get_queues = rnpvf_get_queues_vf, + .stop_adapter = rnpvf_stop_hw_vf, + .setup_link = rnpvf_setup_mac_link_vf, + .check_link = rnpvf_check_mac_link_vf, + .set_rar = rnpvf_set_rar_vf, + .update_mc_addr_list = rnpvf_update_mc_addr_list_vf, + .set_uc_addr = rnpvf_set_uc_addr_vf, + .set_vfta = rnpvf_set_vfta_vf, + .set_vlan_strip = rnpvf_set_vlan_strip, + .read_eth_reg = rnpvf_read_eth_reg, + .get_mtu = rnpvf_get_mtu, + .set_mtu = rnpvf_set_mtu, + .req_reset_pf = rnpvf_reset_pf, + .set_promisc_mode = rnpvf_set_promisc_mode, +}; + +const struct rnpvf_info rnp_n10_vf_info = { + .mac = rnp_mac_2port_40G, + .mac_ops = &rnpvf_mac_ops, + .board_type = rnp_board_n10, + .get_invariants = &rnpvf_get_invariants_n10, +}; diff --git a/drivers/net/ethernet/mucse/rnpvf/vf.h b/drivers/net/ethernet/mucse/rnpvf/vf.h new file mode 100644 index 0000000000000..ca84f37e862e7 --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpvf/vf.h @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2022 - 2024 Mucse Corporation. */ + +#ifndef __RNP_VF_H__ +#define __RNP_VF_H__ + +#include +#include +#include +#include +#include + +#include "defines.h" +#include "regs.h" +#include "mbx.h" + +struct rnpvf_hw; + +/* iterator type for walking multicast address lists */ +typedef u8 *(*rnp_mc_addr_itr)(struct rnpvf_hw *hw, u8 **mc_addr_ptr, + u32 *vmdq); +struct rnp_mac_operations { + s32 (*init_hw)(struct rnpvf_hw *); + s32 (*reset_hw)(struct rnpvf_hw *); + s32 (*start_hw)(struct rnpvf_hw *); + s32 (*clear_hw_cntrs)(struct rnpvf_hw *); + enum rnp_media_type (*get_media_type)(struct rnpvf_hw *); + u32 (*get_supported_physical_layer)(struct rnpvf_hw *); + s32 (*get_mac_addr)(struct rnpvf_hw *, u8 *); + s32 (*get_queues)(struct rnpvf_hw *); + s32 (*stop_adapter)(struct rnpvf_hw *); + s32 (*get_bus_info)(struct rnpvf_hw *); + int (*read_eth_reg)(struct rnpvf_hw *, int, u32 *); + int (*get_mtu)(struct rnpvf_hw *); + int (*set_mtu)(struct rnpvf_hw *, int); + int (*req_reset_pf)(struct rnpvf_hw *); + /* Link */ + s32 (*setup_link)(struct rnpvf_hw *, rnp_link_speed, bool, bool); + s32 (*check_link)(struct rnpvf_hw *, rnp_link_speed *, bool *, + bool); + s32 (*get_link_capabilities)(struct rnpvf_hw *, rnp_link_speed *, + bool *); + /* RAR, Multicast, VLAN */ + s32 (*set_rar)(struct rnpvf_hw *, u32, u8 *, u32); + s32 (*set_uc_addr)(struct rnpvf_hw *, u32, u8 *); + s32 (*init_rx_addrs)(struct rnpvf_hw *); + s32 (*update_mc_addr_list)(struct rnpvf_hw *, struct net_device *); + s32 (*enable_mc)(struct rnpvf_hw *); + s32 (*disable_mc)(struct rnpvf_hw *); + s32 (*clear_vfta)(struct rnpvf_hw *); + s32 (*set_vfta)(struct rnpvf_hw *, u32, u32, bool); + s32 (*set_vlan_strip)(struct rnpvf_hw *, bool); + s32 (*set_promisc_mode)(struct rnpvf_hw *, bool); +}; + +enum rnp_mac_type { + rnp_mac_unknown = 0, + rnp_mac_2port_10G, + rnp_mac_2port_40G, + rnp_mac_4port_10G, + rnp_mac_8port_10G, + rnp_num_macs +}; + +enum rnp_board_type { + rnp_board_n10, +}; + +struct rnp_mac_info { + struct rnp_mac_operations ops; + u8 addr[6]; + u8 perm_addr[6]; + enum rnp_mac_type type; + s32 mc_filter_type; + u32 dma_version; + bool get_link_status; + u32 max_tx_queues; + u32 max_rx_queues; + u32 max_msix_vectors; +}; + +#define RNP_MAX_TRAFFIC_CLASS 4 +enum rnp_fc_mode { + rnp_fc_none = 0, + rnp_fc_rx_pause, + rnp_fc_tx_pause, + rnp_fc_full, + rnp_fc_default +}; + +struct rnp_fc_info { + u32 high_water[RNP_MAX_TRAFFIC_CLASS]; /* Flow Control High-water */ + u32 low_water[RNP_MAX_TRAFFIC_CLASS]; /* Flow Control Low-water */ + u16 pause_time; /* Flow Control Pause timer */ + bool send_xon; /* Flow control send XON */ + bool strict_ieee; /* Strict IEEE mode */ + bool disable_fc_autoneg; /* Do not autonegotiate FC */ + bool fc_was_autonegged; /* Is current_mode the result of autonegging? */ + enum rnp_fc_mode current_mode; /* FC mode in effect */ + enum rnp_fc_mode requested_mode; /* FC mode requested by caller */ +}; + +struct rnp_mbx_operations { + s32 (*init_params)(struct rnpvf_hw *hw); + s32 (*read)(struct rnpvf_hw *, u32 *, u16, bool); + s32 (*write)(struct rnpvf_hw *, u32 *, u16, bool); + s32 (*read_posted)(struct rnpvf_hw *, u32 *, u16, bool); + s32 (*write_posted)(struct rnpvf_hw *, u32 *, u16, bool); + s32 (*check_for_msg)(struct rnpvf_hw *, bool); + s32 (*check_for_ack)(struct rnpvf_hw *, bool); + s32 (*check_for_rst)(struct rnpvf_hw *, bool); + s32 (*configure)(struct rnpvf_hw *hw, int nr_vec, bool enable); +}; + +struct rnpvf_hw_operations { + void (*set_veb_mac)(struct rnpvf_hw *hw, u8 *, u32, u32); + void (*set_veb_vlan)(struct rnpvf_hw *hw, u16, u32); +}; + +struct rnp_mbx_stats { + u32 msgs_tx; + u32 msgs_rx; + u32 acks; + u32 reqs; + u32 rsts; +}; + +struct rnp_mbx_info { + struct rnp_mbx_operations ops; + struct rnp_mbx_stats stats; + u32 timeout; + u32 udelay; + u32 v2p_mailbox; + u16 size; + u16 pf_req; + u16 pf_ack; + u16 cpu_req; + u16 cpu_ack; + u32 vf_num_mask; + int mbx_size; + int mbx_mem_size; + /* cm3 <-> pf mbx */ + u32 cpu_pf_shm_base; + u32 pf2cpu_mbox_ctrl; + u32 pf2cpu_mbox_mask; + u32 cpu_pf_mbox_mask; + u32 cpu2pf_mbox_vec; + /* cm3 <-> vf mbx */ + u32 cpu_vf_shm_base; + u32 cpu2vf_mbox_vec_base; + u32 cpu_vf_mbox_mask_lo_base; + u32 cpu_vf_mbox_mask_hi_base; + /* pf <--> vf mbx */ + u32 pf_vf_shm_base; + u32 vf2cpu_mbox_ctrl_base; + u32 pf2vf_mbox_ctrl_base; + u32 pf_vf_mbox_mask_lo; + u32 pf_vf_mbox_mask_hi; + u32 pf2vf_mbox_vec_base; + u32 vf2pf_mbox_vec_base; + u32 vf2pf_mbox_ctrl_base; +}; + +struct rnpvf_hw_stats_own { + u64 vlan_add_cnt; + u64 vlan_strip_cnt; + u64 csum_err; + u64 csum_good; + u64 spoof_dropped; +}; + +struct rnpvf_hw_stats { + u64 base_vfgprc; + u64 base_vfgptc; + u64 base_vfgorc; + u64 base_vfgotc; + u64 base_vfmprc; + u64 last_vfgprc; + u64 last_vfgptc; + u64 last_vfgorc; + u64 last_vfgotc; + u64 last_vfmprc; + u64 vfgprc; + u64 vfgptc; + u64 vfgorc; + u64 vfgotc; + u64 vfmprc; + u64 saved_reset_vfgprc; + u64 saved_reset_vfgptc; + u64 saved_reset_vfgorc; + u64 saved_reset_vfgotc; + u64 saved_reset_vfmprc; +}; + +struct rnpvf_info { + enum rnp_mac_type mac; + enum rnp_board_type board_type; + const struct rnp_mac_operations *mac_ops; + s32 (*get_invariants)(struct rnpvf_hw *); +}; + +void rnpvf_rlpml_set_vf(struct rnpvf_hw *hw, u16 max_size); +#endif /* __RNP_VF_H__ */