X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fi40e%2Fi40e_ethdev.c;h=56bd2c6c36f0b64ab95b63ce34821f6898528f64;hb=c830cb295411fc71c3f3d3591308d531937dd58f;hp=ce4efb257116d47120294b6613a8c8a8e71982d2;hpb=c8b9a3e3fe1bf161003b57af6a5bce17daf37a4e;p=dpdk.git diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c index ce4efb2571..56bd2c6c36 100644 --- a/drivers/net/i40e/i40e_ethdev.c +++ b/drivers/net/i40e/i40e_ethdev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,7 +31,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include #include #include #include @@ -39,6 +38,7 @@ #include #include #include +#include #include #include @@ -50,6 +50,7 @@ #include #include #include +#include #include "i40e_logs.h" #include "base/i40e_prototype.h" @@ -60,9 +61,11 @@ #include "i40e_ethdev.h" #include "i40e_rxtx.h" #include "i40e_pf.h" +#include "i40e_regs.h" + +#define ETH_I40E_FLOATING_VEB_ARG "enable_floating_veb" +#define ETH_I40E_FLOATING_VEB_LIST_ARG "floating_veb_list" -/* Maximun number of MAC addresses */ -#define I40E_NUM_MACADDR_MAX 64 #define I40E_CLEAR_PXE_WAIT_MS 200 /* Maximun number of capability elements */ @@ -75,11 +78,6 @@ /* Maximun number of VSI */ #define I40E_MAX_NUM_VSIS (384UL) -/* Default queue interrupt throttling time in microseconds */ -#define I40E_ITR_INDEX_DEFAULT 0 -#define I40E_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */ -#define I40E_QUEUE_ITR_INTERVAL_MAX 8160 /* 8160 us */ - #define I40E_PRE_TX_Q_CFG_WAIT_US 10 /* 10 us */ /* Flow control default timer */ @@ -129,16 +127,172 @@ (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \ (1UL << RTE_ETH_FLOW_L2_PAYLOAD)) -#define I40E_PTP_40GB_INCVAL 0x0199999999ULL -#define I40E_PTP_10GB_INCVAL 0x0333333333ULL -#define I40E_PTP_1GB_INCVAL 0x2000000000ULL -#define I40E_PRTTSYN_TSYNENA 0x80000000 -#define I40E_PRTTSYN_TSYNTYPE 0x0e000000 +/* Additional timesync values. */ +#define I40E_PTP_40GB_INCVAL 0x0199999999ULL +#define I40E_PTP_10GB_INCVAL 0x0333333333ULL +#define I40E_PTP_1GB_INCVAL 0x2000000000ULL +#define I40E_PRTTSYN_TSYNENA 0x80000000 +#define I40E_PRTTSYN_TSYNTYPE 0x0e000000 +#define I40E_CYCLECOUNTER_MASK 0xffffffffffffffffULL #define I40E_MAX_PERCENT 100 #define I40E_DEFAULT_DCB_APP_NUM 1 #define I40E_DEFAULT_DCB_APP_PRIO 3 +#define I40E_INSET_NONE 0x00000000000000000ULL + +/* bit0 ~ bit 7 */ +#define I40E_INSET_DMAC 0x0000000000000001ULL +#define I40E_INSET_SMAC 0x0000000000000002ULL +#define I40E_INSET_VLAN_OUTER 0x0000000000000004ULL +#define I40E_INSET_VLAN_INNER 0x0000000000000008ULL +#define I40E_INSET_VLAN_TUNNEL 0x0000000000000010ULL + +/* bit 8 ~ bit 15 */ +#define I40E_INSET_IPV4_SRC 0x0000000000000100ULL +#define I40E_INSET_IPV4_DST 0x0000000000000200ULL +#define I40E_INSET_IPV6_SRC 0x0000000000000400ULL +#define I40E_INSET_IPV6_DST 0x0000000000000800ULL +#define I40E_INSET_SRC_PORT 0x0000000000001000ULL +#define I40E_INSET_DST_PORT 0x0000000000002000ULL +#define I40E_INSET_SCTP_VT 0x0000000000004000ULL + +/* bit 16 ~ bit 31 */ +#define I40E_INSET_IPV4_TOS 0x0000000000010000ULL +#define I40E_INSET_IPV4_PROTO 0x0000000000020000ULL +#define I40E_INSET_IPV4_TTL 0x0000000000040000ULL +#define I40E_INSET_IPV6_TC 0x0000000000080000ULL +#define I40E_INSET_IPV6_FLOW 0x0000000000100000ULL +#define I40E_INSET_IPV6_NEXT_HDR 0x0000000000200000ULL +#define I40E_INSET_IPV6_HOP_LIMIT 0x0000000000400000ULL +#define I40E_INSET_TCP_FLAGS 0x0000000000800000ULL + +/* bit 32 ~ bit 47, tunnel fields */ +#define I40E_INSET_TUNNEL_IPV4_DST 0x0000000100000000ULL +#define I40E_INSET_TUNNEL_IPV6_DST 0x0000000200000000ULL +#define I40E_INSET_TUNNEL_DMAC 0x0000000400000000ULL +#define I40E_INSET_TUNNEL_SRC_PORT 0x0000000800000000ULL +#define I40E_INSET_TUNNEL_DST_PORT 0x0000001000000000ULL +#define I40E_INSET_TUNNEL_ID 0x0000002000000000ULL + +/* bit 48 ~ bit 55 */ +#define I40E_INSET_LAST_ETHER_TYPE 0x0001000000000000ULL + +/* bit 56 ~ bit 63, Flex Payload */ +#define I40E_INSET_FLEX_PAYLOAD_W1 0x0100000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W2 0x0200000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W3 0x0400000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W4 0x0800000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W5 0x1000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W6 0x2000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W7 0x4000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD_W8 0x8000000000000000ULL +#define I40E_INSET_FLEX_PAYLOAD \ + (I40E_INSET_FLEX_PAYLOAD_W1 | I40E_INSET_FLEX_PAYLOAD_W2 | \ + I40E_INSET_FLEX_PAYLOAD_W3 | I40E_INSET_FLEX_PAYLOAD_W4 | \ + I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \ + I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8) + +/** + * Below are values for writing un-exposed registers suggested + * by silicon experts + */ +/* Destination MAC address */ +#define I40E_REG_INSET_L2_DMAC 0xE000000000000000ULL +/* Source MAC address */ +#define I40E_REG_INSET_L2_SMAC 0x1C00000000000000ULL +/* Outer (S-Tag) VLAN tag in the outer L2 header */ +#define I40E_REG_INSET_L2_OUTER_VLAN 0x0200000000000000ULL +/* Inner (C-Tag) or single VLAN tag in the outer L2 header */ +#define I40E_REG_INSET_L2_INNER_VLAN 0x0080000000000000ULL +/* Single VLAN tag in the inner L2 header */ +#define I40E_REG_INSET_TUNNEL_VLAN 0x0100000000000000ULL +/* Source IPv4 address */ +#define I40E_REG_INSET_L3_SRC_IP4 0x0001800000000000ULL +/* Destination IPv4 address */ +#define I40E_REG_INSET_L3_DST_IP4 0x0000001800000000ULL +/* IPv4 Type of Service (TOS) */ +#define I40E_REG_INSET_L3_IP4_TOS 0x0040000000000000ULL +/* IPv4 Protocol */ +#define I40E_REG_INSET_L3_IP4_PROTO 0x0004000000000000ULL +/* IPv4 Time to Live */ +#define I40E_REG_INSET_L3_IP4_TTL 0x0004000000000000ULL +/* Source IPv6 address */ +#define I40E_REG_INSET_L3_SRC_IP6 0x0007F80000000000ULL +/* Destination IPv6 address */ +#define I40E_REG_INSET_L3_DST_IP6 0x000007F800000000ULL +/* IPv6 Traffic Class (TC) */ +#define I40E_REG_INSET_L3_IP6_TC 0x0040000000000000ULL +/* IPv6 Next Header */ +#define I40E_REG_INSET_L3_IP6_NEXT_HDR 0x0008000000000000ULL +/* IPv6 Hop Limit */ +#define I40E_REG_INSET_L3_IP6_HOP_LIMIT 0x0008000000000000ULL +/* Source L4 port */ +#define I40E_REG_INSET_L4_SRC_PORT 0x0000000400000000ULL +/* Destination L4 port */ +#define I40E_REG_INSET_L4_DST_PORT 0x0000000200000000ULL +/* SCTP verification tag */ +#define I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG 0x0000000180000000ULL +/* Inner destination MAC address (MAC-in-UDP/MAC-in-GRE)*/ +#define I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC 0x0000000001C00000ULL +/* Source port of tunneling UDP */ +#define I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT 0x0000000000200000ULL +/* Destination port of tunneling UDP */ +#define I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT 0x0000000000100000ULL +/* UDP Tunneling ID, NVGRE/GRE key */ +#define I40E_REG_INSET_TUNNEL_ID 0x00000000000C0000ULL +/* Last ether type */ +#define I40E_REG_INSET_LAST_ETHER_TYPE 0x0000000000004000ULL +/* Tunneling outer destination IPv4 address */ +#define I40E_REG_INSET_TUNNEL_L3_DST_IP4 0x00000000000000C0ULL +/* Tunneling outer destination IPv6 address */ +#define I40E_REG_INSET_TUNNEL_L3_DST_IP6 0x0000000000003FC0ULL +/* 1st word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD1 0x0000000000002000ULL +/* 2nd word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD2 0x0000000000001000ULL +/* 3rd word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD3 0x0000000000000800ULL +/* 4th word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD4 0x0000000000000400ULL +/* 5th word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD5 0x0000000000000200ULL +/* 6th word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD6 0x0000000000000100ULL +/* 7th word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD7 0x0000000000000080ULL +/* 8th word of flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORD8 0x0000000000000040ULL +/* all 8 words flex payload */ +#define I40E_REG_INSET_FLEX_PAYLOAD_WORDS 0x0000000000003FC0ULL +#define I40E_REG_INSET_MASK_DEFAULT 0x0000000000000000ULL + +#define I40E_TRANSLATE_INSET 0 +#define I40E_TRANSLATE_REG 1 + +#define I40E_INSET_IPV4_TOS_MASK 0x0009FF00UL +#define I40E_INSET_IPv4_TTL_MASK 0x000D00FFUL +#define I40E_INSET_IPV4_PROTO_MASK 0x000DFF00UL +#define I40E_INSET_IPV6_TC_MASK 0x0009F00FUL +#define I40E_INSET_IPV6_HOP_LIMIT_MASK 0x000CFF00UL +#define I40E_INSET_IPV6_NEXT_HDR_MASK 0x000C00FFUL + +#define I40E_GL_SWT_L2TAGCTRL(_i) (0x001C0A70 + ((_i) * 4)) +#define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT 16 +#define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK \ + I40E_MASK(0xFFFF, I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT) + +/* PCI offset for querying capability */ +#define PCI_DEV_CAP_REG 0xA4 +/* PCI offset for enabling/disabling Extended Tag */ +#define PCI_DEV_CTRL_REG 0xA8 +/* Bit mask of Extended Tag capability */ +#define PCI_DEV_CAP_EXT_TAG_MASK 0x20 +/* Bit shift of Extended Tag enable/disable */ +#define PCI_DEV_CTRL_EXT_TAG_SHIFT 8 +/* Bit mask of Extended Tag enable/disable */ +#define PCI_DEV_CTRL_EXT_TAG_MASK (1 << PCI_DEV_CTRL_EXT_TAG_SHIFT) + static int eth_i40e_dev_init(struct rte_eth_dev *eth_dev); static int eth_i40e_dev_uninit(struct rte_eth_dev *eth_dev); static int i40e_dev_configure(struct rte_eth_dev *dev); @@ -153,6 +307,11 @@ static int i40e_dev_set_link_up(struct rte_eth_dev *dev); static int i40e_dev_set_link_down(struct rte_eth_dev *dev); static void i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); +static int i40e_dev_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *xstats, unsigned n); +static int i40e_dev_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned limit); static void i40e_dev_stats_reset(struct rte_eth_dev *dev); static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev, uint16_t queue_id, @@ -163,7 +322,9 @@ static void i40e_dev_info_get(struct rte_eth_dev *dev, static int i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); -static void i40e_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid); +static int i40e_vlan_tpid_set(struct rte_eth_dev *dev, + enum rte_vlan_type vlan_type, + uint16_t tpid); static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask); static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, @@ -204,7 +365,7 @@ static void i40e_stat_update_48(struct i40e_hw *hw, bool offset_loaded, uint64_t *offset, uint64_t *stat); -static void i40e_pf_config_irq0(struct i40e_hw *hw); +static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue); static void i40e_dev_interrupt_handler( __rte_unused struct rte_intr_handle *handle, void *param); static int i40e_res_pool_init(struct i40e_res_pool_info *pool, @@ -233,10 +394,11 @@ static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); -static int i40e_dev_udp_tunnel_add(struct rte_eth_dev *dev, - struct rte_eth_udp_tunnel *udp_tunnel); -static int i40e_dev_udp_tunnel_del(struct rte_eth_dev *dev, - struct rte_eth_udp_tunnel *udp_tunnel); +static int i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, + struct rte_eth_udp_tunnel *udp_tunnel); +static int i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, + struct rte_eth_udp_tunnel *udp_tunnel); +static void i40e_filter_input_set_init(struct i40e_pf *pf); static int i40e_ethertype_filter_set(struct i40e_pf *pf, struct rte_eth_ethertype_filter *filter, bool add); @@ -247,8 +409,10 @@ static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type filter_type, enum rte_filter_op filter_op, void *arg); +static int i40e_dev_get_dcb_info(struct rte_eth_dev *dev, + struct rte_eth_dcb_info *dcb_info); static void i40e_configure_registers(struct i40e_hw *hw); -static void i40e_hw_init(struct i40e_hw *hw); +static void i40e_hw_init(struct rte_eth_dev *dev); static int i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi); static int i40e_mirror_rule_set(struct rte_eth_dev *dev, struct rte_eth_mirror_conf *mirror_conf, @@ -262,11 +426,56 @@ static int i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev, uint32_t flags); static int i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev, struct timespec *timestamp); +static void i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw); + +static int i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta); + +static int i40e_timesync_read_time(struct rte_eth_dev *dev, + struct timespec *timestamp); +static int i40e_timesync_write_time(struct rte_eth_dev *dev, + const struct timespec *timestamp); + +static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, + uint16_t queue_id); +static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, + uint16_t queue_id); + +static int i40e_get_regs(struct rte_eth_dev *dev, + struct rte_dev_reg_info *regs); + +static int i40e_get_eeprom_length(struct rte_eth_dev *dev); + +static int i40e_get_eeprom(struct rte_eth_dev *dev, + struct rte_dev_eeprom_info *eeprom); + +static void i40e_set_default_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *mac_addr); + +static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static const struct rte_pci_id pci_id_i40e_map[] = { -#define RTE_PCI_DEV_ID_DECL_I40E(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, -#include "rte_pci_dev_ids.h" -{ .vendor_id = 0, /* sentinel */ }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QEMU) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2_A) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_B) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_SFP28) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722) }, + { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_I_X722) }, + { .vendor_id = 0, /* sentinel */ }, }; static const struct eth_dev_ops i40e_eth_dev_ops = { @@ -282,9 +491,13 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .dev_set_link_down = i40e_dev_set_link_down, .link_update = i40e_dev_link_update, .stats_get = i40e_dev_stats_get, + .xstats_get = i40e_dev_xstats_get, + .xstats_get_names = i40e_dev_xstats_get_names, .stats_reset = i40e_dev_stats_reset, + .xstats_reset = i40e_dev_stats_reset, .queue_stats_mapping_set = i40e_dev_queue_stats_mapping_set, .dev_infos_get = i40e_dev_info_get, + .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get, .vlan_filter_set = i40e_vlan_filter_set, .vlan_tpid_set = i40e_vlan_tpid_set, .vlan_offload_set = i40e_vlan_offload_set, @@ -295,6 +508,8 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .tx_queue_start = i40e_dev_tx_queue_start, .tx_queue_stop = i40e_dev_tx_queue_stop, .rx_queue_setup = i40e_dev_rx_queue_setup, + .rx_queue_intr_enable = i40e_dev_rx_queue_intr_enable, + .rx_queue_intr_disable = i40e_dev_rx_queue_intr_disable, .rx_queue_release = i40e_dev_rx_queue_release, .rx_queue_count = i40e_dev_rx_queue_count, .rx_descriptor_done = i40e_dev_rx_descriptor_done, @@ -311,23 +526,149 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .reta_query = i40e_dev_rss_reta_query, .rss_hash_update = i40e_dev_rss_hash_update, .rss_hash_conf_get = i40e_dev_rss_hash_conf_get, - .udp_tunnel_add = i40e_dev_udp_tunnel_add, - .udp_tunnel_del = i40e_dev_udp_tunnel_del, + .udp_tunnel_port_add = i40e_dev_udp_tunnel_port_add, + .udp_tunnel_port_del = i40e_dev_udp_tunnel_port_del, .filter_ctrl = i40e_dev_filter_ctrl, + .rxq_info_get = i40e_rxq_info_get, + .txq_info_get = i40e_txq_info_get, .mirror_rule_set = i40e_mirror_rule_set, .mirror_rule_reset = i40e_mirror_rule_reset, .timesync_enable = i40e_timesync_enable, .timesync_disable = i40e_timesync_disable, .timesync_read_rx_timestamp = i40e_timesync_read_rx_timestamp, .timesync_read_tx_timestamp = i40e_timesync_read_tx_timestamp, + .get_dcb_info = i40e_dev_get_dcb_info, + .timesync_adjust_time = i40e_timesync_adjust_time, + .timesync_read_time = i40e_timesync_read_time, + .timesync_write_time = i40e_timesync_write_time, + .get_reg = i40e_get_regs, + .get_eeprom_length = i40e_get_eeprom_length, + .get_eeprom = i40e_get_eeprom, + .mac_addr_set = i40e_set_default_mac_addr, + .mtu_set = i40e_dev_mtu_set, +}; + +/* store statistics names and its offset in stats structure */ +struct rte_i40e_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + unsigned offset; }; +static const struct rte_i40e_xstats_name_off rte_i40e_stats_strings[] = { + {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)}, + {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)}, + {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)}, + {"rx_dropped", offsetof(struct i40e_eth_stats, rx_discards)}, + {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats, + rx_unknown_protocol)}, + {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)}, + {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)}, + {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)}, + {"tx_dropped", offsetof(struct i40e_eth_stats, tx_discards)}, +}; + +#define I40E_NB_ETH_XSTATS (sizeof(rte_i40e_stats_strings) / \ + sizeof(rte_i40e_stats_strings[0])) + +static const struct rte_i40e_xstats_name_off rte_i40e_hw_port_strings[] = { + {"tx_link_down_dropped", offsetof(struct i40e_hw_port_stats, + tx_dropped_link_down)}, + {"rx_crc_errors", offsetof(struct i40e_hw_port_stats, crc_errors)}, + {"rx_illegal_byte_errors", offsetof(struct i40e_hw_port_stats, + illegal_bytes)}, + {"rx_error_bytes", offsetof(struct i40e_hw_port_stats, error_bytes)}, + {"mac_local_errors", offsetof(struct i40e_hw_port_stats, + mac_local_faults)}, + {"mac_remote_errors", offsetof(struct i40e_hw_port_stats, + mac_remote_faults)}, + {"rx_length_errors", offsetof(struct i40e_hw_port_stats, + rx_length_errors)}, + {"tx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_tx)}, + {"rx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_rx)}, + {"tx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_tx)}, + {"rx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_rx)}, + {"rx_size_64_packets", offsetof(struct i40e_hw_port_stats, rx_size_64)}, + {"rx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats, + rx_size_127)}, + {"rx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats, + rx_size_255)}, + {"rx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats, + rx_size_511)}, + {"rx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats, + rx_size_1023)}, + {"rx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats, + rx_size_1522)}, + {"rx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats, + rx_size_big)}, + {"rx_undersized_errors", offsetof(struct i40e_hw_port_stats, + rx_undersize)}, + {"rx_oversize_errors", offsetof(struct i40e_hw_port_stats, + rx_oversize)}, + {"rx_mac_short_dropped", offsetof(struct i40e_hw_port_stats, + mac_short_packet_dropped)}, + {"rx_fragmented_errors", offsetof(struct i40e_hw_port_stats, + rx_fragments)}, + {"rx_jabber_errors", offsetof(struct i40e_hw_port_stats, rx_jabber)}, + {"tx_size_64_packets", offsetof(struct i40e_hw_port_stats, tx_size_64)}, + {"tx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats, + tx_size_127)}, + {"tx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats, + tx_size_255)}, + {"tx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats, + tx_size_511)}, + {"tx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats, + tx_size_1023)}, + {"tx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats, + tx_size_1522)}, + {"tx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats, + tx_size_big)}, + {"rx_flow_director_atr_match_packets", + offsetof(struct i40e_hw_port_stats, fd_atr_match)}, + {"rx_flow_director_sb_match_packets", + offsetof(struct i40e_hw_port_stats, fd_sb_match)}, + {"tx_low_power_idle_status", offsetof(struct i40e_hw_port_stats, + tx_lpi_status)}, + {"rx_low_power_idle_status", offsetof(struct i40e_hw_port_stats, + rx_lpi_status)}, + {"tx_low_power_idle_count", offsetof(struct i40e_hw_port_stats, + tx_lpi_count)}, + {"rx_low_power_idle_count", offsetof(struct i40e_hw_port_stats, + rx_lpi_count)}, +}; + +#define I40E_NB_HW_PORT_XSTATS (sizeof(rte_i40e_hw_port_strings) / \ + sizeof(rte_i40e_hw_port_strings[0])) + +static const struct rte_i40e_xstats_name_off rte_i40e_rxq_prio_strings[] = { + {"xon_packets", offsetof(struct i40e_hw_port_stats, + priority_xon_rx)}, + {"xoff_packets", offsetof(struct i40e_hw_port_stats, + priority_xoff_rx)}, +}; + +#define I40E_NB_RXQ_PRIO_XSTATS (sizeof(rte_i40e_rxq_prio_strings) / \ + sizeof(rte_i40e_rxq_prio_strings[0])) + +static const struct rte_i40e_xstats_name_off rte_i40e_txq_prio_strings[] = { + {"xon_packets", offsetof(struct i40e_hw_port_stats, + priority_xon_tx)}, + {"xoff_packets", offsetof(struct i40e_hw_port_stats, + priority_xoff_tx)}, + {"xon_to_xoff_packets", offsetof(struct i40e_hw_port_stats, + priority_xon_2_xoff)}, +}; + +#define I40E_NB_TXQ_PRIO_XSTATS (sizeof(rte_i40e_txq_prio_strings) / \ + sizeof(rte_i40e_txq_prio_strings[0])) + static struct eth_driver rte_i40e_pmd = { .pci_drv = { .name = "rte_i40e_pmd", .id_table = pci_id_i40e_map, .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_DETACHABLE, + .probe = rte_eth_dev_pci_probe, + .remove = rte_eth_dev_pci_remove, }, .eth_dev_init = eth_i40e_dev_init, .eth_dev_uninit = eth_i40e_dev_uninit, @@ -362,27 +703,8 @@ rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev, return 0; } -/* - * Driver initialization routine. - * Invoked once at EAL init time. - * Register itself as the [Poll Mode] Driver of PCI IXGBE devices. - */ -static int -rte_i40e_pmd_init(const char *name __rte_unused, - const char *params __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); - rte_eth_driver_register(&rte_i40e_pmd); - - return 0; -} - -static struct rte_driver rte_i40e_driver = { - .type = PMD_PDEV, - .init = rte_i40e_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_i40e_driver); +DRIVER_REGISTER_PCI(net_i40e, rte_i40e_pmd.pci_drv); +DRIVER_REGISTER_PCI_TABLE(net_i40e, pci_id_i40e_map); /* * Initialize registers for flexible payload, which should be set by NVM. @@ -437,6 +759,161 @@ i40e_add_tx_flow_control_drop_filter(struct i40e_pf *pf) " frames from VSIs."); } +static int +floating_veb_list_handler(__rte_unused const char *key, + const char *floating_veb_value, + void *opaque) +{ + int idx = 0; + unsigned int count = 0; + char *end = NULL; + int min, max; + bool *vf_floating_veb = opaque; + + while (isblank(*floating_veb_value)) + floating_veb_value++; + + /* Reset floating VEB configuration for VFs */ + for (idx = 0; idx < I40E_MAX_VF; idx++) + vf_floating_veb[idx] = false; + + min = I40E_MAX_VF; + do { + while (isblank(*floating_veb_value)) + floating_veb_value++; + if (*floating_veb_value == '\0') + return -1; + errno = 0; + idx = strtoul(floating_veb_value, &end, 10); + if (errno || end == NULL) + return -1; + while (isblank(*end)) + end++; + if (*end == '-') { + min = idx; + } else if ((*end == ';') || (*end == '\0')) { + max = idx; + if (min == I40E_MAX_VF) + min = idx; + if (max >= I40E_MAX_VF) + max = I40E_MAX_VF - 1; + for (idx = min; idx <= max; idx++) { + vf_floating_veb[idx] = true; + count++; + } + min = I40E_MAX_VF; + } else { + return -1; + } + floating_veb_value = end + 1; + } while (*end != '\0'); + + if (count == 0) + return -1; + + return 0; +} + +static void +config_vf_floating_veb(struct rte_devargs *devargs, + uint16_t floating_veb, + bool *vf_floating_veb) +{ + struct rte_kvargs *kvlist; + int i; + const char *floating_veb_list = ETH_I40E_FLOATING_VEB_LIST_ARG; + + if (!floating_veb) + return; + /* All the VFs attach to the floating VEB by default + * when the floating VEB is enabled. + */ + for (i = 0; i < I40E_MAX_VF; i++) + vf_floating_veb[i] = true; + + if (devargs == NULL) + return; + + kvlist = rte_kvargs_parse(devargs->args, NULL); + if (kvlist == NULL) + return; + + if (!rte_kvargs_count(kvlist, floating_veb_list)) { + rte_kvargs_free(kvlist); + return; + } + /* When the floating_veb_list parameter exists, all the VFs + * will attach to the legacy VEB firstly, then configure VFs + * to the floating VEB according to the floating_veb_list. + */ + if (rte_kvargs_process(kvlist, floating_veb_list, + floating_veb_list_handler, + vf_floating_veb) < 0) { + rte_kvargs_free(kvlist); + return; + } + rte_kvargs_free(kvlist); +} + +static int +i40e_check_floating_handler(__rte_unused const char *key, + const char *value, + __rte_unused void *opaque) +{ + if (strcmp(value, "1")) + return -1; + + return 0; +} + +static int +is_floating_veb_supported(struct rte_devargs *devargs) +{ + struct rte_kvargs *kvlist; + const char *floating_veb_key = ETH_I40E_FLOATING_VEB_ARG; + + if (devargs == NULL) + return 0; + + kvlist = rte_kvargs_parse(devargs->args, NULL); + if (kvlist == NULL) + return 0; + + if (!rte_kvargs_count(kvlist, floating_veb_key)) { + rte_kvargs_free(kvlist); + return 0; + } + /* Floating VEB is enabled when there's key-value: + * enable_floating_veb=1 + */ + if (rte_kvargs_process(kvlist, floating_veb_key, + i40e_check_floating_handler, NULL) < 0) { + rte_kvargs_free(kvlist); + return 0; + } + rte_kvargs_free(kvlist); + + return 1; +} + +static void +config_floating_veb(struct rte_eth_dev *dev) +{ + struct rte_pci_device *pci_dev = dev->pci_dev; + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + memset(pf->floating_veb_list, 0, sizeof(pf->floating_veb_list)); + + if (hw->aq.fw_maj_ver >= FLOATING_VEB_SUPPORTED_FW_MAJ) { + pf->floating_veb = is_floating_veb_supported(pci_dev->devargs); + config_vf_floating_veb(pci_dev->devargs, pf->floating_veb, + pf->floating_veb_list); + } else { + pf->floating_veb = false; + } +} + static int eth_i40e_dev_init(struct rte_eth_dev *dev) { @@ -463,6 +940,9 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) return 0; } pci_dev = dev->pci_dev; + + rte_eth_copy_pci_info(dev, pci_dev); + pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); pf->adapter->eth_dev = dev; pf->dev_data = dev->data; @@ -487,7 +967,7 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) i40e_clear_hw(hw); /* Initialize the hardware */ - i40e_hw_init(hw); + i40e_hw_init(dev); /* Reset here to make sure all is clean for each PF */ ret = i40e_pf_reset(hw); @@ -510,6 +990,9 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) */ i40e_flex_payload_reg_init(hw); + /* Initialize the input set for filters (hash and fd) to default value */ + i40e_filter_input_set_init(pf); + /* Initialize the parameters for adminq */ i40e_init_adminq_parameter(hw); ret = i40e_init_adminq(hw); @@ -524,6 +1007,8 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) ((hw->nvm.version >> 4) & 0xff), (hw->nvm.version & 0xf), hw->nvm.eetrack); + /* Need the special FW version to support floating VEB */ + config_floating_veb(dev); /* Clear PXE mode */ i40e_clear_pxe_mode(hw); @@ -594,6 +1079,14 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) hw->fc.requested_mode = I40E_FC_NONE; i40e_set_fc(hw, &aq_fail, TRUE); + /* Set the global registers with default ether type value */ + ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, ETHER_TYPE_VLAN); + if (ret != I40E_SUCCESS) { + PMD_INIT_LOG(ERR, "Failed to set the default outer " + "VLAN ether type"); + goto err_setup_pf_switch; + } + /* PF setup, which includes VSI setup */ ret = i40e_pf_setup(pf); if (ret) { @@ -601,6 +1094,9 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) goto err_setup_pf_switch; } + /* reset all stats of the device, including pf and main vsi */ + i40e_dev_stats_reset(dev); + vsi = pf->main_vsi; /* Disable double vlan by default */ @@ -629,7 +1125,7 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) i40e_dev_interrupt_handler, (void *)dev); /* configure and enable device interrupt */ - i40e_pf_config_irq0(hw); + i40e_pf_config_irq0(hw, TRUE); i40e_pf_enable_irq0(hw); /* enable uio intr after callback register */ @@ -641,6 +1137,11 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) */ i40e_add_tx_flow_control_drop_filter(pf); + /* Set the max frame size to 0x2600 by default, + * in case other drivers changed the default value. + */ + i40e_aq_set_mac_config(hw, I40E_FRAME_SIZE_MAX, TRUE, 0, NULL); + /* initialize mirror rule list */ TAILQ_INIT(&pf->mirror_list); @@ -810,6 +1311,8 @@ err: void i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi) { + struct rte_eth_dev *dev = vsi->adapter->eth_dev; + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t msix_vect = vsi->msix_intr; uint16_t i; @@ -821,52 +1324,50 @@ i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi) } if (vsi->type != I40E_VSI_SRIOV) { - I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1), 0); - I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT, - msix_vect - 1), 0); + if (!rte_intr_allow_others(intr_handle)) { + I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, + I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK); + I40E_WRITE_REG(hw, + I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT), + 0); + } else { + I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1), + I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK); + I40E_WRITE_REG(hw, + I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT, + msix_vect - 1), 0); + } } else { uint32_t reg; reg = (hw->func_caps.num_msix_vectors_vf - 1) * vsi->user_param + (msix_vect - 1); - I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), 0); + I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), + I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); } I40E_WRITE_FLUSH(hw); } -static inline uint16_t -i40e_calc_itr_interval(int16_t interval) -{ - if (interval < 0 || interval > I40E_QUEUE_ITR_INTERVAL_MAX) - interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT; - - /* Convert to hardware count, as writing each 1 represents 2 us */ - return (interval/2); -} - -void -i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) +static void +__vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect, + int base_queue, int nb_queue) { + int i; uint32_t val; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); - uint16_t msix_vect = vsi->msix_intr; - int i; - - for (i = 0; i < vsi->nb_qps; i++) - I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0); /* Bind all RX queues to allocated MSIX interrupt */ - for (i = 0; i < vsi->nb_qps; i++) { + for (i = 0; i < nb_queue; i++) { val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | I40E_QINT_RQCTL_ITR_INDX_MASK | - ((vsi->base_queue + i + 1) << - I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + ((base_queue + i + 1) << + I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | I40E_QINT_RQCTL_CAUSE_ENA_MASK; - if (i == vsi->nb_qps - 1) + if (i == nb_queue - 1) val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK; - I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), val); + I40E_WRITE_REG(hw, I40E_QINT_RQCTL(base_queue + i), val); } /* Write first RX queue to Link list register as the head element */ @@ -874,91 +1375,203 @@ i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) uint16_t interval = i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL); - I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1), - (vsi->base_queue << - I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | - (0x0 << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); - - I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT, - msix_vect - 1), interval); - -#ifndef I40E_GLINT_CTL -#define I40E_GLINT_CTL 0x0003F800 -#define I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK 0x4 -#endif - /* Disable auto-mask on enabling of all none-zero interrupt */ - I40E_WRITE_REG(hw, I40E_GLINT_CTL, - I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK); + if (msix_vect == I40E_MISC_VEC_ID) { + I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, + (base_queue << + I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | + (0x0 << + I40E_PFINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); + I40E_WRITE_REG(hw, + I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT), + interval); + } else { + I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1), + (base_queue << + I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | + (0x0 << + I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); + I40E_WRITE_REG(hw, + I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT, + msix_vect - 1), + interval); + } } else { uint32_t reg; - /* num_msix_vectors_vf needs to minus irq0 */ - reg = (hw->func_caps.num_msix_vectors_vf - 1) * - vsi->user_param + (msix_vect - 1); + if (msix_vect == I40E_MISC_VEC_ID) { + I40E_WRITE_REG(hw, + I40E_VPINT_LNKLST0(vsi->user_param), + (base_queue << + I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | + (0x0 << + I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); + } else { + /* num_msix_vectors_vf needs to minus irq0 */ + reg = (hw->func_caps.num_msix_vectors_vf - 1) * + vsi->user_param + (msix_vect - 1); - I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), (vsi->base_queue << + I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), + (base_queue << I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | - (0x0 << I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); + (0x0 << + I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)); + } } I40E_WRITE_FLUSH(hw); } +void +i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi) +{ + struct rte_eth_dev *dev = vsi->adapter->eth_dev; + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + uint16_t msix_vect = vsi->msix_intr; + uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd); + uint16_t queue_idx = 0; + int record = 0; + uint32_t val; + int i; + + for (i = 0; i < vsi->nb_qps; i++) { + I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0); + I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0); + } + + /* INTENA flag is not auto-cleared for interrupt */ + val = I40E_READ_REG(hw, I40E_GLINT_CTL); + val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK | + I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK | + I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK; + I40E_WRITE_REG(hw, I40E_GLINT_CTL, val); + + /* VF bind interrupt */ + if (vsi->type == I40E_VSI_SRIOV) { + __vsi_queues_bind_intr(vsi, msix_vect, + vsi->base_queue, vsi->nb_qps); + return; + } + + /* PF & VMDq bind interrupt */ + if (rte_intr_dp_is_en(intr_handle)) { + if (vsi->type == I40E_VSI_MAIN) { + queue_idx = 0; + record = 1; + } else if (vsi->type == I40E_VSI_VMDQ2) { + struct i40e_vsi *main_vsi = + I40E_DEV_PRIVATE_TO_MAIN_VSI(vsi->adapter); + queue_idx = vsi->base_queue - main_vsi->nb_qps; + record = 1; + } + } + + for (i = 0; i < vsi->nb_used_qps; i++) { + if (nb_msix <= 1) { + if (!rte_intr_allow_others(intr_handle)) + /* allow to share MISC_VEC_ID */ + msix_vect = I40E_MISC_VEC_ID; + + /* no enough msix_vect, map all to one */ + __vsi_queues_bind_intr(vsi, msix_vect, + vsi->base_queue + i, + vsi->nb_used_qps - i); + for (; !!record && i < vsi->nb_used_qps; i++) + intr_handle->intr_vec[queue_idx + i] = + msix_vect; + break; + } + /* 1:1 queue/msix_vect mapping */ + __vsi_queues_bind_intr(vsi, msix_vect, + vsi->base_queue + i, 1); + if (!!record) + intr_handle->intr_vec[queue_idx + i] = msix_vect; + + msix_vect++; + nb_msix--; + } +} + static void i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi) { + struct rte_eth_dev *dev = vsi->adapter->eth_dev; + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); uint16_t interval = i40e_calc_itr_interval(\ - RTE_LIBRTE_I40E_ITR_INTERVAL); - - I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1), - I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | + RTE_LIBRTE_I40E_ITR_INTERVAL); + uint16_t msix_intr, i; + + if (rte_intr_allow_others(intr_handle)) + for (i = 0; i < vsi->nb_msix; i++) { + msix_intr = vsi->msix_intr + i; + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1), + I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | - (interval << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT)); + (interval << + I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT)); + } + else + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + (0 << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT) | + (interval << + I40E_PFINT_DYN_CTL0_INTERVAL_SHIFT)); + + I40E_WRITE_FLUSH(hw); } static void i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi) { + struct rte_eth_dev *dev = vsi->adapter->eth_dev; + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + uint16_t msix_intr, i; + + if (rte_intr_allow_others(intr_handle)) + for (i = 0; i < vsi->nb_msix; i++) { + msix_intr = vsi->msix_intr + i; + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1), + 0); + } + else + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0); - I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1), 0); + I40E_WRITE_FLUSH(hw); } static inline uint8_t -i40e_parse_link_speed(uint16_t eth_link_speed) +i40e_parse_link_speeds(uint16_t link_speeds) { uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN; - switch (eth_link_speed) { - case ETH_LINK_SPEED_40G: - link_speed = I40E_LINK_SPEED_40GB; - break; - case ETH_LINK_SPEED_20G: - link_speed = I40E_LINK_SPEED_20GB; - break; - case ETH_LINK_SPEED_10G: - link_speed = I40E_LINK_SPEED_10GB; - break; - case ETH_LINK_SPEED_1000: - link_speed = I40E_LINK_SPEED_1GB; - break; - case ETH_LINK_SPEED_100: - link_speed = I40E_LINK_SPEED_100MB; - break; - } + if (link_speeds & ETH_LINK_SPEED_40G) + link_speed |= I40E_LINK_SPEED_40GB; + if (link_speeds & ETH_LINK_SPEED_20G) + link_speed |= I40E_LINK_SPEED_20GB; + if (link_speeds & ETH_LINK_SPEED_10G) + link_speed |= I40E_LINK_SPEED_10GB; + if (link_speeds & ETH_LINK_SPEED_1G) + link_speed |= I40E_LINK_SPEED_1GB; + if (link_speeds & ETH_LINK_SPEED_100M) + link_speed |= I40E_LINK_SPEED_100MB; return link_speed; } static int -i40e_phy_conf_link(struct i40e_hw *hw, uint8_t abilities, uint8_t force_speed) +i40e_phy_conf_link(struct i40e_hw *hw, + uint8_t abilities, + uint8_t force_speed) { enum i40e_status_code status; struct i40e_aq_get_phy_abilities_resp phy_ab; struct i40e_aq_set_phy_config phy_conf; const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX | + I40E_AQ_PHY_FLAG_PAUSE_RX | I40E_AQ_PHY_FLAG_PAUSE_RX | I40E_AQ_PHY_FLAG_LOW_POWER; const uint8_t advt = I40E_LINK_SPEED_40GB | @@ -967,9 +1580,6 @@ i40e_phy_conf_link(struct i40e_hw *hw, uint8_t abilities, uint8_t force_speed) I40E_LINK_SPEED_100MB; int ret = -ENOTSUP; - /* Skip it on 40G interfaces, as a workaround for the link issue */ - if (i40e_is_40G_device(hw->device_id)) - return I40E_SUCCESS; status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab, NULL); @@ -1016,12 +1626,17 @@ i40e_apply_link_speed(struct rte_eth_dev *dev) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_conf *conf = &dev->data->dev_conf; - speed = i40e_parse_link_speed(conf->link_speed); + speed = i40e_parse_link_speeds(conf->link_speeds); abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK; - if (conf->link_speed == ETH_LINK_SPEED_AUTONEG) + if (!(conf->link_speeds & ETH_LINK_SPEED_FIXED)) abilities |= I40E_AQ_PHY_AN_ENABLED; - else - abilities |= I40E_AQ_PHY_LINK_ENABLED; + abilities |= I40E_AQ_PHY_LINK_ENABLED; + + /* Skip changing speed on 40G interfaces, FW does not support */ + if (i40e_is_40G_device(hw->device_id)) { + speed = I40E_LINK_SPEED_UNKNOWN; + abilities |= I40E_AQ_PHY_AN_ENABLED; + } return i40e_phy_conf_link(hw, abilities, speed); } @@ -1033,17 +1648,39 @@ i40e_dev_start(struct rte_eth_dev *dev) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct i40e_vsi *main_vsi = pf->main_vsi; int ret, i; + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + uint32_t intr_vector = 0; hw->adapter_stopped = 0; - if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) && - (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) { - PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu", - dev->data->dev_conf.link_duplex, + if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) { + PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; autonegotiation disabled", dev->data->port_id); return -EINVAL; } + rte_intr_disable(intr_handle); + + if ((rte_intr_cap_multiple(intr_handle) || + !RTE_ETH_DEV_SRIOV(dev).active) && + dev->data->dev_conf.intr_conf.rxq != 0) { + intr_vector = dev->data->nb_rx_queues; + if (rte_intr_efd_enable(intr_handle, intr_vector)) + return -1; + } + + if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { + intr_handle->intr_vec = + rte_zmalloc("intr_vec", + dev->data->nb_rx_queues * sizeof(int), + 0); + if (!intr_handle->intr_vec) { + PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" + " intr_vec\n", dev->data->nb_rx_queues); + return -ENOMEM; + } + } + /* Initialize VSI */ ret = i40e_dev_rxtx_init(pf); if (ret != I40E_SUCCESS) { @@ -1052,11 +1689,14 @@ i40e_dev_start(struct rte_eth_dev *dev) } /* Map queues with MSIX interrupt */ + main_vsi->nb_used_qps = dev->data->nb_rx_queues - + pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; i40e_vsi_queues_bind_intr(main_vsi); i40e_vsi_enable_queues_intr(main_vsi); /* Map VMDQ VSI queues with MSIX interrupt */ for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) { + pf->vmdq[i].vsi->nb_used_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi); i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi); } @@ -1087,12 +1727,34 @@ i40e_dev_start(struct rte_eth_dev *dev) } /* Apply link configure */ + if (dev->data->dev_conf.link_speeds & ~(ETH_LINK_SPEED_100M | + ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | + ETH_LINK_SPEED_20G | ETH_LINK_SPEED_40G)) { + PMD_DRV_LOG(ERR, "Invalid link setting"); + goto err_up; + } ret = i40e_apply_link_speed(dev); if (I40E_SUCCESS != ret) { PMD_DRV_LOG(ERR, "Fail to apply link setting"); goto err_up; } + if (!rte_intr_allow_others(intr_handle)) { + rte_intr_callback_unregister(intr_handle, + i40e_dev_interrupt_handler, + (void *)dev); + /* configure and enable device interrupt */ + i40e_pf_config_irq0(hw, FALSE); + i40e_pf_enable_irq0(hw); + + if (dev->data->dev_conf.intr_conf.lsc != 0) + PMD_INIT_LOG(INFO, "lsc won't enable because of" + " no intr multiplex\n"); + } + + /* enable uio intr after callback register */ + rte_intr_enable(intr_handle); + return I40E_SUCCESS; err_up: @@ -1108,6 +1770,7 @@ i40e_dev_stop(struct rte_eth_dev *dev) struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_vsi *main_vsi = pf->main_vsi; struct i40e_mirror_rule *p_mirror; + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; int i; /* Disable all queues */ @@ -1123,8 +1786,8 @@ i40e_dev_stop(struct rte_eth_dev *dev) } if (pf->fdir.fdir_vsi) { - i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi); - i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi); + i40e_vsi_queues_unbind_intr(pf->fdir.fdir_vsi); + i40e_vsi_disable_queues_intr(pf->fdir.fdir_vsi); } /* Clear all queues and release memory */ i40e_dev_clear_queues(dev); @@ -1139,6 +1802,18 @@ i40e_dev_stop(struct rte_eth_dev *dev) } pf->nb_mirror_rule = 0; + if (!rte_intr_allow_others(intr_handle)) + /* resume to the default handler */ + rte_intr_callback_register(intr_handle, + i40e_dev_interrupt_handler, + (void *)dev); + + /* Clean datapath event and queue/vec mapping */ + rte_intr_efd_disable(intr_handle); + if (intr_handle->intr_vec) { + rte_free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; + } } static void @@ -1197,7 +1872,7 @@ i40e_dev_promiscuous_enable(struct rte_eth_dev *dev) int status; status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid, - true, NULL); + true, NULL, true); if (status != I40E_SUCCESS) PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous"); @@ -1217,7 +1892,7 @@ i40e_dev_promiscuous_disable(struct rte_eth_dev *dev) int status; status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid, - false, NULL); + false, NULL, true); if (status != I40E_SUCCESS) PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous"); @@ -1271,7 +1946,7 @@ i40e_dev_set_link_up(struct rte_eth_dev *dev) * Set device link down. */ static int -i40e_dev_set_link_down(__rte_unused struct rte_eth_dev *dev) +i40e_dev_set_link_down(struct rte_eth_dev *dev) { uint8_t speed = I40E_LINK_SPEED_UNKNOWN; uint8_t abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK; @@ -1301,7 +1976,7 @@ i40e_dev_link_update(struct rte_eth_dev *dev, /* Get link status information from hardware */ status = i40e_aq_get_link_info(hw, false, &link_status, NULL); if (status != I40E_SUCCESS) { - link.link_speed = ETH_LINK_SPEED_100; + link.link_speed = ETH_SPEED_NUM_100M; link.link_duplex = ETH_LINK_FULL_DUPLEX; PMD_DRV_LOG(ERR, "Failed to get link info"); goto out; @@ -1323,25 +1998,28 @@ i40e_dev_link_update(struct rte_eth_dev *dev, /* Parse the link status */ switch (link_status.link_speed) { case I40E_LINK_SPEED_100MB: - link.link_speed = ETH_LINK_SPEED_100; + link.link_speed = ETH_SPEED_NUM_100M; break; case I40E_LINK_SPEED_1GB: - link.link_speed = ETH_LINK_SPEED_1000; + link.link_speed = ETH_SPEED_NUM_1G; break; case I40E_LINK_SPEED_10GB: - link.link_speed = ETH_LINK_SPEED_10G; + link.link_speed = ETH_SPEED_NUM_10G; break; case I40E_LINK_SPEED_20GB: - link.link_speed = ETH_LINK_SPEED_20G; + link.link_speed = ETH_SPEED_NUM_20G; break; case I40E_LINK_SPEED_40GB: - link.link_speed = ETH_LINK_SPEED_40G; + link.link_speed = ETH_SPEED_NUM_40G; break; default: - link.link_speed = ETH_LINK_SPEED_100; + link.link_speed = ETH_SPEED_NUM_100M; break; } + link.link_autoneg = !(dev->data->dev_conf.link_speeds & + ETH_LINK_SPEED_FIXED); + out: rte_i40e_dev_atomic_write_link_status(dev, &link); if (link.link_status == old.link_status) @@ -1414,13 +2092,10 @@ i40e_update_vsi_stats(struct i40e_vsi *vsi) vsi->vsi_id); } -/* Get all statistics of a port */ static void -i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw) { - uint32_t i; - struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); - struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + unsigned int i; struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */ struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */ @@ -1441,6 +2116,12 @@ i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) I40E_GLPRT_BPRCL(hw->port), pf->offset_loaded, &os->eth.rx_broadcast, &ns->eth.rx_broadcast); + /* Workaround: CRC size should not be included in byte statistics, + * so subtract ETHER_CRC_LEN from the byte counter for each rx packet. + */ + ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast + + ns->eth.rx_broadcast) * ETHER_CRC_LEN; + i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port), pf->offset_loaded, &os->eth.rx_discards, &ns->eth.rx_discards); @@ -1466,6 +2147,8 @@ i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) I40E_GLPRT_BPTCL(hw->port), pf->offset_loaded, &os->eth.tx_broadcast, &ns->eth.tx_broadcast); + ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast + + ns->eth.tx_broadcast) * ETHER_CRC_LEN; /* GLPRT_TEPC not supported */ /* additional port specific stats */ @@ -1602,23 +2285,38 @@ i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) if (pf->main_vsi) i40e_update_vsi_stats(pf->main_vsi); +} - stats->ipackets = ns->eth.rx_unicast + ns->eth.rx_multicast + - ns->eth.rx_broadcast; - stats->opackets = ns->eth.tx_unicast + ns->eth.tx_multicast + - ns->eth.tx_broadcast; +/* Get all statistics of a port */ +static void +i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */ + unsigned i; + + /* call read registers - updates values, now write them to struct */ + i40e_read_stats_registers(pf, hw); + + stats->ipackets = pf->main_vsi->eth_stats.rx_unicast + + pf->main_vsi->eth_stats.rx_multicast + + pf->main_vsi->eth_stats.rx_broadcast - + pf->main_vsi->eth_stats.rx_discards; + stats->opackets = pf->main_vsi->eth_stats.tx_unicast + + pf->main_vsi->eth_stats.tx_multicast + + pf->main_vsi->eth_stats.tx_broadcast; stats->ibytes = ns->eth.rx_bytes; stats->obytes = ns->eth.tx_bytes; - stats->oerrors = ns->eth.tx_errors; - stats->imcasts = ns->eth.rx_multicast; - stats->fdirmatch = ns->fd_sb_match; + stats->oerrors = ns->eth.tx_errors + + pf->main_vsi->eth_stats.tx_errors; /* Rx Errors */ - stats->ibadcrc = ns->crc_errors; - stats->ibadlen = ns->rx_length_errors + ns->rx_undersize + + stats->imissed = ns->eth.rx_discards + + pf->main_vsi->eth_stats.rx_discards; + stats->ierrors = ns->crc_errors + + ns->rx_length_errors + ns->rx_undersize + ns->rx_oversize + ns->rx_fragments + ns->rx_jabber; - stats->imissed = ns->eth.rx_discards; - stats->ierrors = stats->ibadcrc + stats->ibadlen + stats->imissed; PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************"); PMD_DRV_LOG(DEBUG, "rx_bytes: %"PRIu64"", ns->eth.rx_bytes); @@ -1696,27 +2394,149 @@ static void i40e_dev_stats_reset(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - /* It results in reloading the start point of each counter */ + /* Mark PF and VSI stats to update the offset, aka "reset" */ pf->offset_loaded = false; + if (pf->main_vsi) + pf->main_vsi->offset_loaded = false; + + /* read the stats, reading current register values into offset */ + i40e_read_stats_registers(pf, hw); } -static int -i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev, - __rte_unused uint16_t queue_id, - __rte_unused uint8_t stat_idx, - __rte_unused uint8_t is_rx) +static uint32_t +i40e_xstats_calc_num(void) { - PMD_INIT_FUNC_TRACE(); + return I40E_NB_ETH_XSTATS + I40E_NB_HW_PORT_XSTATS + + (I40E_NB_RXQ_PRIO_XSTATS * 8) + + (I40E_NB_TXQ_PRIO_XSTATS * 8); +} - return -ENOSYS; +static int i40e_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + __rte_unused unsigned limit) +{ + unsigned count = 0; + unsigned i, prio; + + if (xstats_names == NULL) + return i40e_xstats_calc_num(); + + /* Note: limit checked in rte_eth_xstats_names() */ + + /* Get stats from i40e_eth_stats struct */ + for (i = 0; i < I40E_NB_ETH_XSTATS; i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", rte_i40e_stats_strings[i].name); + count++; + } + + /* Get individiual stats from i40e_hw_port struct */ + for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "%s", rte_i40e_hw_port_strings[i].name); + count++; + } + + for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) { + for (prio = 0; prio < 8; prio++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "rx_priority%u_%s", prio, + rte_i40e_rxq_prio_strings[i].name); + count++; + } + } + + for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) { + for (prio = 0; prio < 8; prio++) { + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "tx_priority%u_%s", prio, + rte_i40e_txq_prio_strings[i].name); + count++; + } + } + return count; } -static void -i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +static int +i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, + unsigned n) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); - struct i40e_vsi *vsi = pf->main_vsi; + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + unsigned i, count, prio; + struct i40e_hw_port_stats *hw_stats = &pf->stats; + + count = i40e_xstats_calc_num(); + if (n < count) + return count; + + i40e_read_stats_registers(pf, hw); + + if (xstats == NULL) + return 0; + + count = 0; + + /* Get stats from i40e_eth_stats struct */ + for (i = 0; i < I40E_NB_ETH_XSTATS; i++) { + xstats[count].value = *(uint64_t *)(((char *)&hw_stats->eth) + + rte_i40e_stats_strings[i].offset); + count++; + } + + /* Get individiual stats from i40e_hw_port struct */ + for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) { + xstats[count].value = *(uint64_t *)(((char *)hw_stats) + + rte_i40e_hw_port_strings[i].offset); + count++; + } + + for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) { + for (prio = 0; prio < 8; prio++) { + xstats[count].value = + *(uint64_t *)(((char *)hw_stats) + + rte_i40e_rxq_prio_strings[i].offset + + (sizeof(uint64_t) * prio)); + count++; + } + } + + for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) { + for (prio = 0; prio < 8; prio++) { + xstats[count].value = + *(uint64_t *)(((char *)hw_stats) + + rte_i40e_txq_prio_strings[i].offset + + (sizeof(uint64_t) * prio)); + count++; + } + } + + return count; +} + +static int +i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev, + __rte_unused uint16_t queue_id, + __rte_unused uint8_t stat_idx, + __rte_unused uint8_t is_rx) +{ + PMD_INIT_FUNC_TRACE(); + + return -ENOSYS; +} + +static void +i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_vsi *vsi = pf->main_vsi; dev_info->max_rx_queues = vsi->nb_qps; dev_info->max_tx_queues = vsi->nb_qps; @@ -1766,6 +2586,18 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) ETH_TXQ_FLAGS_NOOFFLOADS, }; + dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = I40E_MAX_RING_DESC, + .nb_min = I40E_MIN_RING_DESC, + .nb_align = I40E_ALIGN_RING_DESC, + }; + + dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = I40E_MAX_RING_DESC, + .nb_min = I40E_MIN_RING_DESC, + .nb_align = I40E_ALIGN_RING_DESC, + }; + if (pf->flags & I40E_FLAG_VMDQ) { dev_info->max_vmdq_pools = pf->max_nb_vmdq_vsi; dev_info->vmdq_queue_base = dev_info->max_rx_queues; @@ -1775,6 +2607,13 @@ i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_rx_queues += dev_info->vmdq_queue_num; dev_info->max_tx_queues += dev_info->vmdq_queue_num; } + + if (i40e_is_40G_device(hw->device_id)) + /* For XL710 */ + dev_info->speed_capa = ETH_LINK_SPEED_40G; + else + /* For X710 */ + dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; } static int @@ -1790,11 +2629,70 @@ i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) return i40e_vsi_delete_vlan(vsi, vlan_id); } -static void -i40e_vlan_tpid_set(__rte_unused struct rte_eth_dev *dev, - __rte_unused uint16_t tpid) +static int +i40e_vlan_tpid_set(struct rte_eth_dev *dev, + enum rte_vlan_type vlan_type, + uint16_t tpid) { - PMD_INIT_FUNC_TRACE(); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint64_t reg_r = 0, reg_w = 0; + uint16_t reg_id = 0; + int ret = 0; + int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend; + + switch (vlan_type) { + case ETH_VLAN_TYPE_OUTER: + if (qinq) + reg_id = 2; + else + reg_id = 3; + break; + case ETH_VLAN_TYPE_INNER: + if (qinq) + reg_id = 3; + else { + ret = -EINVAL; + PMD_DRV_LOG(ERR, + "Unsupported vlan type in single vlan.\n"); + return ret; + } + break; + default: + ret = -EINVAL; + PMD_DRV_LOG(ERR, "Unsupported vlan type %d", vlan_type); + return ret; + } + ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id), + ®_r, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "Fail to debug read from " + "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id); + ret = -EIO; + return ret; + } + PMD_DRV_LOG(DEBUG, "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: " + "0x%08"PRIx64"", reg_id, reg_r); + + reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK)); + reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT); + if (reg_r == reg_w) { + ret = 0; + PMD_DRV_LOG(DEBUG, "No need to write"); + return ret; + } + + ret = i40e_aq_debug_write_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id), + reg_w, NULL); + if (ret != I40E_SUCCESS) { + ret = -EIO; + PMD_DRV_LOG(ERR, "Fail to debug write to " + "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id); + return ret; + } + PMD_DRV_LOG(DEBUG, "Debug write 0x%08"PRIx64" to " + "I40E_GL_SWT_L2TAGCTRL[%d]", reg_w, reg_id); + + return ret; } static void @@ -1803,6 +2701,13 @@ i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask) struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_vsi *vsi = pf->main_vsi; + if (mask & ETH_VLAN_FILTER_MASK) { + if (dev->data->dev_conf.rxmode.hw_vlan_filter) + i40e_vsi_config_vlan_filter(vsi, TRUE); + else + i40e_vsi_config_vlan_filter(vsi, FALSE); + } + if (mask & ETH_VLAN_STRIP_MASK) { /* Enable or disable VLAN stripping */ if (dev->data->dev_conf.rxmode.hw_vlan_strip) @@ -1812,8 +2717,14 @@ i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask) } if (mask & ETH_VLAN_EXTEND_MASK) { - if (dev->data->dev_conf.rxmode.hw_vlan_extend) + if (dev->data->dev_conf.rxmode.hw_vlan_extend) { i40e_vsi_config_double_vlan(vsi, TRUE); + /* Set global registers with default ether type value */ + i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, + ETHER_TYPE_VLAN); + i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER, + ETHER_TYPE_VLAN); + } else i40e_vsi_config_double_vlan(vsi, FALSE); } @@ -2040,9 +2951,10 @@ i40e_macaddr_add(struct rte_eth_dev *dev, int ret; /* If VMDQ not enabled or configured, return */ - if (pool != 0 && (!(pf->flags | I40E_FLAG_VMDQ) || !pf->nb_cfg_vmdq_vsi)) { + if (pool != 0 && (!(pf->flags & I40E_FLAG_VMDQ) || + !pf->nb_cfg_vmdq_vsi)) { PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u", - pf->flags | I40E_FLAG_VMDQ ? "configured" : "enabled", + pf->flags & I40E_FLAG_VMDQ ? "configured" : "enabled", pool); return; } @@ -2054,7 +2966,10 @@ i40e_macaddr_add(struct rte_eth_dev *dev, } (void)rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN); - mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; + if (dev->data->dev_conf.rxmode.hw_vlan_filter) + mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH; + else + mac_filter.filter_type = RTE_MAC_PERFECT_MATCH; if (pool == 0) vsi = pf->main_vsi; @@ -2090,7 +3005,7 @@ i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index) vsi = pf->main_vsi; else { /* No VMDQ pool enabled or configured */ - if (!(pf->flags | I40E_FLAG_VMDQ) || + if (!(pf->flags & I40E_FLAG_VMDQ) || (i > pf->nb_cfg_vmdq_vsi)) { PMD_DRV_LOG(ERR, "No VMDQ pool enabled" "/configured"); @@ -2221,17 +3136,76 @@ i40e_mac_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, return ret; } +static int +i40e_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size) +{ + struct i40e_pf *pf = I40E_VSI_TO_PF(vsi); + struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + int ret; + + if (!lut) + return -EINVAL; + + if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { + ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, TRUE, + lut, lut_size); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to get RSS lookup table"); + return ret; + } + } else { + uint32_t *lut_dw = (uint32_t *)lut; + uint16_t i, lut_size_dw = lut_size / 4; + + for (i = 0; i < lut_size_dw; i++) + lut_dw[i] = I40E_READ_REG(hw, I40E_PFQF_HLUT(i)); + } + + return 0; +} + +static int +i40e_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size) +{ + struct i40e_pf *pf; + struct i40e_hw *hw; + int ret; + + if (!vsi || !lut) + return -EINVAL; + + pf = I40E_VSI_TO_PF(vsi); + hw = I40E_VSI_TO_HW(vsi); + + if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { + ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, TRUE, + lut, lut_size); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to set RSS lookup table"); + return ret; + } + } else { + uint32_t *lut_dw = (uint32_t *)lut; + uint16_t i, lut_size_dw = lut_size / 4; + + for (i = 0; i < lut_size_dw; i++) + I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i), lut_dw[i]); + I40E_WRITE_FLUSH(hw); + } + + return 0; +} + static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); - struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t lut, l; - uint16_t i, j, lut_size = pf->hash_lut_size; + uint16_t i, lut_size = pf->hash_lut_size; uint16_t idx, shift; - uint8_t mask; + uint8_t *lut; + int ret; if (reta_size != lut_size || reta_size > ETH_RSS_RETA_SIZE_512) { @@ -2241,28 +3215,26 @@ i40e_dev_rss_reta_update(struct rte_eth_dev *dev, return -EINVAL; } - for (i = 0; i < reta_size; i += I40E_4_BIT_WIDTH) { + lut = rte_zmalloc("i40e_rss_lut", reta_size, 0); + if (!lut) { + PMD_DRV_LOG(ERR, "No memory can be allocated"); + return -ENOMEM; + } + ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size); + if (ret) + goto out; + for (i = 0; i < reta_size; i++) { idx = i / RTE_RETA_GROUP_SIZE; shift = i % RTE_RETA_GROUP_SIZE; - mask = (uint8_t)((reta_conf[idx].mask >> shift) & - I40E_4_BIT_MASK); - if (!mask) - continue; - if (mask == I40E_4_BIT_MASK) - l = 0; - else - l = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2)); - for (j = 0, lut = 0; j < I40E_4_BIT_WIDTH; j++) { - if (mask & (0x1 << j)) - lut |= reta_conf[idx].reta[shift + j] << - (CHAR_BIT * j); - else - lut |= l & (I40E_8_BIT_MASK << (CHAR_BIT * j)); - } - I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut); + if (reta_conf[idx].mask & (1ULL << shift)) + lut[i] = reta_conf[idx].reta[shift]; } + ret = i40e_set_rss_lut(pf->main_vsi, lut, reta_size); - return 0; +out: + rte_free(lut); + + return ret; } static int @@ -2271,11 +3243,10 @@ i40e_dev_rss_reta_query(struct rte_eth_dev *dev, uint16_t reta_size) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); - struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t lut; - uint16_t i, j, lut_size = pf->hash_lut_size; + uint16_t i, lut_size = pf->hash_lut_size; uint16_t idx, shift; - uint8_t mask; + uint8_t *lut; + int ret; if (reta_size != lut_size || reta_size > ETH_RSS_RETA_SIZE_512) { @@ -2285,23 +3256,26 @@ i40e_dev_rss_reta_query(struct rte_eth_dev *dev, return -EINVAL; } - for (i = 0; i < reta_size; i += I40E_4_BIT_WIDTH) { + lut = rte_zmalloc("i40e_rss_lut", reta_size, 0); + if (!lut) { + PMD_DRV_LOG(ERR, "No memory can be allocated"); + return -ENOMEM; + } + + ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size); + if (ret) + goto out; + for (i = 0; i < reta_size; i++) { idx = i / RTE_RETA_GROUP_SIZE; shift = i % RTE_RETA_GROUP_SIZE; - mask = (uint8_t)((reta_conf[idx].mask >> shift) & - I40E_4_BIT_MASK); - if (!mask) - continue; - - lut = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2)); - for (j = 0; j < I40E_4_BIT_WIDTH; j++) { - if (mask & (0x1 << j)) - reta_conf[idx].reta[shift + j] = ((lut >> - (CHAR_BIT * j)) & I40E_8_BIT_MASK); - } + if (reta_conf[idx].mask & (1ULL << shift)) + reta_conf[idx].reta[shift] = lut[i]; } - return 0; +out: + rte_free(lut); + + return ret; } /** @@ -2317,33 +3291,24 @@ i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw, u64 size, u32 alignment) { - static uint64_t id = 0; const struct rte_memzone *mz = NULL; char z_name[RTE_MEMZONE_NAMESIZE]; if (!mem) return I40E_ERR_PARAM; - id++; - snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, id); -#ifdef RTE_LIBRTE_XEN_DOM0 + snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand()); mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0, alignment, RTE_PGSIZE_2M); -#else - mz = rte_memzone_reserve_aligned(z_name, size, SOCKET_ID_ANY, 0, - alignment); -#endif if (!mz) return I40E_ERR_NO_MEMORY; - mem->id = id; mem->size = size; mem->va = mz->addr; -#ifdef RTE_LIBRTE_XEN_DOM0 mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr); -#else - mem->pa = mz->phys_addr; -#endif + mem->zone = (const void *)mz; + PMD_DRV_LOG(DEBUG, "memzone %s allocated with physical address: " + "%"PRIu64, mz->name, mem->pa); return I40E_SUCCESS; } @@ -2357,9 +3322,14 @@ enum i40e_status_code i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw, struct i40e_dma_mem *mem) { - if (!mem || !mem->va) + if (!mem) return I40E_ERR_PARAM; + PMD_DRV_LOG(DEBUG, "memzone %s to be freed with physical address: " + "%"PRIu64, ((const struct rte_memzone *)mem->zone)->name, + mem->pa); + rte_memzone_free((const struct rte_memzone *)mem->zone); + mem->zone = NULL; mem->va = NULL; mem->pa = (u64)0; @@ -2468,9 +3438,8 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_PF_TO_HW(pf); - uint16_t sum_queues = 0, sum_vsis, left_queues; + uint16_t qp_count = 0, vsi_count = 0; - /* First check if FW support SRIOV */ if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) { PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV"); return -EINVAL; @@ -2481,107 +3450,104 @@ i40e_pf_parameter_init(struct rte_eth_dev *dev) pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_LOW_WATER; pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED; - pf->max_num_vsi = RTE_MIN(hw->func_caps.num_vsis, I40E_MAX_NUM_VSIS); - PMD_INIT_LOG(INFO, "Max supported VSIs:%u", pf->max_num_vsi); - /* Allocate queues for pf */ - if (hw->func_caps.rss) { - pf->flags |= I40E_FLAG_RSS; - pf->lan_nb_qps = RTE_MIN(hw->func_caps.num_tx_qp, - (uint32_t)(1 << hw->func_caps.rss_table_entry_width)); - pf->lan_nb_qps = i40e_align_floor(pf->lan_nb_qps); - } else + pf->max_num_vsi = hw->func_caps.num_vsis; + pf->lan_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF; + pf->vmdq_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; + pf->vf_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF; + + /* FDir queue/VSI allocation */ + pf->fdir_qp_offset = 0; + if (hw->func_caps.fd) { + pf->flags |= I40E_FLAG_FDIR; + pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR; + } else { + pf->fdir_nb_qps = 0; + } + qp_count += pf->fdir_nb_qps; + vsi_count += 1; + + /* LAN queue/VSI allocation */ + pf->lan_qp_offset = pf->fdir_qp_offset + pf->fdir_nb_qps; + if (!hw->func_caps.rss) { pf->lan_nb_qps = 1; - sum_queues = pf->lan_nb_qps; - /* Default VSI is not counted in */ - sum_vsis = 0; - PMD_INIT_LOG(INFO, "PF queue pairs:%u", pf->lan_nb_qps); + } else { + pf->flags |= I40E_FLAG_RSS; + if (hw->mac.type == I40E_MAC_X722) + pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE; + pf->lan_nb_qps = pf->lan_nb_qp_max; + } + qp_count += pf->lan_nb_qps; + vsi_count += 1; + /* VF queue/VSI allocation */ + pf->vf_qp_offset = pf->lan_qp_offset + pf->lan_nb_qps; if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) { pf->flags |= I40E_FLAG_SRIOV; pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF; - if (dev->pci_dev->max_vfs > hw->func_caps.num_vfs) { - PMD_INIT_LOG(ERR, "Config VF number %u, " - "max supported %u.", - dev->pci_dev->max_vfs, - hw->func_caps.num_vfs); - return -EINVAL; - } - if (pf->vf_nb_qps > I40E_MAX_QP_NUM_PER_VF) { - PMD_INIT_LOG(ERR, "FVL VF queue %u, " - "max support %u queues.", - pf->vf_nb_qps, I40E_MAX_QP_NUM_PER_VF); - return -EINVAL; - } pf->vf_num = dev->pci_dev->max_vfs; - sum_queues += pf->vf_nb_qps * pf->vf_num; - sum_vsis += pf->vf_num; - PMD_INIT_LOG(INFO, "Max VF num:%u each has queue pairs:%u", - pf->vf_num, pf->vf_nb_qps); - } else + PMD_DRV_LOG(DEBUG, "%u VF VSIs, %u queues per VF VSI, " + "in total %u queues", pf->vf_num, pf->vf_nb_qps, + pf->vf_nb_qps * pf->vf_num); + } else { + pf->vf_nb_qps = 0; pf->vf_num = 0; + } + qp_count += pf->vf_nb_qps * pf->vf_num; + vsi_count += pf->vf_num; + /* VMDq queue/VSI allocation */ + pf->vmdq_qp_offset = pf->vf_qp_offset + pf->vf_nb_qps * pf->vf_num; + pf->vmdq_nb_qps = 0; + pf->max_nb_vmdq_vsi = 0; if (hw->func_caps.vmdq) { - pf->flags |= I40E_FLAG_VMDQ; - pf->vmdq_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; - pf->max_nb_vmdq_vsi = 1; - /* - * If VMDQ available, assume a single VSI can be created. Will adjust - * later. - */ - sum_queues += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi; - sum_vsis += pf->max_nb_vmdq_vsi; - } else { - pf->vmdq_nb_qps = 0; - pf->max_nb_vmdq_vsi = 0; - } - pf->nb_cfg_vmdq_vsi = 0; + if (qp_count < hw->func_caps.num_tx_qp && + vsi_count < hw->func_caps.num_vsis) { + pf->max_nb_vmdq_vsi = (hw->func_caps.num_tx_qp - + qp_count) / pf->vmdq_nb_qp_max; - if (hw->func_caps.fd) { - pf->flags |= I40E_FLAG_FDIR; - pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR; - /** - * Each flow director consumes one VSI and one queue, - * but can't calculate out predictably here. - */ + /* Limit the maximum number of VMDq vsi to the maximum + * ethdev can support + */ + pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi, + hw->func_caps.num_vsis - vsi_count); + pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi, + ETH_64_POOLS); + if (pf->max_nb_vmdq_vsi) { + pf->flags |= I40E_FLAG_VMDQ; + pf->vmdq_nb_qps = pf->vmdq_nb_qp_max; + PMD_DRV_LOG(DEBUG, "%u VMDQ VSIs, %u queues " + "per VMDQ VSI, in total %u queues", + pf->max_nb_vmdq_vsi, + pf->vmdq_nb_qps, pf->vmdq_nb_qps * + pf->max_nb_vmdq_vsi); + } else { + PMD_DRV_LOG(INFO, "No enough queues left for " + "VMDq"); + } + } else { + PMD_DRV_LOG(INFO, "No queue or VSI left for VMDq"); + } } + qp_count += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi; + vsi_count += pf->max_nb_vmdq_vsi; if (hw->func_caps.dcb) pf->flags |= I40E_FLAG_DCB; - if (sum_vsis > pf->max_num_vsi || - sum_queues > hw->func_caps.num_rx_qp) { - PMD_INIT_LOG(ERR, "VSI/QUEUE setting can't be satisfied"); - PMD_INIT_LOG(ERR, "Max VSIs: %u, asked:%u", - pf->max_num_vsi, sum_vsis); - PMD_INIT_LOG(ERR, "Total queue pairs:%u, asked:%u", - hw->func_caps.num_rx_qp, sum_queues); + if (qp_count > hw->func_caps.num_tx_qp) { + PMD_DRV_LOG(ERR, "Failed to allocate %u queues, which exceeds " + "the hardware maximum %u", qp_count, + hw->func_caps.num_tx_qp); return -EINVAL; } - - /* Adjust VMDQ setting to support as many VMs as possible */ - if (pf->flags & I40E_FLAG_VMDQ) { - left_queues = hw->func_caps.num_rx_qp - sum_queues; - - pf->max_nb_vmdq_vsi += RTE_MIN(left_queues / pf->vmdq_nb_qps, - pf->max_num_vsi - sum_vsis); - - /* Limit the max VMDQ number that rte_ether that can support */ - pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi, - ETH_64_POOLS - 1); - - PMD_INIT_LOG(INFO, "Max VMDQ VSI num:%u", - pf->max_nb_vmdq_vsi); - PMD_INIT_LOG(INFO, "VMDQ queue pairs:%u", pf->vmdq_nb_qps); - } - - /* Each VSI occupy 1 MSIX interrupt at least, plus IRQ0 for misc intr - * cause */ - if (sum_vsis > hw->func_caps.num_msix_vectors - 1) { - PMD_INIT_LOG(ERR, "Too many VSIs(%u), MSIX intr(%u) not enough", - sum_vsis, hw->func_caps.num_msix_vectors); + if (vsi_count > hw->func_caps.num_vsis) { + PMD_DRV_LOG(ERR, "Failed to allocate %u VSIs, which exceeds " + "the hardware maximum %u", vsi_count, + hw->func_caps.num_vsis); return -EINVAL; } - return I40E_SUCCESS; + + return 0; } static int @@ -2660,17 +3626,21 @@ i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base, static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool) { - struct pool_entry *entry; + struct pool_entry *entry, *next_entry; if (pool == NULL) return; - LIST_FOREACH(entry, &pool->alloc_list, next) { + for (entry = LIST_FIRST(&pool->alloc_list); + entry && (next_entry = LIST_NEXT(entry, next), 1); + entry = next_entry) { LIST_REMOVE(entry, next); rte_free(entry); } - LIST_FOREACH(entry, &pool->free_list, next) { + for (entry = LIST_FIRST(&pool->free_list); + entry && (next_entry = LIST_NEXT(entry, next), 1); + entry = next_entry) { LIST_REMOVE(entry, next); rte_free(entry); } @@ -2836,7 +3806,7 @@ i40e_res_pool_alloc(struct i40e_res_pool_info *pool, pool->num_free -= valid_entry->len; pool->num_alloc += valid_entry->len; - return (valid_entry->base + pool->base); + return valid_entry->base + pool->base; } /** @@ -2848,7 +3818,7 @@ bitmap_is_subset(uint8_t src1, uint8_t src2) return !((src1 ^ src2) & src2); } -static int +static enum i40e_status_code validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap) { struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); @@ -2856,14 +3826,14 @@ validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap) /* If DCB is not supported, only default TC is supported */ if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) { PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported"); - return -EINVAL; + return I40E_NOT_SUPPORTED; } if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) { PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to " "HW support 0x%x", hw->func_caps.enabled_tcmap, enabled_tcmap); - return -EINVAL; + return I40E_NOT_SUPPORTED; } return I40E_SUCCESS; } @@ -2948,12 +3918,13 @@ i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap) return I40E_SUCCESS; } -static int +static enum i40e_status_code i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi, struct i40e_aqc_vsi_properties_data *info, uint8_t enabled_tcmap) { - int ret, i, total_tc = 0; + enum i40e_status_code ret; + int i, total_tc = 0; uint16_t qpnum_per_tc, bsf, qp_idx; ret = validate_tcmap_parameter(vsi, enabled_tcmap); @@ -2971,7 +3942,8 @@ i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi, bsf = rte_bsf32(qpnum_per_tc); /* Adjust the queue number to actual queues that can be applied */ - vsi->nb_qps = qpnum_per_tc * total_tc; + if (!(vsi->type == I40E_VSI_MAIN && total_tc == 1)) + vsi->nb_qps = qpnum_per_tc * total_tc; /** * Configure TC and queue mapping parameters, for enabled TC, @@ -3013,21 +3985,27 @@ i40e_veb_release(struct i40e_veb *veb) struct i40e_vsi *vsi; struct i40e_hw *hw; - if (veb == NULL || veb->associate_vsi == NULL) + if (veb == NULL) return -EINVAL; if (!TAILQ_EMPTY(&veb->head)) { PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove"); return -EACCES; } + /* associate_vsi field is NULL for floating VEB */ + if (veb->associate_vsi != NULL) { + vsi = veb->associate_vsi; + hw = I40E_VSI_TO_HW(vsi); - vsi = veb->associate_vsi; - hw = I40E_VSI_TO_HW(vsi); + vsi->uplink_seid = veb->uplink_seid; + vsi->veb = NULL; + } else { + veb->associate_pf->main_vsi->floating_veb = NULL; + hw = I40E_VSI_TO_HW(veb->associate_pf->main_vsi); + } - vsi->uplink_seid = veb->uplink_seid; i40e_aq_delete_element(hw, veb->seid, NULL); rte_free(veb); - vsi->veb = NULL; return I40E_SUCCESS; } @@ -3039,9 +4017,9 @@ i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi) int ret; struct i40e_hw *hw; - if (NULL == pf || vsi == NULL) { - PMD_DRV_LOG(ERR, "veb setup failed, " - "associated VSI shouldn't null"); + if (pf == NULL) { + PMD_DRV_LOG(ERR, + "veb setup failed, associated PF shouldn't null"); return NULL; } hw = I40E_PF_TO_HW(pf); @@ -3053,11 +4031,19 @@ i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi) } veb->associate_vsi = vsi; + veb->associate_pf = pf; TAILQ_INIT(&veb->head); - veb->uplink_seid = vsi->uplink_seid; + veb->uplink_seid = vsi ? vsi->uplink_seid : 0; - ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid, - I40E_DEFAULT_TCMAP, false, false, &veb->seid, NULL); + /* create floating veb if vsi is NULL */ + if (vsi != NULL) { + ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid, + I40E_DEFAULT_TCMAP, false, + &veb->seid, false, NULL); + } else { + ret = i40e_aq_add_veb(hw, 0, 0, I40E_DEFAULT_TCMAP, + true, &veb->seid, false, NULL); + } if (ret != I40E_SUCCESS) { PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d", @@ -3073,10 +4059,10 @@ i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi) hw->aq.asq_last_status); goto fail; } - /* Get VEB bandwidth, to be implemented */ /* Now associated vsi binding to the VEB, set uplink to this VEB */ - vsi->uplink_seid = veb->seid; + if (vsi) + vsi->uplink_seid = veb->seid; return veb; fail: @@ -3090,8 +4076,10 @@ i40e_vsi_release(struct i40e_vsi *vsi) struct i40e_pf *pf; struct i40e_hw *hw; struct i40e_vsi_list *vsi_list; + void *temp; int ret; struct i40e_mac_filter *f; + uint16_t user_param = vsi->user_param; if (!vsi) return I40E_SUCCESS; @@ -3101,20 +4089,28 @@ i40e_vsi_release(struct i40e_vsi *vsi) /* VSI has child to attach, release child first */ if (vsi->veb) { - TAILQ_FOREACH(vsi_list, &vsi->veb->head, list) { + TAILQ_FOREACH_SAFE(vsi_list, &vsi->veb->head, list, temp) { if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS) return -1; - TAILQ_REMOVE(&vsi->veb->head, vsi_list, list); } i40e_veb_release(vsi->veb); } + if (vsi->floating_veb) { + TAILQ_FOREACH_SAFE(vsi_list, &vsi->floating_veb->head, list, temp) { + if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS) + return -1; + } + } + /* Remove all macvlan filters of the VSI */ i40e_vsi_remove_all_macvlan_filter(vsi); - TAILQ_FOREACH(f, &vsi->mac_list, next) + TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) rte_free(f); - if (vsi->type != I40E_VSI_MAIN) { + if (vsi->type != I40E_VSI_MAIN && + ((vsi->type != I40E_VSI_SRIOV) || + !pf->floating_veb_list[user_param])) { /* Remove vsi from parent's sibling list */ if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) { PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL"); @@ -3128,6 +4124,24 @@ i40e_vsi_release(struct i40e_vsi *vsi) if (ret != I40E_SUCCESS) PMD_DRV_LOG(ERR, "Failed to delete element"); } + + if ((vsi->type == I40E_VSI_SRIOV) && + pf->floating_veb_list[user_param]) { + /* Remove vsi from parent's sibling list */ + if (vsi->parent_vsi == NULL || + vsi->parent_vsi->floating_veb == NULL) { + PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL"); + return I40E_ERR_PARAM; + } + TAILQ_REMOVE(&vsi->parent_vsi->floating_veb->head, + &vsi->sib_vsi_list, list); + + /* Remove all switch element of the VSI */ + ret = i40e_aq_delete_element(hw, vsi->seid, NULL); + if (ret != I40E_SUCCESS) + PMD_DRV_LOG(ERR, "Failed to delete element"); + } + i40e_res_pool_free(&pf->qp_pool, vsi->base_queue); if (vsi->type != I40E_VSI_SRIOV) @@ -3181,14 +4195,21 @@ i40e_update_default_filter_setting(struct i40e_vsi *vsi) return i40e_vsi_add_mac(vsi, &filter); } -static int -i40e_vsi_dump_bw_config(struct i40e_vsi *vsi) +/* + * i40e_vsi_get_bw_config - Query VSI BW Information + * @vsi: the VSI to be queried + * + * Returns 0 on success, negative value on failure + */ +static enum i40e_status_code +i40e_vsi_get_bw_config(struct i40e_vsi *vsi) { struct i40e_aqc_query_vsi_bw_config_resp bw_config; struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config; struct i40e_hw *hw = &vsi->adapter->hw; i40e_status ret; int i; + uint32_t bw_max; memset(&bw_config, 0, sizeof(bw_config)); ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); @@ -3207,20 +4228,71 @@ i40e_vsi_dump_bw_config(struct i40e_vsi *vsi) return ret; } - /* Not store the info yet, just print out */ - PMD_DRV_LOG(INFO, "VSI bw limit:%u", bw_config.port_bw_limit); - PMD_DRV_LOG(INFO, "VSI max_bw:%u", bw_config.max_bw); + /* store and print out BW info */ + vsi->bw_info.bw_limit = rte_le_to_cpu_16(bw_config.port_bw_limit); + vsi->bw_info.bw_max = bw_config.max_bw; + PMD_DRV_LOG(DEBUG, "VSI bw limit:%u", vsi->bw_info.bw_limit); + PMD_DRV_LOG(DEBUG, "VSI max_bw:%u", vsi->bw_info.bw_max); + bw_max = rte_le_to_cpu_16(ets_sla_config.tc_bw_max[0]) | + (rte_le_to_cpu_16(ets_sla_config.tc_bw_max[1]) << + I40E_16_BIT_WIDTH); for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { - PMD_DRV_LOG(INFO, "\tVSI TC%u:share credits %u", i, - ets_sla_config.share_credits[i]); - PMD_DRV_LOG(INFO, "\tVSI TC%u:credits %u", i, - rte_le_to_cpu_16(ets_sla_config.credits[i])); - PMD_DRV_LOG(INFO, "\tVSI TC%u: max credits: %u", i, - rte_le_to_cpu_16(ets_sla_config.credits[i / 4]) >> - (i * 4)); + vsi->bw_info.bw_ets_share_credits[i] = + ets_sla_config.share_credits[i]; + vsi->bw_info.bw_ets_credits[i] = + rte_le_to_cpu_16(ets_sla_config.credits[i]); + /* 4 bits per TC, 4th bit is reserved */ + vsi->bw_info.bw_ets_max[i] = + (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) & + RTE_LEN2MASK(3, uint8_t)); + PMD_DRV_LOG(DEBUG, "\tVSI TC%u:share credits %u", i, + vsi->bw_info.bw_ets_share_credits[i]); + PMD_DRV_LOG(DEBUG, "\tVSI TC%u:credits %u", i, + vsi->bw_info.bw_ets_credits[i]); + PMD_DRV_LOG(DEBUG, "\tVSI TC%u: max credits: %u", i, + vsi->bw_info.bw_ets_max[i]); } - return 0; + return I40E_SUCCESS; +} + +/* i40e_enable_pf_lb + * @pf: pointer to the pf structure + * + * allow loopback on pf + */ +static inline void +i40e_enable_pf_lb(struct i40e_pf *pf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + struct i40e_vsi_context ctxt; + int ret; + + /* Use the FW API if FW >= v5.0 */ + if (hw->aq.fw_maj_ver < 5) { + PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback"); + return; + } + + memset(&ctxt, 0, sizeof(ctxt)); + ctxt.seid = pf->main_vsi_seid; + ctxt.pf_num = hw->pf_id; + ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL); + if (ret) { + PMD_DRV_LOG(ERR, "cannot get pf vsi config, err %d, aq_err %d", + ret, hw->aq.asq_last_status); + return; + } + ctxt.flags = I40E_AQ_VSI_TYPE_PF; + ctxt.info.valid_sections = + rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID); + ctxt.info.switch_id |= + rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); + + ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (ret) + PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d\n", + hw->aq.asq_last_status); } /* Setup a VSI */ @@ -3238,7 +4310,8 @@ i40e_vsi_setup(struct i40e_pf *pf, struct ether_addr broadcast = {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}; - if (type != I40E_VSI_MAIN && uplink_vsi == NULL) { + if (type != I40E_VSI_MAIN && type != I40E_VSI_SRIOV && + uplink_vsi == NULL) { PMD_DRV_LOG(ERR, "VSI setup failed, " "VSI link shouldn't be NULL"); return NULL; @@ -3250,11 +4323,30 @@ i40e_vsi_setup(struct i40e_pf *pf, return NULL; } - /* If uplink vsi didn't setup VEB, create one first */ - if (type != I40E_VSI_MAIN && uplink_vsi->veb == NULL) { + /* two situations + * 1.type is not MAIN and uplink vsi is not NULL + * If uplink vsi didn't setup VEB, create one first under veb field + * 2.type is SRIOV and the uplink is NULL + * If floating VEB is NULL, create one veb under floating veb field + */ + + if (type != I40E_VSI_MAIN && uplink_vsi != NULL && + uplink_vsi->veb == NULL) { uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi); - if (NULL == uplink_vsi->veb) { + if (uplink_vsi->veb == NULL) { + PMD_DRV_LOG(ERR, "VEB setup failed"); + return NULL; + } + /* set ALLOWLOOPBACk on pf, when veb is created */ + i40e_enable_pf_lb(pf); + } + + if (type == I40E_VSI_SRIOV && uplink_vsi == NULL && + pf->main_vsi->floating_veb == NULL) { + pf->main_vsi->floating_veb = i40e_veb_setup(pf, uplink_vsi); + + if (pf->main_vsi->floating_veb == NULL) { PMD_DRV_LOG(ERR, "VEB setup failed"); return NULL; } @@ -3269,7 +4361,7 @@ i40e_vsi_setup(struct i40e_pf *pf, vsi->type = type; vsi->adapter = I40E_PF_TO_ADAPTER(pf); vsi->max_macaddrs = I40E_NUM_MACADDR_MAX; - vsi->parent_vsi = uplink_vsi; + vsi->parent_vsi = uplink_vsi ? uplink_vsi : pf->main_vsi; vsi->user_param = user_param; /* Allocate queues */ switch (vsi->type) { @@ -3308,15 +4400,30 @@ i40e_vsi_setup(struct i40e_pf *pf, vsi->base_queue = I40E_FDIR_QUEUE_ID; /* VF has MSIX interrupt in VF range, don't allocate here */ - if (type != I40E_VSI_SRIOV) { + if (type == I40E_VSI_MAIN) { + ret = i40e_res_pool_alloc(&pf->msix_pool, + RTE_MIN(vsi->nb_qps, + RTE_MAX_RXTX_INTR_VEC_ID)); + if (ret < 0) { + PMD_DRV_LOG(ERR, "VSI MAIN %d get heap failed %d", + vsi->seid, ret); + goto fail_queue_alloc; + } + vsi->msix_intr = ret; + vsi->nb_msix = RTE_MIN(vsi->nb_qps, RTE_MAX_RXTX_INTR_VEC_ID); + } else if (type != I40E_VSI_SRIOV) { ret = i40e_res_pool_alloc(&pf->msix_pool, 1); if (ret < 0) { PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret); goto fail_queue_alloc; } vsi->msix_intr = ret; - } else + vsi->nb_msix = 1; + } else { vsi->msix_intr = 0; + vsi->nb_msix = 0; + } + /* Add VSI */ if (type == I40E_VSI_MAIN) { /* For main VSI, no need to add since it's default one */ @@ -3408,21 +4515,24 @@ i40e_vsi_setup(struct i40e_pf *pf, * For other VSI, the uplink_seid equals to uplink VSI's * uplink_seid since they share same VEB */ - vsi->uplink_seid = uplink_vsi->uplink_seid; + if (uplink_vsi == NULL) + vsi->uplink_seid = pf->main_vsi->floating_veb->seid; + else + vsi->uplink_seid = uplink_vsi->uplink_seid; ctxt.pf_num = hw->pf_id; ctxt.vf_num = hw->func_caps.vf_base_id + user_param; ctxt.uplink_seid = vsi->uplink_seid; ctxt.connection_type = 0x1; ctxt.flags = I40E_AQ_VSI_TYPE_VF; - /** - * Do not configure switch ID to enable VEB switch by - * I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB. Because in Fortville, - * if the source mac address of packet sent from VF is not - * listed in the VEB's mac table, the VEB will switch the - * packet back to the VF. Need to enable it when HW issue - * is fixed. - */ + /* Use the VEB configuration if FW >= v5.0 */ + if (hw->aq.fw_maj_ver >= 5) { + /* Configure switch ID */ + ctxt.info.valid_sections |= + rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID); + ctxt.info.switch_id = + rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); + } /* Configure port/vlan */ ctxt.info.valid_sections |= @@ -3516,8 +4626,13 @@ i40e_vsi_setup(struct i40e_pf *pf, vsi->seid = ctxt.seid; vsi->vsi_id = ctxt.vsi_number; vsi->sib_vsi_list.vsi = vsi; - TAILQ_INSERT_TAIL(&uplink_vsi->veb->head, - &vsi->sib_vsi_list, list); + if (vsi->type == I40E_VSI_SRIOV && uplink_vsi == NULL) { + TAILQ_INSERT_TAIL(&pf->main_vsi->floating_veb->head, + &vsi->sib_vsi_list, list); + } else { + TAILQ_INSERT_TAIL(&uplink_vsi->veb->head, + &vsi->sib_vsi_list, list); + } } /* MAC/VLAN configuration */ @@ -3531,7 +4646,7 @@ i40e_vsi_setup(struct i40e_pf *pf, } /* Get VSI BW information */ - i40e_vsi_dump_bw_config(vsi); + i40e_vsi_get_bw_config(vsi); return vsi; fail_msix_alloc: i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr); @@ -3542,6 +4657,64 @@ fail_mem: return NULL; } +/* Configure vlan filter on or off */ +int +i40e_vsi_config_vlan_filter(struct i40e_vsi *vsi, bool on) +{ + int i, num; + struct i40e_mac_filter *f; + void *temp; + struct i40e_mac_filter_info *mac_filter; + enum rte_mac_filter_type desired_filter; + int ret = I40E_SUCCESS; + + if (on) { + /* Filter to match MAC and VLAN */ + desired_filter = RTE_MACVLAN_PERFECT_MATCH; + } else { + /* Filter to match only MAC */ + desired_filter = RTE_MAC_PERFECT_MATCH; + } + + num = vsi->mac_num; + + mac_filter = rte_zmalloc("mac_filter_info_data", + num * sizeof(*mac_filter), 0); + if (mac_filter == NULL) { + PMD_DRV_LOG(ERR, "failed to allocate memory"); + return I40E_ERR_NO_MEMORY; + } + + i = 0; + + /* Remove all existing mac */ + TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) { + mac_filter[i] = f->mac_info; + ret = i40e_vsi_delete_mac(vsi, &f->mac_info.mac_addr); + if (ret) { + PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter", + on ? "enable" : "disable"); + goto DONE; + } + i++; + } + + /* Override with new filter */ + for (i = 0; i < num; i++) { + mac_filter[i].filter_type = desired_filter; + ret = i40e_vsi_add_mac(vsi, &mac_filter[i]); + if (ret) { + PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter", + on ? "enable" : "disable"); + goto DONE; + } + } + +DONE: + rte_free(mac_filter); + return ret; +} + /* Configure vlan stripping on or off */ int i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on) @@ -3589,9 +4762,11 @@ i40e_dev_init_vlan(struct rte_eth_dev *dev) { struct rte_eth_dev_data *data = dev->data; int ret; + int mask = 0; /* Apply vlan offload setting */ - i40e_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK); + mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK; + i40e_vlan_offload_set(dev, mask); /* Apply double-vlan setting, not implemented yet */ @@ -4160,7 +5335,7 @@ i40e_pf_enable_irq0(struct i40e_hw *hw) } static void -i40e_pf_config_irq0(struct i40e_hw *hw) +i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue) { /* read pending request and disable first */ i40e_pf_disable_irq0(hw); @@ -4168,9 +5343,10 @@ i40e_pf_config_irq0(struct i40e_hw *hw) I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0, I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK); - /* Link no queues with irq0 */ - I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, - I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK); + if (no_queue) + /* Link no queues with irq0 */ + I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, + I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK); } static void @@ -5024,39 +6200,99 @@ i40e_pf_disable_rss(struct i40e_pf *pf) struct i40e_hw *hw = I40E_PF_TO_HW(pf); uint64_t hena; - hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0)); - hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32; + hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); + hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; hena &= ~I40E_RSS_HENA_ALL; - I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena); - I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena); + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); I40E_WRITE_FLUSH(hw); } static int -i40e_hw_rss_hash_set(struct i40e_hw *hw, struct rte_eth_rss_conf *rss_conf) +i40e_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len) { - uint32_t *hash_key; - uint8_t hash_key_len; - uint64_t rss_hf; - uint16_t i; - uint64_t hena; + struct i40e_pf *pf = I40E_VSI_TO_PF(vsi); + struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + int ret = 0; + + if (!key || key_len == 0) { + PMD_DRV_LOG(DEBUG, "No key to be configured"); + return 0; + } else if (key_len != (I40E_PFQF_HKEY_MAX_INDEX + 1) * + sizeof(uint32_t)) { + PMD_DRV_LOG(ERR, "Invalid key length %u", key_len); + return -EINVAL; + } + + if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { + struct i40e_aqc_get_set_rss_key_data *key_dw = + (struct i40e_aqc_get_set_rss_key_data *)key; + + ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw); + if (ret) + PMD_INIT_LOG(ERR, "Failed to configure RSS key " + "via AQ"); + } else { + uint32_t *hash_key = (uint32_t *)key; + uint16_t i; + + for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) + i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), hash_key[i]); + I40E_WRITE_FLUSH(hw); + } + + return ret; +} + +static int +i40e_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len) +{ + struct i40e_pf *pf = I40E_VSI_TO_PF(vsi); + struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); + int ret; + + if (!key || !key_len) + return -EINVAL; + + if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { + ret = i40e_aq_get_rss_key(hw, vsi->vsi_id, + (struct i40e_aqc_get_set_rss_key_data *)key); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ"); + return ret; + } + } else { + uint32_t *key_dw = (uint32_t *)key; + uint16_t i; - hash_key = (uint32_t *)(rss_conf->rss_key); - hash_key_len = rss_conf->rss_key_len; - if (hash_key != NULL && hash_key_len >= - (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) { - /* Fill in RSS hash key */ for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) - I40E_WRITE_REG(hw, I40E_PFQF_HKEY(i), hash_key[i]); + key_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); } + *key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t); + + return 0; +} + +static int +i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + uint64_t rss_hf; + uint64_t hena; + int ret; + + ret = i40e_set_rss_key(pf->main_vsi, rss_conf->rss_key, + rss_conf->rss_key_len); + if (ret) + return ret; rss_hf = rss_conf->rss_hf; - hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0)); - hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32; + hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); + hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; hena &= ~I40E_RSS_HENA_ALL; hena |= i40e_config_hena(rss_hf); - I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena); - I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena); + i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); I40E_WRITE_FLUSH(hw); return 0; @@ -5066,12 +6302,13 @@ static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL; uint64_t hena; - hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0)); - hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32; + hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); + hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */ if (rss_hf != 0) /* Enable RSS */ return -EINVAL; @@ -5081,25 +6318,22 @@ i40e_dev_rss_hash_update(struct rte_eth_dev *dev, if (rss_hf == 0) /* Disable RSS */ return -EINVAL; - return i40e_hw_rss_hash_set(hw, rss_conf); + return i40e_hw_rss_hash_set(pf, rss_conf); } static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t *hash_key = (uint32_t *)(rss_conf->rss_key); uint64_t hena; - uint16_t i; - if (hash_key != NULL) { - for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) - hash_key[i] = I40E_READ_REG(hw, I40E_PFQF_HKEY(i)); - rss_conf->rss_key_len = i * sizeof(uint32_t); - } - hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0)); - hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32; + i40e_get_rss_key(pf->main_vsi, rss_conf->rss_key, + &rss_conf->rss_key_len); + + hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)); + hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32; rss_conf->rss_hf = i40e_parse_hena(hena); return 0; @@ -5124,6 +6358,12 @@ i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag) case ETH_TUNNEL_FILTER_IMAC: *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC; break; + case ETH_TUNNEL_FILTER_OIP: + *flag = I40E_AQC_ADD_CLOUD_FILTER_OIP; + break; + case ETH_TUNNEL_FILTER_IIP: + *flag = I40E_AQC_ADD_CLOUD_FILTER_IIP; + break; default: PMD_DRV_LOG(ERR, "invalid tunnel filter type"); return -EINVAL; @@ -5138,7 +6378,10 @@ i40e_dev_tunnel_filter_set(struct i40e_pf *pf, uint8_t add) { uint16_t ip_type; - uint8_t tun_type = 0; + uint32_t ipv4_addr; + uint8_t i, tun_type = 0; + /* internal varialbe to convert ipv6 byte order */ + uint32_t convert_ipv6[4]; int val, ret = 0; struct i40e_hw *hw = I40E_PF_TO_HW(pf); struct i40e_vsi *vsi = pf->main_vsi; @@ -5155,32 +6398,37 @@ i40e_dev_tunnel_filter_set(struct i40e_pf *pf, } pfilter = cld_filter; - (void)rte_memcpy(&pfilter->outer_mac, tunnel_filter->outer_mac, - sizeof(struct ether_addr)); - (void)rte_memcpy(&pfilter->inner_mac, tunnel_filter->inner_mac, - sizeof(struct ether_addr)); + ether_addr_copy(&tunnel_filter->outer_mac, (struct ether_addr*)&pfilter->outer_mac); + ether_addr_copy(&tunnel_filter->inner_mac, (struct ether_addr*)&pfilter->inner_mac); - pfilter->inner_vlan = tunnel_filter->inner_vlan; + pfilter->inner_vlan = rte_cpu_to_le_16(tunnel_filter->inner_vlan); if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) { ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4; - (void)rte_memcpy(&pfilter->ipaddr.v4.data, - &tunnel_filter->ip_addr, + ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr); + rte_memcpy(&pfilter->ipaddr.v4.data, + &rte_cpu_to_le_32(ipv4_addr), sizeof(pfilter->ipaddr.v4.data)); } else { ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6; - (void)rte_memcpy(&pfilter->ipaddr.v6.data, - &tunnel_filter->ip_addr, + for (i = 0; i < 4; i++) { + convert_ipv6[i] = + rte_cpu_to_le_32(rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv6_addr[i])); + } + rte_memcpy(&pfilter->ipaddr.v6.data, &convert_ipv6, sizeof(pfilter->ipaddr.v6.data)); } /* check tunneled type */ switch (tunnel_filter->tunnel_type) { case RTE_TUNNEL_TYPE_VXLAN: - tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_XVLAN; + tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN; break; case RTE_TUNNEL_TYPE_NVGRE: tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC; break; + case RTE_TUNNEL_TYPE_IP_IN_GRE: + tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP; + break; default: /* Other tunnel types is not supported. */ PMD_DRV_LOG(ERR, "tunnel type is not supported."); @@ -5195,10 +6443,11 @@ i40e_dev_tunnel_filter_set(struct i40e_pf *pf, return -EINVAL; } - pfilter->flags |= I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE | ip_type | - (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); - pfilter->tenant_id = tunnel_filter->tenant_id; - pfilter->queue_number = tunnel_filter->queue_id; + pfilter->flags |= rte_cpu_to_le_16( + I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE | + ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT)); + pfilter->tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id); + pfilter->queue_number = rte_cpu_to_le_16(tunnel_filter->queue_id); if (add) ret = i40e_aq_add_cloud_filters(hw, vsi->seid, cld_filter, 1); @@ -5303,8 +6552,8 @@ i40e_del_vxlan_port(struct i40e_pf *pf, uint16_t port) /* Add UDP tunneling port */ static int -i40e_dev_udp_tunnel_add(struct rte_eth_dev *dev, - struct rte_eth_udp_tunnel *udp_tunnel) +i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev, + struct rte_eth_udp_tunnel *udp_tunnel) { int ret = 0; struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); @@ -5334,8 +6583,8 @@ i40e_dev_udp_tunnel_add(struct rte_eth_dev *dev, /* Remove UDP tunneling port */ static int -i40e_dev_udp_tunnel_del(struct rte_eth_dev *dev, - struct rte_eth_udp_tunnel *udp_tunnel) +i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev, + struct rte_eth_udp_tunnel *udp_tunnel) { int ret = 0; struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); @@ -5394,12 +6643,12 @@ i40e_pf_config_rss(struct i40e_pf *pf) * If both VMDQ and RSS enabled, not all of PF queues are configured. * It's necessary to calulate the actual PF queues that are configured. */ - if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) { + if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) num = i40e_pf_calc_configured_queues_num(pf); - num = i40e_align_floor(num); - } else - num = i40e_align_floor(pf->dev_data->nb_rx_queues); + else + num = pf->dev_data->nb_rx_queues; + num = RTE_MIN(num, I40E_MAX_Q_PER_TC); PMD_INIT_LOG(INFO, "Max of contiguous %u PF queues are configured", num); @@ -5435,12 +6684,12 @@ i40e_pf_config_rss(struct i40e_pf *pf) sizeof(uint32_t); } - return i40e_hw_rss_hash_set(hw, &rss_conf); + return i40e_hw_rss_hash_set(pf, &rss_conf); } static int i40e_tunnel_filter_param_check(struct i40e_pf *pf, - struct rte_eth_tunnel_filter_conf *filter) + struct rte_eth_tunnel_filter_conf *filter) { if (pf == NULL || filter == NULL) { PMD_DRV_LOG(ERR, "Invalid parameter"); @@ -5458,13 +6707,13 @@ i40e_tunnel_filter_param_check(struct i40e_pf *pf, } if ((filter->filter_type & ETH_TUNNEL_FILTER_OMAC) && - (is_zero_ether_addr(filter->outer_mac))) { + (is_zero_ether_addr(&filter->outer_mac))) { PMD_DRV_LOG(ERR, "Cannot add NULL outer MAC address"); return -EINVAL; } if ((filter->filter_type & ETH_TUNNEL_FILTER_IMAC) && - (is_zero_ether_addr(filter->inner_mac))) { + (is_zero_ether_addr(&filter->inner_mac))) { PMD_DRV_LOG(ERR, "Cannot add NULL inner MAC address"); return -EINVAL; } @@ -5472,9 +6721,85 @@ i40e_tunnel_filter_param_check(struct i40e_pf *pf, return 0; } +#define I40E_GL_PRS_FVBM_MSK_ENA 0x80000000 +#define I40E_GL_PRS_FVBM(_i) (0x00269760 + ((_i) * 4)) +static int +i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len) +{ + uint32_t val, reg; + int ret = -EINVAL; + + val = I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2)); + PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x\n", val); + + if (len == 3) { + reg = val | I40E_GL_PRS_FVBM_MSK_ENA; + } else if (len == 4) { + reg = val & ~I40E_GL_PRS_FVBM_MSK_ENA; + } else { + PMD_DRV_LOG(ERR, "Unsupported GRE key length of %u", len); + return ret; + } + + if (reg != val) { + ret = i40e_aq_debug_write_register(hw, I40E_GL_PRS_FVBM(2), + reg, NULL); + if (ret != 0) + return ret; + } else { + ret = 0; + } + PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x\n", + I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2))); + + return ret; +} + +static int +i40e_dev_global_config_set(struct i40e_hw *hw, struct rte_eth_global_cfg *cfg) +{ + int ret = -EINVAL; + + if (!hw || !cfg) + return -EINVAL; + + switch (cfg->cfg_type) { + case RTE_ETH_GLOBAL_CFG_TYPE_GRE_KEY_LEN: + ret = i40e_dev_set_gre_key_len(hw, cfg->cfg.gre_key_len); + break; + default: + PMD_DRV_LOG(ERR, "Unknown config type %u", cfg->cfg_type); + break; + } + + return ret; +} + +static int +i40e_filter_ctrl_global_config(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret = I40E_ERR_PARAM; + + switch (filter_op) { + case RTE_ETH_FILTER_SET: + ret = i40e_dev_global_config_set(hw, + (struct rte_eth_global_cfg *)arg); + break; + default: + PMD_DRV_LOG(ERR, "unknown operation %u", filter_op); + break; + } + + return ret; +} + static int -i40e_tunnel_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, - void *arg) +i40e_tunnel_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) { struct rte_eth_tunnel_filter_conf *filter; struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); @@ -5489,6 +6814,7 @@ i40e_tunnel_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op, case RTE_ETH_FILTER_NOP: if (!(pf->flags & I40E_FLAG_VXLAN)) ret = I40E_NOT_SUPPORTED; + break; case RTE_ETH_FILTER_ADD: ret = i40e_dev_tunnel_filter_set(pf, filter, 1); break; @@ -5523,7 +6849,7 @@ i40e_pf_config_mq_rx(struct i40e_pf *pf) static void i40e_get_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t *enable) { - uint32_t reg = I40E_READ_REG(hw, I40E_PRTQF_CTL_0); + uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0); *enable = reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK ? 1 : 0; } @@ -5532,7 +6858,7 @@ i40e_get_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t *enable) static void i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable) { - uint32_t reg = I40E_READ_REG(hw, I40E_PRTQF_CTL_0); + uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0); if (enable > 0) { if (reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK) { @@ -5549,7 +6875,7 @@ i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable) } reg &= ~I40E_PRTQF_CTL_0_HSYM_ENA_MASK; } - I40E_WRITE_REG(hw, I40E_PRTQF_CTL_0, reg); + i40e_write_rx_ctl(hw, I40E_PRTQF_CTL_0, reg); I40E_WRITE_FLUSH(hw); } @@ -5567,7 +6893,7 @@ i40e_get_hash_filter_global_config(struct i40e_hw *hw, enum i40e_filter_pctype pctype; memset(g_cfg, 0, sizeof(*g_cfg)); - reg = I40E_READ_REG(hw, I40E_GLQF_CTL); + reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL); if (reg & I40E_GLQF_CTL_HTOEP_MASK) g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; else @@ -5581,8 +6907,11 @@ i40e_get_hash_filter_global_config(struct i40e_hw *hw, mask &= ~(1UL << i); /* Bit set indicats the coresponding flow type is supported */ g_cfg->valid_bit_mask[0] |= (1UL << i); + /* if flowtype is invalid, continue */ + if (!I40E_VALID_FLOW(i)) + continue; pctype = i40e_flowtype_to_pctype(i); - reg = I40E_READ_REG(hw, I40E_GLQF_HSYM(pctype)); + reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype)); if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK) g_cfg->sym_hash_enable_mask[0] |= (1UL << i); } @@ -5652,13 +6981,16 @@ i40e_set_hash_filter_global_config(struct i40e_hw *hw, if (!(mask0 & (1UL << i))) continue; mask0 &= ~(1UL << i); + /* if flowtype is invalid, continue */ + if (!I40E_VALID_FLOW(i)) + continue; pctype = i40e_flowtype_to_pctype(i); reg = (g_cfg->sym_hash_enable_mask[0] & (1UL << i)) ? I40E_GLQF_HSYM_SYMH_ENA_MASK : 0; - I40E_WRITE_REG(hw, I40E_GLQF_HSYM(pctype), reg); + i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg); } - reg = I40E_READ_REG(hw, I40E_GLQF_CTL); + reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL); if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ) { /* Toeplitz */ if (reg & I40E_GLQF_CTL_HTOEP_MASK) { @@ -5679,7 +7011,7 @@ i40e_set_hash_filter_global_config(struct i40e_hw *hw, /* Use the default, and keep it as it is */ goto out; - I40E_WRITE_REG(hw, I40E_GLQF_CTL, reg); + i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg); out: I40E_WRITE_FLUSH(hw); @@ -5687,77 +7019,734 @@ out: return 0; } -static int -i40e_hash_filter_get(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info) -{ - int ret = 0; +/** + * Valid input sets for hash and flow director filters per PCTYPE + */ +static uint64_t +i40e_get_valid_input_set(enum i40e_filter_pctype pctype, + enum rte_filter_type filter) +{ + uint64_t valid; + + static const uint64_t valid_hash_inset_table[] = { + [I40E_FILTER_PCTYPE_FRAG_IPV4] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_SRC | + I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS | + I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL | + I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_FRAG_IPV6] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_TUNNEL_DMAC | + I40E_INSET_TUNNEL_ID | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | + I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | + I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT | + I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC | + I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC | + I40E_INSET_IPV6_DST | I40E_INSET_TUNNEL_ID | + I40E_INSET_FLEX_PAYLOAD, + [I40E_FILTER_PCTYPE_L2_PAYLOAD] = + I40E_INSET_DMAC | I40E_INSET_SMAC | + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_VLAN_TUNNEL | I40E_INSET_LAST_ETHER_TYPE | + I40E_INSET_FLEX_PAYLOAD, + }; - if (!hw || !info) { - PMD_DRV_LOG(ERR, "Invalid pointer"); - return -EFAULT; - } + /** + * Flow director supports only fields defined in + * union rte_eth_fdir_flow. + */ + static const uint64_t valid_fdir_inset_table[] = { + [I40E_FILTER_PCTYPE_FRAG_IPV4] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO | + I40E_INSET_IPV4_TTL, + [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_SCTP_VT, + [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO | + I40E_INSET_IPV4_TTL, + [I40E_FILTER_PCTYPE_FRAG_IPV6] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT, + [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_SCTP_VT, + [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR | + I40E_INSET_IPV6_HOP_LIMIT, + [I40E_FILTER_PCTYPE_L2_PAYLOAD] = + I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER | + I40E_INSET_LAST_ETHER_TYPE, + }; - switch (info->info_type) { - case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT: - i40e_get_symmetric_hash_enable_per_port(hw, - &(info->info.enable)); - break; - case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG: - ret = i40e_get_hash_filter_global_config(hw, - &(info->info.global_conf)); - break; - default: - PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported", - info->info_type); - ret = -EINVAL; - break; - } + if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD) + return 0; + if (filter == RTE_ETH_FILTER_HASH) + valid = valid_hash_inset_table[pctype]; + else + valid = valid_fdir_inset_table[pctype]; - return ret; + return valid; } +/** + * Validate if the input set is allowed for a specific PCTYPE + */ static int -i40e_hash_filter_set(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info) +i40e_validate_input_set(enum i40e_filter_pctype pctype, + enum rte_filter_type filter, uint64_t inset) { - int ret = 0; - - if (!hw || !info) { - PMD_DRV_LOG(ERR, "Invalid pointer"); - return -EFAULT; - } + uint64_t valid; - switch (info->info_type) { - case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT: - i40e_set_symmetric_hash_enable_per_port(hw, info->info.enable); - break; - case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG: - ret = i40e_set_hash_filter_global_config(hw, - &(info->info.global_conf)); - break; - default: - PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported", - info->info_type); - ret = -EINVAL; - break; - } + valid = i40e_get_valid_input_set(pctype, filter); + if (inset & (~valid)) + return -EINVAL; - return ret; + return 0; } -/* Operations for hash function */ -static int -i40e_hash_filter_ctrl(struct rte_eth_dev *dev, - enum rte_filter_op filter_op, - void *arg) +/* default input set fields combination per pctype */ +static uint64_t +i40e_get_default_input_set(uint16_t pctype) { - struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - int ret = 0; + static const uint64_t default_inset_table[] = { + [I40E_FILTER_PCTYPE_FRAG_IPV4] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST, + [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_SCTP_VT, + [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] = + I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST, + [I40E_FILTER_PCTYPE_FRAG_IPV6] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST, + [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT, + [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST | + I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | + I40E_INSET_SCTP_VT, + [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] = + I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST, + [I40E_FILTER_PCTYPE_L2_PAYLOAD] = + I40E_INSET_LAST_ETHER_TYPE, + }; - switch (filter_op) { - case RTE_ETH_FILTER_NOP: - break; - case RTE_ETH_FILTER_GET: - ret = i40e_hash_filter_get(hw, + if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD) + return 0; + + return default_inset_table[pctype]; +} + +/** + * Parse the input set from index to logical bit masks + */ +static int +i40e_parse_input_set(uint64_t *inset, + enum i40e_filter_pctype pctype, + enum rte_eth_input_set_field *field, + uint16_t size) +{ + uint16_t i, j; + int ret = -EINVAL; + + static const struct { + enum rte_eth_input_set_field field; + uint64_t inset; + } inset_convert_table[] = { + {RTE_ETH_INPUT_SET_NONE, I40E_INSET_NONE}, + {RTE_ETH_INPUT_SET_L2_SRC_MAC, I40E_INSET_SMAC}, + {RTE_ETH_INPUT_SET_L2_DST_MAC, I40E_INSET_DMAC}, + {RTE_ETH_INPUT_SET_L2_OUTER_VLAN, I40E_INSET_VLAN_OUTER}, + {RTE_ETH_INPUT_SET_L2_INNER_VLAN, I40E_INSET_VLAN_INNER}, + {RTE_ETH_INPUT_SET_L2_ETHERTYPE, I40E_INSET_LAST_ETHER_TYPE}, + {RTE_ETH_INPUT_SET_L3_SRC_IP4, I40E_INSET_IPV4_SRC}, + {RTE_ETH_INPUT_SET_L3_DST_IP4, I40E_INSET_IPV4_DST}, + {RTE_ETH_INPUT_SET_L3_IP4_TOS, I40E_INSET_IPV4_TOS}, + {RTE_ETH_INPUT_SET_L3_IP4_PROTO, I40E_INSET_IPV4_PROTO}, + {RTE_ETH_INPUT_SET_L3_IP4_TTL, I40E_INSET_IPV4_TTL}, + {RTE_ETH_INPUT_SET_L3_SRC_IP6, I40E_INSET_IPV6_SRC}, + {RTE_ETH_INPUT_SET_L3_DST_IP6, I40E_INSET_IPV6_DST}, + {RTE_ETH_INPUT_SET_L3_IP6_TC, I40E_INSET_IPV6_TC}, + {RTE_ETH_INPUT_SET_L3_IP6_NEXT_HEADER, + I40E_INSET_IPV6_NEXT_HDR}, + {RTE_ETH_INPUT_SET_L3_IP6_HOP_LIMITS, + I40E_INSET_IPV6_HOP_LIMIT}, + {RTE_ETH_INPUT_SET_L4_UDP_SRC_PORT, I40E_INSET_SRC_PORT}, + {RTE_ETH_INPUT_SET_L4_TCP_SRC_PORT, I40E_INSET_SRC_PORT}, + {RTE_ETH_INPUT_SET_L4_SCTP_SRC_PORT, I40E_INSET_SRC_PORT}, + {RTE_ETH_INPUT_SET_L4_UDP_DST_PORT, I40E_INSET_DST_PORT}, + {RTE_ETH_INPUT_SET_L4_TCP_DST_PORT, I40E_INSET_DST_PORT}, + {RTE_ETH_INPUT_SET_L4_SCTP_DST_PORT, I40E_INSET_DST_PORT}, + {RTE_ETH_INPUT_SET_L4_SCTP_VERIFICATION_TAG, + I40E_INSET_SCTP_VT}, + {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_DST_MAC, + I40E_INSET_TUNNEL_DMAC}, + {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_VLAN, + I40E_INSET_VLAN_TUNNEL}, + {RTE_ETH_INPUT_SET_TUNNEL_L4_UDP_KEY, + I40E_INSET_TUNNEL_ID}, + {RTE_ETH_INPUT_SET_TUNNEL_GRE_KEY, I40E_INSET_TUNNEL_ID}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_1ST_WORD, + I40E_INSET_FLEX_PAYLOAD_W1}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_2ND_WORD, + I40E_INSET_FLEX_PAYLOAD_W2}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_3RD_WORD, + I40E_INSET_FLEX_PAYLOAD_W3}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_4TH_WORD, + I40E_INSET_FLEX_PAYLOAD_W4}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_5TH_WORD, + I40E_INSET_FLEX_PAYLOAD_W5}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_6TH_WORD, + I40E_INSET_FLEX_PAYLOAD_W6}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_7TH_WORD, + I40E_INSET_FLEX_PAYLOAD_W7}, + {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_8TH_WORD, + I40E_INSET_FLEX_PAYLOAD_W8}, + }; + + if (!inset || !field || size > RTE_ETH_INSET_SIZE_MAX) + return ret; + + /* Only one item allowed for default or all */ + if (size == 1) { + if (field[0] == RTE_ETH_INPUT_SET_DEFAULT) { + *inset = i40e_get_default_input_set(pctype); + return 0; + } else if (field[0] == RTE_ETH_INPUT_SET_NONE) { + *inset = I40E_INSET_NONE; + return 0; + } + } + + for (i = 0, *inset = 0; i < size; i++) { + for (j = 0; j < RTE_DIM(inset_convert_table); j++) { + if (field[i] == inset_convert_table[j].field) { + *inset |= inset_convert_table[j].inset; + break; + } + } + + /* It contains unsupported input set, return immediately */ + if (j == RTE_DIM(inset_convert_table)) + return ret; + } + + return 0; +} + +/** + * Translate the input set from bit masks to register aware bit masks + * and vice versa + */ +static uint64_t +i40e_translate_input_set_reg(uint64_t input) +{ + uint64_t val = 0; + uint16_t i; + + static const struct { + uint64_t inset; + uint64_t inset_reg; + } inset_map[] = { + {I40E_INSET_DMAC, I40E_REG_INSET_L2_DMAC}, + {I40E_INSET_SMAC, I40E_REG_INSET_L2_SMAC}, + {I40E_INSET_VLAN_OUTER, I40E_REG_INSET_L2_OUTER_VLAN}, + {I40E_INSET_VLAN_INNER, I40E_REG_INSET_L2_INNER_VLAN}, + {I40E_INSET_LAST_ETHER_TYPE, I40E_REG_INSET_LAST_ETHER_TYPE}, + {I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4}, + {I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4}, + {I40E_INSET_IPV4_TOS, I40E_REG_INSET_L3_IP4_TOS}, + {I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO}, + {I40E_INSET_IPV4_TTL, I40E_REG_INSET_L3_IP4_TTL}, + {I40E_INSET_IPV6_SRC, I40E_REG_INSET_L3_SRC_IP6}, + {I40E_INSET_IPV6_DST, I40E_REG_INSET_L3_DST_IP6}, + {I40E_INSET_IPV6_TC, I40E_REG_INSET_L3_IP6_TC}, + {I40E_INSET_IPV6_NEXT_HDR, I40E_REG_INSET_L3_IP6_NEXT_HDR}, + {I40E_INSET_IPV6_HOP_LIMIT, I40E_REG_INSET_L3_IP6_HOP_LIMIT}, + {I40E_INSET_SRC_PORT, I40E_REG_INSET_L4_SRC_PORT}, + {I40E_INSET_DST_PORT, I40E_REG_INSET_L4_DST_PORT}, + {I40E_INSET_SCTP_VT, I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG}, + {I40E_INSET_TUNNEL_ID, I40E_REG_INSET_TUNNEL_ID}, + {I40E_INSET_TUNNEL_DMAC, + I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC}, + {I40E_INSET_TUNNEL_IPV4_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP4}, + {I40E_INSET_TUNNEL_IPV6_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP6}, + {I40E_INSET_TUNNEL_SRC_PORT, + I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT}, + {I40E_INSET_TUNNEL_DST_PORT, + I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT}, + {I40E_INSET_VLAN_TUNNEL, I40E_REG_INSET_TUNNEL_VLAN}, + {I40E_INSET_FLEX_PAYLOAD_W1, I40E_REG_INSET_FLEX_PAYLOAD_WORD1}, + {I40E_INSET_FLEX_PAYLOAD_W2, I40E_REG_INSET_FLEX_PAYLOAD_WORD2}, + {I40E_INSET_FLEX_PAYLOAD_W3, I40E_REG_INSET_FLEX_PAYLOAD_WORD3}, + {I40E_INSET_FLEX_PAYLOAD_W4, I40E_REG_INSET_FLEX_PAYLOAD_WORD4}, + {I40E_INSET_FLEX_PAYLOAD_W5, I40E_REG_INSET_FLEX_PAYLOAD_WORD5}, + {I40E_INSET_FLEX_PAYLOAD_W6, I40E_REG_INSET_FLEX_PAYLOAD_WORD6}, + {I40E_INSET_FLEX_PAYLOAD_W7, I40E_REG_INSET_FLEX_PAYLOAD_WORD7}, + {I40E_INSET_FLEX_PAYLOAD_W8, I40E_REG_INSET_FLEX_PAYLOAD_WORD8}, + }; + + if (input == 0) + return val; + + /* Translate input set to register aware inset */ + for (i = 0; i < RTE_DIM(inset_map); i++) { + if (input & inset_map[i].inset) + val |= inset_map[i].inset_reg; + } + + return val; +} + +static int +i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem) +{ + uint8_t i, idx = 0; + uint64_t inset_need_mask = inset; + + static const struct { + uint64_t inset; + uint32_t mask; + } inset_mask_map[] = { + {I40E_INSET_IPV4_TOS, I40E_INSET_IPV4_TOS_MASK}, + {I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL, 0}, + {I40E_INSET_IPV4_PROTO, I40E_INSET_IPV4_PROTO_MASK}, + {I40E_INSET_IPV4_TTL, I40E_INSET_IPv4_TTL_MASK}, + {I40E_INSET_IPV6_TC, I40E_INSET_IPV6_TC_MASK}, + {I40E_INSET_IPV6_NEXT_HDR | I40E_INSET_IPV6_HOP_LIMIT, 0}, + {I40E_INSET_IPV6_NEXT_HDR, I40E_INSET_IPV6_NEXT_HDR_MASK}, + {I40E_INSET_IPV6_HOP_LIMIT, I40E_INSET_IPV6_HOP_LIMIT_MASK}, + }; + + if (!inset || !mask || !nb_elem) + return 0; + + for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) { + /* Clear the inset bit, if no MASK is required, + * for example proto + ttl + */ + if ((inset & inset_mask_map[i].inset) == + inset_mask_map[i].inset && inset_mask_map[i].mask == 0) + inset_need_mask &= ~inset_mask_map[i].inset; + if (!inset_need_mask) + return 0; + } + for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) { + if ((inset_need_mask & inset_mask_map[i].inset) == + inset_mask_map[i].inset) { + if (idx >= nb_elem) { + PMD_DRV_LOG(ERR, "exceed maximal number of bitmasks"); + return -EINVAL; + } + mask[idx] = inset_mask_map[i].mask; + idx++; + } + } + + return idx; +} + +static void +i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val) +{ + uint32_t reg = i40e_read_rx_ctl(hw, addr); + + PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x\n", addr, reg); + if (reg != val) + i40e_write_rx_ctl(hw, addr, val); + PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x\n", addr, + (uint32_t)i40e_read_rx_ctl(hw, addr)); +} + +static void +i40e_filter_input_set_init(struct i40e_pf *pf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + enum i40e_filter_pctype pctype; + uint64_t input_set, inset_reg; + uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0}; + int num, i; + + for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; + pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) { + if (!I40E_VALID_PCTYPE(pctype)) + continue; + input_set = i40e_get_default_input_set(pctype); + + num = i40e_generate_inset_mask_reg(input_set, mask_reg, + I40E_INSET_MASK_NUM_REG); + if (num < 0) + return; + inset_reg = i40e_translate_input_set_reg(input_set); + + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); + i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); + + for (i = 0; i < num; i++) { + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), + mask_reg[i]); + i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), + mask_reg[i]); + } + /*clear unused mask registers of the pctype */ + for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) { + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), + 0); + i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), + 0); + } + I40E_WRITE_FLUSH(hw); + + /* store the default input set */ + pf->hash_input_set[pctype] = input_set; + pf->fdir.input_set[pctype] = input_set; + } +} + +int +i40e_hash_filter_inset_select(struct i40e_hw *hw, + struct rte_eth_input_set_conf *conf) +{ + struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf; + enum i40e_filter_pctype pctype; + uint64_t input_set, inset_reg = 0; + uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0}; + int ret, i, num; + + if (!conf) { + PMD_DRV_LOG(ERR, "Invalid pointer"); + return -EFAULT; + } + if (conf->op != RTE_ETH_INPUT_SET_SELECT && + conf->op != RTE_ETH_INPUT_SET_ADD) { + PMD_DRV_LOG(ERR, "Unsupported input set operation"); + return -EINVAL; + } + + if (!I40E_VALID_FLOW(conf->flow_type)) { + PMD_DRV_LOG(ERR, "invalid flow_type input."); + return -EINVAL; + } + pctype = i40e_flowtype_to_pctype(conf->flow_type); + ret = i40e_parse_input_set(&input_set, pctype, conf->field, + conf->inset_size); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to parse input set"); + return -EINVAL; + } + if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_HASH, + input_set) != 0) { + PMD_DRV_LOG(ERR, "Invalid input set"); + return -EINVAL; + } + if (conf->op == RTE_ETH_INPUT_SET_ADD) { + /* get inset value in register */ + inset_reg = i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, pctype)); + inset_reg <<= I40E_32_BIT_WIDTH; + inset_reg |= i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, pctype)); + input_set |= pf->hash_input_set[pctype]; + } + num = i40e_generate_inset_mask_reg(input_set, mask_reg, + I40E_INSET_MASK_NUM_REG); + if (num < 0) + return -EINVAL; + + inset_reg |= i40e_translate_input_set_reg(input_set); + + i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); + + for (i = 0; i < num; i++) + i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), + mask_reg[i]); + /*clear unused mask registers of the pctype */ + for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) + i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype), + 0); + I40E_WRITE_FLUSH(hw); + + pf->hash_input_set[pctype] = input_set; + return 0; +} + +int +i40e_fdir_filter_inset_select(struct i40e_pf *pf, + struct rte_eth_input_set_conf *conf) +{ + struct i40e_hw *hw = I40E_PF_TO_HW(pf); + enum i40e_filter_pctype pctype; + uint64_t input_set, inset_reg = 0; + uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0}; + int ret, i, num; + + if (!hw || !conf) { + PMD_DRV_LOG(ERR, "Invalid pointer"); + return -EFAULT; + } + if (conf->op != RTE_ETH_INPUT_SET_SELECT && + conf->op != RTE_ETH_INPUT_SET_ADD) { + PMD_DRV_LOG(ERR, "Unsupported input set operation"); + return -EINVAL; + } + + if (!I40E_VALID_FLOW(conf->flow_type)) { + PMD_DRV_LOG(ERR, "invalid flow_type input."); + return -EINVAL; + } + pctype = i40e_flowtype_to_pctype(conf->flow_type); + ret = i40e_parse_input_set(&input_set, pctype, conf->field, + conf->inset_size); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to parse input set"); + return -EINVAL; + } + if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR, + input_set) != 0) { + PMD_DRV_LOG(ERR, "Invalid input set"); + return -EINVAL; + } + + /* get inset value in register */ + inset_reg = i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 1)); + inset_reg <<= I40E_32_BIT_WIDTH; + inset_reg |= i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 0)); + + /* Can not change the inset reg for flex payload for fdir, + * it is done by writing I40E_PRTQF_FD_FLXINSET + * in i40e_set_flex_mask_on_pctype. + */ + if (conf->op == RTE_ETH_INPUT_SET_SELECT) + inset_reg &= I40E_REG_INSET_FLEX_PAYLOAD_WORDS; + else + input_set |= pf->fdir.input_set[pctype]; + num = i40e_generate_inset_mask_reg(input_set, mask_reg, + I40E_INSET_MASK_NUM_REG); + if (num < 0) + return -EINVAL; + + inset_reg |= i40e_translate_input_set_reg(input_set); + + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0), + (uint32_t)(inset_reg & UINT32_MAX)); + i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1), + (uint32_t)((inset_reg >> + I40E_32_BIT_WIDTH) & UINT32_MAX)); + + for (i = 0; i < num; i++) + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), + mask_reg[i]); + /*clear unused mask registers of the pctype */ + for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) + i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype), + 0); + I40E_WRITE_FLUSH(hw); + + pf->fdir.input_set[pctype] = input_set; + return 0; +} + +static int +i40e_hash_filter_get(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info) +{ + int ret = 0; + + if (!hw || !info) { + PMD_DRV_LOG(ERR, "Invalid pointer"); + return -EFAULT; + } + + switch (info->info_type) { + case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT: + i40e_get_symmetric_hash_enable_per_port(hw, + &(info->info.enable)); + break; + case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG: + ret = i40e_get_hash_filter_global_config(hw, + &(info->info.global_conf)); + break; + default: + PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported", + info->info_type); + ret = -EINVAL; + break; + } + + return ret; +} + +static int +i40e_hash_filter_set(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info) +{ + int ret = 0; + + if (!hw || !info) { + PMD_DRV_LOG(ERR, "Invalid pointer"); + return -EFAULT; + } + + switch (info->info_type) { + case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT: + i40e_set_symmetric_hash_enable_per_port(hw, info->info.enable); + break; + case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG: + ret = i40e_set_hash_filter_global_config(hw, + &(info->info.global_conf)); + break; + case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT: + ret = i40e_hash_filter_inset_select(hw, + &(info->info.input_set_conf)); + break; + + default: + PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported", + info->info_type); + ret = -EINVAL; + break; + } + + return ret; +} + +/* Operations for hash function */ +static int +i40e_hash_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret = 0; + + switch (filter_op) { + case RTE_ETH_FILTER_NOP: + break; + case RTE_ETH_FILTER_GET: + ret = i40e_hash_filter_get(hw, (struct rte_eth_hash_filter_info *)arg); break; case RTE_ETH_FILTER_SET: @@ -5876,6 +7865,10 @@ i40e_dev_filter_ctrl(struct rte_eth_dev *dev, return -EINVAL; switch (filter_type) { + case RTE_ETH_FILTER_NONE: + /* For global configuration */ + ret = i40e_filter_ctrl_global_config(dev, filter_op, arg); + break; case RTE_ETH_FILTER_HASH: ret = i40e_hash_filter_ctrl(dev, filter_op, arg); break; @@ -5901,16 +7894,64 @@ i40e_dev_filter_ctrl(struct rte_eth_dev *dev, return ret; } +/* + * Check and enable Extended Tag. + * Enabling Extended Tag is important for 40G performance. + */ +static void +i40e_enable_extended_tag(struct rte_eth_dev *dev) +{ + uint32_t buf = 0; + int ret; + + ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf), + PCI_DEV_CAP_REG); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", + PCI_DEV_CAP_REG); + return; + } + if (!(buf & PCI_DEV_CAP_EXT_TAG_MASK)) { + PMD_DRV_LOG(ERR, "Does not support Extended Tag"); + return; + } + + buf = 0; + ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf), + PCI_DEV_CTRL_REG); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", + PCI_DEV_CTRL_REG); + return; + } + if (buf & PCI_DEV_CTRL_EXT_TAG_MASK) { + PMD_DRV_LOG(DEBUG, "Extended Tag has already been enabled"); + return; + } + buf |= PCI_DEV_CTRL_EXT_TAG_MASK; + ret = rte_eal_pci_write_config(dev->pci_dev, &buf, sizeof(buf), + PCI_DEV_CTRL_REG); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to write PCI offset 0x%x", + PCI_DEV_CTRL_REG); + return; + } +} + /* * As some registers wouldn't be reset unless a global hardware reset, * hardware initialization is needed to put those registers into an * expected initial state. */ static void -i40e_hw_init(struct i40e_hw *hw) +i40e_hw_init(struct rte_eth_dev *dev) { + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + i40e_enable_extended_tag(dev); + /* clear the PF Queue Filter control register */ - I40E_WRITE_REG(hw, I40E_PFQF_CTL_0, 0); + i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, 0); /* Disable symmetric hash per port */ i40e_set_symmetric_hash_enable_per_port(hw, 0); @@ -6411,26 +8452,70 @@ i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id) return 0; } -static int -i40e_timesync_enable(struct rte_eth_dev *dev) +static uint64_t +i40e_read_systime_cyclecounter(struct rte_eth_dev *dev) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct rte_eth_link *link = &dev->data->dev_link; - uint32_t tsync_ctl_l; - uint32_t tsync_ctl_h; + uint64_t systim_cycles; + + systim_cycles = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_L); + systim_cycles |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_H) + << 32; + + return systim_cycles; +} + +static uint64_t +i40e_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev, uint8_t index) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint64_t rx_tstamp; + + rx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(index)); + rx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(index)) + << 32; + + return rx_tstamp; +} + +static uint64_t +i40e_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint64_t tx_tstamp; + + tx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_L); + tx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H) + << 32; + + return tx_tstamp; +} + +static void +i40e_start_timecounters(struct rte_eth_dev *dev) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_adapter *adapter = + (struct i40e_adapter *)dev->data->dev_private; + struct rte_eth_link link; uint32_t tsync_inc_l; uint32_t tsync_inc_h; - switch (link->link_speed) { - case ETH_LINK_SPEED_40G: + /* Get current link speed. */ + memset(&link, 0, sizeof(link)); + i40e_dev_link_update(dev, 1); + rte_i40e_dev_atomic_read_link_status(dev, &link); + + switch (link.link_speed) { + case ETH_SPEED_NUM_40G: tsync_inc_l = I40E_PTP_40GB_INCVAL & 0xFFFFFFFF; tsync_inc_h = I40E_PTP_40GB_INCVAL >> 32; break; - case ETH_LINK_SPEED_10G: + case ETH_SPEED_NUM_10G: tsync_inc_l = I40E_PTP_10GB_INCVAL & 0xFFFFFFFF; tsync_inc_h = I40E_PTP_10GB_INCVAL >> 32; break; - case ETH_LINK_SPEED_1000: + case ETH_SPEED_NUM_1G: tsync_inc_l = I40E_PTP_1GB_INCVAL & 0xFFFFFFFF; tsync_inc_h = I40E_PTP_1GB_INCVAL >> 32; break; @@ -6439,19 +8524,95 @@ i40e_timesync_enable(struct rte_eth_dev *dev) tsync_inc_h = 0x0; } - /* Clear timesync registers. */ - I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0); - I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H); - I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(0)); - I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(1)); - I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(2)); - I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(3)); - I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H); - /* Set the timesync increment value. */ I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, tsync_inc_l); I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, tsync_inc_h); + memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter)); + memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter)); + memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter)); + + adapter->systime_tc.cc_mask = I40E_CYCLECOUNTER_MASK; + adapter->systime_tc.cc_shift = 0; + adapter->systime_tc.nsec_mask = 0; + + adapter->rx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK; + adapter->rx_tstamp_tc.cc_shift = 0; + adapter->rx_tstamp_tc.nsec_mask = 0; + + adapter->tx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK; + adapter->tx_tstamp_tc.cc_shift = 0; + adapter->tx_tstamp_tc.nsec_mask = 0; +} + +static int +i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta) +{ + struct i40e_adapter *adapter = + (struct i40e_adapter *)dev->data->dev_private; + + adapter->systime_tc.nsec += delta; + adapter->rx_tstamp_tc.nsec += delta; + adapter->tx_tstamp_tc.nsec += delta; + + return 0; +} + +static int +i40e_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts) +{ + uint64_t ns; + struct i40e_adapter *adapter = + (struct i40e_adapter *)dev->data->dev_private; + + ns = rte_timespec_to_ns(ts); + + /* Set the timecounters to a new value. */ + adapter->systime_tc.nsec = ns; + adapter->rx_tstamp_tc.nsec = ns; + adapter->tx_tstamp_tc.nsec = ns; + + return 0; +} + +static int +i40e_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts) +{ + uint64_t ns, systime_cycles; + struct i40e_adapter *adapter = + (struct i40e_adapter *)dev->data->dev_private; + + systime_cycles = i40e_read_systime_cyclecounter(dev); + ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles); + *ts = rte_ns_to_timespec(ns); + + return 0; +} + +static int +i40e_timesync_enable(struct rte_eth_dev *dev) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t tsync_ctl_l; + uint32_t tsync_ctl_h; + + /* Stop the timesync system time. */ + I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0); + I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0); + /* Reset the timesync system time value. */ + I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_L, 0x0); + I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_H, 0x0); + + i40e_start_timecounters(dev); + + /* Clear timesync registers. */ + I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0); + I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H); + I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(0)); + I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(1)); + I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(2)); + I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(3)); + /* Enable timestamping of PTP packets. */ tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0); tsync_ctl_l |= I40E_PRTTSYN_TSYNENA; @@ -6483,7 +8644,7 @@ i40e_timesync_disable(struct rte_eth_dev *dev) I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l); I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h); - /* Set the timesync increment value. */ + /* Reset the timesync increment value. */ I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0); I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0); @@ -6495,22 +8656,23 @@ i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev, struct timespec *timestamp, uint32_t flags) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_adapter *adapter = + (struct i40e_adapter *)dev->data->dev_private; + uint32_t sync_status; - uint32_t rx_stmpl; - uint32_t rx_stmph; uint32_t index = flags & 0x03; + uint64_t rx_tstamp_cycles; + uint64_t ns; sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_1); if ((sync_status & (1 << index)) == 0) return -EINVAL; - rx_stmpl = I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(index)); - rx_stmph = I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(index)); + rx_tstamp_cycles = i40e_read_rx_tstamp_cyclecounter(dev, index); + ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles); + *timestamp = rte_ns_to_timespec(ns); - timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl); - timestamp->tv_nsec = 0; - - return 0; + return 0; } static int @@ -6518,21 +8680,22 @@ i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev, struct timespec *timestamp) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_adapter *adapter = + (struct i40e_adapter *)dev->data->dev_private; + uint32_t sync_status; - uint32_t tx_stmpl; - uint32_t tx_stmph; + uint64_t tx_tstamp_cycles; + uint64_t ns; sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0); if ((sync_status & I40E_PRTTSYN_STAT_0_TXTIME_MASK) == 0) return -EINVAL; - tx_stmpl = I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_L); - tx_stmph = I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H); - - timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl); - timestamp->tv_nsec = 0; + tx_tstamp_cycles = i40e_read_tx_tstamp_cyclecounter(dev); + ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles); + *timestamp = rte_ns_to_timespec(ns); - return 0; + return 0; } /* @@ -6595,78 +8758,18 @@ i40e_parse_dcb_configure(struct rte_eth_dev *dev, return 0; } -/* - * i40e_vsi_get_bw_info - Query VSI BW Information - * @vsi: the VSI being queried - * - * Returns 0 on success, negative value on failure - */ -static int -i40e_vsi_get_bw_info(struct i40e_vsi *vsi) -{ - struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; - struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; - struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); - int i, ret; - uint32_t tc_bw_max; - - /* Get the VSI level BW configuration */ - ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); - if (ret) { - PMD_INIT_LOG(ERR, - "couldn't get PF vsi bw config, err %s aq_err %s\n", - i40e_stat_str(hw, ret), - i40e_aq_str(hw, hw->aq.asq_last_status)); - return -EINVAL; - } - - /* Get the VSI level BW configuration per TC */ - ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, - NULL); - if (ret) { - PMD_INIT_LOG(ERR, - "couldn't get PF vsi ets bw config, err %s aq_err %s\n", - i40e_stat_str(hw, ret), - i40e_aq_str(hw, hw->aq.asq_last_status)); - return -EINVAL; - } - - if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { - PMD_INIT_LOG(WARNING, - "Enabled TCs mismatch from querying VSI BW info" - " 0x%08x 0x%08x\n", bw_config.tc_valid_bits, - bw_ets_config.tc_valid_bits); - /* Still continuing */ - } - - vsi->bw_info.bw_limit = rte_le_to_cpu_16(bw_config.port_bw_limit); - vsi->bw_info.bw_max_quanta = bw_config.max_bw; - tc_bw_max = rte_le_to_cpu_16(bw_ets_config.tc_bw_max[0]) | - (rte_le_to_cpu_16(bw_ets_config.tc_bw_max[1]) << 16); - for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { - vsi->bw_info.bw_ets_share_credits[i] = - bw_ets_config.share_credits[i]; - vsi->bw_info.bw_ets_limit_credits[i] = - rte_le_to_cpu_16(bw_ets_config.credits[i]); - /* 3 bits out of 4 for each TC */ - vsi->bw_info.bw_ets_max_quanta[i] = - (uint8_t)((tc_bw_max >> (i * 4)) & 0x7); - PMD_INIT_LOG(DEBUG, - "%s: vsi seid = %d, TC = %d, qset = 0x%x\n", - __func__, vsi->seid, i, bw_config.qs_handles[i]); - } - return 0; -} - -static int +static enum i40e_status_code i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi, struct i40e_aqc_vsi_properties_data *info, uint8_t enabled_tcmap) { - int ret, i, total_tc = 0; + enum i40e_status_code ret; + int i, total_tc = 0; uint16_t qpnum_per_tc, bsf, qp_idx; struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi); + struct i40e_pf *pf = I40E_VSI_TO_PF(vsi); + uint16_t used_queues; ret = validate_tcmap_parameter(vsi, enabled_tcmap); if (ret != I40E_SUCCESS) @@ -6680,7 +8783,18 @@ i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi, total_tc = 1; vsi->enabled_tc = enabled_tcmap; - qpnum_per_tc = dev_data->nb_rx_queues / total_tc; + /* different VSI has different queues assigned */ + if (vsi->type == I40E_VSI_MAIN) + used_queues = dev_data->nb_rx_queues - + pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; + else if (vsi->type == I40E_VSI_VMDQ2) + used_queues = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM; + else { + PMD_INIT_LOG(ERR, "unsupported VSI type."); + return I40E_ERR_NO_AVAILABLE_VSI; + } + + qpnum_per_tc = used_queues / total_tc; /* Number of queues per enabled TC */ if (qpnum_per_tc == 0) { PMD_INIT_LOG(ERR, " number of queues is less that tcs."); @@ -6724,6 +8838,93 @@ i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi, return I40E_SUCCESS; } +/* + * i40e_config_switch_comp_tc - Configure VEB tc setting for given TC map + * @veb: VEB to be configured + * @tc_map: enabled TC bitmap + * + * Returns 0 on success, negative value on failure + */ +static enum i40e_status_code +i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map) +{ + struct i40e_aqc_configure_switching_comp_bw_config_data veb_bw; + struct i40e_aqc_query_switching_comp_bw_config_resp bw_query; + struct i40e_aqc_query_switching_comp_ets_config_resp ets_query; + struct i40e_hw *hw = I40E_VSI_TO_HW(veb->associate_vsi); + enum i40e_status_code ret = I40E_SUCCESS; + int i; + uint32_t bw_max; + + /* Check if enabled_tc is same as existing or new TCs */ + if (veb->enabled_tc == tc_map) + return ret; + + /* configure tc bandwidth */ + memset(&veb_bw, 0, sizeof(veb_bw)); + veb_bw.tc_valid_bits = tc_map; + /* Enable ETS TCs with equal BW Share for now across all VSIs */ + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (tc_map & BIT_ULL(i)) + veb_bw.tc_bw_share_credits[i] = 1; + } + ret = i40e_aq_config_switch_comp_bw_config(hw, veb->seid, + &veb_bw, NULL); + if (ret) { + PMD_INIT_LOG(ERR, "AQ command Config switch_comp BW allocation" + " per TC failed = %d", + hw->aq.asq_last_status); + return ret; + } + + memset(&ets_query, 0, sizeof(ets_query)); + ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, + &ets_query, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "Failed to get switch_comp ETS" + " configuration %u", hw->aq.asq_last_status); + return ret; + } + memset(&bw_query, 0, sizeof(bw_query)); + ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, + &bw_query, NULL); + if (ret != I40E_SUCCESS) { + PMD_DRV_LOG(ERR, "Failed to get switch_comp bandwidth" + " configuration %u", hw->aq.asq_last_status); + return ret; + } + + /* store and print out BW info */ + veb->bw_info.bw_limit = rte_le_to_cpu_16(ets_query.port_bw_limit); + veb->bw_info.bw_max = ets_query.tc_bw_max; + PMD_DRV_LOG(DEBUG, "switch_comp bw limit:%u", veb->bw_info.bw_limit); + PMD_DRV_LOG(DEBUG, "switch_comp max_bw:%u", veb->bw_info.bw_max); + bw_max = rte_le_to_cpu_16(bw_query.tc_bw_max[0]) | + (rte_le_to_cpu_16(bw_query.tc_bw_max[1]) << + I40E_16_BIT_WIDTH); + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + veb->bw_info.bw_ets_share_credits[i] = + bw_query.tc_bw_share_credits[i]; + veb->bw_info.bw_ets_credits[i] = + rte_le_to_cpu_16(bw_query.tc_bw_limits[i]); + /* 4 bits per TC, 4th bit is reserved */ + veb->bw_info.bw_ets_max[i] = + (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) & + RTE_LEN2MASK(3, uint8_t)); + PMD_DRV_LOG(DEBUG, "\tVEB TC%u:share credits %u", i, + veb->bw_info.bw_ets_share_credits[i]); + PMD_DRV_LOG(DEBUG, "\tVEB TC%u:credits %u", i, + veb->bw_info.bw_ets_credits[i]); + PMD_DRV_LOG(DEBUG, "\tVEB TC%u: max credits: %u", i, + veb->bw_info.bw_ets_max[i]); + } + + veb->enabled_tc = tc_map; + + return ret; +} + + /* * i40e_vsi_config_tc - Configure VSI tc setting for given TC map * @vsi: VSI to be configured @@ -6731,13 +8932,13 @@ i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi, * * Returns 0 on success, negative value on failure */ -static int -i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 tc_map) +static enum i40e_status_code +i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map) { struct i40e_aqc_configure_vsi_tc_bw_data bw_data; struct i40e_vsi_context ctxt; struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); - int ret = 0; + enum i40e_status_code ret = I40E_SUCCESS; int i; /* Check if enabled_tc is same as existing or new TCs */ @@ -6790,8 +8991,8 @@ i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 tc_map) vsi->info.mapping_flags = ctxt.info.mapping_flags; vsi->info.valid_sections = 0; - /* Update current VSI BW information */ - ret = i40e_vsi_get_bw_info(vsi); + /* query and update current VSI BW information */ + ret = i40e_vsi_get_bw_config(vsi); if (ret) { PMD_INIT_LOG(ERR, "Failed updating vsi bw info, err %s aq_err %s", @@ -6823,11 +9024,13 @@ i40e_dcb_hw_configure(struct i40e_pf *pf, struct i40e_dcbx_config *old_cfg = &hw->local_dcbx_config; struct i40e_vsi *main_vsi = pf->main_vsi; struct i40e_vsi_list *vsi_list; - int i, ret; + enum i40e_status_code ret; + int i; uint32_t val; /* Use the FW API if FW > v4.4*/ - if (!((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4))) { + if (!(((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4)) || + (hw->aq.fw_maj_ver >= 5))) { PMD_INIT_LOG(ERR, "FW < v4.4, can not use FW LLDP API" " to configure DCB"); return I40E_ERR_FIRMWARE_API_VERSION; @@ -6872,15 +9075,27 @@ i40e_dcb_hw_configure(struct i40e_pf *pf, i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_LOCAL, 0, &hw->local_dcbx_config); + /* if Veb is created, need to update TC of it at first */ + if (main_vsi->veb) { + ret = i40e_config_switch_comp_tc(main_vsi->veb, tc_map); + if (ret) + PMD_INIT_LOG(WARNING, + "Failed configuring TC for VEB seid=%d\n", + main_vsi->veb->seid); + } /* Update each VSI */ i40e_vsi_config_tc(main_vsi, tc_map); if (main_vsi->veb) { TAILQ_FOREACH(vsi_list, &main_vsi->veb->head, list) { - /* Beside main VSI, only enable default + /* Beside main VSI and VMDQ VSIs, only enable default * TC for other VSIs */ - ret = i40e_vsi_config_tc(vsi_list->vsi, - I40E_DEFAULT_TCMAP); + if (vsi_list->vsi->type == I40E_VSI_VMDQ2) + ret = i40e_vsi_config_tc(vsi_list->vsi, + tc_map); + else + ret = i40e_vsi_config_tc(vsi_list->vsi, + I40E_DEFAULT_TCMAP); if (ret) PMD_INIT_LOG(WARNING, "Failed configuring TC for VSI seid=%d\n", @@ -7000,9 +9215,8 @@ i40e_dcb_setup(struct rte_eth_dev *dev) return -ENOTSUP; } - if (pf->vf_num != 0 || - (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)) - PMD_INIT_LOG(DEBUG, " DCB only works on main vsi."); + if (pf->vf_num != 0) + PMD_INIT_LOG(DEBUG, " DCB only works on pf and vmdq vsis."); ret = i40e_parse_dcb_configure(dev, &dcb_cfg, &tc_map); if (ret) { @@ -7014,5 +9228,253 @@ i40e_dcb_setup(struct rte_eth_dev *dev) PMD_INIT_LOG(ERR, "dcb sw configure fails"); return -ENOSYS; } + + return 0; +} + +static int +i40e_dev_get_dcb_info(struct rte_eth_dev *dev, + struct rte_eth_dcb_info *dcb_info) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct i40e_vsi *vsi = pf->main_vsi; + struct i40e_dcbx_config *dcb_cfg = &hw->local_dcbx_config; + uint16_t bsf, tc_mapping; + int i, j = 0; + + if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG) + dcb_info->nb_tcs = rte_bsf32(vsi->enabled_tc + 1); + else + dcb_info->nb_tcs = 1; + for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) + dcb_info->prio_tc[i] = dcb_cfg->etscfg.prioritytable[i]; + for (i = 0; i < dcb_info->nb_tcs; i++) + dcb_info->tc_bws[i] = dcb_cfg->etscfg.tcbwtable[i]; + + /* get queue mapping if vmdq is disabled */ + if (!pf->nb_cfg_vmdq_vsi) { + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (!(vsi->enabled_tc & (1 << i))) + continue; + tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]); + dcb_info->tc_queue.tc_rxq[j][i].base = + (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >> + I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT; + dcb_info->tc_queue.tc_txq[j][i].base = + dcb_info->tc_queue.tc_rxq[j][i].base; + bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >> + I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT; + dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf; + dcb_info->tc_queue.tc_txq[j][i].nb_queue = + dcb_info->tc_queue.tc_rxq[j][i].nb_queue; + } + return 0; + } + + /* get queue mapping if vmdq is enabled */ + do { + vsi = pf->vmdq[j].vsi; + for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { + if (!(vsi->enabled_tc & (1 << i))) + continue; + tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]); + dcb_info->tc_queue.tc_rxq[j][i].base = + (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >> + I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT; + dcb_info->tc_queue.tc_txq[j][i].base = + dcb_info->tc_queue.tc_rxq[j][i].base; + bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >> + I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT; + dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf; + dcb_info->tc_queue.tc_txq[j][i].nb_queue = + dcb_info->tc_queue.tc_rxq[j][i].nb_queue; + } + j++; + } while (j < RTE_MIN(pf->nb_cfg_vmdq_vsi, ETH_MAX_VMDQ_POOL)); + return 0; +} + +static int +i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint16_t interval = + i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL); + uint16_t msix_intr; + + msix_intr = intr_handle->intr_vec[queue_id]; + if (msix_intr == I40E_MISC_VEC_ID) + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | + (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | + (interval << + I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT)); + else + I40E_WRITE_REG(hw, + I40E_PFINT_DYN_CTLN(msix_intr - + I40E_RX_VEC_START), + I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | + (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | + (interval << + I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT)); + + I40E_WRITE_FLUSH(hw); + rte_intr_enable(&dev->pci_dev->intr_handle); + + return 0; +} + +static int +i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) +{ + struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle; + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint16_t msix_intr; + + msix_intr = intr_handle->intr_vec[queue_id]; + if (msix_intr == I40E_MISC_VEC_ID) + I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0); + else + I40E_WRITE_REG(hw, + I40E_PFINT_DYN_CTLN(msix_intr - + I40E_RX_VEC_START), + 0); + I40E_WRITE_FLUSH(hw); + + return 0; +} + +static int i40e_get_regs(struct rte_eth_dev *dev, + struct rte_dev_reg_info *regs) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t *ptr_data = regs->data; + uint32_t reg_idx, arr_idx, arr_idx2, reg_offset; + const struct i40e_reg_info *reg_info; + + if (ptr_data == NULL) { + regs->length = I40E_GLGEN_STAT_CLEAR + 4; + regs->width = sizeof(uint32_t); + return 0; + } + + /* The first few registers have to be read using AQ operations */ + reg_idx = 0; + while (i40e_regs_adminq[reg_idx].name) { + reg_info = &i40e_regs_adminq[reg_idx++]; + for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++) + for (arr_idx2 = 0; + arr_idx2 <= reg_info->count2; + arr_idx2++) { + reg_offset = arr_idx * reg_info->stride1 + + arr_idx2 * reg_info->stride2; + reg_offset += reg_info->base_addr; + ptr_data[reg_offset >> 2] = + i40e_read_rx_ctl(hw, reg_offset); + } + } + + /* The remaining registers can be read using primitives */ + reg_idx = 0; + while (i40e_regs_others[reg_idx].name) { + reg_info = &i40e_regs_others[reg_idx++]; + for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++) + for (arr_idx2 = 0; + arr_idx2 <= reg_info->count2; + arr_idx2++) { + reg_offset = arr_idx * reg_info->stride1 + + arr_idx2 * reg_info->stride2; + reg_offset += reg_info->base_addr; + ptr_data[reg_offset >> 2] = + I40E_READ_REG(hw, reg_offset); + } + } + + return 0; +} + +static int i40e_get_eeprom_length(struct rte_eth_dev *dev) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + /* Convert word count to byte count */ + return hw->nvm.sr_size << 1; +} + +static int i40e_get_eeprom(struct rte_eth_dev *dev, + struct rte_dev_eeprom_info *eeprom) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint16_t *data = eeprom->data; + uint16_t offset, length, cnt_words; + int ret_code; + + offset = eeprom->offset >> 1; + length = eeprom->length >> 1; + cnt_words = length; + + if (offset > hw->nvm.sr_size || + offset + length > hw->nvm.sr_size) { + PMD_DRV_LOG(ERR, "Requested EEPROM bytes out of range."); + return -EINVAL; + } + + eeprom->magic = hw->vendor_id | (hw->device_id << 16); + + ret_code = i40e_read_nvm_buffer(hw, offset, &cnt_words, data); + if (ret_code != I40E_SUCCESS || cnt_words != length) { + PMD_DRV_LOG(ERR, "EEPROM read failed."); + return -EIO; + } + return 0; } + +static void i40e_set_default_mac_addr(struct rte_eth_dev *dev, + struct ether_addr *mac_addr) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (!is_valid_assigned_ether_addr(mac_addr)) { + PMD_DRV_LOG(ERR, "Tried to set invalid MAC address."); + return; + } + + /* Flags: 0x3 updates port address */ + i40e_aq_mac_address_write(hw, 0x3, mac_addr->addr_bytes, NULL); +} + +static int +i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct rte_eth_dev_data *dev_data = pf->dev_data; + uint32_t frame_size = mtu + ETHER_HDR_LEN + + ETHER_CRC_LEN + I40E_VLAN_TAG_SIZE; + int ret = 0; + + /* check if mtu is within the allowed range */ + if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX)) + return -EINVAL; + + /* mtu setting is forbidden if port is start */ + if (dev_data->dev_started) { + PMD_DRV_LOG(ERR, + "port %d must be stopped before configuration\n", + dev_data->port_id); + return -EBUSY; + } + + if (frame_size > ETHER_MAX_LEN) + dev_data->dev_conf.rxmode.jumbo_frame = 1; + else + dev_data->dev_conf.rxmode.jumbo_frame = 0; + + dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size; + + return ret; +}