X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_pmd_e1000%2Figb_ethdev.c;h=4415155c02ea02208282b7e40dd975f03c8e76c9;hb=6065355a03fc743234d4c11fc8e17093bf072617;hp=f13da4041273f83acd0918f640b684029a84278b;hpb=5caeb1b143da211d545bd9e59e9a93c830fc778f;p=dpdk.git diff --git a/lib/librte_pmd_e1000/igb_ethdev.c b/lib/librte_pmd_e1000/igb_ethdev.c index f13da40412..4415155c02 100644 --- a/lib/librte_pmd_e1000/igb_ethdev.c +++ b/lib/librte_pmd_e1000/igb_ethdev.c @@ -1,13 +1,13 @@ /*- * BSD LICENSE - * - * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. * All rights reserved. - * + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: - * + * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright @@ -17,7 +17,7 @@ * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -47,15 +47,33 @@ #include #include #include -#include #include #include #include +#include #include "e1000_logs.h" #include "e1000/e1000_api.h" #include "e1000_ethdev.h" +/* + * Default values for port configuration + */ +#define IGB_DEFAULT_RX_FREE_THRESH 32 +#define IGB_DEFAULT_RX_PTHRESH 8 +#define IGB_DEFAULT_RX_HTHRESH 8 +#define IGB_DEFAULT_RX_WTHRESH 0 + +#define IGB_DEFAULT_TX_PTHRESH 32 +#define IGB_DEFAULT_TX_HTHRESH 0 +#define IGB_DEFAULT_TX_WTHRESH 0 + +/* Bit shift and mask */ +#define IGB_4_BIT_WIDTH (CHAR_BIT / 2) +#define IGB_4_BIT_MASK RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t) +#define IGB_8_BIT_WIDTH CHAR_BIT +#define IGB_8_BIT_MASK UINT8_MAX + static int eth_igb_configure(struct rte_eth_dev *dev); static int eth_igb_start(struct rte_eth_dev *dev); static void eth_igb_stop(struct rte_eth_dev *dev); @@ -70,7 +88,11 @@ static void eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats); static void eth_igb_stats_reset(struct rte_eth_dev *dev); static void eth_igb_infos_get(struct rte_eth_dev *dev, + struct rte_eth_dev_info *dev_info); +static void eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); +static int eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf); static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev); @@ -84,6 +106,8 @@ static void igb_hw_control_release(struct e1000_hw *hw); static void igb_init_manageability(struct e1000_hw *hw); static void igb_release_manageability(struct e1000_hw *hw); +static int eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); + static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id); @@ -114,14 +138,61 @@ static void igbvf_dev_close(struct rte_eth_dev *dev); static int eth_igbvf_link_update(struct e1000_hw *hw); static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats); static void eth_igbvf_stats_reset(struct rte_eth_dev *dev); -static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, +static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on); static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on); static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, - struct rte_eth_rss_reta *reta_conf); + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, - struct rte_eth_rss_reta *reta_conf); + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); + +static int eth_igb_syn_filter_set(struct rte_eth_dev *dev, + struct rte_eth_syn_filter *filter, + bool add); +static int eth_igb_syn_filter_get(struct rte_eth_dev *dev, + struct rte_eth_syn_filter *filter); +static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg); +static int igb_add_2tuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter); +static int igb_remove_2tuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter); +static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, + struct rte_eth_flex_filter *filter, + bool add); +static int eth_igb_get_flex_filter(struct rte_eth_dev *dev, + struct rte_eth_flex_filter *filter); +static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg); +static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter); +static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter); +static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *filter, + bool add); +static int igb_get_ntuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *filter); +static int igb_ntuple_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg); +static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev, + struct rte_eth_ethertype_filter *filter, + bool add); +static int igb_ethertype_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg); +static int igb_get_ethertype_filter(struct rte_eth_dev *dev, + struct rte_eth_ethertype_filter *filter); +static int eth_igb_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg); /* * Define VF Stats MACRO for Non "cleared on read" register @@ -145,26 +216,26 @@ static enum e1000_fc_mode igb_fc_setting = e1000_fc_full; /* * The set of PCI devices this driver supports */ -static struct rte_pci_id pci_id_igb_map[] = { +static const struct rte_pci_id pci_id_igb_map[] = { #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, #include "rte_pci_dev_ids.h" -{.device_id = 0}, +{0}, }; /* * The set of PCI devices this driver supports (for 82576&I350 VF) */ -static struct rte_pci_id pci_id_igbvf_map[] = { +static const struct rte_pci_id pci_id_igbvf_map[] = { #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, #include "rte_pci_dev_ids.h" -{.device_id = 0}, +{0}, }; -static struct eth_dev_ops eth_igb_ops = { +static const struct eth_dev_ops eth_igb_ops = { .dev_configure = eth_igb_configure, .dev_start = eth_igb_start, .dev_stop = eth_igb_stop, @@ -177,6 +248,7 @@ static struct eth_dev_ops eth_igb_ops = { .stats_get = eth_igb_stats_get, .stats_reset = eth_igb_stats_reset, .dev_infos_get = eth_igb_infos_get, + .mtu_set = eth_igb_mtu_set, .vlan_filter_set = eth_igb_vlan_filter_set, .vlan_tpid_set = eth_igb_vlan_tpid_set, .vlan_offload_set = eth_igb_vlan_offload_set, @@ -188,18 +260,22 @@ static struct eth_dev_ops eth_igb_ops = { .tx_queue_release = eth_igb_tx_queue_release, .dev_led_on = eth_igb_led_on, .dev_led_off = eth_igb_led_off, + .flow_ctrl_get = eth_igb_flow_ctrl_get, .flow_ctrl_set = eth_igb_flow_ctrl_set, .mac_addr_add = eth_igb_rar_set, .mac_addr_remove = eth_igb_rar_clear, .reta_update = eth_igb_rss_reta_update, .reta_query = eth_igb_rss_reta_query, + .rss_hash_update = eth_igb_rss_hash_update, + .rss_hash_conf_get = eth_igb_rss_hash_conf_get, + .filter_ctrl = eth_igb_filter_ctrl, }; /* * dev_ops for virtual function, bare necessities for basic vf * operation have been implemented */ -static struct eth_dev_ops igbvf_eth_dev_ops = { +static const struct eth_dev_ops igbvf_eth_dev_ops = { .dev_configure = igbvf_dev_configure, .dev_start = igbvf_dev_start, .dev_stop = igbvf_dev_stop, @@ -208,7 +284,7 @@ static struct eth_dev_ops igbvf_eth_dev_ops = { .stats_get = eth_igbvf_stats_get, .stats_reset = eth_igbvf_stats_reset, .vlan_filter_set = igbvf_vlan_filter_set, - .dev_infos_get = eth_igb_infos_get, + .dev_infos_get = eth_igbvf_infos_get, .rx_queue_setup = eth_igb_rx_queue_setup, .rx_queue_release = eth_igb_rx_queue_release, .tx_queue_setup = eth_igb_tx_queue_setup, @@ -274,7 +350,7 @@ igb_intr_enable(struct rte_eth_dev *dev) E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - + E1000_WRITE_REG(hw, E1000_IMS, intr->mask); E1000_WRITE_FLUSH(hw); } @@ -291,18 +367,18 @@ igb_pf_reset_hw(struct e1000_hw *hw) { uint32_t ctrl_ext; int32_t status; - + status = e1000_reset_hw(hw); - + ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); /* Set PF Reset Done bit so PF/VF Mail Ops can work */ ctrl_ext |= E1000_CTRL_EXT_PFRSTD; E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); E1000_WRITE_FLUSH(hw); - + return status; } - + static void igb_identify_hardware(struct rte_eth_dev *dev) { @@ -320,8 +396,63 @@ igb_identify_hardware(struct rte_eth_dev *dev) } static int -eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, - struct rte_eth_dev *eth_dev) +igb_reset_swfw_lock(struct e1000_hw *hw) +{ + int ret_val; + + /* + * Do mac ops initialization manually here, since we will need + * some function pointers set by this call. + */ + ret_val = e1000_init_mac_params(hw); + if (ret_val) + return ret_val; + + /* + * SMBI lock should not fail in this early stage. If this is the case, + * it is due to an improper exit of the application. + * So force the release of the faulty lock. + */ + if (e1000_get_hw_semaphore_generic(hw) < 0) { + PMD_DRV_LOG(DEBUG, "SMBI lock released"); + } + e1000_put_hw_semaphore_generic(hw); + + if (hw->mac.ops.acquire_swfw_sync != NULL) { + uint16_t mask; + + /* + * Phy lock should not fail in this early stage. If this is the case, + * it is due to an improper exit of the application. + * So force the release of the faulty lock. + */ + mask = E1000_SWFW_PHY0_SM << hw->bus.func; + if (hw->bus.func > E1000_FUNC_1) + mask <<= 2; + if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { + PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", + hw->bus.func); + } + hw->mac.ops.release_swfw_sync(hw, mask); + + /* + * This one is more tricky since it is common to all ports; but + * swfw_sync retries last long enough (1s) to be almost sure that if + * lock can not be taken it is due to an improper lock of the + * semaphore. + */ + mask = E1000_SWFW_EEP_SM; + if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) { + PMD_DRV_LOG(DEBUG, "SWFW common locks released"); + } + hw->mac.ops.release_swfw_sync(hw, mask); + } + + return E1000_SUCCESS; +} + +static int +eth_igb_dev_init(struct rte_eth_dev *eth_dev) { int error = 0; struct rte_pci_device *pci_dev; @@ -329,6 +460,8 @@ eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); struct e1000_vfta * shadow_vfta = E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private); uint32_t ctrl_ext; pci_dev = eth_dev->pci_dev; @@ -348,13 +481,25 @@ eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, hw->hw_addr= (void *)pci_dev->mem_resource[0].addr; igb_identify_hardware(eth_dev); - if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { + if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) { error = -EIO; goto err_late; } e1000_get_bus_info(hw); + /* Reset any pending lock */ + if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) { + error = -EIO; + goto err_late; + } + + /* Finish initialization */ + if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) { + error = -EIO; + goto err_late; + } + hw->mac.autoneg = 1; hw->phy.autoneg_wait_to_complete = 0; hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX; @@ -428,14 +573,14 @@ eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, /* initialize PF if max_vfs not zero */ igb_pf_host_init(eth_dev); - + ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); /* Set PF Reset Done bit so PF/VF Mail Ops can work */ ctrl_ext |= E1000_CTRL_EXT_PFRSTD; E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); E1000_WRITE_FLUSH(hw); - PMD_INIT_LOG(INFO, "port_id %d vendorID=0x%x deviceID=0x%x\n", + PMD_INIT_LOG(INFO, "port_id %d vendorID=0x%x deviceID=0x%x", eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id); @@ -444,10 +589,17 @@ eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, /* enable uio intr after callback register */ rte_intr_enable(&(pci_dev->intr_handle)); - + /* enable support intr */ igb_intr_enable(eth_dev); - + + TAILQ_INIT(&filter_info->flex_list); + filter_info->flex_mask = 0; + TAILQ_INIT(&filter_info->twotuple_list); + filter_info->twotuple_mask = 0; + TAILQ_INIT(&filter_info->fivetuple_list); + filter_info->fivetuple_mask = 0; + return 0; err_late: @@ -460,15 +612,14 @@ err_late: * Virtual Function device init */ static int -eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, - struct rte_eth_dev *eth_dev) +eth_igbvf_dev_init(struct rte_eth_dev *eth_dev) { struct rte_pci_device *pci_dev; struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); int diag; - PMD_INIT_LOG(DEBUG, "eth_igbvf_dev_init"); + PMD_INIT_FUNC_TRACE(); eth_dev->dev_ops = &igbvf_eth_dev_ops; eth_dev->rx_pkt_burst = ð_igb_recv_pkts; @@ -489,7 +640,7 @@ eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, hw->vendor_id = pci_dev->id.vendor_id; hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; - /* Initialize the shared code */ + /* Initialize the shared code (base driver) */ diag = e1000_setup_init_funcs(hw, TRUE); if (diag != 0) { PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d", @@ -502,7 +653,7 @@ eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, /* Disable the interrupts for VF */ igbvf_intr_disable(hw); - + diag = hw->mac.ops.reset_hw(hw); /* Allocate memory for storing MAC addresses */ @@ -515,16 +666,15 @@ eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv, ETHER_ADDR_LEN * hw->mac.rar_entry_count); return -ENOMEM; } - + /* Copy the permanent MAC address */ ether_addr_copy((struct ether_addr *) hw->mac.perm_addr, ð_dev->data->mac_addrs[0]); - PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x " - "mac.type=%s\n", - eth_dev->data->port_id, pci_dev->id.vendor_id, - pci_dev->id.device_id, - "igb_mac_82576_vf"); + PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x " + "mac.type=%s", + eth_dev->data->port_id, pci_dev->id.vendor_id, + pci_dev->id.device_id, "igb_mac_82576_vf"); return 0; } @@ -533,9 +683,7 @@ static struct eth_driver rte_igb_pmd = { { .name = "rte_igb_pmd", .id_table = pci_id_igb_map, -#ifdef RTE_EAL_UNBIND_PORTS - .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO, -#endif + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, }, .eth_dev_init = eth_igb_dev_init, .dev_private_size = sizeof(struct e1000_adapter), @@ -548,30 +696,39 @@ static struct eth_driver rte_igbvf_pmd = { { .name = "rte_igbvf_pmd", .id_table = pci_id_igbvf_map, -#ifdef RTE_EAL_UNBIND_PORTS - .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO, -#endif + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, }, .eth_dev_init = eth_igbvf_dev_init, .dev_private_size = sizeof(struct e1000_adapter), }; -int -rte_igb_pmd_init(void) +static int +rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused) { rte_eth_driver_register(&rte_igb_pmd); return 0; } +static void +igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = + E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + /* RCTL: enable VLAN filter since VMDq always use VLAN filter */ + uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL); + rctl |= E1000_RCTL_VFE; + E1000_WRITE_REG(hw, E1000_RCTL, rctl); +} + /* * VF Driver initialization routine. * Invoked one at EAL init time. * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices. */ -int -rte_igbvf_pmd_init(void) +static int +rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused) { - DEBUGFUNC("rte_igbvf_pmd_init"); + PMD_INIT_FUNC_TRACE(); rte_eth_driver_register(&rte_igbvf_pmd); return (0); @@ -583,11 +740,9 @@ eth_igb_configure(struct rte_eth_dev *dev) struct e1000_interrupt *intr = E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private); - PMD_INIT_LOG(DEBUG, ">>"); - + PMD_INIT_FUNC_TRACE(); intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; - - PMD_INIT_LOG(DEBUG, "<<"); + PMD_INIT_FUNC_TRACE(); return (0); } @@ -600,7 +755,7 @@ eth_igb_start(struct rte_eth_dev *dev) int ret, i, mask; uint32_t ctrl_ext; - PMD_INIT_LOG(DEBUG, ">>"); + PMD_INIT_FUNC_TRACE(); /* Power up the phy. Needed to make the link go Up */ e1000_power_up_phy(hw); @@ -626,7 +781,7 @@ eth_igb_start(struct rte_eth_dev *dev) return (-EIO); } - E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN); + E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); /* Set PF Reset Done bit so PF/VF Mail Ops can work */ @@ -659,6 +814,11 @@ eth_igb_start(struct rte_eth_dev *dev) ETH_VLAN_EXTEND_MASK; eth_igb_vlan_offload_set(dev, mask); + if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) { + /* Enable VLAN filter since VMDq always use VLAN filter */ + igb_vmdq_vlan_hw_filter_enable(dev); + } + /* * Configure the Interrupt Moderation register (EITR) with the maximum * possible value (0xFFFF) to minimize "System Partial Write" issued by @@ -677,7 +837,8 @@ eth_igb_start(struct rte_eth_dev *dev) * value of Write-Back Threshold registers. */ if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) || - (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210)) { + (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) || + (hw->mac.type == e1000_i211)) { uint32_t ivar; /* Enable all RX & TX queues in the IVAR registers */ @@ -746,9 +907,9 @@ eth_igb_start(struct rte_eth_dev *dev) return (0); error_invalid_config: - PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u\n", - dev->data->dev_conf.link_speed, - dev->data->dev_conf.link_duplex, dev->data->port_id); + PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u", + dev->data->dev_conf.link_speed, + dev->data->dev_conf.link_duplex, dev->data->port_id); igb_dev_clear_queues(dev); return (-EINVAL); } @@ -763,7 +924,12 @@ static void eth_igb_stop(struct rte_eth_dev *dev) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); struct rte_eth_link link; + struct e1000_flex_filter *p_flex; + struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next; + struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next; igb_intr_disable(hw); igb_pf_reset_hw(hw); @@ -779,13 +945,41 @@ eth_igb_stop(struct rte_eth_dev *dev) } /* Power down the phy. Needed to make the link go Down */ - e1000_power_down_phy(hw); + if (hw->phy.media_type == e1000_media_type_copper) + e1000_power_down_phy(hw); + else + e1000_shutdown_fiber_serdes_link(hw); igb_dev_clear_queues(dev); /* clear the recorded link status */ memset(&link, 0, sizeof(link)); rte_igb_dev_atomic_write_link_status(dev, &link); + + /* Remove all flex filters of the device */ + while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) { + TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries); + rte_free(p_flex); + } + filter_info->flex_mask = 0; + + /* Remove all ntuple filters of the device */ + for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list); + p_5tuple != NULL; p_5tuple = p_5tuple_next) { + p_5tuple_next = TAILQ_NEXT(p_5tuple, entries); + TAILQ_REMOVE(&filter_info->fivetuple_list, + p_5tuple, entries); + rte_free(p_5tuple); + } + filter_info->fivetuple_mask = 0; + for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list); + p_2tuple != NULL; p_2tuple = p_2tuple_next) { + p_2tuple_next = TAILQ_NEXT(p_2tuple, entries); + TAILQ_REMOVE(&filter_info->twotuple_list, + p_2tuple, entries); + rte_free(p_2tuple); + } + filter_info->twotuple_mask = 0; } static void @@ -825,7 +1019,7 @@ igb_get_rx_buffer_size(struct e1000_hw *hw) rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf); rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size); rx_buf_size = (rx_buf_size << 10); - } else if (hw->mac.type == e1000_i210) { + } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) { rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10; } else { rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10; @@ -855,7 +1049,7 @@ igb_hardware_init(struct e1000_hw *hw) * frames to be received after sending an XOFF. * - Low water mark works best when it is very near the high water mark. * This allows the receiver to restart by sending XON when it has - * drained a bit. Here we use an arbitary value of 1500 which will + * drained a bit. Here we use an arbitrary value of 1500 which will * restart after one full frame is pulled from the buffer. There * could be several smaller frames in the buffer and if so they will * not trigger the XON until their total number reduces the buffer @@ -883,7 +1077,7 @@ igb_hardware_init(struct e1000_hw *hw) if (diag < 0) return (diag); - E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN); + E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN); e1000_get_phy_info(hw); e1000_check_for_link(hw); @@ -1005,12 +1199,23 @@ eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) return; /* Rx Errors */ - rte_stats->ierrors = stats->rxerrc + stats->crcerrs + stats->algnerrc + - stats->ruc + stats->roc + stats->mpc + stats->cexterr; + rte_stats->ibadcrc = stats->crcerrs; + rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc; + rte_stats->imissed = stats->mpc; + rte_stats->ierrors = rte_stats->ibadcrc + + rte_stats->ibadlen + + rte_stats->imissed + + stats->rxerrc + stats->algnerrc + stats->cexterr; /* Tx Errors */ rte_stats->oerrors = stats->ecol + stats->latecol; + /* XON/XOFF pause frames */ + rte_stats->tx_pause_xon = stats->xontxc; + rte_stats->rx_pause_xon = stats->xonrxc; + rte_stats->tx_pause_xoff = stats->xofftxc; + rte_stats->rx_pause_xoff = stats->xoffrxc; + rte_stats->ipackets = stats->gprc; rte_stats->opackets = stats->gptc; rte_stats->ibytes = stats->gorc; @@ -1076,7 +1281,6 @@ eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats) if (rte_stats == NULL) return; - memset(rte_stats, 0, sizeof(*rte_stats)); rte_stats->ipackets = hw_stats->gprc; rte_stats->ibytes = hw_stats->gorc; rte_stats->opackets = hw_stats->gptc; @@ -1105,32 +1309,54 @@ eth_igbvf_stats_reset(struct rte_eth_dev *dev) } static void -eth_igb_infos_get(struct rte_eth_dev *dev, - struct rte_eth_dev_info *dev_info) +eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ dev_info->max_mac_addrs = hw->mac.rar_entry_count; + dev_info->rx_offload_capa = + DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + dev_info->tx_offload_capa = + DEV_TX_OFFLOAD_VLAN_INSERT | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_SCTP_CKSUM; switch (hw->mac.type) { case e1000_82575: dev_info->max_rx_queues = 4; dev_info->max_tx_queues = 4; + dev_info->max_vmdq_pools = 0; break; case e1000_82576: dev_info->max_rx_queues = 16; dev_info->max_tx_queues = 16; + dev_info->max_vmdq_pools = ETH_8_POOLS; + dev_info->vmdq_queue_num = 16; break; case e1000_82580: dev_info->max_rx_queues = 8; dev_info->max_tx_queues = 8; + dev_info->max_vmdq_pools = ETH_8_POOLS; + dev_info->vmdq_queue_num = 8; break; case e1000_i350: + dev_info->max_rx_queues = 8; + dev_info->max_tx_queues = 8; + dev_info->max_vmdq_pools = ETH_8_POOLS; + dev_info->vmdq_queue_num = 8; + break; + + case e1000_i354: dev_info->max_rx_queues = 8; dev_info->max_tx_queues = 8; break; @@ -1138,23 +1364,91 @@ eth_igb_infos_get(struct rte_eth_dev *dev, case e1000_i210: dev_info->max_rx_queues = 4; dev_info->max_tx_queues = 4; + dev_info->max_vmdq_pools = 0; break; - case e1000_vfadapt: + case e1000_i211: dev_info->max_rx_queues = 2; dev_info->max_tx_queues = 2; + dev_info->max_vmdq_pools = 0; + break; + + default: + /* Should not happen */ break; + } + dev_info->reta_size = ETH_RSS_RETA_SIZE_128; + dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL; + + dev_info->default_rxconf = (struct rte_eth_rxconf) { + .rx_thresh = { + .pthresh = IGB_DEFAULT_RX_PTHRESH, + .hthresh = IGB_DEFAULT_RX_HTHRESH, + .wthresh = IGB_DEFAULT_RX_WTHRESH, + }, + .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, + .rx_drop_en = 0, + }; + + dev_info->default_txconf = (struct rte_eth_txconf) { + .tx_thresh = { + .pthresh = IGB_DEFAULT_TX_PTHRESH, + .hthresh = IGB_DEFAULT_TX_HTHRESH, + .wthresh = IGB_DEFAULT_TX_WTHRESH, + }, + .txq_flags = 0, + }; +} + +static void +eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */ + dev_info->max_rx_pktlen = 0x3FFF; /* See RLPML register. */ + dev_info->max_mac_addrs = hw->mac.rar_entry_count; + dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_SCTP_CKSUM; + switch (hw->mac.type) { + case e1000_vfadapt: + dev_info->max_rx_queues = 2; + dev_info->max_tx_queues = 2; + break; case e1000_vfadapt_i350: dev_info->max_rx_queues = 1; dev_info->max_tx_queues = 1; break; - default: /* Should not happen */ - dev_info->max_rx_queues = 0; - dev_info->max_tx_queues = 0; + break; } + + dev_info->default_rxconf = (struct rte_eth_rxconf) { + .rx_thresh = { + .pthresh = IGB_DEFAULT_RX_PTHRESH, + .hthresh = IGB_DEFAULT_RX_HTHRESH, + .wthresh = IGB_DEFAULT_RX_WTHRESH, + }, + .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH, + .rx_drop_en = 0, + }; + + dev_info->default_txconf = (struct rte_eth_txconf) { + .tx_thresh = { + .pthresh = IGB_DEFAULT_TX_PTHRESH, + .hthresh = IGB_DEFAULT_TX_HTHRESH, + .wthresh = IGB_DEFAULT_TX_WTHRESH, + }, + .txq_flags = 0, + }; } /* return 0 means link status changed, -1 means not changed */ @@ -1495,14 +1789,14 @@ eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask) else igb_vlan_hw_strip_disable(dev); } - + if(mask & ETH_VLAN_FILTER_MASK){ if (dev->data->dev_conf.rxmode.hw_vlan_filter) igb_vlan_hw_filter_enable(dev); else igb_vlan_hw_filter_disable(dev); } - + if(mask & ETH_VLAN_EXTEND_MASK){ if (dev->data->dev_conf.rxmode.hw_vlan_extend) igb_vlan_hw_extend_enable(dev); @@ -1563,7 +1857,7 @@ eth_igb_interrupt_get_status(struct rte_eth_dev *dev) intr->flags |= E1000_FLAG_NEED_LINK_UPDATE; } - if (icr & E1000_ICR_VMMB) + if (icr & E1000_ICR_VMMB) intr->flags |= E1000_FLAG_MAILBOX; return 0; @@ -1613,19 +1907,20 @@ eth_igb_interrupt_action(struct rte_eth_dev *dev) rte_igb_dev_atomic_read_link_status(dev, &link); if (link.link_status) { PMD_INIT_LOG(INFO, - " Port %d: Link Up - speed %u Mbps - %s\n", - dev->data->port_id, (unsigned)link.link_speed, - link.link_duplex == ETH_LINK_FULL_DUPLEX ? - "full-duplex" : "half-duplex"); + " Port %d: Link Up - speed %u Mbps - %s", + dev->data->port_id, + (unsigned)link.link_speed, + link.link_duplex == ETH_LINK_FULL_DUPLEX ? + "full-duplex" : "half-duplex"); } else { - PMD_INIT_LOG(INFO, " Port %d: Link Down\n", - dev->data->port_id); + PMD_INIT_LOG(INFO, " Port %d: Link Down", + dev->data->port_id); } PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d", - dev->pci_dev->addr.domain, - dev->pci_dev->addr.bus, - dev->pci_dev->addr.devid, - dev->pci_dev->addr.function); + dev->pci_dev->addr.domain, + dev->pci_dev->addr.bus, + dev->pci_dev->addr.devid, + dev->pci_dev->addr.function); tctl = E1000_READ_REG(hw, E1000_TCTL); rctl = E1000_READ_REG(hw, E1000_RCTL); if (link.link_status) { @@ -1685,6 +1980,48 @@ eth_igb_led_off(struct rte_eth_dev *dev) return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP); } +static int +eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) +{ + struct e1000_hw *hw; + uint32_t ctrl; + int tx_pause; + int rx_pause; + + hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + fc_conf->pause_time = hw->fc.pause_time; + fc_conf->high_water = hw->fc.high_water; + fc_conf->low_water = hw->fc.low_water; + fc_conf->send_xon = hw->fc.send_xon; + fc_conf->autoneg = hw->mac.autoneg; + + /* + * Return rx_pause and tx_pause status according to actual setting of + * the TFCE and RFCE bits in the CTRL register. + */ + ctrl = E1000_READ_REG(hw, E1000_CTRL); + if (ctrl & E1000_CTRL_TFCE) + tx_pause = 1; + else + tx_pause = 0; + + if (ctrl & E1000_CTRL_RFCE) + rx_pause = 1; + else + rx_pause = 0; + + if (rx_pause && tx_pause) + fc_conf->mode = RTE_FC_FULL; + else if (rx_pause) + fc_conf->mode = RTE_FC_RX_PAUSE; + else if (tx_pause) + fc_conf->mode = RTE_FC_TX_PAUSE; + else + fc_conf->mode = RTE_FC_NONE; + + return 0; +} + static int eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { @@ -1698,17 +2035,20 @@ eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) }; uint32_t rx_buf_size; uint32_t max_high_water; + uint32_t rctl; hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + if (fc_conf->autoneg != hw->mac.autoneg) + return -ENOTSUP; rx_buf_size = igb_get_rx_buffer_size(hw); - PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size); + PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size); /* At least reserve one Ethernet frame for watermark */ max_high_water = rx_buf_size - ETHER_MAX_LEN; if ((fc_conf->high_water > max_high_water) || - (fc_conf->high_water < fc_conf->low_water)) { - PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value \n"); - PMD_INIT_LOG(ERR, "high water must <= 0x%x \n", max_high_water); + (fc_conf->high_water < fc_conf->low_water)) { + PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value"); + PMD_INIT_LOG(ERR, "high water must <= 0x%x", max_high_water); return (-EINVAL); } @@ -1720,10 +2060,25 @@ eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) err = e1000_setup_link_generic(hw); if (err == E1000_SUCCESS) { + + /* check if we want to forward MAC frames - driver doesn't have native + * capability to do that, so we'll write the registers ourselves */ + + rctl = E1000_READ_REG(hw, E1000_RCTL); + + /* set or clear MFLCN.PMCF bit depending on configuration */ + if (fc_conf->mac_ctrl_frame_fwd != 0) + rctl |= E1000_RCTL_PMCF; + else + rctl &= ~E1000_RCTL_PMCF; + + E1000_WRITE_REG(hw, E1000_RCTL, rctl); + E1000_WRITE_FLUSH(hw); + return 0; } - PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x \n", err); + PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err); return (-EIO); } @@ -1758,7 +2113,7 @@ eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index) static void igbvf_intr_disable(struct e1000_hw *hw) { - PMD_INIT_LOG(DEBUG, "igbvf_intr_disable"); + PMD_INIT_FUNC_TRACE(); /* Clear interrupt mask to stop from interrupts being generated */ E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF); @@ -1775,7 +2130,7 @@ igbvf_stop_adapter(struct rte_eth_dev *dev) struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); memset(&dev_info, 0, sizeof(dev_info)); - eth_igb_infos_get(dev, &dev_info); + eth_igbvf_infos_get(dev, &dev_info); /* Clear interrupt mask to stop from interrupts being generated */ igbvf_intr_disable(hw); @@ -1840,8 +2195,8 @@ igbvf_dev_configure(struct rte_eth_dev *dev) { struct rte_eth_conf* conf = &dev->data->dev_conf; - PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n", - dev->data->port_id); + PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d", + dev->data->port_id); /* * VF has no ability to enable/disable HW CRC @@ -1849,12 +2204,12 @@ igbvf_dev_configure(struct rte_eth_dev *dev) */ #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC if (!conf->rxmode.hw_strip_crc) { - PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n"); + PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip"); conf->rxmode.hw_strip_crc = 1; } #else if (conf->rxmode.hw_strip_crc) { - PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n"); + PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip"); conf->rxmode.hw_strip_crc = 0; } #endif @@ -1865,17 +2220,17 @@ igbvf_dev_configure(struct rte_eth_dev *dev) static int igbvf_dev_start(struct rte_eth_dev *dev) { - struct e1000_hw *hw = + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); int ret; - PMD_INIT_LOG(DEBUG, "igbvf_dev_start"); + PMD_INIT_FUNC_TRACE(); hw->mac.ops.reset_hw(hw); /* Set all vfta */ igbvf_set_vfta_all(dev,1); - + eth_igbvf_tx_init(dev); /* This can fail when allocating mbufs for descriptor rings */ @@ -1892,12 +2247,12 @@ igbvf_dev_start(struct rte_eth_dev *dev) static void igbvf_dev_stop(struct rte_eth_dev *dev) { - PMD_INIT_LOG(DEBUG, "igbvf_dev_stop"); + PMD_INIT_FUNC_TRACE(); igbvf_stop_adapter(dev); - - /* - * Clear what we set, but we still keep shadow_vfta to + + /* + * Clear what we set, but we still keep shadow_vfta to * restore after device starts */ igbvf_set_vfta_all(dev,0); @@ -1910,7 +2265,7 @@ igbvf_dev_close(struct rte_eth_dev *dev) { struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - PMD_INIT_LOG(DEBUG, "igbvf_dev_close"); + PMD_INIT_FUNC_TRACE(); e1000_reset_hw(hw); @@ -1922,7 +2277,7 @@ static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) struct e1000_mbx_info *mbx = &hw->mbx; uint32_t msgbuf[2]; - /* After set vlan, vlan strip will also be enabled in igb driver*/ + /* After set vlan, vlan strip will also be enabled in igb driver*/ msgbuf[0] = E1000_VF_SET_VLAN; msgbuf[1] = vid; /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */ @@ -1934,7 +2289,7 @@ static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on) static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) { - struct e1000_hw *hw = + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_vfta * shadow_vfta = E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); @@ -1958,15 +2313,15 @@ static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on) static int igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) { - struct e1000_hw *hw = + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct e1000_vfta * shadow_vfta = E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private); uint32_t vid_idx = 0; uint32_t vid_bit = 0; int ret = 0; - - PMD_INIT_LOG(DEBUG, "igbvf_vlan_filter_set"); + + PMD_INIT_FUNC_TRACE(); /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/ ret = igbvf_set_vfta(hw, vlan_id, !!on); @@ -1988,38 +2343,40 @@ igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) static int eth_igb_rss_reta_update(struct rte_eth_dev *dev, - struct rte_eth_rss_reta *reta_conf) + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) { - uint8_t i,j,mask; - uint32_t reta; - struct e1000_hw *hw = - E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - - /* - * Update Redirection Table RETA[n],n=0...31,The redirection table has - * 128-entries in 32 registers - */ - for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) { - if (i < ETH_RSS_RETA_NUM_ENTRIES/2) - mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF); + uint8_t i, j, mask; + uint32_t reta, r; + uint16_t idx, shift; + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (reta_size != ETH_RSS_RETA_SIZE_128) { + PMD_DRV_LOG(ERR, "The size of hash lookup table configured " + "(%d) doesn't match the number hardware can supported " + "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128); + return -EINVAL; + } + + for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { + idx = i / RTE_RETA_GROUP_SIZE; + shift = i % RTE_RETA_GROUP_SIZE; + mask = (uint8_t)((reta_conf[idx].mask >> shift) & + IGB_4_BIT_MASK); + if (!mask) + continue; + if (mask == IGB_4_BIT_MASK) + r = 0; else - mask = (uint8_t)((reta_conf->mask_hi >> - (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF); - if (mask != 0) { - reta = 0; - /* If all 4 entries were set,don't need read RETA register */ - if (mask != 0xF) - reta = E1000_READ_REG(hw,E1000_RETA(i >> 2)); - - for (j = 0; j < 4; j++) { - if (mask & (0x1 << j)) { - if (mask != 0xF) - reta &= ~(0xFF << 8 * j); - reta |= reta_conf->reta[i + j] << 8 * j; - } - } - E1000_WRITE_REG(hw, E1000_RETA(i >> 2),reta); + r = E1000_READ_REG(hw, E1000_RETA(i >> 2)); + for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) { + if (mask & (0x1 << j)) + reta |= reta_conf[idx].reta[shift + j] << + (CHAR_BIT * j); + else + reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j)); } + E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); } return 0; @@ -2027,33 +2384,1273 @@ eth_igb_rss_reta_update(struct rte_eth_dev *dev, static int eth_igb_rss_reta_query(struct rte_eth_dev *dev, - struct rte_eth_rss_reta *reta_conf) + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) { - uint8_t i,j,mask; + uint8_t i, j, mask; uint32_t reta; - struct e1000_hw *hw = - E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint16_t idx, shift; + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); - /* - * Read Redirection Table RETA[n],n=0...31,The redirection table has - * 128-entries in 32 registers - */ - for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) { - if (i < ETH_RSS_RETA_NUM_ENTRIES/2) - mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF); + if (reta_size != ETH_RSS_RETA_SIZE_128) { + PMD_DRV_LOG(ERR, "The size of hash lookup table configured " + "(%d) doesn't match the number hardware can supported " + "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128); + return -EINVAL; + } + + for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) { + idx = i / RTE_RETA_GROUP_SIZE; + shift = i % RTE_RETA_GROUP_SIZE; + mask = (uint8_t)((reta_conf[idx].mask >> shift) & + IGB_4_BIT_MASK); + if (!mask) + continue; + reta = E1000_READ_REG(hw, E1000_RETA(i >> 2)); + for (j = 0; j < IGB_4_BIT_WIDTH; j++) { + if (mask & (0x1 << j)) + reta_conf[idx].reta[shift + j] = + ((reta >> (CHAR_BIT * j)) & + IGB_8_BIT_MASK); + } + } + + return 0; +} + +#define MAC_TYPE_FILTER_SUP(type) do {\ + if ((type) != e1000_82580 && (type) != e1000_i350 &&\ + (type) != e1000_82576)\ + return -ENOTSUP;\ +} while (0) + +static int +eth_igb_syn_filter_set(struct rte_eth_dev *dev, + struct rte_eth_syn_filter *filter, + bool add) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t synqf, rfctl; + + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) + return -EINVAL; + + synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); + + if (add) { + if (synqf & E1000_SYN_FILTER_ENABLE) + return -EINVAL; + + synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) & + E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE); + + rfctl = E1000_READ_REG(hw, E1000_RFCTL); + if (filter->hig_pri) + rfctl |= E1000_RFCTL_SYNQFP; else - mask = (uint8_t)((reta_conf->mask_hi >> - (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF); - - if (mask != 0) { - reta = E1000_READ_REG(hw,E1000_RETA(i >> 2)); - for (j = 0; j < 4; j++) { - if (mask & (0x1 << j)) - reta_conf->reta[i + j] = - (uint8_t)((reta >> 8 * j) & 0xFF); - } + rfctl &= ~E1000_RFCTL_SYNQFP; + + E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); + } else { + if (!(synqf & E1000_SYN_FILTER_ENABLE)) + return -ENOENT; + synqf = 0; + } + + E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf); + E1000_WRITE_FLUSH(hw); + return 0; +} + +static int +eth_igb_syn_filter_get(struct rte_eth_dev *dev, + struct rte_eth_syn_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t synqf, rfctl; + + synqf = E1000_READ_REG(hw, E1000_SYNQF(0)); + if (synqf & E1000_SYN_FILTER_ENABLE) { + rfctl = E1000_READ_REG(hw, E1000_RFCTL); + filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0; + filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >> + E1000_SYN_FILTER_QUEUE_SHIFT); + return 0; + } + + return -ENOENT; +} + +static int +eth_igb_syn_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + if (filter_op == RTE_ETH_FILTER_NOP) + return 0; + + if (arg == NULL) { + PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", + filter_op); + return -EINVAL; + } + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + ret = eth_igb_syn_filter_set(dev, + (struct rte_eth_syn_filter *)arg, + TRUE); + break; + case RTE_ETH_FILTER_DELETE: + ret = eth_igb_syn_filter_set(dev, + (struct rte_eth_syn_filter *)arg, + FALSE); + break; + case RTE_ETH_FILTER_GET: + ret = eth_igb_syn_filter_get(dev, + (struct rte_eth_syn_filter *)arg); + break; + default: + PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op); + ret = -EINVAL; + break; + } + + return ret; +} + +#define MAC_TYPE_FILTER_SUP_EXT(type) do {\ + if ((type) != e1000_82580 && (type) != e1000_i350)\ + return -ENOSYS; \ +} while (0) + +/* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/ +static inline int +ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter, + struct e1000_2tuple_filter_info *filter_info) +{ + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM) + return -EINVAL; + if (filter->priority > E1000_2TUPLE_MAX_PRI) + return -EINVAL; /* filter index is out of range. */ + if (filter->tcp_flags > TCP_FLAG_ALL) + return -EINVAL; /* flags is invalid. */ + + switch (filter->dst_port_mask) { + case UINT16_MAX: + filter_info->dst_port_mask = 0; + filter_info->dst_port = filter->dst_port; + break; + case 0: + filter_info->dst_port_mask = 1; + break; + default: + PMD_DRV_LOG(ERR, "invalid dst_port mask."); + return -EINVAL; + } + + switch (filter->proto_mask) { + case UINT8_MAX: + filter_info->proto_mask = 0; + filter_info->proto = filter->proto; + break; + case 0: + filter_info->proto_mask = 1; + break; + default: + PMD_DRV_LOG(ERR, "invalid protocol mask."); + return -EINVAL; + } + + filter_info->priority = (uint8_t)filter->priority; + if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) + filter_info->tcp_flags = filter->tcp_flags; + else + filter_info->tcp_flags = 0; + + return 0; +} + +static inline struct e1000_2tuple_filter * +igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list, + struct e1000_2tuple_filter_info *key) +{ + struct e1000_2tuple_filter *it; + + TAILQ_FOREACH(it, filter_list, entries) { + if (memcmp(key, &it->filter_info, + sizeof(struct e1000_2tuple_filter_info)) == 0) { + return it; + } + } + return NULL; +} + +/* + * igb_add_2tuple_filter - add a 2tuple filter + * + * @param + * dev: Pointer to struct rte_eth_dev. + * ntuple_filter: ponter to the filter that will be added. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +igb_add_2tuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_2tuple_filter *filter; + uint32_t ttqf = E1000_TTQF_DISABLE_MASK; + uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP; + int i, ret; + + filter = rte_zmalloc("e1000_2tuple_filter", + sizeof(struct e1000_2tuple_filter), 0); + if (filter == NULL) + return -ENOMEM; + + ret = ntuple_filter_to_2tuple(ntuple_filter, + &filter->filter_info); + if (ret < 0) { + rte_free(filter); + return ret; + } + if (igb_2tuple_filter_lookup(&filter_info->twotuple_list, + &filter->filter_info) != NULL) { + PMD_DRV_LOG(ERR, "filter exists."); + rte_free(filter); + return -EEXIST; + } + filter->queue = ntuple_filter->queue; + + /* + * look for an unused 2tuple filter index, + * and insert the filter to list. + */ + for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) { + if (!(filter_info->twotuple_mask & (1 << i))) { + filter_info->twotuple_mask |= 1 << i; + filter->index = i; + TAILQ_INSERT_TAIL(&filter_info->twotuple_list, + filter, + entries); + break; } } - + if (i >= E1000_MAX_TTQF_FILTERS) { + PMD_DRV_LOG(ERR, "2tuple filters are full."); + rte_free(filter); + return -ENOSYS; + } + + imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); + if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ + imir |= E1000_IMIR_PORT_BP; + else + imir &= ~E1000_IMIR_PORT_BP; + + imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + + ttqf |= E1000_TTQF_QUEUE_ENABLE; + ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT); + ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK); + if (filter->filter_info.proto_mask == 0) + ttqf &= ~E1000_TTQF_MASK_ENABLE; + + /* tcp flags bits setting. */ + if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { + if (filter->filter_info.tcp_flags & TCP_URG_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_URG; + if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_ACK; + if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_PSH; + if (filter->filter_info.tcp_flags & TCP_RST_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_RST; + if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_SYN; + if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_FIN; + } else + imir_ext |= E1000_IMIREXT_CTRL_BP; + E1000_WRITE_REG(hw, E1000_IMIR(i), imir); + E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf); + E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); return 0; } + +/* + * igb_remove_2tuple_filter - remove a 2tuple filter + * + * @param + * dev: Pointer to struct rte_eth_dev. + * ntuple_filter: ponter to the filter that will be removed. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +igb_remove_2tuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_2tuple_filter_info filter_2tuple; + struct e1000_2tuple_filter *filter; + int ret; + + memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info)); + ret = ntuple_filter_to_2tuple(ntuple_filter, + &filter_2tuple); + if (ret < 0) + return ret; + + filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list, + &filter_2tuple); + if (filter == NULL) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + return -ENOENT; + } + + filter_info->twotuple_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries); + rte_free(filter); + + E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK); + E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); + return 0; +} + +static inline struct e1000_flex_filter * +eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list, + struct e1000_flex_filter_info *key) +{ + struct e1000_flex_filter *it; + + TAILQ_FOREACH(it, filter_list, entries) { + if (memcmp(key, &it->filter_info, + sizeof(struct e1000_flex_filter_info)) == 0) + return it; + } + + return NULL; +} + +static int +eth_igb_add_del_flex_filter(struct rte_eth_dev *dev, + struct rte_eth_flex_filter *filter, + bool add) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_flex_filter *flex_filter, *it; + uint32_t wufc, queueing, mask; + uint32_t reg_off; + uint8_t shift, i, j = 0; + + flex_filter = rte_zmalloc("e1000_flex_filter", + sizeof(struct e1000_flex_filter), 0); + if (flex_filter == NULL) + return -ENOMEM; + + flex_filter->filter_info.len = filter->len; + flex_filter->filter_info.priority = filter->priority; + memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len); + for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) { + mask = 0; + /* reverse bits in flex filter's mask*/ + for (shift = 0; shift < CHAR_BIT; shift++) { + if (filter->mask[i] & (0x01 << shift)) + mask |= (0x80 >> shift); + } + flex_filter->filter_info.mask[i] = mask; + } + + wufc = E1000_READ_REG(hw, E1000_WUFC); + if (flex_filter->index < E1000_MAX_FHFT) + reg_off = E1000_FHFT(flex_filter->index); + else + reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT); + + if (add) { + if (eth_igb_flex_filter_lookup(&filter_info->flex_list, + &flex_filter->filter_info) != NULL) { + PMD_DRV_LOG(ERR, "filter exists."); + rte_free(flex_filter); + return -EEXIST; + } + flex_filter->queue = filter->queue; + /* + * look for an unused flex filter index + * and insert the filter into the list. + */ + for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) { + if (!(filter_info->flex_mask & (1 << i))) { + filter_info->flex_mask |= 1 << i; + flex_filter->index = i; + TAILQ_INSERT_TAIL(&filter_info->flex_list, + flex_filter, + entries); + break; + } + } + if (i >= E1000_MAX_FLEX_FILTERS) { + PMD_DRV_LOG(ERR, "flex filters are full."); + rte_free(flex_filter); + return -ENOSYS; + } + + E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ | + (E1000_WUFC_FLX0 << flex_filter->index)); + queueing = filter->len | + (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) | + (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT); + E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, + queueing); + for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) { + E1000_WRITE_REG(hw, reg_off, + flex_filter->filter_info.dwords[j]); + reg_off += sizeof(uint32_t); + E1000_WRITE_REG(hw, reg_off, + flex_filter->filter_info.dwords[++j]); + reg_off += sizeof(uint32_t); + E1000_WRITE_REG(hw, reg_off, + (uint32_t)flex_filter->filter_info.mask[i]); + reg_off += sizeof(uint32_t) * 2; + ++j; + } + } else { + it = eth_igb_flex_filter_lookup(&filter_info->flex_list, + &flex_filter->filter_info); + if (it == NULL) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + rte_free(flex_filter); + return -ENOENT; + } + + for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++) + E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0); + E1000_WRITE_REG(hw, E1000_WUFC, wufc & + (~(E1000_WUFC_FLX0 << it->index))); + + filter_info->flex_mask &= ~(1 << it->index); + TAILQ_REMOVE(&filter_info->flex_list, it, entries); + rte_free(it); + rte_free(flex_filter); + } + + return 0; +} + +static int +eth_igb_get_flex_filter(struct rte_eth_dev *dev, + struct rte_eth_flex_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_flex_filter flex_filter, *it; + uint32_t wufc, queueing, wufc_en = 0; + + memset(&flex_filter, 0, sizeof(struct e1000_flex_filter)); + flex_filter.filter_info.len = filter->len; + flex_filter.filter_info.priority = filter->priority; + memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len); + memcpy(flex_filter.filter_info.mask, filter->mask, + RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char)); + + it = eth_igb_flex_filter_lookup(&filter_info->flex_list, + &flex_filter.filter_info); + if (it == NULL) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + return -ENOENT; + } + + wufc = E1000_READ_REG(hw, E1000_WUFC); + wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index); + + if ((wufc & wufc_en) == wufc_en) { + uint32_t reg_off = 0; + if (it->index < E1000_MAX_FHFT) + reg_off = E1000_FHFT(it->index); + else + reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT); + + queueing = E1000_READ_REG(hw, + reg_off + E1000_FHFT_QUEUEING_OFFSET); + filter->len = queueing & E1000_FHFT_QUEUEING_LEN; + filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >> + E1000_FHFT_QUEUEING_PRIO_SHIFT; + filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >> + E1000_FHFT_QUEUEING_QUEUE_SHIFT; + return 0; + } + return -ENOENT; +} + +static int +eth_igb_flex_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_eth_flex_filter *filter; + int ret = 0; + + MAC_TYPE_FILTER_SUP_EXT(hw->mac.type); + + if (filter_op == RTE_ETH_FILTER_NOP) + return ret; + + if (arg == NULL) { + PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u", + filter_op); + return -EINVAL; + } + + filter = (struct rte_eth_flex_filter *)arg; + if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN + || filter->len % sizeof(uint64_t) != 0) { + PMD_DRV_LOG(ERR, "filter's length is out of range"); + return -EINVAL; + } + if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) { + PMD_DRV_LOG(ERR, "filter's priority is out of range"); + return -EINVAL; + } + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + ret = eth_igb_add_del_flex_filter(dev, filter, TRUE); + break; + case RTE_ETH_FILTER_DELETE: + ret = eth_igb_add_del_flex_filter(dev, filter, FALSE); + break; + case RTE_ETH_FILTER_GET: + ret = eth_igb_get_flex_filter(dev, filter); + break; + default: + PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op); + ret = -EINVAL; + break; + } + + return ret; +} + +/* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/ +static inline int +ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter, + struct e1000_5tuple_filter_info *filter_info) +{ + if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576) + return -EINVAL; + if (filter->priority > E1000_2TUPLE_MAX_PRI) + return -EINVAL; /* filter index is out of range. */ + if (filter->tcp_flags > TCP_FLAG_ALL) + return -EINVAL; /* flags is invalid. */ + + switch (filter->dst_ip_mask) { + case UINT32_MAX: + filter_info->dst_ip_mask = 0; + filter_info->dst_ip = filter->dst_ip; + break; + case 0: + filter_info->dst_ip_mask = 1; + break; + default: + PMD_DRV_LOG(ERR, "invalid dst_ip mask."); + return -EINVAL; + } + + switch (filter->src_ip_mask) { + case UINT32_MAX: + filter_info->src_ip_mask = 0; + filter_info->src_ip = filter->src_ip; + break; + case 0: + filter_info->src_ip_mask = 1; + break; + default: + PMD_DRV_LOG(ERR, "invalid src_ip mask."); + return -EINVAL; + } + + switch (filter->dst_port_mask) { + case UINT16_MAX: + filter_info->dst_port_mask = 0; + filter_info->dst_port = filter->dst_port; + break; + case 0: + filter_info->dst_port_mask = 1; + break; + default: + PMD_DRV_LOG(ERR, "invalid dst_port mask."); + return -EINVAL; + } + + switch (filter->src_port_mask) { + case UINT16_MAX: + filter_info->src_port_mask = 0; + filter_info->src_port = filter->src_port; + break; + case 0: + filter_info->src_port_mask = 1; + break; + default: + PMD_DRV_LOG(ERR, "invalid src_port mask."); + return -EINVAL; + } + + switch (filter->proto_mask) { + case UINT8_MAX: + filter_info->proto_mask = 0; + filter_info->proto = filter->proto; + break; + case 0: + filter_info->proto_mask = 1; + break; + default: + PMD_DRV_LOG(ERR, "invalid protocol mask."); + return -EINVAL; + } + + filter_info->priority = (uint8_t)filter->priority; + if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) + filter_info->tcp_flags = filter->tcp_flags; + else + filter_info->tcp_flags = 0; + + return 0; +} + +static inline struct e1000_5tuple_filter * +igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list, + struct e1000_5tuple_filter_info *key) +{ + struct e1000_5tuple_filter *it; + + TAILQ_FOREACH(it, filter_list, entries) { + if (memcmp(key, &it->filter_info, + sizeof(struct e1000_5tuple_filter_info)) == 0) { + return it; + } + } + return NULL; +} + +/* + * igb_add_5tuple_filter_82576 - add a 5tuple filter + * + * @param + * dev: Pointer to struct rte_eth_dev. + * ntuple_filter: ponter to the filter that will be added. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +igb_add_5tuple_filter_82576(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_5tuple_filter *filter; + uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK; + uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP; + uint8_t i; + int ret; + + filter = rte_zmalloc("e1000_5tuple_filter", + sizeof(struct e1000_5tuple_filter), 0); + if (filter == NULL) + return -ENOMEM; + + ret = ntuple_filter_to_5tuple_82576(ntuple_filter, + &filter->filter_info); + if (ret < 0) { + rte_free(filter); + return ret; + } + + if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, + &filter->filter_info) != NULL) { + PMD_DRV_LOG(ERR, "filter exists."); + rte_free(filter); + return -EEXIST; + } + filter->queue = ntuple_filter->queue; + + /* + * look for an unused 5tuple filter index, + * and insert the filter to list. + */ + for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) { + if (!(filter_info->fivetuple_mask & (1 << i))) { + filter_info->fivetuple_mask |= 1 << i; + filter->index = i; + TAILQ_INSERT_TAIL(&filter_info->fivetuple_list, + filter, + entries); + break; + } + } + if (i >= E1000_MAX_FTQF_FILTERS) { + PMD_DRV_LOG(ERR, "5tuple filters are full."); + rte_free(filter); + return -ENOSYS; + } + + ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK; + if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */ + ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP; + if (filter->filter_info.dst_ip_mask == 0) + ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP; + if (filter->filter_info.src_port_mask == 0) + ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; + if (filter->filter_info.proto_mask == 0) + ftqf &= ~E1000_FTQF_MASK_PROTO_BP; + ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) & + E1000_FTQF_QUEUE_MASK; + ftqf |= E1000_FTQF_QUEUE_ENABLE; + E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf); + E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip); + E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip); + + spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT; + E1000_WRITE_REG(hw, E1000_SPQF(i), spqf); + + imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT); + if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */ + imir |= E1000_IMIR_PORT_BP; + else + imir &= ~E1000_IMIR_PORT_BP; + imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT; + + /* tcp flags bits setting. */ + if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) { + if (filter->filter_info.tcp_flags & TCP_URG_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_URG; + if (filter->filter_info.tcp_flags & TCP_ACK_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_ACK; + if (filter->filter_info.tcp_flags & TCP_PSH_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_PSH; + if (filter->filter_info.tcp_flags & TCP_RST_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_RST; + if (filter->filter_info.tcp_flags & TCP_SYN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_SYN; + if (filter->filter_info.tcp_flags & TCP_FIN_FLAG) + imir_ext |= E1000_IMIREXT_CTRL_FIN; + } else + imir_ext |= E1000_IMIREXT_CTRL_BP; + E1000_WRITE_REG(hw, E1000_IMIR(i), imir); + E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext); + return 0; +} + +/* + * igb_remove_5tuple_filter_82576 - remove a 5tuple filter + * + * @param + * dev: Pointer to struct rte_eth_dev. + * ntuple_filter: ponter to the filter that will be removed. + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_5tuple_filter_info filter_5tuple; + struct e1000_5tuple_filter *filter; + int ret; + + memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info)); + ret = ntuple_filter_to_5tuple_82576(ntuple_filter, + &filter_5tuple); + if (ret < 0) + return ret; + + filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list, + &filter_5tuple); + if (filter == NULL) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + return -ENOENT; + } + + filter_info->fivetuple_mask &= ~(1 << filter->index); + TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries); + rte_free(filter); + + E1000_WRITE_REG(hw, E1000_FTQF(filter->index), + E1000_FTQF_VF_BP | E1000_FTQF_MASK); + E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0); + E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0); + return 0; +} + +static int +eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + uint32_t rctl; + struct e1000_hw *hw; + struct rte_eth_dev_info dev_info; + uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + + VLAN_TAG_SIZE); + + hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + +#ifdef RTE_LIBRTE_82571_SUPPORT + /* XXX: not bigger than max_rx_pktlen */ + if (hw->mac.type == e1000_82571) + return -ENOTSUP; +#endif + eth_igb_infos_get(dev, &dev_info); + + /* check that mtu is within the allowed range */ + if ((mtu < ETHER_MIN_MTU) || + (frame_size > dev_info.max_rx_pktlen)) + return -EINVAL; + + /* refuse mtu that requires the support of scattered packets when this + * feature has not been enabled before. */ + if (!dev->data->scattered_rx && + frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) + return -EINVAL; + + rctl = E1000_READ_REG(hw, E1000_RCTL); + + /* switch to jumbo mode if needed */ + if (frame_size > ETHER_MAX_LEN) { + dev->data->dev_conf.rxmode.jumbo_frame = 1; + rctl |= E1000_RCTL_LPE; + } else { + dev->data->dev_conf.rxmode.jumbo_frame = 0; + rctl &= ~E1000_RCTL_LPE; + } + E1000_WRITE_REG(hw, E1000_RCTL, rctl); + + /* update max frame size */ + dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; + + E1000_WRITE_REG(hw, E1000_RLPML, + dev->data->dev_conf.rxmode.max_rx_pkt_len); + + return 0; +} + +/* + * igb_add_del_ntuple_filter - add or delete a ntuple filter + * + * @param + * dev: Pointer to struct rte_eth_dev. + * ntuple_filter: Pointer to struct rte_eth_ntuple_filter + * add: if true, add filter, if false, remove filter + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +igb_add_del_ntuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter, + bool add) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + switch (ntuple_filter->flags) { + case RTE_5TUPLE_FLAGS: + case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): + if (hw->mac.type != e1000_82576) + return -ENOTSUP; + if (add) + ret = igb_add_5tuple_filter_82576(dev, + ntuple_filter); + else + ret = igb_remove_5tuple_filter_82576(dev, + ntuple_filter); + break; + case RTE_2TUPLE_FLAGS: + case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): + if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) + return -ENOTSUP; + if (add) + ret = igb_add_2tuple_filter(dev, ntuple_filter); + else + ret = igb_remove_2tuple_filter(dev, ntuple_filter); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +/* + * igb_get_ntuple_filter - get a ntuple filter + * + * @param + * dev: Pointer to struct rte_eth_dev. + * ntuple_filter: Pointer to struct rte_eth_ntuple_filter + * + * @return + * - On success, zero. + * - On failure, a negative value. + */ +static int +igb_get_ntuple_filter(struct rte_eth_dev *dev, + struct rte_eth_ntuple_filter *ntuple_filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + struct e1000_5tuple_filter_info filter_5tuple; + struct e1000_2tuple_filter_info filter_2tuple; + struct e1000_5tuple_filter *p_5tuple_filter; + struct e1000_2tuple_filter *p_2tuple_filter; + int ret; + + switch (ntuple_filter->flags) { + case RTE_5TUPLE_FLAGS: + case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): + if (hw->mac.type != e1000_82576) + return -ENOTSUP; + memset(&filter_5tuple, + 0, + sizeof(struct e1000_5tuple_filter_info)); + ret = ntuple_filter_to_5tuple_82576(ntuple_filter, + &filter_5tuple); + if (ret < 0) + return ret; + p_5tuple_filter = igb_5tuple_filter_lookup_82576( + &filter_info->fivetuple_list, + &filter_5tuple); + if (p_5tuple_filter == NULL) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + return -ENOENT; + } + ntuple_filter->queue = p_5tuple_filter->queue; + break; + case RTE_2TUPLE_FLAGS: + case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG): + if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350) + return -ENOTSUP; + memset(&filter_2tuple, + 0, + sizeof(struct e1000_2tuple_filter_info)); + ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple); + if (ret < 0) + return ret; + p_2tuple_filter = igb_2tuple_filter_lookup( + &filter_info->twotuple_list, + &filter_2tuple); + if (p_2tuple_filter == NULL) { + PMD_DRV_LOG(ERR, "filter doesn't exist."); + return -ENOENT; + } + ntuple_filter->queue = p_2tuple_filter->queue; + break; + default: + ret = -EINVAL; + break; + } + + return 0; +} + +/* + * igb_ntuple_filter_handle - Handle operations for ntuple filter. + * @dev: pointer to rte_eth_dev structure + * @filter_op:operation will be taken. + * @arg: a pointer to specific structure corresponding to the filter_op + */ +static int +igb_ntuple_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + if (filter_op == RTE_ETH_FILTER_NOP) + return 0; + + if (arg == NULL) { + PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", + filter_op); + return -EINVAL; + } + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + ret = igb_add_del_ntuple_filter(dev, + (struct rte_eth_ntuple_filter *)arg, + TRUE); + break; + case RTE_ETH_FILTER_DELETE: + ret = igb_add_del_ntuple_filter(dev, + (struct rte_eth_ntuple_filter *)arg, + FALSE); + break; + case RTE_ETH_FILTER_GET: + ret = igb_get_ntuple_filter(dev, + (struct rte_eth_ntuple_filter *)arg); + break; + default: + PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); + ret = -EINVAL; + break; + } + return ret; +} + +static inline int +igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info, + uint16_t ethertype) +{ + int i; + + for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { + if (filter_info->ethertype_filters[i] == ethertype && + (filter_info->ethertype_mask & (1 << i))) + return i; + } + return -1; +} + +static inline int +igb_ethertype_filter_insert(struct e1000_filter_info *filter_info, + uint16_t ethertype) +{ + int i; + + for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) { + if (!(filter_info->ethertype_mask & (1 << i))) { + filter_info->ethertype_mask |= 1 << i; + filter_info->ethertype_filters[i] = ethertype; + return i; + } + } + return -1; +} + +static inline int +igb_ethertype_filter_remove(struct e1000_filter_info *filter_info, + uint8_t idx) +{ + if (idx >= E1000_MAX_ETQF_FILTERS) + return -1; + filter_info->ethertype_mask &= ~(1 << idx); + filter_info->ethertype_filters[idx] = 0; + return idx; +} + + +static int +igb_add_del_ethertype_filter(struct rte_eth_dev *dev, + struct rte_eth_ethertype_filter *filter, + bool add) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + uint32_t etqf = 0; + int ret; + + if (filter->ether_type == ETHER_TYPE_IPv4 || + filter->ether_type == ETHER_TYPE_IPv6) { + PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in" + " ethertype filter.", filter->ether_type); + return -EINVAL; + } + + if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) { + PMD_DRV_LOG(ERR, "mac compare is unsupported."); + return -EINVAL; + } + if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) { + PMD_DRV_LOG(ERR, "drop option is unsupported."); + return -EINVAL; + } + + ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); + if (ret >= 0 && add) { + PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.", + filter->ether_type); + return -EEXIST; + } + if (ret < 0 && !add) { + PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", + filter->ether_type); + return -ENOENT; + } + + if (add) { + ret = igb_ethertype_filter_insert(filter_info, + filter->ether_type); + if (ret < 0) { + PMD_DRV_LOG(ERR, "ethertype filters are full."); + return -ENOSYS; + } + + etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE; + etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE); + etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT; + } else { + ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret); + if (ret < 0) + return -ENOSYS; + } + E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf); + E1000_WRITE_FLUSH(hw); + + return 0; +} + +static int +igb_get_ethertype_filter(struct rte_eth_dev *dev, + struct rte_eth_ethertype_filter *filter) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct e1000_filter_info *filter_info = + E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); + uint32_t etqf; + int ret; + + ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type); + if (ret < 0) { + PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.", + filter->ether_type); + return -ENOENT; + } + + etqf = E1000_READ_REG(hw, E1000_ETQF(ret)); + if (etqf & E1000_ETQF_FILTER_ENABLE) { + filter->ether_type = etqf & E1000_ETQF_ETHERTYPE; + filter->flags = 0; + filter->queue = (etqf & E1000_ETQF_QUEUE) >> + E1000_ETQF_QUEUE_SHIFT; + return 0; + } + + return -ENOENT; +} + +/* + * igb_ethertype_filter_handle - Handle operations for ethertype filter. + * @dev: pointer to rte_eth_dev structure + * @filter_op:operation will be taken. + * @arg: a pointer to specific structure corresponding to the filter_op + */ +static int +igb_ethertype_filter_handle(struct rte_eth_dev *dev, + enum rte_filter_op filter_op, + void *arg) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int ret; + + MAC_TYPE_FILTER_SUP(hw->mac.type); + + if (filter_op == RTE_ETH_FILTER_NOP) + return 0; + + if (arg == NULL) { + PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.", + filter_op); + return -EINVAL; + } + + switch (filter_op) { + case RTE_ETH_FILTER_ADD: + ret = igb_add_del_ethertype_filter(dev, + (struct rte_eth_ethertype_filter *)arg, + TRUE); + break; + case RTE_ETH_FILTER_DELETE: + ret = igb_add_del_ethertype_filter(dev, + (struct rte_eth_ethertype_filter *)arg, + FALSE); + break; + case RTE_ETH_FILTER_GET: + ret = igb_get_ethertype_filter(dev, + (struct rte_eth_ethertype_filter *)arg); + break; + default: + PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op); + ret = -EINVAL; + break; + } + return ret; +} + +static int +eth_igb_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + int ret = -EINVAL; + + switch (filter_type) { + case RTE_ETH_FILTER_NTUPLE: + ret = igb_ntuple_filter_handle(dev, filter_op, arg); + break; + case RTE_ETH_FILTER_ETHERTYPE: + ret = igb_ethertype_filter_handle(dev, filter_op, arg); + break; + case RTE_ETH_FILTER_SYN: + ret = eth_igb_syn_filter_handle(dev, filter_op, arg); + break; + case RTE_ETH_FILTER_FLEXIBLE: + ret = eth_igb_flex_filter_handle(dev, filter_op, arg); + break; + default: + PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", + filter_type); + break; + } + + return ret; +} + +static struct rte_driver pmd_igb_drv = { + .type = PMD_PDEV, + .init = rte_igb_pmd_init, +}; + +static struct rte_driver pmd_igbvf_drv = { + .type = PMD_PDEV, + .init = rte_igbvf_pmd_init, +}; + +PMD_REGISTER_DRIVER(pmd_igb_drv); +PMD_REGISTER_DRIVER(pmd_igbvf_drv);