drivers/net/i40e/i40e_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <assert.h>
  43
  44 #include <rte_string_fns.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memzone.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_alarm.h>
  52 #include <rte_dev.h>
  53 #include <rte_eth_ctrl.h>
  54
  55 #include "i40e_logs.h"
  56 #include "base/i40e_prototype.h"
  57 #include "base/i40e_adminq_cmd.h"
  58 #include "base/i40e_type.h"
  59 #include "base/i40e_register.h"
  60 #include "base/i40e_dcb.h"
  61 #include "i40e_ethdev.h"
  62 #include "i40e_rxtx.h"
  63 #include "i40e_pf.h"
  64 #include "i40e_regs.h"
  65
  66 #define ETH_I40E_FLOATING_VEB_ARG       "enable_floating_veb"
  67 #define ETH_I40E_FLOATING_VEB_LIST_ARG  "floating_veb_list"
  68
  69 #define I40E_CLEAR_PXE_WAIT_MS     200
  70
  71 /* Maximun number of capability elements */
  72 #define I40E_MAX_CAP_ELE_NUM       128
  73
  74 /* Wait count and inteval */
  75 #define I40E_CHK_Q_ENA_COUNT       1000
  76 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
  77
  78 /* Maximun number of VSI */
  79 #define I40E_MAX_NUM_VSIS          (384UL)
  80
  81 #define I40E_PRE_TX_Q_CFG_WAIT_US       10 /* 10 us */
  82
  83 /* Flow control default timer */
  84 #define I40E_DEFAULT_PAUSE_TIME 0xFFFFU
  85
  86 /* Flow control default high water */
  87 #define I40E_DEFAULT_HIGH_WATER (0x1C40/1024)
  88
  89 /* Flow control default low water */
  90 #define I40E_DEFAULT_LOW_WATER  (0x1A40/1024)
  91
  92 /* Flow control enable fwd bit */
  93 #define I40E_PRTMAC_FWD_CTRL   0x00000001
  94
  95 /* Receive Packet Buffer size */
  96 #define I40E_RXPBSIZE (968 * 1024)
  97
  98 /* Kilobytes shift */
  99 #define I40E_KILOSHIFT 10
 100
 101 /* Receive Average Packet Size in Byte*/
 102 #define I40E_PACKET_AVERAGE_SIZE 128
 103
 104 /* Mask of PF interrupt causes */
 105 #define I40E_PFINT_ICR0_ENA_MASK ( \
 106                 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | \
 107                 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | \
 108                 I40E_PFINT_ICR0_ENA_GRST_MASK | \
 109                 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | \
 110                 I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK | \
 111                 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK | \
 112                 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | \
 113                 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | \
 114                 I40E_PFINT_ICR0_ENA_VFLR_MASK | \
 115                 I40E_PFINT_ICR0_ENA_ADMINQ_MASK)
 116
 117 #define I40E_FLOW_TYPES ( \
 118         (1UL << RTE_ETH_FLOW_FRAG_IPV4) | \
 119         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
 120         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
 121         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
 122         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
 123         (1UL << RTE_ETH_FLOW_FRAG_IPV6) | \
 124         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
 125         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
 126         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
 127         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
 128         (1UL << RTE_ETH_FLOW_L2_PAYLOAD))
 129
 130 /* Additional timesync values. */
 131 #define I40E_PTP_40GB_INCVAL     0x0199999999ULL
 132 #define I40E_PTP_10GB_INCVAL     0x0333333333ULL
 133 #define I40E_PTP_1GB_INCVAL      0x2000000000ULL
 134 #define I40E_PRTTSYN_TSYNENA     0x80000000
 135 #define I40E_PRTTSYN_TSYNTYPE    0x0e000000
 136 #define I40E_CYCLECOUNTER_MASK   0xffffffffffffffffULL
 137
 138 #define I40E_MAX_PERCENT            100
 139 #define I40E_DEFAULT_DCB_APP_NUM    1
 140 #define I40E_DEFAULT_DCB_APP_PRIO   3
 141
 142 #define I40E_INSET_NONE            0x00000000000000000ULL
 143
 144 /* bit0 ~ bit 7 */
 145 #define I40E_INSET_DMAC            0x0000000000000001ULL
 146 #define I40E_INSET_SMAC            0x0000000000000002ULL
 147 #define I40E_INSET_VLAN_OUTER      0x0000000000000004ULL
 148 #define I40E_INSET_VLAN_INNER      0x0000000000000008ULL
 149 #define I40E_INSET_VLAN_TUNNEL     0x0000000000000010ULL
 150
 151 /* bit 8 ~ bit 15 */
 152 #define I40E_INSET_IPV4_SRC        0x0000000000000100ULL
 153 #define I40E_INSET_IPV4_DST        0x0000000000000200ULL
 154 #define I40E_INSET_IPV6_SRC        0x0000000000000400ULL
 155 #define I40E_INSET_IPV6_DST        0x0000000000000800ULL
 156 #define I40E_INSET_SRC_PORT        0x0000000000001000ULL
 157 #define I40E_INSET_DST_PORT        0x0000000000002000ULL
 158 #define I40E_INSET_SCTP_VT         0x0000000000004000ULL
 159
 160 /* bit 16 ~ bit 31 */
 161 #define I40E_INSET_IPV4_TOS        0x0000000000010000ULL
 162 #define I40E_INSET_IPV4_PROTO      0x0000000000020000ULL
 163 #define I40E_INSET_IPV4_TTL        0x0000000000040000ULL
 164 #define I40E_INSET_IPV6_TC         0x0000000000080000ULL
 165 #define I40E_INSET_IPV6_FLOW       0x0000000000100000ULL
 166 #define I40E_INSET_IPV6_NEXT_HDR   0x0000000000200000ULL
 167 #define I40E_INSET_IPV6_HOP_LIMIT  0x0000000000400000ULL
 168 #define I40E_INSET_TCP_FLAGS       0x0000000000800000ULL
 169
 170 /* bit 32 ~ bit 47, tunnel fields */
 171 #define I40E_INSET_TUNNEL_IPV4_DST       0x0000000100000000ULL
 172 #define I40E_INSET_TUNNEL_IPV6_DST       0x0000000200000000ULL
 173 #define I40E_INSET_TUNNEL_DMAC           0x0000000400000000ULL
 174 #define I40E_INSET_TUNNEL_SRC_PORT       0x0000000800000000ULL
 175 #define I40E_INSET_TUNNEL_DST_PORT       0x0000001000000000ULL
 176 #define I40E_INSET_TUNNEL_ID             0x0000002000000000ULL
 177
 178 /* bit 48 ~ bit 55 */
 179 #define I40E_INSET_LAST_ETHER_TYPE 0x0001000000000000ULL
 180
 181 /* bit 56 ~ bit 63, Flex Payload */
 182 #define I40E_INSET_FLEX_PAYLOAD_W1 0x0100000000000000ULL
 183 #define I40E_INSET_FLEX_PAYLOAD_W2 0x0200000000000000ULL
 184 #define I40E_INSET_FLEX_PAYLOAD_W3 0x0400000000000000ULL
 185 #define I40E_INSET_FLEX_PAYLOAD_W4 0x0800000000000000ULL
 186 #define I40E_INSET_FLEX_PAYLOAD_W5 0x1000000000000000ULL
 187 #define I40E_INSET_FLEX_PAYLOAD_W6 0x2000000000000000ULL
 188 #define I40E_INSET_FLEX_PAYLOAD_W7 0x4000000000000000ULL
 189 #define I40E_INSET_FLEX_PAYLOAD_W8 0x8000000000000000ULL
 190 #define I40E_INSET_FLEX_PAYLOAD \
 191         (I40E_INSET_FLEX_PAYLOAD_W1 | I40E_INSET_FLEX_PAYLOAD_W2 | \
 192         I40E_INSET_FLEX_PAYLOAD_W3 | I40E_INSET_FLEX_PAYLOAD_W4 | \
 193         I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \
 194         I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8)
 195
 196 /**
 197  * Below are values for writing un-exposed registers suggested
 198  * by silicon experts
 199  */
 200 /* Destination MAC address */
 201 #define I40E_REG_INSET_L2_DMAC                   0xE000000000000000ULL
 202 /* Source MAC address */
 203 #define I40E_REG_INSET_L2_SMAC                   0x1C00000000000000ULL
 204 /* Outer (S-Tag) VLAN tag in the outer L2 header */
 205 #define I40E_REG_INSET_L2_OUTER_VLAN             0x0200000000000000ULL
 206 /* Inner (C-Tag) or single VLAN tag in the outer L2 header */
 207 #define I40E_REG_INSET_L2_INNER_VLAN             0x0080000000000000ULL
 208 /* Single VLAN tag in the inner L2 header */
 209 #define I40E_REG_INSET_TUNNEL_VLAN               0x0100000000000000ULL
 210 /* Source IPv4 address */
 211 #define I40E_REG_INSET_L3_SRC_IP4                0x0001800000000000ULL
 212 /* Destination IPv4 address */
 213 #define I40E_REG_INSET_L3_DST_IP4                0x0000001800000000ULL
 214 /* IPv4 Type of Service (TOS) */
 215 #define I40E_REG_INSET_L3_IP4_TOS                0x0040000000000000ULL
 216 /* IPv4 Protocol */
 217 #define I40E_REG_INSET_L3_IP4_PROTO              0x0004000000000000ULL
 218 /* IPv4 Time to Live */
 219 #define I40E_REG_INSET_L3_IP4_TTL                0x0004000000000000ULL
 220 /* Source IPv6 address */
 221 #define I40E_REG_INSET_L3_SRC_IP6                0x0007F80000000000ULL
 222 /* Destination IPv6 address */
 223 #define I40E_REG_INSET_L3_DST_IP6                0x000007F800000000ULL
 224 /* IPv6 Traffic Class (TC) */
 225 #define I40E_REG_INSET_L3_IP6_TC                 0x0040000000000000ULL
 226 /* IPv6 Next Header */
 227 #define I40E_REG_INSET_L3_IP6_NEXT_HDR           0x0008000000000000ULL
 228 /* IPv6 Hop Limit */
 229 #define I40E_REG_INSET_L3_IP6_HOP_LIMIT          0x0008000000000000ULL
 230 /* Source L4 port */
 231 #define I40E_REG_INSET_L4_SRC_PORT               0x0000000400000000ULL
 232 /* Destination L4 port */
 233 #define I40E_REG_INSET_L4_DST_PORT               0x0000000200000000ULL
 234 /* SCTP verification tag */
 235 #define I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG  0x0000000180000000ULL
 236 /* Inner destination MAC address (MAC-in-UDP/MAC-in-GRE)*/
 237 #define I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC   0x0000000001C00000ULL
 238 /* Source port of tunneling UDP */
 239 #define I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT    0x0000000000200000ULL
 240 /* Destination port of tunneling UDP */
 241 #define I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT    0x0000000000100000ULL
 242 /* UDP Tunneling ID, NVGRE/GRE key */
 243 #define I40E_REG_INSET_TUNNEL_ID                 0x00000000000C0000ULL
 244 /* Last ether type */
 245 #define I40E_REG_INSET_LAST_ETHER_TYPE           0x0000000000004000ULL
 246 /* Tunneling outer destination IPv4 address */
 247 #define I40E_REG_INSET_TUNNEL_L3_DST_IP4         0x00000000000000C0ULL
 248 /* Tunneling outer destination IPv6 address */
 249 #define I40E_REG_INSET_TUNNEL_L3_DST_IP6         0x0000000000003FC0ULL
 250 /* 1st word of flex payload */
 251 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD1        0x0000000000002000ULL
 252 /* 2nd word of flex payload */
 253 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD2        0x0000000000001000ULL
 254 /* 3rd word of flex payload */
 255 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD3        0x0000000000000800ULL
 256 /* 4th word of flex payload */
 257 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD4        0x0000000000000400ULL
 258 /* 5th word of flex payload */
 259 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD5        0x0000000000000200ULL
 260 /* 6th word of flex payload */
 261 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD6        0x0000000000000100ULL
 262 /* 7th word of flex payload */
 263 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD7        0x0000000000000080ULL
 264 /* 8th word of flex payload */
 265 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD8        0x0000000000000040ULL
 266 /* all 8 words flex payload */
 267 #define I40E_REG_INSET_FLEX_PAYLOAD_WORDS        0x0000000000003FC0ULL
 268 #define I40E_REG_INSET_MASK_DEFAULT              0x0000000000000000ULL
 269
 270 #define I40E_TRANSLATE_INSET 0
 271 #define I40E_TRANSLATE_REG   1
 272
 273 #define I40E_INSET_IPV4_TOS_MASK        0x0009FF00UL
 274 #define I40E_INSET_IPv4_TTL_MASK        0x000D00FFUL
 275 #define I40E_INSET_IPV4_PROTO_MASK      0x000DFF00UL
 276 #define I40E_INSET_IPV6_TC_MASK         0x0009F00FUL
 277 #define I40E_INSET_IPV6_HOP_LIMIT_MASK  0x000CFF00UL
 278 #define I40E_INSET_IPV6_NEXT_HDR_MASK   0x000C00FFUL
 279
 280 #define I40E_GL_SWT_L2TAGCTRL(_i)             (0x001C0A70 + ((_i) * 4))
 281 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT 16
 282 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK  \
 283         I40E_MASK(0xFFFF, I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT)
 284
 285 /* PCI offset for querying capability */
 286 #define PCI_DEV_CAP_REG            0xA4
 287 /* PCI offset for enabling/disabling Extended Tag */
 288 #define PCI_DEV_CTRL_REG           0xA8
 289 /* Bit mask of Extended Tag capability */
 290 #define PCI_DEV_CAP_EXT_TAG_MASK   0x20
 291 /* Bit shift of Extended Tag enable/disable */
 292 #define PCI_DEV_CTRL_EXT_TAG_SHIFT 8
 293 /* Bit mask of Extended Tag enable/disable */
 294 #define PCI_DEV_CTRL_EXT_TAG_MASK  (1 << PCI_DEV_CTRL_EXT_TAG_SHIFT)
 295
 296 static int eth_i40e_dev_init(struct rte_eth_dev *eth_dev);
 297 static int eth_i40e_dev_uninit(struct rte_eth_dev *eth_dev);
 298 static int i40e_dev_configure(struct rte_eth_dev *dev);
 299 static int i40e_dev_start(struct rte_eth_dev *dev);
 300 static void i40e_dev_stop(struct rte_eth_dev *dev);
 301 static void i40e_dev_close(struct rte_eth_dev *dev);
 302 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
 303 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
 304 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
 305 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
 306 static int i40e_dev_set_link_up(struct rte_eth_dev *dev);
 307 static int i40e_dev_set_link_down(struct rte_eth_dev *dev);
 308 static void i40e_dev_stats_get(struct rte_eth_dev *dev,
 309                                struct rte_eth_stats *stats);
 310 static int i40e_dev_xstats_get(struct rte_eth_dev *dev,
 311                                struct rte_eth_xstat *xstats, unsigned n);
 312 static int i40e_dev_xstats_get_names(struct rte_eth_dev *dev,
 313                                      struct rte_eth_xstat_name *xstats_names,
 314                                      unsigned limit);
 315 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
 316 static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev,
 317                                             uint16_t queue_id,
 318                                             uint8_t stat_idx,
 319                                             uint8_t is_rx);
 320 static void i40e_dev_info_get(struct rte_eth_dev *dev,
 321                               struct rte_eth_dev_info *dev_info);
 322 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
 323                                 uint16_t vlan_id,
 324                                 int on);
 325 static int i40e_vlan_tpid_set(struct rte_eth_dev *dev,
 326                               enum rte_vlan_type vlan_type,
 327                               uint16_t tpid);
 328 static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 329 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
 330                                       uint16_t queue,
 331                                       int on);
 332 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
 333 static int i40e_dev_led_on(struct rte_eth_dev *dev);
 334 static int i40e_dev_led_off(struct rte_eth_dev *dev);
 335 static int i40e_flow_ctrl_get(struct rte_eth_dev *dev,
 336                               struct rte_eth_fc_conf *fc_conf);
 337 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
 338                               struct rte_eth_fc_conf *fc_conf);
 339 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 340                                        struct rte_eth_pfc_conf *pfc_conf);
 341 static void i40e_macaddr_add(struct rte_eth_dev *dev,
 342                           struct ether_addr *mac_addr,
 343                           uint32_t index,
 344                           uint32_t pool);
 345 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 346 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
 347                                     struct rte_eth_rss_reta_entry64 *reta_conf,
 348                                     uint16_t reta_size);
 349 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
 350                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 351                                    uint16_t reta_size);
 352
 353 static int i40e_get_cap(struct i40e_hw *hw);
 354 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
 355 static int i40e_pf_setup(struct i40e_pf *pf);
 356 static int i40e_dev_rxtx_init(struct i40e_pf *pf);
 357 static int i40e_vmdq_setup(struct rte_eth_dev *dev);
 358 static int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb);
 359 static int i40e_dcb_setup(struct rte_eth_dev *dev);
 360 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
 361                 bool offset_loaded, uint64_t *offset, uint64_t *stat);
 362 static void i40e_stat_update_48(struct i40e_hw *hw,
 363                                uint32_t hireg,
 364                                uint32_t loreg,
 365                                bool offset_loaded,
 366                                uint64_t *offset,
 367                                uint64_t *stat);
 368 static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue);
 369 static void i40e_dev_interrupt_handler(
 370                 __rte_unused struct rte_intr_handle *handle, void *param);
 371 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 372                                 uint32_t base, uint32_t num);
 373 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
 374 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
 375                         uint32_t base);
 376 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
 377                         uint16_t num);
 378 static int i40e_dev_init_vlan(struct rte_eth_dev *dev);
 379 static int i40e_veb_release(struct i40e_veb *veb);
 380 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
 381                                                 struct i40e_vsi *vsi);
 382 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
 383 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
 384 static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
 385                                              struct i40e_macvlan_filter *mv_f,
 386                                              int num,
 387                                              struct ether_addr *addr);
 388 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
 389                                              struct i40e_macvlan_filter *mv_f,
 390                                              int num,
 391                                              uint16_t vlan);
 392 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
 393 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
 394                                     struct rte_eth_rss_conf *rss_conf);
 395 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 396                                       struct rte_eth_rss_conf *rss_conf);
 397 static int i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
 398                                         struct rte_eth_udp_tunnel *udp_tunnel);
 399 static int i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
 400                                         struct rte_eth_udp_tunnel *udp_tunnel);
 401 static void i40e_filter_input_set_init(struct i40e_pf *pf);
 402 static int i40e_ethertype_filter_set(struct i40e_pf *pf,
 403                         struct rte_eth_ethertype_filter *filter,
 404                         bool add);
 405 static int i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
 406                                 enum rte_filter_op filter_op,
 407                                 void *arg);
 408 static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
 409                                 enum rte_filter_type filter_type,
 410                                 enum rte_filter_op filter_op,
 411                                 void *arg);
 412 static int i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
 413                                   struct rte_eth_dcb_info *dcb_info);
 414 static void i40e_configure_registers(struct i40e_hw *hw);
 415 static void i40e_hw_init(struct rte_eth_dev *dev);
 416 static int i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi);
 417 static int i40e_mirror_rule_set(struct rte_eth_dev *dev,
 418                         struct rte_eth_mirror_conf *mirror_conf,
 419                         uint8_t sw_id, uint8_t on);
 420 static int i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id);
 421
 422 static int i40e_timesync_enable(struct rte_eth_dev *dev);
 423 static int i40e_timesync_disable(struct rte_eth_dev *dev);
 424 static int i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 425                                            struct timespec *timestamp,
 426                                            uint32_t flags);
 427 static int i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 428                                            struct timespec *timestamp);
 429 static void i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw);
 430
 431 static int i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 432
 433 static int i40e_timesync_read_time(struct rte_eth_dev *dev,
 434                                    struct timespec *timestamp);
 435 static int i40e_timesync_write_time(struct rte_eth_dev *dev,
 436                                     const struct timespec *timestamp);
 437
 438 static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 439                                          uint16_t queue_id);
 440 static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 441                                           uint16_t queue_id);
 442
 443 static int i40e_get_reg_length(struct rte_eth_dev *dev);
 444
 445 static int i40e_get_regs(struct rte_eth_dev *dev,
 446                          struct rte_dev_reg_info *regs);
 447
 448 static int i40e_get_eeprom_length(struct rte_eth_dev *dev);
 449
 450 static int i40e_get_eeprom(struct rte_eth_dev *dev,
 451                            struct rte_dev_eeprom_info *eeprom);
 452
 453 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
 454                                       struct ether_addr *mac_addr);
 455
 456 static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 457
 458 static const struct rte_pci_id pci_id_i40e_map[] = {
 459 #define RTE_PCI_DEV_ID_DECL_I40E(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 460 #include "rte_pci_dev_ids.h"
 461 { .vendor_id = 0, /* sentinel */ },
 462 };
 463
 464 static const struct eth_dev_ops i40e_eth_dev_ops = {
 465         .dev_configure                = i40e_dev_configure,
 466         .dev_start                    = i40e_dev_start,
 467         .dev_stop                     = i40e_dev_stop,
 468         .dev_close                    = i40e_dev_close,
 469         .promiscuous_enable           = i40e_dev_promiscuous_enable,
 470         .promiscuous_disable          = i40e_dev_promiscuous_disable,
 471         .allmulticast_enable          = i40e_dev_allmulticast_enable,
 472         .allmulticast_disable         = i40e_dev_allmulticast_disable,
 473         .dev_set_link_up              = i40e_dev_set_link_up,
 474         .dev_set_link_down            = i40e_dev_set_link_down,
 475         .link_update                  = i40e_dev_link_update,
 476         .stats_get                    = i40e_dev_stats_get,
 477         .xstats_get                   = i40e_dev_xstats_get,
 478         .xstats_get_names             = i40e_dev_xstats_get_names,
 479         .stats_reset                  = i40e_dev_stats_reset,
 480         .xstats_reset                 = i40e_dev_stats_reset,
 481         .queue_stats_mapping_set      = i40e_dev_queue_stats_mapping_set,
 482         .dev_infos_get                = i40e_dev_info_get,
 483         .dev_supported_ptypes_get     = i40e_dev_supported_ptypes_get,
 484         .vlan_filter_set              = i40e_vlan_filter_set,
 485         .vlan_tpid_set                = i40e_vlan_tpid_set,
 486         .vlan_offload_set             = i40e_vlan_offload_set,
 487         .vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
 488         .vlan_pvid_set                = i40e_vlan_pvid_set,
 489         .rx_queue_start               = i40e_dev_rx_queue_start,
 490         .rx_queue_stop                = i40e_dev_rx_queue_stop,
 491         .tx_queue_start               = i40e_dev_tx_queue_start,
 492         .tx_queue_stop                = i40e_dev_tx_queue_stop,
 493         .rx_queue_setup               = i40e_dev_rx_queue_setup,
 494         .rx_queue_intr_enable         = i40e_dev_rx_queue_intr_enable,
 495         .rx_queue_intr_disable        = i40e_dev_rx_queue_intr_disable,
 496         .rx_queue_release             = i40e_dev_rx_queue_release,
 497         .rx_queue_count               = i40e_dev_rx_queue_count,
 498         .rx_descriptor_done           = i40e_dev_rx_descriptor_done,
 499         .tx_queue_setup               = i40e_dev_tx_queue_setup,
 500         .tx_queue_release             = i40e_dev_tx_queue_release,
 501         .dev_led_on                   = i40e_dev_led_on,
 502         .dev_led_off                  = i40e_dev_led_off,
 503         .flow_ctrl_get                = i40e_flow_ctrl_get,
 504         .flow_ctrl_set                = i40e_flow_ctrl_set,
 505         .priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
 506         .mac_addr_add                 = i40e_macaddr_add,
 507         .mac_addr_remove              = i40e_macaddr_remove,
 508         .reta_update                  = i40e_dev_rss_reta_update,
 509         .reta_query                   = i40e_dev_rss_reta_query,
 510         .rss_hash_update              = i40e_dev_rss_hash_update,
 511         .rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
 512         .udp_tunnel_port_add          = i40e_dev_udp_tunnel_port_add,
 513         .udp_tunnel_port_del          = i40e_dev_udp_tunnel_port_del,
 514         .filter_ctrl                  = i40e_dev_filter_ctrl,
 515         .rxq_info_get                 = i40e_rxq_info_get,
 516         .txq_info_get                 = i40e_txq_info_get,
 517         .mirror_rule_set              = i40e_mirror_rule_set,
 518         .mirror_rule_reset            = i40e_mirror_rule_reset,
 519         .timesync_enable              = i40e_timesync_enable,
 520         .timesync_disable             = i40e_timesync_disable,
 521         .timesync_read_rx_timestamp   = i40e_timesync_read_rx_timestamp,
 522         .timesync_read_tx_timestamp   = i40e_timesync_read_tx_timestamp,
 523         .get_dcb_info                 = i40e_dev_get_dcb_info,
 524         .timesync_adjust_time         = i40e_timesync_adjust_time,
 525         .timesync_read_time           = i40e_timesync_read_time,
 526         .timesync_write_time          = i40e_timesync_write_time,
 527         .get_reg_length               = i40e_get_reg_length,
 528         .get_reg                      = i40e_get_regs,
 529         .get_eeprom_length            = i40e_get_eeprom_length,
 530         .get_eeprom                   = i40e_get_eeprom,
 531         .mac_addr_set                 = i40e_set_default_mac_addr,
 532         .mtu_set                      = i40e_dev_mtu_set,
 533 };
 534
 535 /* store statistics names and its offset in stats structure */
 536 struct rte_i40e_xstats_name_off {
 537         char name[RTE_ETH_XSTATS_NAME_SIZE];
 538         unsigned offset;
 539 };
 540
 541 static const struct rte_i40e_xstats_name_off rte_i40e_stats_strings[] = {
 542         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 543         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 544         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 545         {"rx_dropped", offsetof(struct i40e_eth_stats, rx_discards)},
 546         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 547                 rx_unknown_protocol)},
 548         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)},
 549         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)},
 550         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)},
 551         {"tx_dropped", offsetof(struct i40e_eth_stats, tx_discards)},
 552 };
 553
 554 #define I40E_NB_ETH_XSTATS (sizeof(rte_i40e_stats_strings) / \
 555                 sizeof(rte_i40e_stats_strings[0]))
 556
 557 static const struct rte_i40e_xstats_name_off rte_i40e_hw_port_strings[] = {
 558         {"tx_link_down_dropped", offsetof(struct i40e_hw_port_stats,
 559                 tx_dropped_link_down)},
 560         {"rx_crc_errors", offsetof(struct i40e_hw_port_stats, crc_errors)},
 561         {"rx_illegal_byte_errors", offsetof(struct i40e_hw_port_stats,
 562                 illegal_bytes)},
 563         {"rx_error_bytes", offsetof(struct i40e_hw_port_stats, error_bytes)},
 564         {"mac_local_errors", offsetof(struct i40e_hw_port_stats,
 565                 mac_local_faults)},
 566         {"mac_remote_errors", offsetof(struct i40e_hw_port_stats,
 567                 mac_remote_faults)},
 568         {"rx_length_errors", offsetof(struct i40e_hw_port_stats,
 569                 rx_length_errors)},
 570         {"tx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_tx)},
 571         {"rx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_rx)},
 572         {"tx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_tx)},
 573         {"rx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_rx)},
 574         {"rx_size_64_packets", offsetof(struct i40e_hw_port_stats, rx_size_64)},
 575         {"rx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 576                 rx_size_127)},
 577         {"rx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 578                 rx_size_255)},
 579         {"rx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 580                 rx_size_511)},
 581         {"rx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 582                 rx_size_1023)},
 583         {"rx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 584                 rx_size_1522)},
 585         {"rx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 586                 rx_size_big)},
 587         {"rx_undersized_errors", offsetof(struct i40e_hw_port_stats,
 588                 rx_undersize)},
 589         {"rx_oversize_errors", offsetof(struct i40e_hw_port_stats,
 590                 rx_oversize)},
 591         {"rx_mac_short_dropped", offsetof(struct i40e_hw_port_stats,
 592                 mac_short_packet_dropped)},
 593         {"rx_fragmented_errors", offsetof(struct i40e_hw_port_stats,
 594                 rx_fragments)},
 595         {"rx_jabber_errors", offsetof(struct i40e_hw_port_stats, rx_jabber)},
 596         {"tx_size_64_packets", offsetof(struct i40e_hw_port_stats, tx_size_64)},
 597         {"tx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 598                 tx_size_127)},
 599         {"tx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 600                 tx_size_255)},
 601         {"tx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 602                 tx_size_511)},
 603         {"tx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 604                 tx_size_1023)},
 605         {"tx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 606                 tx_size_1522)},
 607         {"tx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 608                 tx_size_big)},
 609         {"rx_flow_director_atr_match_packets",
 610                 offsetof(struct i40e_hw_port_stats, fd_atr_match)},
 611         {"rx_flow_director_sb_match_packets",
 612                 offsetof(struct i40e_hw_port_stats, fd_sb_match)},
 613         {"tx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 614                 tx_lpi_status)},
 615         {"rx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 616                 rx_lpi_status)},
 617         {"tx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 618                 tx_lpi_count)},
 619         {"rx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 620                 rx_lpi_count)},
 621 };
 622
 623 #define I40E_NB_HW_PORT_XSTATS (sizeof(rte_i40e_hw_port_strings) / \
 624                 sizeof(rte_i40e_hw_port_strings[0]))
 625
 626 static const struct rte_i40e_xstats_name_off rte_i40e_rxq_prio_strings[] = {
 627         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 628                 priority_xon_rx)},
 629         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 630                 priority_xoff_rx)},
 631 };
 632
 633 #define I40E_NB_RXQ_PRIO_XSTATS (sizeof(rte_i40e_rxq_prio_strings) / \
 634                 sizeof(rte_i40e_rxq_prio_strings[0]))
 635
 636 static const struct rte_i40e_xstats_name_off rte_i40e_txq_prio_strings[] = {
 637         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 638                 priority_xon_tx)},
 639         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 640                 priority_xoff_tx)},
 641         {"xon_to_xoff_packets", offsetof(struct i40e_hw_port_stats,
 642                 priority_xon_2_xoff)},
 643 };
 644
 645 #define I40E_NB_TXQ_PRIO_XSTATS (sizeof(rte_i40e_txq_prio_strings) / \
 646                 sizeof(rte_i40e_txq_prio_strings[0]))
 647
 648 static struct eth_driver rte_i40e_pmd = {
 649         .pci_drv = {
 650                 .name = "rte_i40e_pmd",
 651                 .id_table = pci_id_i40e_map,
 652                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 653                         RTE_PCI_DRV_DETACHABLE,
 654         },
 655         .eth_dev_init = eth_i40e_dev_init,
 656         .eth_dev_uninit = eth_i40e_dev_uninit,
 657         .dev_private_size = sizeof(struct i40e_adapter),
 658 };
 659
 660 static inline int
 661 rte_i40e_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 662                                      struct rte_eth_link *link)
 663 {
 664         struct rte_eth_link *dst = link;
 665         struct rte_eth_link *src = &(dev->data->dev_link);
 666
 667         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 668                                         *(uint64_t *)src) == 0)
 669                 return -1;
 670
 671         return 0;
 672 }
 673
 674 static inline int
 675 rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 676                                       struct rte_eth_link *link)
 677 {
 678         struct rte_eth_link *dst = &(dev->data->dev_link);
 679         struct rte_eth_link *src = link;
 680
 681         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 682                                         *(uint64_t *)src) == 0)
 683                 return -1;
 684
 685         return 0;
 686 }
 687
 688 /*
 689  * Driver initialization routine.
 690  * Invoked once at EAL init time.
 691  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 692  */
 693 static int
 694 rte_i40e_pmd_init(const char *name __rte_unused,
 695                   const char *params __rte_unused)
 696 {
 697         PMD_INIT_FUNC_TRACE();
 698         rte_eth_driver_register(&rte_i40e_pmd);
 699
 700         return 0;
 701 }
 702
 703 static struct rte_driver rte_i40e_driver = {
 704         .type = PMD_PDEV,
 705         .init = rte_i40e_pmd_init,
 706 };
 707
 708 PMD_REGISTER_DRIVER(rte_i40e_driver, i40e);
 709 DRIVER_REGISTER_PCI_TABLE(i40e, pci_id_i40e_map);
 710
 711 /*
 712  * Initialize registers for flexible payload, which should be set by NVM.
 713  * This should be removed from code once it is fixed in NVM.
 714  */
 715 #ifndef I40E_GLQF_ORT
 716 #define I40E_GLQF_ORT(_i)    (0x00268900 + ((_i) * 4))
 717 #endif
 718 #ifndef I40E_GLQF_PIT
 719 #define I40E_GLQF_PIT(_i)    (0x00268C80 + ((_i) * 4))
 720 #endif
 721
 722 static inline void i40e_flex_payload_reg_init(struct i40e_hw *hw)
 723 {
 724         I40E_WRITE_REG(hw, I40E_GLQF_ORT(18), 0x00000030);
 725         I40E_WRITE_REG(hw, I40E_GLQF_ORT(19), 0x00000030);
 726         I40E_WRITE_REG(hw, I40E_GLQF_ORT(26), 0x0000002B);
 727         I40E_WRITE_REG(hw, I40E_GLQF_ORT(30), 0x0000002B);
 728         I40E_WRITE_REG(hw, I40E_GLQF_ORT(33), 0x000000E0);
 729         I40E_WRITE_REG(hw, I40E_GLQF_ORT(34), 0x000000E3);
 730         I40E_WRITE_REG(hw, I40E_GLQF_ORT(35), 0x000000E6);
 731         I40E_WRITE_REG(hw, I40E_GLQF_ORT(20), 0x00000031);
 732         I40E_WRITE_REG(hw, I40E_GLQF_ORT(23), 0x00000031);
 733         I40E_WRITE_REG(hw, I40E_GLQF_ORT(63), 0x0000002D);
 734
 735         /* GLQF_PIT Registers */
 736         I40E_WRITE_REG(hw, I40E_GLQF_PIT(16), 0x00007480);
 737         I40E_WRITE_REG(hw, I40E_GLQF_PIT(17), 0x00007440);
 738 }
 739
 740 #define I40E_FLOW_CONTROL_ETHERTYPE  0x8808
 741
 742 /*
 743  * Add a ethertype filter to drop all flow control frames transmitted
 744  * from VSIs.
 745 */
 746 static void
 747 i40e_add_tx_flow_control_drop_filter(struct i40e_pf *pf)
 748 {
 749         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 750         uint16_t flags = I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC |
 751                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP |
 752                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX;
 753         int ret;
 754
 755         ret = i40e_aq_add_rem_control_packet_filter(hw, NULL,
 756                                 I40E_FLOW_CONTROL_ETHERTYPE, flags,
 757                                 pf->main_vsi_seid, 0,
 758                                 TRUE, NULL, NULL);
 759         if (ret)
 760                 PMD_INIT_LOG(ERR, "Failed to add filter to drop flow control "
 761                                   " frames from VSIs.");
 762 }
 763
 764 static int
 765 floating_veb_list_handler(__rte_unused const char *key,
 766                           const char *floating_veb_value,
 767                           void *opaque)
 768 {
 769         int idx = 0;
 770         unsigned int count = 0;
 771         char *end = NULL;
 772         int min, max;
 773         bool *vf_floating_veb = opaque;
 774
 775         while (isblank(*floating_veb_value))
 776                 floating_veb_value++;
 777
 778         /* Reset floating VEB configuration for VFs */
 779         for (idx = 0; idx < I40E_MAX_VF; idx++)
 780                 vf_floating_veb[idx] = false;
 781
 782         min = I40E_MAX_VF;
 783         do {
 784                 while (isblank(*floating_veb_value))
 785                         floating_veb_value++;
 786                 if (*floating_veb_value == '\0')
 787                         return -1;
 788                 errno = 0;
 789                 idx = strtoul(floating_veb_value, &end, 10);
 790                 if (errno || end == NULL)
 791                         return -1;
 792                 while (isblank(*end))
 793                         end++;
 794                 if (*end == '-') {
 795                         min = idx;
 796                 } else if ((*end == ';') || (*end == '\0')) {
 797                         max = idx;
 798                         if (min == I40E_MAX_VF)
 799                                 min = idx;
 800                         if (max >= I40E_MAX_VF)
 801                                 max = I40E_MAX_VF - 1;
 802                         for (idx = min; idx <= max; idx++) {
 803                                 vf_floating_veb[idx] = true;
 804                                 count++;
 805                         }
 806                         min = I40E_MAX_VF;
 807                 } else {
 808                         return -1;
 809                 }
 810                 floating_veb_value = end + 1;
 811         } while (*end != '\0');
 812
 813         if (count == 0)
 814                 return -1;
 815
 816         return 0;
 817 }
 818
 819 static void
 820 config_vf_floating_veb(struct rte_devargs *devargs,
 821                        uint16_t floating_veb,
 822                        bool *vf_floating_veb)
 823 {
 824         struct rte_kvargs *kvlist;
 825         int i;
 826         const char *floating_veb_list = ETH_I40E_FLOATING_VEB_LIST_ARG;
 827
 828         if (!floating_veb)
 829                 return;
 830         /* All the VFs attach to the floating VEB by default
 831          * when the floating VEB is enabled.
 832          */
 833         for (i = 0; i < I40E_MAX_VF; i++)
 834                 vf_floating_veb[i] = true;
 835
 836         if (devargs == NULL)
 837                 return;
 838
 839         kvlist = rte_kvargs_parse(devargs->args, NULL);
 840         if (kvlist == NULL)
 841                 return;
 842
 843         if (!rte_kvargs_count(kvlist, floating_veb_list)) {
 844                 rte_kvargs_free(kvlist);
 845                 return;
 846         }
 847         /* When the floating_veb_list parameter exists, all the VFs
 848          * will attach to the legacy VEB firstly, then configure VFs
 849          * to the floating VEB according to the floating_veb_list.
 850          */
 851         if (rte_kvargs_process(kvlist, floating_veb_list,
 852                                floating_veb_list_handler,
 853                                vf_floating_veb) < 0) {
 854                 rte_kvargs_free(kvlist);
 855                 return;
 856         }
 857         rte_kvargs_free(kvlist);
 858 }
 859
 860 static int
 861 i40e_check_floating_handler(__rte_unused const char *key,
 862                             const char *value,
 863                             __rte_unused void *opaque)
 864 {
 865         if (strcmp(value, "1"))
 866                 return -1;
 867
 868         return 0;
 869 }
 870
 871 static int
 872 is_floating_veb_supported(struct rte_devargs *devargs)
 873 {
 874         struct rte_kvargs *kvlist;
 875         const char *floating_veb_key = ETH_I40E_FLOATING_VEB_ARG;
 876
 877         if (devargs == NULL)
 878                 return 0;
 879
 880         kvlist = rte_kvargs_parse(devargs->args, NULL);
 881         if (kvlist == NULL)
 882                 return 0;
 883
 884         if (!rte_kvargs_count(kvlist, floating_veb_key)) {
 885                 rte_kvargs_free(kvlist);
 886                 return 0;
 887         }
 888         /* Floating VEB is enabled when there's key-value:
 889          * enable_floating_veb=1
 890          */
 891         if (rte_kvargs_process(kvlist, floating_veb_key,
 892                                i40e_check_floating_handler, NULL) < 0) {
 893                 rte_kvargs_free(kvlist);
 894                 return 0;
 895         }
 896         rte_kvargs_free(kvlist);
 897
 898         return 1;
 899 }
 900
 901 static void
 902 config_floating_veb(struct rte_eth_dev *dev)
 903 {
 904         struct rte_pci_device *pci_dev = dev->pci_dev;
 905         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 906         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 907
 908         memset(pf->floating_veb_list, 0, sizeof(pf->floating_veb_list));
 909
 910         if (hw->aq.fw_maj_ver >= FLOATING_VEB_SUPPORTED_FW_MAJ) {
 911                 pf->floating_veb = is_floating_veb_supported(pci_dev->devargs);
 912                 config_vf_floating_veb(pci_dev->devargs, pf->floating_veb,
 913                                        pf->floating_veb_list);
 914         } else {
 915                 pf->floating_veb = false;
 916         }
 917 }
 918
 919 static int
 920 eth_i40e_dev_init(struct rte_eth_dev *dev)
 921 {
 922         struct rte_pci_device *pci_dev;
 923         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 924         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 925         struct i40e_vsi *vsi;
 926         int ret;
 927         uint32_t len;
 928         uint8_t aq_fail = 0;
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         dev->dev_ops = &i40e_eth_dev_ops;
 933         dev->rx_pkt_burst = i40e_recv_pkts;
 934         dev->tx_pkt_burst = i40e_xmit_pkts;
 935
 936         /* for secondary processes, we don't initialise any further as primary
 937          * has already done this work. Only check we don't need a different
 938          * RX function */
 939         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 940                 i40e_set_rx_function(dev);
 941                 i40e_set_tx_function(dev);
 942                 return 0;
 943         }
 944         pci_dev = dev->pci_dev;
 945
 946         rte_eth_copy_pci_info(dev, pci_dev);
 947
 948         pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 949         pf->adapter->eth_dev = dev;
 950         pf->dev_data = dev->data;
 951
 952         hw->back = I40E_PF_TO_ADAPTER(pf);
 953         hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
 954         if (!hw->hw_addr) {
 955                 PMD_INIT_LOG(ERR, "Hardware is not available, "
 956                              "as address is NULL");
 957                 return -ENODEV;
 958         }
 959
 960         hw->vendor_id = pci_dev->id.vendor_id;
 961         hw->device_id = pci_dev->id.device_id;
 962         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 963         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 964         hw->bus.device = pci_dev->addr.devid;
 965         hw->bus.func = pci_dev->addr.function;
 966         hw->adapter_stopped = 0;
 967
 968         /* Make sure all is clean before doing PF reset */
 969         i40e_clear_hw(hw);
 970
 971         /* Initialize the hardware */
 972         i40e_hw_init(dev);
 973
 974         /* Reset here to make sure all is clean for each PF */
 975         ret = i40e_pf_reset(hw);
 976         if (ret) {
 977                 PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
 978                 return ret;
 979         }
 980
 981         /* Initialize the shared code (base driver) */
 982         ret = i40e_init_shared_code(hw);
 983         if (ret) {
 984                 PMD_INIT_LOG(ERR, "Failed to init shared code (base driver): %d", ret);
 985                 return ret;
 986         }
 987
 988         /*
 989          * To work around the NVM issue,initialize registers
 990          * for flexible payload by software.
 991          * It should be removed once issues are fixed in NVM.
 992          */
 993         i40e_flex_payload_reg_init(hw);
 994
 995         /* Initialize the input set for filters (hash and fd) to default value */
 996         i40e_filter_input_set_init(pf);
 997
 998         /* Initialize the parameters for adminq */
 999         i40e_init_adminq_parameter(hw);
1000         ret = i40e_init_adminq(hw);
1001         if (ret != I40E_SUCCESS) {
1002                 PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
1003                 return -EIO;
1004         }
1005         PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
1006                      hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
1007                      hw->aq.api_maj_ver, hw->aq.api_min_ver,
1008                      ((hw->nvm.version >> 12) & 0xf),
1009                      ((hw->nvm.version >> 4) & 0xff),
1010                      (hw->nvm.version & 0xf), hw->nvm.eetrack);
1011
1012         /* Need the special FW version to support floating VEB */
1013         config_floating_veb(dev);
1014         /* Clear PXE mode */
1015         i40e_clear_pxe_mode(hw);
1016
1017         /*
1018          * On X710, performance number is far from the expectation on recent
1019          * firmware versions. The fix for this issue may not be integrated in
1020          * the following firmware version. So the workaround in software driver
1021          * is needed. It needs to modify the initial values of 3 internal only
1022          * registers. Note that the workaround can be removed when it is fixed
1023          * in firmware in the future.
1024          */
1025         i40e_configure_registers(hw);
1026
1027         /* Get hw capabilities */
1028         ret = i40e_get_cap(hw);
1029         if (ret != I40E_SUCCESS) {
1030                 PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
1031                 goto err_get_capabilities;
1032         }
1033
1034         /* Initialize parameters for PF */
1035         ret = i40e_pf_parameter_init(dev);
1036         if (ret != 0) {
1037                 PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
1038                 goto err_parameter_init;
1039         }
1040
1041         /* Initialize the queue management */
1042         ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
1043         if (ret < 0) {
1044                 PMD_INIT_LOG(ERR, "Failed to init queue pool");
1045                 goto err_qp_pool_init;
1046         }
1047         ret = i40e_res_pool_init(&pf->msix_pool, 1,
1048                                 hw->func_caps.num_msix_vectors - 1);
1049         if (ret < 0) {
1050                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
1051                 goto err_msix_pool_init;
1052         }
1053
1054         /* Initialize lan hmc */
1055         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1056                                 hw->func_caps.num_rx_qp, 0, 0);
1057         if (ret != I40E_SUCCESS) {
1058                 PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
1059                 goto err_init_lan_hmc;
1060         }
1061
1062         /* Configure lan hmc */
1063         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1064         if (ret != I40E_SUCCESS) {
1065                 PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
1066                 goto err_configure_lan_hmc;
1067         }
1068
1069         /* Get and check the mac address */
1070         i40e_get_mac_addr(hw, hw->mac.addr);
1071         if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
1072                 PMD_INIT_LOG(ERR, "mac address is not valid");
1073                 ret = -EIO;
1074                 goto err_get_mac_addr;
1075         }
1076         /* Copy the permanent MAC address */
1077         ether_addr_copy((struct ether_addr *) hw->mac.addr,
1078                         (struct ether_addr *) hw->mac.perm_addr);
1079
1080         /* Disable flow control */
1081         hw->fc.requested_mode = I40E_FC_NONE;
1082         i40e_set_fc(hw, &aq_fail, TRUE);
1083
1084         /* Set the global registers with default ether type value */
1085         ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, ETHER_TYPE_VLAN);
1086         if (ret != I40E_SUCCESS) {
1087                 PMD_INIT_LOG(ERR, "Failed to set the default outer "
1088                              "VLAN ether type");
1089                 goto err_setup_pf_switch;
1090         }
1091
1092         /* PF setup, which includes VSI setup */
1093         ret = i40e_pf_setup(pf);
1094         if (ret) {
1095                 PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
1096                 goto err_setup_pf_switch;
1097         }
1098
1099         /* reset all stats of the device, including pf and main vsi */
1100         i40e_dev_stats_reset(dev);
1101
1102         vsi = pf->main_vsi;
1103
1104         /* Disable double vlan by default */
1105         i40e_vsi_config_double_vlan(vsi, FALSE);
1106
1107         if (!vsi->max_macaddrs)
1108                 len = ETHER_ADDR_LEN;
1109         else
1110                 len = ETHER_ADDR_LEN * vsi->max_macaddrs;
1111
1112         /* Should be after VSI initialized */
1113         dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
1114         if (!dev->data->mac_addrs) {
1115                 PMD_INIT_LOG(ERR, "Failed to allocated memory "
1116                                         "for storing mac address");
1117                 goto err_mac_alloc;
1118         }
1119         ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
1120                                         &dev->data->mac_addrs[0]);
1121
1122         /* initialize pf host driver to setup SRIOV resource if applicable */
1123         i40e_pf_host_init(dev);
1124
1125         /* register callback func to eal lib */
1126         rte_intr_callback_register(&(pci_dev->intr_handle),
1127                 i40e_dev_interrupt_handler, (void *)dev);
1128
1129         /* configure and enable device interrupt */
1130         i40e_pf_config_irq0(hw, TRUE);
1131         i40e_pf_enable_irq0(hw);
1132
1133         /* enable uio intr after callback register */
1134         rte_intr_enable(&(pci_dev->intr_handle));
1135         /*
1136          * Add an ethertype filter to drop all flow control frames transmitted
1137          * from VSIs. By doing so, we stop VF from sending out PAUSE or PFC
1138          * frames to wire.
1139          */
1140         i40e_add_tx_flow_control_drop_filter(pf);
1141
1142         /* Set the max frame size to 0x2600 by default,
1143          * in case other drivers changed the default value.
1144          */
1145         i40e_aq_set_mac_config(hw, I40E_FRAME_SIZE_MAX, TRUE, 0, NULL);
1146
1147         /* initialize mirror rule list */
1148         TAILQ_INIT(&pf->mirror_list);
1149
1150         /* Init dcb to sw mode by default */
1151         ret = i40e_dcb_init_configure(dev, TRUE);
1152         if (ret != I40E_SUCCESS) {
1153                 PMD_INIT_LOG(INFO, "Failed to init dcb.");
1154                 pf->flags &= ~I40E_FLAG_DCB;
1155         }
1156
1157         return 0;
1158
1159 err_mac_alloc:
1160         i40e_vsi_release(pf->main_vsi);
1161 err_setup_pf_switch:
1162 err_get_mac_addr:
1163 err_configure_lan_hmc:
1164         (void)i40e_shutdown_lan_hmc(hw);
1165 err_init_lan_hmc:
1166         i40e_res_pool_destroy(&pf->msix_pool);
1167 err_msix_pool_init:
1168         i40e_res_pool_destroy(&pf->qp_pool);
1169 err_qp_pool_init:
1170 err_parameter_init:
1171 err_get_capabilities:
1172         (void)i40e_shutdown_adminq(hw);
1173
1174         return ret;
1175 }
1176
1177 static int
1178 eth_i40e_dev_uninit(struct rte_eth_dev *dev)
1179 {
1180         struct rte_pci_device *pci_dev;
1181         struct i40e_hw *hw;
1182         struct i40e_filter_control_settings settings;
1183         int ret;
1184         uint8_t aq_fail = 0;
1185
1186         PMD_INIT_FUNC_TRACE();
1187
1188         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1189                 return 0;
1190
1191         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1192         pci_dev = dev->pci_dev;
1193
1194         if (hw->adapter_stopped == 0)
1195                 i40e_dev_close(dev);
1196
1197         dev->dev_ops = NULL;
1198         dev->rx_pkt_burst = NULL;
1199         dev->tx_pkt_burst = NULL;
1200
1201         /* Disable LLDP */
1202         ret = i40e_aq_stop_lldp(hw, true, NULL);
1203         if (ret != I40E_SUCCESS) /* Its failure can be ignored */
1204                 PMD_INIT_LOG(INFO, "Failed to stop lldp");
1205
1206         /* Clear PXE mode */
1207         i40e_clear_pxe_mode(hw);
1208
1209         /* Unconfigure filter control */
1210         memset(&settings, 0, sizeof(settings));
1211         ret = i40e_set_filter_control(hw, &settings);
1212         if (ret)
1213                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
1214                                         ret);
1215
1216         /* Disable flow control */
1217         hw->fc.requested_mode = I40E_FC_NONE;
1218         i40e_set_fc(hw, &aq_fail, TRUE);
1219
1220         /* uninitialize pf host driver */
1221         i40e_pf_host_uninit(dev);
1222
1223         rte_free(dev->data->mac_addrs);
1224         dev->data->mac_addrs = NULL;
1225
1226         /* disable uio intr before callback unregister */
1227         rte_intr_disable(&(pci_dev->intr_handle));
1228
1229         /* register callback func to eal lib */
1230         rte_intr_callback_unregister(&(pci_dev->intr_handle),
1231                 i40e_dev_interrupt_handler, (void *)dev);
1232
1233         return 0;
1234 }
1235
1236 static int
1237 i40e_dev_configure(struct rte_eth_dev *dev)
1238 {
1239         struct i40e_adapter *ad =
1240                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1241         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1242         enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1243         int i, ret;
1244
1245         /* Initialize to TRUE. If any of Rx queues doesn't meet the
1246          * bulk allocation or vector Rx preconditions we will reset it.
1247          */
1248         ad->rx_bulk_alloc_allowed = true;
1249         ad->rx_vec_allowed = true;
1250         ad->tx_simple_allowed = true;
1251         ad->tx_vec_allowed = true;
1252
1253         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
1254                 ret = i40e_fdir_setup(pf);
1255                 if (ret != I40E_SUCCESS) {
1256                         PMD_DRV_LOG(ERR, "Failed to setup flow director.");
1257                         return -ENOTSUP;
1258                 }
1259                 ret = i40e_fdir_configure(dev);
1260                 if (ret < 0) {
1261                         PMD_DRV_LOG(ERR, "failed to configure fdir.");
1262                         goto err;
1263                 }
1264         } else
1265                 i40e_fdir_teardown(pf);
1266
1267         ret = i40e_dev_init_vlan(dev);
1268         if (ret < 0)
1269                 goto err;
1270
1271         /* VMDQ setup.
1272          *  Needs to move VMDQ setting out of i40e_pf_config_mq_rx() as VMDQ and
1273          *  RSS setting have different requirements.
1274          *  General PMD driver call sequence are NIC init, configure,
1275          *  rx/tx_queue_setup and dev_start. In rx/tx_queue_setup() function, it
1276          *  will try to lookup the VSI that specific queue belongs to if VMDQ
1277          *  applicable. So, VMDQ setting has to be done before
1278          *  rx/tx_queue_setup(). This function is good  to place vmdq_setup.
1279          *  For RSS setting, it will try to calculate actual configured RX queue
1280          *  number, which will be available after rx_queue_setup(). dev_start()
1281          *  function is good to place RSS setup.
1282          */
1283         if (mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
1284                 ret = i40e_vmdq_setup(dev);
1285                 if (ret)
1286                         goto err;
1287         }
1288
1289         if (mq_mode & ETH_MQ_RX_DCB_FLAG) {
1290                 ret = i40e_dcb_setup(dev);
1291                 if (ret) {
1292                         PMD_DRV_LOG(ERR, "failed to configure DCB.");
1293                         goto err_dcb;
1294                 }
1295         }
1296
1297         return 0;
1298
1299 err_dcb:
1300         /* need to release vmdq resource if exists */
1301         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1302                 i40e_vsi_release(pf->vmdq[i].vsi);
1303                 pf->vmdq[i].vsi = NULL;
1304         }
1305         rte_free(pf->vmdq);
1306         pf->vmdq = NULL;
1307 err:
1308         /* need to release fdir resource if exists */
1309         i40e_fdir_teardown(pf);
1310         return ret;
1311 }
1312
1313 void
1314 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
1315 {
1316         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1317         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1318         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1319         uint16_t msix_vect = vsi->msix_intr;
1320         uint16_t i;
1321
1322         for (i = 0; i < vsi->nb_qps; i++) {
1323                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1324                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1325                 rte_wmb();
1326         }
1327
1328         if (vsi->type != I40E_VSI_SRIOV) {
1329                 if (!rte_intr_allow_others(intr_handle)) {
1330                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1331                                        I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
1332                         I40E_WRITE_REG(hw,
1333                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1334                                        0);
1335                 } else {
1336                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1337                                        I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK);
1338                         I40E_WRITE_REG(hw,
1339                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1340                                                        msix_vect - 1), 0);
1341                 }
1342         } else {
1343                 uint32_t reg;
1344                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1345                         vsi->user_param + (msix_vect - 1);
1346
1347                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1348                                I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
1349         }
1350         I40E_WRITE_FLUSH(hw);
1351 }
1352
1353 static void
1354 __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect,
1355                        int base_queue, int nb_queue)
1356 {
1357         int i;
1358         uint32_t val;
1359         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1360
1361         /* Bind all RX queues to allocated MSIX interrupt */
1362         for (i = 0; i < nb_queue; i++) {
1363                 val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
1364                         I40E_QINT_RQCTL_ITR_INDX_MASK |
1365                         ((base_queue + i + 1) <<
1366                          I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
1367                         (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
1368                         I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1369
1370                 if (i == nb_queue - 1)
1371                         val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
1372                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(base_queue + i), val);
1373         }
1374
1375         /* Write first RX queue to Link list register as the head element */
1376         if (vsi->type != I40E_VSI_SRIOV) {
1377                 uint16_t interval =
1378                         i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
1379
1380                 if (msix_vect == I40E_MISC_VEC_ID) {
1381                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1382                                        (base_queue <<
1383                                         I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1384                                        (0x0 <<
1385                                         I40E_PFINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1386                         I40E_WRITE_REG(hw,
1387                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1388                                        interval);
1389                 } else {
1390                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1391                                        (base_queue <<
1392                                         I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1393                                        (0x0 <<
1394                                         I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1395                         I40E_WRITE_REG(hw,
1396                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1397                                                        msix_vect - 1),
1398                                        interval);
1399                 }
1400         } else {
1401                 uint32_t reg;
1402
1403                 if (msix_vect == I40E_MISC_VEC_ID) {
1404                         I40E_WRITE_REG(hw,
1405                                        I40E_VPINT_LNKLST0(vsi->user_param),
1406                                        (base_queue <<
1407                                         I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1408                                        (0x0 <<
1409                                         I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1410                 } else {
1411                         /* num_msix_vectors_vf needs to minus irq0 */
1412                         reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1413                                 vsi->user_param + (msix_vect - 1);
1414
1415                         I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1416                                        (base_queue <<
1417                                         I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1418                                        (0x0 <<
1419                                         I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1420                 }
1421         }
1422
1423         I40E_WRITE_FLUSH(hw);
1424 }
1425
1426 void
1427 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi)
1428 {
1429         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1430         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1431         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1432         uint16_t msix_vect = vsi->msix_intr;
1433         uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd);
1434         uint16_t queue_idx = 0;
1435         int record = 0;
1436         uint32_t val;
1437         int i;
1438
1439         for (i = 0; i < vsi->nb_qps; i++) {
1440                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1441                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1442         }
1443
1444         /* INTENA flag is not auto-cleared for interrupt */
1445         val = I40E_READ_REG(hw, I40E_GLINT_CTL);
1446         val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK |
1447                 I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK |
1448                 I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK;
1449         I40E_WRITE_REG(hw, I40E_GLINT_CTL, val);
1450
1451         /* VF bind interrupt */
1452         if (vsi->type == I40E_VSI_SRIOV) {
1453                 __vsi_queues_bind_intr(vsi, msix_vect,
1454                                        vsi->base_queue, vsi->nb_qps);
1455                 return;
1456         }
1457
1458         /* PF & VMDq bind interrupt */
1459         if (rte_intr_dp_is_en(intr_handle)) {
1460                 if (vsi->type == I40E_VSI_MAIN) {
1461                         queue_idx = 0;
1462                         record = 1;
1463                 } else if (vsi->type == I40E_VSI_VMDQ2) {
1464                         struct i40e_vsi *main_vsi =
1465                                 I40E_DEV_PRIVATE_TO_MAIN_VSI(vsi->adapter);
1466                         queue_idx = vsi->base_queue - main_vsi->nb_qps;
1467                         record = 1;
1468                 }
1469         }
1470
1471         for (i = 0; i < vsi->nb_used_qps; i++) {
1472                 if (nb_msix <= 1) {
1473                         if (!rte_intr_allow_others(intr_handle))
1474                                 /* allow to share MISC_VEC_ID */
1475                                 msix_vect = I40E_MISC_VEC_ID;
1476
1477                         /* no enough msix_vect, map all to one */
1478                         __vsi_queues_bind_intr(vsi, msix_vect,
1479                                                vsi->base_queue + i,
1480                                                vsi->nb_used_qps - i);
1481                         for (; !!record && i < vsi->nb_used_qps; i++)
1482                                 intr_handle->intr_vec[queue_idx + i] =
1483                                         msix_vect;
1484                         break;
1485                 }
1486                 /* 1:1 queue/msix_vect mapping */
1487                 __vsi_queues_bind_intr(vsi, msix_vect,
1488                                        vsi->base_queue + i, 1);
1489                 if (!!record)
1490                         intr_handle->intr_vec[queue_idx + i] = msix_vect;
1491
1492                 msix_vect++;
1493                 nb_msix--;
1494         }
1495 }
1496
1497 static void
1498 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
1499 {
1500         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1501         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1502         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1503         uint16_t interval = i40e_calc_itr_interval(\
1504                 RTE_LIBRTE_I40E_ITR_INTERVAL);
1505         uint16_t msix_intr, i;
1506
1507         if (rte_intr_allow_others(intr_handle))
1508                 for (i = 0; i < vsi->nb_msix; i++) {
1509                         msix_intr = vsi->msix_intr + i;
1510                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1511                                 I40E_PFINT_DYN_CTLN_INTENA_MASK |
1512                                 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
1513                                 (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
1514                                 (interval <<
1515                                  I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
1516                 }
1517         else
1518                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
1519                                I40E_PFINT_DYN_CTL0_INTENA_MASK |
1520                                I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
1521                                (0 << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT) |
1522                                (interval <<
1523                                 I40E_PFINT_DYN_CTL0_INTERVAL_SHIFT));
1524
1525         I40E_WRITE_FLUSH(hw);
1526 }
1527
1528 static void
1529 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
1530 {
1531         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1532         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1533         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1534         uint16_t msix_intr, i;
1535
1536         if (rte_intr_allow_others(intr_handle))
1537                 for (i = 0; i < vsi->nb_msix; i++) {
1538                         msix_intr = vsi->msix_intr + i;
1539                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1540                                        0);
1541                 }
1542         else
1543                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
1544
1545         I40E_WRITE_FLUSH(hw);
1546 }
1547
1548 static inline uint8_t
1549 i40e_parse_link_speeds(uint16_t link_speeds)
1550 {
1551         uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN;
1552
1553         if (link_speeds & ETH_LINK_SPEED_40G)
1554                 link_speed |= I40E_LINK_SPEED_40GB;
1555         if (link_speeds & ETH_LINK_SPEED_20G)
1556                 link_speed |= I40E_LINK_SPEED_20GB;
1557         if (link_speeds & ETH_LINK_SPEED_10G)
1558                 link_speed |= I40E_LINK_SPEED_10GB;
1559         if (link_speeds & ETH_LINK_SPEED_1G)
1560                 link_speed |= I40E_LINK_SPEED_1GB;
1561         if (link_speeds & ETH_LINK_SPEED_100M)
1562                 link_speed |= I40E_LINK_SPEED_100MB;
1563
1564         return link_speed;
1565 }
1566
1567 static int
1568 i40e_phy_conf_link(struct i40e_hw *hw,
1569                    uint8_t abilities,
1570                    uint8_t force_speed)
1571 {
1572         enum i40e_status_code status;
1573         struct i40e_aq_get_phy_abilities_resp phy_ab;
1574         struct i40e_aq_set_phy_config phy_conf;
1575         const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX |
1576                         I40E_AQ_PHY_FLAG_PAUSE_RX |
1577                         I40E_AQ_PHY_FLAG_PAUSE_RX |
1578                         I40E_AQ_PHY_FLAG_LOW_POWER;
1579         const uint8_t advt = I40E_LINK_SPEED_40GB |
1580                         I40E_LINK_SPEED_10GB |
1581                         I40E_LINK_SPEED_1GB |
1582                         I40E_LINK_SPEED_100MB;
1583         int ret = -ENOTSUP;
1584
1585
1586         status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab,
1587                                               NULL);
1588         if (status)
1589                 return ret;
1590
1591         memset(&phy_conf, 0, sizeof(phy_conf));
1592
1593         /* bits 0-2 use the values from get_phy_abilities_resp */
1594         abilities &= ~mask;
1595         abilities |= phy_ab.abilities & mask;
1596
1597         /* update ablities and speed */
1598         if (abilities & I40E_AQ_PHY_AN_ENABLED)
1599                 phy_conf.link_speed = advt;
1600         else
1601                 phy_conf.link_speed = force_speed;
1602
1603         phy_conf.abilities = abilities;
1604
1605         /* use get_phy_abilities_resp value for the rest */
1606         phy_conf.phy_type = phy_ab.phy_type;
1607         phy_conf.eee_capability = phy_ab.eee_capability;
1608         phy_conf.eeer = phy_ab.eeer_val;
1609         phy_conf.low_power_ctrl = phy_ab.d3_lpan;
1610
1611         PMD_DRV_LOG(DEBUG, "\tCurrent: abilities %x, link_speed %x",
1612                     phy_ab.abilities, phy_ab.link_speed);
1613         PMD_DRV_LOG(DEBUG, "\tConfig:  abilities %x, link_speed %x",
1614                     phy_conf.abilities, phy_conf.link_speed);
1615
1616         status = i40e_aq_set_phy_config(hw, &phy_conf, NULL);
1617         if (status)
1618                 return ret;
1619
1620         return I40E_SUCCESS;
1621 }
1622
1623 static int
1624 i40e_apply_link_speed(struct rte_eth_dev *dev)
1625 {
1626         uint8_t speed;
1627         uint8_t abilities = 0;
1628         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1629         struct rte_eth_conf *conf = &dev->data->dev_conf;
1630
1631         speed = i40e_parse_link_speeds(conf->link_speeds);
1632         abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
1633         if (!(conf->link_speeds & ETH_LINK_SPEED_FIXED))
1634                 abilities |= I40E_AQ_PHY_AN_ENABLED;
1635         abilities |= I40E_AQ_PHY_LINK_ENABLED;
1636
1637         /* Skip changing speed on 40G interfaces, FW does not support */
1638         if (i40e_is_40G_device(hw->device_id)) {
1639                 speed =  I40E_LINK_SPEED_UNKNOWN;
1640                 abilities |= I40E_AQ_PHY_AN_ENABLED;
1641         }
1642
1643         return i40e_phy_conf_link(hw, abilities, speed);
1644 }
1645
1646 static int
1647 i40e_dev_start(struct rte_eth_dev *dev)
1648 {
1649         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1650         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1651         struct i40e_vsi *main_vsi = pf->main_vsi;
1652         int ret, i;
1653         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1654         uint32_t intr_vector = 0;
1655
1656         hw->adapter_stopped = 0;
1657
1658         if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
1659                 PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; autonegotiation disabled",
1660                              dev->data->port_id);
1661                 return -EINVAL;
1662         }
1663
1664         rte_intr_disable(intr_handle);
1665
1666         if ((rte_intr_cap_multiple(intr_handle) ||
1667              !RTE_ETH_DEV_SRIOV(dev).active) &&
1668             dev->data->dev_conf.intr_conf.rxq != 0) {
1669                 intr_vector = dev->data->nb_rx_queues;
1670                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1671                         return -1;
1672         }
1673
1674         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1675                 intr_handle->intr_vec =
1676                         rte_zmalloc("intr_vec",
1677                                     dev->data->nb_rx_queues * sizeof(int),
1678                                     0);
1679                 if (!intr_handle->intr_vec) {
1680                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1681                                      " intr_vec\n", dev->data->nb_rx_queues);
1682                         return -ENOMEM;
1683                 }
1684         }
1685
1686         /* Initialize VSI */
1687         ret = i40e_dev_rxtx_init(pf);
1688         if (ret != I40E_SUCCESS) {
1689                 PMD_DRV_LOG(ERR, "Failed to init rx/tx queues");
1690                 goto err_up;
1691         }
1692
1693         /* Map queues with MSIX interrupt */
1694         main_vsi->nb_used_qps = dev->data->nb_rx_queues -
1695                 pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1696         i40e_vsi_queues_bind_intr(main_vsi);
1697         i40e_vsi_enable_queues_intr(main_vsi);
1698
1699         /* Map VMDQ VSI queues with MSIX interrupt */
1700         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1701                 pf->vmdq[i].vsi->nb_used_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1702                 i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi);
1703                 i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi);
1704         }
1705
1706         /* enable FDIR MSIX interrupt */
1707         if (pf->fdir.fdir_vsi) {
1708                 i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi);
1709                 i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi);
1710         }
1711
1712         /* Enable all queues which have been configured */
1713         ret = i40e_dev_switch_queues(pf, TRUE);
1714         if (ret != I40E_SUCCESS) {
1715                 PMD_DRV_LOG(ERR, "Failed to enable VSI");
1716                 goto err_up;
1717         }
1718
1719         /* Enable receiving broadcast packets */
1720         ret = i40e_aq_set_vsi_broadcast(hw, main_vsi->seid, true, NULL);
1721         if (ret != I40E_SUCCESS)
1722                 PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1723
1724         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1725                 ret = i40e_aq_set_vsi_broadcast(hw, pf->vmdq[i].vsi->seid,
1726                                                 true, NULL);
1727                 if (ret != I40E_SUCCESS)
1728                         PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1729         }
1730
1731         /* Apply link configure */
1732         if (dev->data->dev_conf.link_speeds & ~(ETH_LINK_SPEED_100M |
1733                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G |
1734                                 ETH_LINK_SPEED_20G | ETH_LINK_SPEED_40G)) {
1735                 PMD_DRV_LOG(ERR, "Invalid link setting");
1736                 goto err_up;
1737         }
1738         ret = i40e_apply_link_speed(dev);
1739         if (I40E_SUCCESS != ret) {
1740                 PMD_DRV_LOG(ERR, "Fail to apply link setting");
1741                 goto err_up;
1742         }
1743
1744         if (!rte_intr_allow_others(intr_handle)) {
1745                 rte_intr_callback_unregister(intr_handle,
1746                                              i40e_dev_interrupt_handler,
1747                                              (void *)dev);
1748                 /* configure and enable device interrupt */
1749                 i40e_pf_config_irq0(hw, FALSE);
1750                 i40e_pf_enable_irq0(hw);
1751
1752                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1753                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1754                                      " no intr multiplex\n");
1755         }
1756
1757         /* enable uio intr after callback register */
1758         rte_intr_enable(intr_handle);
1759
1760         return I40E_SUCCESS;
1761
1762 err_up:
1763         i40e_dev_switch_queues(pf, FALSE);
1764         i40e_dev_clear_queues(dev);
1765
1766         return ret;
1767 }
1768
1769 static void
1770 i40e_dev_stop(struct rte_eth_dev *dev)
1771 {
1772         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1773         struct i40e_vsi *main_vsi = pf->main_vsi;
1774         struct i40e_mirror_rule *p_mirror;
1775         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1776         int i;
1777
1778         /* Disable all queues */
1779         i40e_dev_switch_queues(pf, FALSE);
1780
1781         /* un-map queues with interrupt registers */
1782         i40e_vsi_disable_queues_intr(main_vsi);
1783         i40e_vsi_queues_unbind_intr(main_vsi);
1784
1785         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1786                 i40e_vsi_disable_queues_intr(pf->vmdq[i].vsi);
1787                 i40e_vsi_queues_unbind_intr(pf->vmdq[i].vsi);
1788         }
1789
1790         if (pf->fdir.fdir_vsi) {
1791                 i40e_vsi_queues_unbind_intr(pf->fdir.fdir_vsi);
1792                 i40e_vsi_disable_queues_intr(pf->fdir.fdir_vsi);
1793         }
1794         /* Clear all queues and release memory */
1795         i40e_dev_clear_queues(dev);
1796
1797         /* Set link down */
1798         i40e_dev_set_link_down(dev);
1799
1800         /* Remove all mirror rules */
1801         while ((p_mirror = TAILQ_FIRST(&pf->mirror_list))) {
1802                 TAILQ_REMOVE(&pf->mirror_list, p_mirror, rules);
1803                 rte_free(p_mirror);
1804         }
1805         pf->nb_mirror_rule = 0;
1806
1807         if (!rte_intr_allow_others(intr_handle))
1808                 /* resume to the default handler */
1809                 rte_intr_callback_register(intr_handle,
1810                                            i40e_dev_interrupt_handler,
1811                                            (void *)dev);
1812
1813         /* Clean datapath event and queue/vec mapping */
1814         rte_intr_efd_disable(intr_handle);
1815         if (intr_handle->intr_vec) {
1816                 rte_free(intr_handle->intr_vec);
1817                 intr_handle->intr_vec = NULL;
1818         }
1819 }
1820
1821 static void
1822 i40e_dev_close(struct rte_eth_dev *dev)
1823 {
1824         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1825         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1826         uint32_t reg;
1827         int i;
1828
1829         PMD_INIT_FUNC_TRACE();
1830
1831         i40e_dev_stop(dev);
1832         hw->adapter_stopped = 1;
1833         i40e_dev_free_queues(dev);
1834
1835         /* Disable interrupt */
1836         i40e_pf_disable_irq0(hw);
1837         rte_intr_disable(&(dev->pci_dev->intr_handle));
1838
1839         /* shutdown and destroy the HMC */
1840         i40e_shutdown_lan_hmc(hw);
1841
1842         /* release all the existing VSIs and VEBs */
1843         i40e_fdir_teardown(pf);
1844         i40e_vsi_release(pf->main_vsi);
1845
1846         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1847                 i40e_vsi_release(pf->vmdq[i].vsi);
1848                 pf->vmdq[i].vsi = NULL;
1849         }
1850
1851         rte_free(pf->vmdq);
1852         pf->vmdq = NULL;
1853
1854         /* shutdown the adminq */
1855         i40e_aq_queue_shutdown(hw, true);
1856         i40e_shutdown_adminq(hw);
1857
1858         i40e_res_pool_destroy(&pf->qp_pool);
1859         i40e_res_pool_destroy(&pf->msix_pool);
1860
1861         /* force a PF reset to clean anything leftover */
1862         reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
1863         I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
1864                         (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
1865         I40E_WRITE_FLUSH(hw);
1866 }
1867
1868 static void
1869 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
1870 {
1871         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1872         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1873         struct i40e_vsi *vsi = pf->main_vsi;
1874         int status;
1875
1876         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
1877                                                      true, NULL, true);
1878         if (status != I40E_SUCCESS)
1879                 PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous");
1880
1881         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
1882                                                         TRUE, NULL);
1883         if (status != I40E_SUCCESS)
1884                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
1885
1886 }
1887
1888 static void
1889 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
1890 {
1891         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1892         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1893         struct i40e_vsi *vsi = pf->main_vsi;
1894         int status;
1895
1896         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
1897                                                      false, NULL, true);
1898         if (status != I40E_SUCCESS)
1899                 PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
1900
1901         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
1902                                                         false, NULL);
1903         if (status != I40E_SUCCESS)
1904                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1905 }
1906
1907 static void
1908 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
1909 {
1910         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1911         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1912         struct i40e_vsi *vsi = pf->main_vsi;
1913         int ret;
1914
1915         ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
1916         if (ret != I40E_SUCCESS)
1917                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
1918 }
1919
1920 static void
1921 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
1922 {
1923         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1924         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1925         struct i40e_vsi *vsi = pf->main_vsi;
1926         int ret;
1927
1928         if (dev->data->promiscuous == 1)
1929                 return; /* must remain in all_multicast mode */
1930
1931         ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
1932                                 vsi->seid, FALSE, NULL);
1933         if (ret != I40E_SUCCESS)
1934                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1935 }
1936
1937 /*
1938  * Set device link up.
1939  */
1940 static int
1941 i40e_dev_set_link_up(struct rte_eth_dev *dev)
1942 {
1943         /* re-apply link speed setting */
1944         return i40e_apply_link_speed(dev);
1945 }
1946
1947 /*
1948  * Set device link down.
1949  */
1950 static int
1951 i40e_dev_set_link_down(struct rte_eth_dev *dev)
1952 {
1953         uint8_t speed = I40E_LINK_SPEED_UNKNOWN;
1954         uint8_t abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
1955         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1956
1957         return i40e_phy_conf_link(hw, abilities, speed);
1958 }
1959
1960 int
1961 i40e_dev_link_update(struct rte_eth_dev *dev,
1962                      int wait_to_complete)
1963 {
1964 #define CHECK_INTERVAL 100  /* 100ms */
1965 #define MAX_REPEAT_TIME 10  /* 1s (10 * 100ms) in total */
1966         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1967         struct i40e_link_status link_status;
1968         struct rte_eth_link link, old;
1969         int status;
1970         unsigned rep_cnt = MAX_REPEAT_TIME;
1971
1972         memset(&link, 0, sizeof(link));
1973         memset(&old, 0, sizeof(old));
1974         memset(&link_status, 0, sizeof(link_status));
1975         rte_i40e_dev_atomic_read_link_status(dev, &old);
1976
1977         do {
1978                 /* Get link status information from hardware */
1979                 status = i40e_aq_get_link_info(hw, false, &link_status, NULL);
1980                 if (status != I40E_SUCCESS) {
1981                         link.link_speed = ETH_SPEED_NUM_100M;
1982                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1983                         PMD_DRV_LOG(ERR, "Failed to get link info");
1984                         goto out;
1985                 }
1986
1987                 link.link_status = link_status.link_info & I40E_AQ_LINK_UP;
1988                 if (!wait_to_complete)
1989                         break;
1990
1991                 rte_delay_ms(CHECK_INTERVAL);
1992         } while (!link.link_status && rep_cnt--);
1993
1994         if (!link.link_status)
1995                 goto out;
1996
1997         /* i40e uses full duplex only */
1998         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1999
2000         /* Parse the link status */
2001         switch (link_status.link_speed) {
2002         case I40E_LINK_SPEED_100MB:
2003                 link.link_speed = ETH_SPEED_NUM_100M;
2004                 break;
2005         case I40E_LINK_SPEED_1GB:
2006                 link.link_speed = ETH_SPEED_NUM_1G;
2007                 break;
2008         case I40E_LINK_SPEED_10GB:
2009                 link.link_speed = ETH_SPEED_NUM_10G;
2010                 break;
2011         case I40E_LINK_SPEED_20GB:
2012                 link.link_speed = ETH_SPEED_NUM_20G;
2013                 break;
2014         case I40E_LINK_SPEED_40GB:
2015                 link.link_speed = ETH_SPEED_NUM_40G;
2016                 break;
2017         default:
2018                 link.link_speed = ETH_SPEED_NUM_100M;
2019                 break;
2020         }
2021
2022         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
2023                         ETH_LINK_SPEED_FIXED);
2024
2025 out:
2026         rte_i40e_dev_atomic_write_link_status(dev, &link);
2027         if (link.link_status == old.link_status)
2028                 return -1;
2029
2030         return 0;
2031 }
2032
2033 /* Get all the statistics of a VSI */
2034 void
2035 i40e_update_vsi_stats(struct i40e_vsi *vsi)
2036 {
2037         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
2038         struct i40e_eth_stats *nes = &vsi->eth_stats;
2039         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2040         int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
2041
2042         i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
2043                             vsi->offset_loaded, &oes->rx_bytes,
2044                             &nes->rx_bytes);
2045         i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
2046                             vsi->offset_loaded, &oes->rx_unicast,
2047                             &nes->rx_unicast);
2048         i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
2049                             vsi->offset_loaded, &oes->rx_multicast,
2050                             &nes->rx_multicast);
2051         i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
2052                             vsi->offset_loaded, &oes->rx_broadcast,
2053                             &nes->rx_broadcast);
2054         i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
2055                             &oes->rx_discards, &nes->rx_discards);
2056         /* GLV_REPC not supported */
2057         /* GLV_RMPC not supported */
2058         i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
2059                             &oes->rx_unknown_protocol,
2060                             &nes->rx_unknown_protocol);
2061         i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
2062                             vsi->offset_loaded, &oes->tx_bytes,
2063                             &nes->tx_bytes);
2064         i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
2065                             vsi->offset_loaded, &oes->tx_unicast,
2066                             &nes->tx_unicast);
2067         i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
2068                             vsi->offset_loaded, &oes->tx_multicast,
2069                             &nes->tx_multicast);
2070         i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
2071                             vsi->offset_loaded,  &oes->tx_broadcast,
2072                             &nes->tx_broadcast);
2073         /* GLV_TDPC not supported */
2074         i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
2075                             &oes->tx_errors, &nes->tx_errors);
2076         vsi->offset_loaded = true;
2077
2078         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats start *******************",
2079                     vsi->vsi_id);
2080         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", nes->rx_bytes);
2081         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", nes->rx_unicast);
2082         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", nes->rx_multicast);
2083         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", nes->rx_broadcast);
2084         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", nes->rx_discards);
2085         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2086                     nes->rx_unknown_protocol);
2087         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", nes->tx_bytes);
2088         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", nes->tx_unicast);
2089         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", nes->tx_multicast);
2090         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", nes->tx_broadcast);
2091         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", nes->tx_discards);
2092         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", nes->tx_errors);
2093         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats end *******************",
2094                     vsi->vsi_id);
2095 }
2096
2097 static void
2098 i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw)
2099 {
2100         unsigned int i;
2101         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2102         struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
2103
2104         /* Get statistics of struct i40e_eth_stats */
2105         i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
2106                             I40E_GLPRT_GORCL(hw->port),
2107                             pf->offset_loaded, &os->eth.rx_bytes,
2108                             &ns->eth.rx_bytes);
2109         i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
2110                             I40E_GLPRT_UPRCL(hw->port),
2111                             pf->offset_loaded, &os->eth.rx_unicast,
2112                             &ns->eth.rx_unicast);
2113         i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
2114                             I40E_GLPRT_MPRCL(hw->port),
2115                             pf->offset_loaded, &os->eth.rx_multicast,
2116                             &ns->eth.rx_multicast);
2117         i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
2118                             I40E_GLPRT_BPRCL(hw->port),
2119                             pf->offset_loaded, &os->eth.rx_broadcast,
2120                             &ns->eth.rx_broadcast);
2121         /* Workaround: CRC size should not be included in byte statistics,
2122          * so subtract ETHER_CRC_LEN from the byte counter for each rx packet.
2123          */
2124         ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast +
2125                 ns->eth.rx_broadcast) * ETHER_CRC_LEN;
2126
2127         i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
2128                             pf->offset_loaded, &os->eth.rx_discards,
2129                             &ns->eth.rx_discards);
2130         /* GLPRT_REPC not supported */
2131         /* GLPRT_RMPC not supported */
2132         i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
2133                             pf->offset_loaded,
2134                             &os->eth.rx_unknown_protocol,
2135                             &ns->eth.rx_unknown_protocol);
2136         i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
2137                             I40E_GLPRT_GOTCL(hw->port),
2138                             pf->offset_loaded, &os->eth.tx_bytes,
2139                             &ns->eth.tx_bytes);
2140         i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
2141                             I40E_GLPRT_UPTCL(hw->port),
2142                             pf->offset_loaded, &os->eth.tx_unicast,
2143                             &ns->eth.tx_unicast);
2144         i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
2145                             I40E_GLPRT_MPTCL(hw->port),
2146                             pf->offset_loaded, &os->eth.tx_multicast,
2147                             &ns->eth.tx_multicast);
2148         i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
2149                             I40E_GLPRT_BPTCL(hw->port),
2150                             pf->offset_loaded, &os->eth.tx_broadcast,
2151                             &ns->eth.tx_broadcast);
2152         ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast +
2153                 ns->eth.tx_broadcast) * ETHER_CRC_LEN;
2154         /* GLPRT_TEPC not supported */
2155
2156         /* additional port specific stats */
2157         i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
2158                             pf->offset_loaded, &os->tx_dropped_link_down,
2159                             &ns->tx_dropped_link_down);
2160         i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
2161                             pf->offset_loaded, &os->crc_errors,
2162                             &ns->crc_errors);
2163         i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
2164                             pf->offset_loaded, &os->illegal_bytes,
2165                             &ns->illegal_bytes);
2166         /* GLPRT_ERRBC not supported */
2167         i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
2168                             pf->offset_loaded, &os->mac_local_faults,
2169                             &ns->mac_local_faults);
2170         i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
2171                             pf->offset_loaded, &os->mac_remote_faults,
2172                             &ns->mac_remote_faults);
2173         i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
2174                             pf->offset_loaded, &os->rx_length_errors,
2175                             &ns->rx_length_errors);
2176         i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
2177                             pf->offset_loaded, &os->link_xon_rx,
2178                             &ns->link_xon_rx);
2179         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
2180                             pf->offset_loaded, &os->link_xoff_rx,
2181                             &ns->link_xoff_rx);
2182         for (i = 0; i < 8; i++) {
2183                 i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
2184                                     pf->offset_loaded,
2185                                     &os->priority_xon_rx[i],
2186                                     &ns->priority_xon_rx[i]);
2187                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
2188                                     pf->offset_loaded,
2189                                     &os->priority_xoff_rx[i],
2190                                     &ns->priority_xoff_rx[i]);
2191         }
2192         i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
2193                             pf->offset_loaded, &os->link_xon_tx,
2194                             &ns->link_xon_tx);
2195         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
2196                             pf->offset_loaded, &os->link_xoff_tx,
2197                             &ns->link_xoff_tx);
2198         for (i = 0; i < 8; i++) {
2199                 i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
2200                                     pf->offset_loaded,
2201                                     &os->priority_xon_tx[i],
2202                                     &ns->priority_xon_tx[i]);
2203                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
2204                                     pf->offset_loaded,
2205                                     &os->priority_xoff_tx[i],
2206                                     &ns->priority_xoff_tx[i]);
2207                 i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
2208                                     pf->offset_loaded,
2209                                     &os->priority_xon_2_xoff[i],
2210                                     &ns->priority_xon_2_xoff[i]);
2211         }
2212         i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
2213                             I40E_GLPRT_PRC64L(hw->port),
2214                             pf->offset_loaded, &os->rx_size_64,
2215                             &ns->rx_size_64);
2216         i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
2217                             I40E_GLPRT_PRC127L(hw->port),
2218                             pf->offset_loaded, &os->rx_size_127,
2219                             &ns->rx_size_127);
2220         i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
2221                             I40E_GLPRT_PRC255L(hw->port),
2222                             pf->offset_loaded, &os->rx_size_255,
2223                             &ns->rx_size_255);
2224         i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
2225                             I40E_GLPRT_PRC511L(hw->port),
2226                             pf->offset_loaded, &os->rx_size_511,
2227                             &ns->rx_size_511);
2228         i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
2229                             I40E_GLPRT_PRC1023L(hw->port),
2230                             pf->offset_loaded, &os->rx_size_1023,
2231                             &ns->rx_size_1023);
2232         i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
2233                             I40E_GLPRT_PRC1522L(hw->port),
2234                             pf->offset_loaded, &os->rx_size_1522,
2235                             &ns->rx_size_1522);
2236         i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
2237                             I40E_GLPRT_PRC9522L(hw->port),
2238                             pf->offset_loaded, &os->rx_size_big,
2239                             &ns->rx_size_big);
2240         i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
2241                             pf->offset_loaded, &os->rx_undersize,
2242                             &ns->rx_undersize);
2243         i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
2244                             pf->offset_loaded, &os->rx_fragments,
2245                             &ns->rx_fragments);
2246         i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
2247                             pf->offset_loaded, &os->rx_oversize,
2248                             &ns->rx_oversize);
2249         i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
2250                             pf->offset_loaded, &os->rx_jabber,
2251                             &ns->rx_jabber);
2252         i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
2253                             I40E_GLPRT_PTC64L(hw->port),
2254                             pf->offset_loaded, &os->tx_size_64,
2255                             &ns->tx_size_64);
2256         i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
2257                             I40E_GLPRT_PTC127L(hw->port),
2258                             pf->offset_loaded, &os->tx_size_127,
2259                             &ns->tx_size_127);
2260         i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
2261                             I40E_GLPRT_PTC255L(hw->port),
2262                             pf->offset_loaded, &os->tx_size_255,
2263                             &ns->tx_size_255);
2264         i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
2265                             I40E_GLPRT_PTC511L(hw->port),
2266                             pf->offset_loaded, &os->tx_size_511,
2267                             &ns->tx_size_511);
2268         i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
2269                             I40E_GLPRT_PTC1023L(hw->port),
2270                             pf->offset_loaded, &os->tx_size_1023,
2271                             &ns->tx_size_1023);
2272         i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
2273                             I40E_GLPRT_PTC1522L(hw->port),
2274                             pf->offset_loaded, &os->tx_size_1522,
2275                             &ns->tx_size_1522);
2276         i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
2277                             I40E_GLPRT_PTC9522L(hw->port),
2278                             pf->offset_loaded, &os->tx_size_big,
2279                             &ns->tx_size_big);
2280         i40e_stat_update_32(hw, I40E_GLQF_PCNT(pf->fdir.match_counter_index),
2281                            pf->offset_loaded,
2282                            &os->fd_sb_match, &ns->fd_sb_match);
2283         /* GLPRT_MSPDC not supported */
2284         /* GLPRT_XEC not supported */
2285
2286         pf->offset_loaded = true;
2287
2288         if (pf->main_vsi)
2289                 i40e_update_vsi_stats(pf->main_vsi);
2290 }
2291
2292 /* Get all statistics of a port */
2293 static void
2294 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2295 {
2296         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2297         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2298         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2299         unsigned i;
2300
2301         /* call read registers - updates values, now write them to struct */
2302         i40e_read_stats_registers(pf, hw);
2303
2304         stats->ipackets = pf->main_vsi->eth_stats.rx_unicast +
2305                         pf->main_vsi->eth_stats.rx_multicast +
2306                         pf->main_vsi->eth_stats.rx_broadcast -
2307                         pf->main_vsi->eth_stats.rx_discards;
2308         stats->opackets = pf->main_vsi->eth_stats.tx_unicast +
2309                         pf->main_vsi->eth_stats.tx_multicast +
2310                         pf->main_vsi->eth_stats.tx_broadcast;
2311         stats->ibytes   = ns->eth.rx_bytes;
2312         stats->obytes   = ns->eth.tx_bytes;
2313         stats->oerrors  = ns->eth.tx_errors +
2314                         pf->main_vsi->eth_stats.tx_errors;
2315
2316         /* Rx Errors */
2317         stats->imissed  = ns->eth.rx_discards +
2318                         pf->main_vsi->eth_stats.rx_discards;
2319         stats->ierrors  = ns->crc_errors +
2320                         ns->rx_length_errors + ns->rx_undersize +
2321                         ns->rx_oversize + ns->rx_fragments + ns->rx_jabber;
2322
2323         PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************");
2324         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", ns->eth.rx_bytes);
2325         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", ns->eth.rx_unicast);
2326         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", ns->eth.rx_multicast);
2327         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", ns->eth.rx_broadcast);
2328         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", ns->eth.rx_discards);
2329         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2330                     ns->eth.rx_unknown_protocol);
2331         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", ns->eth.tx_bytes);
2332         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", ns->eth.tx_unicast);
2333         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", ns->eth.tx_multicast);
2334         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", ns->eth.tx_broadcast);
2335         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", ns->eth.tx_discards);
2336         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", ns->eth.tx_errors);
2337
2338         PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:     %"PRIu64"",
2339                     ns->tx_dropped_link_down);
2340         PMD_DRV_LOG(DEBUG, "crc_errors:               %"PRIu64"", ns->crc_errors);
2341         PMD_DRV_LOG(DEBUG, "illegal_bytes:            %"PRIu64"",
2342                     ns->illegal_bytes);
2343         PMD_DRV_LOG(DEBUG, "error_bytes:              %"PRIu64"", ns->error_bytes);
2344         PMD_DRV_LOG(DEBUG, "mac_local_faults:         %"PRIu64"",
2345                     ns->mac_local_faults);
2346         PMD_DRV_LOG(DEBUG, "mac_remote_faults:        %"PRIu64"",
2347                     ns->mac_remote_faults);
2348         PMD_DRV_LOG(DEBUG, "rx_length_errors:         %"PRIu64"",
2349                     ns->rx_length_errors);
2350         PMD_DRV_LOG(DEBUG, "link_xon_rx:              %"PRIu64"", ns->link_xon_rx);
2351         PMD_DRV_LOG(DEBUG, "link_xoff_rx:             %"PRIu64"", ns->link_xoff_rx);
2352         for (i = 0; i < 8; i++) {
2353                 PMD_DRV_LOG(DEBUG, "priority_xon_rx[%d]:      %"PRIu64"",
2354                                 i, ns->priority_xon_rx[i]);
2355                 PMD_DRV_LOG(DEBUG, "priority_xoff_rx[%d]:     %"PRIu64"",
2356                                 i, ns->priority_xoff_rx[i]);
2357         }
2358         PMD_DRV_LOG(DEBUG, "link_xon_tx:              %"PRIu64"", ns->link_xon_tx);
2359         PMD_DRV_LOG(DEBUG, "link_xoff_tx:             %"PRIu64"", ns->link_xoff_tx);
2360         for (i = 0; i < 8; i++) {
2361                 PMD_DRV_LOG(DEBUG, "priority_xon_tx[%d]:      %"PRIu64"",
2362                                 i, ns->priority_xon_tx[i]);
2363                 PMD_DRV_LOG(DEBUG, "priority_xoff_tx[%d]:     %"PRIu64"",
2364                                 i, ns->priority_xoff_tx[i]);
2365                 PMD_DRV_LOG(DEBUG, "priority_xon_2_xoff[%d]:  %"PRIu64"",
2366                                 i, ns->priority_xon_2_xoff[i]);
2367         }
2368         PMD_DRV_LOG(DEBUG, "rx_size_64:               %"PRIu64"", ns->rx_size_64);
2369         PMD_DRV_LOG(DEBUG, "rx_size_127:              %"PRIu64"", ns->rx_size_127);
2370         PMD_DRV_LOG(DEBUG, "rx_size_255:              %"PRIu64"", ns->rx_size_255);
2371         PMD_DRV_LOG(DEBUG, "rx_size_511:              %"PRIu64"", ns->rx_size_511);
2372         PMD_DRV_LOG(DEBUG, "rx_size_1023:             %"PRIu64"", ns->rx_size_1023);
2373         PMD_DRV_LOG(DEBUG, "rx_size_1522:             %"PRIu64"", ns->rx_size_1522);
2374         PMD_DRV_LOG(DEBUG, "rx_size_big:              %"PRIu64"", ns->rx_size_big);
2375         PMD_DRV_LOG(DEBUG, "rx_undersize:             %"PRIu64"", ns->rx_undersize);
2376         PMD_DRV_LOG(DEBUG, "rx_fragments:             %"PRIu64"", ns->rx_fragments);
2377         PMD_DRV_LOG(DEBUG, "rx_oversize:              %"PRIu64"", ns->rx_oversize);
2378         PMD_DRV_LOG(DEBUG, "rx_jabber:                %"PRIu64"", ns->rx_jabber);
2379         PMD_DRV_LOG(DEBUG, "tx_size_64:               %"PRIu64"", ns->tx_size_64);
2380         PMD_DRV_LOG(DEBUG, "tx_size_127:              %"PRIu64"", ns->tx_size_127);
2381         PMD_DRV_LOG(DEBUG, "tx_size_255:              %"PRIu64"", ns->tx_size_255);
2382         PMD_DRV_LOG(DEBUG, "tx_size_511:              %"PRIu64"", ns->tx_size_511);
2383         PMD_DRV_LOG(DEBUG, "tx_size_1023:             %"PRIu64"", ns->tx_size_1023);
2384         PMD_DRV_LOG(DEBUG, "tx_size_1522:             %"PRIu64"", ns->tx_size_1522);
2385         PMD_DRV_LOG(DEBUG, "tx_size_big:              %"PRIu64"", ns->tx_size_big);
2386         PMD_DRV_LOG(DEBUG, "mac_short_packet_dropped: %"PRIu64"",
2387                         ns->mac_short_packet_dropped);
2388         PMD_DRV_LOG(DEBUG, "checksum_error:           %"PRIu64"",
2389                     ns->checksum_error);
2390         PMD_DRV_LOG(DEBUG, "fdir_match:               %"PRIu64"", ns->fd_sb_match);
2391         PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************");
2392 }
2393
2394 /* Reset the statistics */
2395 static void
2396 i40e_dev_stats_reset(struct rte_eth_dev *dev)
2397 {
2398         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2399         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2400
2401         /* Mark PF and VSI stats to update the offset, aka "reset" */
2402         pf->offset_loaded = false;
2403         if (pf->main_vsi)
2404                 pf->main_vsi->offset_loaded = false;
2405
2406         /* read the stats, reading current register values into offset */
2407         i40e_read_stats_registers(pf, hw);
2408 }
2409
2410 static uint32_t
2411 i40e_xstats_calc_num(void)
2412 {
2413         return I40E_NB_ETH_XSTATS + I40E_NB_HW_PORT_XSTATS +
2414                 (I40E_NB_RXQ_PRIO_XSTATS * 8) +
2415                 (I40E_NB_TXQ_PRIO_XSTATS * 8);
2416 }
2417
2418 static int i40e_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
2419                                      struct rte_eth_xstat_name *xstats_names,
2420                                      __rte_unused unsigned limit)
2421 {
2422         unsigned count = 0;
2423         unsigned i, prio;
2424
2425         if (xstats_names == NULL)
2426                 return i40e_xstats_calc_num();
2427
2428         /* Note: limit checked in rte_eth_xstats_names() */
2429
2430         /* Get stats from i40e_eth_stats struct */
2431         for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
2432                 snprintf(xstats_names[count].name,
2433                          sizeof(xstats_names[count].name),
2434                          "%s", rte_i40e_stats_strings[i].name);
2435                 count++;
2436         }
2437
2438         /* Get individiual stats from i40e_hw_port struct */
2439         for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
2440                 snprintf(xstats_names[count].name,
2441                         sizeof(xstats_names[count].name),
2442                          "%s", rte_i40e_hw_port_strings[i].name);
2443                 count++;
2444         }
2445
2446         for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
2447                 for (prio = 0; prio < 8; prio++) {
2448                         snprintf(xstats_names[count].name,
2449                                  sizeof(xstats_names[count].name),
2450                                  "rx_priority%u_%s", prio,
2451                                  rte_i40e_rxq_prio_strings[i].name);
2452                         count++;
2453                 }
2454         }
2455
2456         for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
2457                 for (prio = 0; prio < 8; prio++) {
2458                         snprintf(xstats_names[count].name,
2459                                  sizeof(xstats_names[count].name),
2460                                  "tx_priority%u_%s", prio,
2461                                  rte_i40e_txq_prio_strings[i].name);
2462                         count++;
2463                 }
2464         }
2465         return count;
2466 }
2467
2468 static int
2469 i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
2470                     unsigned n)
2471 {
2472         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2473         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2474         unsigned i, count, prio;
2475         struct i40e_hw_port_stats *hw_stats = &pf->stats;
2476
2477         count = i40e_xstats_calc_num();
2478         if (n < count)
2479                 return count;
2480
2481         i40e_read_stats_registers(pf, hw);
2482
2483         if (xstats == NULL)
2484                 return 0;
2485
2486         count = 0;
2487
2488         /* Get stats from i40e_eth_stats struct */
2489         for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
2490                 xstats[count].value = *(uint64_t *)(((char *)&hw_stats->eth) +
2491                         rte_i40e_stats_strings[i].offset);
2492                 count++;
2493         }
2494
2495         /* Get individiual stats from i40e_hw_port struct */
2496         for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
2497                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2498                         rte_i40e_hw_port_strings[i].offset);
2499                 count++;
2500         }
2501
2502         for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
2503                 for (prio = 0; prio < 8; prio++) {
2504                         xstats[count].value =
2505                                 *(uint64_t *)(((char *)hw_stats) +
2506                                 rte_i40e_rxq_prio_strings[i].offset +
2507                                 (sizeof(uint64_t) * prio));
2508                         count++;
2509                 }
2510         }
2511
2512         for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
2513                 for (prio = 0; prio < 8; prio++) {
2514                         xstats[count].value =
2515                                 *(uint64_t *)(((char *)hw_stats) +
2516                                 rte_i40e_txq_prio_strings[i].offset +
2517                                 (sizeof(uint64_t) * prio));
2518                         count++;
2519                 }
2520         }
2521
2522         return count;
2523 }
2524
2525 static int
2526 i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev,
2527                                  __rte_unused uint16_t queue_id,
2528                                  __rte_unused uint8_t stat_idx,
2529                                  __rte_unused uint8_t is_rx)
2530 {
2531         PMD_INIT_FUNC_TRACE();
2532
2533         return -ENOSYS;
2534 }
2535
2536 static void
2537 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2538 {
2539         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2540         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2541         struct i40e_vsi *vsi = pf->main_vsi;
2542
2543         dev_info->max_rx_queues = vsi->nb_qps;
2544         dev_info->max_tx_queues = vsi->nb_qps;
2545         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2546         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2547         dev_info->max_mac_addrs = vsi->max_macaddrs;
2548         dev_info->max_vfs = dev->pci_dev->max_vfs;
2549         dev_info->rx_offload_capa =
2550                 DEV_RX_OFFLOAD_VLAN_STRIP |
2551                 DEV_RX_OFFLOAD_QINQ_STRIP |
2552                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2553                 DEV_RX_OFFLOAD_UDP_CKSUM |
2554                 DEV_RX_OFFLOAD_TCP_CKSUM;
2555         dev_info->tx_offload_capa =
2556                 DEV_TX_OFFLOAD_VLAN_INSERT |
2557                 DEV_TX_OFFLOAD_QINQ_INSERT |
2558                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2559                 DEV_TX_OFFLOAD_UDP_CKSUM |
2560                 DEV_TX_OFFLOAD_TCP_CKSUM |
2561                 DEV_TX_OFFLOAD_SCTP_CKSUM |
2562                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
2563                 DEV_TX_OFFLOAD_TCP_TSO;
2564         dev_info->hash_key_size = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
2565                                                 sizeof(uint32_t);
2566         dev_info->reta_size = pf->hash_lut_size;
2567         dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2568
2569         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2570                 .rx_thresh = {
2571                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2572                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2573                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2574                 },
2575                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2576                 .rx_drop_en = 0,
2577         };
2578
2579         dev_info->default_txconf = (struct rte_eth_txconf) {
2580                 .tx_thresh = {
2581                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2582                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2583                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2584                 },
2585                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2586                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2587                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2588                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2589         };
2590
2591         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2592                 .nb_max = I40E_MAX_RING_DESC,
2593                 .nb_min = I40E_MIN_RING_DESC,
2594                 .nb_align = I40E_ALIGN_RING_DESC,
2595         };
2596
2597         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2598                 .nb_max = I40E_MAX_RING_DESC,
2599                 .nb_min = I40E_MIN_RING_DESC,
2600                 .nb_align = I40E_ALIGN_RING_DESC,
2601         };
2602
2603         if (pf->flags & I40E_FLAG_VMDQ) {
2604                 dev_info->max_vmdq_pools = pf->max_nb_vmdq_vsi;
2605                 dev_info->vmdq_queue_base = dev_info->max_rx_queues;
2606                 dev_info->vmdq_queue_num = pf->vmdq_nb_qps *
2607                                                 pf->max_nb_vmdq_vsi;
2608                 dev_info->vmdq_pool_base = I40E_VMDQ_POOL_BASE;
2609                 dev_info->max_rx_queues += dev_info->vmdq_queue_num;
2610                 dev_info->max_tx_queues += dev_info->vmdq_queue_num;
2611         }
2612
2613         if (i40e_is_40G_device(hw->device_id))
2614                 /* For XL710 */
2615                 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
2616         else
2617                 /* For X710 */
2618                 dev_info->speed_capa = ETH_LINK_SPEED_10G | ETH_LINK_SPEED_40G;
2619 }
2620
2621 static int
2622 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2623 {
2624         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2625         struct i40e_vsi *vsi = pf->main_vsi;
2626         PMD_INIT_FUNC_TRACE();
2627
2628         if (on)
2629                 return i40e_vsi_add_vlan(vsi, vlan_id);
2630         else
2631                 return i40e_vsi_delete_vlan(vsi, vlan_id);
2632 }
2633
2634 static int
2635 i40e_vlan_tpid_set(struct rte_eth_dev *dev,
2636                    enum rte_vlan_type vlan_type,
2637                    uint16_t tpid)
2638 {
2639         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2640         uint64_t reg_r = 0, reg_w = 0;
2641         uint16_t reg_id = 0;
2642         int ret = 0;
2643         int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend;
2644
2645         switch (vlan_type) {
2646         case ETH_VLAN_TYPE_OUTER:
2647                 if (qinq)
2648                         reg_id = 2;
2649                 else
2650                         reg_id = 3;
2651                 break;
2652         case ETH_VLAN_TYPE_INNER:
2653                 if (qinq)
2654                         reg_id = 3;
2655                 else {
2656                         ret = -EINVAL;
2657                         PMD_DRV_LOG(ERR,
2658                                 "Unsupported vlan type in single vlan.\n");
2659                         return ret;
2660                 }
2661                 break;
2662         default:
2663                 ret = -EINVAL;
2664                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d", vlan_type);
2665                 return ret;
2666         }
2667         ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2668                                           &reg_r, NULL);
2669         if (ret != I40E_SUCCESS) {
2670                 PMD_DRV_LOG(ERR, "Fail to debug read from "
2671                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2672                 ret = -EIO;
2673                 return ret;
2674         }
2675         PMD_DRV_LOG(DEBUG, "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: "
2676                     "0x%08"PRIx64"", reg_id, reg_r);
2677
2678         reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK));
2679         reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT);
2680         if (reg_r == reg_w) {
2681                 ret = 0;
2682                 PMD_DRV_LOG(DEBUG, "No need to write");
2683                 return ret;
2684         }
2685
2686         ret = i40e_aq_debug_write_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2687                                            reg_w, NULL);
2688         if (ret != I40E_SUCCESS) {
2689                 ret = -EIO;
2690                 PMD_DRV_LOG(ERR, "Fail to debug write to "
2691                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2692                 return ret;
2693         }
2694         PMD_DRV_LOG(DEBUG, "Debug write 0x%08"PRIx64" to "
2695                     "I40E_GL_SWT_L2TAGCTRL[%d]", reg_w, reg_id);
2696
2697         return ret;
2698 }
2699
2700 static void
2701 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2702 {
2703         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2704         struct i40e_vsi *vsi = pf->main_vsi;
2705
2706         if (mask & ETH_VLAN_FILTER_MASK) {
2707                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2708                         i40e_vsi_config_vlan_filter(vsi, TRUE);
2709                 else
2710                         i40e_vsi_config_vlan_filter(vsi, FALSE);
2711         }
2712
2713         if (mask & ETH_VLAN_STRIP_MASK) {
2714                 /* Enable or disable VLAN stripping */
2715                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2716                         i40e_vsi_config_vlan_stripping(vsi, TRUE);
2717                 else
2718                         i40e_vsi_config_vlan_stripping(vsi, FALSE);
2719         }
2720
2721         if (mask & ETH_VLAN_EXTEND_MASK) {
2722                 if (dev->data->dev_conf.rxmode.hw_vlan_extend) {
2723                         i40e_vsi_config_double_vlan(vsi, TRUE);
2724                         /* Set global registers with default ether type value */
2725                         i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER,
2726                                            ETHER_TYPE_VLAN);
2727                         i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER,
2728                                            ETHER_TYPE_VLAN);
2729                 }
2730                 else
2731                         i40e_vsi_config_double_vlan(vsi, FALSE);
2732         }
2733 }
2734
2735 static void
2736 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
2737                           __rte_unused uint16_t queue,
2738                           __rte_unused int on)
2739 {
2740         PMD_INIT_FUNC_TRACE();
2741 }
2742
2743 static int
2744 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
2745 {
2746         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2747         struct i40e_vsi *vsi = pf->main_vsi;
2748         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
2749         struct i40e_vsi_vlan_pvid_info info;
2750
2751         memset(&info, 0, sizeof(info));
2752         info.on = on;
2753         if (info.on)
2754                 info.config.pvid = pvid;
2755         else {
2756                 info.config.reject.tagged =
2757                                 data->dev_conf.txmode.hw_vlan_reject_tagged;
2758                 info.config.reject.untagged =
2759                                 data->dev_conf.txmode.hw_vlan_reject_untagged;
2760         }
2761
2762         return i40e_vsi_vlan_pvid_set(vsi, &info);
2763 }
2764
2765 static int
2766 i40e_dev_led_on(struct rte_eth_dev *dev)
2767 {
2768         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2769         uint32_t mode = i40e_led_get(hw);
2770
2771         if (mode == 0)
2772                 i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
2773
2774         return 0;
2775 }
2776
2777 static int
2778 i40e_dev_led_off(struct rte_eth_dev *dev)
2779 {
2780         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2781         uint32_t mode = i40e_led_get(hw);
2782
2783         if (mode != 0)
2784                 i40e_led_set(hw, 0, false);
2785
2786         return 0;
2787 }
2788
2789 static int
2790 i40e_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2791 {
2792         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2793         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2794
2795         fc_conf->pause_time = pf->fc_conf.pause_time;
2796         fc_conf->high_water =  pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS];
2797         fc_conf->low_water = pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS];
2798
2799          /* Return current mode according to actual setting*/
2800         switch (hw->fc.current_mode) {
2801         case I40E_FC_FULL:
2802                 fc_conf->mode = RTE_FC_FULL;
2803                 break;
2804         case I40E_FC_TX_PAUSE:
2805                 fc_conf->mode = RTE_FC_TX_PAUSE;
2806                 break;
2807         case I40E_FC_RX_PAUSE:
2808                 fc_conf->mode = RTE_FC_RX_PAUSE;
2809                 break;
2810         case I40E_FC_NONE:
2811         default:
2812                 fc_conf->mode = RTE_FC_NONE;
2813         };
2814
2815         return 0;
2816 }
2817
2818 static int
2819 i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2820 {
2821         uint32_t mflcn_reg, fctrl_reg, reg;
2822         uint32_t max_high_water;
2823         uint8_t i, aq_failure;
2824         int err;
2825         struct i40e_hw *hw;
2826         struct i40e_pf *pf;
2827         enum i40e_fc_mode rte_fcmode_2_i40e_fcmode[] = {
2828                 [RTE_FC_NONE] = I40E_FC_NONE,
2829                 [RTE_FC_RX_PAUSE] = I40E_FC_RX_PAUSE,
2830                 [RTE_FC_TX_PAUSE] = I40E_FC_TX_PAUSE,
2831                 [RTE_FC_FULL] = I40E_FC_FULL
2832         };
2833
2834         /* high_water field in the rte_eth_fc_conf using the kilobytes unit */
2835
2836         max_high_water = I40E_RXPBSIZE >> I40E_KILOSHIFT;
2837         if ((fc_conf->high_water > max_high_water) ||
2838                         (fc_conf->high_water < fc_conf->low_water)) {
2839                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB, "
2840                         "High_water must <= %d.", max_high_water);
2841                 return -EINVAL;
2842         }
2843
2844         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2845         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2846         hw->fc.requested_mode = rte_fcmode_2_i40e_fcmode[fc_conf->mode];
2847
2848         pf->fc_conf.pause_time = fc_conf->pause_time;
2849         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->high_water;
2850         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->low_water;
2851
2852         PMD_INIT_FUNC_TRACE();
2853
2854         /* All the link flow control related enable/disable register
2855          * configuration is handle by the F/W
2856          */
2857         err = i40e_set_fc(hw, &aq_failure, true);
2858         if (err < 0)
2859                 return -ENOSYS;
2860
2861         if (i40e_is_40G_device(hw->device_id)) {
2862                 /* Configure flow control refresh threshold,
2863                  * the value for stat_tx_pause_refresh_timer[8]
2864                  * is used for global pause operation.
2865                  */
2866
2867                 I40E_WRITE_REG(hw,
2868                                I40E_PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(8),
2869                                pf->fc_conf.pause_time);
2870
2871                 /* configure the timer value included in transmitted pause
2872                  * frame,
2873                  * the value for stat_tx_pause_quanta[8] is used for global
2874                  * pause operation
2875                  */
2876                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(8),
2877                                pf->fc_conf.pause_time);
2878
2879                 fctrl_reg = I40E_READ_REG(hw,
2880                                           I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL);
2881
2882                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2883                         fctrl_reg |= I40E_PRTMAC_FWD_CTRL;
2884                 else
2885                         fctrl_reg &= ~I40E_PRTMAC_FWD_CTRL;
2886
2887                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL,
2888                                fctrl_reg);
2889         } else {
2890                 /* Configure pause time (2 TCs per register) */
2891                 reg = (uint32_t)pf->fc_conf.pause_time * (uint32_t)0x00010001;
2892                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS / 2; i++)
2893                         I40E_WRITE_REG(hw, I40E_PRTDCB_FCTTVN(i), reg);
2894
2895                 /* Configure flow control refresh threshold value */
2896                 I40E_WRITE_REG(hw, I40E_PRTDCB_FCRTV,
2897                                pf->fc_conf.pause_time / 2);
2898
2899                 mflcn_reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
2900
2901                 /* set or clear MFLCN.PMCF & MFLCN.DPF bits
2902                  *depending on configuration
2903                  */
2904                 if (fc_conf->mac_ctrl_frame_fwd != 0) {
2905                         mflcn_reg |= I40E_PRTDCB_MFLCN_PMCF_MASK;
2906                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_DPF_MASK;
2907                 } else {
2908                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_PMCF_MASK;
2909                         mflcn_reg |= I40E_PRTDCB_MFLCN_DPF_MASK;
2910                 }
2911
2912                 I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, mflcn_reg);
2913         }
2914
2915         /* config the water marker both based on the packets and bytes */
2916         I40E_WRITE_REG(hw, I40E_GLRPB_PHW,
2917                        (pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
2918                        << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
2919         I40E_WRITE_REG(hw, I40E_GLRPB_PLW,
2920                        (pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
2921                        << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
2922         I40E_WRITE_REG(hw, I40E_GLRPB_GHW,
2923                        pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
2924                        << I40E_KILOSHIFT);
2925         I40E_WRITE_REG(hw, I40E_GLRPB_GLW,
2926                        pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
2927                        << I40E_KILOSHIFT);
2928
2929         I40E_WRITE_FLUSH(hw);
2930
2931         return 0;
2932 }
2933
2934 static int
2935 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
2936                             __rte_unused struct rte_eth_pfc_conf *pfc_conf)
2937 {
2938         PMD_INIT_FUNC_TRACE();
2939
2940         return -ENOSYS;
2941 }
2942
2943 /* Add a MAC address, and update filters */
2944 static void
2945 i40e_macaddr_add(struct rte_eth_dev *dev,
2946                  struct ether_addr *mac_addr,
2947                  __rte_unused uint32_t index,
2948                  uint32_t pool)
2949 {
2950         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2951         struct i40e_mac_filter_info mac_filter;
2952         struct i40e_vsi *vsi;
2953         int ret;
2954
2955         /* If VMDQ not enabled or configured, return */
2956         if (pool != 0 && (!(pf->flags | I40E_FLAG_VMDQ) || !pf->nb_cfg_vmdq_vsi)) {
2957                 PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u",
2958                         pf->flags | I40E_FLAG_VMDQ ? "configured" : "enabled",
2959                         pool);
2960                 return;
2961         }
2962
2963         if (pool > pf->nb_cfg_vmdq_vsi) {
2964                 PMD_DRV_LOG(ERR, "Pool number %u invalid. Max pool is %u",
2965                                 pool, pf->nb_cfg_vmdq_vsi);
2966                 return;
2967         }
2968
2969         (void)rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN);
2970         if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2971                 mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
2972         else
2973                 mac_filter.filter_type = RTE_MAC_PERFECT_MATCH;
2974
2975         if (pool == 0)
2976                 vsi = pf->main_vsi;
2977         else
2978                 vsi = pf->vmdq[pool - 1].vsi;
2979
2980         ret = i40e_vsi_add_mac(vsi, &mac_filter);
2981         if (ret != I40E_SUCCESS) {
2982                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
2983                 return;
2984         }
2985 }
2986
2987 /* Remove a MAC address, and update filters */
2988 static void
2989 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
2990 {
2991         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2992         struct i40e_vsi *vsi;
2993         struct rte_eth_dev_data *data = dev->data;
2994         struct ether_addr *macaddr;
2995         int ret;
2996         uint32_t i;
2997         uint64_t pool_sel;
2998
2999         macaddr = &(data->mac_addrs[index]);
3000
3001         pool_sel = dev->data->mac_pool_sel[index];
3002
3003         for (i = 0; i < sizeof(pool_sel) * CHAR_BIT; i++) {
3004                 if (pool_sel & (1ULL << i)) {
3005                         if (i == 0)
3006                                 vsi = pf->main_vsi;
3007                         else {
3008                                 /* No VMDQ pool enabled or configured */
3009                                 if (!(pf->flags | I40E_FLAG_VMDQ) ||
3010                                         (i > pf->nb_cfg_vmdq_vsi)) {
3011                                         PMD_DRV_LOG(ERR, "No VMDQ pool enabled"
3012                                                         "/configured");
3013                                         return;
3014                                 }
3015                                 vsi = pf->vmdq[i - 1].vsi;
3016                         }
3017                         ret = i40e_vsi_delete_mac(vsi, macaddr);
3018
3019                         if (ret) {
3020                                 PMD_DRV_LOG(ERR, "Failed to remove MACVLAN filter");
3021                                 return;
3022                         }
3023                 }
3024         }
3025 }
3026
3027 /* Set perfect match or hash match of MAC and VLAN for a VF */
3028 static int
3029 i40e_vf_mac_filter_set(struct i40e_pf *pf,
3030                  struct rte_eth_mac_filter *filter,
3031                  bool add)
3032 {
3033         struct i40e_hw *hw;
3034         struct i40e_mac_filter_info mac_filter;
3035         struct ether_addr old_mac;
3036         struct ether_addr *new_mac;
3037         struct i40e_pf_vf *vf = NULL;
3038         uint16_t vf_id;
3039         int ret;
3040
3041         if (pf == NULL) {
3042                 PMD_DRV_LOG(ERR, "Invalid PF argument.");
3043                 return -EINVAL;
3044         }
3045         hw = I40E_PF_TO_HW(pf);
3046
3047         if (filter == NULL) {
3048                 PMD_DRV_LOG(ERR, "Invalid mac filter argument.");
3049                 return -EINVAL;
3050         }
3051
3052         new_mac = &filter->mac_addr;
3053
3054         if (is_zero_ether_addr(new_mac)) {
3055                 PMD_DRV_LOG(ERR, "Invalid ethernet address.");
3056                 return -EINVAL;
3057         }
3058
3059         vf_id = filter->dst_id;
3060
3061         if (vf_id > pf->vf_num - 1 || !pf->vfs) {
3062                 PMD_DRV_LOG(ERR, "Invalid argument.");
3063                 return -EINVAL;
3064         }
3065         vf = &pf->vfs[vf_id];
3066
3067         if (add && is_same_ether_addr(new_mac, &(pf->dev_addr))) {
3068                 PMD_DRV_LOG(INFO, "Ignore adding permanent MAC address.");
3069                 return -EINVAL;
3070         }
3071
3072         if (add) {
3073                 (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
3074                 (void)rte_memcpy(hw->mac.addr, new_mac->addr_bytes,
3075                                 ETHER_ADDR_LEN);
3076                 (void)rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr,
3077                                  ETHER_ADDR_LEN);
3078
3079                 mac_filter.filter_type = filter->filter_type;
3080                 ret = i40e_vsi_add_mac(vf->vsi, &mac_filter);
3081                 if (ret != I40E_SUCCESS) {
3082                         PMD_DRV_LOG(ERR, "Failed to add MAC filter.");
3083                         return -1;
3084                 }
3085                 ether_addr_copy(new_mac, &pf->dev_addr);
3086         } else {
3087                 (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr,
3088                                 ETHER_ADDR_LEN);
3089                 ret = i40e_vsi_delete_mac(vf->vsi, &filter->mac_addr);
3090                 if (ret != I40E_SUCCESS) {
3091                         PMD_DRV_LOG(ERR, "Failed to delete MAC filter.");
3092                         return -1;
3093                 }
3094
3095                 /* Clear device address as it has been removed */
3096                 if (is_same_ether_addr(&(pf->dev_addr), new_mac))
3097                         memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
3098         }
3099
3100         return 0;
3101 }
3102
3103 /* MAC filter handle */
3104 static int
3105 i40e_mac_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
3106                 void *arg)
3107 {
3108         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3109         struct rte_eth_mac_filter *filter;
3110         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3111         int ret = I40E_NOT_SUPPORTED;
3112
3113         filter = (struct rte_eth_mac_filter *)(arg);
3114
3115         switch (filter_op) {
3116         case RTE_ETH_FILTER_NOP:
3117                 ret = I40E_SUCCESS;
3118                 break;
3119         case RTE_ETH_FILTER_ADD:
3120                 i40e_pf_disable_irq0(hw);
3121                 if (filter->is_vf)
3122                         ret = i40e_vf_mac_filter_set(pf, filter, 1);
3123                 i40e_pf_enable_irq0(hw);
3124                 break;
3125         case RTE_ETH_FILTER_DELETE:
3126                 i40e_pf_disable_irq0(hw);
3127                 if (filter->is_vf)
3128                         ret = i40e_vf_mac_filter_set(pf, filter, 0);
3129                 i40e_pf_enable_irq0(hw);
3130                 break;
3131         default:
3132                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
3133                 ret = I40E_ERR_PARAM;
3134                 break;
3135         }
3136
3137         return ret;
3138 }
3139
3140 static int
3141 i40e_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
3142 {
3143         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
3144         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3145         int ret;
3146
3147         if (!lut)
3148                 return -EINVAL;
3149
3150         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
3151                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, TRUE,
3152                                           lut, lut_size);
3153                 if (ret) {
3154                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
3155                         return ret;
3156                 }
3157         } else {
3158                 uint32_t *lut_dw = (uint32_t *)lut;
3159                 uint16_t i, lut_size_dw = lut_size / 4;
3160
3161                 for (i = 0; i < lut_size_dw; i++)
3162                         lut_dw[i] = I40E_READ_REG(hw, I40E_PFQF_HLUT(i));
3163         }
3164
3165         return 0;
3166 }
3167
3168 static int
3169 i40e_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
3170 {
3171         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
3172         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3173         int ret;
3174
3175         if (!vsi || !lut)
3176                 return -EINVAL;
3177
3178         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
3179                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, TRUE,
3180                                           lut, lut_size);
3181                 if (ret) {
3182                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
3183                         return ret;
3184                 }
3185         } else {
3186                 uint32_t *lut_dw = (uint32_t *)lut;
3187                 uint16_t i, lut_size_dw = lut_size / 4;
3188
3189                 for (i = 0; i < lut_size_dw; i++)
3190                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
3191                 I40E_WRITE_FLUSH(hw);
3192         }
3193
3194         return 0;
3195 }
3196
3197 static int
3198 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
3199                          struct rte_eth_rss_reta_entry64 *reta_conf,
3200                          uint16_t reta_size)
3201 {
3202         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3203         uint16_t i, lut_size = pf->hash_lut_size;
3204         uint16_t idx, shift;
3205         uint8_t *lut;
3206         int ret;
3207
3208         if (reta_size != lut_size ||
3209                 reta_size > ETH_RSS_RETA_SIZE_512) {
3210                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3211                         "(%d) doesn't match the number hardware can supported "
3212                                         "(%d)\n", reta_size, lut_size);
3213                 return -EINVAL;
3214         }
3215
3216         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
3217         if (!lut) {
3218                 PMD_DRV_LOG(ERR, "No memory can be allocated");
3219                 return -ENOMEM;
3220         }
3221         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
3222         if (ret)
3223                 goto out;
3224         for (i = 0; i < reta_size; i++) {
3225                 idx = i / RTE_RETA_GROUP_SIZE;
3226                 shift = i % RTE_RETA_GROUP_SIZE;
3227                 if (reta_conf[idx].mask & (1ULL << shift))
3228                         lut[i] = reta_conf[idx].reta[shift];
3229         }
3230         ret = i40e_set_rss_lut(pf->main_vsi, lut, reta_size);
3231
3232 out:
3233         rte_free(lut);
3234
3235         return ret;
3236 }
3237
3238 static int
3239 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
3240                         struct rte_eth_rss_reta_entry64 *reta_conf,
3241                         uint16_t reta_size)
3242 {
3243         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3244         uint16_t i, lut_size = pf->hash_lut_size;
3245         uint16_t idx, shift;
3246         uint8_t *lut;
3247         int ret;
3248
3249         if (reta_size != lut_size ||
3250                 reta_size > ETH_RSS_RETA_SIZE_512) {
3251                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3252                         "(%d) doesn't match the number hardware can supported "
3253                                         "(%d)\n", reta_size, lut_size);
3254                 return -EINVAL;
3255         }
3256
3257         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
3258         if (!lut) {
3259                 PMD_DRV_LOG(ERR, "No memory can be allocated");
3260                 return -ENOMEM;
3261         }
3262
3263         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
3264         if (ret)
3265                 goto out;
3266         for (i = 0; i < reta_size; i++) {
3267                 idx = i / RTE_RETA_GROUP_SIZE;
3268                 shift = i % RTE_RETA_GROUP_SIZE;
3269                 if (reta_conf[idx].mask & (1ULL << shift))
3270                         reta_conf[idx].reta[shift] = lut[i];
3271         }
3272
3273 out:
3274         rte_free(lut);
3275
3276         return ret;
3277 }
3278
3279 /**
3280  * i40e_allocate_dma_mem_d - specific memory alloc for shared code (base driver)
3281  * @hw:   pointer to the HW structure
3282  * @mem:  pointer to mem struct to fill out
3283  * @size: size of memory requested
3284  * @alignment: what to align the allocation to
3285  **/
3286 enum i40e_status_code
3287 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3288                         struct i40e_dma_mem *mem,
3289                         u64 size,
3290                         u32 alignment)
3291 {
3292         const struct rte_memzone *mz = NULL;
3293         char z_name[RTE_MEMZONE_NAMESIZE];
3294
3295         if (!mem)
3296                 return I40E_ERR_PARAM;
3297
3298         snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand());
3299         mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
3300                                          alignment, RTE_PGSIZE_2M);
3301         if (!mz)
3302                 return I40E_ERR_NO_MEMORY;
3303
3304         mem->size = size;
3305         mem->va = mz->addr;
3306         mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
3307         mem->zone = (const void *)mz;
3308         PMD_DRV_LOG(DEBUG, "memzone %s allocated with physical address: "
3309                     "%"PRIu64, mz->name, mem->pa);
3310
3311         return I40E_SUCCESS;
3312 }
3313
3314 /**
3315  * i40e_free_dma_mem_d - specific memory free for shared code (base driver)
3316  * @hw:   pointer to the HW structure
3317  * @mem:  ptr to mem struct to free
3318  **/
3319 enum i40e_status_code
3320 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3321                     struct i40e_dma_mem *mem)
3322 {
3323         if (!mem)
3324                 return I40E_ERR_PARAM;
3325
3326         PMD_DRV_LOG(DEBUG, "memzone %s to be freed with physical address: "
3327                     "%"PRIu64, ((const struct rte_memzone *)mem->zone)->name,
3328                     mem->pa);
3329         rte_memzone_free((const struct rte_memzone *)mem->zone);
3330         mem->zone = NULL;
3331         mem->va = NULL;
3332         mem->pa = (u64)0;
3333
3334         return I40E_SUCCESS;
3335 }
3336
3337 /**
3338  * i40e_allocate_virt_mem_d - specific memory alloc for shared code (base driver)
3339  * @hw:   pointer to the HW structure
3340  * @mem:  pointer to mem struct to fill out
3341  * @size: size of memory requested
3342  **/
3343 enum i40e_status_code
3344 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3345                          struct i40e_virt_mem *mem,
3346                          u32 size)
3347 {
3348         if (!mem)
3349                 return I40E_ERR_PARAM;
3350
3351         mem->size = size;
3352         mem->va = rte_zmalloc("i40e", size, 0);
3353
3354         if (mem->va)
3355                 return I40E_SUCCESS;
3356         else
3357                 return I40E_ERR_NO_MEMORY;
3358 }
3359
3360 /**
3361  * i40e_free_virt_mem_d - specific memory free for shared code (base driver)
3362  * @hw:   pointer to the HW structure
3363  * @mem:  pointer to mem struct to free
3364  **/
3365 enum i40e_status_code
3366 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3367                      struct i40e_virt_mem *mem)
3368 {
3369         if (!mem)
3370                 return I40E_ERR_PARAM;
3371
3372         rte_free(mem->va);
3373         mem->va = NULL;
3374
3375         return I40E_SUCCESS;
3376 }
3377
3378 void
3379 i40e_init_spinlock_d(struct i40e_spinlock *sp)
3380 {
3381         rte_spinlock_init(&sp->spinlock);
3382 }
3383
3384 void
3385 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
3386 {
3387         rte_spinlock_lock(&sp->spinlock);
3388 }
3389
3390 void
3391 i40e_release_spinlock_d(struct i40e_spinlock *sp)
3392 {
3393         rte_spinlock_unlock(&sp->spinlock);
3394 }
3395
3396 void
3397 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
3398 {
3399         return;
3400 }
3401
3402 /**
3403  * Get the hardware capabilities, which will be parsed
3404  * and saved into struct i40e_hw.
3405  */
3406 static int
3407 i40e_get_cap(struct i40e_hw *hw)
3408 {
3409         struct i40e_aqc_list_capabilities_element_resp *buf;
3410         uint16_t len, size = 0;
3411         int ret;
3412
3413         /* Calculate a huge enough buff for saving response data temporarily */
3414         len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
3415                                                 I40E_MAX_CAP_ELE_NUM;
3416         buf = rte_zmalloc("i40e", len, 0);
3417         if (!buf) {
3418                 PMD_DRV_LOG(ERR, "Failed to allocate memory");
3419                 return I40E_ERR_NO_MEMORY;
3420         }
3421
3422         /* Get, parse the capabilities and save it to hw */
3423         ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
3424                         i40e_aqc_opc_list_func_capabilities, NULL);
3425         if (ret != I40E_SUCCESS)
3426                 PMD_DRV_LOG(ERR, "Failed to discover capabilities");
3427
3428         /* Free the temporary buffer after being used */
3429         rte_free(buf);
3430
3431         return ret;
3432 }
3433
3434 static int
3435 i40e_pf_parameter_init(struct rte_eth_dev *dev)
3436 {
3437         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3438         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3439         uint16_t qp_count = 0, vsi_count = 0;
3440
3441         if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
3442                 PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV");
3443                 return -EINVAL;
3444         }
3445         /* Add the parameter init for LFC */
3446         pf->fc_conf.pause_time = I40E_DEFAULT_PAUSE_TIME;
3447         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_HIGH_WATER;
3448         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_LOW_WATER;
3449
3450         pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
3451         pf->max_num_vsi = hw->func_caps.num_vsis;
3452         pf->lan_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF;
3453         pf->vmdq_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
3454         pf->vf_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3455
3456         /* FDir queue/VSI allocation */
3457         pf->fdir_qp_offset = 0;
3458         if (hw->func_caps.fd) {
3459                 pf->flags |= I40E_FLAG_FDIR;
3460                 pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
3461         } else {
3462                 pf->fdir_nb_qps = 0;
3463         }
3464         qp_count += pf->fdir_nb_qps;
3465         vsi_count += 1;
3466
3467         /* LAN queue/VSI allocation */
3468         pf->lan_qp_offset = pf->fdir_qp_offset + pf->fdir_nb_qps;
3469         if (!hw->func_caps.rss) {
3470                 pf->lan_nb_qps = 1;
3471         } else {
3472                 pf->flags |= I40E_FLAG_RSS;
3473                 if (hw->mac.type == I40E_MAC_X722)
3474                         pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE;
3475                 pf->lan_nb_qps = pf->lan_nb_qp_max;
3476         }
3477         qp_count += pf->lan_nb_qps;
3478         vsi_count += 1;
3479
3480         /* VF queue/VSI allocation */
3481         pf->vf_qp_offset = pf->lan_qp_offset + pf->lan_nb_qps;
3482         if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) {
3483                 pf->flags |= I40E_FLAG_SRIOV;
3484                 pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3485                 pf->vf_num = dev->pci_dev->max_vfs;
3486                 PMD_DRV_LOG(DEBUG, "%u VF VSIs, %u queues per VF VSI, "
3487                             "in total %u queues", pf->vf_num, pf->vf_nb_qps,
3488                             pf->vf_nb_qps * pf->vf_num);
3489         } else {
3490                 pf->vf_nb_qps = 0;
3491                 pf->vf_num = 0;
3492         }
3493         qp_count += pf->vf_nb_qps * pf->vf_num;
3494         vsi_count += pf->vf_num;
3495
3496         /* VMDq queue/VSI allocation */
3497         pf->vmdq_qp_offset = pf->vf_qp_offset + pf->vf_nb_qps * pf->vf_num;
3498         pf->vmdq_nb_qps = 0;
3499         pf->max_nb_vmdq_vsi = 0;
3500         if (hw->func_caps.vmdq) {
3501                 if (qp_count < hw->func_caps.num_tx_qp &&
3502                         vsi_count < hw->func_caps.num_vsis) {
3503                         pf->max_nb_vmdq_vsi = (hw->func_caps.num_tx_qp -
3504                                 qp_count) / pf->vmdq_nb_qp_max;
3505
3506                         /* Limit the maximum number of VMDq vsi to the maximum
3507                          * ethdev can support
3508                          */
3509                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3510                                 hw->func_caps.num_vsis - vsi_count);
3511                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3512                                 ETH_64_POOLS);
3513                         if (pf->max_nb_vmdq_vsi) {
3514                                 pf->flags |= I40E_FLAG_VMDQ;
3515                                 pf->vmdq_nb_qps = pf->vmdq_nb_qp_max;
3516                                 PMD_DRV_LOG(DEBUG, "%u VMDQ VSIs, %u queues "
3517                                             "per VMDQ VSI, in total %u queues",
3518                                             pf->max_nb_vmdq_vsi,
3519                                             pf->vmdq_nb_qps, pf->vmdq_nb_qps *
3520                                             pf->max_nb_vmdq_vsi);
3521                         } else {
3522                                 PMD_DRV_LOG(INFO, "No enough queues left for "
3523                                             "VMDq");
3524                         }
3525                 } else {
3526                         PMD_DRV_LOG(INFO, "No queue or VSI left for VMDq");
3527                 }
3528         }
3529         qp_count += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi;
3530         vsi_count += pf->max_nb_vmdq_vsi;
3531
3532         if (hw->func_caps.dcb)
3533                 pf->flags |= I40E_FLAG_DCB;
3534
3535         if (qp_count > hw->func_caps.num_tx_qp) {
3536                 PMD_DRV_LOG(ERR, "Failed to allocate %u queues, which exceeds "
3537                             "the hardware maximum %u", qp_count,
3538                             hw->func_caps.num_tx_qp);
3539                 return -EINVAL;
3540         }
3541         if (vsi_count > hw->func_caps.num_vsis) {
3542                 PMD_DRV_LOG(ERR, "Failed to allocate %u VSIs, which exceeds "
3543                             "the hardware maximum %u", vsi_count,
3544                             hw->func_caps.num_vsis);
3545                 return -EINVAL;
3546         }
3547
3548         return 0;
3549 }
3550
3551 static int
3552 i40e_pf_get_switch_config(struct i40e_pf *pf)
3553 {
3554         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3555         struct i40e_aqc_get_switch_config_resp *switch_config;
3556         struct i40e_aqc_switch_config_element_resp *element;
3557         uint16_t start_seid = 0, num_reported;
3558         int ret;
3559
3560         switch_config = (struct i40e_aqc_get_switch_config_resp *)\
3561                         rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
3562         if (!switch_config) {
3563                 PMD_DRV_LOG(ERR, "Failed to allocated memory");
3564                 return -ENOMEM;
3565         }
3566
3567         /* Get the switch configurations */
3568         ret = i40e_aq_get_switch_config(hw, switch_config,
3569                 I40E_AQ_LARGE_BUF, &start_seid, NULL);
3570         if (ret != I40E_SUCCESS) {
3571                 PMD_DRV_LOG(ERR, "Failed to get switch configurations");
3572                 goto fail;
3573         }
3574         num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
3575         if (num_reported != 1) { /* The number should be 1 */
3576                 PMD_DRV_LOG(ERR, "Wrong number of switch config reported");
3577                 goto fail;
3578         }
3579
3580         /* Parse the switch configuration elements */
3581         element = &(switch_config->element[0]);
3582         if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
3583                 pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
3584                 pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
3585         } else
3586                 PMD_DRV_LOG(INFO, "Unknown element type");
3587
3588 fail:
3589         rte_free(switch_config);
3590
3591         return ret;
3592 }
3593
3594 static int
3595 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
3596                         uint32_t num)
3597 {
3598         struct pool_entry *entry;
3599
3600         if (pool == NULL || num == 0)
3601                 return -EINVAL;
3602
3603         entry = rte_zmalloc("i40e", sizeof(*entry), 0);
3604         if (entry == NULL) {
3605                 PMD_DRV_LOG(ERR, "Failed to allocate memory for resource pool");
3606                 return -ENOMEM;
3607         }
3608
3609         /* queue heap initialize */
3610         pool->num_free = num;
3611         pool->num_alloc = 0;
3612         pool->base = base;
3613         LIST_INIT(&pool->alloc_list);
3614         LIST_INIT(&pool->free_list);
3615
3616         /* Initialize element  */
3617         entry->base = 0;
3618         entry->len = num;
3619
3620         LIST_INSERT_HEAD(&pool->free_list, entry, next);
3621         return 0;
3622 }
3623
3624 static void
3625 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
3626 {
3627         struct pool_entry *entry, *next_entry;
3628
3629         if (pool == NULL)
3630                 return;
3631
3632         for (entry = LIST_FIRST(&pool->alloc_list);
3633                         entry && (next_entry = LIST_NEXT(entry, next), 1);
3634                         entry = next_entry) {
3635                 LIST_REMOVE(entry, next);
3636                 rte_free(entry);
3637         }
3638
3639         for (entry = LIST_FIRST(&pool->free_list);
3640                         entry && (next_entry = LIST_NEXT(entry, next), 1);
3641                         entry = next_entry) {
3642                 LIST_REMOVE(entry, next);
3643                 rte_free(entry);
3644         }
3645
3646         pool->num_free = 0;
3647         pool->num_alloc = 0;
3648         pool->base = 0;
3649         LIST_INIT(&pool->alloc_list);
3650         LIST_INIT(&pool->free_list);
3651 }
3652
3653 static int
3654 i40e_res_pool_free(struct i40e_res_pool_info *pool,
3655                        uint32_t base)
3656 {
3657         struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
3658         uint32_t pool_offset;
3659         int insert;
3660
3661         if (pool == NULL) {
3662                 PMD_DRV_LOG(ERR, "Invalid parameter");
3663                 return -EINVAL;
3664         }
3665
3666         pool_offset = base - pool->base;
3667         /* Lookup in alloc list */
3668         LIST_FOREACH(entry, &pool->alloc_list, next) {
3669                 if (entry->base == pool_offset) {
3670                         valid_entry = entry;
3671                         LIST_REMOVE(entry, next);
3672                         break;
3673                 }
3674         }
3675
3676         /* Not find, return */
3677         if (valid_entry == NULL) {
3678                 PMD_DRV_LOG(ERR, "Failed to find entry");
3679                 return -EINVAL;
3680         }
3681
3682         /**
3683          * Found it, move it to free list  and try to merge.
3684          * In order to make merge easier, always sort it by qbase.
3685          * Find adjacent prev and last entries.
3686          */
3687         prev = next = NULL;
3688         LIST_FOREACH(entry, &pool->free_list, next) {
3689                 if (entry->base > valid_entry->base) {
3690                         next = entry;
3691                         break;
3692                 }
3693                 prev = entry;
3694         }
3695
3696         insert = 0;
3697         /* Try to merge with next one*/
3698         if (next != NULL) {
3699                 /* Merge with next one */
3700                 if (valid_entry->base + valid_entry->len == next->base) {
3701                         next->base = valid_entry->base;
3702                         next->len += valid_entry->len;
3703                         rte_free(valid_entry);
3704                         valid_entry = next;
3705                         insert = 1;
3706                 }
3707         }
3708
3709         if (prev != NULL) {
3710                 /* Merge with previous one */
3711                 if (prev->base + prev->len == valid_entry->base) {
3712                         prev->len += valid_entry->len;
3713                         /* If it merge with next one, remove next node */
3714                         if (insert == 1) {
3715                                 LIST_REMOVE(valid_entry, next);
3716                                 rte_free(valid_entry);
3717                         } else {
3718                                 rte_free(valid_entry);
3719                                 insert = 1;
3720                         }
3721                 }
3722         }
3723
3724         /* Not find any entry to merge, insert */
3725         if (insert == 0) {
3726                 if (prev != NULL)
3727                         LIST_INSERT_AFTER(prev, valid_entry, next);
3728                 else if (next != NULL)
3729                         LIST_INSERT_BEFORE(next, valid_entry, next);
3730                 else /* It's empty list, insert to head */
3731                         LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
3732         }
3733
3734         pool->num_free += valid_entry->len;
3735         pool->num_alloc -= valid_entry->len;
3736
3737         return 0;
3738 }
3739
3740 static int
3741 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
3742                        uint16_t num)
3743 {
3744         struct pool_entry *entry, *valid_entry;
3745
3746         if (pool == NULL || num == 0) {
3747                 PMD_DRV_LOG(ERR, "Invalid parameter");
3748                 return -EINVAL;
3749         }
3750
3751         if (pool->num_free < num) {
3752                 PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u",
3753                             num, pool->num_free);
3754                 return -ENOMEM;
3755         }
3756
3757         valid_entry = NULL;
3758         /* Lookup  in free list and find most fit one */
3759         LIST_FOREACH(entry, &pool->free_list, next) {
3760                 if (entry->len >= num) {
3761                         /* Find best one */
3762                         if (entry->len == num) {
3763                                 valid_entry = entry;
3764                                 break;
3765                         }
3766                         if (valid_entry == NULL || valid_entry->len > entry->len)
3767                                 valid_entry = entry;
3768                 }
3769         }
3770
3771         /* Not find one to satisfy the request, return */
3772         if (valid_entry == NULL) {
3773                 PMD_DRV_LOG(ERR, "No valid entry found");
3774                 return -ENOMEM;
3775         }
3776         /**
3777          * The entry have equal queue number as requested,
3778          * remove it from alloc_list.
3779          */
3780         if (valid_entry->len == num) {
3781                 LIST_REMOVE(valid_entry, next);
3782         } else {
3783                 /**
3784                  * The entry have more numbers than requested,
3785                  * create a new entry for alloc_list and minus its
3786                  * queue base and number in free_list.
3787                  */
3788                 entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
3789                 if (entry == NULL) {
3790                         PMD_DRV_LOG(ERR, "Failed to allocate memory for "
3791                                     "resource pool");
3792                         return -ENOMEM;
3793                 }
3794                 entry->base = valid_entry->base;
3795                 entry->len = num;
3796                 valid_entry->base += num;
3797                 valid_entry->len -= num;
3798                 valid_entry = entry;
3799         }
3800
3801         /* Insert it into alloc list, not sorted */
3802         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
3803
3804         pool->num_free -= valid_entry->len;
3805         pool->num_alloc += valid_entry->len;
3806
3807         return valid_entry->base + pool->base;
3808 }
3809
3810 /**
3811  * bitmap_is_subset - Check whether src2 is subset of src1
3812  **/
3813 static inline int
3814 bitmap_is_subset(uint8_t src1, uint8_t src2)
3815 {
3816         return !((src1 ^ src2) & src2);
3817 }
3818
3819 static enum i40e_status_code
3820 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
3821 {
3822         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3823
3824         /* If DCB is not supported, only default TC is supported */
3825         if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
3826                 PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported");
3827                 return I40E_NOT_SUPPORTED;
3828         }
3829
3830         if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
3831                 PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to "
3832                             "HW support 0x%x", hw->func_caps.enabled_tcmap,
3833                             enabled_tcmap);
3834                 return I40E_NOT_SUPPORTED;
3835         }
3836         return I40E_SUCCESS;
3837 }
3838
3839 int
3840 i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi,
3841                                 struct i40e_vsi_vlan_pvid_info *info)
3842 {
3843         struct i40e_hw *hw;
3844         struct i40e_vsi_context ctxt;
3845         uint8_t vlan_flags = 0;
3846         int ret;
3847
3848         if (vsi == NULL || info == NULL) {
3849                 PMD_DRV_LOG(ERR, "invalid parameters");
3850                 return I40E_ERR_PARAM;
3851         }
3852
3853         if (info->on) {
3854                 vsi->info.pvid = info->config.pvid;
3855                 /**
3856                  * If insert pvid is enabled, only tagged pkts are
3857                  * allowed to be sent out.
3858                  */
3859                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
3860                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
3861         } else {
3862                 vsi->info.pvid = 0;
3863                 if (info->config.reject.tagged == 0)
3864                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
3865
3866                 if (info->config.reject.untagged == 0)
3867                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
3868         }
3869         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
3870                                         I40E_AQ_VSI_PVLAN_MODE_MASK);
3871         vsi->info.port_vlan_flags |= vlan_flags;
3872         vsi->info.valid_sections =
3873                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
3874         memset(&ctxt, 0, sizeof(ctxt));
3875         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3876         ctxt.seid = vsi->seid;
3877
3878         hw = I40E_VSI_TO_HW(vsi);
3879         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3880         if (ret != I40E_SUCCESS)
3881                 PMD_DRV_LOG(ERR, "Failed to update VSI params");
3882
3883         return ret;
3884 }
3885
3886 static int
3887 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
3888 {
3889         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3890         int i, ret;
3891         struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
3892
3893         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
3894         if (ret != I40E_SUCCESS)
3895                 return ret;
3896
3897         if (!vsi->seid) {
3898                 PMD_DRV_LOG(ERR, "seid not valid");
3899                 return -EINVAL;
3900         }
3901
3902         memset(&tc_bw_data, 0, sizeof(tc_bw_data));
3903         tc_bw_data.tc_valid_bits = enabled_tcmap;
3904         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3905                 tc_bw_data.tc_bw_credits[i] =
3906                         (enabled_tcmap & (1 << i)) ? 1 : 0;
3907
3908         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
3909         if (ret != I40E_SUCCESS) {
3910                 PMD_DRV_LOG(ERR, "Failed to configure TC BW");
3911                 return ret;
3912         }
3913
3914         (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
3915                                         sizeof(vsi->info.qs_handle));
3916         return I40E_SUCCESS;
3917 }
3918
3919 static enum i40e_status_code
3920 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
3921                                  struct i40e_aqc_vsi_properties_data *info,
3922                                  uint8_t enabled_tcmap)
3923 {
3924         enum i40e_status_code ret;
3925         int i, total_tc = 0;
3926         uint16_t qpnum_per_tc, bsf, qp_idx;
3927
3928         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
3929         if (ret != I40E_SUCCESS)
3930                 return ret;
3931
3932         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3933                 if (enabled_tcmap & (1 << i))
3934                         total_tc++;
3935         vsi->enabled_tc = enabled_tcmap;
3936
3937         /* Number of queues per enabled TC */
3938         qpnum_per_tc = i40e_align_floor(vsi->nb_qps / total_tc);
3939         qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
3940         bsf = rte_bsf32(qpnum_per_tc);
3941
3942         /* Adjust the queue number to actual queues that can be applied */
3943         if (!(vsi->type == I40E_VSI_MAIN && total_tc == 1))
3944                 vsi->nb_qps = qpnum_per_tc * total_tc;
3945
3946         /**
3947          * Configure TC and queue mapping parameters, for enabled TC,
3948          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
3949          * default queue will serve it.
3950          */
3951         qp_idx = 0;
3952         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3953                 if (vsi->enabled_tc & (1 << i)) {
3954                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
3955                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
3956                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
3957                         qp_idx += qpnum_per_tc;
3958                 } else
3959                         info->tc_mapping[i] = 0;
3960         }
3961
3962         /* Associate queue number with VSI */
3963         if (vsi->type == I40E_VSI_SRIOV) {
3964                 info->mapping_flags |=
3965                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
3966                 for (i = 0; i < vsi->nb_qps; i++)
3967                         info->queue_mapping[i] =
3968                                 rte_cpu_to_le_16(vsi->base_queue + i);
3969         } else {
3970                 info->mapping_flags |=
3971                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
3972                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
3973         }
3974         info->valid_sections |=
3975                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
3976
3977         return I40E_SUCCESS;
3978 }
3979
3980 static int
3981 i40e_veb_release(struct i40e_veb *veb)
3982 {
3983         struct i40e_vsi *vsi;
3984         struct i40e_hw *hw;
3985
3986         if (veb == NULL)
3987                 return -EINVAL;
3988
3989         if (!TAILQ_EMPTY(&veb->head)) {
3990                 PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove");
3991                 return -EACCES;
3992         }
3993         /* associate_vsi field is NULL for floating VEB */
3994         if (veb->associate_vsi != NULL) {
3995                 vsi = veb->associate_vsi;
3996                 hw = I40E_VSI_TO_HW(vsi);
3997
3998                 vsi->uplink_seid = veb->uplink_seid;
3999                 vsi->veb = NULL;
4000         } else {
4001                 veb->associate_pf->main_vsi->floating_veb = NULL;
4002                 hw = I40E_VSI_TO_HW(veb->associate_pf->main_vsi);
4003         }
4004
4005         i40e_aq_delete_element(hw, veb->seid, NULL);
4006         rte_free(veb);
4007         return I40E_SUCCESS;
4008 }
4009
4010 /* Setup a veb */
4011 static struct i40e_veb *
4012 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
4013 {
4014         struct i40e_veb *veb;
4015         int ret;
4016         struct i40e_hw *hw;
4017
4018         if (pf == NULL) {
4019                 PMD_DRV_LOG(ERR,
4020                             "veb setup failed, associated PF shouldn't null");
4021                 return NULL;
4022         }
4023         hw = I40E_PF_TO_HW(pf);
4024
4025         veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
4026         if (!veb) {
4027                 PMD_DRV_LOG(ERR, "Failed to allocate memory for veb");
4028                 goto fail;
4029         }
4030
4031         veb->associate_vsi = vsi;
4032         veb->associate_pf = pf;
4033         TAILQ_INIT(&veb->head);
4034         veb->uplink_seid = vsi ? vsi->uplink_seid : 0;
4035
4036         /* create floating veb if vsi is NULL */
4037         if (vsi != NULL) {
4038                 ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
4039                                       I40E_DEFAULT_TCMAP, false,
4040                                       &veb->seid, false, NULL);
4041         } else {
4042                 ret = i40e_aq_add_veb(hw, 0, 0, I40E_DEFAULT_TCMAP,
4043                                       true, &veb->seid, false, NULL);
4044         }
4045
4046         if (ret != I40E_SUCCESS) {
4047                 PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d",
4048                             hw->aq.asq_last_status);
4049                 goto fail;
4050         }
4051
4052         /* get statistics index */
4053         ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
4054                                 &veb->stats_idx, NULL, NULL, NULL);
4055         if (ret != I40E_SUCCESS) {
4056                 PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d",
4057                             hw->aq.asq_last_status);
4058                 goto fail;
4059         }
4060         /* Get VEB bandwidth, to be implemented */
4061         /* Now associated vsi binding to the VEB, set uplink to this VEB */
4062         if (vsi)
4063                 vsi->uplink_seid = veb->seid;
4064
4065         return veb;
4066 fail:
4067         rte_free(veb);
4068         return NULL;
4069 }
4070
4071 int
4072 i40e_vsi_release(struct i40e_vsi *vsi)
4073 {
4074         struct i40e_pf *pf;
4075         struct i40e_hw *hw;
4076         struct i40e_vsi_list *vsi_list;
4077         int ret;
4078         struct i40e_mac_filter *f;
4079         uint16_t user_param = vsi->user_param;
4080
4081         if (!vsi)
4082                 return I40E_SUCCESS;
4083
4084         pf = I40E_VSI_TO_PF(vsi);
4085         hw = I40E_VSI_TO_HW(vsi);
4086
4087         /* VSI has child to attach, release child first */
4088         if (vsi->veb) {
4089                 TAILQ_FOREACH(vsi_list, &vsi->veb->head, list) {
4090                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
4091                                 return -1;
4092                         TAILQ_REMOVE(&vsi->veb->head, vsi_list, list);
4093                 }
4094                 i40e_veb_release(vsi->veb);
4095         }
4096
4097         if (vsi->floating_veb) {
4098                 TAILQ_FOREACH(vsi_list, &vsi->floating_veb->head, list) {
4099                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
4100                                 return -1;
4101                         TAILQ_REMOVE(&vsi->floating_veb->head, vsi_list, list);
4102                 }
4103         }
4104
4105         /* Remove all macvlan filters of the VSI */
4106         i40e_vsi_remove_all_macvlan_filter(vsi);
4107         TAILQ_FOREACH(f, &vsi->mac_list, next)
4108                 rte_free(f);
4109
4110         if (vsi->type != I40E_VSI_MAIN &&
4111             ((vsi->type != I40E_VSI_SRIOV) ||
4112             !pf->floating_veb_list[user_param])) {
4113                 /* Remove vsi from parent's sibling list */
4114                 if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
4115                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
4116                         return I40E_ERR_PARAM;
4117                 }
4118                 TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
4119                                 &vsi->sib_vsi_list, list);
4120
4121                 /* Remove all switch element of the VSI */
4122                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
4123                 if (ret != I40E_SUCCESS)
4124                         PMD_DRV_LOG(ERR, "Failed to delete element");
4125         }
4126
4127         if ((vsi->type == I40E_VSI_SRIOV) &&
4128             pf->floating_veb_list[user_param]) {
4129                 /* Remove vsi from parent's sibling list */
4130                 if (vsi->parent_vsi == NULL ||
4131                     vsi->parent_vsi->floating_veb == NULL) {
4132                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
4133                         return I40E_ERR_PARAM;
4134                 }
4135                 TAILQ_REMOVE(&vsi->parent_vsi->floating_veb->head,
4136                              &vsi->sib_vsi_list, list);
4137
4138                 /* Remove all switch element of the VSI */
4139                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
4140                 if (ret != I40E_SUCCESS)
4141                         PMD_DRV_LOG(ERR, "Failed to delete element");
4142         }
4143
4144         i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
4145
4146         if (vsi->type != I40E_VSI_SRIOV)
4147                 i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
4148         rte_free(vsi);
4149
4150         return I40E_SUCCESS;
4151 }
4152
4153 static int
4154 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
4155 {
4156         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4157         struct i40e_aqc_remove_macvlan_element_data def_filter;
4158         struct i40e_mac_filter_info filter;
4159         int ret;
4160
4161         if (vsi->type != I40E_VSI_MAIN)
4162                 return I40E_ERR_CONFIG;
4163         memset(&def_filter, 0, sizeof(def_filter));
4164         (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
4165                                         ETH_ADDR_LEN);
4166         def_filter.vlan_tag = 0;
4167         def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
4168                                 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
4169         ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
4170         if (ret != I40E_SUCCESS) {
4171                 struct i40e_mac_filter *f;
4172                 struct ether_addr *mac;
4173
4174                 PMD_DRV_LOG(WARNING, "Cannot remove the default "
4175                             "macvlan filter");
4176                 /* It needs to add the permanent mac into mac list */
4177                 f = rte_zmalloc("macv_filter", sizeof(*f), 0);
4178                 if (f == NULL) {
4179                         PMD_DRV_LOG(ERR, "failed to allocate memory");
4180                         return I40E_ERR_NO_MEMORY;
4181                 }
4182                 mac = &f->mac_info.mac_addr;
4183                 (void)rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr,
4184                                 ETH_ADDR_LEN);
4185                 f->mac_info.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4186                 TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
4187                 vsi->mac_num++;
4188
4189                 return ret;
4190         }
4191         (void)rte_memcpy(&filter.mac_addr,
4192                 (struct ether_addr *)(hw->mac.perm_addr), ETH_ADDR_LEN);
4193         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4194         return i40e_vsi_add_mac(vsi, &filter);
4195 }
4196
4197 /*
4198  * i40e_vsi_get_bw_config - Query VSI BW Information
4199  * @vsi: the VSI to be queried
4200  *
4201  * Returns 0 on success, negative value on failure
4202  */
4203 static enum i40e_status_code
4204 i40e_vsi_get_bw_config(struct i40e_vsi *vsi)
4205 {
4206         struct i40e_aqc_query_vsi_bw_config_resp bw_config;
4207         struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
4208         struct i40e_hw *hw = &vsi->adapter->hw;
4209         i40e_status ret;
4210         int i;
4211         uint32_t bw_max;
4212
4213         memset(&bw_config, 0, sizeof(bw_config));
4214         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4215         if (ret != I40E_SUCCESS) {
4216                 PMD_DRV_LOG(ERR, "VSI failed to get bandwidth configuration %u",
4217                             hw->aq.asq_last_status);
4218                 return ret;
4219         }
4220
4221         memset(&ets_sla_config, 0, sizeof(ets_sla_config));
4222         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
4223                                         &ets_sla_config, NULL);
4224         if (ret != I40E_SUCCESS) {
4225                 PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith "
4226                             "configuration %u", hw->aq.asq_last_status);
4227                 return ret;
4228         }
4229
4230         /* store and print out BW info */
4231         vsi->bw_info.bw_limit = rte_le_to_cpu_16(bw_config.port_bw_limit);
4232         vsi->bw_info.bw_max = bw_config.max_bw;
4233         PMD_DRV_LOG(DEBUG, "VSI bw limit:%u", vsi->bw_info.bw_limit);
4234         PMD_DRV_LOG(DEBUG, "VSI max_bw:%u", vsi->bw_info.bw_max);
4235         bw_max = rte_le_to_cpu_16(ets_sla_config.tc_bw_max[0]) |
4236                     (rte_le_to_cpu_16(ets_sla_config.tc_bw_max[1]) <<
4237                      I40E_16_BIT_WIDTH);
4238         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4239                 vsi->bw_info.bw_ets_share_credits[i] =
4240                                 ets_sla_config.share_credits[i];
4241                 vsi->bw_info.bw_ets_credits[i] =
4242                                 rte_le_to_cpu_16(ets_sla_config.credits[i]);
4243                 /* 4 bits per TC, 4th bit is reserved */
4244                 vsi->bw_info.bw_ets_max[i] =
4245                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
4246                                   RTE_LEN2MASK(3, uint8_t));
4247                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:share credits %u", i,
4248                             vsi->bw_info.bw_ets_share_credits[i]);
4249                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:credits %u", i,
4250                             vsi->bw_info.bw_ets_credits[i]);
4251                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u: max credits: %u", i,
4252                             vsi->bw_info.bw_ets_max[i]);
4253         }
4254
4255         return I40E_SUCCESS;
4256 }
4257
4258 /* i40e_enable_pf_lb
4259  * @pf: pointer to the pf structure
4260  *
4261  * allow loopback on pf
4262  */
4263 static inline void
4264 i40e_enable_pf_lb(struct i40e_pf *pf)
4265 {
4266         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4267         struct i40e_vsi_context ctxt;
4268         int ret;
4269
4270         /* Use the FW API if FW >= v5.0 */
4271         if (hw->aq.fw_maj_ver < 5) {
4272                 PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback");
4273                 return;
4274         }
4275
4276         memset(&ctxt, 0, sizeof(ctxt));
4277         ctxt.seid = pf->main_vsi_seid;
4278         ctxt.pf_num = hw->pf_id;
4279         ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
4280         if (ret) {
4281                 PMD_DRV_LOG(ERR, "cannot get pf vsi config, err %d, aq_err %d",
4282                             ret, hw->aq.asq_last_status);
4283                 return;
4284         }
4285         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
4286         ctxt.info.valid_sections =
4287                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4288         ctxt.info.switch_id |=
4289                 rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4290
4291         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4292         if (ret)
4293                 PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d\n",
4294                             hw->aq.asq_last_status);
4295 }
4296
4297 /* Setup a VSI */
4298 struct i40e_vsi *
4299 i40e_vsi_setup(struct i40e_pf *pf,
4300                enum i40e_vsi_type type,
4301                struct i40e_vsi *uplink_vsi,
4302                uint16_t user_param)
4303 {
4304         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4305         struct i40e_vsi *vsi;
4306         struct i40e_mac_filter_info filter;
4307         int ret;
4308         struct i40e_vsi_context ctxt;
4309         struct ether_addr broadcast =
4310                 {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
4311
4312         if (type != I40E_VSI_MAIN && type != I40E_VSI_SRIOV &&
4313             uplink_vsi == NULL) {
4314                 PMD_DRV_LOG(ERR, "VSI setup failed, "
4315                             "VSI link shouldn't be NULL");
4316                 return NULL;
4317         }
4318
4319         if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
4320                 PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI "
4321                             "uplink VSI should be NULL");
4322                 return NULL;
4323         }
4324
4325         /* two situations
4326          * 1.type is not MAIN and uplink vsi is not NULL
4327          * If uplink vsi didn't setup VEB, create one first under veb field
4328          * 2.type is SRIOV and the uplink is NULL
4329          * If floating VEB is NULL, create one veb under floating veb field
4330          */
4331
4332         if (type != I40E_VSI_MAIN && uplink_vsi != NULL &&
4333             uplink_vsi->veb == NULL) {
4334                 uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
4335
4336                 if (uplink_vsi->veb == NULL) {
4337                         PMD_DRV_LOG(ERR, "VEB setup failed");
4338                         return NULL;
4339                 }
4340                 /* set ALLOWLOOPBACk on pf, when veb is created */
4341                 i40e_enable_pf_lb(pf);
4342         }
4343
4344         if (type == I40E_VSI_SRIOV && uplink_vsi == NULL &&
4345             pf->main_vsi->floating_veb == NULL) {
4346                 pf->main_vsi->floating_veb = i40e_veb_setup(pf, uplink_vsi);
4347
4348                 if (pf->main_vsi->floating_veb == NULL) {
4349                         PMD_DRV_LOG(ERR, "VEB setup failed");
4350                         return NULL;
4351                 }
4352         }
4353
4354         vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
4355         if (!vsi) {
4356                 PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi");
4357                 return NULL;
4358         }
4359         TAILQ_INIT(&vsi->mac_list);
4360         vsi->type = type;
4361         vsi->adapter = I40E_PF_TO_ADAPTER(pf);
4362         vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
4363         vsi->parent_vsi = uplink_vsi ? uplink_vsi : pf->main_vsi;
4364         vsi->user_param = user_param;
4365         /* Allocate queues */
4366         switch (vsi->type) {
4367         case I40E_VSI_MAIN  :
4368                 vsi->nb_qps = pf->lan_nb_qps;
4369                 break;
4370         case I40E_VSI_SRIOV :
4371                 vsi->nb_qps = pf->vf_nb_qps;
4372                 break;
4373         case I40E_VSI_VMDQ2:
4374                 vsi->nb_qps = pf->vmdq_nb_qps;
4375                 break;
4376         case I40E_VSI_FDIR:
4377                 vsi->nb_qps = pf->fdir_nb_qps;
4378                 break;
4379         default:
4380                 goto fail_mem;
4381         }
4382         /*
4383          * The filter status descriptor is reported in rx queue 0,
4384          * while the tx queue for fdir filter programming has no
4385          * such constraints, can be non-zero queues.
4386          * To simplify it, choose FDIR vsi use queue 0 pair.
4387          * To make sure it will use queue 0 pair, queue allocation
4388          * need be done before this function is called
4389          */
4390         if (type != I40E_VSI_FDIR) {
4391                 ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
4392                         if (ret < 0) {
4393                                 PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
4394                                                 vsi->seid, ret);
4395                                 goto fail_mem;
4396                         }
4397                         vsi->base_queue = ret;
4398         } else
4399                 vsi->base_queue = I40E_FDIR_QUEUE_ID;
4400
4401         /* VF has MSIX interrupt in VF range, don't allocate here */
4402         if (type == I40E_VSI_MAIN) {
4403                 ret = i40e_res_pool_alloc(&pf->msix_pool,
4404                                           RTE_MIN(vsi->nb_qps,
4405                                                   RTE_MAX_RXTX_INTR_VEC_ID));
4406                 if (ret < 0) {
4407                         PMD_DRV_LOG(ERR, "VSI MAIN %d get heap failed %d",
4408                                     vsi->seid, ret);
4409                         goto fail_queue_alloc;
4410                 }
4411                 vsi->msix_intr = ret;
4412                 vsi->nb_msix = RTE_MIN(vsi->nb_qps, RTE_MAX_RXTX_INTR_VEC_ID);
4413         } else if (type != I40E_VSI_SRIOV) {
4414                 ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
4415                 if (ret < 0) {
4416                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
4417                         goto fail_queue_alloc;
4418                 }
4419                 vsi->msix_intr = ret;
4420                 vsi->nb_msix = 1;
4421         } else {
4422                 vsi->msix_intr = 0;
4423                 vsi->nb_msix = 0;
4424         }
4425
4426         /* Add VSI */
4427         if (type == I40E_VSI_MAIN) {
4428                 /* For main VSI, no need to add since it's default one */
4429                 vsi->uplink_seid = pf->mac_seid;
4430                 vsi->seid = pf->main_vsi_seid;
4431                 /* Bind queues with specific MSIX interrupt */
4432                 /**
4433                  * Needs 2 interrupt at least, one for misc cause which will
4434                  * enabled from OS side, Another for queues binding the
4435                  * interrupt from device side only.
4436                  */
4437
4438                 /* Get default VSI parameters from hardware */
4439                 memset(&ctxt, 0, sizeof(ctxt));
4440                 ctxt.seid = vsi->seid;
4441                 ctxt.pf_num = hw->pf_id;
4442                 ctxt.uplink_seid = vsi->uplink_seid;
4443                 ctxt.vf_num = 0;
4444                 ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
4445                 if (ret != I40E_SUCCESS) {
4446                         PMD_DRV_LOG(ERR, "Failed to get VSI params");
4447                         goto fail_msix_alloc;
4448                 }
4449                 (void)rte_memcpy(&vsi->info, &ctxt.info,
4450                         sizeof(struct i40e_aqc_vsi_properties_data));
4451                 vsi->vsi_id = ctxt.vsi_number;
4452                 vsi->info.valid_sections = 0;
4453
4454                 /* Configure tc, enabled TC0 only */
4455                 if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
4456                         I40E_SUCCESS) {
4457                         PMD_DRV_LOG(ERR, "Failed to update TC bandwidth");
4458                         goto fail_msix_alloc;
4459                 }
4460
4461                 /* TC, queue mapping */
4462                 memset(&ctxt, 0, sizeof(ctxt));
4463                 vsi->info.valid_sections |=
4464                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4465                 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
4466                                         I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
4467                 (void)rte_memcpy(&ctxt.info, &vsi->info,
4468                         sizeof(struct i40e_aqc_vsi_properties_data));
4469                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4470                                                 I40E_DEFAULT_TCMAP);
4471                 if (ret != I40E_SUCCESS) {
4472                         PMD_DRV_LOG(ERR, "Failed to configure "
4473                                     "TC queue mapping");
4474                         goto fail_msix_alloc;
4475                 }
4476                 ctxt.seid = vsi->seid;
4477                 ctxt.pf_num = hw->pf_id;
4478                 ctxt.uplink_seid = vsi->uplink_seid;
4479                 ctxt.vf_num = 0;
4480
4481                 /* Update VSI parameters */
4482                 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4483                 if (ret != I40E_SUCCESS) {
4484                         PMD_DRV_LOG(ERR, "Failed to update VSI params");
4485                         goto fail_msix_alloc;
4486                 }
4487
4488                 (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
4489                                                 sizeof(vsi->info.tc_mapping));
4490                 (void)rte_memcpy(&vsi->info.queue_mapping,
4491                                 &ctxt.info.queue_mapping,
4492                         sizeof(vsi->info.queue_mapping));
4493                 vsi->info.mapping_flags = ctxt.info.mapping_flags;
4494                 vsi->info.valid_sections = 0;
4495
4496                 (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
4497                                 ETH_ADDR_LEN);
4498
4499                 /**
4500                  * Updating default filter settings are necessary to prevent
4501                  * reception of tagged packets.
4502                  * Some old firmware configurations load a default macvlan
4503                  * filter which accepts both tagged and untagged packets.
4504                  * The updating is to use a normal filter instead if needed.
4505                  * For NVM 4.2.2 or after, the updating is not needed anymore.
4506                  * The firmware with correct configurations load the default
4507                  * macvlan filter which is expected and cannot be removed.
4508                  */
4509                 i40e_update_default_filter_setting(vsi);
4510                 i40e_config_qinq(hw, vsi);
4511         } else if (type == I40E_VSI_SRIOV) {
4512                 memset(&ctxt, 0, sizeof(ctxt));
4513                 /**
4514                  * For other VSI, the uplink_seid equals to uplink VSI's
4515                  * uplink_seid since they share same VEB
4516                  */
4517                 if (uplink_vsi == NULL)
4518                         vsi->uplink_seid = pf->main_vsi->floating_veb->seid;
4519                 else
4520                         vsi->uplink_seid = uplink_vsi->uplink_seid;
4521                 ctxt.pf_num = hw->pf_id;
4522                 ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
4523                 ctxt.uplink_seid = vsi->uplink_seid;
4524                 ctxt.connection_type = 0x1;
4525                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
4526
4527                 /* Use the VEB configuration if FW >= v5.0 */
4528                 if (hw->aq.fw_maj_ver >= 5) {
4529                         /* Configure switch ID */
4530                         ctxt.info.valid_sections |=
4531                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4532                         ctxt.info.switch_id =
4533                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4534                 }
4535
4536                 /* Configure port/vlan */
4537                 ctxt.info.valid_sections |=
4538                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4539                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4540                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4541                                                 I40E_DEFAULT_TCMAP);
4542                 if (ret != I40E_SUCCESS) {
4543                         PMD_DRV_LOG(ERR, "Failed to configure "
4544                                     "TC queue mapping");
4545                         goto fail_msix_alloc;
4546                 }
4547                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4548                 ctxt.info.valid_sections |=
4549                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4550                 /**
4551                  * Since VSI is not created yet, only configure parameter,
4552                  * will add vsi below.
4553                  */
4554
4555                 i40e_config_qinq(hw, vsi);
4556         } else if (type == I40E_VSI_VMDQ2) {
4557                 memset(&ctxt, 0, sizeof(ctxt));
4558                 /*
4559                  * For other VSI, the uplink_seid equals to uplink VSI's
4560                  * uplink_seid since they share same VEB
4561                  */
4562                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4563                 ctxt.pf_num = hw->pf_id;
4564                 ctxt.vf_num = 0;
4565                 ctxt.uplink_seid = vsi->uplink_seid;
4566                 ctxt.connection_type = 0x1;
4567                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
4568
4569                 ctxt.info.valid_sections |=
4570                                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4571                 /* user_param carries flag to enable loop back */
4572                 if (user_param) {
4573                         ctxt.info.switch_id =
4574                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
4575                         ctxt.info.switch_id |=
4576                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4577                 }
4578
4579                 /* Configure port/vlan */
4580                 ctxt.info.valid_sections |=
4581                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4582                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4583                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4584                                                 I40E_DEFAULT_TCMAP);
4585                 if (ret != I40E_SUCCESS) {
4586                         PMD_DRV_LOG(ERR, "Failed to configure "
4587                                         "TC queue mapping");
4588                         goto fail_msix_alloc;
4589                 }
4590                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4591                 ctxt.info.valid_sections |=
4592                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4593         } else if (type == I40E_VSI_FDIR) {
4594                 memset(&ctxt, 0, sizeof(ctxt));
4595                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4596                 ctxt.pf_num = hw->pf_id;
4597                 ctxt.vf_num = 0;
4598                 ctxt.uplink_seid = vsi->uplink_seid;
4599                 ctxt.connection_type = 0x1;     /* regular data port */
4600                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
4601                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4602                                                 I40E_DEFAULT_TCMAP);
4603                 if (ret != I40E_SUCCESS) {
4604                         PMD_DRV_LOG(ERR, "Failed to configure "
4605                                         "TC queue mapping.");
4606                         goto fail_msix_alloc;
4607                 }
4608                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4609                 ctxt.info.valid_sections |=
4610                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4611         } else {
4612                 PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet");
4613                 goto fail_msix_alloc;
4614         }
4615
4616         if (vsi->type != I40E_VSI_MAIN) {
4617                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
4618                 if (ret != I40E_SUCCESS) {
4619                         PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d",
4620                                     hw->aq.asq_last_status);
4621                         goto fail_msix_alloc;
4622                 }
4623                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
4624                 vsi->info.valid_sections = 0;
4625                 vsi->seid = ctxt.seid;
4626                 vsi->vsi_id = ctxt.vsi_number;
4627                 vsi->sib_vsi_list.vsi = vsi;
4628                 if (vsi->type == I40E_VSI_SRIOV && uplink_vsi == NULL) {
4629                         TAILQ_INSERT_TAIL(&pf->main_vsi->floating_veb->head,
4630                                           &vsi->sib_vsi_list, list);
4631                 } else {
4632                         TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
4633                                           &vsi->sib_vsi_list, list);
4634                 }
4635         }
4636
4637         /* MAC/VLAN configuration */
4638         (void)rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN);
4639         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4640
4641         ret = i40e_vsi_add_mac(vsi, &filter);
4642         if (ret != I40E_SUCCESS) {
4643                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
4644                 goto fail_msix_alloc;
4645         }
4646
4647         /* Get VSI BW information */
4648         i40e_vsi_get_bw_config(vsi);
4649         return vsi;
4650 fail_msix_alloc:
4651         i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
4652 fail_queue_alloc:
4653         i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
4654 fail_mem:
4655         rte_free(vsi);
4656         return NULL;
4657 }
4658
4659 /* Configure vlan filter on or off */
4660 int
4661 i40e_vsi_config_vlan_filter(struct i40e_vsi *vsi, bool on)
4662 {
4663         int i, num;
4664         struct i40e_mac_filter *f;
4665         struct i40e_mac_filter_info *mac_filter;
4666         enum rte_mac_filter_type desired_filter;
4667         int ret = I40E_SUCCESS;
4668
4669         if (on) {
4670                 /* Filter to match MAC and VLAN */
4671                 desired_filter = RTE_MACVLAN_PERFECT_MATCH;
4672         } else {
4673                 /* Filter to match only MAC */
4674                 desired_filter = RTE_MAC_PERFECT_MATCH;
4675         }
4676
4677         num = vsi->mac_num;
4678
4679         mac_filter = rte_zmalloc("mac_filter_info_data",
4680                                  num * sizeof(*mac_filter), 0);
4681         if (mac_filter == NULL) {
4682                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4683                 return I40E_ERR_NO_MEMORY;
4684         }
4685
4686         i = 0;
4687
4688         /* Remove all existing mac */
4689         TAILQ_FOREACH(f, &vsi->mac_list, next) {
4690                 mac_filter[i] = f->mac_info;
4691                 ret = i40e_vsi_delete_mac(vsi, &f->mac_info.mac_addr);
4692                 if (ret) {
4693                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
4694                                     on ? "enable" : "disable");
4695                         goto DONE;
4696                 }
4697                 i++;
4698         }
4699
4700         /* Override with new filter */
4701         for (i = 0; i < num; i++) {
4702                 mac_filter[i].filter_type = desired_filter;
4703                 ret = i40e_vsi_add_mac(vsi, &mac_filter[i]);
4704                 if (ret) {
4705                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
4706                                     on ? "enable" : "disable");
4707                         goto DONE;
4708                 }
4709         }
4710
4711 DONE:
4712         rte_free(mac_filter);
4713         return ret;
4714 }
4715
4716 /* Configure vlan stripping on or off */
4717 int
4718 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
4719 {
4720         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4721         struct i40e_vsi_context ctxt;
4722         uint8_t vlan_flags;
4723         int ret = I40E_SUCCESS;
4724
4725         /* Check if it has been already on or off */
4726         if (vsi->info.valid_sections &
4727                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
4728                 if (on) {
4729                         if ((vsi->info.port_vlan_flags &
4730                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
4731                                 return 0; /* already on */
4732                 } else {
4733                         if ((vsi->info.port_vlan_flags &
4734                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
4735                                 I40E_AQ_VSI_PVLAN_EMOD_MASK)
4736                                 return 0; /* already off */
4737                 }
4738         }
4739
4740         if (on)
4741                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
4742         else
4743                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
4744         vsi->info.valid_sections =
4745                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4746         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
4747         vsi->info.port_vlan_flags |= vlan_flags;
4748         ctxt.seid = vsi->seid;
4749         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
4750         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4751         if (ret)
4752                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping",
4753                             on ? "enable" : "disable");
4754
4755         return ret;
4756 }
4757
4758 static int
4759 i40e_dev_init_vlan(struct rte_eth_dev *dev)
4760 {
4761         struct rte_eth_dev_data *data = dev->data;
4762         int ret;
4763         int mask = 0;
4764
4765         /* Apply vlan offload setting */
4766         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK;
4767         i40e_vlan_offload_set(dev, mask);
4768
4769         /* Apply double-vlan setting, not implemented yet */
4770
4771         /* Apply pvid setting */
4772         ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
4773                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
4774         if (ret)
4775                 PMD_DRV_LOG(INFO, "Failed to update VSI params");
4776
4777         return ret;
4778 }
4779
4780 static int
4781 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
4782 {
4783         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4784
4785         return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
4786 }
4787
4788 static int
4789 i40e_update_flow_control(struct i40e_hw *hw)
4790 {
4791 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
4792         struct i40e_link_status link_status;
4793         uint32_t rxfc = 0, txfc = 0, reg;
4794         uint8_t an_info;
4795         int ret;
4796
4797         memset(&link_status, 0, sizeof(link_status));
4798         ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
4799         if (ret != I40E_SUCCESS) {
4800                 PMD_DRV_LOG(ERR, "Failed to get link status information");
4801                 goto write_reg; /* Disable flow control */
4802         }
4803
4804         an_info = hw->phy.link_info.an_info;
4805         if (!(an_info & I40E_AQ_AN_COMPLETED)) {
4806                 PMD_DRV_LOG(INFO, "Link auto negotiation not completed");
4807                 ret = I40E_ERR_NOT_READY;
4808                 goto write_reg; /* Disable flow control */
4809         }
4810         /**
4811          * If link auto negotiation is enabled, flow control needs to
4812          * be configured according to it
4813          */
4814         switch (an_info & I40E_LINK_PAUSE_RXTX) {
4815         case I40E_LINK_PAUSE_RXTX:
4816                 rxfc = 1;
4817                 txfc = 1;
4818                 hw->fc.current_mode = I40E_FC_FULL;
4819                 break;
4820         case I40E_AQ_LINK_PAUSE_RX:
4821                 rxfc = 1;
4822                 hw->fc.current_mode = I40E_FC_RX_PAUSE;
4823                 break;
4824         case I40E_AQ_LINK_PAUSE_TX:
4825                 txfc = 1;
4826                 hw->fc.current_mode = I40E_FC_TX_PAUSE;
4827                 break;
4828         default:
4829                 hw->fc.current_mode = I40E_FC_NONE;
4830                 break;
4831         }
4832
4833 write_reg:
4834         I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
4835                 txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
4836         reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
4837         reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
4838         reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
4839         I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
4840
4841         return ret;
4842 }
4843
4844 /* PF setup */
4845 static int
4846 i40e_pf_setup(struct i40e_pf *pf)
4847 {
4848         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4849         struct i40e_filter_control_settings settings;
4850         struct i40e_vsi *vsi;
4851         int ret;
4852
4853         /* Clear all stats counters */
4854         pf->offset_loaded = FALSE;
4855         memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
4856         memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
4857
4858         ret = i40e_pf_get_switch_config(pf);
4859         if (ret != I40E_SUCCESS) {
4860                 PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
4861                 return ret;
4862         }
4863         if (pf->flags & I40E_FLAG_FDIR) {
4864                 /* make queue allocated first, let FDIR use queue pair 0*/
4865                 ret = i40e_res_pool_alloc(&pf->qp_pool, I40E_DEFAULT_QP_NUM_FDIR);
4866                 if (ret != I40E_FDIR_QUEUE_ID) {
4867                         PMD_DRV_LOG(ERR, "queue allocation fails for FDIR :"
4868                                     " ret =%d", ret);
4869                         pf->flags &= ~I40E_FLAG_FDIR;
4870                 }
4871         }
4872         /*  main VSI setup */
4873         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
4874         if (!vsi) {
4875                 PMD_DRV_LOG(ERR, "Setup of main vsi failed");
4876                 return I40E_ERR_NOT_READY;
4877         }
4878         pf->main_vsi = vsi;
4879
4880         /* Configure filter control */
4881         memset(&settings, 0, sizeof(settings));
4882         if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_128)
4883                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
4884         else if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_512)
4885                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_512;
4886         else {
4887                 PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported\n",
4888                                                 hw->func_caps.rss_table_size);
4889                 return I40E_ERR_PARAM;
4890         }
4891         PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table "
4892                         "size: %u\n", hw->func_caps.rss_table_size);
4893         pf->hash_lut_size = hw->func_caps.rss_table_size;
4894
4895         /* Enable ethtype and macvlan filters */
4896         settings.enable_ethtype = TRUE;
4897         settings.enable_macvlan = TRUE;
4898         ret = i40e_set_filter_control(hw, &settings);
4899         if (ret)
4900                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
4901                                                                 ret);
4902
4903         /* Update flow control according to the auto negotiation */
4904         i40e_update_flow_control(hw);
4905
4906         return I40E_SUCCESS;
4907 }
4908
4909 int
4910 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
4911 {
4912         uint32_t reg;
4913         uint16_t j;
4914
4915         /**
4916          * Set or clear TX Queue Disable flags,
4917          * which is required by hardware.
4918          */
4919         i40e_pre_tx_queue_cfg(hw, q_idx, on);
4920         rte_delay_us(I40E_PRE_TX_Q_CFG_WAIT_US);
4921
4922         /* Wait until the request is finished */
4923         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4924                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4925                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
4926                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
4927                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
4928                                                         & 0x1))) {
4929                         break;
4930                 }
4931         }
4932         if (on) {
4933                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4934                         return I40E_SUCCESS; /* already on, skip next steps */
4935
4936                 I40E_WRITE_REG(hw, I40E_QTX_HEAD(q_idx), 0);
4937                 reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4938         } else {
4939                 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4940                         return I40E_SUCCESS; /* already off, skip next steps */
4941                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4942         }
4943         /* Write the register */
4944         I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
4945         /* Check the result */
4946         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4947                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4948                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
4949                 if (on) {
4950                         if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
4951                                 (reg & I40E_QTX_ENA_QENA_STAT_MASK))
4952                                 break;
4953                 } else {
4954                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
4955                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4956                                 break;
4957                 }
4958         }
4959         /* Check if it is timeout */
4960         if (j >= I40E_CHK_Q_ENA_COUNT) {
4961                 PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]",
4962                             (on ? "enable" : "disable"), q_idx);
4963                 return I40E_ERR_TIMEOUT;
4964         }
4965
4966         return I40E_SUCCESS;
4967 }
4968
4969 /* Swith on or off the tx queues */
4970 static int
4971 i40e_dev_switch_tx_queues(struct i40e_pf *pf, bool on)
4972 {
4973         struct rte_eth_dev_data *dev_data = pf->dev_data;
4974         struct i40e_tx_queue *txq;
4975         struct rte_eth_dev *dev = pf->adapter->eth_dev;
4976         uint16_t i;
4977         int ret;
4978
4979         for (i = 0; i < dev_data->nb_tx_queues; i++) {
4980                 txq = dev_data->tx_queues[i];
4981                 /* Don't operate the queue if not configured or
4982                  * if starting only per queue */
4983                 if (!txq || !txq->q_set || (on && txq->tx_deferred_start))
4984                         continue;
4985                 if (on)
4986                         ret = i40e_dev_tx_queue_start(dev, i);
4987                 else
4988                         ret = i40e_dev_tx_queue_stop(dev, i);
4989                 if ( ret != I40E_SUCCESS)
4990                         return ret;
4991         }
4992
4993         return I40E_SUCCESS;
4994 }
4995
4996 int
4997 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
4998 {
4999         uint32_t reg;
5000         uint16_t j;
5001
5002         /* Wait until the request is finished */
5003         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5004                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5005                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
5006                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
5007                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
5008                         break;
5009         }
5010
5011         if (on) {
5012                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
5013                         return I40E_SUCCESS; /* Already on, skip next steps */
5014                 reg |= I40E_QRX_ENA_QENA_REQ_MASK;
5015         } else {
5016                 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
5017                         return I40E_SUCCESS; /* Already off, skip next steps */
5018                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
5019         }
5020
5021         /* Write the register */
5022         I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
5023         /* Check the result */
5024         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5025                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5026                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
5027                 if (on) {
5028                         if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
5029                                 (reg & I40E_QRX_ENA_QENA_STAT_MASK))
5030                                 break;
5031                 } else {
5032                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
5033                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
5034                                 break;
5035                 }
5036         }
5037
5038         /* Check if it is timeout */
5039         if (j >= I40E_CHK_Q_ENA_COUNT) {
5040                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
5041                             (on ? "enable" : "disable"), q_idx);
5042                 return I40E_ERR_TIMEOUT;
5043         }
5044
5045         return I40E_SUCCESS;
5046 }
5047 /* Switch on or off the rx queues */
5048 static int
5049 i40e_dev_switch_rx_queues(struct i40e_pf *pf, bool on)
5050 {
5051         struct rte_eth_dev_data *dev_data = pf->dev_data;
5052         struct i40e_rx_queue *rxq;
5053         struct rte_eth_dev *dev = pf->adapter->eth_dev;
5054         uint16_t i;
5055         int ret;
5056
5057         for (i = 0; i < dev_data->nb_rx_queues; i++) {
5058                 rxq = dev_data->rx_queues[i];
5059                 /* Don't operate the queue if not configured or
5060                  * if starting only per queue */
5061                 if (!rxq || !rxq->q_set || (on && rxq->rx_deferred_start))
5062                         continue;
5063                 if (on)
5064                         ret = i40e_dev_rx_queue_start(dev, i);
5065                 else
5066                         ret = i40e_dev_rx_queue_stop(dev, i);
5067                 if (ret != I40E_SUCCESS)
5068                         return ret;
5069         }
5070
5071         return I40E_SUCCESS;
5072 }
5073
5074 /* Switch on or off all the rx/tx queues */
5075 int
5076 i40e_dev_switch_queues(struct i40e_pf *pf, bool on)
5077 {
5078         int ret;
5079
5080         if (on) {
5081                 /* enable rx queues before enabling tx queues */
5082                 ret = i40e_dev_switch_rx_queues(pf, on);
5083                 if (ret) {
5084                         PMD_DRV_LOG(ERR, "Failed to switch rx queues");
5085                         return ret;
5086                 }
5087                 ret = i40e_dev_switch_tx_queues(pf, on);
5088         } else {
5089                 /* Stop tx queues before stopping rx queues */
5090                 ret = i40e_dev_switch_tx_queues(pf, on);
5091                 if (ret) {
5092                         PMD_DRV_LOG(ERR, "Failed to switch tx queues");
5093                         return ret;
5094                 }
5095                 ret = i40e_dev_switch_rx_queues(pf, on);
5096         }
5097
5098         return ret;
5099 }
5100
5101 /* Initialize VSI for TX */
5102 static int
5103 i40e_dev_tx_init(struct i40e_pf *pf)
5104 {
5105         struct rte_eth_dev_data *data = pf->dev_data;
5106         uint16_t i;
5107         uint32_t ret = I40E_SUCCESS;
5108         struct i40e_tx_queue *txq;
5109
5110         for (i = 0; i < data->nb_tx_queues; i++) {
5111                 txq = data->tx_queues[i];
5112                 if (!txq || !txq->q_set)
5113                         continue;
5114                 ret = i40e_tx_queue_init(txq);
5115                 if (ret != I40E_SUCCESS)
5116                         break;
5117         }
5118         if (ret == I40E_SUCCESS)
5119                 i40e_set_tx_function(container_of(pf, struct i40e_adapter, pf)
5120                                      ->eth_dev);
5121
5122         return ret;
5123 }
5124
5125 /* Initialize VSI for RX */
5126 static int
5127 i40e_dev_rx_init(struct i40e_pf *pf)
5128 {
5129         struct rte_eth_dev_data *data = pf->dev_data;
5130         int ret = I40E_SUCCESS;
5131         uint16_t i;
5132         struct i40e_rx_queue *rxq;
5133
5134         i40e_pf_config_mq_rx(pf);
5135         for (i = 0; i < data->nb_rx_queues; i++) {
5136                 rxq = data->rx_queues[i];
5137                 if (!rxq || !rxq->q_set)
5138                         continue;
5139
5140                 ret = i40e_rx_queue_init(rxq);
5141                 if (ret != I40E_SUCCESS) {
5142                         PMD_DRV_LOG(ERR, "Failed to do RX queue "
5143                                     "initialization");
5144                         break;
5145                 }
5146         }
5147         if (ret == I40E_SUCCESS)
5148                 i40e_set_rx_function(container_of(pf, struct i40e_adapter, pf)
5149                                      ->eth_dev);
5150
5151         return ret;
5152 }
5153
5154 static int
5155 i40e_dev_rxtx_init(struct i40e_pf *pf)
5156 {
5157         int err;
5158
5159         err = i40e_dev_tx_init(pf);
5160         if (err) {
5161                 PMD_DRV_LOG(ERR, "Failed to do TX initialization");
5162                 return err;
5163         }
5164         err = i40e_dev_rx_init(pf);
5165         if (err) {
5166                 PMD_DRV_LOG(ERR, "Failed to do RX initialization");
5167                 return err;
5168         }
5169
5170         return err;
5171 }
5172
5173 static int
5174 i40e_vmdq_setup(struct rte_eth_dev *dev)
5175 {
5176         struct rte_eth_conf *conf = &dev->data->dev_conf;
5177         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5178         int i, err, conf_vsis, j, loop;
5179         struct i40e_vsi *vsi;
5180         struct i40e_vmdq_info *vmdq_info;
5181         struct rte_eth_vmdq_rx_conf *vmdq_conf;
5182         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5183
5184         /*
5185          * Disable interrupt to avoid message from VF. Furthermore, it will
5186          * avoid race condition in VSI creation/destroy.
5187          */
5188         i40e_pf_disable_irq0(hw);
5189
5190         if ((pf->flags & I40E_FLAG_VMDQ) == 0) {
5191                 PMD_INIT_LOG(ERR, "FW doesn't support VMDQ");
5192                 return -ENOTSUP;
5193         }
5194
5195         conf_vsis = conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
5196         if (conf_vsis > pf->max_nb_vmdq_vsi) {
5197                 PMD_INIT_LOG(ERR, "VMDQ config: %u, max support:%u",
5198                         conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools,
5199                         pf->max_nb_vmdq_vsi);
5200                 return -ENOTSUP;
5201         }
5202
5203         if (pf->vmdq != NULL) {
5204                 PMD_INIT_LOG(INFO, "VMDQ already configured");
5205                 return 0;
5206         }
5207
5208         pf->vmdq = rte_zmalloc("vmdq_info_struct",
5209                                 sizeof(*vmdq_info) * conf_vsis, 0);
5210
5211         if (pf->vmdq == NULL) {
5212                 PMD_INIT_LOG(ERR, "Failed to allocate memory");
5213                 return -ENOMEM;
5214         }
5215
5216         vmdq_conf = &conf->rx_adv_conf.vmdq_rx_conf;
5217
5218         /* Create VMDQ VSI */
5219         for (i = 0; i < conf_vsis; i++) {
5220                 vsi = i40e_vsi_setup(pf, I40E_VSI_VMDQ2, pf->main_vsi,
5221                                 vmdq_conf->enable_loop_back);
5222                 if (vsi == NULL) {
5223                         PMD_INIT_LOG(ERR, "Failed to create VMDQ VSI");
5224                         err = -1;
5225                         goto err_vsi_setup;
5226                 }
5227                 vmdq_info = &pf->vmdq[i];
5228                 vmdq_info->pf = pf;
5229                 vmdq_info->vsi = vsi;
5230         }
5231         pf->nb_cfg_vmdq_vsi = conf_vsis;
5232
5233         /* Configure Vlan */
5234         loop = sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT;
5235         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
5236                 for (j = 0; j < loop && j < pf->nb_cfg_vmdq_vsi; j++) {
5237                         if (vmdq_conf->pool_map[i].pools & (1UL << j)) {
5238                                 PMD_INIT_LOG(INFO, "Add vlan %u to vmdq pool %u",
5239                                         vmdq_conf->pool_map[i].vlan_id, j);
5240
5241                                 err = i40e_vsi_add_vlan(pf->vmdq[j].vsi,
5242                                                 vmdq_conf->pool_map[i].vlan_id);
5243                                 if (err) {
5244                                         PMD_INIT_LOG(ERR, "Failed to add vlan");
5245                                         err = -1;
5246                                         goto err_vsi_setup;
5247                                 }
5248                         }
5249                 }
5250         }
5251
5252         i40e_pf_enable_irq0(hw);
5253
5254         return 0;
5255
5256 err_vsi_setup:
5257         for (i = 0; i < conf_vsis; i++)
5258                 if (pf->vmdq[i].vsi == NULL)
5259                         break;
5260                 else
5261                         i40e_vsi_release(pf->vmdq[i].vsi);
5262
5263         rte_free(pf->vmdq);
5264         pf->vmdq = NULL;
5265         i40e_pf_enable_irq0(hw);
5266         return err;
5267 }
5268
5269 static void
5270 i40e_stat_update_32(struct i40e_hw *hw,
5271                    uint32_t reg,
5272                    bool offset_loaded,
5273                    uint64_t *offset,
5274                    uint64_t *stat)
5275 {
5276         uint64_t new_data;
5277
5278         new_data = (uint64_t)I40E_READ_REG(hw, reg);
5279         if (!offset_loaded)
5280                 *offset = new_data;
5281
5282         if (new_data >= *offset)
5283                 *stat = (uint64_t)(new_data - *offset);
5284         else
5285                 *stat = (uint64_t)((new_data +
5286                         ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset);
5287 }
5288
5289 static void
5290 i40e_stat_update_48(struct i40e_hw *hw,
5291                    uint32_t hireg,
5292                    uint32_t loreg,
5293                    bool offset_loaded,
5294                    uint64_t *offset,
5295                    uint64_t *stat)
5296 {
5297         uint64_t new_data;
5298
5299         new_data = (uint64_t)I40E_READ_REG(hw, loreg);
5300         new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
5301                         I40E_16_BIT_MASK)) << I40E_32_BIT_WIDTH;
5302
5303         if (!offset_loaded)
5304                 *offset = new_data;
5305
5306         if (new_data >= *offset)
5307                 *stat = new_data - *offset;
5308         else
5309                 *stat = (uint64_t)((new_data +
5310                         ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset);
5311
5312         *stat &= I40E_48_BIT_MASK;
5313 }
5314
5315 /* Disable IRQ0 */
5316 void
5317 i40e_pf_disable_irq0(struct i40e_hw *hw)
5318 {
5319         /* Disable all interrupt types */
5320         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
5321         I40E_WRITE_FLUSH(hw);
5322 }
5323
5324 /* Enable IRQ0 */
5325 void
5326 i40e_pf_enable_irq0(struct i40e_hw *hw)
5327 {
5328         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
5329                 I40E_PFINT_DYN_CTL0_INTENA_MASK |
5330                 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
5331                 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
5332         I40E_WRITE_FLUSH(hw);
5333 }
5334
5335 static void
5336 i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue)
5337 {
5338         /* read pending request and disable first */
5339         i40e_pf_disable_irq0(hw);
5340         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, I40E_PFINT_ICR0_ENA_MASK);
5341         I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
5342                 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
5343
5344         if (no_queue)
5345                 /* Link no queues with irq0 */
5346                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
5347                                I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
5348 }
5349
5350 static void
5351 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
5352 {
5353         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5354         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5355         int i;
5356         uint16_t abs_vf_id;
5357         uint32_t index, offset, val;
5358
5359         if (!pf->vfs)
5360                 return;
5361         /**
5362          * Try to find which VF trigger a reset, use absolute VF id to access
5363          * since the reg is global register.
5364          */
5365         for (i = 0; i < pf->vf_num; i++) {
5366                 abs_vf_id = hw->func_caps.vf_base_id + i;
5367                 index = abs_vf_id / I40E_UINT32_BIT_SIZE;
5368                 offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
5369                 val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
5370                 /* VFR event occured */
5371                 if (val & (0x1 << offset)) {
5372                         int ret;
5373
5374                         /* Clear the event first */
5375                         I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
5376                                                         (0x1 << offset));
5377                         PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id);
5378                         /**
5379                          * Only notify a VF reset event occured,
5380                          * don't trigger another SW reset
5381                          */
5382                         ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
5383                         if (ret != I40E_SUCCESS)
5384                                 PMD_DRV_LOG(ERR, "Failed to do VF reset");
5385                 }
5386         }
5387 }
5388
5389 static void
5390 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
5391 {
5392         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5393         struct i40e_arq_event_info info;
5394         uint16_t pending, opcode;
5395         int ret;
5396
5397         info.buf_len = I40E_AQ_BUF_SZ;
5398         info.msg_buf = rte_zmalloc("msg_buffer", info.buf_len, 0);
5399         if (!info.msg_buf) {
5400                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
5401                 return;
5402         }
5403
5404         pending = 1;
5405         while (pending) {
5406                 ret = i40e_clean_arq_element(hw, &info, &pending);
5407
5408                 if (ret != I40E_SUCCESS) {
5409                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
5410                                     "aq_err: %u", hw->aq.asq_last_status);
5411                         break;
5412                 }
5413                 opcode = rte_le_to_cpu_16(info.desc.opcode);
5414
5415                 switch (opcode) {
5416                 case i40e_aqc_opc_send_msg_to_pf:
5417                         /* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
5418                         i40e_pf_host_handle_vf_msg(dev,
5419                                         rte_le_to_cpu_16(info.desc.retval),
5420                                         rte_le_to_cpu_32(info.desc.cookie_high),
5421                                         rte_le_to_cpu_32(info.desc.cookie_low),
5422                                         info.msg_buf,
5423                                         info.msg_len);
5424                         break;
5425                 default:
5426                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
5427                                     opcode);
5428                         break;
5429                 }
5430         }
5431         rte_free(info.msg_buf);
5432 }
5433
5434 /*
5435  * Interrupt handler is registered as the alarm callback for handling LSC
5436  * interrupt in a definite of time, in order to wait the NIC into a stable
5437  * state. Currently it waits 1 sec in i40e for the link up interrupt, and
5438  * no need for link down interrupt.
5439  */
5440 static void
5441 i40e_dev_interrupt_delayed_handler(void *param)
5442 {
5443         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
5444         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5445         uint32_t icr0;
5446
5447         /* read interrupt causes again */
5448         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
5449
5450 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
5451         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
5452                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error\n");
5453         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
5454                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected\n");
5455         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
5456                 PMD_DRV_LOG(INFO, "ICR0: global reset requested\n");
5457         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
5458                 PMD_DRV_LOG(INFO, "ICR0: PCI exception\n activated\n");
5459         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
5460                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control "
5461                                                                 "state\n");
5462         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
5463                 PMD_DRV_LOG(ERR, "ICR0: HMC error\n");
5464         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
5465                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error\n");
5466 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
5467
5468         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
5469                 PMD_DRV_LOG(INFO, "INT:VF reset detected\n");
5470                 i40e_dev_handle_vfr_event(dev);
5471         }
5472         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
5473                 PMD_DRV_LOG(INFO, "INT:ADMINQ event\n");
5474                 i40e_dev_handle_aq_msg(dev);
5475         }
5476
5477         /* handle the link up interrupt in an alarm callback */
5478         i40e_dev_link_update(dev, 0);
5479         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
5480
5481         i40e_pf_enable_irq0(hw);
5482         rte_intr_enable(&(dev->pci_dev->intr_handle));
5483 }
5484
5485 /**
5486  * Interrupt handler triggered by NIC  for handling
5487  * specific interrupt.
5488  *
5489  * @param handle
5490  *  Pointer to interrupt handle.
5491  * @param param
5492  *  The address of parameter (struct rte_eth_dev *) regsitered before.
5493  *
5494  * @return
5495  *  void
5496  */
5497 static void
5498 i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
5499                            void *param)
5500 {
5501         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
5502         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5503         uint32_t icr0;
5504
5505         /* Disable interrupt */
5506         i40e_pf_disable_irq0(hw);
5507
5508         /* read out interrupt causes */
5509         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
5510
5511         /* No interrupt event indicated */
5512         if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK)) {
5513                 PMD_DRV_LOG(INFO, "No interrupt event");
5514                 goto done;
5515         }
5516 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
5517         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
5518                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
5519         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
5520                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
5521         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
5522                 PMD_DRV_LOG(INFO, "ICR0: global reset requested");
5523         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
5524                 PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
5525         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
5526                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
5527         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
5528                 PMD_DRV_LOG(ERR, "ICR0: HMC error");
5529         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
5530                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
5531 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
5532
5533         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
5534                 PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
5535                 i40e_dev_handle_vfr_event(dev);
5536         }
5537         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
5538                 PMD_DRV_LOG(INFO, "ICR0: adminq event");
5539                 i40e_dev_handle_aq_msg(dev);
5540         }
5541
5542         /* Link Status Change interrupt */
5543         if (icr0 & I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK) {
5544 #define I40E_US_PER_SECOND 1000000
5545                 struct rte_eth_link link;
5546
5547                 PMD_DRV_LOG(INFO, "ICR0: link status changed\n");
5548                 memset(&link, 0, sizeof(link));
5549                 rte_i40e_dev_atomic_read_link_status(dev, &link);
5550                 i40e_dev_link_update(dev, 0);
5551
5552                 /*
5553                  * For link up interrupt, it needs to wait 1 second to let the
5554                  * hardware be a stable state. Otherwise several consecutive
5555                  * interrupts can be observed.
5556                  * For link down interrupt, no need to wait.
5557                  */
5558                 if (!link.link_status && rte_eal_alarm_set(I40E_US_PER_SECOND,
5559                         i40e_dev_interrupt_delayed_handler, (void *)dev) >= 0)
5560                         return;
5561                 else
5562                         _rte_eth_dev_callback_process(dev,
5563                                 RTE_ETH_EVENT_INTR_LSC);
5564         }
5565
5566 done:
5567         /* Enable interrupt */
5568         i40e_pf_enable_irq0(hw);
5569         rte_intr_enable(&(dev->pci_dev->intr_handle));
5570 }
5571
5572 static int
5573 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
5574                          struct i40e_macvlan_filter *filter,
5575                          int total)
5576 {
5577         int ele_num, ele_buff_size;
5578         int num, actual_num, i;
5579         uint16_t flags;
5580         int ret = I40E_SUCCESS;
5581         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5582         struct i40e_aqc_add_macvlan_element_data *req_list;
5583
5584         if (filter == NULL  || total == 0)
5585                 return I40E_ERR_PARAM;
5586         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5587         ele_buff_size = hw->aq.asq_buf_size;
5588
5589         req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
5590         if (req_list == NULL) {
5591                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5592                 return I40E_ERR_NO_MEMORY;
5593         }
5594
5595         num = 0;
5596         do {
5597                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5598                 memset(req_list, 0, ele_buff_size);
5599
5600                 for (i = 0; i < actual_num; i++) {
5601                         (void)rte_memcpy(req_list[i].mac_addr,
5602                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5603                         req_list[i].vlan_tag =
5604                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5605
5606                         switch (filter[num + i].filter_type) {
5607                         case RTE_MAC_PERFECT_MATCH:
5608                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH |
5609                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5610                                 break;
5611                         case RTE_MACVLAN_PERFECT_MATCH:
5612                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
5613                                 break;
5614                         case RTE_MAC_HASH_MATCH:
5615                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH |
5616                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5617                                 break;
5618                         case RTE_MACVLAN_HASH_MATCH:
5619                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH;
5620                                 break;
5621                         default:
5622                                 PMD_DRV_LOG(ERR, "Invalid MAC match type\n");
5623                                 ret = I40E_ERR_PARAM;
5624                                 goto DONE;
5625                         }
5626
5627                         req_list[i].queue_number = 0;
5628
5629                         req_list[i].flags = rte_cpu_to_le_16(flags);
5630                 }
5631
5632                 ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
5633                                                 actual_num, NULL);
5634                 if (ret != I40E_SUCCESS) {
5635                         PMD_DRV_LOG(ERR, "Failed to add macvlan filter");
5636                         goto DONE;
5637                 }
5638                 num += actual_num;
5639         } while (num < total);
5640
5641 DONE:
5642         rte_free(req_list);
5643         return ret;
5644 }
5645
5646 static int
5647 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
5648                             struct i40e_macvlan_filter *filter,
5649                             int total)
5650 {
5651         int ele_num, ele_buff_size;
5652         int num, actual_num, i;
5653         uint16_t flags;
5654         int ret = I40E_SUCCESS;
5655         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5656         struct i40e_aqc_remove_macvlan_element_data *req_list;
5657
5658         if (filter == NULL  || total == 0)
5659                 return I40E_ERR_PARAM;
5660
5661         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5662         ele_buff_size = hw->aq.asq_buf_size;
5663
5664         req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
5665         if (req_list == NULL) {
5666                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5667                 return I40E_ERR_NO_MEMORY;
5668         }
5669
5670         num = 0;
5671         do {
5672                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5673                 memset(req_list, 0, ele_buff_size);
5674
5675                 for (i = 0; i < actual_num; i++) {
5676                         (void)rte_memcpy(req_list[i].mac_addr,
5677                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5678                         req_list[i].vlan_tag =
5679                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5680
5681                         switch (filter[num + i].filter_type) {
5682                         case RTE_MAC_PERFECT_MATCH:
5683                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
5684                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5685                                 break;
5686                         case RTE_MACVLAN_PERFECT_MATCH:
5687                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
5688                                 break;
5689                         case RTE_MAC_HASH_MATCH:
5690                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH |
5691                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5692                                 break;
5693                         case RTE_MACVLAN_HASH_MATCH:
5694                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH;
5695                                 break;
5696                         default:
5697                                 PMD_DRV_LOG(ERR, "Invalid MAC filter type\n");
5698                                 ret = I40E_ERR_PARAM;
5699                                 goto DONE;
5700                         }
5701                         req_list[i].flags = rte_cpu_to_le_16(flags);
5702                 }
5703
5704                 ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
5705                                                 actual_num, NULL);
5706                 if (ret != I40E_SUCCESS) {
5707                         PMD_DRV_LOG(ERR, "Failed to remove macvlan filter");
5708                         goto DONE;
5709                 }
5710                 num += actual_num;
5711         } while (num < total);
5712
5713 DONE:
5714         rte_free(req_list);
5715         return ret;
5716 }
5717
5718 /* Find out specific MAC filter */
5719 static struct i40e_mac_filter *
5720 i40e_find_mac_filter(struct i40e_vsi *vsi,
5721                          struct ether_addr *macaddr)
5722 {
5723         struct i40e_mac_filter *f;
5724
5725         TAILQ_FOREACH(f, &vsi->mac_list, next) {
5726                 if (is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
5727                         return f;
5728         }
5729
5730         return NULL;
5731 }
5732
5733 static bool
5734 i40e_find_vlan_filter(struct i40e_vsi *vsi,
5735                          uint16_t vlan_id)
5736 {
5737         uint32_t vid_idx, vid_bit;
5738
5739         if (vlan_id > ETH_VLAN_ID_MAX)
5740                 return 0;
5741
5742         vid_idx = I40E_VFTA_IDX(vlan_id);
5743         vid_bit = I40E_VFTA_BIT(vlan_id);
5744
5745         if (vsi->vfta[vid_idx] & vid_bit)
5746                 return 1;
5747         else
5748                 return 0;
5749 }
5750
5751 static void
5752 i40e_set_vlan_filter(struct i40e_vsi *vsi,
5753                          uint16_t vlan_id, bool on)
5754 {
5755         uint32_t vid_idx, vid_bit;
5756
5757         if (vlan_id > ETH_VLAN_ID_MAX)
5758                 return;
5759
5760         vid_idx = I40E_VFTA_IDX(vlan_id);
5761         vid_bit = I40E_VFTA_BIT(vlan_id);
5762
5763         if (on)
5764                 vsi->vfta[vid_idx] |= vid_bit;
5765         else
5766                 vsi->vfta[vid_idx] &= ~vid_bit;
5767 }
5768
5769 /**
5770  * Find all vlan options for specific mac addr,
5771  * return with actual vlan found.
5772  */
5773 static inline int
5774 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
5775                            struct i40e_macvlan_filter *mv_f,
5776                            int num, struct ether_addr *addr)
5777 {
5778         int i;
5779         uint32_t j, k;
5780
5781         /**
5782          * Not to use i40e_find_vlan_filter to decrease the loop time,
5783          * although the code looks complex.
5784           */
5785         if (num < vsi->vlan_num)
5786                 return I40E_ERR_PARAM;
5787
5788         i = 0;
5789         for (j = 0; j < I40E_VFTA_SIZE; j++) {
5790                 if (vsi->vfta[j]) {
5791                         for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
5792                                 if (vsi->vfta[j] & (1 << k)) {
5793                                         if (i > num - 1) {
5794                                                 PMD_DRV_LOG(ERR, "vlan number "
5795                                                             "not match");
5796                                                 return I40E_ERR_PARAM;
5797                                         }
5798                                         (void)rte_memcpy(&mv_f[i].macaddr,
5799                                                         addr, ETH_ADDR_LEN);
5800                                         mv_f[i].vlan_id =
5801                                                 j * I40E_UINT32_BIT_SIZE + k;
5802                                         i++;
5803                                 }
5804                         }
5805                 }
5806         }
5807         return I40E_SUCCESS;
5808 }
5809
5810 static inline int
5811 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
5812                            struct i40e_macvlan_filter *mv_f,
5813                            int num,
5814                            uint16_t vlan)
5815 {
5816         int i = 0;
5817         struct i40e_mac_filter *f;
5818
5819         if (num < vsi->mac_num)
5820                 return I40E_ERR_PARAM;
5821
5822         TAILQ_FOREACH(f, &vsi->mac_list, next) {
5823                 if (i > num - 1) {
5824                         PMD_DRV_LOG(ERR, "buffer number not match");
5825                         return I40E_ERR_PARAM;
5826                 }
5827                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
5828                                 ETH_ADDR_LEN);
5829                 mv_f[i].vlan_id = vlan;
5830                 mv_f[i].filter_type = f->mac_info.filter_type;
5831                 i++;
5832         }
5833
5834         return I40E_SUCCESS;
5835 }
5836
5837 static int
5838 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
5839 {
5840         int i, num;
5841         struct i40e_mac_filter *f;
5842         struct i40e_macvlan_filter *mv_f;
5843         int ret = I40E_SUCCESS;
5844
5845         if (vsi == NULL || vsi->mac_num == 0)
5846                 return I40E_ERR_PARAM;
5847
5848         /* Case that no vlan is set */
5849         if (vsi->vlan_num == 0)
5850                 num = vsi->mac_num;
5851         else
5852                 num = vsi->mac_num * vsi->vlan_num;
5853
5854         mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
5855         if (mv_f == NULL) {
5856                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5857                 return I40E_ERR_NO_MEMORY;
5858         }
5859
5860         i = 0;
5861         if (vsi->vlan_num == 0) {
5862                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
5863                         (void)rte_memcpy(&mv_f[i].macaddr,
5864                                 &f->mac_info.mac_addr, ETH_ADDR_LEN);
5865                         mv_f[i].vlan_id = 0;
5866                         i++;
5867                 }
5868         } else {
5869                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
5870                         ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
5871                                         vsi->vlan_num, &f->mac_info.mac_addr);
5872                         if (ret != I40E_SUCCESS)
5873                                 goto DONE;
5874                         i += vsi->vlan_num;
5875                 }
5876         }
5877
5878         ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
5879 DONE:
5880         rte_free(mv_f);
5881
5882         return ret;
5883 }
5884
5885 int
5886 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
5887 {
5888         struct i40e_macvlan_filter *mv_f;
5889         int mac_num;
5890         int ret = I40E_SUCCESS;
5891
5892         if (!vsi || vlan > ETHER_MAX_VLAN_ID)
5893                 return I40E_ERR_PARAM;
5894
5895         /* If it's already set, just return */
5896         if (i40e_find_vlan_filter(vsi,vlan))
5897                 return I40E_SUCCESS;
5898
5899         mac_num = vsi->mac_num;
5900
5901         if (mac_num == 0) {
5902                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
5903                 return I40E_ERR_PARAM;
5904         }
5905
5906         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
5907
5908         if (mv_f == NULL) {
5909                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5910                 return I40E_ERR_NO_MEMORY;
5911         }
5912
5913         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
5914
5915         if (ret != I40E_SUCCESS)
5916                 goto DONE;
5917
5918         ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
5919
5920         if (ret != I40E_SUCCESS)
5921                 goto DONE;
5922
5923         i40e_set_vlan_filter(vsi, vlan, 1);
5924
5925         vsi->vlan_num++;
5926         ret = I40E_SUCCESS;
5927 DONE:
5928         rte_free(mv_f);
5929         return ret;
5930 }
5931
5932 int
5933 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
5934 {
5935         struct i40e_macvlan_filter *mv_f;
5936         int mac_num;
5937         int ret = I40E_SUCCESS;
5938
5939         /**
5940          * Vlan 0 is the generic filter for untagged packets
5941          * and can't be removed.
5942          */
5943         if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
5944                 return I40E_ERR_PARAM;
5945
5946         /* If can't find it, just return */
5947         if (!i40e_find_vlan_filter(vsi, vlan))
5948                 return I40E_ERR_PARAM;
5949
5950         mac_num = vsi->mac_num;
5951
5952         if (mac_num == 0) {
5953                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
5954                 return I40E_ERR_PARAM;
5955         }
5956
5957         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
5958
5959         if (mv_f == NULL) {
5960                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5961                 return I40E_ERR_NO_MEMORY;
5962         }
5963
5964         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
5965
5966         if (ret != I40E_SUCCESS)
5967                 goto DONE;
5968
5969         ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
5970
5971         if (ret != I40E_SUCCESS)
5972                 goto DONE;
5973
5974         /* This is last vlan to remove, replace all mac filter with vlan 0 */
5975         if (vsi->vlan_num == 1) {
5976                 ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
5977                 if (ret != I40E_SUCCESS)
5978                         goto DONE;
5979
5980                 ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
5981                 if (ret != I40E_SUCCESS)
5982                         goto DONE;
5983         }
5984
5985         i40e_set_vlan_filter(vsi, vlan, 0);
5986
5987         vsi->vlan_num--;
5988         ret = I40E_SUCCESS;
5989 DONE:
5990         rte_free(mv_f);
5991         return ret;
5992 }
5993
5994 int
5995 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter)
5996 {
5997         struct i40e_mac_filter *f;
5998         struct i40e_macvlan_filter *mv_f;
5999         int i, vlan_num = 0;
6000         int ret = I40E_SUCCESS;
6001
6002         /* If it's add and we've config it, return */
6003         f = i40e_find_mac_filter(vsi, &mac_filter->mac_addr);
6004         if (f != NULL)
6005                 return I40E_SUCCESS;
6006         if ((mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH) ||
6007                 (mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH)) {
6008
6009                 /**
6010                  * If vlan_num is 0, that's the first time to add mac,
6011                  * set mask for vlan_id 0.
6012                  */
6013                 if (vsi->vlan_num == 0) {
6014                         i40e_set_vlan_filter(vsi, 0, 1);
6015                         vsi->vlan_num = 1;
6016                 }
6017                 vlan_num = vsi->vlan_num;
6018         } else if ((mac_filter->filter_type == RTE_MAC_PERFECT_MATCH) ||
6019                         (mac_filter->filter_type == RTE_MAC_HASH_MATCH))
6020                 vlan_num = 1;
6021
6022         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
6023         if (mv_f == NULL) {
6024                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6025                 return I40E_ERR_NO_MEMORY;
6026         }
6027
6028         for (i = 0; i < vlan_num; i++) {
6029                 mv_f[i].filter_type = mac_filter->filter_type;
6030                 (void)rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr,
6031                                 ETH_ADDR_LEN);
6032         }
6033
6034         if (mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6035                 mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH) {
6036                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num,
6037                                         &mac_filter->mac_addr);
6038                 if (ret != I40E_SUCCESS)
6039                         goto DONE;
6040         }
6041
6042         ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
6043         if (ret != I40E_SUCCESS)
6044                 goto DONE;
6045
6046         /* Add the mac addr into mac list */
6047         f = rte_zmalloc("macv_filter", sizeof(*f), 0);
6048         if (f == NULL) {
6049                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6050                 ret = I40E_ERR_NO_MEMORY;
6051                 goto DONE;
6052         }
6053         (void)rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr,
6054                         ETH_ADDR_LEN);
6055         f->mac_info.filter_type = mac_filter->filter_type;
6056         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
6057         vsi->mac_num++;
6058
6059         ret = I40E_SUCCESS;
6060 DONE:
6061         rte_free(mv_f);
6062
6063         return ret;
6064 }
6065
6066 int
6067 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
6068 {
6069         struct i40e_mac_filter *f;
6070         struct i40e_macvlan_filter *mv_f;
6071         int i, vlan_num;
6072         enum rte_mac_filter_type filter_type;
6073         int ret = I40E_SUCCESS;
6074
6075         /* Can't find it, return an error */
6076         f = i40e_find_mac_filter(vsi, addr);
6077         if (f == NULL)
6078                 return I40E_ERR_PARAM;
6079
6080         vlan_num = vsi->vlan_num;
6081         filter_type = f->mac_info.filter_type;
6082         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6083                 filter_type == RTE_MACVLAN_HASH_MATCH) {
6084                 if (vlan_num == 0) {
6085                         PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0\n");
6086                         return I40E_ERR_PARAM;
6087                 }
6088         } else if (filter_type == RTE_MAC_PERFECT_MATCH ||
6089                         filter_type == RTE_MAC_HASH_MATCH)
6090                 vlan_num = 1;
6091
6092         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
6093         if (mv_f == NULL) {
6094                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6095                 return I40E_ERR_NO_MEMORY;
6096         }
6097
6098         for (i = 0; i < vlan_num; i++) {
6099                 mv_f[i].filter_type = filter_type;
6100                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
6101                                 ETH_ADDR_LEN);
6102         }
6103         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6104                         filter_type == RTE_MACVLAN_HASH_MATCH) {
6105                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
6106                 if (ret != I40E_SUCCESS)
6107                         goto DONE;
6108         }
6109
6110         ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
6111         if (ret != I40E_SUCCESS)
6112                 goto DONE;
6113
6114         /* Remove the mac addr into mac list */
6115         TAILQ_REMOVE(&vsi->mac_list, f, next);
6116         rte_free(f);
6117         vsi->mac_num--;
6118
6119         ret = I40E_SUCCESS;
6120 DONE:
6121         rte_free(mv_f);
6122         return ret;
6123 }
6124
6125 /* Configure hash enable flags for RSS */
6126 uint64_t
6127 i40e_config_hena(uint64_t flags)
6128 {
6129         uint64_t hena = 0;
6130
6131         if (!flags)
6132                 return hena;
6133
6134         if (flags & ETH_RSS_FRAG_IPV4)
6135                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4;
6136         if (flags & ETH_RSS_NONFRAG_IPV4_TCP)
6137                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
6138         if (flags & ETH_RSS_NONFRAG_IPV4_UDP)
6139                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
6140         if (flags & ETH_RSS_NONFRAG_IPV4_SCTP)
6141                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP;
6142         if (flags & ETH_RSS_NONFRAG_IPV4_OTHER)
6143                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
6144         if (flags & ETH_RSS_FRAG_IPV6)
6145                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6;
6146         if (flags & ETH_RSS_NONFRAG_IPV6_TCP)
6147                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
6148         if (flags & ETH_RSS_NONFRAG_IPV6_UDP)
6149                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
6150         if (flags & ETH_RSS_NONFRAG_IPV6_SCTP)
6151                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP;
6152         if (flags & ETH_RSS_NONFRAG_IPV6_OTHER)
6153                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
6154         if (flags & ETH_RSS_L2_PAYLOAD)
6155                 hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD;
6156
6157         return hena;
6158 }
6159
6160 /* Parse the hash enable flags */
6161 uint64_t
6162 i40e_parse_hena(uint64_t flags)
6163 {
6164         uint64_t rss_hf = 0;
6165
6166         if (!flags)
6167                 return rss_hf;
6168         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4))
6169                 rss_hf |= ETH_RSS_FRAG_IPV4;
6170         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP))
6171                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
6172         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP))
6173                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
6174         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP))
6175                 rss_hf |= ETH_RSS_NONFRAG_IPV4_SCTP;
6176         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER))
6177                 rss_hf |= ETH_RSS_NONFRAG_IPV4_OTHER;
6178         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6))
6179                 rss_hf |= ETH_RSS_FRAG_IPV6;
6180         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP))
6181                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
6182         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP))
6183                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
6184         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP))
6185                 rss_hf |= ETH_RSS_NONFRAG_IPV6_SCTP;
6186         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER))
6187                 rss_hf |= ETH_RSS_NONFRAG_IPV6_OTHER;
6188         if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
6189                 rss_hf |= ETH_RSS_L2_PAYLOAD;
6190
6191         return rss_hf;
6192 }
6193
6194 /* Disable RSS */
6195 static void
6196 i40e_pf_disable_rss(struct i40e_pf *pf)
6197 {
6198         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6199         uint64_t hena;
6200
6201         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6202         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6203         hena &= ~I40E_RSS_HENA_ALL;
6204         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
6205         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
6206         I40E_WRITE_FLUSH(hw);
6207 }
6208
6209 static int
6210 i40e_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
6211 {
6212         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
6213         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6214         int ret = 0;
6215
6216         if (!key || key_len == 0) {
6217                 PMD_DRV_LOG(DEBUG, "No key to be configured");
6218                 return 0;
6219         } else if (key_len != (I40E_PFQF_HKEY_MAX_INDEX + 1) *
6220                 sizeof(uint32_t)) {
6221                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
6222                 return -EINVAL;
6223         }
6224
6225         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
6226                 struct i40e_aqc_get_set_rss_key_data *key_dw =
6227                         (struct i40e_aqc_get_set_rss_key_data *)key;
6228
6229                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
6230                 if (ret)
6231                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
6232                                      "via AQ");
6233         } else {
6234                 uint32_t *hash_key = (uint32_t *)key;
6235                 uint16_t i;
6236
6237                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6238                         i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), hash_key[i]);
6239                 I40E_WRITE_FLUSH(hw);
6240         }
6241
6242         return ret;
6243 }
6244
6245 static int
6246 i40e_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
6247 {
6248         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
6249         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6250         int ret;
6251
6252         if (!key || !key_len)
6253                 return -EINVAL;
6254
6255         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
6256                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
6257                         (struct i40e_aqc_get_set_rss_key_data *)key);
6258                 if (ret) {
6259                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
6260                         return ret;
6261                 }
6262         } else {
6263                 uint32_t *key_dw = (uint32_t *)key;
6264                 uint16_t i;
6265
6266                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6267                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
6268         }
6269         *key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
6270
6271         return 0;
6272 }
6273
6274 static int
6275 i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf)
6276 {
6277         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6278         uint64_t rss_hf;
6279         uint64_t hena;
6280         int ret;
6281
6282         ret = i40e_set_rss_key(pf->main_vsi, rss_conf->rss_key,
6283                                rss_conf->rss_key_len);
6284         if (ret)
6285                 return ret;
6286
6287         rss_hf = rss_conf->rss_hf;
6288         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6289         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6290         hena &= ~I40E_RSS_HENA_ALL;
6291         hena |= i40e_config_hena(rss_hf);
6292         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
6293         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
6294         I40E_WRITE_FLUSH(hw);
6295
6296         return 0;
6297 }
6298
6299 static int
6300 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
6301                          struct rte_eth_rss_conf *rss_conf)
6302 {
6303         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6304         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6305         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
6306         uint64_t hena;
6307
6308         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6309         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6310         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
6311                 if (rss_hf != 0) /* Enable RSS */
6312                         return -EINVAL;
6313                 return 0; /* Nothing to do */
6314         }
6315         /* RSS enabled */
6316         if (rss_hf == 0) /* Disable RSS */
6317                 return -EINVAL;
6318
6319         return i40e_hw_rss_hash_set(pf, rss_conf);
6320 }
6321
6322 static int
6323 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
6324                            struct rte_eth_rss_conf *rss_conf)
6325 {
6326         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6327         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6328         uint64_t hena;
6329
6330         i40e_get_rss_key(pf->main_vsi, rss_conf->rss_key,
6331                          &rss_conf->rss_key_len);
6332
6333         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6334         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6335         rss_conf->rss_hf = i40e_parse_hena(hena);
6336
6337         return 0;
6338 }
6339
6340 static int
6341 i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag)
6342 {
6343         switch (filter_type) {
6344         case RTE_TUNNEL_FILTER_IMAC_IVLAN:
6345                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN;
6346                 break;
6347         case RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID:
6348                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID;
6349                 break;
6350         case RTE_TUNNEL_FILTER_IMAC_TENID:
6351                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID;
6352                 break;
6353         case RTE_TUNNEL_FILTER_OMAC_TENID_IMAC:
6354                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC;
6355                 break;
6356         case ETH_TUNNEL_FILTER_IMAC:
6357                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC;
6358                 break;
6359         case ETH_TUNNEL_FILTER_OIP:
6360                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OIP;
6361                 break;
6362         case ETH_TUNNEL_FILTER_IIP:
6363                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IIP;
6364                 break;
6365         default:
6366                 PMD_DRV_LOG(ERR, "invalid tunnel filter type");
6367                 return -EINVAL;
6368         }
6369
6370         return 0;
6371 }
6372
6373 static int
6374 i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
6375                         struct rte_eth_tunnel_filter_conf *tunnel_filter,
6376                         uint8_t add)
6377 {
6378         uint16_t ip_type;
6379         uint32_t ipv4_addr;
6380         uint8_t i, tun_type = 0;
6381         /* internal varialbe to convert ipv6 byte order */
6382         uint32_t convert_ipv6[4];
6383         int val, ret = 0;
6384         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6385         struct i40e_vsi *vsi = pf->main_vsi;
6386         struct i40e_aqc_add_remove_cloud_filters_element_data  *cld_filter;
6387         struct i40e_aqc_add_remove_cloud_filters_element_data  *pfilter;
6388
6389         cld_filter = rte_zmalloc("tunnel_filter",
6390                 sizeof(struct i40e_aqc_add_remove_cloud_filters_element_data),
6391                 0);
6392
6393         if (NULL == cld_filter) {
6394                 PMD_DRV_LOG(ERR, "Failed to alloc memory.");
6395                 return -EINVAL;
6396         }
6397         pfilter = cld_filter;
6398
6399         ether_addr_copy(&tunnel_filter->outer_mac, (struct ether_addr*)&pfilter->outer_mac);
6400         ether_addr_copy(&tunnel_filter->inner_mac, (struct ether_addr*)&pfilter->inner_mac);
6401
6402         pfilter->inner_vlan = rte_cpu_to_le_16(tunnel_filter->inner_vlan);
6403         if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) {
6404                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4;
6405                 ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr);
6406                 rte_memcpy(&pfilter->ipaddr.v4.data,
6407                                 &rte_cpu_to_le_32(ipv4_addr),
6408                                 sizeof(pfilter->ipaddr.v4.data));
6409         } else {
6410                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6;
6411                 for (i = 0; i < 4; i++) {
6412                         convert_ipv6[i] =
6413                         rte_cpu_to_le_32(rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv6_addr[i]));
6414                 }
6415                 rte_memcpy(&pfilter->ipaddr.v6.data, &convert_ipv6,
6416                                 sizeof(pfilter->ipaddr.v6.data));
6417         }
6418
6419         /* check tunneled type */
6420         switch (tunnel_filter->tunnel_type) {
6421         case RTE_TUNNEL_TYPE_VXLAN:
6422                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN;
6423                 break;
6424         case RTE_TUNNEL_TYPE_NVGRE:
6425                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC;
6426                 break;
6427         case RTE_TUNNEL_TYPE_IP_IN_GRE:
6428                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP;
6429                 break;
6430         default:
6431                 /* Other tunnel types is not supported. */
6432                 PMD_DRV_LOG(ERR, "tunnel type is not supported.");
6433                 rte_free(cld_filter);
6434                 return -EINVAL;
6435         }
6436
6437         val = i40e_dev_get_filter_type(tunnel_filter->filter_type,
6438                                                 &pfilter->flags);
6439         if (val < 0) {
6440                 rte_free(cld_filter);
6441                 return -EINVAL;
6442         }
6443
6444         pfilter->flags |= rte_cpu_to_le_16(
6445                 I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE |
6446                 ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT));
6447         pfilter->tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id);
6448         pfilter->queue_number = rte_cpu_to_le_16(tunnel_filter->queue_id);
6449
6450         if (add)
6451                 ret = i40e_aq_add_cloud_filters(hw, vsi->seid, cld_filter, 1);
6452         else
6453                 ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
6454                                                 cld_filter, 1);
6455
6456         rte_free(cld_filter);
6457         return ret;
6458 }
6459
6460 static int
6461 i40e_get_vxlan_port_idx(struct i40e_pf *pf, uint16_t port)
6462 {
6463         uint8_t i;
6464
6465         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
6466                 if (pf->vxlan_ports[i] == port)
6467                         return i;
6468         }
6469
6470         return -1;
6471 }
6472
6473 static int
6474 i40e_add_vxlan_port(struct i40e_pf *pf, uint16_t port)
6475 {
6476         int  idx, ret;
6477         uint8_t filter_idx;
6478         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6479
6480         idx = i40e_get_vxlan_port_idx(pf, port);
6481
6482         /* Check if port already exists */
6483         if (idx >= 0) {
6484                 PMD_DRV_LOG(ERR, "Port %d already offloaded", port);
6485                 return -EINVAL;
6486         }
6487
6488         /* Now check if there is space to add the new port */
6489         idx = i40e_get_vxlan_port_idx(pf, 0);
6490         if (idx < 0) {
6491                 PMD_DRV_LOG(ERR, "Maximum number of UDP ports reached,"
6492                         "not adding port %d", port);
6493                 return -ENOSPC;
6494         }
6495
6496         ret =  i40e_aq_add_udp_tunnel(hw, port, I40E_AQC_TUNNEL_TYPE_VXLAN,
6497                                         &filter_idx, NULL);
6498         if (ret < 0) {
6499                 PMD_DRV_LOG(ERR, "Failed to add VXLAN UDP port %d", port);
6500                 return -1;
6501         }
6502
6503         PMD_DRV_LOG(INFO, "Added port %d with AQ command with index %d",
6504                          port,  filter_idx);
6505
6506         /* New port: add it and mark its index in the bitmap */
6507         pf->vxlan_ports[idx] = port;
6508         pf->vxlan_bitmap |= (1 << idx);
6509
6510         if (!(pf->flags & I40E_FLAG_VXLAN))
6511                 pf->flags |= I40E_FLAG_VXLAN;
6512
6513         return 0;
6514 }
6515
6516 static int
6517 i40e_del_vxlan_port(struct i40e_pf *pf, uint16_t port)
6518 {
6519         int idx;
6520         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6521
6522         if (!(pf->flags & I40E_FLAG_VXLAN)) {
6523                 PMD_DRV_LOG(ERR, "VXLAN UDP port was not configured.");
6524                 return -EINVAL;
6525         }
6526
6527         idx = i40e_get_vxlan_port_idx(pf, port);
6528
6529         if (idx < 0) {
6530                 PMD_DRV_LOG(ERR, "Port %d doesn't exist", port);
6531                 return -EINVAL;
6532         }
6533
6534         if (i40e_aq_del_udp_tunnel(hw, idx, NULL) < 0) {
6535                 PMD_DRV_LOG(ERR, "Failed to delete VXLAN UDP port %d", port);
6536                 return -1;
6537         }
6538
6539         PMD_DRV_LOG(INFO, "Deleted port %d with AQ command with index %d",
6540                         port, idx);
6541
6542         pf->vxlan_ports[idx] = 0;
6543         pf->vxlan_bitmap &= ~(1 << idx);
6544
6545         if (!pf->vxlan_bitmap)
6546                 pf->flags &= ~I40E_FLAG_VXLAN;
6547
6548         return 0;
6549 }
6550
6551 /* Add UDP tunneling port */
6552 static int
6553 i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
6554                              struct rte_eth_udp_tunnel *udp_tunnel)
6555 {
6556         int ret = 0;
6557         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6558
6559         if (udp_tunnel == NULL)
6560                 return -EINVAL;
6561
6562         switch (udp_tunnel->prot_type) {
6563         case RTE_TUNNEL_TYPE_VXLAN:
6564                 ret = i40e_add_vxlan_port(pf, udp_tunnel->udp_port);
6565                 break;
6566
6567         case RTE_TUNNEL_TYPE_GENEVE:
6568         case RTE_TUNNEL_TYPE_TEREDO:
6569                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6570                 ret = -1;
6571                 break;
6572
6573         default:
6574                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6575                 ret = -1;
6576                 break;
6577         }
6578
6579         return ret;
6580 }
6581
6582 /* Remove UDP tunneling port */
6583 static int
6584 i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
6585                              struct rte_eth_udp_tunnel *udp_tunnel)
6586 {
6587         int ret = 0;
6588         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6589
6590         if (udp_tunnel == NULL)
6591                 return -EINVAL;
6592
6593         switch (udp_tunnel->prot_type) {
6594         case RTE_TUNNEL_TYPE_VXLAN:
6595                 ret = i40e_del_vxlan_port(pf, udp_tunnel->udp_port);
6596                 break;
6597         case RTE_TUNNEL_TYPE_GENEVE:
6598         case RTE_TUNNEL_TYPE_TEREDO:
6599                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6600                 ret = -1;
6601                 break;
6602         default:
6603                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6604                 ret = -1;
6605                 break;
6606         }
6607
6608         return ret;
6609 }
6610
6611 /* Calculate the maximum number of contiguous PF queues that are configured */
6612 static int
6613 i40e_pf_calc_configured_queues_num(struct i40e_pf *pf)
6614 {
6615         struct rte_eth_dev_data *data = pf->dev_data;
6616         int i, num;
6617         struct i40e_rx_queue *rxq;
6618
6619         num = 0;
6620         for (i = 0; i < pf->lan_nb_qps; i++) {
6621                 rxq = data->rx_queues[i];
6622                 if (rxq && rxq->q_set)
6623                         num++;
6624                 else
6625                         break;
6626         }
6627
6628         return num;
6629 }
6630
6631 /* Configure RSS */
6632 static int
6633 i40e_pf_config_rss(struct i40e_pf *pf)
6634 {
6635         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6636         struct rte_eth_rss_conf rss_conf;
6637         uint32_t i, lut = 0;
6638         uint16_t j, num;
6639
6640         /*
6641          * If both VMDQ and RSS enabled, not all of PF queues are configured.
6642          * It's necessary to calulate the actual PF queues that are configured.
6643          */
6644         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
6645                 num = i40e_pf_calc_configured_queues_num(pf);
6646         else
6647                 num = pf->dev_data->nb_rx_queues;
6648
6649         num = RTE_MIN(num, I40E_MAX_Q_PER_TC);
6650         PMD_INIT_LOG(INFO, "Max of contiguous %u PF queues are configured",
6651                         num);
6652
6653         if (num == 0) {
6654                 PMD_INIT_LOG(ERR, "No PF queues are configured to enable RSS");
6655                 return -ENOTSUP;
6656         }
6657
6658         for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
6659                 if (j == num)
6660                         j = 0;
6661                 lut = (lut << 8) | (j & ((0x1 <<
6662                         hw->func_caps.rss_table_entry_width) - 1));
6663                 if ((i & 3) == 3)
6664                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
6665         }
6666
6667         rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
6668         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
6669                 i40e_pf_disable_rss(pf);
6670                 return 0;
6671         }
6672         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
6673                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
6674                 /* Random default keys */
6675                 static uint32_t rss_key_default[] = {0x6b793944,
6676                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
6677                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
6678                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
6679
6680                 rss_conf.rss_key = (uint8_t *)rss_key_default;
6681                 rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
6682                                                         sizeof(uint32_t);
6683         }
6684
6685         return i40e_hw_rss_hash_set(pf, &rss_conf);
6686 }
6687
6688 static int
6689 i40e_tunnel_filter_param_check(struct i40e_pf *pf,
6690                                struct rte_eth_tunnel_filter_conf *filter)
6691 {
6692         if (pf == NULL || filter == NULL) {
6693                 PMD_DRV_LOG(ERR, "Invalid parameter");
6694                 return -EINVAL;
6695         }
6696
6697         if (filter->queue_id >= pf->dev_data->nb_rx_queues) {
6698                 PMD_DRV_LOG(ERR, "Invalid queue ID");
6699                 return -EINVAL;
6700         }
6701
6702         if (filter->inner_vlan > ETHER_MAX_VLAN_ID) {
6703                 PMD_DRV_LOG(ERR, "Invalid inner VLAN ID");
6704                 return -EINVAL;
6705         }
6706
6707         if ((filter->filter_type & ETH_TUNNEL_FILTER_OMAC) &&
6708                 (is_zero_ether_addr(&filter->outer_mac))) {
6709                 PMD_DRV_LOG(ERR, "Cannot add NULL outer MAC address");
6710                 return -EINVAL;
6711         }
6712
6713         if ((filter->filter_type & ETH_TUNNEL_FILTER_IMAC) &&
6714                 (is_zero_ether_addr(&filter->inner_mac))) {
6715                 PMD_DRV_LOG(ERR, "Cannot add NULL inner MAC address");
6716                 return -EINVAL;
6717         }
6718
6719         return 0;
6720 }
6721
6722 #define I40E_GL_PRS_FVBM_MSK_ENA 0x80000000
6723 #define I40E_GL_PRS_FVBM(_i)     (0x00269760 + ((_i) * 4))
6724 static int
6725 i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len)
6726 {
6727         uint32_t val, reg;
6728         int ret = -EINVAL;
6729
6730         val = I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2));
6731         PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x\n", val);
6732
6733         if (len == 3) {
6734                 reg = val | I40E_GL_PRS_FVBM_MSK_ENA;
6735         } else if (len == 4) {
6736                 reg = val & ~I40E_GL_PRS_FVBM_MSK_ENA;
6737         } else {
6738                 PMD_DRV_LOG(ERR, "Unsupported GRE key length of %u", len);
6739                 return ret;
6740         }
6741
6742         if (reg != val) {
6743                 ret = i40e_aq_debug_write_register(hw, I40E_GL_PRS_FVBM(2),
6744                                                    reg, NULL);
6745                 if (ret != 0)
6746                         return ret;
6747         } else {
6748                 ret = 0;
6749         }
6750         PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x\n",
6751                     I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2)));
6752
6753         return ret;
6754 }
6755
6756 static int
6757 i40e_dev_global_config_set(struct i40e_hw *hw, struct rte_eth_global_cfg *cfg)
6758 {
6759         int ret = -EINVAL;
6760
6761         if (!hw || !cfg)
6762                 return -EINVAL;
6763
6764         switch (cfg->cfg_type) {
6765         case RTE_ETH_GLOBAL_CFG_TYPE_GRE_KEY_LEN:
6766                 ret = i40e_dev_set_gre_key_len(hw, cfg->cfg.gre_key_len);
6767                 break;
6768         default:
6769                 PMD_DRV_LOG(ERR, "Unknown config type %u", cfg->cfg_type);
6770                 break;
6771         }
6772
6773         return ret;
6774 }
6775
6776 static int
6777 i40e_filter_ctrl_global_config(struct rte_eth_dev *dev,
6778                                enum rte_filter_op filter_op,
6779                                void *arg)
6780 {
6781         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6782         int ret = I40E_ERR_PARAM;
6783
6784         switch (filter_op) {
6785         case RTE_ETH_FILTER_SET:
6786                 ret = i40e_dev_global_config_set(hw,
6787                         (struct rte_eth_global_cfg *)arg);
6788                 break;
6789         default:
6790                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
6791                 break;
6792         }
6793
6794         return ret;
6795 }
6796
6797 static int
6798 i40e_tunnel_filter_handle(struct rte_eth_dev *dev,
6799                           enum rte_filter_op filter_op,
6800                           void *arg)
6801 {
6802         struct rte_eth_tunnel_filter_conf *filter;
6803         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6804         int ret = I40E_SUCCESS;
6805
6806         filter = (struct rte_eth_tunnel_filter_conf *)(arg);
6807
6808         if (i40e_tunnel_filter_param_check(pf, filter) < 0)
6809                 return I40E_ERR_PARAM;
6810
6811         switch (filter_op) {
6812         case RTE_ETH_FILTER_NOP:
6813                 if (!(pf->flags & I40E_FLAG_VXLAN))
6814                         ret = I40E_NOT_SUPPORTED;
6815                 break;
6816         case RTE_ETH_FILTER_ADD:
6817                 ret = i40e_dev_tunnel_filter_set(pf, filter, 1);
6818                 break;
6819         case RTE_ETH_FILTER_DELETE:
6820                 ret = i40e_dev_tunnel_filter_set(pf, filter, 0);
6821                 break;
6822         default:
6823                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
6824                 ret = I40E_ERR_PARAM;
6825                 break;
6826         }
6827
6828         return ret;
6829 }
6830
6831 static int
6832 i40e_pf_config_mq_rx(struct i40e_pf *pf)
6833 {
6834         int ret = 0;
6835         enum rte_eth_rx_mq_mode mq_mode = pf->dev_data->dev_conf.rxmode.mq_mode;
6836
6837         /* RSS setup */
6838         if (mq_mode & ETH_MQ_RX_RSS_FLAG)
6839                 ret = i40e_pf_config_rss(pf);
6840         else
6841                 i40e_pf_disable_rss(pf);
6842
6843         return ret;
6844 }
6845
6846 /* Get the symmetric hash enable configurations per port */
6847 static void
6848 i40e_get_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t *enable)
6849 {
6850         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
6851
6852         *enable = reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK ? 1 : 0;
6853 }
6854
6855 /* Set the symmetric hash enable configurations per port */
6856 static void
6857 i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable)
6858 {
6859         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
6860
6861         if (enable > 0) {
6862                 if (reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK) {
6863                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
6864                                                         "been enabled");
6865                         return;
6866                 }
6867                 reg |= I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
6868         } else {
6869                 if (!(reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK)) {
6870                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
6871                                                         "been disabled");
6872                         return;
6873                 }
6874                 reg &= ~I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
6875         }
6876         i40e_write_rx_ctl(hw, I40E_PRTQF_CTL_0, reg);
6877         I40E_WRITE_FLUSH(hw);
6878 }
6879
6880 /*
6881  * Get global configurations of hash function type and symmetric hash enable
6882  * per flow type (pctype). Note that global configuration means it affects all
6883  * the ports on the same NIC.
6884  */
6885 static int
6886 i40e_get_hash_filter_global_config(struct i40e_hw *hw,
6887                                    struct rte_eth_hash_global_conf *g_cfg)
6888 {
6889         uint32_t reg, mask = I40E_FLOW_TYPES;
6890         uint16_t i;
6891         enum i40e_filter_pctype pctype;
6892
6893         memset(g_cfg, 0, sizeof(*g_cfg));
6894         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
6895         if (reg & I40E_GLQF_CTL_HTOEP_MASK)
6896                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ;
6897         else
6898                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
6899         PMD_DRV_LOG(DEBUG, "Hash function is %s",
6900                 (reg & I40E_GLQF_CTL_HTOEP_MASK) ? "Toeplitz" : "Simple XOR");
6901
6902         for (i = 0; mask && i < RTE_ETH_FLOW_MAX; i++) {
6903                 if (!(mask & (1UL << i)))
6904                         continue;
6905                 mask &= ~(1UL << i);
6906                 /* Bit set indicats the coresponding flow type is supported */
6907                 g_cfg->valid_bit_mask[0] |= (1UL << i);
6908                 pctype = i40e_flowtype_to_pctype(i);
6909                 reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
6910                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
6911                         g_cfg->sym_hash_enable_mask[0] |= (1UL << i);
6912         }
6913
6914         return 0;
6915 }
6916
6917 static int
6918 i40e_hash_global_config_check(struct rte_eth_hash_global_conf *g_cfg)
6919 {
6920         uint32_t i;
6921         uint32_t mask0, i40e_mask = I40E_FLOW_TYPES;
6922
6923         if (g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_TOEPLITZ &&
6924                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR &&
6925                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
6926                 PMD_DRV_LOG(ERR, "Unsupported hash function type %d",
6927                                                 g_cfg->hash_func);
6928                 return -EINVAL;
6929         }
6930
6931         /*
6932          * As i40e supports less than 32 flow types, only first 32 bits need to
6933          * be checked.
6934          */
6935         mask0 = g_cfg->valid_bit_mask[0];
6936         for (i = 0; i < RTE_SYM_HASH_MASK_ARRAY_SIZE; i++) {
6937                 if (i == 0) {
6938                         /* Check if any unsupported flow type configured */
6939                         if ((mask0 | i40e_mask) ^ i40e_mask)
6940                                 goto mask_err;
6941                 } else {
6942                         if (g_cfg->valid_bit_mask[i])
6943                                 goto mask_err;
6944                 }
6945         }
6946
6947         return 0;
6948
6949 mask_err:
6950         PMD_DRV_LOG(ERR, "i40e unsupported flow type bit(s) configured");
6951
6952         return -EINVAL;
6953 }
6954
6955 /*
6956  * Set global configurations of hash function type and symmetric hash enable
6957  * per flow type (pctype). Note any modifying global configuration will affect
6958  * all the ports on the same NIC.
6959  */
6960 static int
6961 i40e_set_hash_filter_global_config(struct i40e_hw *hw,
6962                                    struct rte_eth_hash_global_conf *g_cfg)
6963 {
6964         int ret;
6965         uint16_t i;
6966         uint32_t reg;
6967         uint32_t mask0 = g_cfg->valid_bit_mask[0];
6968         enum i40e_filter_pctype pctype;
6969
6970         /* Check the input parameters */
6971         ret = i40e_hash_global_config_check(g_cfg);
6972         if (ret < 0)
6973                 return ret;
6974
6975         for (i = 0; mask0 && i < UINT32_BIT; i++) {
6976                 if (!(mask0 & (1UL << i)))
6977                         continue;
6978                 mask0 &= ~(1UL << i);
6979                 pctype = i40e_flowtype_to_pctype(i);
6980                 reg = (g_cfg->sym_hash_enable_mask[0] & (1UL << i)) ?
6981                                 I40E_GLQF_HSYM_SYMH_ENA_MASK : 0;
6982                 i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
6983         }
6984
6985         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
6986         if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
6987                 /* Toeplitz */
6988                 if (reg & I40E_GLQF_CTL_HTOEP_MASK) {
6989                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
6990                                                                 "Toeplitz");
6991                         goto out;
6992                 }
6993                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
6994         } else if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
6995                 /* Simple XOR */
6996                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK)) {
6997                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
6998                                                         "Simple XOR");
6999                         goto out;
7000                 }
7001                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
7002         } else
7003                 /* Use the default, and keep it as it is */
7004                 goto out;
7005
7006         i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
7007
7008 out:
7009         I40E_WRITE_FLUSH(hw);
7010
7011         return 0;
7012 }
7013
7014 /**
7015  * Valid input sets for hash and flow director filters per PCTYPE
7016  */
7017 static uint64_t
7018 i40e_get_valid_input_set(enum i40e_filter_pctype pctype,
7019                 enum rte_filter_type filter)
7020 {
7021         uint64_t valid;
7022
7023         static const uint64_t valid_hash_inset_table[] = {
7024                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7025                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7026                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7027                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_SRC |
7028                         I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS |
7029                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7030                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7031                         I40E_INSET_FLEX_PAYLOAD,
7032                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7033                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7034                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7035                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7036                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7037                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7038                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7039                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7040                         I40E_INSET_FLEX_PAYLOAD,
7041                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7042                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7043                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7044                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7045                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7046                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7047                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7048                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7049                         I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
7050                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7051                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7052                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7053                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7054                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7055                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7056                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7057                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7058                         I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD,
7059                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7060                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7061                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7062                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7063                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7064                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7065                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7066                         I40E_INSET_FLEX_PAYLOAD,
7067                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7068                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7069                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7070                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7071                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7072                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_TUNNEL_DMAC |
7073                         I40E_INSET_TUNNEL_ID | I40E_INSET_IPV6_SRC |
7074                         I40E_INSET_IPV6_DST | I40E_INSET_FLEX_PAYLOAD,
7075                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7076                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7077                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7078                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7079                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7080                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7081                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7082                         I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD,
7083                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7084                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7085                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7086                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7087                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7088                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7089                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7090                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7091                         I40E_INSET_FLEX_PAYLOAD,
7092                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7093                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7094                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7095                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7096                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7097                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7098                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7099                         I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT |
7100                         I40E_INSET_FLEX_PAYLOAD,
7101                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7102                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7103                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7104                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7105                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7106                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7107                         I40E_INSET_IPV6_DST | I40E_INSET_TUNNEL_ID |
7108                         I40E_INSET_FLEX_PAYLOAD,
7109                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7110                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7111                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7112                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_LAST_ETHER_TYPE |
7113                         I40E_INSET_FLEX_PAYLOAD,
7114         };
7115
7116         /**
7117          * Flow director supports only fields defined in
7118          * union rte_eth_fdir_flow.
7119          */
7120         static const uint64_t valid_fdir_inset_table[] = {
7121                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7122                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7123                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7124                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
7125                 I40E_INSET_IPV4_TTL,
7126                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7127                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7128                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7129                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7130                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7131                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7132                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7133                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7134                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7135                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7136                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7137                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7138                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7139                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7140                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7141                 I40E_INSET_SCTP_VT,
7142                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7143                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7144                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7145                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
7146                 I40E_INSET_IPV4_TTL,
7147                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7148                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7149                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7150                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
7151                 I40E_INSET_IPV6_HOP_LIMIT,
7152                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7153                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7154                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7155                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7156                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7157                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7158                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7159                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7160                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7161                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7162                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7163                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7164                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7165                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7166                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7167                 I40E_INSET_SCTP_VT,
7168                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7169                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7170                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7171                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
7172                 I40E_INSET_IPV6_HOP_LIMIT,
7173                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7174                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7175                 I40E_INSET_LAST_ETHER_TYPE,
7176         };
7177
7178         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
7179                 return 0;
7180         if (filter == RTE_ETH_FILTER_HASH)
7181                 valid = valid_hash_inset_table[pctype];
7182         else
7183                 valid = valid_fdir_inset_table[pctype];
7184
7185         return valid;
7186 }
7187
7188 /**
7189  * Validate if the input set is allowed for a specific PCTYPE
7190  */
7191 static int
7192 i40e_validate_input_set(enum i40e_filter_pctype pctype,
7193                 enum rte_filter_type filter, uint64_t inset)
7194 {
7195         uint64_t valid;
7196
7197         valid = i40e_get_valid_input_set(pctype, filter);
7198         if (inset & (~valid))
7199                 return -EINVAL;
7200
7201         return 0;
7202 }
7203
7204 /* default input set fields combination per pctype */
7205 static uint64_t
7206 i40e_get_default_input_set(uint16_t pctype)
7207 {
7208         static const uint64_t default_inset_table[] = {
7209                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7210                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
7211                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7212                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7213                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7214                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7215                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7216                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7217                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7218                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7219                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7220                         I40E_INSET_SCTP_VT,
7221                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7222                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
7223                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7224                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
7225                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7226                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7227                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7228                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7229                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7230                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7231                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7232                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7233                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7234                         I40E_INSET_SCTP_VT,
7235                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7236                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
7237                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7238                         I40E_INSET_LAST_ETHER_TYPE,
7239         };
7240
7241         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
7242                 return 0;
7243
7244         return default_inset_table[pctype];
7245 }
7246
7247 /**
7248  * Parse the input set from index to logical bit masks
7249  */
7250 static int
7251 i40e_parse_input_set(uint64_t *inset,
7252                      enum i40e_filter_pctype pctype,
7253                      enum rte_eth_input_set_field *field,
7254                      uint16_t size)
7255 {
7256         uint16_t i, j;
7257         int ret = -EINVAL;
7258
7259         static const struct {
7260                 enum rte_eth_input_set_field field;
7261                 uint64_t inset;
7262         } inset_convert_table[] = {
7263                 {RTE_ETH_INPUT_SET_NONE, I40E_INSET_NONE},
7264                 {RTE_ETH_INPUT_SET_L2_SRC_MAC, I40E_INSET_SMAC},
7265                 {RTE_ETH_INPUT_SET_L2_DST_MAC, I40E_INSET_DMAC},
7266                 {RTE_ETH_INPUT_SET_L2_OUTER_VLAN, I40E_INSET_VLAN_OUTER},
7267                 {RTE_ETH_INPUT_SET_L2_INNER_VLAN, I40E_INSET_VLAN_INNER},
7268                 {RTE_ETH_INPUT_SET_L2_ETHERTYPE, I40E_INSET_LAST_ETHER_TYPE},
7269                 {RTE_ETH_INPUT_SET_L3_SRC_IP4, I40E_INSET_IPV4_SRC},
7270                 {RTE_ETH_INPUT_SET_L3_DST_IP4, I40E_INSET_IPV4_DST},
7271                 {RTE_ETH_INPUT_SET_L3_IP4_TOS, I40E_INSET_IPV4_TOS},
7272                 {RTE_ETH_INPUT_SET_L3_IP4_PROTO, I40E_INSET_IPV4_PROTO},
7273                 {RTE_ETH_INPUT_SET_L3_IP4_TTL, I40E_INSET_IPV4_TTL},
7274                 {RTE_ETH_INPUT_SET_L3_SRC_IP6, I40E_INSET_IPV6_SRC},
7275                 {RTE_ETH_INPUT_SET_L3_DST_IP6, I40E_INSET_IPV6_DST},
7276                 {RTE_ETH_INPUT_SET_L3_IP6_TC, I40E_INSET_IPV6_TC},
7277                 {RTE_ETH_INPUT_SET_L3_IP6_NEXT_HEADER,
7278                         I40E_INSET_IPV6_NEXT_HDR},
7279                 {RTE_ETH_INPUT_SET_L3_IP6_HOP_LIMITS,
7280                         I40E_INSET_IPV6_HOP_LIMIT},
7281                 {RTE_ETH_INPUT_SET_L4_UDP_SRC_PORT, I40E_INSET_SRC_PORT},
7282                 {RTE_ETH_INPUT_SET_L4_TCP_SRC_PORT, I40E_INSET_SRC_PORT},
7283                 {RTE_ETH_INPUT_SET_L4_SCTP_SRC_PORT, I40E_INSET_SRC_PORT},
7284                 {RTE_ETH_INPUT_SET_L4_UDP_DST_PORT, I40E_INSET_DST_PORT},
7285                 {RTE_ETH_INPUT_SET_L4_TCP_DST_PORT, I40E_INSET_DST_PORT},
7286                 {RTE_ETH_INPUT_SET_L4_SCTP_DST_PORT, I40E_INSET_DST_PORT},
7287                 {RTE_ETH_INPUT_SET_L4_SCTP_VERIFICATION_TAG,
7288                         I40E_INSET_SCTP_VT},
7289                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_DST_MAC,
7290                         I40E_INSET_TUNNEL_DMAC},
7291                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_VLAN,
7292                         I40E_INSET_VLAN_TUNNEL},
7293                 {RTE_ETH_INPUT_SET_TUNNEL_L4_UDP_KEY,
7294                         I40E_INSET_TUNNEL_ID},
7295                 {RTE_ETH_INPUT_SET_TUNNEL_GRE_KEY, I40E_INSET_TUNNEL_ID},
7296                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_1ST_WORD,
7297                         I40E_INSET_FLEX_PAYLOAD_W1},
7298                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_2ND_WORD,
7299                         I40E_INSET_FLEX_PAYLOAD_W2},
7300                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_3RD_WORD,
7301                         I40E_INSET_FLEX_PAYLOAD_W3},
7302                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_4TH_WORD,
7303                         I40E_INSET_FLEX_PAYLOAD_W4},
7304                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_5TH_WORD,
7305                         I40E_INSET_FLEX_PAYLOAD_W5},
7306                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_6TH_WORD,
7307                         I40E_INSET_FLEX_PAYLOAD_W6},
7308                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_7TH_WORD,
7309                         I40E_INSET_FLEX_PAYLOAD_W7},
7310                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_8TH_WORD,
7311                         I40E_INSET_FLEX_PAYLOAD_W8},
7312         };
7313
7314         if (!inset || !field || size > RTE_ETH_INSET_SIZE_MAX)
7315                 return ret;
7316
7317         /* Only one item allowed for default or all */
7318         if (size == 1) {
7319                 if (field[0] == RTE_ETH_INPUT_SET_DEFAULT) {
7320                         *inset = i40e_get_default_input_set(pctype);
7321                         return 0;
7322                 } else if (field[0] == RTE_ETH_INPUT_SET_NONE) {
7323                         *inset = I40E_INSET_NONE;
7324                         return 0;
7325                 }
7326         }
7327
7328         for (i = 0, *inset = 0; i < size; i++) {
7329                 for (j = 0; j < RTE_DIM(inset_convert_table); j++) {
7330                         if (field[i] == inset_convert_table[j].field) {
7331                                 *inset |= inset_convert_table[j].inset;
7332                                 break;
7333                         }
7334                 }
7335
7336                 /* It contains unsupported input set, return immediately */
7337                 if (j == RTE_DIM(inset_convert_table))
7338                         return ret;
7339         }
7340
7341         return 0;
7342 }
7343
7344 /**
7345  * Translate the input set from bit masks to register aware bit masks
7346  * and vice versa
7347  */
7348 static uint64_t
7349 i40e_translate_input_set_reg(uint64_t input)
7350 {
7351         uint64_t val = 0;
7352         uint16_t i;
7353
7354         static const struct {
7355                 uint64_t inset;
7356                 uint64_t inset_reg;
7357         } inset_map[] = {
7358                 {I40E_INSET_DMAC, I40E_REG_INSET_L2_DMAC},
7359                 {I40E_INSET_SMAC, I40E_REG_INSET_L2_SMAC},
7360                 {I40E_INSET_VLAN_OUTER, I40E_REG_INSET_L2_OUTER_VLAN},
7361                 {I40E_INSET_VLAN_INNER, I40E_REG_INSET_L2_INNER_VLAN},
7362                 {I40E_INSET_LAST_ETHER_TYPE, I40E_REG_INSET_LAST_ETHER_TYPE},
7363                 {I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4},
7364                 {I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4},
7365                 {I40E_INSET_IPV4_TOS, I40E_REG_INSET_L3_IP4_TOS},
7366                 {I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO},
7367                 {I40E_INSET_IPV4_TTL, I40E_REG_INSET_L3_IP4_TTL},
7368                 {I40E_INSET_IPV6_SRC, I40E_REG_INSET_L3_SRC_IP6},
7369                 {I40E_INSET_IPV6_DST, I40E_REG_INSET_L3_DST_IP6},
7370                 {I40E_INSET_IPV6_TC, I40E_REG_INSET_L3_IP6_TC},
7371                 {I40E_INSET_IPV6_NEXT_HDR, I40E_REG_INSET_L3_IP6_NEXT_HDR},
7372                 {I40E_INSET_IPV6_HOP_LIMIT, I40E_REG_INSET_L3_IP6_HOP_LIMIT},
7373                 {I40E_INSET_SRC_PORT, I40E_REG_INSET_L4_SRC_PORT},
7374                 {I40E_INSET_DST_PORT, I40E_REG_INSET_L4_DST_PORT},
7375                 {I40E_INSET_SCTP_VT, I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG},
7376                 {I40E_INSET_TUNNEL_ID, I40E_REG_INSET_TUNNEL_ID},
7377                 {I40E_INSET_TUNNEL_DMAC,
7378                         I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC},
7379                 {I40E_INSET_TUNNEL_IPV4_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP4},
7380                 {I40E_INSET_TUNNEL_IPV6_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP6},
7381                 {I40E_INSET_TUNNEL_SRC_PORT,
7382                         I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT},
7383                 {I40E_INSET_TUNNEL_DST_PORT,
7384                         I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT},
7385                 {I40E_INSET_VLAN_TUNNEL, I40E_REG_INSET_TUNNEL_VLAN},
7386                 {I40E_INSET_FLEX_PAYLOAD_W1, I40E_REG_INSET_FLEX_PAYLOAD_WORD1},
7387                 {I40E_INSET_FLEX_PAYLOAD_W2, I40E_REG_INSET_FLEX_PAYLOAD_WORD2},
7388                 {I40E_INSET_FLEX_PAYLOAD_W3, I40E_REG_INSET_FLEX_PAYLOAD_WORD3},
7389                 {I40E_INSET_FLEX_PAYLOAD_W4, I40E_REG_INSET_FLEX_PAYLOAD_WORD4},
7390                 {I40E_INSET_FLEX_PAYLOAD_W5, I40E_REG_INSET_FLEX_PAYLOAD_WORD5},
7391                 {I40E_INSET_FLEX_PAYLOAD_W6, I40E_REG_INSET_FLEX_PAYLOAD_WORD6},
7392                 {I40E_INSET_FLEX_PAYLOAD_W7, I40E_REG_INSET_FLEX_PAYLOAD_WORD7},
7393                 {I40E_INSET_FLEX_PAYLOAD_W8, I40E_REG_INSET_FLEX_PAYLOAD_WORD8},
7394         };
7395
7396         if (input == 0)
7397                 return val;
7398
7399         /* Translate input set to register aware inset */
7400         for (i = 0; i < RTE_DIM(inset_map); i++) {
7401                 if (input & inset_map[i].inset)
7402                         val |= inset_map[i].inset_reg;
7403         }
7404
7405         return val;
7406 }
7407
7408 static int
7409 i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem)
7410 {
7411         uint8_t i, idx = 0;
7412         uint64_t inset_need_mask = inset;
7413
7414         static const struct {
7415                 uint64_t inset;
7416                 uint32_t mask;
7417         } inset_mask_map[] = {
7418                 {I40E_INSET_IPV4_TOS, I40E_INSET_IPV4_TOS_MASK},
7419                 {I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL, 0},
7420                 {I40E_INSET_IPV4_PROTO, I40E_INSET_IPV4_PROTO_MASK},
7421                 {I40E_INSET_IPV4_TTL, I40E_INSET_IPv4_TTL_MASK},
7422                 {I40E_INSET_IPV6_TC, I40E_INSET_IPV6_TC_MASK},
7423                 {I40E_INSET_IPV6_NEXT_HDR | I40E_INSET_IPV6_HOP_LIMIT, 0},
7424                 {I40E_INSET_IPV6_NEXT_HDR, I40E_INSET_IPV6_NEXT_HDR_MASK},
7425                 {I40E_INSET_IPV6_HOP_LIMIT, I40E_INSET_IPV6_HOP_LIMIT_MASK},
7426         };
7427
7428         if (!inset || !mask || !nb_elem)
7429                 return 0;
7430
7431         for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
7432                 /* Clear the inset bit, if no MASK is required,
7433                  * for example proto + ttl
7434                  */
7435                 if ((inset & inset_mask_map[i].inset) ==
7436                      inset_mask_map[i].inset && inset_mask_map[i].mask == 0)
7437                         inset_need_mask &= ~inset_mask_map[i].inset;
7438                 if (!inset_need_mask)
7439                         return 0;
7440         }
7441         for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
7442                 if ((inset_need_mask & inset_mask_map[i].inset) ==
7443                     inset_mask_map[i].inset) {
7444                         if (idx >= nb_elem) {
7445                                 PMD_DRV_LOG(ERR, "exceed maximal number of bitmasks");
7446                                 return -EINVAL;
7447                         }
7448                         mask[idx] = inset_mask_map[i].mask;
7449                         idx++;
7450                 }
7451         }
7452
7453         return idx;
7454 }
7455
7456 static void
7457 i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
7458 {
7459         uint32_t reg = i40e_read_rx_ctl(hw, addr);
7460
7461         PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x\n", addr, reg);
7462         if (reg != val)
7463                 i40e_write_rx_ctl(hw, addr, val);
7464         PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x\n", addr,
7465                     (uint32_t)i40e_read_rx_ctl(hw, addr));
7466 }
7467
7468 static void
7469 i40e_filter_input_set_init(struct i40e_pf *pf)
7470 {
7471         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7472         enum i40e_filter_pctype pctype;
7473         uint64_t input_set, inset_reg;
7474         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
7475         int num, i;
7476
7477         for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
7478              pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) {
7479                 if (!I40E_VALID_PCTYPE(pctype))
7480                         continue;
7481                 input_set = i40e_get_default_input_set(pctype);
7482
7483                 num = i40e_generate_inset_mask_reg(input_set, mask_reg,
7484                                                    I40E_INSET_MASK_NUM_REG);
7485                 if (num < 0)
7486                         return;
7487                 inset_reg = i40e_translate_input_set_reg(input_set);
7488
7489                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
7490                                       (uint32_t)(inset_reg & UINT32_MAX));
7491                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
7492                                      (uint32_t)((inset_reg >>
7493                                      I40E_32_BIT_WIDTH) & UINT32_MAX));
7494                 i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype),
7495                                       (uint32_t)(inset_reg & UINT32_MAX));
7496                 i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
7497                                      (uint32_t)((inset_reg >>
7498                                      I40E_32_BIT_WIDTH) & UINT32_MAX));
7499
7500                 for (i = 0; i < num; i++) {
7501                         i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7502                                              mask_reg[i]);
7503                         i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7504                                              mask_reg[i]);
7505                 }
7506                 /*clear unused mask registers of the pctype */
7507                 for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) {
7508                         i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7509                                              0);
7510                         i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7511                                              0);
7512                 }
7513                 I40E_WRITE_FLUSH(hw);
7514
7515                 /* store the default input set */
7516                 pf->hash_input_set[pctype] = input_set;
7517                 pf->fdir.input_set[pctype] = input_set;
7518         }
7519 }
7520
7521 int
7522 i40e_hash_filter_inset_select(struct i40e_hw *hw,
7523                          struct rte_eth_input_set_conf *conf)
7524 {
7525         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
7526         enum i40e_filter_pctype pctype;
7527         uint64_t input_set, inset_reg = 0;
7528         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
7529         int ret, i, num;
7530
7531         if (!conf) {
7532                 PMD_DRV_LOG(ERR, "Invalid pointer");
7533                 return -EFAULT;
7534         }
7535         if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
7536             conf->op != RTE_ETH_INPUT_SET_ADD) {
7537                 PMD_DRV_LOG(ERR, "Unsupported input set operation");
7538                 return -EINVAL;
7539         }
7540
7541         pctype = i40e_flowtype_to_pctype(conf->flow_type);
7542         if (pctype == 0 || pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD) {
7543                 PMD_DRV_LOG(ERR, "Not supported flow type (%u)",
7544                             conf->flow_type);
7545                 return -EINVAL;
7546         }
7547
7548         ret = i40e_parse_input_set(&input_set, pctype, conf->field,
7549                                    conf->inset_size);
7550         if (ret) {
7551                 PMD_DRV_LOG(ERR, "Failed to parse input set");
7552                 return -EINVAL;
7553         }
7554         if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_HASH,
7555                                     input_set) != 0) {
7556                 PMD_DRV_LOG(ERR, "Invalid input set");
7557                 return -EINVAL;
7558         }
7559         if (conf->op == RTE_ETH_INPUT_SET_ADD) {
7560                 /* get inset value in register */
7561                 inset_reg = i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, pctype));
7562                 inset_reg <<= I40E_32_BIT_WIDTH;
7563                 inset_reg |= i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, pctype));
7564                 input_set |= pf->hash_input_set[pctype];
7565         }
7566         num = i40e_generate_inset_mask_reg(input_set, mask_reg,
7567                                            I40E_INSET_MASK_NUM_REG);
7568         if (num < 0)
7569                 return -EINVAL;
7570
7571         inset_reg |= i40e_translate_input_set_reg(input_set);
7572
7573         i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype),
7574                               (uint32_t)(inset_reg & UINT32_MAX));
7575         i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
7576                              (uint32_t)((inset_reg >>
7577                              I40E_32_BIT_WIDTH) & UINT32_MAX));
7578
7579         for (i = 0; i < num; i++)
7580                 i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7581                                      mask_reg[i]);
7582         /*clear unused mask registers of the pctype */
7583         for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
7584                 i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7585                                      0);
7586         I40E_WRITE_FLUSH(hw);
7587
7588         pf->hash_input_set[pctype] = input_set;
7589         return 0;
7590 }
7591
7592 int
7593 i40e_fdir_filter_inset_select(struct i40e_pf *pf,
7594                          struct rte_eth_input_set_conf *conf)
7595 {
7596         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7597         enum i40e_filter_pctype pctype;
7598         uint64_t input_set, inset_reg = 0;
7599         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
7600         int ret, i, num;
7601
7602         if (!hw || !conf) {
7603                 PMD_DRV_LOG(ERR, "Invalid pointer");
7604                 return -EFAULT;
7605         }
7606         if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
7607             conf->op != RTE_ETH_INPUT_SET_ADD) {
7608                 PMD_DRV_LOG(ERR, "Unsupported input set operation");
7609                 return -EINVAL;
7610         }
7611
7612         pctype = i40e_flowtype_to_pctype(conf->flow_type);
7613         if (pctype == 0 || pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD) {
7614                 PMD_DRV_LOG(ERR, "Not supported flow type (%u)",
7615                             conf->flow_type);
7616                 return -EINVAL;
7617         }
7618         ret = i40e_parse_input_set(&input_set, pctype, conf->field,
7619                                    conf->inset_size);
7620         if (ret) {
7621                 PMD_DRV_LOG(ERR, "Failed to parse input set");
7622                 return -EINVAL;
7623         }
7624         if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR,
7625                                     input_set) != 0) {
7626                 PMD_DRV_LOG(ERR, "Invalid input set");
7627                 return -EINVAL;
7628         }
7629
7630         /* get inset value in register */
7631         inset_reg = i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 1));
7632         inset_reg <<= I40E_32_BIT_WIDTH;
7633         inset_reg |= i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 0));
7634
7635         /* Can not change the inset reg for flex payload for fdir,
7636          * it is done by writing I40E_PRTQF_FD_FLXINSET
7637          * in i40e_set_flex_mask_on_pctype.
7638          */
7639         if (conf->op == RTE_ETH_INPUT_SET_SELECT)
7640                 inset_reg &= I40E_REG_INSET_FLEX_PAYLOAD_WORDS;
7641         else
7642                 input_set |= pf->fdir.input_set[pctype];
7643         num = i40e_generate_inset_mask_reg(input_set, mask_reg,
7644                                            I40E_INSET_MASK_NUM_REG);
7645         if (num < 0)
7646                 return -EINVAL;
7647
7648         inset_reg |= i40e_translate_input_set_reg(input_set);
7649
7650         i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
7651                               (uint32_t)(inset_reg & UINT32_MAX));
7652         i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
7653                              (uint32_t)((inset_reg >>
7654                              I40E_32_BIT_WIDTH) & UINT32_MAX));
7655
7656         for (i = 0; i < num; i++)
7657                 i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7658                                      mask_reg[i]);
7659         /*clear unused mask registers of the pctype */
7660         for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
7661                 i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7662                                      0);
7663         I40E_WRITE_FLUSH(hw);
7664
7665         pf->fdir.input_set[pctype] = input_set;
7666         return 0;
7667 }
7668
7669 static int
7670 i40e_hash_filter_get(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
7671 {
7672         int ret = 0;
7673
7674         if (!hw || !info) {
7675                 PMD_DRV_LOG(ERR, "Invalid pointer");
7676                 return -EFAULT;
7677         }
7678
7679         switch (info->info_type) {
7680         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
7681                 i40e_get_symmetric_hash_enable_per_port(hw,
7682                                         &(info->info.enable));
7683                 break;
7684         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
7685                 ret = i40e_get_hash_filter_global_config(hw,
7686                                 &(info->info.global_conf));
7687                 break;
7688         default:
7689                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
7690                                                         info->info_type);
7691                 ret = -EINVAL;
7692                 break;
7693         }
7694
7695         return ret;
7696 }
7697
7698 static int
7699 i40e_hash_filter_set(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
7700 {
7701         int ret = 0;
7702
7703         if (!hw || !info) {
7704                 PMD_DRV_LOG(ERR, "Invalid pointer");
7705                 return -EFAULT;
7706         }
7707
7708         switch (info->info_type) {
7709         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
7710                 i40e_set_symmetric_hash_enable_per_port(hw, info->info.enable);
7711                 break;
7712         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
7713                 ret = i40e_set_hash_filter_global_config(hw,
7714                                 &(info->info.global_conf));
7715                 break;
7716         case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT:
7717                 ret = i40e_hash_filter_inset_select(hw,
7718                                                &(info->info.input_set_conf));
7719                 break;
7720
7721         default:
7722                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
7723                                                         info->info_type);
7724                 ret = -EINVAL;
7725                 break;
7726         }
7727
7728         return ret;
7729 }
7730
7731 /* Operations for hash function */
7732 static int
7733 i40e_hash_filter_ctrl(struct rte_eth_dev *dev,
7734                       enum rte_filter_op filter_op,
7735                       void *arg)
7736 {
7737         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7738         int ret = 0;
7739
7740         switch (filter_op) {
7741         case RTE_ETH_FILTER_NOP:
7742                 break;
7743         case RTE_ETH_FILTER_GET:
7744                 ret = i40e_hash_filter_get(hw,
7745                         (struct rte_eth_hash_filter_info *)arg);
7746                 break;
7747         case RTE_ETH_FILTER_SET:
7748                 ret = i40e_hash_filter_set(hw,
7749                         (struct rte_eth_hash_filter_info *)arg);
7750                 break;
7751         default:
7752                 PMD_DRV_LOG(WARNING, "Filter operation (%d) not supported",
7753                                                                 filter_op);
7754                 ret = -ENOTSUP;
7755                 break;
7756         }
7757
7758         return ret;
7759 }
7760
7761 /*
7762  * Configure ethertype filter, which can director packet by filtering
7763  * with mac address and ether_type or only ether_type
7764  */
7765 static int
7766 i40e_ethertype_filter_set(struct i40e_pf *pf,
7767                         struct rte_eth_ethertype_filter *filter,
7768                         bool add)
7769 {
7770         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7771         struct i40e_control_filter_stats stats;
7772         uint16_t flags = 0;
7773         int ret;
7774
7775         if (filter->queue >= pf->dev_data->nb_rx_queues) {
7776                 PMD_DRV_LOG(ERR, "Invalid queue ID");
7777                 return -EINVAL;
7778         }
7779         if (filter->ether_type == ETHER_TYPE_IPv4 ||
7780                 filter->ether_type == ETHER_TYPE_IPv6) {
7781                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
7782                         " control packet filter.", filter->ether_type);
7783                 return -EINVAL;
7784         }
7785         if (filter->ether_type == ETHER_TYPE_VLAN)
7786                 PMD_DRV_LOG(WARNING, "filter vlan ether_type in first tag is"
7787                         " not supported.");
7788
7789         if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC))
7790                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC;
7791         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP)
7792                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP;
7793         flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE;
7794
7795         memset(&stats, 0, sizeof(stats));
7796         ret = i40e_aq_add_rem_control_packet_filter(hw,
7797                         filter->mac_addr.addr_bytes,
7798                         filter->ether_type, flags,
7799                         pf->main_vsi->seid,
7800                         filter->queue, add, &stats, NULL);
7801
7802         PMD_DRV_LOG(INFO, "add/rem control packet filter, return %d,"
7803                          " mac_etype_used = %u, etype_used = %u,"
7804                          " mac_etype_free = %u, etype_free = %u\n",
7805                          ret, stats.mac_etype_used, stats.etype_used,
7806                          stats.mac_etype_free, stats.etype_free);
7807         if (ret < 0)
7808                 return -ENOSYS;
7809         return 0;
7810 }
7811
7812 /*
7813  * Handle operations for ethertype filter.
7814  */
7815 static int
7816 i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
7817                                 enum rte_filter_op filter_op,
7818                                 void *arg)
7819 {
7820         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
7821         int ret = 0;
7822
7823         if (filter_op == RTE_ETH_FILTER_NOP)
7824                 return ret;
7825
7826         if (arg == NULL) {
7827                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
7828                             filter_op);
7829                 return -EINVAL;
7830         }
7831
7832         switch (filter_op) {
7833         case RTE_ETH_FILTER_ADD:
7834                 ret = i40e_ethertype_filter_set(pf,
7835                         (struct rte_eth_ethertype_filter *)arg,
7836                         TRUE);
7837                 break;
7838         case RTE_ETH_FILTER_DELETE:
7839                 ret = i40e_ethertype_filter_set(pf,
7840                         (struct rte_eth_ethertype_filter *)arg,
7841                         FALSE);
7842                 break;
7843         default:
7844                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
7845                 ret = -ENOSYS;
7846                 break;
7847         }
7848         return ret;
7849 }
7850
7851 static int
7852 i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
7853                      enum rte_filter_type filter_type,
7854                      enum rte_filter_op filter_op,
7855                      void *arg)
7856 {
7857         int ret = 0;
7858
7859         if (dev == NULL)
7860                 return -EINVAL;
7861
7862         switch (filter_type) {
7863         case RTE_ETH_FILTER_NONE:
7864                 /* For global configuration */
7865                 ret = i40e_filter_ctrl_global_config(dev, filter_op, arg);
7866                 break;
7867         case RTE_ETH_FILTER_HASH:
7868                 ret = i40e_hash_filter_ctrl(dev, filter_op, arg);
7869                 break;
7870         case RTE_ETH_FILTER_MACVLAN:
7871                 ret = i40e_mac_filter_handle(dev, filter_op, arg);
7872                 break;
7873         case RTE_ETH_FILTER_ETHERTYPE:
7874                 ret = i40e_ethertype_filter_handle(dev, filter_op, arg);
7875                 break;
7876         case RTE_ETH_FILTER_TUNNEL:
7877                 ret = i40e_tunnel_filter_handle(dev, filter_op, arg);
7878                 break;
7879         case RTE_ETH_FILTER_FDIR:
7880                 ret = i40e_fdir_ctrl_func(dev, filter_op, arg);
7881                 break;
7882         default:
7883                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
7884                                                         filter_type);
7885                 ret = -EINVAL;
7886                 break;
7887         }
7888
7889         return ret;
7890 }
7891
7892 /*
7893  * Check and enable Extended Tag.
7894  * Enabling Extended Tag is important for 40G performance.
7895  */
7896 static void
7897 i40e_enable_extended_tag(struct rte_eth_dev *dev)
7898 {
7899         uint32_t buf = 0;
7900         int ret;
7901
7902         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
7903                                       PCI_DEV_CAP_REG);
7904         if (ret < 0) {
7905                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
7906                             PCI_DEV_CAP_REG);
7907                 return;
7908         }
7909         if (!(buf & PCI_DEV_CAP_EXT_TAG_MASK)) {
7910                 PMD_DRV_LOG(ERR, "Does not support Extended Tag");
7911                 return;
7912         }
7913
7914         buf = 0;
7915         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
7916                                       PCI_DEV_CTRL_REG);
7917         if (ret < 0) {
7918                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
7919                             PCI_DEV_CTRL_REG);
7920                 return;
7921         }
7922         if (buf & PCI_DEV_CTRL_EXT_TAG_MASK) {
7923                 PMD_DRV_LOG(DEBUG, "Extended Tag has already been enabled");
7924                 return;
7925         }
7926         buf |= PCI_DEV_CTRL_EXT_TAG_MASK;
7927         ret = rte_eal_pci_write_config(dev->pci_dev, &buf, sizeof(buf),
7928                                        PCI_DEV_CTRL_REG);
7929         if (ret < 0) {
7930                 PMD_DRV_LOG(ERR, "Failed to write PCI offset 0x%x",
7931                             PCI_DEV_CTRL_REG);
7932                 return;
7933         }
7934 }
7935
7936 /*
7937  * As some registers wouldn't be reset unless a global hardware reset,
7938  * hardware initialization is needed to put those registers into an
7939  * expected initial state.
7940  */
7941 static void
7942 i40e_hw_init(struct rte_eth_dev *dev)
7943 {
7944         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7945
7946         i40e_enable_extended_tag(dev);
7947
7948         /* clear the PF Queue Filter control register */
7949         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, 0);
7950
7951         /* Disable symmetric hash per port */
7952         i40e_set_symmetric_hash_enable_per_port(hw, 0);
7953 }
7954
7955 enum i40e_filter_pctype
7956 i40e_flowtype_to_pctype(uint16_t flow_type)
7957 {
7958         static const enum i40e_filter_pctype pctype_table[] = {
7959                 [RTE_ETH_FLOW_FRAG_IPV4] = I40E_FILTER_PCTYPE_FRAG_IPV4,
7960                 [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] =
7961                         I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
7962                 [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] =
7963                         I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
7964                 [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] =
7965                         I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
7966                 [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] =
7967                         I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
7968                 [RTE_ETH_FLOW_FRAG_IPV6] = I40E_FILTER_PCTYPE_FRAG_IPV6,
7969                 [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] =
7970                         I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
7971                 [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] =
7972                         I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
7973                 [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] =
7974                         I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
7975                 [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] =
7976                         I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
7977                 [RTE_ETH_FLOW_L2_PAYLOAD] = I40E_FILTER_PCTYPE_L2_PAYLOAD,
7978         };
7979
7980         return pctype_table[flow_type];
7981 }
7982
7983 uint16_t
7984 i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype)
7985 {
7986         static const uint16_t flowtype_table[] = {
7987                 [I40E_FILTER_PCTYPE_FRAG_IPV4] = RTE_ETH_FLOW_FRAG_IPV4,
7988                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7989                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
7990                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7991                         RTE_ETH_FLOW_NONFRAG_IPV4_TCP,
7992                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7993                         RTE_ETH_FLOW_NONFRAG_IPV4_SCTP,
7994                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7995                         RTE_ETH_FLOW_NONFRAG_IPV4_OTHER,
7996                 [I40E_FILTER_PCTYPE_FRAG_IPV6] = RTE_ETH_FLOW_FRAG_IPV6,
7997                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7998                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
7999                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
8000                         RTE_ETH_FLOW_NONFRAG_IPV6_TCP,
8001                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
8002                         RTE_ETH_FLOW_NONFRAG_IPV6_SCTP,
8003                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
8004                         RTE_ETH_FLOW_NONFRAG_IPV6_OTHER,
8005                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] = RTE_ETH_FLOW_L2_PAYLOAD,
8006         };
8007
8008         return flowtype_table[pctype];
8009 }
8010
8011 /*
8012  * On X710, performance number is far from the expectation on recent firmware
8013  * versions; on XL710, performance number is also far from the expectation on
8014  * recent firmware versions, if promiscuous mode is disabled, or promiscuous
8015  * mode is enabled and port MAC address is equal to the packet destination MAC
8016  * address. The fix for this issue may not be integrated in the following
8017  * firmware version. So the workaround in software driver is needed. It needs
8018  * to modify the initial values of 3 internal only registers for both X710 and
8019  * XL710. Note that the values for X710 or XL710 could be different, and the
8020  * workaround can be removed when it is fixed in firmware in the future.
8021  */
8022
8023 /* For both X710 and XL710 */
8024 #define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x10000200
8025 #define I40E_GL_SWR_PRI_JOIN_MAP_0       0x26CE00
8026
8027 #define I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x011f0200
8028 #define I40E_GL_SWR_PRI_JOIN_MAP_2       0x26CE08
8029
8030 /* For X710 */
8031 #define I40E_GL_SWR_PM_UP_THR_EF_VALUE   0x03030303
8032 /* For XL710 */
8033 #define I40E_GL_SWR_PM_UP_THR_SF_VALUE   0x06060606
8034 #define I40E_GL_SWR_PM_UP_THR            0x269FBC
8035
8036 static void
8037 i40e_configure_registers(struct i40e_hw *hw)
8038 {
8039         static struct {
8040                 uint32_t addr;
8041                 uint64_t val;
8042         } reg_table[] = {
8043                 {I40E_GL_SWR_PRI_JOIN_MAP_0, I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE},
8044                 {I40E_GL_SWR_PRI_JOIN_MAP_2, I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE},
8045                 {I40E_GL_SWR_PM_UP_THR, 0}, /* Compute value dynamically */
8046         };
8047         uint64_t reg;
8048         uint32_t i;
8049         int ret;
8050
8051         for (i = 0; i < RTE_DIM(reg_table); i++) {
8052                 if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) {
8053                         if (i40e_is_40G_device(hw->device_id)) /* For XL710 */
8054                                 reg_table[i].val =
8055                                         I40E_GL_SWR_PM_UP_THR_SF_VALUE;
8056                         else /* For X710 */
8057                                 reg_table[i].val =
8058                                         I40E_GL_SWR_PM_UP_THR_EF_VALUE;
8059                 }
8060
8061                 ret = i40e_aq_debug_read_register(hw, reg_table[i].addr,
8062                                                         &reg, NULL);
8063                 if (ret < 0) {
8064                         PMD_DRV_LOG(ERR, "Failed to read from 0x%"PRIx32,
8065                                                         reg_table[i].addr);
8066                         break;
8067                 }
8068                 PMD_DRV_LOG(DEBUG, "Read from 0x%"PRIx32": 0x%"PRIx64,
8069                                                 reg_table[i].addr, reg);
8070                 if (reg == reg_table[i].val)
8071                         continue;
8072
8073                 ret = i40e_aq_debug_write_register(hw, reg_table[i].addr,
8074                                                 reg_table[i].val, NULL);
8075                 if (ret < 0) {
8076                         PMD_DRV_LOG(ERR, "Failed to write 0x%"PRIx64" to the "
8077                                 "address of 0x%"PRIx32, reg_table[i].val,
8078                                                         reg_table[i].addr);
8079                         break;
8080                 }
8081                 PMD_DRV_LOG(DEBUG, "Write 0x%"PRIx64" to the address of "
8082                         "0x%"PRIx32, reg_table[i].val, reg_table[i].addr);
8083         }
8084 }
8085
8086 #define I40E_VSI_TSR(_i)            (0x00050800 + ((_i) * 4))
8087 #define I40E_VSI_TSR_QINQ_CONFIG    0xc030
8088 #define I40E_VSI_L2TAGSTXVALID(_i)  (0x00042800 + ((_i) * 4))
8089 #define I40E_VSI_L2TAGSTXVALID_QINQ 0xab
8090 static int
8091 i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi)
8092 {
8093         uint32_t reg;
8094         int ret;
8095
8096         if (vsi->vsi_id >= I40E_MAX_NUM_VSIS) {
8097                 PMD_DRV_LOG(ERR, "VSI ID exceeds the maximum");
8098                 return -EINVAL;
8099         }
8100
8101         /* Configure for double VLAN RX stripping */
8102         reg = I40E_READ_REG(hw, I40E_VSI_TSR(vsi->vsi_id));
8103         if ((reg & I40E_VSI_TSR_QINQ_CONFIG) != I40E_VSI_TSR_QINQ_CONFIG) {
8104                 reg |= I40E_VSI_TSR_QINQ_CONFIG;
8105                 ret = i40e_aq_debug_write_register(hw,
8106                                                    I40E_VSI_TSR(vsi->vsi_id),
8107                                                    reg, NULL);
8108                 if (ret < 0) {
8109                         PMD_DRV_LOG(ERR, "Failed to update VSI_TSR[%d]",
8110                                     vsi->vsi_id);
8111                         return I40E_ERR_CONFIG;
8112                 }
8113         }
8114
8115         /* Configure for double VLAN TX insertion */
8116         reg = I40E_READ_REG(hw, I40E_VSI_L2TAGSTXVALID(vsi->vsi_id));
8117         if ((reg & 0xff) != I40E_VSI_L2TAGSTXVALID_QINQ) {
8118                 reg = I40E_VSI_L2TAGSTXVALID_QINQ;
8119                 ret = i40e_aq_debug_write_register(hw,
8120                                                    I40E_VSI_L2TAGSTXVALID(
8121                                                    vsi->vsi_id), reg, NULL);
8122                 if (ret < 0) {
8123                         PMD_DRV_LOG(ERR, "Failed to update "
8124                                 "VSI_L2TAGSTXVALID[%d]", vsi->vsi_id);
8125                         return I40E_ERR_CONFIG;
8126                 }
8127         }
8128
8129         return 0;
8130 }
8131
8132 /**
8133  * i40e_aq_add_mirror_rule
8134  * @hw: pointer to the hardware structure
8135  * @seid: VEB seid to add mirror rule to
8136  * @dst_id: destination vsi seid
8137  * @entries: Buffer which contains the entities to be mirrored
8138  * @count: number of entities contained in the buffer
8139  * @rule_id:the rule_id of the rule to be added
8140  *
8141  * Add a mirror rule for a given veb.
8142  *
8143  **/
8144 static enum i40e_status_code
8145 i40e_aq_add_mirror_rule(struct i40e_hw *hw,
8146                         uint16_t seid, uint16_t dst_id,
8147                         uint16_t rule_type, uint16_t *entries,
8148                         uint16_t count, uint16_t *rule_id)
8149 {
8150         struct i40e_aq_desc desc;
8151         struct i40e_aqc_add_delete_mirror_rule cmd;
8152         struct i40e_aqc_add_delete_mirror_rule_completion *resp =
8153                 (struct i40e_aqc_add_delete_mirror_rule_completion *)
8154                 &desc.params.raw;
8155         uint16_t buff_len;
8156         enum i40e_status_code status;
8157
8158         i40e_fill_default_direct_cmd_desc(&desc,
8159                                           i40e_aqc_opc_add_mirror_rule);
8160         memset(&cmd, 0, sizeof(cmd));
8161
8162         buff_len = sizeof(uint16_t) * count;
8163         desc.datalen = rte_cpu_to_le_16(buff_len);
8164         if (buff_len > 0)
8165                 desc.flags |= rte_cpu_to_le_16(
8166                         (uint16_t)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD));
8167         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
8168                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
8169         cmd.num_entries = rte_cpu_to_le_16(count);
8170         cmd.seid = rte_cpu_to_le_16(seid);
8171         cmd.destination = rte_cpu_to_le_16(dst_id);
8172
8173         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
8174         status = i40e_asq_send_command(hw, &desc, entries, buff_len, NULL);
8175         PMD_DRV_LOG(INFO, "i40e_aq_add_mirror_rule, aq_status %d,"
8176                          "rule_id = %u"
8177                          " mirror_rules_used = %u, mirror_rules_free = %u,",
8178                          hw->aq.asq_last_status, resp->rule_id,
8179                          resp->mirror_rules_used, resp->mirror_rules_free);
8180         *rule_id = rte_le_to_cpu_16(resp->rule_id);
8181
8182         return status;
8183 }
8184
8185 /**
8186  * i40e_aq_del_mirror_rule
8187  * @hw: pointer to the hardware structure
8188  * @seid: VEB seid to add mirror rule to
8189  * @entries: Buffer which contains the entities to be mirrored
8190  * @count: number of entities contained in the buffer
8191  * @rule_id:the rule_id of the rule to be delete
8192  *
8193  * Delete a mirror rule for a given veb.
8194  *
8195  **/
8196 static enum i40e_status_code
8197 i40e_aq_del_mirror_rule(struct i40e_hw *hw,
8198                 uint16_t seid, uint16_t rule_type, uint16_t *entries,
8199                 uint16_t count, uint16_t rule_id)
8200 {
8201         struct i40e_aq_desc desc;
8202         struct i40e_aqc_add_delete_mirror_rule cmd;
8203         uint16_t buff_len = 0;
8204         enum i40e_status_code status;
8205         void *buff = NULL;
8206
8207         i40e_fill_default_direct_cmd_desc(&desc,
8208                                           i40e_aqc_opc_delete_mirror_rule);
8209         memset(&cmd, 0, sizeof(cmd));
8210         if (rule_type == I40E_AQC_MIRROR_RULE_TYPE_VLAN) {
8211                 desc.flags |= rte_cpu_to_le_16((uint16_t)(I40E_AQ_FLAG_BUF |
8212                                                           I40E_AQ_FLAG_RD));
8213                 cmd.num_entries = count;
8214                 buff_len = sizeof(uint16_t) * count;
8215                 desc.datalen = rte_cpu_to_le_16(buff_len);
8216                 buff = (void *)entries;
8217         } else
8218                 /* rule id is filled in destination field for deleting mirror rule */
8219                 cmd.destination = rte_cpu_to_le_16(rule_id);
8220
8221         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
8222                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
8223         cmd.seid = rte_cpu_to_le_16(seid);
8224
8225         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
8226         status = i40e_asq_send_command(hw, &desc, buff, buff_len, NULL);
8227
8228         return status;
8229 }
8230
8231 /**
8232  * i40e_mirror_rule_set
8233  * @dev: pointer to the hardware structure
8234  * @mirror_conf: mirror rule info
8235  * @sw_id: mirror rule's sw_id
8236  * @on: enable/disable
8237  *
8238  * set a mirror rule.
8239  *
8240  **/
8241 static int
8242 i40e_mirror_rule_set(struct rte_eth_dev *dev,
8243                         struct rte_eth_mirror_conf *mirror_conf,
8244                         uint8_t sw_id, uint8_t on)
8245 {
8246         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8247         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8248         struct i40e_mirror_rule *it, *mirr_rule = NULL;
8249         struct i40e_mirror_rule *parent = NULL;
8250         uint16_t seid, dst_seid, rule_id;
8251         uint16_t i, j = 0;
8252         int ret;
8253
8254         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_set: sw_id = %d.", sw_id);
8255
8256         if (pf->main_vsi->veb == NULL || pf->vfs == NULL) {
8257                 PMD_DRV_LOG(ERR, "mirror rule can not be configured"
8258                         " without veb or vfs.");
8259                 return -ENOSYS;
8260         }
8261         if (pf->nb_mirror_rule > I40E_MAX_MIRROR_RULES) {
8262                 PMD_DRV_LOG(ERR, "mirror table is full.");
8263                 return -ENOSPC;
8264         }
8265         if (mirror_conf->dst_pool > pf->vf_num) {
8266                 PMD_DRV_LOG(ERR, "invalid destination pool %u.",
8267                                  mirror_conf->dst_pool);
8268                 return -EINVAL;
8269         }
8270
8271         seid = pf->main_vsi->veb->seid;
8272
8273         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
8274                 if (sw_id <= it->index) {
8275                         mirr_rule = it;
8276                         break;
8277                 }
8278                 parent = it;
8279         }
8280         if (mirr_rule && sw_id == mirr_rule->index) {
8281                 if (on) {
8282                         PMD_DRV_LOG(ERR, "mirror rule exists.");
8283                         return -EEXIST;
8284                 } else {
8285                         ret = i40e_aq_del_mirror_rule(hw, seid,
8286                                         mirr_rule->rule_type,
8287                                         mirr_rule->entries,
8288                                         mirr_rule->num_entries, mirr_rule->id);
8289                         if (ret < 0) {
8290                                 PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
8291                                                    " ret = %d, aq_err = %d.",
8292                                                    ret, hw->aq.asq_last_status);
8293                                 return -ENOSYS;
8294                         }
8295                         TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
8296                         rte_free(mirr_rule);
8297                         pf->nb_mirror_rule--;
8298                         return 0;
8299                 }
8300         } else if (!on) {
8301                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
8302                 return -ENOENT;
8303         }
8304
8305         mirr_rule = rte_zmalloc("i40e_mirror_rule",
8306                                 sizeof(struct i40e_mirror_rule) , 0);
8307         if (!mirr_rule) {
8308                 PMD_DRV_LOG(ERR, "failed to allocate memory");
8309                 return I40E_ERR_NO_MEMORY;
8310         }
8311         switch (mirror_conf->rule_type) {
8312         case ETH_MIRROR_VLAN:
8313                 for (i = 0, j = 0; i < ETH_MIRROR_MAX_VLANS; i++) {
8314                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
8315                                 mirr_rule->entries[j] =
8316                                         mirror_conf->vlan.vlan_id[i];
8317                                 j++;
8318                         }
8319                 }
8320                 if (j == 0) {
8321                         PMD_DRV_LOG(ERR, "vlan is not specified.");
8322                         rte_free(mirr_rule);
8323                         return -EINVAL;
8324                 }
8325                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_VLAN;
8326                 break;
8327         case ETH_MIRROR_VIRTUAL_POOL_UP:
8328         case ETH_MIRROR_VIRTUAL_POOL_DOWN:
8329                 /* check if the specified pool bit is out of range */
8330                 if (mirror_conf->pool_mask > (uint64_t)(1ULL << (pf->vf_num + 1))) {
8331                         PMD_DRV_LOG(ERR, "pool mask is out of range.");
8332                         rte_free(mirr_rule);
8333                         return -EINVAL;
8334                 }
8335                 for (i = 0, j = 0; i < pf->vf_num; i++) {
8336                         if (mirror_conf->pool_mask & (1ULL << i)) {
8337                                 mirr_rule->entries[j] = pf->vfs[i].vsi->seid;
8338                                 j++;
8339                         }
8340                 }
8341                 if (mirror_conf->pool_mask & (1ULL << pf->vf_num)) {
8342                         /* add pf vsi to entries */
8343                         mirr_rule->entries[j] = pf->main_vsi_seid;
8344                         j++;
8345                 }
8346                 if (j == 0) {
8347                         PMD_DRV_LOG(ERR, "pool is not specified.");
8348                         rte_free(mirr_rule);
8349                         return -EINVAL;
8350                 }
8351                 /* egress and ingress in aq commands means from switch but not port */
8352                 mirr_rule->rule_type =
8353                         (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) ?
8354                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_EGRESS :
8355                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_INGRESS;
8356                 break;
8357         case ETH_MIRROR_UPLINK_PORT:
8358                 /* egress and ingress in aq commands means from switch but not port*/
8359                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_EGRESS;
8360                 break;
8361         case ETH_MIRROR_DOWNLINK_PORT:
8362                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_INGRESS;
8363                 break;
8364         default:
8365                 PMD_DRV_LOG(ERR, "unsupported mirror type %d.",
8366                         mirror_conf->rule_type);
8367                 rte_free(mirr_rule);
8368                 return -EINVAL;
8369         }
8370
8371         /* If the dst_pool is equal to vf_num, consider it as PF */
8372         if (mirror_conf->dst_pool == pf->vf_num)
8373                 dst_seid = pf->main_vsi_seid;
8374         else
8375                 dst_seid = pf->vfs[mirror_conf->dst_pool].vsi->seid;
8376
8377         ret = i40e_aq_add_mirror_rule(hw, seid, dst_seid,
8378                                       mirr_rule->rule_type, mirr_rule->entries,
8379                                       j, &rule_id);
8380         if (ret < 0) {
8381                 PMD_DRV_LOG(ERR, "failed to add mirror rule:"
8382                                    " ret = %d, aq_err = %d.",
8383                                    ret, hw->aq.asq_last_status);
8384                 rte_free(mirr_rule);
8385                 return -ENOSYS;
8386         }
8387
8388         mirr_rule->index = sw_id;
8389         mirr_rule->num_entries = j;
8390         mirr_rule->id = rule_id;
8391         mirr_rule->dst_vsi_seid = dst_seid;
8392
8393         if (parent)
8394                 TAILQ_INSERT_AFTER(&pf->mirror_list, parent, mirr_rule, rules);
8395         else
8396                 TAILQ_INSERT_HEAD(&pf->mirror_list, mirr_rule, rules);
8397
8398         pf->nb_mirror_rule++;
8399         return 0;
8400 }
8401
8402 /**
8403  * i40e_mirror_rule_reset
8404  * @dev: pointer to the device
8405  * @sw_id: mirror rule's sw_id
8406  *
8407  * reset a mirror rule.
8408  *
8409  **/
8410 static int
8411 i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id)
8412 {
8413         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8414         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8415         struct i40e_mirror_rule *it, *mirr_rule = NULL;
8416         uint16_t seid;
8417         int ret;
8418
8419         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_reset: sw_id = %d.", sw_id);
8420
8421         seid = pf->main_vsi->veb->seid;
8422
8423         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
8424                 if (sw_id == it->index) {
8425                         mirr_rule = it;
8426                         break;
8427                 }
8428         }
8429         if (mirr_rule) {
8430                 ret = i40e_aq_del_mirror_rule(hw, seid,
8431                                 mirr_rule->rule_type,
8432                                 mirr_rule->entries,
8433                                 mirr_rule->num_entries, mirr_rule->id);
8434                 if (ret < 0) {
8435                         PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
8436                                            " status = %d, aq_err = %d.",
8437                                            ret, hw->aq.asq_last_status);
8438                         return -ENOSYS;
8439                 }
8440                 TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
8441                 rte_free(mirr_rule);
8442                 pf->nb_mirror_rule--;
8443         } else {
8444                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
8445                 return -ENOENT;
8446         }
8447         return 0;
8448 }
8449
8450 static uint64_t
8451 i40e_read_systime_cyclecounter(struct rte_eth_dev *dev)
8452 {
8453         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8454         uint64_t systim_cycles;
8455
8456         systim_cycles = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_L);
8457         systim_cycles |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_H)
8458                         << 32;
8459
8460         return systim_cycles;
8461 }
8462
8463 static uint64_t
8464 i40e_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev, uint8_t index)
8465 {
8466         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8467         uint64_t rx_tstamp;
8468
8469         rx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(index));
8470         rx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(index))
8471                         << 32;
8472
8473         return rx_tstamp;
8474 }
8475
8476 static uint64_t
8477 i40e_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
8478 {
8479         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8480         uint64_t tx_tstamp;
8481
8482         tx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_L);
8483         tx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H)
8484                         << 32;
8485
8486         return tx_tstamp;
8487 }
8488
8489 static void
8490 i40e_start_timecounters(struct rte_eth_dev *dev)
8491 {
8492         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8493         struct i40e_adapter *adapter =
8494                         (struct i40e_adapter *)dev->data->dev_private;
8495         struct rte_eth_link link;
8496         uint32_t tsync_inc_l;
8497         uint32_t tsync_inc_h;
8498
8499         /* Get current link speed. */
8500         memset(&link, 0, sizeof(link));
8501         i40e_dev_link_update(dev, 1);
8502         rte_i40e_dev_atomic_read_link_status(dev, &link);
8503
8504         switch (link.link_speed) {
8505         case ETH_SPEED_NUM_40G:
8506                 tsync_inc_l = I40E_PTP_40GB_INCVAL & 0xFFFFFFFF;
8507                 tsync_inc_h = I40E_PTP_40GB_INCVAL >> 32;
8508                 break;
8509         case ETH_SPEED_NUM_10G:
8510                 tsync_inc_l = I40E_PTP_10GB_INCVAL & 0xFFFFFFFF;
8511                 tsync_inc_h = I40E_PTP_10GB_INCVAL >> 32;
8512                 break;
8513         case ETH_SPEED_NUM_1G:
8514                 tsync_inc_l = I40E_PTP_1GB_INCVAL & 0xFFFFFFFF;
8515                 tsync_inc_h = I40E_PTP_1GB_INCVAL >> 32;
8516                 break;
8517         default:
8518                 tsync_inc_l = 0x0;
8519                 tsync_inc_h = 0x0;
8520         }
8521
8522         /* Set the timesync increment value. */
8523         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, tsync_inc_l);
8524         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, tsync_inc_h);
8525
8526         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
8527         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
8528         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
8529
8530         adapter->systime_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8531         adapter->systime_tc.cc_shift = 0;
8532         adapter->systime_tc.nsec_mask = 0;
8533
8534         adapter->rx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8535         adapter->rx_tstamp_tc.cc_shift = 0;
8536         adapter->rx_tstamp_tc.nsec_mask = 0;
8537
8538         adapter->tx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8539         adapter->tx_tstamp_tc.cc_shift = 0;
8540         adapter->tx_tstamp_tc.nsec_mask = 0;
8541 }
8542
8543 static int
8544 i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
8545 {
8546         struct i40e_adapter *adapter =
8547                         (struct i40e_adapter *)dev->data->dev_private;
8548
8549         adapter->systime_tc.nsec += delta;
8550         adapter->rx_tstamp_tc.nsec += delta;
8551         adapter->tx_tstamp_tc.nsec += delta;
8552
8553         return 0;
8554 }
8555
8556 static int
8557 i40e_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
8558 {
8559         uint64_t ns;
8560         struct i40e_adapter *adapter =
8561                         (struct i40e_adapter *)dev->data->dev_private;
8562
8563         ns = rte_timespec_to_ns(ts);
8564
8565         /* Set the timecounters to a new value. */
8566         adapter->systime_tc.nsec = ns;
8567         adapter->rx_tstamp_tc.nsec = ns;
8568         adapter->tx_tstamp_tc.nsec = ns;
8569
8570         return 0;
8571 }
8572
8573 static int
8574 i40e_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
8575 {
8576         uint64_t ns, systime_cycles;
8577         struct i40e_adapter *adapter =
8578                         (struct i40e_adapter *)dev->data->dev_private;
8579
8580         systime_cycles = i40e_read_systime_cyclecounter(dev);
8581         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
8582         *ts = rte_ns_to_timespec(ns);
8583
8584         return 0;
8585 }
8586
8587 static int
8588 i40e_timesync_enable(struct rte_eth_dev *dev)
8589 {
8590         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8591         uint32_t tsync_ctl_l;
8592         uint32_t tsync_ctl_h;
8593
8594         /* Stop the timesync system time. */
8595         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
8596         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
8597         /* Reset the timesync system time value. */
8598         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_L, 0x0);
8599         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_H, 0x0);
8600
8601         i40e_start_timecounters(dev);
8602
8603         /* Clear timesync registers. */
8604         I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
8605         I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H);
8606         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(0));
8607         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(1));
8608         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(2));
8609         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(3));
8610
8611         /* Enable timestamping of PTP packets. */
8612         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
8613         tsync_ctl_l |= I40E_PRTTSYN_TSYNENA;
8614
8615         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
8616         tsync_ctl_h |= I40E_PRTTSYN_TSYNENA;
8617         tsync_ctl_h |= I40E_PRTTSYN_TSYNTYPE;
8618
8619         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
8620         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
8621
8622         return 0;
8623 }
8624
8625 static int
8626 i40e_timesync_disable(struct rte_eth_dev *dev)
8627 {
8628         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8629         uint32_t tsync_ctl_l;
8630         uint32_t tsync_ctl_h;
8631
8632         /* Disable timestamping of transmitted PTP packets. */
8633         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
8634         tsync_ctl_l &= ~I40E_PRTTSYN_TSYNENA;
8635
8636         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
8637         tsync_ctl_h &= ~I40E_PRTTSYN_TSYNENA;
8638
8639         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
8640         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
8641
8642         /* Reset the timesync increment value. */
8643         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
8644         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
8645
8646         return 0;
8647 }
8648
8649 static int
8650 i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
8651                                 struct timespec *timestamp, uint32_t flags)
8652 {
8653         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8654         struct i40e_adapter *adapter =
8655                 (struct i40e_adapter *)dev->data->dev_private;
8656
8657         uint32_t sync_status;
8658         uint32_t index = flags & 0x03;
8659         uint64_t rx_tstamp_cycles;
8660         uint64_t ns;
8661
8662         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_1);
8663         if ((sync_status & (1 << index)) == 0)
8664                 return -EINVAL;
8665
8666         rx_tstamp_cycles = i40e_read_rx_tstamp_cyclecounter(dev, index);
8667         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
8668         *timestamp = rte_ns_to_timespec(ns);
8669
8670         return 0;
8671 }
8672
8673 static int
8674 i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
8675                                 struct timespec *timestamp)
8676 {
8677         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8678         struct i40e_adapter *adapter =
8679                 (struct i40e_adapter *)dev->data->dev_private;
8680
8681         uint32_t sync_status;
8682         uint64_t tx_tstamp_cycles;
8683         uint64_t ns;
8684
8685         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
8686         if ((sync_status & I40E_PRTTSYN_STAT_0_TXTIME_MASK) == 0)
8687                 return -EINVAL;
8688
8689         tx_tstamp_cycles = i40e_read_tx_tstamp_cyclecounter(dev);
8690         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
8691         *timestamp = rte_ns_to_timespec(ns);
8692
8693         return 0;
8694 }
8695
8696 /*
8697  * i40e_parse_dcb_configure - parse dcb configure from user
8698  * @dev: the device being configured
8699  * @dcb_cfg: pointer of the result of parse
8700  * @*tc_map: bit map of enabled traffic classes
8701  *
8702  * Returns 0 on success, negative value on failure
8703  */
8704 static int
8705 i40e_parse_dcb_configure(struct rte_eth_dev *dev,
8706                          struct i40e_dcbx_config *dcb_cfg,
8707                          uint8_t *tc_map)
8708 {
8709         struct rte_eth_dcb_rx_conf *dcb_rx_conf;
8710         uint8_t i, tc_bw, bw_lf;
8711
8712         memset(dcb_cfg, 0, sizeof(struct i40e_dcbx_config));
8713
8714         dcb_rx_conf = &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
8715         if (dcb_rx_conf->nb_tcs > I40E_MAX_TRAFFIC_CLASS) {
8716                 PMD_INIT_LOG(ERR, "number of tc exceeds max.");
8717                 return -EINVAL;
8718         }
8719
8720         /* assume each tc has the same bw */
8721         tc_bw = I40E_MAX_PERCENT / dcb_rx_conf->nb_tcs;
8722         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
8723                 dcb_cfg->etscfg.tcbwtable[i] = tc_bw;
8724         /* to ensure the sum of tcbw is equal to 100 */
8725         bw_lf = I40E_MAX_PERCENT % dcb_rx_conf->nb_tcs;
8726         for (i = 0; i < bw_lf; i++)
8727                 dcb_cfg->etscfg.tcbwtable[i]++;
8728
8729         /* assume each tc has the same Transmission Selection Algorithm */
8730         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
8731                 dcb_cfg->etscfg.tsatable[i] = I40E_IEEE_TSA_ETS;
8732
8733         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
8734                 dcb_cfg->etscfg.prioritytable[i] =
8735                                 dcb_rx_conf->dcb_tc[i];
8736
8737         /* FW needs one App to configure HW */
8738         dcb_cfg->numapps = I40E_DEFAULT_DCB_APP_NUM;
8739         dcb_cfg->app[0].selector = I40E_APP_SEL_ETHTYPE;
8740         dcb_cfg->app[0].priority = I40E_DEFAULT_DCB_APP_PRIO;
8741         dcb_cfg->app[0].protocolid = I40E_APP_PROTOID_FCOE;
8742
8743         if (dcb_rx_conf->nb_tcs == 0)
8744                 *tc_map = 1; /* tc0 only */
8745         else
8746                 *tc_map = RTE_LEN2MASK(dcb_rx_conf->nb_tcs, uint8_t);
8747
8748         if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
8749                 dcb_cfg->pfc.willing = 0;
8750                 dcb_cfg->pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
8751                 dcb_cfg->pfc.pfcenable = *tc_map;
8752         }
8753         return 0;
8754 }
8755
8756
8757 static enum i40e_status_code
8758 i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi,
8759                               struct i40e_aqc_vsi_properties_data *info,
8760                               uint8_t enabled_tcmap)
8761 {
8762         enum i40e_status_code ret;
8763         int i, total_tc = 0;
8764         uint16_t qpnum_per_tc, bsf, qp_idx;
8765         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
8766         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
8767         uint16_t used_queues;
8768
8769         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
8770         if (ret != I40E_SUCCESS)
8771                 return ret;
8772
8773         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8774                 if (enabled_tcmap & (1 << i))
8775                         total_tc++;
8776         }
8777         if (total_tc == 0)
8778                 total_tc = 1;
8779         vsi->enabled_tc = enabled_tcmap;
8780
8781         /* different VSI has different queues assigned */
8782         if (vsi->type == I40E_VSI_MAIN)
8783                 used_queues = dev_data->nb_rx_queues -
8784                         pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
8785         else if (vsi->type == I40E_VSI_VMDQ2)
8786                 used_queues = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
8787         else {
8788                 PMD_INIT_LOG(ERR, "unsupported VSI type.");
8789                 return I40E_ERR_NO_AVAILABLE_VSI;
8790         }
8791
8792         qpnum_per_tc = used_queues / total_tc;
8793         /* Number of queues per enabled TC */
8794         if (qpnum_per_tc == 0) {
8795                 PMD_INIT_LOG(ERR, " number of queues is less that tcs.");
8796                 return I40E_ERR_INVALID_QP_ID;
8797         }
8798         qpnum_per_tc = RTE_MIN(i40e_align_floor(qpnum_per_tc),
8799                                 I40E_MAX_Q_PER_TC);
8800         bsf = rte_bsf32(qpnum_per_tc);
8801
8802         /**
8803          * Configure TC and queue mapping parameters, for enabled TC,
8804          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
8805          * default queue will serve it.
8806          */
8807         qp_idx = 0;
8808         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8809                 if (vsi->enabled_tc & (1 << i)) {
8810                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
8811                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
8812                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
8813                         qp_idx += qpnum_per_tc;
8814                 } else
8815                         info->tc_mapping[i] = 0;
8816         }
8817
8818         /* Associate queue number with VSI, Keep vsi->nb_qps unchanged */
8819         if (vsi->type == I40E_VSI_SRIOV) {
8820                 info->mapping_flags |=
8821                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
8822                 for (i = 0; i < vsi->nb_qps; i++)
8823                         info->queue_mapping[i] =
8824                                 rte_cpu_to_le_16(vsi->base_queue + i);
8825         } else {
8826                 info->mapping_flags |=
8827                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
8828                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
8829         }
8830         info->valid_sections |=
8831                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
8832
8833         return I40E_SUCCESS;
8834 }
8835
8836 /*
8837  * i40e_config_switch_comp_tc - Configure VEB tc setting for given TC map
8838  * @veb: VEB to be configured
8839  * @tc_map: enabled TC bitmap
8840  *
8841  * Returns 0 on success, negative value on failure
8842  */
8843 static enum i40e_status_code
8844 i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map)
8845 {
8846         struct i40e_aqc_configure_switching_comp_bw_config_data veb_bw;
8847         struct i40e_aqc_query_switching_comp_bw_config_resp bw_query;
8848         struct i40e_aqc_query_switching_comp_ets_config_resp ets_query;
8849         struct i40e_hw *hw = I40E_VSI_TO_HW(veb->associate_vsi);
8850         enum i40e_status_code ret = I40E_SUCCESS;
8851         int i;
8852         uint32_t bw_max;
8853
8854         /* Check if enabled_tc is same as existing or new TCs */
8855         if (veb->enabled_tc == tc_map)
8856                 return ret;
8857
8858         /* configure tc bandwidth */
8859         memset(&veb_bw, 0, sizeof(veb_bw));
8860         veb_bw.tc_valid_bits = tc_map;
8861         /* Enable ETS TCs with equal BW Share for now across all VSIs */
8862         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8863                 if (tc_map & BIT_ULL(i))
8864                         veb_bw.tc_bw_share_credits[i] = 1;
8865         }
8866         ret = i40e_aq_config_switch_comp_bw_config(hw, veb->seid,
8867                                                    &veb_bw, NULL);
8868         if (ret) {
8869                 PMD_INIT_LOG(ERR, "AQ command Config switch_comp BW allocation"
8870                                   " per TC failed = %d",
8871                                   hw->aq.asq_last_status);
8872                 return ret;
8873         }
8874
8875         memset(&ets_query, 0, sizeof(ets_query));
8876         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
8877                                                    &ets_query, NULL);
8878         if (ret != I40E_SUCCESS) {
8879                 PMD_DRV_LOG(ERR, "Failed to get switch_comp ETS"
8880                                  " configuration %u", hw->aq.asq_last_status);
8881                 return ret;
8882         }
8883         memset(&bw_query, 0, sizeof(bw_query));
8884         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
8885                                                   &bw_query, NULL);
8886         if (ret != I40E_SUCCESS) {
8887                 PMD_DRV_LOG(ERR, "Failed to get switch_comp bandwidth"
8888                                  " configuration %u", hw->aq.asq_last_status);
8889                 return ret;
8890         }
8891
8892         /* store and print out BW info */
8893         veb->bw_info.bw_limit = rte_le_to_cpu_16(ets_query.port_bw_limit);
8894         veb->bw_info.bw_max = ets_query.tc_bw_max;
8895         PMD_DRV_LOG(DEBUG, "switch_comp bw limit:%u", veb->bw_info.bw_limit);
8896         PMD_DRV_LOG(DEBUG, "switch_comp max_bw:%u", veb->bw_info.bw_max);
8897         bw_max = rte_le_to_cpu_16(bw_query.tc_bw_max[0]) |
8898                     (rte_le_to_cpu_16(bw_query.tc_bw_max[1]) <<
8899                      I40E_16_BIT_WIDTH);
8900         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8901                 veb->bw_info.bw_ets_share_credits[i] =
8902                                 bw_query.tc_bw_share_credits[i];
8903                 veb->bw_info.bw_ets_credits[i] =
8904                                 rte_le_to_cpu_16(bw_query.tc_bw_limits[i]);
8905                 /* 4 bits per TC, 4th bit is reserved */
8906                 veb->bw_info.bw_ets_max[i] =
8907                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
8908                                   RTE_LEN2MASK(3, uint8_t));
8909                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:share credits %u", i,
8910                             veb->bw_info.bw_ets_share_credits[i]);
8911                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:credits %u", i,
8912                             veb->bw_info.bw_ets_credits[i]);
8913                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u: max credits: %u", i,
8914                             veb->bw_info.bw_ets_max[i]);
8915         }
8916
8917         veb->enabled_tc = tc_map;
8918
8919         return ret;
8920 }
8921
8922
8923 /*
8924  * i40e_vsi_config_tc - Configure VSI tc setting for given TC map
8925  * @vsi: VSI to be configured
8926  * @tc_map: enabled TC bitmap
8927  *
8928  * Returns 0 on success, negative value on failure
8929  */
8930 static enum i40e_status_code
8931 i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map)
8932 {
8933         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
8934         struct i40e_vsi_context ctxt;
8935         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
8936         enum i40e_status_code ret = I40E_SUCCESS;
8937         int i;
8938
8939         /* Check if enabled_tc is same as existing or new TCs */
8940         if (vsi->enabled_tc == tc_map)
8941                 return ret;
8942
8943         /* configure tc bandwidth */
8944         memset(&bw_data, 0, sizeof(bw_data));
8945         bw_data.tc_valid_bits = tc_map;
8946         /* Enable ETS TCs with equal BW Share for now across all VSIs */
8947         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8948                 if (tc_map & BIT_ULL(i))
8949                         bw_data.tc_bw_credits[i] = 1;
8950         }
8951         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &bw_data, NULL);
8952         if (ret) {
8953                 PMD_INIT_LOG(ERR, "AQ command Config VSI BW allocation"
8954                         " per TC failed = %d",
8955                         hw->aq.asq_last_status);
8956                 goto out;
8957         }
8958         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
8959                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
8960
8961         /* Update Queue Pairs Mapping for currently enabled UPs */
8962         ctxt.seid = vsi->seid;
8963         ctxt.pf_num = hw->pf_id;
8964         ctxt.vf_num = 0;
8965         ctxt.uplink_seid = vsi->uplink_seid;
8966         ctxt.info = vsi->info;
8967         i40e_get_cap(hw);
8968         ret = i40e_vsi_update_queue_mapping(vsi, &ctxt.info, tc_map);
8969         if (ret)
8970                 goto out;
8971
8972         /* Update the VSI after updating the VSI queue-mapping information */
8973         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
8974         if (ret) {
8975                 PMD_INIT_LOG(ERR, "Failed to configure "
8976                             "TC queue mapping = %d",
8977                             hw->aq.asq_last_status);
8978                 goto out;
8979         }
8980         /* update the local VSI info with updated queue map */
8981         (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
8982                                         sizeof(vsi->info.tc_mapping));
8983         (void)rte_memcpy(&vsi->info.queue_mapping,
8984                         &ctxt.info.queue_mapping,
8985                 sizeof(vsi->info.queue_mapping));
8986         vsi->info.mapping_flags = ctxt.info.mapping_flags;
8987         vsi->info.valid_sections = 0;
8988
8989         /* query and update current VSI BW information */
8990         ret = i40e_vsi_get_bw_config(vsi);
8991         if (ret) {
8992                 PMD_INIT_LOG(ERR,
8993                          "Failed updating vsi bw info, err %s aq_err %s",
8994                          i40e_stat_str(hw, ret),
8995                          i40e_aq_str(hw, hw->aq.asq_last_status));
8996                 goto out;
8997         }
8998
8999         vsi->enabled_tc = tc_map;
9000
9001 out:
9002         return ret;
9003 }
9004
9005 /*
9006  * i40e_dcb_hw_configure - program the dcb setting to hw
9007  * @pf: pf the configuration is taken on
9008  * @new_cfg: new configuration
9009  * @tc_map: enabled TC bitmap
9010  *
9011  * Returns 0 on success, negative value on failure
9012  */
9013 static enum i40e_status_code
9014 i40e_dcb_hw_configure(struct i40e_pf *pf,
9015                       struct i40e_dcbx_config *new_cfg,
9016                       uint8_t tc_map)
9017 {
9018         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
9019         struct i40e_dcbx_config *old_cfg = &hw->local_dcbx_config;
9020         struct i40e_vsi *main_vsi = pf->main_vsi;
9021         struct i40e_vsi_list *vsi_list;
9022         enum i40e_status_code ret;
9023         int i;
9024         uint32_t val;
9025
9026         /* Use the FW API if FW > v4.4*/
9027         if (!(((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4)) ||
9028               (hw->aq.fw_maj_ver >= 5))) {
9029                 PMD_INIT_LOG(ERR, "FW < v4.4, can not use FW LLDP API"
9030                                   " to configure DCB");
9031                 return I40E_ERR_FIRMWARE_API_VERSION;
9032         }
9033
9034         /* Check if need reconfiguration */
9035         if (!memcmp(new_cfg, old_cfg, sizeof(struct i40e_dcbx_config))) {
9036                 PMD_INIT_LOG(ERR, "No Change in DCB Config required.");
9037                 return I40E_SUCCESS;
9038         }
9039
9040         /* Copy the new config to the current config */
9041         *old_cfg = *new_cfg;
9042         old_cfg->etsrec = old_cfg->etscfg;
9043         ret = i40e_set_dcb_config(hw);
9044         if (ret) {
9045                 PMD_INIT_LOG(ERR,
9046                          "Set DCB Config failed, err %s aq_err %s\n",
9047                          i40e_stat_str(hw, ret),
9048                          i40e_aq_str(hw, hw->aq.asq_last_status));
9049                 return ret;
9050         }
9051         /* set receive Arbiter to RR mode and ETS scheme by default */
9052         for (i = 0; i <= I40E_PRTDCB_RETSTCC_MAX_INDEX; i++) {
9053                 val = I40E_READ_REG(hw, I40E_PRTDCB_RETSTCC(i));
9054                 val &= ~(I40E_PRTDCB_RETSTCC_BWSHARE_MASK     |
9055                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK |
9056                          I40E_PRTDCB_RETSTCC_ETSTC_SHIFT);
9057                 val |= ((uint32_t)old_cfg->etscfg.tcbwtable[i] <<
9058                         I40E_PRTDCB_RETSTCC_BWSHARE_SHIFT) &
9059                          I40E_PRTDCB_RETSTCC_BWSHARE_MASK;
9060                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_UPINTC_MODE_SHIFT) &
9061                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK;
9062                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_ETSTC_SHIFT) &
9063                          I40E_PRTDCB_RETSTCC_ETSTC_MASK;
9064                 I40E_WRITE_REG(hw, I40E_PRTDCB_RETSTCC(i), val);
9065         }
9066         /* get local mib to check whether it is configured correctly */
9067         /* IEEE mode */
9068         hw->local_dcbx_config.dcbx_mode = I40E_DCBX_MODE_IEEE;
9069         /* Get Local DCB Config */
9070         i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_LOCAL, 0,
9071                                      &hw->local_dcbx_config);
9072
9073         /* if Veb is created, need to update TC of it at first */
9074         if (main_vsi->veb) {
9075                 ret = i40e_config_switch_comp_tc(main_vsi->veb, tc_map);
9076                 if (ret)
9077                         PMD_INIT_LOG(WARNING,
9078                                  "Failed configuring TC for VEB seid=%d\n",
9079                                  main_vsi->veb->seid);
9080         }
9081         /* Update each VSI */
9082         i40e_vsi_config_tc(main_vsi, tc_map);
9083         if (main_vsi->veb) {
9084                 TAILQ_FOREACH(vsi_list, &main_vsi->veb->head, list) {
9085                         /* Beside main VSI and VMDQ VSIs, only enable default
9086                          * TC for other VSIs
9087                          */
9088                         if (vsi_list->vsi->type == I40E_VSI_VMDQ2)
9089                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
9090                                                          tc_map);
9091                         else
9092                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
9093                                                          I40E_DEFAULT_TCMAP);
9094                         if (ret)
9095                                 PMD_INIT_LOG(WARNING,
9096                                          "Failed configuring TC for VSI seid=%d\n",
9097                                          vsi_list->vsi->seid);
9098                         /* continue */
9099                 }
9100         }
9101         return I40E_SUCCESS;
9102 }
9103
9104 /*
9105  * i40e_dcb_init_configure - initial dcb config
9106  * @dev: device being configured
9107  * @sw_dcb: indicate whether dcb is sw configured or hw offload
9108  *
9109  * Returns 0 on success, negative value on failure
9110  */
9111 static int
9112 i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb)
9113 {
9114         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9115         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9116         int ret = 0;
9117
9118         if ((pf->flags & I40E_FLAG_DCB) == 0) {
9119                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
9120                 return -ENOTSUP;
9121         }
9122
9123         /* DCB initialization:
9124          * Update DCB configuration from the Firmware and configure
9125          * LLDP MIB change event.
9126          */
9127         if (sw_dcb == TRUE) {
9128                 ret = i40e_aq_stop_lldp(hw, TRUE, NULL);
9129                 if (ret != I40E_SUCCESS)
9130                         PMD_INIT_LOG(DEBUG, "Failed to stop lldp");
9131
9132                 ret = i40e_init_dcb(hw);
9133                 /* if sw_dcb, lldp agent is stopped, the return from
9134                  * i40e_init_dcb we expect is failure with I40E_AQ_RC_EPERM
9135                  * adminq status.
9136                  */
9137                 if (ret != I40E_SUCCESS &&
9138                     hw->aq.asq_last_status == I40E_AQ_RC_EPERM) {
9139                         memset(&hw->local_dcbx_config, 0,
9140                                 sizeof(struct i40e_dcbx_config));
9141                         /* set dcb default configuration */
9142                         hw->local_dcbx_config.etscfg.willing = 0;
9143                         hw->local_dcbx_config.etscfg.maxtcs = 0;
9144                         hw->local_dcbx_config.etscfg.tcbwtable[0] = 100;
9145                         hw->local_dcbx_config.etscfg.tsatable[0] =
9146                                                 I40E_IEEE_TSA_ETS;
9147                         hw->local_dcbx_config.etsrec =
9148                                 hw->local_dcbx_config.etscfg;
9149                         hw->local_dcbx_config.pfc.willing = 0;
9150                         hw->local_dcbx_config.pfc.pfccap =
9151                                                 I40E_MAX_TRAFFIC_CLASS;
9152                         /* FW needs one App to configure HW */
9153                         hw->local_dcbx_config.numapps = 1;
9154                         hw->local_dcbx_config.app[0].selector =
9155                                                 I40E_APP_SEL_ETHTYPE;
9156                         hw->local_dcbx_config.app[0].priority = 3;
9157                         hw->local_dcbx_config.app[0].protocolid =
9158                                                 I40E_APP_PROTOID_FCOE;
9159                         ret = i40e_set_dcb_config(hw);
9160                         if (ret) {
9161                                 PMD_INIT_LOG(ERR, "default dcb config fails."
9162                                         " err = %d, aq_err = %d.", ret,
9163                                           hw->aq.asq_last_status);
9164                                 return -ENOSYS;
9165                         }
9166                 } else {
9167                         PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d,"
9168                                           " aq_err = %d.", ret,
9169                                           hw->aq.asq_last_status);
9170                         return -ENOTSUP;
9171                 }
9172         } else {
9173                 ret = i40e_aq_start_lldp(hw, NULL);
9174                 if (ret != I40E_SUCCESS)
9175                         PMD_INIT_LOG(DEBUG, "Failed to start lldp");
9176
9177                 ret = i40e_init_dcb(hw);
9178                 if (!ret) {
9179                         if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) {
9180                                 PMD_INIT_LOG(ERR, "HW doesn't support"
9181                                                   " DCBX offload.");
9182                                 return -ENOTSUP;
9183                         }
9184                 } else {
9185                         PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d,"
9186                                           " aq_err = %d.", ret,
9187                                           hw->aq.asq_last_status);
9188                         return -ENOTSUP;
9189                 }
9190         }
9191         return 0;
9192 }
9193
9194 /*
9195  * i40e_dcb_setup - setup dcb related config
9196  * @dev: device being configured
9197  *
9198  * Returns 0 on success, negative value on failure
9199  */
9200 static int
9201 i40e_dcb_setup(struct rte_eth_dev *dev)
9202 {
9203         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9204         struct i40e_dcbx_config dcb_cfg;
9205         uint8_t tc_map = 0;
9206         int ret = 0;
9207
9208         if ((pf->flags & I40E_FLAG_DCB) == 0) {
9209                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
9210                 return -ENOTSUP;
9211         }
9212
9213         if (pf->vf_num != 0)
9214                 PMD_INIT_LOG(DEBUG, " DCB only works on pf and vmdq vsis.");
9215
9216         ret = i40e_parse_dcb_configure(dev, &dcb_cfg, &tc_map);
9217         if (ret) {
9218                 PMD_INIT_LOG(ERR, "invalid dcb config");
9219                 return -EINVAL;
9220         }
9221         ret = i40e_dcb_hw_configure(pf, &dcb_cfg, tc_map);
9222         if (ret) {
9223                 PMD_INIT_LOG(ERR, "dcb sw configure fails");
9224                 return -ENOSYS;
9225         }
9226
9227         return 0;
9228 }
9229
9230 static int
9231 i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
9232                       struct rte_eth_dcb_info *dcb_info)
9233 {
9234         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9235         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9236         struct i40e_vsi *vsi = pf->main_vsi;
9237         struct i40e_dcbx_config *dcb_cfg = &hw->local_dcbx_config;
9238         uint16_t bsf, tc_mapping;
9239         int i, j = 0;
9240
9241         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
9242                 dcb_info->nb_tcs = rte_bsf32(vsi->enabled_tc + 1);
9243         else
9244                 dcb_info->nb_tcs = 1;
9245         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
9246                 dcb_info->prio_tc[i] = dcb_cfg->etscfg.prioritytable[i];
9247         for (i = 0; i < dcb_info->nb_tcs; i++)
9248                 dcb_info->tc_bws[i] = dcb_cfg->etscfg.tcbwtable[i];
9249
9250         /* get queue mapping if vmdq is disabled */
9251         if (!pf->nb_cfg_vmdq_vsi) {
9252                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9253                         if (!(vsi->enabled_tc & (1 << i)))
9254                                 continue;
9255                         tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
9256                         dcb_info->tc_queue.tc_rxq[j][i].base =
9257                                 (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
9258                                 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
9259                         dcb_info->tc_queue.tc_txq[j][i].base =
9260                                 dcb_info->tc_queue.tc_rxq[j][i].base;
9261                         bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
9262                                 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
9263                         dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
9264                         dcb_info->tc_queue.tc_txq[j][i].nb_queue =
9265                                 dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
9266                 }
9267                 return 0;
9268         }
9269
9270         /* get queue mapping if vmdq is enabled */
9271         do {
9272                 vsi = pf->vmdq[j].vsi;
9273                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9274                         if (!(vsi->enabled_tc & (1 << i)))
9275                                 continue;
9276                         tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
9277                         dcb_info->tc_queue.tc_rxq[j][i].base =
9278                                 (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
9279                                 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
9280                         dcb_info->tc_queue.tc_txq[j][i].base =
9281                                 dcb_info->tc_queue.tc_rxq[j][i].base;
9282                         bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
9283                                 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
9284                         dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
9285                         dcb_info->tc_queue.tc_txq[j][i].nb_queue =
9286                                 dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
9287                 }
9288                 j++;
9289         } while (j < RTE_MIN(pf->nb_cfg_vmdq_vsi, ETH_MAX_VMDQ_POOL));
9290         return 0;
9291 }
9292
9293 static int
9294 i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
9295 {
9296         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
9297         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9298         uint16_t interval =
9299                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
9300         uint16_t msix_intr;
9301
9302         msix_intr = intr_handle->intr_vec[queue_id];
9303         if (msix_intr == I40E_MISC_VEC_ID)
9304                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
9305                                I40E_PFINT_DYN_CTLN_INTENA_MASK |
9306                                I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
9307                                (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
9308                                (interval <<
9309                                 I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
9310         else
9311                 I40E_WRITE_REG(hw,
9312                                I40E_PFINT_DYN_CTLN(msix_intr -
9313                                                    I40E_RX_VEC_START),
9314                                I40E_PFINT_DYN_CTLN_INTENA_MASK |
9315                                I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
9316                                (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
9317                                (interval <<
9318                                 I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
9319
9320         I40E_WRITE_FLUSH(hw);
9321         rte_intr_enable(&dev->pci_dev->intr_handle);
9322
9323         return 0;
9324 }
9325
9326 static int
9327 i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
9328 {
9329         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
9330         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9331         uint16_t msix_intr;
9332
9333         msix_intr = intr_handle->intr_vec[queue_id];
9334         if (msix_intr == I40E_MISC_VEC_ID)
9335                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
9336         else
9337                 I40E_WRITE_REG(hw,
9338                                I40E_PFINT_DYN_CTLN(msix_intr -
9339                                                    I40E_RX_VEC_START),
9340                                0);
9341         I40E_WRITE_FLUSH(hw);
9342
9343         return 0;
9344 }
9345
9346 static int i40e_get_reg_length(__rte_unused struct rte_eth_dev *dev)
9347 {
9348         /* Highest base addr + 32-bit word */
9349         return I40E_GLGEN_STAT_CLEAR + 4;
9350 }
9351
9352 static int i40e_get_regs(struct rte_eth_dev *dev,
9353                          struct rte_dev_reg_info *regs)
9354 {
9355         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9356         uint32_t *ptr_data = regs->data;
9357         uint32_t reg_idx, arr_idx, arr_idx2, reg_offset;
9358         const struct i40e_reg_info *reg_info;
9359
9360         /* The first few registers have to be read using AQ operations */
9361         reg_idx = 0;
9362         while (i40e_regs_adminq[reg_idx].name) {
9363                 reg_info = &i40e_regs_adminq[reg_idx++];
9364                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
9365                         for (arr_idx2 = 0;
9366                                         arr_idx2 <= reg_info->count2;
9367                                         arr_idx2++) {
9368                                 reg_offset = arr_idx * reg_info->stride1 +
9369                                         arr_idx2 * reg_info->stride2;
9370                                 reg_offset += reg_info->base_addr;
9371                                 ptr_data[reg_offset >> 2] =
9372                                         i40e_read_rx_ctl(hw, reg_offset);
9373                         }
9374         }
9375
9376         /* The remaining registers can be read using primitives */
9377         reg_idx = 0;
9378         while (i40e_regs_others[reg_idx].name) {
9379                 reg_info = &i40e_regs_others[reg_idx++];
9380                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
9381                         for (arr_idx2 = 0;
9382                                         arr_idx2 <= reg_info->count2;
9383                                         arr_idx2++) {
9384                                 reg_offset = arr_idx * reg_info->stride1 +
9385                                         arr_idx2 * reg_info->stride2;
9386                                 reg_offset += reg_info->base_addr;
9387                                 ptr_data[reg_offset >> 2] =
9388                                         I40E_READ_REG(hw, reg_offset);
9389                         }
9390         }
9391
9392         return 0;
9393 }
9394
9395 static int i40e_get_eeprom_length(struct rte_eth_dev *dev)
9396 {
9397         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9398
9399         /* Convert word count to byte count */
9400         return hw->nvm.sr_size << 1;
9401 }
9402
9403 static int i40e_get_eeprom(struct rte_eth_dev *dev,
9404                            struct rte_dev_eeprom_info *eeprom)
9405 {
9406         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9407         uint16_t *data = eeprom->data;
9408         uint16_t offset, length, cnt_words;
9409         int ret_code;
9410
9411         offset = eeprom->offset >> 1;
9412         length = eeprom->length >> 1;
9413         cnt_words = length;
9414
9415         if (offset > hw->nvm.sr_size ||
9416                 offset + length > hw->nvm.sr_size) {
9417                 PMD_DRV_LOG(ERR, "Requested EEPROM bytes out of range.");
9418                 return -EINVAL;
9419         }
9420
9421         eeprom->magic = hw->vendor_id | (hw->device_id << 16);
9422
9423         ret_code = i40e_read_nvm_buffer(hw, offset, &cnt_words, data);
9424         if (ret_code != I40E_SUCCESS || cnt_words != length) {
9425                 PMD_DRV_LOG(ERR, "EEPROM read failed.");
9426                 return -EIO;
9427         }
9428
9429         return 0;
9430 }
9431
9432 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
9433                                       struct ether_addr *mac_addr)
9434 {
9435         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9436
9437         if (!is_valid_assigned_ether_addr(mac_addr)) {
9438                 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
9439                 return;
9440         }
9441
9442         /* Flags: 0x3 updates port address */
9443         i40e_aq_mac_address_write(hw, 0x3, mac_addr->addr_bytes, NULL);
9444 }
9445
9446 static int
9447 i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
9448 {
9449         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9450         struct rte_eth_dev_data *dev_data = pf->dev_data;
9451         uint32_t frame_size = mtu + ETHER_HDR_LEN
9452                               + ETHER_CRC_LEN + I40E_VLAN_TAG_SIZE;
9453         int ret = 0;
9454
9455         /* check if mtu is within the allowed range */
9456         if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX))
9457                 return -EINVAL;
9458
9459         /* mtu setting is forbidden if port is start */
9460         if (dev_data->dev_started) {
9461                 PMD_DRV_LOG(ERR,
9462                             "port %d must be stopped before configuration\n",
9463                             dev_data->port_id);
9464                 return -EBUSY;
9465         }
9466
9467         if (frame_size > ETHER_MAX_LEN)
9468                 dev_data->dev_conf.rxmode.jumbo_frame = 1;
9469         else
9470                 dev_data->dev_conf.rxmode.jumbo_frame = 0;
9471
9472         dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
9473
9474         return ret;
9475 }