drivers/net/i40e/i40e_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <errno.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39 #include <stdarg.h>
  40 #include <inttypes.h>
  41 #include <assert.h>
  42
  43 #include <rte_string_fns.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_memzone.h>
  48 #include <rte_malloc.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_alarm.h>
  51 #include <rte_dev.h>
  52 #include <rte_eth_ctrl.h>
  53 #include <rte_tailq.h>
  54
  55 #include "i40e_logs.h"
  56 #include "base/i40e_prototype.h"
  57 #include "base/i40e_adminq_cmd.h"
  58 #include "base/i40e_type.h"
  59 #include "base/i40e_register.h"
  60 #include "base/i40e_dcb.h"
  61 #include "i40e_ethdev.h"
  62 #include "i40e_rxtx.h"
  63 #include "i40e_pf.h"
  64 #include "i40e_regs.h"
  65
  66 #define ETH_I40E_FLOATING_VEB_ARG       "enable_floating_veb"
  67 #define ETH_I40E_FLOATING_VEB_LIST_ARG  "floating_veb_list"
  68
  69 #define I40E_CLEAR_PXE_WAIT_MS     200
  70
  71 /* Maximun number of capability elements */
  72 #define I40E_MAX_CAP_ELE_NUM       128
  73
  74 /* Wait count and inteval */
  75 #define I40E_CHK_Q_ENA_COUNT       1000
  76 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
  77
  78 /* Maximun number of VSI */
  79 #define I40E_MAX_NUM_VSIS          (384UL)
  80
  81 #define I40E_PRE_TX_Q_CFG_WAIT_US       10 /* 10 us */
  82
  83 /* Flow control default timer */
  84 #define I40E_DEFAULT_PAUSE_TIME 0xFFFFU
  85
  86 /* Flow control default high water */
  87 #define I40E_DEFAULT_HIGH_WATER (0x1C40/1024)
  88
  89 /* Flow control default low water */
  90 #define I40E_DEFAULT_LOW_WATER  (0x1A40/1024)
  91
  92 /* Flow control enable fwd bit */
  93 #define I40E_PRTMAC_FWD_CTRL   0x00000001
  94
  95 /* Receive Packet Buffer size */
  96 #define I40E_RXPBSIZE (968 * 1024)
  97
  98 /* Kilobytes shift */
  99 #define I40E_KILOSHIFT 10
 100
 101 /* Receive Average Packet Size in Byte*/
 102 #define I40E_PACKET_AVERAGE_SIZE 128
 103
 104 /* Mask of PF interrupt causes */
 105 #define I40E_PFINT_ICR0_ENA_MASK ( \
 106                 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | \
 107                 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | \
 108                 I40E_PFINT_ICR0_ENA_GRST_MASK | \
 109                 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | \
 110                 I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK | \
 111                 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK | \
 112                 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | \
 113                 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | \
 114                 I40E_PFINT_ICR0_ENA_VFLR_MASK | \
 115                 I40E_PFINT_ICR0_ENA_ADMINQ_MASK)
 116
 117 #define I40E_FLOW_TYPES ( \
 118         (1UL << RTE_ETH_FLOW_FRAG_IPV4) | \
 119         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
 120         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
 121         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
 122         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
 123         (1UL << RTE_ETH_FLOW_FRAG_IPV6) | \
 124         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
 125         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
 126         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
 127         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
 128         (1UL << RTE_ETH_FLOW_L2_PAYLOAD))
 129
 130 /* Additional timesync values. */
 131 #define I40E_PTP_40GB_INCVAL     0x0199999999ULL
 132 #define I40E_PTP_10GB_INCVAL     0x0333333333ULL
 133 #define I40E_PTP_1GB_INCVAL      0x2000000000ULL
 134 #define I40E_PRTTSYN_TSYNENA     0x80000000
 135 #define I40E_PRTTSYN_TSYNTYPE    0x0e000000
 136 #define I40E_CYCLECOUNTER_MASK   0xffffffffffffffffULL
 137
 138 #define I40E_MAX_PERCENT            100
 139 #define I40E_DEFAULT_DCB_APP_NUM    1
 140 #define I40E_DEFAULT_DCB_APP_PRIO   3
 141
 142 #define I40E_INSET_NONE            0x00000000000000000ULL
 143
 144 /* bit0 ~ bit 7 */
 145 #define I40E_INSET_DMAC            0x0000000000000001ULL
 146 #define I40E_INSET_SMAC            0x0000000000000002ULL
 147 #define I40E_INSET_VLAN_OUTER      0x0000000000000004ULL
 148 #define I40E_INSET_VLAN_INNER      0x0000000000000008ULL
 149 #define I40E_INSET_VLAN_TUNNEL     0x0000000000000010ULL
 150
 151 /* bit 8 ~ bit 15 */
 152 #define I40E_INSET_IPV4_SRC        0x0000000000000100ULL
 153 #define I40E_INSET_IPV4_DST        0x0000000000000200ULL
 154 #define I40E_INSET_IPV6_SRC        0x0000000000000400ULL
 155 #define I40E_INSET_IPV6_DST        0x0000000000000800ULL
 156 #define I40E_INSET_SRC_PORT        0x0000000000001000ULL
 157 #define I40E_INSET_DST_PORT        0x0000000000002000ULL
 158 #define I40E_INSET_SCTP_VT         0x0000000000004000ULL
 159
 160 /* bit 16 ~ bit 31 */
 161 #define I40E_INSET_IPV4_TOS        0x0000000000010000ULL
 162 #define I40E_INSET_IPV4_PROTO      0x0000000000020000ULL
 163 #define I40E_INSET_IPV4_TTL        0x0000000000040000ULL
 164 #define I40E_INSET_IPV6_TC         0x0000000000080000ULL
 165 #define I40E_INSET_IPV6_FLOW       0x0000000000100000ULL
 166 #define I40E_INSET_IPV6_NEXT_HDR   0x0000000000200000ULL
 167 #define I40E_INSET_IPV6_HOP_LIMIT  0x0000000000400000ULL
 168 #define I40E_INSET_TCP_FLAGS       0x0000000000800000ULL
 169
 170 /* bit 32 ~ bit 47, tunnel fields */
 171 #define I40E_INSET_TUNNEL_IPV4_DST       0x0000000100000000ULL
 172 #define I40E_INSET_TUNNEL_IPV6_DST       0x0000000200000000ULL
 173 #define I40E_INSET_TUNNEL_DMAC           0x0000000400000000ULL
 174 #define I40E_INSET_TUNNEL_SRC_PORT       0x0000000800000000ULL
 175 #define I40E_INSET_TUNNEL_DST_PORT       0x0000001000000000ULL
 176 #define I40E_INSET_TUNNEL_ID             0x0000002000000000ULL
 177
 178 /* bit 48 ~ bit 55 */
 179 #define I40E_INSET_LAST_ETHER_TYPE 0x0001000000000000ULL
 180
 181 /* bit 56 ~ bit 63, Flex Payload */
 182 #define I40E_INSET_FLEX_PAYLOAD_W1 0x0100000000000000ULL
 183 #define I40E_INSET_FLEX_PAYLOAD_W2 0x0200000000000000ULL
 184 #define I40E_INSET_FLEX_PAYLOAD_W3 0x0400000000000000ULL
 185 #define I40E_INSET_FLEX_PAYLOAD_W4 0x0800000000000000ULL
 186 #define I40E_INSET_FLEX_PAYLOAD_W5 0x1000000000000000ULL
 187 #define I40E_INSET_FLEX_PAYLOAD_W6 0x2000000000000000ULL
 188 #define I40E_INSET_FLEX_PAYLOAD_W7 0x4000000000000000ULL
 189 #define I40E_INSET_FLEX_PAYLOAD_W8 0x8000000000000000ULL
 190 #define I40E_INSET_FLEX_PAYLOAD \
 191         (I40E_INSET_FLEX_PAYLOAD_W1 | I40E_INSET_FLEX_PAYLOAD_W2 | \
 192         I40E_INSET_FLEX_PAYLOAD_W3 | I40E_INSET_FLEX_PAYLOAD_W4 | \
 193         I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \
 194         I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8)
 195
 196 /**
 197  * Below are values for writing un-exposed registers suggested
 198  * by silicon experts
 199  */
 200 /* Destination MAC address */
 201 #define I40E_REG_INSET_L2_DMAC                   0xE000000000000000ULL
 202 /* Source MAC address */
 203 #define I40E_REG_INSET_L2_SMAC                   0x1C00000000000000ULL
 204 /* Outer (S-Tag) VLAN tag in the outer L2 header */
 205 #define I40E_REG_INSET_L2_OUTER_VLAN             0x0000000004000000ULL
 206 /* Inner (C-Tag) or single VLAN tag in the outer L2 header */
 207 #define I40E_REG_INSET_L2_INNER_VLAN             0x0080000000000000ULL
 208 /* Single VLAN tag in the inner L2 header */
 209 #define I40E_REG_INSET_TUNNEL_VLAN               0x0100000000000000ULL
 210 /* Source IPv4 address */
 211 #define I40E_REG_INSET_L3_SRC_IP4                0x0001800000000000ULL
 212 /* Destination IPv4 address */
 213 #define I40E_REG_INSET_L3_DST_IP4                0x0000001800000000ULL
 214 /* IPv4 Type of Service (TOS) */
 215 #define I40E_REG_INSET_L3_IP4_TOS                0x0040000000000000ULL
 216 /* IPv4 Protocol */
 217 #define I40E_REG_INSET_L3_IP4_PROTO              0x0004000000000000ULL
 218 /* IPv4 Time to Live */
 219 #define I40E_REG_INSET_L3_IP4_TTL                0x0004000000000000ULL
 220 /* Source IPv6 address */
 221 #define I40E_REG_INSET_L3_SRC_IP6                0x0007F80000000000ULL
 222 /* Destination IPv6 address */
 223 #define I40E_REG_INSET_L3_DST_IP6                0x000007F800000000ULL
 224 /* IPv6 Traffic Class (TC) */
 225 #define I40E_REG_INSET_L3_IP6_TC                 0x0040000000000000ULL
 226 /* IPv6 Next Header */
 227 #define I40E_REG_INSET_L3_IP6_NEXT_HDR           0x0008000000000000ULL
 228 /* IPv6 Hop Limit */
 229 #define I40E_REG_INSET_L3_IP6_HOP_LIMIT          0x0008000000000000ULL
 230 /* Source L4 port */
 231 #define I40E_REG_INSET_L4_SRC_PORT               0x0000000400000000ULL
 232 /* Destination L4 port */
 233 #define I40E_REG_INSET_L4_DST_PORT               0x0000000200000000ULL
 234 /* SCTP verification tag */
 235 #define I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG  0x0000000180000000ULL
 236 /* Inner destination MAC address (MAC-in-UDP/MAC-in-GRE)*/
 237 #define I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC   0x0000000001C00000ULL
 238 /* Source port of tunneling UDP */
 239 #define I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT    0x0000000000200000ULL
 240 /* Destination port of tunneling UDP */
 241 #define I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT    0x0000000000100000ULL
 242 /* UDP Tunneling ID, NVGRE/GRE key */
 243 #define I40E_REG_INSET_TUNNEL_ID                 0x00000000000C0000ULL
 244 /* Last ether type */
 245 #define I40E_REG_INSET_LAST_ETHER_TYPE           0x0000000000004000ULL
 246 /* Tunneling outer destination IPv4 address */
 247 #define I40E_REG_INSET_TUNNEL_L3_DST_IP4         0x00000000000000C0ULL
 248 /* Tunneling outer destination IPv6 address */
 249 #define I40E_REG_INSET_TUNNEL_L3_DST_IP6         0x0000000000003FC0ULL
 250 /* 1st word of flex payload */
 251 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD1        0x0000000000002000ULL
 252 /* 2nd word of flex payload */
 253 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD2        0x0000000000001000ULL
 254 /* 3rd word of flex payload */
 255 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD3        0x0000000000000800ULL
 256 /* 4th word of flex payload */
 257 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD4        0x0000000000000400ULL
 258 /* 5th word of flex payload */
 259 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD5        0x0000000000000200ULL
 260 /* 6th word of flex payload */
 261 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD6        0x0000000000000100ULL
 262 /* 7th word of flex payload */
 263 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD7        0x0000000000000080ULL
 264 /* 8th word of flex payload */
 265 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD8        0x0000000000000040ULL
 266 /* all 8 words flex payload */
 267 #define I40E_REG_INSET_FLEX_PAYLOAD_WORDS        0x0000000000003FC0ULL
 268 #define I40E_REG_INSET_MASK_DEFAULT              0x0000000000000000ULL
 269
 270 #define I40E_TRANSLATE_INSET 0
 271 #define I40E_TRANSLATE_REG   1
 272
 273 #define I40E_INSET_IPV4_TOS_MASK        0x0009FF00UL
 274 #define I40E_INSET_IPv4_TTL_MASK        0x000D00FFUL
 275 #define I40E_INSET_IPV4_PROTO_MASK      0x000DFF00UL
 276 #define I40E_INSET_IPV6_TC_MASK         0x0009F00FUL
 277 #define I40E_INSET_IPV6_HOP_LIMIT_MASK  0x000CFF00UL
 278 #define I40E_INSET_IPV6_NEXT_HDR_MASK   0x000C00FFUL
 279
 280 #define I40E_GL_SWT_L2TAGCTRL(_i)             (0x001C0A70 + ((_i) * 4))
 281 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT 16
 282 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK  \
 283         I40E_MASK(0xFFFF, I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT)
 284
 285 /* PCI offset for querying capability */
 286 #define PCI_DEV_CAP_REG            0xA4
 287 /* PCI offset for enabling/disabling Extended Tag */
 288 #define PCI_DEV_CTRL_REG           0xA8
 289 /* Bit mask of Extended Tag capability */
 290 #define PCI_DEV_CAP_EXT_TAG_MASK   0x20
 291 /* Bit shift of Extended Tag enable/disable */
 292 #define PCI_DEV_CTRL_EXT_TAG_SHIFT 8
 293 /* Bit mask of Extended Tag enable/disable */
 294 #define PCI_DEV_CTRL_EXT_TAG_MASK  (1 << PCI_DEV_CTRL_EXT_TAG_SHIFT)
 295
 296 static int eth_i40e_dev_init(struct rte_eth_dev *eth_dev);
 297 static int eth_i40e_dev_uninit(struct rte_eth_dev *eth_dev);
 298 static int i40e_dev_configure(struct rte_eth_dev *dev);
 299 static int i40e_dev_start(struct rte_eth_dev *dev);
 300 static void i40e_dev_stop(struct rte_eth_dev *dev);
 301 static void i40e_dev_close(struct rte_eth_dev *dev);
 302 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
 303 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
 304 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
 305 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
 306 static int i40e_dev_set_link_up(struct rte_eth_dev *dev);
 307 static int i40e_dev_set_link_down(struct rte_eth_dev *dev);
 308 static void i40e_dev_stats_get(struct rte_eth_dev *dev,
 309                                struct rte_eth_stats *stats);
 310 static int i40e_dev_xstats_get(struct rte_eth_dev *dev,
 311                                struct rte_eth_xstat *xstats, unsigned n);
 312 static int i40e_dev_xstats_get_names(struct rte_eth_dev *dev,
 313                                      struct rte_eth_xstat_name *xstats_names,
 314                                      unsigned limit);
 315 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
 316 static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev,
 317                                             uint16_t queue_id,
 318                                             uint8_t stat_idx,
 319                                             uint8_t is_rx);
 320 static void i40e_dev_info_get(struct rte_eth_dev *dev,
 321                               struct rte_eth_dev_info *dev_info);
 322 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
 323                                 uint16_t vlan_id,
 324                                 int on);
 325 static int i40e_vlan_tpid_set(struct rte_eth_dev *dev,
 326                               enum rte_vlan_type vlan_type,
 327                               uint16_t tpid);
 328 static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 329 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
 330                                       uint16_t queue,
 331                                       int on);
 332 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
 333 static int i40e_dev_led_on(struct rte_eth_dev *dev);
 334 static int i40e_dev_led_off(struct rte_eth_dev *dev);
 335 static int i40e_flow_ctrl_get(struct rte_eth_dev *dev,
 336                               struct rte_eth_fc_conf *fc_conf);
 337 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
 338                               struct rte_eth_fc_conf *fc_conf);
 339 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 340                                        struct rte_eth_pfc_conf *pfc_conf);
 341 static void i40e_macaddr_add(struct rte_eth_dev *dev,
 342                           struct ether_addr *mac_addr,
 343                           uint32_t index,
 344                           uint32_t pool);
 345 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 346 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
 347                                     struct rte_eth_rss_reta_entry64 *reta_conf,
 348                                     uint16_t reta_size);
 349 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
 350                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 351                                    uint16_t reta_size);
 352
 353 static int i40e_get_cap(struct i40e_hw *hw);
 354 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
 355 static int i40e_pf_setup(struct i40e_pf *pf);
 356 static int i40e_dev_rxtx_init(struct i40e_pf *pf);
 357 static int i40e_vmdq_setup(struct rte_eth_dev *dev);
 358 static int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb);
 359 static int i40e_dcb_setup(struct rte_eth_dev *dev);
 360 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
 361                 bool offset_loaded, uint64_t *offset, uint64_t *stat);
 362 static void i40e_stat_update_48(struct i40e_hw *hw,
 363                                uint32_t hireg,
 364                                uint32_t loreg,
 365                                bool offset_loaded,
 366                                uint64_t *offset,
 367                                uint64_t *stat);
 368 static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue);
 369 static void i40e_dev_interrupt_handler(
 370                 __rte_unused struct rte_intr_handle *handle, void *param);
 371 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 372                                 uint32_t base, uint32_t num);
 373 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
 374 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
 375                         uint32_t base);
 376 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
 377                         uint16_t num);
 378 static int i40e_dev_init_vlan(struct rte_eth_dev *dev);
 379 static int i40e_veb_release(struct i40e_veb *veb);
 380 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
 381                                                 struct i40e_vsi *vsi);
 382 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
 383 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
 384 static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
 385                                              struct i40e_macvlan_filter *mv_f,
 386                                              int num,
 387                                              struct ether_addr *addr);
 388 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
 389                                              struct i40e_macvlan_filter *mv_f,
 390                                              int num,
 391                                              uint16_t vlan);
 392 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
 393 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
 394                                     struct rte_eth_rss_conf *rss_conf);
 395 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 396                                       struct rte_eth_rss_conf *rss_conf);
 397 static int i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
 398                                         struct rte_eth_udp_tunnel *udp_tunnel);
 399 static int i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
 400                                         struct rte_eth_udp_tunnel *udp_tunnel);
 401 static void i40e_filter_input_set_init(struct i40e_pf *pf);
 402 static int i40e_ethertype_filter_set(struct i40e_pf *pf,
 403                         struct rte_eth_ethertype_filter *filter,
 404                         bool add);
 405 static int i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
 406                                 enum rte_filter_op filter_op,
 407                                 void *arg);
 408 static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
 409                                 enum rte_filter_type filter_type,
 410                                 enum rte_filter_op filter_op,
 411                                 void *arg);
 412 static int i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
 413                                   struct rte_eth_dcb_info *dcb_info);
 414 static void i40e_configure_registers(struct i40e_hw *hw);
 415 static void i40e_hw_init(struct rte_eth_dev *dev);
 416 static int i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi);
 417 static int i40e_mirror_rule_set(struct rte_eth_dev *dev,
 418                         struct rte_eth_mirror_conf *mirror_conf,
 419                         uint8_t sw_id, uint8_t on);
 420 static int i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id);
 421
 422 static int i40e_timesync_enable(struct rte_eth_dev *dev);
 423 static int i40e_timesync_disable(struct rte_eth_dev *dev);
 424 static int i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 425                                            struct timespec *timestamp,
 426                                            uint32_t flags);
 427 static int i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 428                                            struct timespec *timestamp);
 429 static void i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw);
 430
 431 static int i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 432
 433 static int i40e_timesync_read_time(struct rte_eth_dev *dev,
 434                                    struct timespec *timestamp);
 435 static int i40e_timesync_write_time(struct rte_eth_dev *dev,
 436                                     const struct timespec *timestamp);
 437
 438 static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 439                                          uint16_t queue_id);
 440 static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 441                                           uint16_t queue_id);
 442
 443 static int i40e_get_regs(struct rte_eth_dev *dev,
 444                          struct rte_dev_reg_info *regs);
 445
 446 static int i40e_get_eeprom_length(struct rte_eth_dev *dev);
 447
 448 static int i40e_get_eeprom(struct rte_eth_dev *dev,
 449                            struct rte_dev_eeprom_info *eeprom);
 450
 451 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
 452                                       struct ether_addr *mac_addr);
 453
 454 static int i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 455
 456 static const struct rte_pci_id pci_id_i40e_map[] = {
 457         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710) },
 458         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QEMU) },
 459         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B) },
 460         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C) },
 461         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A) },
 462         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B) },
 463         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C) },
 464         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T) },
 465         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2) },
 466         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2_A) },
 467         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4) },
 468         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_B) },
 469         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_SFP28) },
 470         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0) },
 471         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722) },
 472         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722) },
 473         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722) },
 474         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722) },
 475         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722) },
 476         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722) },
 477         { .vendor_id = 0, /* sentinel */ },
 478 };
 479
 480 static const struct eth_dev_ops i40e_eth_dev_ops = {
 481         .dev_configure                = i40e_dev_configure,
 482         .dev_start                    = i40e_dev_start,
 483         .dev_stop                     = i40e_dev_stop,
 484         .dev_close                    = i40e_dev_close,
 485         .promiscuous_enable           = i40e_dev_promiscuous_enable,
 486         .promiscuous_disable          = i40e_dev_promiscuous_disable,
 487         .allmulticast_enable          = i40e_dev_allmulticast_enable,
 488         .allmulticast_disable         = i40e_dev_allmulticast_disable,
 489         .dev_set_link_up              = i40e_dev_set_link_up,
 490         .dev_set_link_down            = i40e_dev_set_link_down,
 491         .link_update                  = i40e_dev_link_update,
 492         .stats_get                    = i40e_dev_stats_get,
 493         .xstats_get                   = i40e_dev_xstats_get,
 494         .xstats_get_names             = i40e_dev_xstats_get_names,
 495         .stats_reset                  = i40e_dev_stats_reset,
 496         .xstats_reset                 = i40e_dev_stats_reset,
 497         .queue_stats_mapping_set      = i40e_dev_queue_stats_mapping_set,
 498         .dev_infos_get                = i40e_dev_info_get,
 499         .dev_supported_ptypes_get     = i40e_dev_supported_ptypes_get,
 500         .vlan_filter_set              = i40e_vlan_filter_set,
 501         .vlan_tpid_set                = i40e_vlan_tpid_set,
 502         .vlan_offload_set             = i40e_vlan_offload_set,
 503         .vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
 504         .vlan_pvid_set                = i40e_vlan_pvid_set,
 505         .rx_queue_start               = i40e_dev_rx_queue_start,
 506         .rx_queue_stop                = i40e_dev_rx_queue_stop,
 507         .tx_queue_start               = i40e_dev_tx_queue_start,
 508         .tx_queue_stop                = i40e_dev_tx_queue_stop,
 509         .rx_queue_setup               = i40e_dev_rx_queue_setup,
 510         .rx_queue_intr_enable         = i40e_dev_rx_queue_intr_enable,
 511         .rx_queue_intr_disable        = i40e_dev_rx_queue_intr_disable,
 512         .rx_queue_release             = i40e_dev_rx_queue_release,
 513         .rx_queue_count               = i40e_dev_rx_queue_count,
 514         .rx_descriptor_done           = i40e_dev_rx_descriptor_done,
 515         .tx_queue_setup               = i40e_dev_tx_queue_setup,
 516         .tx_queue_release             = i40e_dev_tx_queue_release,
 517         .dev_led_on                   = i40e_dev_led_on,
 518         .dev_led_off                  = i40e_dev_led_off,
 519         .flow_ctrl_get                = i40e_flow_ctrl_get,
 520         .flow_ctrl_set                = i40e_flow_ctrl_set,
 521         .priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
 522         .mac_addr_add                 = i40e_macaddr_add,
 523         .mac_addr_remove              = i40e_macaddr_remove,
 524         .reta_update                  = i40e_dev_rss_reta_update,
 525         .reta_query                   = i40e_dev_rss_reta_query,
 526         .rss_hash_update              = i40e_dev_rss_hash_update,
 527         .rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
 528         .udp_tunnel_port_add          = i40e_dev_udp_tunnel_port_add,
 529         .udp_tunnel_port_del          = i40e_dev_udp_tunnel_port_del,
 530         .filter_ctrl                  = i40e_dev_filter_ctrl,
 531         .rxq_info_get                 = i40e_rxq_info_get,
 532         .txq_info_get                 = i40e_txq_info_get,
 533         .mirror_rule_set              = i40e_mirror_rule_set,
 534         .mirror_rule_reset            = i40e_mirror_rule_reset,
 535         .timesync_enable              = i40e_timesync_enable,
 536         .timesync_disable             = i40e_timesync_disable,
 537         .timesync_read_rx_timestamp   = i40e_timesync_read_rx_timestamp,
 538         .timesync_read_tx_timestamp   = i40e_timesync_read_tx_timestamp,
 539         .get_dcb_info                 = i40e_dev_get_dcb_info,
 540         .timesync_adjust_time         = i40e_timesync_adjust_time,
 541         .timesync_read_time           = i40e_timesync_read_time,
 542         .timesync_write_time          = i40e_timesync_write_time,
 543         .get_reg                      = i40e_get_regs,
 544         .get_eeprom_length            = i40e_get_eeprom_length,
 545         .get_eeprom                   = i40e_get_eeprom,
 546         .mac_addr_set                 = i40e_set_default_mac_addr,
 547         .mtu_set                      = i40e_dev_mtu_set,
 548 };
 549
 550 /* store statistics names and its offset in stats structure */
 551 struct rte_i40e_xstats_name_off {
 552         char name[RTE_ETH_XSTATS_NAME_SIZE];
 553         unsigned offset;
 554 };
 555
 556 static const struct rte_i40e_xstats_name_off rte_i40e_stats_strings[] = {
 557         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 558         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 559         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 560         {"rx_dropped", offsetof(struct i40e_eth_stats, rx_discards)},
 561         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 562                 rx_unknown_protocol)},
 563         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)},
 564         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)},
 565         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)},
 566         {"tx_dropped", offsetof(struct i40e_eth_stats, tx_discards)},
 567 };
 568
 569 #define I40E_NB_ETH_XSTATS (sizeof(rte_i40e_stats_strings) / \
 570                 sizeof(rte_i40e_stats_strings[0]))
 571
 572 static const struct rte_i40e_xstats_name_off rte_i40e_hw_port_strings[] = {
 573         {"tx_link_down_dropped", offsetof(struct i40e_hw_port_stats,
 574                 tx_dropped_link_down)},
 575         {"rx_crc_errors", offsetof(struct i40e_hw_port_stats, crc_errors)},
 576         {"rx_illegal_byte_errors", offsetof(struct i40e_hw_port_stats,
 577                 illegal_bytes)},
 578         {"rx_error_bytes", offsetof(struct i40e_hw_port_stats, error_bytes)},
 579         {"mac_local_errors", offsetof(struct i40e_hw_port_stats,
 580                 mac_local_faults)},
 581         {"mac_remote_errors", offsetof(struct i40e_hw_port_stats,
 582                 mac_remote_faults)},
 583         {"rx_length_errors", offsetof(struct i40e_hw_port_stats,
 584                 rx_length_errors)},
 585         {"tx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_tx)},
 586         {"rx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_rx)},
 587         {"tx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_tx)},
 588         {"rx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_rx)},
 589         {"rx_size_64_packets", offsetof(struct i40e_hw_port_stats, rx_size_64)},
 590         {"rx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 591                 rx_size_127)},
 592         {"rx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 593                 rx_size_255)},
 594         {"rx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 595                 rx_size_511)},
 596         {"rx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 597                 rx_size_1023)},
 598         {"rx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 599                 rx_size_1522)},
 600         {"rx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 601                 rx_size_big)},
 602         {"rx_undersized_errors", offsetof(struct i40e_hw_port_stats,
 603                 rx_undersize)},
 604         {"rx_oversize_errors", offsetof(struct i40e_hw_port_stats,
 605                 rx_oversize)},
 606         {"rx_mac_short_dropped", offsetof(struct i40e_hw_port_stats,
 607                 mac_short_packet_dropped)},
 608         {"rx_fragmented_errors", offsetof(struct i40e_hw_port_stats,
 609                 rx_fragments)},
 610         {"rx_jabber_errors", offsetof(struct i40e_hw_port_stats, rx_jabber)},
 611         {"tx_size_64_packets", offsetof(struct i40e_hw_port_stats, tx_size_64)},
 612         {"tx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 613                 tx_size_127)},
 614         {"tx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 615                 tx_size_255)},
 616         {"tx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 617                 tx_size_511)},
 618         {"tx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 619                 tx_size_1023)},
 620         {"tx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 621                 tx_size_1522)},
 622         {"tx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 623                 tx_size_big)},
 624         {"rx_flow_director_atr_match_packets",
 625                 offsetof(struct i40e_hw_port_stats, fd_atr_match)},
 626         {"rx_flow_director_sb_match_packets",
 627                 offsetof(struct i40e_hw_port_stats, fd_sb_match)},
 628         {"tx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 629                 tx_lpi_status)},
 630         {"rx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 631                 rx_lpi_status)},
 632         {"tx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 633                 tx_lpi_count)},
 634         {"rx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 635                 rx_lpi_count)},
 636 };
 637
 638 #define I40E_NB_HW_PORT_XSTATS (sizeof(rte_i40e_hw_port_strings) / \
 639                 sizeof(rte_i40e_hw_port_strings[0]))
 640
 641 static const struct rte_i40e_xstats_name_off rte_i40e_rxq_prio_strings[] = {
 642         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 643                 priority_xon_rx)},
 644         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 645                 priority_xoff_rx)},
 646 };
 647
 648 #define I40E_NB_RXQ_PRIO_XSTATS (sizeof(rte_i40e_rxq_prio_strings) / \
 649                 sizeof(rte_i40e_rxq_prio_strings[0]))
 650
 651 static const struct rte_i40e_xstats_name_off rte_i40e_txq_prio_strings[] = {
 652         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 653                 priority_xon_tx)},
 654         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 655                 priority_xoff_tx)},
 656         {"xon_to_xoff_packets", offsetof(struct i40e_hw_port_stats,
 657                 priority_xon_2_xoff)},
 658 };
 659
 660 #define I40E_NB_TXQ_PRIO_XSTATS (sizeof(rte_i40e_txq_prio_strings) / \
 661                 sizeof(rte_i40e_txq_prio_strings[0]))
 662
 663 static struct eth_driver rte_i40e_pmd = {
 664         .pci_drv = {
 665                 .id_table = pci_id_i40e_map,
 666                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 667                         RTE_PCI_DRV_DETACHABLE,
 668                 .probe = rte_eth_dev_pci_probe,
 669                 .remove = rte_eth_dev_pci_remove,
 670         },
 671         .eth_dev_init = eth_i40e_dev_init,
 672         .eth_dev_uninit = eth_i40e_dev_uninit,
 673         .dev_private_size = sizeof(struct i40e_adapter),
 674 };
 675
 676 static inline int
 677 rte_i40e_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 678                                      struct rte_eth_link *link)
 679 {
 680         struct rte_eth_link *dst = link;
 681         struct rte_eth_link *src = &(dev->data->dev_link);
 682
 683         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 684                                         *(uint64_t *)src) == 0)
 685                 return -1;
 686
 687         return 0;
 688 }
 689
 690 static inline int
 691 rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 692                                       struct rte_eth_link *link)
 693 {
 694         struct rte_eth_link *dst = &(dev->data->dev_link);
 695         struct rte_eth_link *src = link;
 696
 697         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 698                                         *(uint64_t *)src) == 0)
 699                 return -1;
 700
 701         return 0;
 702 }
 703
 704 DRIVER_REGISTER_PCI(net_i40e, rte_i40e_pmd.pci_drv);
 705 DRIVER_REGISTER_PCI_TABLE(net_i40e, pci_id_i40e_map);
 706
 707 #ifndef I40E_GLQF_ORT
 708 #define I40E_GLQF_ORT(_i)    (0x00268900 + ((_i) * 4))
 709 #endif
 710 #ifndef I40E_GLQF_PIT
 711 #define I40E_GLQF_PIT(_i)    (0x00268C80 + ((_i) * 4))
 712 #endif
 713
 714 static inline void i40e_GLQF_reg_init(struct i40e_hw *hw)
 715 {
 716         /*
 717          * Initialize registers for flexible payload, which should be set by NVM.
 718          * This should be removed from code once it is fixed in NVM.
 719          */
 720         I40E_WRITE_REG(hw, I40E_GLQF_ORT(18), 0x00000030);
 721         I40E_WRITE_REG(hw, I40E_GLQF_ORT(19), 0x00000030);
 722         I40E_WRITE_REG(hw, I40E_GLQF_ORT(26), 0x0000002B);
 723         I40E_WRITE_REG(hw, I40E_GLQF_ORT(30), 0x0000002B);
 724         I40E_WRITE_REG(hw, I40E_GLQF_ORT(33), 0x000000E0);
 725         I40E_WRITE_REG(hw, I40E_GLQF_ORT(34), 0x000000E3);
 726         I40E_WRITE_REG(hw, I40E_GLQF_ORT(35), 0x000000E6);
 727         I40E_WRITE_REG(hw, I40E_GLQF_ORT(20), 0x00000031);
 728         I40E_WRITE_REG(hw, I40E_GLQF_ORT(23), 0x00000031);
 729         I40E_WRITE_REG(hw, I40E_GLQF_ORT(63), 0x0000002D);
 730         I40E_WRITE_REG(hw, I40E_GLQF_PIT(16), 0x00007480);
 731         I40E_WRITE_REG(hw, I40E_GLQF_PIT(17), 0x00007440);
 732
 733         /* Initialize registers for parsing packet type of QinQ */
 734         I40E_WRITE_REG(hw, I40E_GLQF_ORT(40), 0x00000029);
 735         I40E_WRITE_REG(hw, I40E_GLQF_PIT(9), 0x00009420);
 736 }
 737
 738 #define I40E_FLOW_CONTROL_ETHERTYPE  0x8808
 739
 740 /*
 741  * Add a ethertype filter to drop all flow control frames transmitted
 742  * from VSIs.
 743 */
 744 static void
 745 i40e_add_tx_flow_control_drop_filter(struct i40e_pf *pf)
 746 {
 747         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 748         uint16_t flags = I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC |
 749                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP |
 750                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX;
 751         int ret;
 752
 753         ret = i40e_aq_add_rem_control_packet_filter(hw, NULL,
 754                                 I40E_FLOW_CONTROL_ETHERTYPE, flags,
 755                                 pf->main_vsi_seid, 0,
 756                                 TRUE, NULL, NULL);
 757         if (ret)
 758                 PMD_INIT_LOG(ERR, "Failed to add filter to drop flow control "
 759                                   " frames from VSIs.");
 760 }
 761
 762 static int
 763 floating_veb_list_handler(__rte_unused const char *key,
 764                           const char *floating_veb_value,
 765                           void *opaque)
 766 {
 767         int idx = 0;
 768         unsigned int count = 0;
 769         char *end = NULL;
 770         int min, max;
 771         bool *vf_floating_veb = opaque;
 772
 773         while (isblank(*floating_veb_value))
 774                 floating_veb_value++;
 775
 776         /* Reset floating VEB configuration for VFs */
 777         for (idx = 0; idx < I40E_MAX_VF; idx++)
 778                 vf_floating_veb[idx] = false;
 779
 780         min = I40E_MAX_VF;
 781         do {
 782                 while (isblank(*floating_veb_value))
 783                         floating_veb_value++;
 784                 if (*floating_veb_value == '\0')
 785                         return -1;
 786                 errno = 0;
 787                 idx = strtoul(floating_veb_value, &end, 10);
 788                 if (errno || end == NULL)
 789                         return -1;
 790                 while (isblank(*end))
 791                         end++;
 792                 if (*end == '-') {
 793                         min = idx;
 794                 } else if ((*end == ';') || (*end == '\0')) {
 795                         max = idx;
 796                         if (min == I40E_MAX_VF)
 797                                 min = idx;
 798                         if (max >= I40E_MAX_VF)
 799                                 max = I40E_MAX_VF - 1;
 800                         for (idx = min; idx <= max; idx++) {
 801                                 vf_floating_veb[idx] = true;
 802                                 count++;
 803                         }
 804                         min = I40E_MAX_VF;
 805                 } else {
 806                         return -1;
 807                 }
 808                 floating_veb_value = end + 1;
 809         } while (*end != '\0');
 810
 811         if (count == 0)
 812                 return -1;
 813
 814         return 0;
 815 }
 816
 817 static void
 818 config_vf_floating_veb(struct rte_devargs *devargs,
 819                        uint16_t floating_veb,
 820                        bool *vf_floating_veb)
 821 {
 822         struct rte_kvargs *kvlist;
 823         int i;
 824         const char *floating_veb_list = ETH_I40E_FLOATING_VEB_LIST_ARG;
 825
 826         if (!floating_veb)
 827                 return;
 828         /* All the VFs attach to the floating VEB by default
 829          * when the floating VEB is enabled.
 830          */
 831         for (i = 0; i < I40E_MAX_VF; i++)
 832                 vf_floating_veb[i] = true;
 833
 834         if (devargs == NULL)
 835                 return;
 836
 837         kvlist = rte_kvargs_parse(devargs->args, NULL);
 838         if (kvlist == NULL)
 839                 return;
 840
 841         if (!rte_kvargs_count(kvlist, floating_veb_list)) {
 842                 rte_kvargs_free(kvlist);
 843                 return;
 844         }
 845         /* When the floating_veb_list parameter exists, all the VFs
 846          * will attach to the legacy VEB firstly, then configure VFs
 847          * to the floating VEB according to the floating_veb_list.
 848          */
 849         if (rte_kvargs_process(kvlist, floating_veb_list,
 850                                floating_veb_list_handler,
 851                                vf_floating_veb) < 0) {
 852                 rte_kvargs_free(kvlist);
 853                 return;
 854         }
 855         rte_kvargs_free(kvlist);
 856 }
 857
 858 static int
 859 i40e_check_floating_handler(__rte_unused const char *key,
 860                             const char *value,
 861                             __rte_unused void *opaque)
 862 {
 863         if (strcmp(value, "1"))
 864                 return -1;
 865
 866         return 0;
 867 }
 868
 869 static int
 870 is_floating_veb_supported(struct rte_devargs *devargs)
 871 {
 872         struct rte_kvargs *kvlist;
 873         const char *floating_veb_key = ETH_I40E_FLOATING_VEB_ARG;
 874
 875         if (devargs == NULL)
 876                 return 0;
 877
 878         kvlist = rte_kvargs_parse(devargs->args, NULL);
 879         if (kvlist == NULL)
 880                 return 0;
 881
 882         if (!rte_kvargs_count(kvlist, floating_veb_key)) {
 883                 rte_kvargs_free(kvlist);
 884                 return 0;
 885         }
 886         /* Floating VEB is enabled when there's key-value:
 887          * enable_floating_veb=1
 888          */
 889         if (rte_kvargs_process(kvlist, floating_veb_key,
 890                                i40e_check_floating_handler, NULL) < 0) {
 891                 rte_kvargs_free(kvlist);
 892                 return 0;
 893         }
 894         rte_kvargs_free(kvlist);
 895
 896         return 1;
 897 }
 898
 899 static void
 900 config_floating_veb(struct rte_eth_dev *dev)
 901 {
 902         struct rte_pci_device *pci_dev = dev->pci_dev;
 903         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 904         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 905
 906         memset(pf->floating_veb_list, 0, sizeof(pf->floating_veb_list));
 907
 908         if (hw->aq.fw_maj_ver >= FLOATING_VEB_SUPPORTED_FW_MAJ) {
 909                 pf->floating_veb =
 910                         is_floating_veb_supported(pci_dev->device.devargs);
 911                 config_vf_floating_veb(pci_dev->device.devargs,
 912                                        pf->floating_veb,
 913                                        pf->floating_veb_list);
 914         } else {
 915                 pf->floating_veb = false;
 916         }
 917 }
 918
 919 #define I40E_L2_TAGS_S_TAG_SHIFT 1
 920 #define I40E_L2_TAGS_S_TAG_MASK I40E_MASK(0x1, I40E_L2_TAGS_S_TAG_SHIFT)
 921
 922 static int
 923 eth_i40e_dev_init(struct rte_eth_dev *dev)
 924 {
 925         struct rte_pci_device *pci_dev;
 926         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 927         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 928         struct i40e_vsi *vsi;
 929         int ret;
 930         uint32_t len;
 931         uint8_t aq_fail = 0;
 932
 933         PMD_INIT_FUNC_TRACE();
 934
 935         dev->dev_ops = &i40e_eth_dev_ops;
 936         dev->rx_pkt_burst = i40e_recv_pkts;
 937         dev->tx_pkt_burst = i40e_xmit_pkts;
 938
 939         /* for secondary processes, we don't initialise any further as primary
 940          * has already done this work. Only check we don't need a different
 941          * RX function */
 942         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 943                 i40e_set_rx_function(dev);
 944                 i40e_set_tx_function(dev);
 945                 return 0;
 946         }
 947         pci_dev = dev->pci_dev;
 948
 949         rte_eth_copy_pci_info(dev, pci_dev);
 950
 951         pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 952         pf->adapter->eth_dev = dev;
 953         pf->dev_data = dev->data;
 954
 955         hw->back = I40E_PF_TO_ADAPTER(pf);
 956         hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
 957         if (!hw->hw_addr) {
 958                 PMD_INIT_LOG(ERR, "Hardware is not available, "
 959                              "as address is NULL");
 960                 return -ENODEV;
 961         }
 962
 963         hw->vendor_id = pci_dev->id.vendor_id;
 964         hw->device_id = pci_dev->id.device_id;
 965         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 966         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 967         hw->bus.device = pci_dev->addr.devid;
 968         hw->bus.func = pci_dev->addr.function;
 969         hw->adapter_stopped = 0;
 970
 971         /* Make sure all is clean before doing PF reset */
 972         i40e_clear_hw(hw);
 973
 974         /* Initialize the hardware */
 975         i40e_hw_init(dev);
 976
 977         /* Reset here to make sure all is clean for each PF */
 978         ret = i40e_pf_reset(hw);
 979         if (ret) {
 980                 PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
 981                 return ret;
 982         }
 983
 984         /* Initialize the shared code (base driver) */
 985         ret = i40e_init_shared_code(hw);
 986         if (ret) {
 987                 PMD_INIT_LOG(ERR, "Failed to init shared code (base driver): %d", ret);
 988                 return ret;
 989         }
 990
 991         /*
 992          * To work around the NVM issue, initialize registers
 993          * for flexible payload and packet type of QinQ by
 994          * software. It should be removed once issues are fixed
 995          * in NVM.
 996          */
 997         i40e_GLQF_reg_init(hw);
 998
 999         /* Initialize the input set for filters (hash and fd) to default value */
1000         i40e_filter_input_set_init(pf);
1001
1002         /* Initialize the parameters for adminq */
1003         i40e_init_adminq_parameter(hw);
1004         ret = i40e_init_adminq(hw);
1005         if (ret != I40E_SUCCESS) {
1006                 PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
1007                 return -EIO;
1008         }
1009         PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
1010                      hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
1011                      hw->aq.api_maj_ver, hw->aq.api_min_ver,
1012                      ((hw->nvm.version >> 12) & 0xf),
1013                      ((hw->nvm.version >> 4) & 0xff),
1014                      (hw->nvm.version & 0xf), hw->nvm.eetrack);
1015
1016         /* Need the special FW version to support floating VEB */
1017         config_floating_veb(dev);
1018         /* Clear PXE mode */
1019         i40e_clear_pxe_mode(hw);
1020
1021         /*
1022          * On X710, performance number is far from the expectation on recent
1023          * firmware versions. The fix for this issue may not be integrated in
1024          * the following firmware version. So the workaround in software driver
1025          * is needed. It needs to modify the initial values of 3 internal only
1026          * registers. Note that the workaround can be removed when it is fixed
1027          * in firmware in the future.
1028          */
1029         i40e_configure_registers(hw);
1030
1031         /* Get hw capabilities */
1032         ret = i40e_get_cap(hw);
1033         if (ret != I40E_SUCCESS) {
1034                 PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
1035                 goto err_get_capabilities;
1036         }
1037
1038         /* Initialize parameters for PF */
1039         ret = i40e_pf_parameter_init(dev);
1040         if (ret != 0) {
1041                 PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
1042                 goto err_parameter_init;
1043         }
1044
1045         /* Initialize the queue management */
1046         ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
1047         if (ret < 0) {
1048                 PMD_INIT_LOG(ERR, "Failed to init queue pool");
1049                 goto err_qp_pool_init;
1050         }
1051         ret = i40e_res_pool_init(&pf->msix_pool, 1,
1052                                 hw->func_caps.num_msix_vectors - 1);
1053         if (ret < 0) {
1054                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
1055                 goto err_msix_pool_init;
1056         }
1057
1058         /* Initialize lan hmc */
1059         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
1060                                 hw->func_caps.num_rx_qp, 0, 0);
1061         if (ret != I40E_SUCCESS) {
1062                 PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
1063                 goto err_init_lan_hmc;
1064         }
1065
1066         /* Configure lan hmc */
1067         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
1068         if (ret != I40E_SUCCESS) {
1069                 PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
1070                 goto err_configure_lan_hmc;
1071         }
1072
1073         /* Get and check the mac address */
1074         i40e_get_mac_addr(hw, hw->mac.addr);
1075         if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
1076                 PMD_INIT_LOG(ERR, "mac address is not valid");
1077                 ret = -EIO;
1078                 goto err_get_mac_addr;
1079         }
1080         /* Copy the permanent MAC address */
1081         ether_addr_copy((struct ether_addr *) hw->mac.addr,
1082                         (struct ether_addr *) hw->mac.perm_addr);
1083
1084         /* Disable flow control */
1085         hw->fc.requested_mode = I40E_FC_NONE;
1086         i40e_set_fc(hw, &aq_fail, TRUE);
1087
1088         /* Set the global registers with default ether type value */
1089         ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, ETHER_TYPE_VLAN);
1090         if (ret != I40E_SUCCESS) {
1091                 PMD_INIT_LOG(ERR, "Failed to set the default outer "
1092                              "VLAN ether type");
1093                 goto err_setup_pf_switch;
1094         }
1095
1096         /* PF setup, which includes VSI setup */
1097         ret = i40e_pf_setup(pf);
1098         if (ret) {
1099                 PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
1100                 goto err_setup_pf_switch;
1101         }
1102
1103         /* reset all stats of the device, including pf and main vsi */
1104         i40e_dev_stats_reset(dev);
1105
1106         vsi = pf->main_vsi;
1107
1108         /* Disable double vlan by default */
1109         i40e_vsi_config_double_vlan(vsi, FALSE);
1110
1111         /* Disable S-TAG identification by default */
1112         ret = I40E_READ_REG(hw, I40E_PRT_L2TAGSEN);
1113         if (ret & I40E_L2_TAGS_S_TAG_MASK) {
1114                 ret &= ~I40E_L2_TAGS_S_TAG_MASK;
1115                 I40E_WRITE_REG(hw, I40E_PRT_L2TAGSEN, ret);
1116         }
1117
1118         if (!vsi->max_macaddrs)
1119                 len = ETHER_ADDR_LEN;
1120         else
1121                 len = ETHER_ADDR_LEN * vsi->max_macaddrs;
1122
1123         /* Should be after VSI initialized */
1124         dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
1125         if (!dev->data->mac_addrs) {
1126                 PMD_INIT_LOG(ERR, "Failed to allocated memory "
1127                                         "for storing mac address");
1128                 goto err_mac_alloc;
1129         }
1130         ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
1131                                         &dev->data->mac_addrs[0]);
1132
1133         /* initialize pf host driver to setup SRIOV resource if applicable */
1134         i40e_pf_host_init(dev);
1135
1136         /* register callback func to eal lib */
1137         rte_intr_callback_register(&(pci_dev->intr_handle),
1138                 i40e_dev_interrupt_handler, (void *)dev);
1139
1140         /* configure and enable device interrupt */
1141         i40e_pf_config_irq0(hw, TRUE);
1142         i40e_pf_enable_irq0(hw);
1143
1144         /* enable uio intr after callback register */
1145         rte_intr_enable(&(pci_dev->intr_handle));
1146         /*
1147          * Add an ethertype filter to drop all flow control frames transmitted
1148          * from VSIs. By doing so, we stop VF from sending out PAUSE or PFC
1149          * frames to wire.
1150          */
1151         i40e_add_tx_flow_control_drop_filter(pf);
1152
1153         /* Set the max frame size to 0x2600 by default,
1154          * in case other drivers changed the default value.
1155          */
1156         i40e_aq_set_mac_config(hw, I40E_FRAME_SIZE_MAX, TRUE, 0, NULL);
1157
1158         /* initialize mirror rule list */
1159         TAILQ_INIT(&pf->mirror_list);
1160
1161         /* Init dcb to sw mode by default */
1162         ret = i40e_dcb_init_configure(dev, TRUE);
1163         if (ret != I40E_SUCCESS) {
1164                 PMD_INIT_LOG(INFO, "Failed to init dcb.");
1165                 pf->flags &= ~I40E_FLAG_DCB;
1166         }
1167
1168         return 0;
1169
1170 err_mac_alloc:
1171         i40e_vsi_release(pf->main_vsi);
1172 err_setup_pf_switch:
1173 err_get_mac_addr:
1174 err_configure_lan_hmc:
1175         (void)i40e_shutdown_lan_hmc(hw);
1176 err_init_lan_hmc:
1177         i40e_res_pool_destroy(&pf->msix_pool);
1178 err_msix_pool_init:
1179         i40e_res_pool_destroy(&pf->qp_pool);
1180 err_qp_pool_init:
1181 err_parameter_init:
1182 err_get_capabilities:
1183         (void)i40e_shutdown_adminq(hw);
1184
1185         return ret;
1186 }
1187
1188 static int
1189 eth_i40e_dev_uninit(struct rte_eth_dev *dev)
1190 {
1191         struct rte_pci_device *pci_dev;
1192         struct i40e_hw *hw;
1193         struct i40e_filter_control_settings settings;
1194         int ret;
1195         uint8_t aq_fail = 0;
1196
1197         PMD_INIT_FUNC_TRACE();
1198
1199         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1200                 return 0;
1201
1202         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1203         pci_dev = dev->pci_dev;
1204
1205         if (hw->adapter_stopped == 0)
1206                 i40e_dev_close(dev);
1207
1208         dev->dev_ops = NULL;
1209         dev->rx_pkt_burst = NULL;
1210         dev->tx_pkt_burst = NULL;
1211
1212         /* Disable LLDP */
1213         ret = i40e_aq_stop_lldp(hw, true, NULL);
1214         if (ret != I40E_SUCCESS) /* Its failure can be ignored */
1215                 PMD_INIT_LOG(INFO, "Failed to stop lldp");
1216
1217         /* Clear PXE mode */
1218         i40e_clear_pxe_mode(hw);
1219
1220         /* Unconfigure filter control */
1221         memset(&settings, 0, sizeof(settings));
1222         ret = i40e_set_filter_control(hw, &settings);
1223         if (ret)
1224                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
1225                                         ret);
1226
1227         /* Disable flow control */
1228         hw->fc.requested_mode = I40E_FC_NONE;
1229         i40e_set_fc(hw, &aq_fail, TRUE);
1230
1231         /* uninitialize pf host driver */
1232         i40e_pf_host_uninit(dev);
1233
1234         rte_free(dev->data->mac_addrs);
1235         dev->data->mac_addrs = NULL;
1236
1237         /* disable uio intr before callback unregister */
1238         rte_intr_disable(&(pci_dev->intr_handle));
1239
1240         /* register callback func to eal lib */
1241         rte_intr_callback_unregister(&(pci_dev->intr_handle),
1242                 i40e_dev_interrupt_handler, (void *)dev);
1243
1244         return 0;
1245 }
1246
1247 static int
1248 i40e_dev_configure(struct rte_eth_dev *dev)
1249 {
1250         struct i40e_adapter *ad =
1251                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1252         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1253         enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1254         int i, ret;
1255
1256         /* Initialize to TRUE. If any of Rx queues doesn't meet the
1257          * bulk allocation or vector Rx preconditions we will reset it.
1258          */
1259         ad->rx_bulk_alloc_allowed = true;
1260         ad->rx_vec_allowed = true;
1261         ad->tx_simple_allowed = true;
1262         ad->tx_vec_allowed = true;
1263
1264         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
1265                 ret = i40e_fdir_setup(pf);
1266                 if (ret != I40E_SUCCESS) {
1267                         PMD_DRV_LOG(ERR, "Failed to setup flow director.");
1268                         return -ENOTSUP;
1269                 }
1270                 ret = i40e_fdir_configure(dev);
1271                 if (ret < 0) {
1272                         PMD_DRV_LOG(ERR, "failed to configure fdir.");
1273                         goto err;
1274                 }
1275         } else
1276                 i40e_fdir_teardown(pf);
1277
1278         ret = i40e_dev_init_vlan(dev);
1279         if (ret < 0)
1280                 goto err;
1281
1282         /* VMDQ setup.
1283          *  Needs to move VMDQ setting out of i40e_pf_config_mq_rx() as VMDQ and
1284          *  RSS setting have different requirements.
1285          *  General PMD driver call sequence are NIC init, configure,
1286          *  rx/tx_queue_setup and dev_start. In rx/tx_queue_setup() function, it
1287          *  will try to lookup the VSI that specific queue belongs to if VMDQ
1288          *  applicable. So, VMDQ setting has to be done before
1289          *  rx/tx_queue_setup(). This function is good  to place vmdq_setup.
1290          *  For RSS setting, it will try to calculate actual configured RX queue
1291          *  number, which will be available after rx_queue_setup(). dev_start()
1292          *  function is good to place RSS setup.
1293          */
1294         if (mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
1295                 ret = i40e_vmdq_setup(dev);
1296                 if (ret)
1297                         goto err;
1298         }
1299
1300         if (mq_mode & ETH_MQ_RX_DCB_FLAG) {
1301                 ret = i40e_dcb_setup(dev);
1302                 if (ret) {
1303                         PMD_DRV_LOG(ERR, "failed to configure DCB.");
1304                         goto err_dcb;
1305                 }
1306         }
1307
1308         return 0;
1309
1310 err_dcb:
1311         /* need to release vmdq resource if exists */
1312         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1313                 i40e_vsi_release(pf->vmdq[i].vsi);
1314                 pf->vmdq[i].vsi = NULL;
1315         }
1316         rte_free(pf->vmdq);
1317         pf->vmdq = NULL;
1318 err:
1319         /* need to release fdir resource if exists */
1320         i40e_fdir_teardown(pf);
1321         return ret;
1322 }
1323
1324 void
1325 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
1326 {
1327         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1328         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1329         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1330         uint16_t msix_vect = vsi->msix_intr;
1331         uint16_t i;
1332
1333         for (i = 0; i < vsi->nb_qps; i++) {
1334                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1335                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1336                 rte_wmb();
1337         }
1338
1339         if (vsi->type != I40E_VSI_SRIOV) {
1340                 if (!rte_intr_allow_others(intr_handle)) {
1341                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1342                                        I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
1343                         I40E_WRITE_REG(hw,
1344                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1345                                        0);
1346                 } else {
1347                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1348                                        I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK);
1349                         I40E_WRITE_REG(hw,
1350                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1351                                                        msix_vect - 1), 0);
1352                 }
1353         } else {
1354                 uint32_t reg;
1355                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1356                         vsi->user_param + (msix_vect - 1);
1357
1358                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1359                                I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
1360         }
1361         I40E_WRITE_FLUSH(hw);
1362 }
1363
1364 static void
1365 __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect,
1366                        int base_queue, int nb_queue)
1367 {
1368         int i;
1369         uint32_t val;
1370         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1371
1372         /* Bind all RX queues to allocated MSIX interrupt */
1373         for (i = 0; i < nb_queue; i++) {
1374                 val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
1375                         I40E_QINT_RQCTL_ITR_INDX_MASK |
1376                         ((base_queue + i + 1) <<
1377                          I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
1378                         (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
1379                         I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1380
1381                 if (i == nb_queue - 1)
1382                         val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
1383                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(base_queue + i), val);
1384         }
1385
1386         /* Write first RX queue to Link list register as the head element */
1387         if (vsi->type != I40E_VSI_SRIOV) {
1388                 uint16_t interval =
1389                         i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
1390
1391                 if (msix_vect == I40E_MISC_VEC_ID) {
1392                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1393                                        (base_queue <<
1394                                         I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1395                                        (0x0 <<
1396                                         I40E_PFINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1397                         I40E_WRITE_REG(hw,
1398                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1399                                        interval);
1400                 } else {
1401                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1402                                        (base_queue <<
1403                                         I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1404                                        (0x0 <<
1405                                         I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1406                         I40E_WRITE_REG(hw,
1407                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1408                                                        msix_vect - 1),
1409                                        interval);
1410                 }
1411         } else {
1412                 uint32_t reg;
1413
1414                 if (msix_vect == I40E_MISC_VEC_ID) {
1415                         I40E_WRITE_REG(hw,
1416                                        I40E_VPINT_LNKLST0(vsi->user_param),
1417                                        (base_queue <<
1418                                         I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1419                                        (0x0 <<
1420                                         I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1421                 } else {
1422                         /* num_msix_vectors_vf needs to minus irq0 */
1423                         reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1424                                 vsi->user_param + (msix_vect - 1);
1425
1426                         I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1427                                        (base_queue <<
1428                                         I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1429                                        (0x0 <<
1430                                         I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1431                 }
1432         }
1433
1434         I40E_WRITE_FLUSH(hw);
1435 }
1436
1437 void
1438 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi)
1439 {
1440         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1441         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1442         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1443         uint16_t msix_vect = vsi->msix_intr;
1444         uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd);
1445         uint16_t queue_idx = 0;
1446         int record = 0;
1447         uint32_t val;
1448         int i;
1449
1450         for (i = 0; i < vsi->nb_qps; i++) {
1451                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1452                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1453         }
1454
1455         /* INTENA flag is not auto-cleared for interrupt */
1456         val = I40E_READ_REG(hw, I40E_GLINT_CTL);
1457         val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK |
1458                 I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK |
1459                 I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK;
1460         I40E_WRITE_REG(hw, I40E_GLINT_CTL, val);
1461
1462         /* VF bind interrupt */
1463         if (vsi->type == I40E_VSI_SRIOV) {
1464                 __vsi_queues_bind_intr(vsi, msix_vect,
1465                                        vsi->base_queue, vsi->nb_qps);
1466                 return;
1467         }
1468
1469         /* PF & VMDq bind interrupt */
1470         if (rte_intr_dp_is_en(intr_handle)) {
1471                 if (vsi->type == I40E_VSI_MAIN) {
1472                         queue_idx = 0;
1473                         record = 1;
1474                 } else if (vsi->type == I40E_VSI_VMDQ2) {
1475                         struct i40e_vsi *main_vsi =
1476                                 I40E_DEV_PRIVATE_TO_MAIN_VSI(vsi->adapter);
1477                         queue_idx = vsi->base_queue - main_vsi->nb_qps;
1478                         record = 1;
1479                 }
1480         }
1481
1482         for (i = 0; i < vsi->nb_used_qps; i++) {
1483                 if (nb_msix <= 1) {
1484                         if (!rte_intr_allow_others(intr_handle))
1485                                 /* allow to share MISC_VEC_ID */
1486                                 msix_vect = I40E_MISC_VEC_ID;
1487
1488                         /* no enough msix_vect, map all to one */
1489                         __vsi_queues_bind_intr(vsi, msix_vect,
1490                                                vsi->base_queue + i,
1491                                                vsi->nb_used_qps - i);
1492                         for (; !!record && i < vsi->nb_used_qps; i++)
1493                                 intr_handle->intr_vec[queue_idx + i] =
1494                                         msix_vect;
1495                         break;
1496                 }
1497                 /* 1:1 queue/msix_vect mapping */
1498                 __vsi_queues_bind_intr(vsi, msix_vect,
1499                                        vsi->base_queue + i, 1);
1500                 if (!!record)
1501                         intr_handle->intr_vec[queue_idx + i] = msix_vect;
1502
1503                 msix_vect++;
1504                 nb_msix--;
1505         }
1506 }
1507
1508 static void
1509 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
1510 {
1511         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1512         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1513         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1514         uint16_t interval = i40e_calc_itr_interval(\
1515                 RTE_LIBRTE_I40E_ITR_INTERVAL);
1516         uint16_t msix_intr, i;
1517
1518         if (rte_intr_allow_others(intr_handle))
1519                 for (i = 0; i < vsi->nb_msix; i++) {
1520                         msix_intr = vsi->msix_intr + i;
1521                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1522                                 I40E_PFINT_DYN_CTLN_INTENA_MASK |
1523                                 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
1524                                 (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
1525                                 (interval <<
1526                                  I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
1527                 }
1528         else
1529                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
1530                                I40E_PFINT_DYN_CTL0_INTENA_MASK |
1531                                I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
1532                                (0 << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT) |
1533                                (interval <<
1534                                 I40E_PFINT_DYN_CTL0_INTERVAL_SHIFT));
1535
1536         I40E_WRITE_FLUSH(hw);
1537 }
1538
1539 static void
1540 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
1541 {
1542         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1543         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1544         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1545         uint16_t msix_intr, i;
1546
1547         if (rte_intr_allow_others(intr_handle))
1548                 for (i = 0; i < vsi->nb_msix; i++) {
1549                         msix_intr = vsi->msix_intr + i;
1550                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1551                                        0);
1552                 }
1553         else
1554                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
1555
1556         I40E_WRITE_FLUSH(hw);
1557 }
1558
1559 static inline uint8_t
1560 i40e_parse_link_speeds(uint16_t link_speeds)
1561 {
1562         uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN;
1563
1564         if (link_speeds & ETH_LINK_SPEED_40G)
1565                 link_speed |= I40E_LINK_SPEED_40GB;
1566         if (link_speeds & ETH_LINK_SPEED_20G)
1567                 link_speed |= I40E_LINK_SPEED_20GB;
1568         if (link_speeds & ETH_LINK_SPEED_10G)
1569                 link_speed |= I40E_LINK_SPEED_10GB;
1570         if (link_speeds & ETH_LINK_SPEED_1G)
1571                 link_speed |= I40E_LINK_SPEED_1GB;
1572         if (link_speeds & ETH_LINK_SPEED_100M)
1573                 link_speed |= I40E_LINK_SPEED_100MB;
1574
1575         return link_speed;
1576 }
1577
1578 static int
1579 i40e_phy_conf_link(struct i40e_hw *hw,
1580                    uint8_t abilities,
1581                    uint8_t force_speed)
1582 {
1583         enum i40e_status_code status;
1584         struct i40e_aq_get_phy_abilities_resp phy_ab;
1585         struct i40e_aq_set_phy_config phy_conf;
1586         const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX |
1587                         I40E_AQ_PHY_FLAG_PAUSE_RX |
1588                         I40E_AQ_PHY_FLAG_PAUSE_RX |
1589                         I40E_AQ_PHY_FLAG_LOW_POWER;
1590         const uint8_t advt = I40E_LINK_SPEED_40GB |
1591                         I40E_LINK_SPEED_10GB |
1592                         I40E_LINK_SPEED_1GB |
1593                         I40E_LINK_SPEED_100MB;
1594         int ret = -ENOTSUP;
1595
1596
1597         status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab,
1598                                               NULL);
1599         if (status)
1600                 return ret;
1601
1602         memset(&phy_conf, 0, sizeof(phy_conf));
1603
1604         /* bits 0-2 use the values from get_phy_abilities_resp */
1605         abilities &= ~mask;
1606         abilities |= phy_ab.abilities & mask;
1607
1608         /* update ablities and speed */
1609         if (abilities & I40E_AQ_PHY_AN_ENABLED)
1610                 phy_conf.link_speed = advt;
1611         else
1612                 phy_conf.link_speed = force_speed;
1613
1614         phy_conf.abilities = abilities;
1615
1616         /* use get_phy_abilities_resp value for the rest */
1617         phy_conf.phy_type = phy_ab.phy_type;
1618         phy_conf.eee_capability = phy_ab.eee_capability;
1619         phy_conf.eeer = phy_ab.eeer_val;
1620         phy_conf.low_power_ctrl = phy_ab.d3_lpan;
1621
1622         PMD_DRV_LOG(DEBUG, "\tCurrent: abilities %x, link_speed %x",
1623                     phy_ab.abilities, phy_ab.link_speed);
1624         PMD_DRV_LOG(DEBUG, "\tConfig:  abilities %x, link_speed %x",
1625                     phy_conf.abilities, phy_conf.link_speed);
1626
1627         status = i40e_aq_set_phy_config(hw, &phy_conf, NULL);
1628         if (status)
1629                 return ret;
1630
1631         return I40E_SUCCESS;
1632 }
1633
1634 static int
1635 i40e_apply_link_speed(struct rte_eth_dev *dev)
1636 {
1637         uint8_t speed;
1638         uint8_t abilities = 0;
1639         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1640         struct rte_eth_conf *conf = &dev->data->dev_conf;
1641
1642         speed = i40e_parse_link_speeds(conf->link_speeds);
1643         abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
1644         if (!(conf->link_speeds & ETH_LINK_SPEED_FIXED))
1645                 abilities |= I40E_AQ_PHY_AN_ENABLED;
1646         abilities |= I40E_AQ_PHY_LINK_ENABLED;
1647
1648         /* Skip changing speed on 40G interfaces, FW does not support */
1649         if (i40e_is_40G_device(hw->device_id)) {
1650                 speed =  I40E_LINK_SPEED_UNKNOWN;
1651                 abilities |= I40E_AQ_PHY_AN_ENABLED;
1652         }
1653
1654         return i40e_phy_conf_link(hw, abilities, speed);
1655 }
1656
1657 static int
1658 i40e_dev_start(struct rte_eth_dev *dev)
1659 {
1660         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1661         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1662         struct i40e_vsi *main_vsi = pf->main_vsi;
1663         int ret, i;
1664         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1665         uint32_t intr_vector = 0;
1666
1667         hw->adapter_stopped = 0;
1668
1669         if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
1670                 PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; autonegotiation disabled",
1671                              dev->data->port_id);
1672                 return -EINVAL;
1673         }
1674
1675         rte_intr_disable(intr_handle);
1676
1677         if ((rte_intr_cap_multiple(intr_handle) ||
1678              !RTE_ETH_DEV_SRIOV(dev).active) &&
1679             dev->data->dev_conf.intr_conf.rxq != 0) {
1680                 intr_vector = dev->data->nb_rx_queues;
1681                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1682                         return -1;
1683         }
1684
1685         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1686                 intr_handle->intr_vec =
1687                         rte_zmalloc("intr_vec",
1688                                     dev->data->nb_rx_queues * sizeof(int),
1689                                     0);
1690                 if (!intr_handle->intr_vec) {
1691                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1692                                      " intr_vec\n", dev->data->nb_rx_queues);
1693                         return -ENOMEM;
1694                 }
1695         }
1696
1697         /* Initialize VSI */
1698         ret = i40e_dev_rxtx_init(pf);
1699         if (ret != I40E_SUCCESS) {
1700                 PMD_DRV_LOG(ERR, "Failed to init rx/tx queues");
1701                 goto err_up;
1702         }
1703
1704         /* Map queues with MSIX interrupt */
1705         main_vsi->nb_used_qps = dev->data->nb_rx_queues -
1706                 pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1707         i40e_vsi_queues_bind_intr(main_vsi);
1708         i40e_vsi_enable_queues_intr(main_vsi);
1709
1710         /* Map VMDQ VSI queues with MSIX interrupt */
1711         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1712                 pf->vmdq[i].vsi->nb_used_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1713                 i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi);
1714                 i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi);
1715         }
1716
1717         /* enable FDIR MSIX interrupt */
1718         if (pf->fdir.fdir_vsi) {
1719                 i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi);
1720                 i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi);
1721         }
1722
1723         /* Enable all queues which have been configured */
1724         ret = i40e_dev_switch_queues(pf, TRUE);
1725         if (ret != I40E_SUCCESS) {
1726                 PMD_DRV_LOG(ERR, "Failed to enable VSI");
1727                 goto err_up;
1728         }
1729
1730         /* Enable receiving broadcast packets */
1731         ret = i40e_aq_set_vsi_broadcast(hw, main_vsi->seid, true, NULL);
1732         if (ret != I40E_SUCCESS)
1733                 PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1734
1735         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1736                 ret = i40e_aq_set_vsi_broadcast(hw, pf->vmdq[i].vsi->seid,
1737                                                 true, NULL);
1738                 if (ret != I40E_SUCCESS)
1739                         PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1740         }
1741
1742         /* Apply link configure */
1743         if (dev->data->dev_conf.link_speeds & ~(ETH_LINK_SPEED_100M |
1744                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G |
1745                                 ETH_LINK_SPEED_20G | ETH_LINK_SPEED_40G)) {
1746                 PMD_DRV_LOG(ERR, "Invalid link setting");
1747                 goto err_up;
1748         }
1749         ret = i40e_apply_link_speed(dev);
1750         if (I40E_SUCCESS != ret) {
1751                 PMD_DRV_LOG(ERR, "Fail to apply link setting");
1752                 goto err_up;
1753         }
1754
1755         if (!rte_intr_allow_others(intr_handle)) {
1756                 rte_intr_callback_unregister(intr_handle,
1757                                              i40e_dev_interrupt_handler,
1758                                              (void *)dev);
1759                 /* configure and enable device interrupt */
1760                 i40e_pf_config_irq0(hw, FALSE);
1761                 i40e_pf_enable_irq0(hw);
1762
1763                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1764                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1765                                      " no intr multiplex\n");
1766         }
1767
1768         /* enable uio intr after callback register */
1769         rte_intr_enable(intr_handle);
1770
1771         return I40E_SUCCESS;
1772
1773 err_up:
1774         i40e_dev_switch_queues(pf, FALSE);
1775         i40e_dev_clear_queues(dev);
1776
1777         return ret;
1778 }
1779
1780 static void
1781 i40e_dev_stop(struct rte_eth_dev *dev)
1782 {
1783         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1784         struct i40e_vsi *main_vsi = pf->main_vsi;
1785         struct i40e_mirror_rule *p_mirror;
1786         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1787         int i;
1788
1789         /* Disable all queues */
1790         i40e_dev_switch_queues(pf, FALSE);
1791
1792         /* un-map queues with interrupt registers */
1793         i40e_vsi_disable_queues_intr(main_vsi);
1794         i40e_vsi_queues_unbind_intr(main_vsi);
1795
1796         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1797                 i40e_vsi_disable_queues_intr(pf->vmdq[i].vsi);
1798                 i40e_vsi_queues_unbind_intr(pf->vmdq[i].vsi);
1799         }
1800
1801         if (pf->fdir.fdir_vsi) {
1802                 i40e_vsi_queues_unbind_intr(pf->fdir.fdir_vsi);
1803                 i40e_vsi_disable_queues_intr(pf->fdir.fdir_vsi);
1804         }
1805         /* Clear all queues and release memory */
1806         i40e_dev_clear_queues(dev);
1807
1808         /* Set link down */
1809         i40e_dev_set_link_down(dev);
1810
1811         /* Remove all mirror rules */
1812         while ((p_mirror = TAILQ_FIRST(&pf->mirror_list))) {
1813                 TAILQ_REMOVE(&pf->mirror_list, p_mirror, rules);
1814                 rte_free(p_mirror);
1815         }
1816         pf->nb_mirror_rule = 0;
1817
1818         if (!rte_intr_allow_others(intr_handle))
1819                 /* resume to the default handler */
1820                 rte_intr_callback_register(intr_handle,
1821                                            i40e_dev_interrupt_handler,
1822                                            (void *)dev);
1823
1824         /* Clean datapath event and queue/vec mapping */
1825         rte_intr_efd_disable(intr_handle);
1826         if (intr_handle->intr_vec) {
1827                 rte_free(intr_handle->intr_vec);
1828                 intr_handle->intr_vec = NULL;
1829         }
1830 }
1831
1832 static void
1833 i40e_dev_close(struct rte_eth_dev *dev)
1834 {
1835         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1836         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1837         uint32_t reg;
1838         int i;
1839
1840         PMD_INIT_FUNC_TRACE();
1841
1842         i40e_dev_stop(dev);
1843         hw->adapter_stopped = 1;
1844         i40e_dev_free_queues(dev);
1845
1846         /* Disable interrupt */
1847         i40e_pf_disable_irq0(hw);
1848         rte_intr_disable(&(dev->pci_dev->intr_handle));
1849
1850         /* shutdown and destroy the HMC */
1851         i40e_shutdown_lan_hmc(hw);
1852
1853         /* release all the existing VSIs and VEBs */
1854         i40e_fdir_teardown(pf);
1855         i40e_vsi_release(pf->main_vsi);
1856
1857         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1858                 i40e_vsi_release(pf->vmdq[i].vsi);
1859                 pf->vmdq[i].vsi = NULL;
1860         }
1861
1862         rte_free(pf->vmdq);
1863         pf->vmdq = NULL;
1864
1865         /* shutdown the adminq */
1866         i40e_aq_queue_shutdown(hw, true);
1867         i40e_shutdown_adminq(hw);
1868
1869         i40e_res_pool_destroy(&pf->qp_pool);
1870         i40e_res_pool_destroy(&pf->msix_pool);
1871
1872         /* force a PF reset to clean anything leftover */
1873         reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
1874         I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
1875                         (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
1876         I40E_WRITE_FLUSH(hw);
1877 }
1878
1879 static void
1880 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
1881 {
1882         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1883         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1884         struct i40e_vsi *vsi = pf->main_vsi;
1885         int status;
1886
1887         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
1888                                                      true, NULL, true);
1889         if (status != I40E_SUCCESS)
1890                 PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous");
1891
1892         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
1893                                                         TRUE, NULL);
1894         if (status != I40E_SUCCESS)
1895                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
1896
1897 }
1898
1899 static void
1900 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
1901 {
1902         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1903         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1904         struct i40e_vsi *vsi = pf->main_vsi;
1905         int status;
1906
1907         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
1908                                                      false, NULL, true);
1909         if (status != I40E_SUCCESS)
1910                 PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
1911
1912         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
1913                                                         false, NULL);
1914         if (status != I40E_SUCCESS)
1915                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1916 }
1917
1918 static void
1919 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
1920 {
1921         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1922         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1923         struct i40e_vsi *vsi = pf->main_vsi;
1924         int ret;
1925
1926         ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
1927         if (ret != I40E_SUCCESS)
1928                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
1929 }
1930
1931 static void
1932 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
1933 {
1934         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1935         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1936         struct i40e_vsi *vsi = pf->main_vsi;
1937         int ret;
1938
1939         if (dev->data->promiscuous == 1)
1940                 return; /* must remain in all_multicast mode */
1941
1942         ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
1943                                 vsi->seid, FALSE, NULL);
1944         if (ret != I40E_SUCCESS)
1945                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1946 }
1947
1948 /*
1949  * Set device link up.
1950  */
1951 static int
1952 i40e_dev_set_link_up(struct rte_eth_dev *dev)
1953 {
1954         /* re-apply link speed setting */
1955         return i40e_apply_link_speed(dev);
1956 }
1957
1958 /*
1959  * Set device link down.
1960  */
1961 static int
1962 i40e_dev_set_link_down(struct rte_eth_dev *dev)
1963 {
1964         uint8_t speed = I40E_LINK_SPEED_UNKNOWN;
1965         uint8_t abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
1966         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1967
1968         return i40e_phy_conf_link(hw, abilities, speed);
1969 }
1970
1971 int
1972 i40e_dev_link_update(struct rte_eth_dev *dev,
1973                      int wait_to_complete)
1974 {
1975 #define CHECK_INTERVAL 100  /* 100ms */
1976 #define MAX_REPEAT_TIME 10  /* 1s (10 * 100ms) in total */
1977         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1978         struct i40e_link_status link_status;
1979         struct rte_eth_link link, old;
1980         int status;
1981         unsigned rep_cnt = MAX_REPEAT_TIME;
1982
1983         memset(&link, 0, sizeof(link));
1984         memset(&old, 0, sizeof(old));
1985         memset(&link_status, 0, sizeof(link_status));
1986         rte_i40e_dev_atomic_read_link_status(dev, &old);
1987
1988         do {
1989                 /* Get link status information from hardware */
1990                 status = i40e_aq_get_link_info(hw, false, &link_status, NULL);
1991                 if (status != I40E_SUCCESS) {
1992                         link.link_speed = ETH_SPEED_NUM_100M;
1993                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1994                         PMD_DRV_LOG(ERR, "Failed to get link info");
1995                         goto out;
1996                 }
1997
1998                 link.link_status = link_status.link_info & I40E_AQ_LINK_UP;
1999                 if (!wait_to_complete)
2000                         break;
2001
2002                 rte_delay_ms(CHECK_INTERVAL);
2003         } while (!link.link_status && rep_cnt--);
2004
2005         if (!link.link_status)
2006                 goto out;
2007
2008         /* i40e uses full duplex only */
2009         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2010
2011         /* Parse the link status */
2012         switch (link_status.link_speed) {
2013         case I40E_LINK_SPEED_100MB:
2014                 link.link_speed = ETH_SPEED_NUM_100M;
2015                 break;
2016         case I40E_LINK_SPEED_1GB:
2017                 link.link_speed = ETH_SPEED_NUM_1G;
2018                 break;
2019         case I40E_LINK_SPEED_10GB:
2020                 link.link_speed = ETH_SPEED_NUM_10G;
2021                 break;
2022         case I40E_LINK_SPEED_20GB:
2023                 link.link_speed = ETH_SPEED_NUM_20G;
2024                 break;
2025         case I40E_LINK_SPEED_40GB:
2026                 link.link_speed = ETH_SPEED_NUM_40G;
2027                 break;
2028         default:
2029                 link.link_speed = ETH_SPEED_NUM_100M;
2030                 break;
2031         }
2032
2033         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
2034                         ETH_LINK_SPEED_FIXED);
2035
2036 out:
2037         rte_i40e_dev_atomic_write_link_status(dev, &link);
2038         if (link.link_status == old.link_status)
2039                 return -1;
2040
2041         return 0;
2042 }
2043
2044 /* Get all the statistics of a VSI */
2045 void
2046 i40e_update_vsi_stats(struct i40e_vsi *vsi)
2047 {
2048         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
2049         struct i40e_eth_stats *nes = &vsi->eth_stats;
2050         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2051         int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
2052
2053         i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
2054                             vsi->offset_loaded, &oes->rx_bytes,
2055                             &nes->rx_bytes);
2056         i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
2057                             vsi->offset_loaded, &oes->rx_unicast,
2058                             &nes->rx_unicast);
2059         i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
2060                             vsi->offset_loaded, &oes->rx_multicast,
2061                             &nes->rx_multicast);
2062         i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
2063                             vsi->offset_loaded, &oes->rx_broadcast,
2064                             &nes->rx_broadcast);
2065         i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
2066                             &oes->rx_discards, &nes->rx_discards);
2067         /* GLV_REPC not supported */
2068         /* GLV_RMPC not supported */
2069         i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
2070                             &oes->rx_unknown_protocol,
2071                             &nes->rx_unknown_protocol);
2072         i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
2073                             vsi->offset_loaded, &oes->tx_bytes,
2074                             &nes->tx_bytes);
2075         i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
2076                             vsi->offset_loaded, &oes->tx_unicast,
2077                             &nes->tx_unicast);
2078         i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
2079                             vsi->offset_loaded, &oes->tx_multicast,
2080                             &nes->tx_multicast);
2081         i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
2082                             vsi->offset_loaded,  &oes->tx_broadcast,
2083                             &nes->tx_broadcast);
2084         /* GLV_TDPC not supported */
2085         i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
2086                             &oes->tx_errors, &nes->tx_errors);
2087         vsi->offset_loaded = true;
2088
2089         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats start *******************",
2090                     vsi->vsi_id);
2091         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", nes->rx_bytes);
2092         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", nes->rx_unicast);
2093         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", nes->rx_multicast);
2094         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", nes->rx_broadcast);
2095         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", nes->rx_discards);
2096         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2097                     nes->rx_unknown_protocol);
2098         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", nes->tx_bytes);
2099         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", nes->tx_unicast);
2100         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", nes->tx_multicast);
2101         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", nes->tx_broadcast);
2102         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", nes->tx_discards);
2103         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", nes->tx_errors);
2104         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats end *******************",
2105                     vsi->vsi_id);
2106 }
2107
2108 static void
2109 i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw)
2110 {
2111         unsigned int i;
2112         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2113         struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
2114
2115         /* Get statistics of struct i40e_eth_stats */
2116         i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
2117                             I40E_GLPRT_GORCL(hw->port),
2118                             pf->offset_loaded, &os->eth.rx_bytes,
2119                             &ns->eth.rx_bytes);
2120         i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
2121                             I40E_GLPRT_UPRCL(hw->port),
2122                             pf->offset_loaded, &os->eth.rx_unicast,
2123                             &ns->eth.rx_unicast);
2124         i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
2125                             I40E_GLPRT_MPRCL(hw->port),
2126                             pf->offset_loaded, &os->eth.rx_multicast,
2127                             &ns->eth.rx_multicast);
2128         i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
2129                             I40E_GLPRT_BPRCL(hw->port),
2130                             pf->offset_loaded, &os->eth.rx_broadcast,
2131                             &ns->eth.rx_broadcast);
2132         /* Workaround: CRC size should not be included in byte statistics,
2133          * so subtract ETHER_CRC_LEN from the byte counter for each rx packet.
2134          */
2135         ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast +
2136                 ns->eth.rx_broadcast) * ETHER_CRC_LEN;
2137
2138         i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
2139                             pf->offset_loaded, &os->eth.rx_discards,
2140                             &ns->eth.rx_discards);
2141         /* GLPRT_REPC not supported */
2142         /* GLPRT_RMPC not supported */
2143         i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
2144                             pf->offset_loaded,
2145                             &os->eth.rx_unknown_protocol,
2146                             &ns->eth.rx_unknown_protocol);
2147         i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
2148                             I40E_GLPRT_GOTCL(hw->port),
2149                             pf->offset_loaded, &os->eth.tx_bytes,
2150                             &ns->eth.tx_bytes);
2151         i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
2152                             I40E_GLPRT_UPTCL(hw->port),
2153                             pf->offset_loaded, &os->eth.tx_unicast,
2154                             &ns->eth.tx_unicast);
2155         i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
2156                             I40E_GLPRT_MPTCL(hw->port),
2157                             pf->offset_loaded, &os->eth.tx_multicast,
2158                             &ns->eth.tx_multicast);
2159         i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
2160                             I40E_GLPRT_BPTCL(hw->port),
2161                             pf->offset_loaded, &os->eth.tx_broadcast,
2162                             &ns->eth.tx_broadcast);
2163         ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast +
2164                 ns->eth.tx_broadcast) * ETHER_CRC_LEN;
2165         /* GLPRT_TEPC not supported */
2166
2167         /* additional port specific stats */
2168         i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
2169                             pf->offset_loaded, &os->tx_dropped_link_down,
2170                             &ns->tx_dropped_link_down);
2171         i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
2172                             pf->offset_loaded, &os->crc_errors,
2173                             &ns->crc_errors);
2174         i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
2175                             pf->offset_loaded, &os->illegal_bytes,
2176                             &ns->illegal_bytes);
2177         /* GLPRT_ERRBC not supported */
2178         i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
2179                             pf->offset_loaded, &os->mac_local_faults,
2180                             &ns->mac_local_faults);
2181         i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
2182                             pf->offset_loaded, &os->mac_remote_faults,
2183                             &ns->mac_remote_faults);
2184         i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
2185                             pf->offset_loaded, &os->rx_length_errors,
2186                             &ns->rx_length_errors);
2187         i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
2188                             pf->offset_loaded, &os->link_xon_rx,
2189                             &ns->link_xon_rx);
2190         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
2191                             pf->offset_loaded, &os->link_xoff_rx,
2192                             &ns->link_xoff_rx);
2193         for (i = 0; i < 8; i++) {
2194                 i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
2195                                     pf->offset_loaded,
2196                                     &os->priority_xon_rx[i],
2197                                     &ns->priority_xon_rx[i]);
2198                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
2199                                     pf->offset_loaded,
2200                                     &os->priority_xoff_rx[i],
2201                                     &ns->priority_xoff_rx[i]);
2202         }
2203         i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
2204                             pf->offset_loaded, &os->link_xon_tx,
2205                             &ns->link_xon_tx);
2206         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
2207                             pf->offset_loaded, &os->link_xoff_tx,
2208                             &ns->link_xoff_tx);
2209         for (i = 0; i < 8; i++) {
2210                 i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
2211                                     pf->offset_loaded,
2212                                     &os->priority_xon_tx[i],
2213                                     &ns->priority_xon_tx[i]);
2214                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
2215                                     pf->offset_loaded,
2216                                     &os->priority_xoff_tx[i],
2217                                     &ns->priority_xoff_tx[i]);
2218                 i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
2219                                     pf->offset_loaded,
2220                                     &os->priority_xon_2_xoff[i],
2221                                     &ns->priority_xon_2_xoff[i]);
2222         }
2223         i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
2224                             I40E_GLPRT_PRC64L(hw->port),
2225                             pf->offset_loaded, &os->rx_size_64,
2226                             &ns->rx_size_64);
2227         i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
2228                             I40E_GLPRT_PRC127L(hw->port),
2229                             pf->offset_loaded, &os->rx_size_127,
2230                             &ns->rx_size_127);
2231         i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
2232                             I40E_GLPRT_PRC255L(hw->port),
2233                             pf->offset_loaded, &os->rx_size_255,
2234                             &ns->rx_size_255);
2235         i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
2236                             I40E_GLPRT_PRC511L(hw->port),
2237                             pf->offset_loaded, &os->rx_size_511,
2238                             &ns->rx_size_511);
2239         i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
2240                             I40E_GLPRT_PRC1023L(hw->port),
2241                             pf->offset_loaded, &os->rx_size_1023,
2242                             &ns->rx_size_1023);
2243         i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
2244                             I40E_GLPRT_PRC1522L(hw->port),
2245                             pf->offset_loaded, &os->rx_size_1522,
2246                             &ns->rx_size_1522);
2247         i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
2248                             I40E_GLPRT_PRC9522L(hw->port),
2249                             pf->offset_loaded, &os->rx_size_big,
2250                             &ns->rx_size_big);
2251         i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
2252                             pf->offset_loaded, &os->rx_undersize,
2253                             &ns->rx_undersize);
2254         i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
2255                             pf->offset_loaded, &os->rx_fragments,
2256                             &ns->rx_fragments);
2257         i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
2258                             pf->offset_loaded, &os->rx_oversize,
2259                             &ns->rx_oversize);
2260         i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
2261                             pf->offset_loaded, &os->rx_jabber,
2262                             &ns->rx_jabber);
2263         i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
2264                             I40E_GLPRT_PTC64L(hw->port),
2265                             pf->offset_loaded, &os->tx_size_64,
2266                             &ns->tx_size_64);
2267         i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
2268                             I40E_GLPRT_PTC127L(hw->port),
2269                             pf->offset_loaded, &os->tx_size_127,
2270                             &ns->tx_size_127);
2271         i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
2272                             I40E_GLPRT_PTC255L(hw->port),
2273                             pf->offset_loaded, &os->tx_size_255,
2274                             &ns->tx_size_255);
2275         i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
2276                             I40E_GLPRT_PTC511L(hw->port),
2277                             pf->offset_loaded, &os->tx_size_511,
2278                             &ns->tx_size_511);
2279         i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
2280                             I40E_GLPRT_PTC1023L(hw->port),
2281                             pf->offset_loaded, &os->tx_size_1023,
2282                             &ns->tx_size_1023);
2283         i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
2284                             I40E_GLPRT_PTC1522L(hw->port),
2285                             pf->offset_loaded, &os->tx_size_1522,
2286                             &ns->tx_size_1522);
2287         i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
2288                             I40E_GLPRT_PTC9522L(hw->port),
2289                             pf->offset_loaded, &os->tx_size_big,
2290                             &ns->tx_size_big);
2291         i40e_stat_update_32(hw, I40E_GLQF_PCNT(pf->fdir.match_counter_index),
2292                            pf->offset_loaded,
2293                            &os->fd_sb_match, &ns->fd_sb_match);
2294         /* GLPRT_MSPDC not supported */
2295         /* GLPRT_XEC not supported */
2296
2297         pf->offset_loaded = true;
2298
2299         if (pf->main_vsi)
2300                 i40e_update_vsi_stats(pf->main_vsi);
2301 }
2302
2303 /* Get all statistics of a port */
2304 static void
2305 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2306 {
2307         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2308         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2309         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2310         unsigned i;
2311
2312         /* call read registers - updates values, now write them to struct */
2313         i40e_read_stats_registers(pf, hw);
2314
2315         stats->ipackets = pf->main_vsi->eth_stats.rx_unicast +
2316                         pf->main_vsi->eth_stats.rx_multicast +
2317                         pf->main_vsi->eth_stats.rx_broadcast -
2318                         pf->main_vsi->eth_stats.rx_discards;
2319         stats->opackets = pf->main_vsi->eth_stats.tx_unicast +
2320                         pf->main_vsi->eth_stats.tx_multicast +
2321                         pf->main_vsi->eth_stats.tx_broadcast;
2322         stats->ibytes   = ns->eth.rx_bytes;
2323         stats->obytes   = ns->eth.tx_bytes;
2324         stats->oerrors  = ns->eth.tx_errors +
2325                         pf->main_vsi->eth_stats.tx_errors;
2326
2327         /* Rx Errors */
2328         stats->imissed  = ns->eth.rx_discards +
2329                         pf->main_vsi->eth_stats.rx_discards;
2330         stats->ierrors  = ns->crc_errors +
2331                         ns->rx_length_errors + ns->rx_undersize +
2332                         ns->rx_oversize + ns->rx_fragments + ns->rx_jabber;
2333
2334         PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************");
2335         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", ns->eth.rx_bytes);
2336         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", ns->eth.rx_unicast);
2337         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", ns->eth.rx_multicast);
2338         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", ns->eth.rx_broadcast);
2339         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", ns->eth.rx_discards);
2340         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2341                     ns->eth.rx_unknown_protocol);
2342         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", ns->eth.tx_bytes);
2343         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", ns->eth.tx_unicast);
2344         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", ns->eth.tx_multicast);
2345         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", ns->eth.tx_broadcast);
2346         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", ns->eth.tx_discards);
2347         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", ns->eth.tx_errors);
2348
2349         PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:     %"PRIu64"",
2350                     ns->tx_dropped_link_down);
2351         PMD_DRV_LOG(DEBUG, "crc_errors:               %"PRIu64"", ns->crc_errors);
2352         PMD_DRV_LOG(DEBUG, "illegal_bytes:            %"PRIu64"",
2353                     ns->illegal_bytes);
2354         PMD_DRV_LOG(DEBUG, "error_bytes:              %"PRIu64"", ns->error_bytes);
2355         PMD_DRV_LOG(DEBUG, "mac_local_faults:         %"PRIu64"",
2356                     ns->mac_local_faults);
2357         PMD_DRV_LOG(DEBUG, "mac_remote_faults:        %"PRIu64"",
2358                     ns->mac_remote_faults);
2359         PMD_DRV_LOG(DEBUG, "rx_length_errors:         %"PRIu64"",
2360                     ns->rx_length_errors);
2361         PMD_DRV_LOG(DEBUG, "link_xon_rx:              %"PRIu64"", ns->link_xon_rx);
2362         PMD_DRV_LOG(DEBUG, "link_xoff_rx:             %"PRIu64"", ns->link_xoff_rx);
2363         for (i = 0; i < 8; i++) {
2364                 PMD_DRV_LOG(DEBUG, "priority_xon_rx[%d]:      %"PRIu64"",
2365                                 i, ns->priority_xon_rx[i]);
2366                 PMD_DRV_LOG(DEBUG, "priority_xoff_rx[%d]:     %"PRIu64"",
2367                                 i, ns->priority_xoff_rx[i]);
2368         }
2369         PMD_DRV_LOG(DEBUG, "link_xon_tx:              %"PRIu64"", ns->link_xon_tx);
2370         PMD_DRV_LOG(DEBUG, "link_xoff_tx:             %"PRIu64"", ns->link_xoff_tx);
2371         for (i = 0; i < 8; i++) {
2372                 PMD_DRV_LOG(DEBUG, "priority_xon_tx[%d]:      %"PRIu64"",
2373                                 i, ns->priority_xon_tx[i]);
2374                 PMD_DRV_LOG(DEBUG, "priority_xoff_tx[%d]:     %"PRIu64"",
2375                                 i, ns->priority_xoff_tx[i]);
2376                 PMD_DRV_LOG(DEBUG, "priority_xon_2_xoff[%d]:  %"PRIu64"",
2377                                 i, ns->priority_xon_2_xoff[i]);
2378         }
2379         PMD_DRV_LOG(DEBUG, "rx_size_64:               %"PRIu64"", ns->rx_size_64);
2380         PMD_DRV_LOG(DEBUG, "rx_size_127:              %"PRIu64"", ns->rx_size_127);
2381         PMD_DRV_LOG(DEBUG, "rx_size_255:              %"PRIu64"", ns->rx_size_255);
2382         PMD_DRV_LOG(DEBUG, "rx_size_511:              %"PRIu64"", ns->rx_size_511);
2383         PMD_DRV_LOG(DEBUG, "rx_size_1023:             %"PRIu64"", ns->rx_size_1023);
2384         PMD_DRV_LOG(DEBUG, "rx_size_1522:             %"PRIu64"", ns->rx_size_1522);
2385         PMD_DRV_LOG(DEBUG, "rx_size_big:              %"PRIu64"", ns->rx_size_big);
2386         PMD_DRV_LOG(DEBUG, "rx_undersize:             %"PRIu64"", ns->rx_undersize);
2387         PMD_DRV_LOG(DEBUG, "rx_fragments:             %"PRIu64"", ns->rx_fragments);
2388         PMD_DRV_LOG(DEBUG, "rx_oversize:              %"PRIu64"", ns->rx_oversize);
2389         PMD_DRV_LOG(DEBUG, "rx_jabber:                %"PRIu64"", ns->rx_jabber);
2390         PMD_DRV_LOG(DEBUG, "tx_size_64:               %"PRIu64"", ns->tx_size_64);
2391         PMD_DRV_LOG(DEBUG, "tx_size_127:              %"PRIu64"", ns->tx_size_127);
2392         PMD_DRV_LOG(DEBUG, "tx_size_255:              %"PRIu64"", ns->tx_size_255);
2393         PMD_DRV_LOG(DEBUG, "tx_size_511:              %"PRIu64"", ns->tx_size_511);
2394         PMD_DRV_LOG(DEBUG, "tx_size_1023:             %"PRIu64"", ns->tx_size_1023);
2395         PMD_DRV_LOG(DEBUG, "tx_size_1522:             %"PRIu64"", ns->tx_size_1522);
2396         PMD_DRV_LOG(DEBUG, "tx_size_big:              %"PRIu64"", ns->tx_size_big);
2397         PMD_DRV_LOG(DEBUG, "mac_short_packet_dropped: %"PRIu64"",
2398                         ns->mac_short_packet_dropped);
2399         PMD_DRV_LOG(DEBUG, "checksum_error:           %"PRIu64"",
2400                     ns->checksum_error);
2401         PMD_DRV_LOG(DEBUG, "fdir_match:               %"PRIu64"", ns->fd_sb_match);
2402         PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************");
2403 }
2404
2405 /* Reset the statistics */
2406 static void
2407 i40e_dev_stats_reset(struct rte_eth_dev *dev)
2408 {
2409         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2410         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2411
2412         /* Mark PF and VSI stats to update the offset, aka "reset" */
2413         pf->offset_loaded = false;
2414         if (pf->main_vsi)
2415                 pf->main_vsi->offset_loaded = false;
2416
2417         /* read the stats, reading current register values into offset */
2418         i40e_read_stats_registers(pf, hw);
2419 }
2420
2421 static uint32_t
2422 i40e_xstats_calc_num(void)
2423 {
2424         return I40E_NB_ETH_XSTATS + I40E_NB_HW_PORT_XSTATS +
2425                 (I40E_NB_RXQ_PRIO_XSTATS * 8) +
2426                 (I40E_NB_TXQ_PRIO_XSTATS * 8);
2427 }
2428
2429 static int i40e_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
2430                                      struct rte_eth_xstat_name *xstats_names,
2431                                      __rte_unused unsigned limit)
2432 {
2433         unsigned count = 0;
2434         unsigned i, prio;
2435
2436         if (xstats_names == NULL)
2437                 return i40e_xstats_calc_num();
2438
2439         /* Note: limit checked in rte_eth_xstats_names() */
2440
2441         /* Get stats from i40e_eth_stats struct */
2442         for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
2443                 snprintf(xstats_names[count].name,
2444                          sizeof(xstats_names[count].name),
2445                          "%s", rte_i40e_stats_strings[i].name);
2446                 count++;
2447         }
2448
2449         /* Get individiual stats from i40e_hw_port struct */
2450         for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
2451                 snprintf(xstats_names[count].name,
2452                         sizeof(xstats_names[count].name),
2453                          "%s", rte_i40e_hw_port_strings[i].name);
2454                 count++;
2455         }
2456
2457         for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
2458                 for (prio = 0; prio < 8; prio++) {
2459                         snprintf(xstats_names[count].name,
2460                                  sizeof(xstats_names[count].name),
2461                                  "rx_priority%u_%s", prio,
2462                                  rte_i40e_rxq_prio_strings[i].name);
2463                         count++;
2464                 }
2465         }
2466
2467         for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
2468                 for (prio = 0; prio < 8; prio++) {
2469                         snprintf(xstats_names[count].name,
2470                                  sizeof(xstats_names[count].name),
2471                                  "tx_priority%u_%s", prio,
2472                                  rte_i40e_txq_prio_strings[i].name);
2473                         count++;
2474                 }
2475         }
2476         return count;
2477 }
2478
2479 static int
2480 i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
2481                     unsigned n)
2482 {
2483         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2484         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2485         unsigned i, count, prio;
2486         struct i40e_hw_port_stats *hw_stats = &pf->stats;
2487
2488         count = i40e_xstats_calc_num();
2489         if (n < count)
2490                 return count;
2491
2492         i40e_read_stats_registers(pf, hw);
2493
2494         if (xstats == NULL)
2495                 return 0;
2496
2497         count = 0;
2498
2499         /* Get stats from i40e_eth_stats struct */
2500         for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
2501                 xstats[count].value = *(uint64_t *)(((char *)&hw_stats->eth) +
2502                         rte_i40e_stats_strings[i].offset);
2503                 count++;
2504         }
2505
2506         /* Get individiual stats from i40e_hw_port struct */
2507         for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
2508                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2509                         rte_i40e_hw_port_strings[i].offset);
2510                 count++;
2511         }
2512
2513         for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
2514                 for (prio = 0; prio < 8; prio++) {
2515                         xstats[count].value =
2516                                 *(uint64_t *)(((char *)hw_stats) +
2517                                 rte_i40e_rxq_prio_strings[i].offset +
2518                                 (sizeof(uint64_t) * prio));
2519                         count++;
2520                 }
2521         }
2522
2523         for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
2524                 for (prio = 0; prio < 8; prio++) {
2525                         xstats[count].value =
2526                                 *(uint64_t *)(((char *)hw_stats) +
2527                                 rte_i40e_txq_prio_strings[i].offset +
2528                                 (sizeof(uint64_t) * prio));
2529                         count++;
2530                 }
2531         }
2532
2533         return count;
2534 }
2535
2536 static int
2537 i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev,
2538                                  __rte_unused uint16_t queue_id,
2539                                  __rte_unused uint8_t stat_idx,
2540                                  __rte_unused uint8_t is_rx)
2541 {
2542         PMD_INIT_FUNC_TRACE();
2543
2544         return -ENOSYS;
2545 }
2546
2547 static void
2548 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2549 {
2550         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2551         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2552         struct i40e_vsi *vsi = pf->main_vsi;
2553
2554         dev_info->max_rx_queues = vsi->nb_qps;
2555         dev_info->max_tx_queues = vsi->nb_qps;
2556         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2557         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2558         dev_info->max_mac_addrs = vsi->max_macaddrs;
2559         dev_info->max_vfs = dev->pci_dev->max_vfs;
2560         dev_info->rx_offload_capa =
2561                 DEV_RX_OFFLOAD_VLAN_STRIP |
2562                 DEV_RX_OFFLOAD_QINQ_STRIP |
2563                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2564                 DEV_RX_OFFLOAD_UDP_CKSUM |
2565                 DEV_RX_OFFLOAD_TCP_CKSUM;
2566         dev_info->tx_offload_capa =
2567                 DEV_TX_OFFLOAD_VLAN_INSERT |
2568                 DEV_TX_OFFLOAD_QINQ_INSERT |
2569                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2570                 DEV_TX_OFFLOAD_UDP_CKSUM |
2571                 DEV_TX_OFFLOAD_TCP_CKSUM |
2572                 DEV_TX_OFFLOAD_SCTP_CKSUM |
2573                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
2574                 DEV_TX_OFFLOAD_TCP_TSO;
2575         dev_info->hash_key_size = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
2576                                                 sizeof(uint32_t);
2577         dev_info->reta_size = pf->hash_lut_size;
2578         dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2579
2580         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2581                 .rx_thresh = {
2582                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2583                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2584                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2585                 },
2586                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2587                 .rx_drop_en = 0,
2588         };
2589
2590         dev_info->default_txconf = (struct rte_eth_txconf) {
2591                 .tx_thresh = {
2592                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2593                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2594                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2595                 },
2596                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2597                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2598                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2599                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2600         };
2601
2602         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2603                 .nb_max = I40E_MAX_RING_DESC,
2604                 .nb_min = I40E_MIN_RING_DESC,
2605                 .nb_align = I40E_ALIGN_RING_DESC,
2606         };
2607
2608         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2609                 .nb_max = I40E_MAX_RING_DESC,
2610                 .nb_min = I40E_MIN_RING_DESC,
2611                 .nb_align = I40E_ALIGN_RING_DESC,
2612         };
2613
2614         if (pf->flags & I40E_FLAG_VMDQ) {
2615                 dev_info->max_vmdq_pools = pf->max_nb_vmdq_vsi;
2616                 dev_info->vmdq_queue_base = dev_info->max_rx_queues;
2617                 dev_info->vmdq_queue_num = pf->vmdq_nb_qps *
2618                                                 pf->max_nb_vmdq_vsi;
2619                 dev_info->vmdq_pool_base = I40E_VMDQ_POOL_BASE;
2620                 dev_info->max_rx_queues += dev_info->vmdq_queue_num;
2621                 dev_info->max_tx_queues += dev_info->vmdq_queue_num;
2622         }
2623
2624         if (i40e_is_40G_device(hw->device_id))
2625                 /* For XL710 */
2626                 dev_info->speed_capa = ETH_LINK_SPEED_40G;
2627         else
2628                 /* For X710 */
2629                 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
2630 }
2631
2632 static int
2633 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2634 {
2635         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2636         struct i40e_vsi *vsi = pf->main_vsi;
2637         PMD_INIT_FUNC_TRACE();
2638
2639         if (on)
2640                 return i40e_vsi_add_vlan(vsi, vlan_id);
2641         else
2642                 return i40e_vsi_delete_vlan(vsi, vlan_id);
2643 }
2644
2645 static int
2646 i40e_vlan_tpid_set(struct rte_eth_dev *dev,
2647                    enum rte_vlan_type vlan_type,
2648                    uint16_t tpid)
2649 {
2650         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2651         uint64_t reg_r = 0, reg_w = 0;
2652         uint16_t reg_id = 0;
2653         int ret = 0;
2654         int qinq = dev->data->dev_conf.rxmode.hw_vlan_extend;
2655
2656         switch (vlan_type) {
2657         case ETH_VLAN_TYPE_OUTER:
2658                 if (qinq)
2659                         reg_id = 2;
2660                 else
2661                         reg_id = 3;
2662                 break;
2663         case ETH_VLAN_TYPE_INNER:
2664                 if (qinq)
2665                         reg_id = 3;
2666                 else {
2667                         ret = -EINVAL;
2668                         PMD_DRV_LOG(ERR,
2669                                 "Unsupported vlan type in single vlan.\n");
2670                         return ret;
2671                 }
2672                 break;
2673         default:
2674                 ret = -EINVAL;
2675                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d", vlan_type);
2676                 return ret;
2677         }
2678         ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2679                                           &reg_r, NULL);
2680         if (ret != I40E_SUCCESS) {
2681                 PMD_DRV_LOG(ERR, "Fail to debug read from "
2682                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2683                 ret = -EIO;
2684                 return ret;
2685         }
2686         PMD_DRV_LOG(DEBUG, "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: "
2687                     "0x%08"PRIx64"", reg_id, reg_r);
2688
2689         reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK));
2690         reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT);
2691         if (reg_r == reg_w) {
2692                 ret = 0;
2693                 PMD_DRV_LOG(DEBUG, "No need to write");
2694                 return ret;
2695         }
2696
2697         ret = i40e_aq_debug_write_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2698                                            reg_w, NULL);
2699         if (ret != I40E_SUCCESS) {
2700                 ret = -EIO;
2701                 PMD_DRV_LOG(ERR, "Fail to debug write to "
2702                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2703                 return ret;
2704         }
2705         PMD_DRV_LOG(DEBUG, "Debug write 0x%08"PRIx64" to "
2706                     "I40E_GL_SWT_L2TAGCTRL[%d]", reg_w, reg_id);
2707
2708         return ret;
2709 }
2710
2711 static void
2712 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2713 {
2714         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2715         struct i40e_vsi *vsi = pf->main_vsi;
2716
2717         if (mask & ETH_VLAN_FILTER_MASK) {
2718                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2719                         i40e_vsi_config_vlan_filter(vsi, TRUE);
2720                 else
2721                         i40e_vsi_config_vlan_filter(vsi, FALSE);
2722         }
2723
2724         if (mask & ETH_VLAN_STRIP_MASK) {
2725                 /* Enable or disable VLAN stripping */
2726                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2727                         i40e_vsi_config_vlan_stripping(vsi, TRUE);
2728                 else
2729                         i40e_vsi_config_vlan_stripping(vsi, FALSE);
2730         }
2731
2732         if (mask & ETH_VLAN_EXTEND_MASK) {
2733                 if (dev->data->dev_conf.rxmode.hw_vlan_extend) {
2734                         i40e_vsi_config_double_vlan(vsi, TRUE);
2735                         /* Set global registers with default ether type value */
2736                         i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER,
2737                                            ETHER_TYPE_VLAN);
2738                         i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER,
2739                                            ETHER_TYPE_VLAN);
2740                 }
2741                 else
2742                         i40e_vsi_config_double_vlan(vsi, FALSE);
2743         }
2744 }
2745
2746 static void
2747 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
2748                           __rte_unused uint16_t queue,
2749                           __rte_unused int on)
2750 {
2751         PMD_INIT_FUNC_TRACE();
2752 }
2753
2754 static int
2755 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
2756 {
2757         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2758         struct i40e_vsi *vsi = pf->main_vsi;
2759         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
2760         struct i40e_vsi_vlan_pvid_info info;
2761
2762         memset(&info, 0, sizeof(info));
2763         info.on = on;
2764         if (info.on)
2765                 info.config.pvid = pvid;
2766         else {
2767                 info.config.reject.tagged =
2768                                 data->dev_conf.txmode.hw_vlan_reject_tagged;
2769                 info.config.reject.untagged =
2770                                 data->dev_conf.txmode.hw_vlan_reject_untagged;
2771         }
2772
2773         return i40e_vsi_vlan_pvid_set(vsi, &info);
2774 }
2775
2776 static int
2777 i40e_dev_led_on(struct rte_eth_dev *dev)
2778 {
2779         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2780         uint32_t mode = i40e_led_get(hw);
2781
2782         if (mode == 0)
2783                 i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
2784
2785         return 0;
2786 }
2787
2788 static int
2789 i40e_dev_led_off(struct rte_eth_dev *dev)
2790 {
2791         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2792         uint32_t mode = i40e_led_get(hw);
2793
2794         if (mode != 0)
2795                 i40e_led_set(hw, 0, false);
2796
2797         return 0;
2798 }
2799
2800 static int
2801 i40e_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2802 {
2803         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2804         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2805
2806         fc_conf->pause_time = pf->fc_conf.pause_time;
2807         fc_conf->high_water =  pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS];
2808         fc_conf->low_water = pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS];
2809
2810          /* Return current mode according to actual setting*/
2811         switch (hw->fc.current_mode) {
2812         case I40E_FC_FULL:
2813                 fc_conf->mode = RTE_FC_FULL;
2814                 break;
2815         case I40E_FC_TX_PAUSE:
2816                 fc_conf->mode = RTE_FC_TX_PAUSE;
2817                 break;
2818         case I40E_FC_RX_PAUSE:
2819                 fc_conf->mode = RTE_FC_RX_PAUSE;
2820                 break;
2821         case I40E_FC_NONE:
2822         default:
2823                 fc_conf->mode = RTE_FC_NONE;
2824         };
2825
2826         return 0;
2827 }
2828
2829 static int
2830 i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2831 {
2832         uint32_t mflcn_reg, fctrl_reg, reg;
2833         uint32_t max_high_water;
2834         uint8_t i, aq_failure;
2835         int err;
2836         struct i40e_hw *hw;
2837         struct i40e_pf *pf;
2838         enum i40e_fc_mode rte_fcmode_2_i40e_fcmode[] = {
2839                 [RTE_FC_NONE] = I40E_FC_NONE,
2840                 [RTE_FC_RX_PAUSE] = I40E_FC_RX_PAUSE,
2841                 [RTE_FC_TX_PAUSE] = I40E_FC_TX_PAUSE,
2842                 [RTE_FC_FULL] = I40E_FC_FULL
2843         };
2844
2845         /* high_water field in the rte_eth_fc_conf using the kilobytes unit */
2846
2847         max_high_water = I40E_RXPBSIZE >> I40E_KILOSHIFT;
2848         if ((fc_conf->high_water > max_high_water) ||
2849                         (fc_conf->high_water < fc_conf->low_water)) {
2850                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB, "
2851                         "High_water must <= %d.", max_high_water);
2852                 return -EINVAL;
2853         }
2854
2855         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2856         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2857         hw->fc.requested_mode = rte_fcmode_2_i40e_fcmode[fc_conf->mode];
2858
2859         pf->fc_conf.pause_time = fc_conf->pause_time;
2860         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->high_water;
2861         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->low_water;
2862
2863         PMD_INIT_FUNC_TRACE();
2864
2865         /* All the link flow control related enable/disable register
2866          * configuration is handle by the F/W
2867          */
2868         err = i40e_set_fc(hw, &aq_failure, true);
2869         if (err < 0)
2870                 return -ENOSYS;
2871
2872         if (i40e_is_40G_device(hw->device_id)) {
2873                 /* Configure flow control refresh threshold,
2874                  * the value for stat_tx_pause_refresh_timer[8]
2875                  * is used for global pause operation.
2876                  */
2877
2878                 I40E_WRITE_REG(hw,
2879                                I40E_PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(8),
2880                                pf->fc_conf.pause_time);
2881
2882                 /* configure the timer value included in transmitted pause
2883                  * frame,
2884                  * the value for stat_tx_pause_quanta[8] is used for global
2885                  * pause operation
2886                  */
2887                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(8),
2888                                pf->fc_conf.pause_time);
2889
2890                 fctrl_reg = I40E_READ_REG(hw,
2891                                           I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL);
2892
2893                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2894                         fctrl_reg |= I40E_PRTMAC_FWD_CTRL;
2895                 else
2896                         fctrl_reg &= ~I40E_PRTMAC_FWD_CTRL;
2897
2898                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL,
2899                                fctrl_reg);
2900         } else {
2901                 /* Configure pause time (2 TCs per register) */
2902                 reg = (uint32_t)pf->fc_conf.pause_time * (uint32_t)0x00010001;
2903                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS / 2; i++)
2904                         I40E_WRITE_REG(hw, I40E_PRTDCB_FCTTVN(i), reg);
2905
2906                 /* Configure flow control refresh threshold value */
2907                 I40E_WRITE_REG(hw, I40E_PRTDCB_FCRTV,
2908                                pf->fc_conf.pause_time / 2);
2909
2910                 mflcn_reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
2911
2912                 /* set or clear MFLCN.PMCF & MFLCN.DPF bits
2913                  *depending on configuration
2914                  */
2915                 if (fc_conf->mac_ctrl_frame_fwd != 0) {
2916                         mflcn_reg |= I40E_PRTDCB_MFLCN_PMCF_MASK;
2917                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_DPF_MASK;
2918                 } else {
2919                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_PMCF_MASK;
2920                         mflcn_reg |= I40E_PRTDCB_MFLCN_DPF_MASK;
2921                 }
2922
2923                 I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, mflcn_reg);
2924         }
2925
2926         /* config the water marker both based on the packets and bytes */
2927         I40E_WRITE_REG(hw, I40E_GLRPB_PHW,
2928                        (pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
2929                        << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
2930         I40E_WRITE_REG(hw, I40E_GLRPB_PLW,
2931                        (pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
2932                        << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
2933         I40E_WRITE_REG(hw, I40E_GLRPB_GHW,
2934                        pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
2935                        << I40E_KILOSHIFT);
2936         I40E_WRITE_REG(hw, I40E_GLRPB_GLW,
2937                        pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
2938                        << I40E_KILOSHIFT);
2939
2940         I40E_WRITE_FLUSH(hw);
2941
2942         return 0;
2943 }
2944
2945 static int
2946 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
2947                             __rte_unused struct rte_eth_pfc_conf *pfc_conf)
2948 {
2949         PMD_INIT_FUNC_TRACE();
2950
2951         return -ENOSYS;
2952 }
2953
2954 /* Add a MAC address, and update filters */
2955 static void
2956 i40e_macaddr_add(struct rte_eth_dev *dev,
2957                  struct ether_addr *mac_addr,
2958                  __rte_unused uint32_t index,
2959                  uint32_t pool)
2960 {
2961         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2962         struct i40e_mac_filter_info mac_filter;
2963         struct i40e_vsi *vsi;
2964         int ret;
2965
2966         /* If VMDQ not enabled or configured, return */
2967         if (pool != 0 && (!(pf->flags & I40E_FLAG_VMDQ) ||
2968                           !pf->nb_cfg_vmdq_vsi)) {
2969                 PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u",
2970                         pf->flags & I40E_FLAG_VMDQ ? "configured" : "enabled",
2971                         pool);
2972                 return;
2973         }
2974
2975         if (pool > pf->nb_cfg_vmdq_vsi) {
2976                 PMD_DRV_LOG(ERR, "Pool number %u invalid. Max pool is %u",
2977                                 pool, pf->nb_cfg_vmdq_vsi);
2978                 return;
2979         }
2980
2981         (void)rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN);
2982         if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2983                 mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
2984         else
2985                 mac_filter.filter_type = RTE_MAC_PERFECT_MATCH;
2986
2987         if (pool == 0)
2988                 vsi = pf->main_vsi;
2989         else
2990                 vsi = pf->vmdq[pool - 1].vsi;
2991
2992         ret = i40e_vsi_add_mac(vsi, &mac_filter);
2993         if (ret != I40E_SUCCESS) {
2994                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
2995                 return;
2996         }
2997 }
2998
2999 /* Remove a MAC address, and update filters */
3000 static void
3001 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
3002 {
3003         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3004         struct i40e_vsi *vsi;
3005         struct rte_eth_dev_data *data = dev->data;
3006         struct ether_addr *macaddr;
3007         int ret;
3008         uint32_t i;
3009         uint64_t pool_sel;
3010
3011         macaddr = &(data->mac_addrs[index]);
3012
3013         pool_sel = dev->data->mac_pool_sel[index];
3014
3015         for (i = 0; i < sizeof(pool_sel) * CHAR_BIT; i++) {
3016                 if (pool_sel & (1ULL << i)) {
3017                         if (i == 0)
3018                                 vsi = pf->main_vsi;
3019                         else {
3020                                 /* No VMDQ pool enabled or configured */
3021                                 if (!(pf->flags & I40E_FLAG_VMDQ) ||
3022                                         (i > pf->nb_cfg_vmdq_vsi)) {
3023                                         PMD_DRV_LOG(ERR, "No VMDQ pool enabled"
3024                                                         "/configured");
3025                                         return;
3026                                 }
3027                                 vsi = pf->vmdq[i - 1].vsi;
3028                         }
3029                         ret = i40e_vsi_delete_mac(vsi, macaddr);
3030
3031                         if (ret) {
3032                                 PMD_DRV_LOG(ERR, "Failed to remove MACVLAN filter");
3033                                 return;
3034                         }
3035                 }
3036         }
3037 }
3038
3039 /* Set perfect match or hash match of MAC and VLAN for a VF */
3040 static int
3041 i40e_vf_mac_filter_set(struct i40e_pf *pf,
3042                  struct rte_eth_mac_filter *filter,
3043                  bool add)
3044 {
3045         struct i40e_hw *hw;
3046         struct i40e_mac_filter_info mac_filter;
3047         struct ether_addr old_mac;
3048         struct ether_addr *new_mac;
3049         struct i40e_pf_vf *vf = NULL;
3050         uint16_t vf_id;
3051         int ret;
3052
3053         if (pf == NULL) {
3054                 PMD_DRV_LOG(ERR, "Invalid PF argument.");
3055                 return -EINVAL;
3056         }
3057         hw = I40E_PF_TO_HW(pf);
3058
3059         if (filter == NULL) {
3060                 PMD_DRV_LOG(ERR, "Invalid mac filter argument.");
3061                 return -EINVAL;
3062         }
3063
3064         new_mac = &filter->mac_addr;
3065
3066         if (is_zero_ether_addr(new_mac)) {
3067                 PMD_DRV_LOG(ERR, "Invalid ethernet address.");
3068                 return -EINVAL;
3069         }
3070
3071         vf_id = filter->dst_id;
3072
3073         if (vf_id > pf->vf_num - 1 || !pf->vfs) {
3074                 PMD_DRV_LOG(ERR, "Invalid argument.");
3075                 return -EINVAL;
3076         }
3077         vf = &pf->vfs[vf_id];
3078
3079         if (add && is_same_ether_addr(new_mac, &(pf->dev_addr))) {
3080                 PMD_DRV_LOG(INFO, "Ignore adding permanent MAC address.");
3081                 return -EINVAL;
3082         }
3083
3084         if (add) {
3085                 (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
3086                 (void)rte_memcpy(hw->mac.addr, new_mac->addr_bytes,
3087                                 ETHER_ADDR_LEN);
3088                 (void)rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr,
3089                                  ETHER_ADDR_LEN);
3090
3091                 mac_filter.filter_type = filter->filter_type;
3092                 ret = i40e_vsi_add_mac(vf->vsi, &mac_filter);
3093                 if (ret != I40E_SUCCESS) {
3094                         PMD_DRV_LOG(ERR, "Failed to add MAC filter.");
3095                         return -1;
3096                 }
3097                 ether_addr_copy(new_mac, &pf->dev_addr);
3098         } else {
3099                 (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr,
3100                                 ETHER_ADDR_LEN);
3101                 ret = i40e_vsi_delete_mac(vf->vsi, &filter->mac_addr);
3102                 if (ret != I40E_SUCCESS) {
3103                         PMD_DRV_LOG(ERR, "Failed to delete MAC filter.");
3104                         return -1;
3105                 }
3106
3107                 /* Clear device address as it has been removed */
3108                 if (is_same_ether_addr(&(pf->dev_addr), new_mac))
3109                         memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
3110         }
3111
3112         return 0;
3113 }
3114
3115 /* MAC filter handle */
3116 static int
3117 i40e_mac_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
3118                 void *arg)
3119 {
3120         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3121         struct rte_eth_mac_filter *filter;
3122         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3123         int ret = I40E_NOT_SUPPORTED;
3124
3125         filter = (struct rte_eth_mac_filter *)(arg);
3126
3127         switch (filter_op) {
3128         case RTE_ETH_FILTER_NOP:
3129                 ret = I40E_SUCCESS;
3130                 break;
3131         case RTE_ETH_FILTER_ADD:
3132                 i40e_pf_disable_irq0(hw);
3133                 if (filter->is_vf)
3134                         ret = i40e_vf_mac_filter_set(pf, filter, 1);
3135                 i40e_pf_enable_irq0(hw);
3136                 break;
3137         case RTE_ETH_FILTER_DELETE:
3138                 i40e_pf_disable_irq0(hw);
3139                 if (filter->is_vf)
3140                         ret = i40e_vf_mac_filter_set(pf, filter, 0);
3141                 i40e_pf_enable_irq0(hw);
3142                 break;
3143         default:
3144                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
3145                 ret = I40E_ERR_PARAM;
3146                 break;
3147         }
3148
3149         return ret;
3150 }
3151
3152 static int
3153 i40e_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
3154 {
3155         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
3156         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3157         int ret;
3158
3159         if (!lut)
3160                 return -EINVAL;
3161
3162         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
3163                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, TRUE,
3164                                           lut, lut_size);
3165                 if (ret) {
3166                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
3167                         return ret;
3168                 }
3169         } else {
3170                 uint32_t *lut_dw = (uint32_t *)lut;
3171                 uint16_t i, lut_size_dw = lut_size / 4;
3172
3173                 for (i = 0; i < lut_size_dw; i++)
3174                         lut_dw[i] = I40E_READ_REG(hw, I40E_PFQF_HLUT(i));
3175         }
3176
3177         return 0;
3178 }
3179
3180 static int
3181 i40e_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
3182 {
3183         struct i40e_pf *pf;
3184         struct i40e_hw *hw;
3185         int ret;
3186
3187         if (!vsi || !lut)
3188                 return -EINVAL;
3189
3190         pf = I40E_VSI_TO_PF(vsi);
3191         hw = I40E_VSI_TO_HW(vsi);
3192
3193         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
3194                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, TRUE,
3195                                           lut, lut_size);
3196                 if (ret) {
3197                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
3198                         return ret;
3199                 }
3200         } else {
3201                 uint32_t *lut_dw = (uint32_t *)lut;
3202                 uint16_t i, lut_size_dw = lut_size / 4;
3203
3204                 for (i = 0; i < lut_size_dw; i++)
3205                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
3206                 I40E_WRITE_FLUSH(hw);
3207         }
3208
3209         return 0;
3210 }
3211
3212 static int
3213 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
3214                          struct rte_eth_rss_reta_entry64 *reta_conf,
3215                          uint16_t reta_size)
3216 {
3217         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3218         uint16_t i, lut_size = pf->hash_lut_size;
3219         uint16_t idx, shift;
3220         uint8_t *lut;
3221         int ret;
3222
3223         if (reta_size != lut_size ||
3224                 reta_size > ETH_RSS_RETA_SIZE_512) {
3225                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3226                         "(%d) doesn't match the number hardware can supported "
3227                                         "(%d)\n", reta_size, lut_size);
3228                 return -EINVAL;
3229         }
3230
3231         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
3232         if (!lut) {
3233                 PMD_DRV_LOG(ERR, "No memory can be allocated");
3234                 return -ENOMEM;
3235         }
3236         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
3237         if (ret)
3238                 goto out;
3239         for (i = 0; i < reta_size; i++) {
3240                 idx = i / RTE_RETA_GROUP_SIZE;
3241                 shift = i % RTE_RETA_GROUP_SIZE;
3242                 if (reta_conf[idx].mask & (1ULL << shift))
3243                         lut[i] = reta_conf[idx].reta[shift];
3244         }
3245         ret = i40e_set_rss_lut(pf->main_vsi, lut, reta_size);
3246
3247 out:
3248         rte_free(lut);
3249
3250         return ret;
3251 }
3252
3253 static int
3254 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
3255                         struct rte_eth_rss_reta_entry64 *reta_conf,
3256                         uint16_t reta_size)
3257 {
3258         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3259         uint16_t i, lut_size = pf->hash_lut_size;
3260         uint16_t idx, shift;
3261         uint8_t *lut;
3262         int ret;
3263
3264         if (reta_size != lut_size ||
3265                 reta_size > ETH_RSS_RETA_SIZE_512) {
3266                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3267                         "(%d) doesn't match the number hardware can supported "
3268                                         "(%d)\n", reta_size, lut_size);
3269                 return -EINVAL;
3270         }
3271
3272         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
3273         if (!lut) {
3274                 PMD_DRV_LOG(ERR, "No memory can be allocated");
3275                 return -ENOMEM;
3276         }
3277
3278         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
3279         if (ret)
3280                 goto out;
3281         for (i = 0; i < reta_size; i++) {
3282                 idx = i / RTE_RETA_GROUP_SIZE;
3283                 shift = i % RTE_RETA_GROUP_SIZE;
3284                 if (reta_conf[idx].mask & (1ULL << shift))
3285                         reta_conf[idx].reta[shift] = lut[i];
3286         }
3287
3288 out:
3289         rte_free(lut);
3290
3291         return ret;
3292 }
3293
3294 /**
3295  * i40e_allocate_dma_mem_d - specific memory alloc for shared code (base driver)
3296  * @hw:   pointer to the HW structure
3297  * @mem:  pointer to mem struct to fill out
3298  * @size: size of memory requested
3299  * @alignment: what to align the allocation to
3300  **/
3301 enum i40e_status_code
3302 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3303                         struct i40e_dma_mem *mem,
3304                         u64 size,
3305                         u32 alignment)
3306 {
3307         const struct rte_memzone *mz = NULL;
3308         char z_name[RTE_MEMZONE_NAMESIZE];
3309
3310         if (!mem)
3311                 return I40E_ERR_PARAM;
3312
3313         snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand());
3314         mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
3315                                          alignment, RTE_PGSIZE_2M);
3316         if (!mz)
3317                 return I40E_ERR_NO_MEMORY;
3318
3319         mem->size = size;
3320         mem->va = mz->addr;
3321         mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
3322         mem->zone = (const void *)mz;
3323         PMD_DRV_LOG(DEBUG, "memzone %s allocated with physical address: "
3324                     "%"PRIu64, mz->name, mem->pa);
3325
3326         return I40E_SUCCESS;
3327 }
3328
3329 /**
3330  * i40e_free_dma_mem_d - specific memory free for shared code (base driver)
3331  * @hw:   pointer to the HW structure
3332  * @mem:  ptr to mem struct to free
3333  **/
3334 enum i40e_status_code
3335 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3336                     struct i40e_dma_mem *mem)
3337 {
3338         if (!mem)
3339                 return I40E_ERR_PARAM;
3340
3341         PMD_DRV_LOG(DEBUG, "memzone %s to be freed with physical address: "
3342                     "%"PRIu64, ((const struct rte_memzone *)mem->zone)->name,
3343                     mem->pa);
3344         rte_memzone_free((const struct rte_memzone *)mem->zone);
3345         mem->zone = NULL;
3346         mem->va = NULL;
3347         mem->pa = (u64)0;
3348
3349         return I40E_SUCCESS;
3350 }
3351
3352 /**
3353  * i40e_allocate_virt_mem_d - specific memory alloc for shared code (base driver)
3354  * @hw:   pointer to the HW structure
3355  * @mem:  pointer to mem struct to fill out
3356  * @size: size of memory requested
3357  **/
3358 enum i40e_status_code
3359 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3360                          struct i40e_virt_mem *mem,
3361                          u32 size)
3362 {
3363         if (!mem)
3364                 return I40E_ERR_PARAM;
3365
3366         mem->size = size;
3367         mem->va = rte_zmalloc("i40e", size, 0);
3368
3369         if (mem->va)
3370                 return I40E_SUCCESS;
3371         else
3372                 return I40E_ERR_NO_MEMORY;
3373 }
3374
3375 /**
3376  * i40e_free_virt_mem_d - specific memory free for shared code (base driver)
3377  * @hw:   pointer to the HW structure
3378  * @mem:  pointer to mem struct to free
3379  **/
3380 enum i40e_status_code
3381 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3382                      struct i40e_virt_mem *mem)
3383 {
3384         if (!mem)
3385                 return I40E_ERR_PARAM;
3386
3387         rte_free(mem->va);
3388         mem->va = NULL;
3389
3390         return I40E_SUCCESS;
3391 }
3392
3393 void
3394 i40e_init_spinlock_d(struct i40e_spinlock *sp)
3395 {
3396         rte_spinlock_init(&sp->spinlock);
3397 }
3398
3399 void
3400 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
3401 {
3402         rte_spinlock_lock(&sp->spinlock);
3403 }
3404
3405 void
3406 i40e_release_spinlock_d(struct i40e_spinlock *sp)
3407 {
3408         rte_spinlock_unlock(&sp->spinlock);
3409 }
3410
3411 void
3412 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
3413 {
3414         return;
3415 }
3416
3417 /**
3418  * Get the hardware capabilities, which will be parsed
3419  * and saved into struct i40e_hw.
3420  */
3421 static int
3422 i40e_get_cap(struct i40e_hw *hw)
3423 {
3424         struct i40e_aqc_list_capabilities_element_resp *buf;
3425         uint16_t len, size = 0;
3426         int ret;
3427
3428         /* Calculate a huge enough buff for saving response data temporarily */
3429         len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
3430                                                 I40E_MAX_CAP_ELE_NUM;
3431         buf = rte_zmalloc("i40e", len, 0);
3432         if (!buf) {
3433                 PMD_DRV_LOG(ERR, "Failed to allocate memory");
3434                 return I40E_ERR_NO_MEMORY;
3435         }
3436
3437         /* Get, parse the capabilities and save it to hw */
3438         ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
3439                         i40e_aqc_opc_list_func_capabilities, NULL);
3440         if (ret != I40E_SUCCESS)
3441                 PMD_DRV_LOG(ERR, "Failed to discover capabilities");
3442
3443         /* Free the temporary buffer after being used */
3444         rte_free(buf);
3445
3446         return ret;
3447 }
3448
3449 static int
3450 i40e_pf_parameter_init(struct rte_eth_dev *dev)
3451 {
3452         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3453         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3454         uint16_t qp_count = 0, vsi_count = 0;
3455
3456         if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
3457                 PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV");
3458                 return -EINVAL;
3459         }
3460         /* Add the parameter init for LFC */
3461         pf->fc_conf.pause_time = I40E_DEFAULT_PAUSE_TIME;
3462         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_HIGH_WATER;
3463         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_LOW_WATER;
3464
3465         pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
3466         pf->max_num_vsi = hw->func_caps.num_vsis;
3467         pf->lan_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF;
3468         pf->vmdq_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
3469         pf->vf_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3470
3471         /* FDir queue/VSI allocation */
3472         pf->fdir_qp_offset = 0;
3473         if (hw->func_caps.fd) {
3474                 pf->flags |= I40E_FLAG_FDIR;
3475                 pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
3476         } else {
3477                 pf->fdir_nb_qps = 0;
3478         }
3479         qp_count += pf->fdir_nb_qps;
3480         vsi_count += 1;
3481
3482         /* LAN queue/VSI allocation */
3483         pf->lan_qp_offset = pf->fdir_qp_offset + pf->fdir_nb_qps;
3484         if (!hw->func_caps.rss) {
3485                 pf->lan_nb_qps = 1;
3486         } else {
3487                 pf->flags |= I40E_FLAG_RSS;
3488                 if (hw->mac.type == I40E_MAC_X722)
3489                         pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE;
3490                 pf->lan_nb_qps = pf->lan_nb_qp_max;
3491         }
3492         qp_count += pf->lan_nb_qps;
3493         vsi_count += 1;
3494
3495         /* VF queue/VSI allocation */
3496         pf->vf_qp_offset = pf->lan_qp_offset + pf->lan_nb_qps;
3497         if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) {
3498                 pf->flags |= I40E_FLAG_SRIOV;
3499                 pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3500                 pf->vf_num = dev->pci_dev->max_vfs;
3501                 PMD_DRV_LOG(DEBUG, "%u VF VSIs, %u queues per VF VSI, "
3502                             "in total %u queues", pf->vf_num, pf->vf_nb_qps,
3503                             pf->vf_nb_qps * pf->vf_num);
3504         } else {
3505                 pf->vf_nb_qps = 0;
3506                 pf->vf_num = 0;
3507         }
3508         qp_count += pf->vf_nb_qps * pf->vf_num;
3509         vsi_count += pf->vf_num;
3510
3511         /* VMDq queue/VSI allocation */
3512         pf->vmdq_qp_offset = pf->vf_qp_offset + pf->vf_nb_qps * pf->vf_num;
3513         pf->vmdq_nb_qps = 0;
3514         pf->max_nb_vmdq_vsi = 0;
3515         if (hw->func_caps.vmdq) {
3516                 if (qp_count < hw->func_caps.num_tx_qp &&
3517                         vsi_count < hw->func_caps.num_vsis) {
3518                         pf->max_nb_vmdq_vsi = (hw->func_caps.num_tx_qp -
3519                                 qp_count) / pf->vmdq_nb_qp_max;
3520
3521                         /* Limit the maximum number of VMDq vsi to the maximum
3522                          * ethdev can support
3523                          */
3524                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3525                                 hw->func_caps.num_vsis - vsi_count);
3526                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3527                                 ETH_64_POOLS);
3528                         if (pf->max_nb_vmdq_vsi) {
3529                                 pf->flags |= I40E_FLAG_VMDQ;
3530                                 pf->vmdq_nb_qps = pf->vmdq_nb_qp_max;
3531                                 PMD_DRV_LOG(DEBUG, "%u VMDQ VSIs, %u queues "
3532                                             "per VMDQ VSI, in total %u queues",
3533                                             pf->max_nb_vmdq_vsi,
3534                                             pf->vmdq_nb_qps, pf->vmdq_nb_qps *
3535                                             pf->max_nb_vmdq_vsi);
3536                         } else {
3537                                 PMD_DRV_LOG(INFO, "No enough queues left for "
3538                                             "VMDq");
3539                         }
3540                 } else {
3541                         PMD_DRV_LOG(INFO, "No queue or VSI left for VMDq");
3542                 }
3543         }
3544         qp_count += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi;
3545         vsi_count += pf->max_nb_vmdq_vsi;
3546
3547         if (hw->func_caps.dcb)
3548                 pf->flags |= I40E_FLAG_DCB;
3549
3550         if (qp_count > hw->func_caps.num_tx_qp) {
3551                 PMD_DRV_LOG(ERR, "Failed to allocate %u queues, which exceeds "
3552                             "the hardware maximum %u", qp_count,
3553                             hw->func_caps.num_tx_qp);
3554                 return -EINVAL;
3555         }
3556         if (vsi_count > hw->func_caps.num_vsis) {
3557                 PMD_DRV_LOG(ERR, "Failed to allocate %u VSIs, which exceeds "
3558                             "the hardware maximum %u", vsi_count,
3559                             hw->func_caps.num_vsis);
3560                 return -EINVAL;
3561         }
3562
3563         return 0;
3564 }
3565
3566 static int
3567 i40e_pf_get_switch_config(struct i40e_pf *pf)
3568 {
3569         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3570         struct i40e_aqc_get_switch_config_resp *switch_config;
3571         struct i40e_aqc_switch_config_element_resp *element;
3572         uint16_t start_seid = 0, num_reported;
3573         int ret;
3574
3575         switch_config = (struct i40e_aqc_get_switch_config_resp *)\
3576                         rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
3577         if (!switch_config) {
3578                 PMD_DRV_LOG(ERR, "Failed to allocated memory");
3579                 return -ENOMEM;
3580         }
3581
3582         /* Get the switch configurations */
3583         ret = i40e_aq_get_switch_config(hw, switch_config,
3584                 I40E_AQ_LARGE_BUF, &start_seid, NULL);
3585         if (ret != I40E_SUCCESS) {
3586                 PMD_DRV_LOG(ERR, "Failed to get switch configurations");
3587                 goto fail;
3588         }
3589         num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
3590         if (num_reported != 1) { /* The number should be 1 */
3591                 PMD_DRV_LOG(ERR, "Wrong number of switch config reported");
3592                 goto fail;
3593         }
3594
3595         /* Parse the switch configuration elements */
3596         element = &(switch_config->element[0]);
3597         if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
3598                 pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
3599                 pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
3600         } else
3601                 PMD_DRV_LOG(INFO, "Unknown element type");
3602
3603 fail:
3604         rte_free(switch_config);
3605
3606         return ret;
3607 }
3608
3609 static int
3610 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
3611                         uint32_t num)
3612 {
3613         struct pool_entry *entry;
3614
3615         if (pool == NULL || num == 0)
3616                 return -EINVAL;
3617
3618         entry = rte_zmalloc("i40e", sizeof(*entry), 0);
3619         if (entry == NULL) {
3620                 PMD_DRV_LOG(ERR, "Failed to allocate memory for resource pool");
3621                 return -ENOMEM;
3622         }
3623
3624         /* queue heap initialize */
3625         pool->num_free = num;
3626         pool->num_alloc = 0;
3627         pool->base = base;
3628         LIST_INIT(&pool->alloc_list);
3629         LIST_INIT(&pool->free_list);
3630
3631         /* Initialize element  */
3632         entry->base = 0;
3633         entry->len = num;
3634
3635         LIST_INSERT_HEAD(&pool->free_list, entry, next);
3636         return 0;
3637 }
3638
3639 static void
3640 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
3641 {
3642         struct pool_entry *entry, *next_entry;
3643
3644         if (pool == NULL)
3645                 return;
3646
3647         for (entry = LIST_FIRST(&pool->alloc_list);
3648                         entry && (next_entry = LIST_NEXT(entry, next), 1);
3649                         entry = next_entry) {
3650                 LIST_REMOVE(entry, next);
3651                 rte_free(entry);
3652         }
3653
3654         for (entry = LIST_FIRST(&pool->free_list);
3655                         entry && (next_entry = LIST_NEXT(entry, next), 1);
3656                         entry = next_entry) {
3657                 LIST_REMOVE(entry, next);
3658                 rte_free(entry);
3659         }
3660
3661         pool->num_free = 0;
3662         pool->num_alloc = 0;
3663         pool->base = 0;
3664         LIST_INIT(&pool->alloc_list);
3665         LIST_INIT(&pool->free_list);
3666 }
3667
3668 static int
3669 i40e_res_pool_free(struct i40e_res_pool_info *pool,
3670                        uint32_t base)
3671 {
3672         struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
3673         uint32_t pool_offset;
3674         int insert;
3675
3676         if (pool == NULL) {
3677                 PMD_DRV_LOG(ERR, "Invalid parameter");
3678                 return -EINVAL;
3679         }
3680
3681         pool_offset = base - pool->base;
3682         /* Lookup in alloc list */
3683         LIST_FOREACH(entry, &pool->alloc_list, next) {
3684                 if (entry->base == pool_offset) {
3685                         valid_entry = entry;
3686                         LIST_REMOVE(entry, next);
3687                         break;
3688                 }
3689         }
3690
3691         /* Not find, return */
3692         if (valid_entry == NULL) {
3693                 PMD_DRV_LOG(ERR, "Failed to find entry");
3694                 return -EINVAL;
3695         }
3696
3697         /**
3698          * Found it, move it to free list  and try to merge.
3699          * In order to make merge easier, always sort it by qbase.
3700          * Find adjacent prev and last entries.
3701          */
3702         prev = next = NULL;
3703         LIST_FOREACH(entry, &pool->free_list, next) {
3704                 if (entry->base > valid_entry->base) {
3705                         next = entry;
3706                         break;
3707                 }
3708                 prev = entry;
3709         }
3710
3711         insert = 0;
3712         /* Try to merge with next one*/
3713         if (next != NULL) {
3714                 /* Merge with next one */
3715                 if (valid_entry->base + valid_entry->len == next->base) {
3716                         next->base = valid_entry->base;
3717                         next->len += valid_entry->len;
3718                         rte_free(valid_entry);
3719                         valid_entry = next;
3720                         insert = 1;
3721                 }
3722         }
3723
3724         if (prev != NULL) {
3725                 /* Merge with previous one */
3726                 if (prev->base + prev->len == valid_entry->base) {
3727                         prev->len += valid_entry->len;
3728                         /* If it merge with next one, remove next node */
3729                         if (insert == 1) {
3730                                 LIST_REMOVE(valid_entry, next);
3731                                 rte_free(valid_entry);
3732                         } else {
3733                                 rte_free(valid_entry);
3734                                 insert = 1;
3735                         }
3736                 }
3737         }
3738
3739         /* Not find any entry to merge, insert */
3740         if (insert == 0) {
3741                 if (prev != NULL)
3742                         LIST_INSERT_AFTER(prev, valid_entry, next);
3743                 else if (next != NULL)
3744                         LIST_INSERT_BEFORE(next, valid_entry, next);
3745                 else /* It's empty list, insert to head */
3746                         LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
3747         }
3748
3749         pool->num_free += valid_entry->len;
3750         pool->num_alloc -= valid_entry->len;
3751
3752         return 0;
3753 }
3754
3755 static int
3756 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
3757                        uint16_t num)
3758 {
3759         struct pool_entry *entry, *valid_entry;
3760
3761         if (pool == NULL || num == 0) {
3762                 PMD_DRV_LOG(ERR, "Invalid parameter");
3763                 return -EINVAL;
3764         }
3765
3766         if (pool->num_free < num) {
3767                 PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u",
3768                             num, pool->num_free);
3769                 return -ENOMEM;
3770         }
3771
3772         valid_entry = NULL;
3773         /* Lookup  in free list and find most fit one */
3774         LIST_FOREACH(entry, &pool->free_list, next) {
3775                 if (entry->len >= num) {
3776                         /* Find best one */
3777                         if (entry->len == num) {
3778                                 valid_entry = entry;
3779                                 break;
3780                         }
3781                         if (valid_entry == NULL || valid_entry->len > entry->len)
3782                                 valid_entry = entry;
3783                 }
3784         }
3785
3786         /* Not find one to satisfy the request, return */
3787         if (valid_entry == NULL) {
3788                 PMD_DRV_LOG(ERR, "No valid entry found");
3789                 return -ENOMEM;
3790         }
3791         /**
3792          * The entry have equal queue number as requested,
3793          * remove it from alloc_list.
3794          */
3795         if (valid_entry->len == num) {
3796                 LIST_REMOVE(valid_entry, next);
3797         } else {
3798                 /**
3799                  * The entry have more numbers than requested,
3800                  * create a new entry for alloc_list and minus its
3801                  * queue base and number in free_list.
3802                  */
3803                 entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
3804                 if (entry == NULL) {
3805                         PMD_DRV_LOG(ERR, "Failed to allocate memory for "
3806                                     "resource pool");
3807                         return -ENOMEM;
3808                 }
3809                 entry->base = valid_entry->base;
3810                 entry->len = num;
3811                 valid_entry->base += num;
3812                 valid_entry->len -= num;
3813                 valid_entry = entry;
3814         }
3815
3816         /* Insert it into alloc list, not sorted */
3817         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
3818
3819         pool->num_free -= valid_entry->len;
3820         pool->num_alloc += valid_entry->len;
3821
3822         return valid_entry->base + pool->base;
3823 }
3824
3825 /**
3826  * bitmap_is_subset - Check whether src2 is subset of src1
3827  **/
3828 static inline int
3829 bitmap_is_subset(uint8_t src1, uint8_t src2)
3830 {
3831         return !((src1 ^ src2) & src2);
3832 }
3833
3834 static enum i40e_status_code
3835 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
3836 {
3837         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3838
3839         /* If DCB is not supported, only default TC is supported */
3840         if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
3841                 PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported");
3842                 return I40E_NOT_SUPPORTED;
3843         }
3844
3845         if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
3846                 PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to "
3847                             "HW support 0x%x", hw->func_caps.enabled_tcmap,
3848                             enabled_tcmap);
3849                 return I40E_NOT_SUPPORTED;
3850         }
3851         return I40E_SUCCESS;
3852 }
3853
3854 int
3855 i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi,
3856                                 struct i40e_vsi_vlan_pvid_info *info)
3857 {
3858         struct i40e_hw *hw;
3859         struct i40e_vsi_context ctxt;
3860         uint8_t vlan_flags = 0;
3861         int ret;
3862
3863         if (vsi == NULL || info == NULL) {
3864                 PMD_DRV_LOG(ERR, "invalid parameters");
3865                 return I40E_ERR_PARAM;
3866         }
3867
3868         if (info->on) {
3869                 vsi->info.pvid = info->config.pvid;
3870                 /**
3871                  * If insert pvid is enabled, only tagged pkts are
3872                  * allowed to be sent out.
3873                  */
3874                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
3875                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
3876         } else {
3877                 vsi->info.pvid = 0;
3878                 if (info->config.reject.tagged == 0)
3879                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
3880
3881                 if (info->config.reject.untagged == 0)
3882                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
3883         }
3884         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
3885                                         I40E_AQ_VSI_PVLAN_MODE_MASK);
3886         vsi->info.port_vlan_flags |= vlan_flags;
3887         vsi->info.valid_sections =
3888                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
3889         memset(&ctxt, 0, sizeof(ctxt));
3890         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3891         ctxt.seid = vsi->seid;
3892
3893         hw = I40E_VSI_TO_HW(vsi);
3894         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3895         if (ret != I40E_SUCCESS)
3896                 PMD_DRV_LOG(ERR, "Failed to update VSI params");
3897
3898         return ret;
3899 }
3900
3901 static int
3902 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
3903 {
3904         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3905         int i, ret;
3906         struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
3907
3908         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
3909         if (ret != I40E_SUCCESS)
3910                 return ret;
3911
3912         if (!vsi->seid) {
3913                 PMD_DRV_LOG(ERR, "seid not valid");
3914                 return -EINVAL;
3915         }
3916
3917         memset(&tc_bw_data, 0, sizeof(tc_bw_data));
3918         tc_bw_data.tc_valid_bits = enabled_tcmap;
3919         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3920                 tc_bw_data.tc_bw_credits[i] =
3921                         (enabled_tcmap & (1 << i)) ? 1 : 0;
3922
3923         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
3924         if (ret != I40E_SUCCESS) {
3925                 PMD_DRV_LOG(ERR, "Failed to configure TC BW");
3926                 return ret;
3927         }
3928
3929         (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
3930                                         sizeof(vsi->info.qs_handle));
3931         return I40E_SUCCESS;
3932 }
3933
3934 static enum i40e_status_code
3935 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
3936                                  struct i40e_aqc_vsi_properties_data *info,
3937                                  uint8_t enabled_tcmap)
3938 {
3939         enum i40e_status_code ret;
3940         int i, total_tc = 0;
3941         uint16_t qpnum_per_tc, bsf, qp_idx;
3942
3943         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
3944         if (ret != I40E_SUCCESS)
3945                 return ret;
3946
3947         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3948                 if (enabled_tcmap & (1 << i))
3949                         total_tc++;
3950         vsi->enabled_tc = enabled_tcmap;
3951
3952         /* Number of queues per enabled TC */
3953         qpnum_per_tc = i40e_align_floor(vsi->nb_qps / total_tc);
3954         qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
3955         bsf = rte_bsf32(qpnum_per_tc);
3956
3957         /* Adjust the queue number to actual queues that can be applied */
3958         if (!(vsi->type == I40E_VSI_MAIN && total_tc == 1))
3959                 vsi->nb_qps = qpnum_per_tc * total_tc;
3960
3961         /**
3962          * Configure TC and queue mapping parameters, for enabled TC,
3963          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
3964          * default queue will serve it.
3965          */
3966         qp_idx = 0;
3967         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3968                 if (vsi->enabled_tc & (1 << i)) {
3969                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
3970                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
3971                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
3972                         qp_idx += qpnum_per_tc;
3973                 } else
3974                         info->tc_mapping[i] = 0;
3975         }
3976
3977         /* Associate queue number with VSI */
3978         if (vsi->type == I40E_VSI_SRIOV) {
3979                 info->mapping_flags |=
3980                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
3981                 for (i = 0; i < vsi->nb_qps; i++)
3982                         info->queue_mapping[i] =
3983                                 rte_cpu_to_le_16(vsi->base_queue + i);
3984         } else {
3985                 info->mapping_flags |=
3986                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
3987                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
3988         }
3989         info->valid_sections |=
3990                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
3991
3992         return I40E_SUCCESS;
3993 }
3994
3995 static int
3996 i40e_veb_release(struct i40e_veb *veb)
3997 {
3998         struct i40e_vsi *vsi;
3999         struct i40e_hw *hw;
4000
4001         if (veb == NULL)
4002                 return -EINVAL;
4003
4004         if (!TAILQ_EMPTY(&veb->head)) {
4005                 PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove");
4006                 return -EACCES;
4007         }
4008         /* associate_vsi field is NULL for floating VEB */
4009         if (veb->associate_vsi != NULL) {
4010                 vsi = veb->associate_vsi;
4011                 hw = I40E_VSI_TO_HW(vsi);
4012
4013                 vsi->uplink_seid = veb->uplink_seid;
4014                 vsi->veb = NULL;
4015         } else {
4016                 veb->associate_pf->main_vsi->floating_veb = NULL;
4017                 hw = I40E_VSI_TO_HW(veb->associate_pf->main_vsi);
4018         }
4019
4020         i40e_aq_delete_element(hw, veb->seid, NULL);
4021         rte_free(veb);
4022         return I40E_SUCCESS;
4023 }
4024
4025 /* Setup a veb */
4026 static struct i40e_veb *
4027 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
4028 {
4029         struct i40e_veb *veb;
4030         int ret;
4031         struct i40e_hw *hw;
4032
4033         if (pf == NULL) {
4034                 PMD_DRV_LOG(ERR,
4035                             "veb setup failed, associated PF shouldn't null");
4036                 return NULL;
4037         }
4038         hw = I40E_PF_TO_HW(pf);
4039
4040         veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
4041         if (!veb) {
4042                 PMD_DRV_LOG(ERR, "Failed to allocate memory for veb");
4043                 goto fail;
4044         }
4045
4046         veb->associate_vsi = vsi;
4047         veb->associate_pf = pf;
4048         TAILQ_INIT(&veb->head);
4049         veb->uplink_seid = vsi ? vsi->uplink_seid : 0;
4050
4051         /* create floating veb if vsi is NULL */
4052         if (vsi != NULL) {
4053                 ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
4054                                       I40E_DEFAULT_TCMAP, false,
4055                                       &veb->seid, false, NULL);
4056         } else {
4057                 ret = i40e_aq_add_veb(hw, 0, 0, I40E_DEFAULT_TCMAP,
4058                                       true, &veb->seid, false, NULL);
4059         }
4060
4061         if (ret != I40E_SUCCESS) {
4062                 PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d",
4063                             hw->aq.asq_last_status);
4064                 goto fail;
4065         }
4066
4067         /* get statistics index */
4068         ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
4069                                 &veb->stats_idx, NULL, NULL, NULL);
4070         if (ret != I40E_SUCCESS) {
4071                 PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d",
4072                             hw->aq.asq_last_status);
4073                 goto fail;
4074         }
4075         /* Get VEB bandwidth, to be implemented */
4076         /* Now associated vsi binding to the VEB, set uplink to this VEB */
4077         if (vsi)
4078                 vsi->uplink_seid = veb->seid;
4079
4080         return veb;
4081 fail:
4082         rte_free(veb);
4083         return NULL;
4084 }
4085
4086 int
4087 i40e_vsi_release(struct i40e_vsi *vsi)
4088 {
4089         struct i40e_pf *pf;
4090         struct i40e_hw *hw;
4091         struct i40e_vsi_list *vsi_list;
4092         void *temp;
4093         int ret;
4094         struct i40e_mac_filter *f;
4095         uint16_t user_param = vsi->user_param;
4096
4097         if (!vsi)
4098                 return I40E_SUCCESS;
4099
4100         pf = I40E_VSI_TO_PF(vsi);
4101         hw = I40E_VSI_TO_HW(vsi);
4102
4103         /* VSI has child to attach, release child first */
4104         if (vsi->veb) {
4105                 TAILQ_FOREACH_SAFE(vsi_list, &vsi->veb->head, list, temp) {
4106                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
4107                                 return -1;
4108                 }
4109                 i40e_veb_release(vsi->veb);
4110         }
4111
4112         if (vsi->floating_veb) {
4113                 TAILQ_FOREACH_SAFE(vsi_list, &vsi->floating_veb->head, list, temp) {
4114                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
4115                                 return -1;
4116                 }
4117         }
4118
4119         /* Remove all macvlan filters of the VSI */
4120         i40e_vsi_remove_all_macvlan_filter(vsi);
4121         TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp)
4122                 rte_free(f);
4123
4124         if (vsi->type != I40E_VSI_MAIN &&
4125             ((vsi->type != I40E_VSI_SRIOV) ||
4126             !pf->floating_veb_list[user_param])) {
4127                 /* Remove vsi from parent's sibling list */
4128                 if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
4129                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
4130                         return I40E_ERR_PARAM;
4131                 }
4132                 TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
4133                                 &vsi->sib_vsi_list, list);
4134
4135                 /* Remove all switch element of the VSI */
4136                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
4137                 if (ret != I40E_SUCCESS)
4138                         PMD_DRV_LOG(ERR, "Failed to delete element");
4139         }
4140
4141         if ((vsi->type == I40E_VSI_SRIOV) &&
4142             pf->floating_veb_list[user_param]) {
4143                 /* Remove vsi from parent's sibling list */
4144                 if (vsi->parent_vsi == NULL ||
4145                     vsi->parent_vsi->floating_veb == NULL) {
4146                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
4147                         return I40E_ERR_PARAM;
4148                 }
4149                 TAILQ_REMOVE(&vsi->parent_vsi->floating_veb->head,
4150                              &vsi->sib_vsi_list, list);
4151
4152                 /* Remove all switch element of the VSI */
4153                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
4154                 if (ret != I40E_SUCCESS)
4155                         PMD_DRV_LOG(ERR, "Failed to delete element");
4156         }
4157
4158         i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
4159
4160         if (vsi->type != I40E_VSI_SRIOV)
4161                 i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
4162         rte_free(vsi);
4163
4164         return I40E_SUCCESS;
4165 }
4166
4167 static int
4168 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
4169 {
4170         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4171         struct i40e_aqc_remove_macvlan_element_data def_filter;
4172         struct i40e_mac_filter_info filter;
4173         int ret;
4174
4175         if (vsi->type != I40E_VSI_MAIN)
4176                 return I40E_ERR_CONFIG;
4177         memset(&def_filter, 0, sizeof(def_filter));
4178         (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
4179                                         ETH_ADDR_LEN);
4180         def_filter.vlan_tag = 0;
4181         def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
4182                                 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
4183         ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
4184         if (ret != I40E_SUCCESS) {
4185                 struct i40e_mac_filter *f;
4186                 struct ether_addr *mac;
4187
4188                 PMD_DRV_LOG(WARNING, "Cannot remove the default "
4189                             "macvlan filter");
4190                 /* It needs to add the permanent mac into mac list */
4191                 f = rte_zmalloc("macv_filter", sizeof(*f), 0);
4192                 if (f == NULL) {
4193                         PMD_DRV_LOG(ERR, "failed to allocate memory");
4194                         return I40E_ERR_NO_MEMORY;
4195                 }
4196                 mac = &f->mac_info.mac_addr;
4197                 (void)rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr,
4198                                 ETH_ADDR_LEN);
4199                 f->mac_info.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4200                 TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
4201                 vsi->mac_num++;
4202
4203                 return ret;
4204         }
4205         (void)rte_memcpy(&filter.mac_addr,
4206                 (struct ether_addr *)(hw->mac.perm_addr), ETH_ADDR_LEN);
4207         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4208         return i40e_vsi_add_mac(vsi, &filter);
4209 }
4210
4211 /*
4212  * i40e_vsi_get_bw_config - Query VSI BW Information
4213  * @vsi: the VSI to be queried
4214  *
4215  * Returns 0 on success, negative value on failure
4216  */
4217 static enum i40e_status_code
4218 i40e_vsi_get_bw_config(struct i40e_vsi *vsi)
4219 {
4220         struct i40e_aqc_query_vsi_bw_config_resp bw_config;
4221         struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
4222         struct i40e_hw *hw = &vsi->adapter->hw;
4223         i40e_status ret;
4224         int i;
4225         uint32_t bw_max;
4226
4227         memset(&bw_config, 0, sizeof(bw_config));
4228         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4229         if (ret != I40E_SUCCESS) {
4230                 PMD_DRV_LOG(ERR, "VSI failed to get bandwidth configuration %u",
4231                             hw->aq.asq_last_status);
4232                 return ret;
4233         }
4234
4235         memset(&ets_sla_config, 0, sizeof(ets_sla_config));
4236         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
4237                                         &ets_sla_config, NULL);
4238         if (ret != I40E_SUCCESS) {
4239                 PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith "
4240                             "configuration %u", hw->aq.asq_last_status);
4241                 return ret;
4242         }
4243
4244         /* store and print out BW info */
4245         vsi->bw_info.bw_limit = rte_le_to_cpu_16(bw_config.port_bw_limit);
4246         vsi->bw_info.bw_max = bw_config.max_bw;
4247         PMD_DRV_LOG(DEBUG, "VSI bw limit:%u", vsi->bw_info.bw_limit);
4248         PMD_DRV_LOG(DEBUG, "VSI max_bw:%u", vsi->bw_info.bw_max);
4249         bw_max = rte_le_to_cpu_16(ets_sla_config.tc_bw_max[0]) |
4250                     (rte_le_to_cpu_16(ets_sla_config.tc_bw_max[1]) <<
4251                      I40E_16_BIT_WIDTH);
4252         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4253                 vsi->bw_info.bw_ets_share_credits[i] =
4254                                 ets_sla_config.share_credits[i];
4255                 vsi->bw_info.bw_ets_credits[i] =
4256                                 rte_le_to_cpu_16(ets_sla_config.credits[i]);
4257                 /* 4 bits per TC, 4th bit is reserved */
4258                 vsi->bw_info.bw_ets_max[i] =
4259                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
4260                                   RTE_LEN2MASK(3, uint8_t));
4261                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:share credits %u", i,
4262                             vsi->bw_info.bw_ets_share_credits[i]);
4263                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:credits %u", i,
4264                             vsi->bw_info.bw_ets_credits[i]);
4265                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u: max credits: %u", i,
4266                             vsi->bw_info.bw_ets_max[i]);
4267         }
4268
4269         return I40E_SUCCESS;
4270 }
4271
4272 /* i40e_enable_pf_lb
4273  * @pf: pointer to the pf structure
4274  *
4275  * allow loopback on pf
4276  */
4277 static inline void
4278 i40e_enable_pf_lb(struct i40e_pf *pf)
4279 {
4280         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4281         struct i40e_vsi_context ctxt;
4282         int ret;
4283
4284         /* Use the FW API if FW >= v5.0 */
4285         if (hw->aq.fw_maj_ver < 5) {
4286                 PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback");
4287                 return;
4288         }
4289
4290         memset(&ctxt, 0, sizeof(ctxt));
4291         ctxt.seid = pf->main_vsi_seid;
4292         ctxt.pf_num = hw->pf_id;
4293         ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
4294         if (ret) {
4295                 PMD_DRV_LOG(ERR, "cannot get pf vsi config, err %d, aq_err %d",
4296                             ret, hw->aq.asq_last_status);
4297                 return;
4298         }
4299         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
4300         ctxt.info.valid_sections =
4301                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4302         ctxt.info.switch_id |=
4303                 rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4304
4305         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4306         if (ret)
4307                 PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d\n",
4308                             hw->aq.asq_last_status);
4309 }
4310
4311 /* Setup a VSI */
4312 struct i40e_vsi *
4313 i40e_vsi_setup(struct i40e_pf *pf,
4314                enum i40e_vsi_type type,
4315                struct i40e_vsi *uplink_vsi,
4316                uint16_t user_param)
4317 {
4318         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4319         struct i40e_vsi *vsi;
4320         struct i40e_mac_filter_info filter;
4321         int ret;
4322         struct i40e_vsi_context ctxt;
4323         struct ether_addr broadcast =
4324                 {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
4325
4326         if (type != I40E_VSI_MAIN && type != I40E_VSI_SRIOV &&
4327             uplink_vsi == NULL) {
4328                 PMD_DRV_LOG(ERR, "VSI setup failed, "
4329                             "VSI link shouldn't be NULL");
4330                 return NULL;
4331         }
4332
4333         if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
4334                 PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI "
4335                             "uplink VSI should be NULL");
4336                 return NULL;
4337         }
4338
4339         /* two situations
4340          * 1.type is not MAIN and uplink vsi is not NULL
4341          * If uplink vsi didn't setup VEB, create one first under veb field
4342          * 2.type is SRIOV and the uplink is NULL
4343          * If floating VEB is NULL, create one veb under floating veb field
4344          */
4345
4346         if (type != I40E_VSI_MAIN && uplink_vsi != NULL &&
4347             uplink_vsi->veb == NULL) {
4348                 uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
4349
4350                 if (uplink_vsi->veb == NULL) {
4351                         PMD_DRV_LOG(ERR, "VEB setup failed");
4352                         return NULL;
4353                 }
4354                 /* set ALLOWLOOPBACk on pf, when veb is created */
4355                 i40e_enable_pf_lb(pf);
4356         }
4357
4358         if (type == I40E_VSI_SRIOV && uplink_vsi == NULL &&
4359             pf->main_vsi->floating_veb == NULL) {
4360                 pf->main_vsi->floating_veb = i40e_veb_setup(pf, uplink_vsi);
4361
4362                 if (pf->main_vsi->floating_veb == NULL) {
4363                         PMD_DRV_LOG(ERR, "VEB setup failed");
4364                         return NULL;
4365                 }
4366         }
4367
4368         vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
4369         if (!vsi) {
4370                 PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi");
4371                 return NULL;
4372         }
4373         TAILQ_INIT(&vsi->mac_list);
4374         vsi->type = type;
4375         vsi->adapter = I40E_PF_TO_ADAPTER(pf);
4376         vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
4377         vsi->parent_vsi = uplink_vsi ? uplink_vsi : pf->main_vsi;
4378         vsi->user_param = user_param;
4379         /* Allocate queues */
4380         switch (vsi->type) {
4381         case I40E_VSI_MAIN  :
4382                 vsi->nb_qps = pf->lan_nb_qps;
4383                 break;
4384         case I40E_VSI_SRIOV :
4385                 vsi->nb_qps = pf->vf_nb_qps;
4386                 break;
4387         case I40E_VSI_VMDQ2:
4388                 vsi->nb_qps = pf->vmdq_nb_qps;
4389                 break;
4390         case I40E_VSI_FDIR:
4391                 vsi->nb_qps = pf->fdir_nb_qps;
4392                 break;
4393         default:
4394                 goto fail_mem;
4395         }
4396         /*
4397          * The filter status descriptor is reported in rx queue 0,
4398          * while the tx queue for fdir filter programming has no
4399          * such constraints, can be non-zero queues.
4400          * To simplify it, choose FDIR vsi use queue 0 pair.
4401          * To make sure it will use queue 0 pair, queue allocation
4402          * need be done before this function is called
4403          */
4404         if (type != I40E_VSI_FDIR) {
4405                 ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
4406                         if (ret < 0) {
4407                                 PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
4408                                                 vsi->seid, ret);
4409                                 goto fail_mem;
4410                         }
4411                         vsi->base_queue = ret;
4412         } else
4413                 vsi->base_queue = I40E_FDIR_QUEUE_ID;
4414
4415         /* VF has MSIX interrupt in VF range, don't allocate here */
4416         if (type == I40E_VSI_MAIN) {
4417                 ret = i40e_res_pool_alloc(&pf->msix_pool,
4418                                           RTE_MIN(vsi->nb_qps,
4419                                                   RTE_MAX_RXTX_INTR_VEC_ID));
4420                 if (ret < 0) {
4421                         PMD_DRV_LOG(ERR, "VSI MAIN %d get heap failed %d",
4422                                     vsi->seid, ret);
4423                         goto fail_queue_alloc;
4424                 }
4425                 vsi->msix_intr = ret;
4426                 vsi->nb_msix = RTE_MIN(vsi->nb_qps, RTE_MAX_RXTX_INTR_VEC_ID);
4427         } else if (type != I40E_VSI_SRIOV) {
4428                 ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
4429                 if (ret < 0) {
4430                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
4431                         goto fail_queue_alloc;
4432                 }
4433                 vsi->msix_intr = ret;
4434                 vsi->nb_msix = 1;
4435         } else {
4436                 vsi->msix_intr = 0;
4437                 vsi->nb_msix = 0;
4438         }
4439
4440         /* Add VSI */
4441         if (type == I40E_VSI_MAIN) {
4442                 /* For main VSI, no need to add since it's default one */
4443                 vsi->uplink_seid = pf->mac_seid;
4444                 vsi->seid = pf->main_vsi_seid;
4445                 /* Bind queues with specific MSIX interrupt */
4446                 /**
4447                  * Needs 2 interrupt at least, one for misc cause which will
4448                  * enabled from OS side, Another for queues binding the
4449                  * interrupt from device side only.
4450                  */
4451
4452                 /* Get default VSI parameters from hardware */
4453                 memset(&ctxt, 0, sizeof(ctxt));
4454                 ctxt.seid = vsi->seid;
4455                 ctxt.pf_num = hw->pf_id;
4456                 ctxt.uplink_seid = vsi->uplink_seid;
4457                 ctxt.vf_num = 0;
4458                 ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
4459                 if (ret != I40E_SUCCESS) {
4460                         PMD_DRV_LOG(ERR, "Failed to get VSI params");
4461                         goto fail_msix_alloc;
4462                 }
4463                 (void)rte_memcpy(&vsi->info, &ctxt.info,
4464                         sizeof(struct i40e_aqc_vsi_properties_data));
4465                 vsi->vsi_id = ctxt.vsi_number;
4466                 vsi->info.valid_sections = 0;
4467
4468                 /* Configure tc, enabled TC0 only */
4469                 if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
4470                         I40E_SUCCESS) {
4471                         PMD_DRV_LOG(ERR, "Failed to update TC bandwidth");
4472                         goto fail_msix_alloc;
4473                 }
4474
4475                 /* TC, queue mapping */
4476                 memset(&ctxt, 0, sizeof(ctxt));
4477                 vsi->info.valid_sections |=
4478                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4479                 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
4480                                         I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
4481                 (void)rte_memcpy(&ctxt.info, &vsi->info,
4482                         sizeof(struct i40e_aqc_vsi_properties_data));
4483                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4484                                                 I40E_DEFAULT_TCMAP);
4485                 if (ret != I40E_SUCCESS) {
4486                         PMD_DRV_LOG(ERR, "Failed to configure "
4487                                     "TC queue mapping");
4488                         goto fail_msix_alloc;
4489                 }
4490                 ctxt.seid = vsi->seid;
4491                 ctxt.pf_num = hw->pf_id;
4492                 ctxt.uplink_seid = vsi->uplink_seid;
4493                 ctxt.vf_num = 0;
4494
4495                 /* Update VSI parameters */
4496                 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4497                 if (ret != I40E_SUCCESS) {
4498                         PMD_DRV_LOG(ERR, "Failed to update VSI params");
4499                         goto fail_msix_alloc;
4500                 }
4501
4502                 (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
4503                                                 sizeof(vsi->info.tc_mapping));
4504                 (void)rte_memcpy(&vsi->info.queue_mapping,
4505                                 &ctxt.info.queue_mapping,
4506                         sizeof(vsi->info.queue_mapping));
4507                 vsi->info.mapping_flags = ctxt.info.mapping_flags;
4508                 vsi->info.valid_sections = 0;
4509
4510                 (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
4511                                 ETH_ADDR_LEN);
4512
4513                 /**
4514                  * Updating default filter settings are necessary to prevent
4515                  * reception of tagged packets.
4516                  * Some old firmware configurations load a default macvlan
4517                  * filter which accepts both tagged and untagged packets.
4518                  * The updating is to use a normal filter instead if needed.
4519                  * For NVM 4.2.2 or after, the updating is not needed anymore.
4520                  * The firmware with correct configurations load the default
4521                  * macvlan filter which is expected and cannot be removed.
4522                  */
4523                 i40e_update_default_filter_setting(vsi);
4524                 i40e_config_qinq(hw, vsi);
4525         } else if (type == I40E_VSI_SRIOV) {
4526                 memset(&ctxt, 0, sizeof(ctxt));
4527                 /**
4528                  * For other VSI, the uplink_seid equals to uplink VSI's
4529                  * uplink_seid since they share same VEB
4530                  */
4531                 if (uplink_vsi == NULL)
4532                         vsi->uplink_seid = pf->main_vsi->floating_veb->seid;
4533                 else
4534                         vsi->uplink_seid = uplink_vsi->uplink_seid;
4535                 ctxt.pf_num = hw->pf_id;
4536                 ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
4537                 ctxt.uplink_seid = vsi->uplink_seid;
4538                 ctxt.connection_type = 0x1;
4539                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
4540
4541                 /* Use the VEB configuration if FW >= v5.0 */
4542                 if (hw->aq.fw_maj_ver >= 5) {
4543                         /* Configure switch ID */
4544                         ctxt.info.valid_sections |=
4545                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4546                         ctxt.info.switch_id =
4547                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4548                 }
4549
4550                 /* Configure port/vlan */
4551                 ctxt.info.valid_sections |=
4552                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4553                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4554                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4555                                                 I40E_DEFAULT_TCMAP);
4556                 if (ret != I40E_SUCCESS) {
4557                         PMD_DRV_LOG(ERR, "Failed to configure "
4558                                     "TC queue mapping");
4559                         goto fail_msix_alloc;
4560                 }
4561                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4562                 ctxt.info.valid_sections |=
4563                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4564                 /**
4565                  * Since VSI is not created yet, only configure parameter,
4566                  * will add vsi below.
4567                  */
4568
4569                 i40e_config_qinq(hw, vsi);
4570         } else if (type == I40E_VSI_VMDQ2) {
4571                 memset(&ctxt, 0, sizeof(ctxt));
4572                 /*
4573                  * For other VSI, the uplink_seid equals to uplink VSI's
4574                  * uplink_seid since they share same VEB
4575                  */
4576                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4577                 ctxt.pf_num = hw->pf_id;
4578                 ctxt.vf_num = 0;
4579                 ctxt.uplink_seid = vsi->uplink_seid;
4580                 ctxt.connection_type = 0x1;
4581                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
4582
4583                 ctxt.info.valid_sections |=
4584                                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4585                 /* user_param carries flag to enable loop back */
4586                 if (user_param) {
4587                         ctxt.info.switch_id =
4588                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
4589                         ctxt.info.switch_id |=
4590                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4591                 }
4592
4593                 /* Configure port/vlan */
4594                 ctxt.info.valid_sections |=
4595                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4596                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4597                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4598                                                 I40E_DEFAULT_TCMAP);
4599                 if (ret != I40E_SUCCESS) {
4600                         PMD_DRV_LOG(ERR, "Failed to configure "
4601                                         "TC queue mapping");
4602                         goto fail_msix_alloc;
4603                 }
4604                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4605                 ctxt.info.valid_sections |=
4606                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4607         } else if (type == I40E_VSI_FDIR) {
4608                 memset(&ctxt, 0, sizeof(ctxt));
4609                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4610                 ctxt.pf_num = hw->pf_id;
4611                 ctxt.vf_num = 0;
4612                 ctxt.uplink_seid = vsi->uplink_seid;
4613                 ctxt.connection_type = 0x1;     /* regular data port */
4614                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
4615                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4616                                                 I40E_DEFAULT_TCMAP);
4617                 if (ret != I40E_SUCCESS) {
4618                         PMD_DRV_LOG(ERR, "Failed to configure "
4619                                         "TC queue mapping.");
4620                         goto fail_msix_alloc;
4621                 }
4622                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4623                 ctxt.info.valid_sections |=
4624                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4625         } else {
4626                 PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet");
4627                 goto fail_msix_alloc;
4628         }
4629
4630         if (vsi->type != I40E_VSI_MAIN) {
4631                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
4632                 if (ret != I40E_SUCCESS) {
4633                         PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d",
4634                                     hw->aq.asq_last_status);
4635                         goto fail_msix_alloc;
4636                 }
4637                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
4638                 vsi->info.valid_sections = 0;
4639                 vsi->seid = ctxt.seid;
4640                 vsi->vsi_id = ctxt.vsi_number;
4641                 vsi->sib_vsi_list.vsi = vsi;
4642                 if (vsi->type == I40E_VSI_SRIOV && uplink_vsi == NULL) {
4643                         TAILQ_INSERT_TAIL(&pf->main_vsi->floating_veb->head,
4644                                           &vsi->sib_vsi_list, list);
4645                 } else {
4646                         TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
4647                                           &vsi->sib_vsi_list, list);
4648                 }
4649         }
4650
4651         /* MAC/VLAN configuration */
4652         (void)rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN);
4653         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4654
4655         ret = i40e_vsi_add_mac(vsi, &filter);
4656         if (ret != I40E_SUCCESS) {
4657                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
4658                 goto fail_msix_alloc;
4659         }
4660
4661         /* Get VSI BW information */
4662         i40e_vsi_get_bw_config(vsi);
4663         return vsi;
4664 fail_msix_alloc:
4665         i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
4666 fail_queue_alloc:
4667         i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
4668 fail_mem:
4669         rte_free(vsi);
4670         return NULL;
4671 }
4672
4673 /* Configure vlan filter on or off */
4674 int
4675 i40e_vsi_config_vlan_filter(struct i40e_vsi *vsi, bool on)
4676 {
4677         int i, num;
4678         struct i40e_mac_filter *f;
4679         void *temp;
4680         struct i40e_mac_filter_info *mac_filter;
4681         enum rte_mac_filter_type desired_filter;
4682         int ret = I40E_SUCCESS;
4683
4684         if (on) {
4685                 /* Filter to match MAC and VLAN */
4686                 desired_filter = RTE_MACVLAN_PERFECT_MATCH;
4687         } else {
4688                 /* Filter to match only MAC */
4689                 desired_filter = RTE_MAC_PERFECT_MATCH;
4690         }
4691
4692         num = vsi->mac_num;
4693
4694         mac_filter = rte_zmalloc("mac_filter_info_data",
4695                                  num * sizeof(*mac_filter), 0);
4696         if (mac_filter == NULL) {
4697                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4698                 return I40E_ERR_NO_MEMORY;
4699         }
4700
4701         i = 0;
4702
4703         /* Remove all existing mac */
4704         TAILQ_FOREACH_SAFE(f, &vsi->mac_list, next, temp) {
4705                 mac_filter[i] = f->mac_info;
4706                 ret = i40e_vsi_delete_mac(vsi, &f->mac_info.mac_addr);
4707                 if (ret) {
4708                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
4709                                     on ? "enable" : "disable");
4710                         goto DONE;
4711                 }
4712                 i++;
4713         }
4714
4715         /* Override with new filter */
4716         for (i = 0; i < num; i++) {
4717                 mac_filter[i].filter_type = desired_filter;
4718                 ret = i40e_vsi_add_mac(vsi, &mac_filter[i]);
4719                 if (ret) {
4720                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
4721                                     on ? "enable" : "disable");
4722                         goto DONE;
4723                 }
4724         }
4725
4726 DONE:
4727         rte_free(mac_filter);
4728         return ret;
4729 }
4730
4731 /* Configure vlan stripping on or off */
4732 int
4733 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
4734 {
4735         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4736         struct i40e_vsi_context ctxt;
4737         uint8_t vlan_flags;
4738         int ret = I40E_SUCCESS;
4739
4740         /* Check if it has been already on or off */
4741         if (vsi->info.valid_sections &
4742                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
4743                 if (on) {
4744                         if ((vsi->info.port_vlan_flags &
4745                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
4746                                 return 0; /* already on */
4747                 } else {
4748                         if ((vsi->info.port_vlan_flags &
4749                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
4750                                 I40E_AQ_VSI_PVLAN_EMOD_MASK)
4751                                 return 0; /* already off */
4752                 }
4753         }
4754
4755         if (on)
4756                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
4757         else
4758                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
4759         vsi->info.valid_sections =
4760                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4761         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
4762         vsi->info.port_vlan_flags |= vlan_flags;
4763         ctxt.seid = vsi->seid;
4764         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
4765         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4766         if (ret)
4767                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping",
4768                             on ? "enable" : "disable");
4769
4770         return ret;
4771 }
4772
4773 static int
4774 i40e_dev_init_vlan(struct rte_eth_dev *dev)
4775 {
4776         struct rte_eth_dev_data *data = dev->data;
4777         int ret;
4778         int mask = 0;
4779
4780         /* Apply vlan offload setting */
4781         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK;
4782         i40e_vlan_offload_set(dev, mask);
4783
4784         /* Apply double-vlan setting, not implemented yet */
4785
4786         /* Apply pvid setting */
4787         ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
4788                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
4789         if (ret)
4790                 PMD_DRV_LOG(INFO, "Failed to update VSI params");
4791
4792         return ret;
4793 }
4794
4795 static int
4796 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
4797 {
4798         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4799
4800         return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
4801 }
4802
4803 static int
4804 i40e_update_flow_control(struct i40e_hw *hw)
4805 {
4806 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
4807         struct i40e_link_status link_status;
4808         uint32_t rxfc = 0, txfc = 0, reg;
4809         uint8_t an_info;
4810         int ret;
4811
4812         memset(&link_status, 0, sizeof(link_status));
4813         ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
4814         if (ret != I40E_SUCCESS) {
4815                 PMD_DRV_LOG(ERR, "Failed to get link status information");
4816                 goto write_reg; /* Disable flow control */
4817         }
4818
4819         an_info = hw->phy.link_info.an_info;
4820         if (!(an_info & I40E_AQ_AN_COMPLETED)) {
4821                 PMD_DRV_LOG(INFO, "Link auto negotiation not completed");
4822                 ret = I40E_ERR_NOT_READY;
4823                 goto write_reg; /* Disable flow control */
4824         }
4825         /**
4826          * If link auto negotiation is enabled, flow control needs to
4827          * be configured according to it
4828          */
4829         switch (an_info & I40E_LINK_PAUSE_RXTX) {
4830         case I40E_LINK_PAUSE_RXTX:
4831                 rxfc = 1;
4832                 txfc = 1;
4833                 hw->fc.current_mode = I40E_FC_FULL;
4834                 break;
4835         case I40E_AQ_LINK_PAUSE_RX:
4836                 rxfc = 1;
4837                 hw->fc.current_mode = I40E_FC_RX_PAUSE;
4838                 break;
4839         case I40E_AQ_LINK_PAUSE_TX:
4840                 txfc = 1;
4841                 hw->fc.current_mode = I40E_FC_TX_PAUSE;
4842                 break;
4843         default:
4844                 hw->fc.current_mode = I40E_FC_NONE;
4845                 break;
4846         }
4847
4848 write_reg:
4849         I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
4850                 txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
4851         reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
4852         reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
4853         reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
4854         I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
4855
4856         return ret;
4857 }
4858
4859 /* PF setup */
4860 static int
4861 i40e_pf_setup(struct i40e_pf *pf)
4862 {
4863         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4864         struct i40e_filter_control_settings settings;
4865         struct i40e_vsi *vsi;
4866         int ret;
4867
4868         /* Clear all stats counters */
4869         pf->offset_loaded = FALSE;
4870         memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
4871         memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
4872
4873         ret = i40e_pf_get_switch_config(pf);
4874         if (ret != I40E_SUCCESS) {
4875                 PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
4876                 return ret;
4877         }
4878         if (pf->flags & I40E_FLAG_FDIR) {
4879                 /* make queue allocated first, let FDIR use queue pair 0*/
4880                 ret = i40e_res_pool_alloc(&pf->qp_pool, I40E_DEFAULT_QP_NUM_FDIR);
4881                 if (ret != I40E_FDIR_QUEUE_ID) {
4882                         PMD_DRV_LOG(ERR, "queue allocation fails for FDIR :"
4883                                     " ret =%d", ret);
4884                         pf->flags &= ~I40E_FLAG_FDIR;
4885                 }
4886         }
4887         /*  main VSI setup */
4888         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
4889         if (!vsi) {
4890                 PMD_DRV_LOG(ERR, "Setup of main vsi failed");
4891                 return I40E_ERR_NOT_READY;
4892         }
4893         pf->main_vsi = vsi;
4894
4895         /* Configure filter control */
4896         memset(&settings, 0, sizeof(settings));
4897         if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_128)
4898                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
4899         else if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_512)
4900                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_512;
4901         else {
4902                 PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported\n",
4903                                                 hw->func_caps.rss_table_size);
4904                 return I40E_ERR_PARAM;
4905         }
4906         PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table "
4907                         "size: %u\n", hw->func_caps.rss_table_size);
4908         pf->hash_lut_size = hw->func_caps.rss_table_size;
4909
4910         /* Enable ethtype and macvlan filters */
4911         settings.enable_ethtype = TRUE;
4912         settings.enable_macvlan = TRUE;
4913         ret = i40e_set_filter_control(hw, &settings);
4914         if (ret)
4915                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
4916                                                                 ret);
4917
4918         /* Update flow control according to the auto negotiation */
4919         i40e_update_flow_control(hw);
4920
4921         return I40E_SUCCESS;
4922 }
4923
4924 int
4925 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
4926 {
4927         uint32_t reg;
4928         uint16_t j;
4929
4930         /**
4931          * Set or clear TX Queue Disable flags,
4932          * which is required by hardware.
4933          */
4934         i40e_pre_tx_queue_cfg(hw, q_idx, on);
4935         rte_delay_us(I40E_PRE_TX_Q_CFG_WAIT_US);
4936
4937         /* Wait until the request is finished */
4938         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4939                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4940                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
4941                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
4942                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
4943                                                         & 0x1))) {
4944                         break;
4945                 }
4946         }
4947         if (on) {
4948                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4949                         return I40E_SUCCESS; /* already on, skip next steps */
4950
4951                 I40E_WRITE_REG(hw, I40E_QTX_HEAD(q_idx), 0);
4952                 reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4953         } else {
4954                 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4955                         return I40E_SUCCESS; /* already off, skip next steps */
4956                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4957         }
4958         /* Write the register */
4959         I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
4960         /* Check the result */
4961         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4962                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4963                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
4964                 if (on) {
4965                         if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
4966                                 (reg & I40E_QTX_ENA_QENA_STAT_MASK))
4967                                 break;
4968                 } else {
4969                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
4970                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4971                                 break;
4972                 }
4973         }
4974         /* Check if it is timeout */
4975         if (j >= I40E_CHK_Q_ENA_COUNT) {
4976                 PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]",
4977                             (on ? "enable" : "disable"), q_idx);
4978                 return I40E_ERR_TIMEOUT;
4979         }
4980
4981         return I40E_SUCCESS;
4982 }
4983
4984 /* Swith on or off the tx queues */
4985 static int
4986 i40e_dev_switch_tx_queues(struct i40e_pf *pf, bool on)
4987 {
4988         struct rte_eth_dev_data *dev_data = pf->dev_data;
4989         struct i40e_tx_queue *txq;
4990         struct rte_eth_dev *dev = pf->adapter->eth_dev;
4991         uint16_t i;
4992         int ret;
4993
4994         for (i = 0; i < dev_data->nb_tx_queues; i++) {
4995                 txq = dev_data->tx_queues[i];
4996                 /* Don't operate the queue if not configured or
4997                  * if starting only per queue */
4998                 if (!txq || !txq->q_set || (on && txq->tx_deferred_start))
4999                         continue;
5000                 if (on)
5001                         ret = i40e_dev_tx_queue_start(dev, i);
5002                 else
5003                         ret = i40e_dev_tx_queue_stop(dev, i);
5004                 if ( ret != I40E_SUCCESS)
5005                         return ret;
5006         }
5007
5008         return I40E_SUCCESS;
5009 }
5010
5011 int
5012 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
5013 {
5014         uint32_t reg;
5015         uint16_t j;
5016
5017         /* Wait until the request is finished */
5018         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5019                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5020                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
5021                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
5022                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
5023                         break;
5024         }
5025
5026         if (on) {
5027                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
5028                         return I40E_SUCCESS; /* Already on, skip next steps */
5029                 reg |= I40E_QRX_ENA_QENA_REQ_MASK;
5030         } else {
5031                 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
5032                         return I40E_SUCCESS; /* Already off, skip next steps */
5033                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
5034         }
5035
5036         /* Write the register */
5037         I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
5038         /* Check the result */
5039         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
5040                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
5041                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
5042                 if (on) {
5043                         if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
5044                                 (reg & I40E_QRX_ENA_QENA_STAT_MASK))
5045                                 break;
5046                 } else {
5047                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
5048                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
5049                                 break;
5050                 }
5051         }
5052
5053         /* Check if it is timeout */
5054         if (j >= I40E_CHK_Q_ENA_COUNT) {
5055                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
5056                             (on ? "enable" : "disable"), q_idx);
5057                 return I40E_ERR_TIMEOUT;
5058         }
5059
5060         return I40E_SUCCESS;
5061 }
5062 /* Switch on or off the rx queues */
5063 static int
5064 i40e_dev_switch_rx_queues(struct i40e_pf *pf, bool on)
5065 {
5066         struct rte_eth_dev_data *dev_data = pf->dev_data;
5067         struct i40e_rx_queue *rxq;
5068         struct rte_eth_dev *dev = pf->adapter->eth_dev;
5069         uint16_t i;
5070         int ret;
5071
5072         for (i = 0; i < dev_data->nb_rx_queues; i++) {
5073                 rxq = dev_data->rx_queues[i];
5074                 /* Don't operate the queue if not configured or
5075                  * if starting only per queue */
5076                 if (!rxq || !rxq->q_set || (on && rxq->rx_deferred_start))
5077                         continue;
5078                 if (on)
5079                         ret = i40e_dev_rx_queue_start(dev, i);
5080                 else
5081                         ret = i40e_dev_rx_queue_stop(dev, i);
5082                 if (ret != I40E_SUCCESS)
5083                         return ret;
5084         }
5085
5086         return I40E_SUCCESS;
5087 }
5088
5089 /* Switch on or off all the rx/tx queues */
5090 int
5091 i40e_dev_switch_queues(struct i40e_pf *pf, bool on)
5092 {
5093         int ret;
5094
5095         if (on) {
5096                 /* enable rx queues before enabling tx queues */
5097                 ret = i40e_dev_switch_rx_queues(pf, on);
5098                 if (ret) {
5099                         PMD_DRV_LOG(ERR, "Failed to switch rx queues");
5100                         return ret;
5101                 }
5102                 ret = i40e_dev_switch_tx_queues(pf, on);
5103         } else {
5104                 /* Stop tx queues before stopping rx queues */
5105                 ret = i40e_dev_switch_tx_queues(pf, on);
5106                 if (ret) {
5107                         PMD_DRV_LOG(ERR, "Failed to switch tx queues");
5108                         return ret;
5109                 }
5110                 ret = i40e_dev_switch_rx_queues(pf, on);
5111         }
5112
5113         return ret;
5114 }
5115
5116 /* Initialize VSI for TX */
5117 static int
5118 i40e_dev_tx_init(struct i40e_pf *pf)
5119 {
5120         struct rte_eth_dev_data *data = pf->dev_data;
5121         uint16_t i;
5122         uint32_t ret = I40E_SUCCESS;
5123         struct i40e_tx_queue *txq;
5124
5125         for (i = 0; i < data->nb_tx_queues; i++) {
5126                 txq = data->tx_queues[i];
5127                 if (!txq || !txq->q_set)
5128                         continue;
5129                 ret = i40e_tx_queue_init(txq);
5130                 if (ret != I40E_SUCCESS)
5131                         break;
5132         }
5133         if (ret == I40E_SUCCESS)
5134                 i40e_set_tx_function(container_of(pf, struct i40e_adapter, pf)
5135                                      ->eth_dev);
5136
5137         return ret;
5138 }
5139
5140 /* Initialize VSI for RX */
5141 static int
5142 i40e_dev_rx_init(struct i40e_pf *pf)
5143 {
5144         struct rte_eth_dev_data *data = pf->dev_data;
5145         int ret = I40E_SUCCESS;
5146         uint16_t i;
5147         struct i40e_rx_queue *rxq;
5148
5149         i40e_pf_config_mq_rx(pf);
5150         for (i = 0; i < data->nb_rx_queues; i++) {
5151                 rxq = data->rx_queues[i];
5152                 if (!rxq || !rxq->q_set)
5153                         continue;
5154
5155                 ret = i40e_rx_queue_init(rxq);
5156                 if (ret != I40E_SUCCESS) {
5157                         PMD_DRV_LOG(ERR, "Failed to do RX queue "
5158                                     "initialization");
5159                         break;
5160                 }
5161         }
5162         if (ret == I40E_SUCCESS)
5163                 i40e_set_rx_function(container_of(pf, struct i40e_adapter, pf)
5164                                      ->eth_dev);
5165
5166         return ret;
5167 }
5168
5169 static int
5170 i40e_dev_rxtx_init(struct i40e_pf *pf)
5171 {
5172         int err;
5173
5174         err = i40e_dev_tx_init(pf);
5175         if (err) {
5176                 PMD_DRV_LOG(ERR, "Failed to do TX initialization");
5177                 return err;
5178         }
5179         err = i40e_dev_rx_init(pf);
5180         if (err) {
5181                 PMD_DRV_LOG(ERR, "Failed to do RX initialization");
5182                 return err;
5183         }
5184
5185         return err;
5186 }
5187
5188 static int
5189 i40e_vmdq_setup(struct rte_eth_dev *dev)
5190 {
5191         struct rte_eth_conf *conf = &dev->data->dev_conf;
5192         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5193         int i, err, conf_vsis, j, loop;
5194         struct i40e_vsi *vsi;
5195         struct i40e_vmdq_info *vmdq_info;
5196         struct rte_eth_vmdq_rx_conf *vmdq_conf;
5197         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5198
5199         /*
5200          * Disable interrupt to avoid message from VF. Furthermore, it will
5201          * avoid race condition in VSI creation/destroy.
5202          */
5203         i40e_pf_disable_irq0(hw);
5204
5205         if ((pf->flags & I40E_FLAG_VMDQ) == 0) {
5206                 PMD_INIT_LOG(ERR, "FW doesn't support VMDQ");
5207                 return -ENOTSUP;
5208         }
5209
5210         conf_vsis = conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
5211         if (conf_vsis > pf->max_nb_vmdq_vsi) {
5212                 PMD_INIT_LOG(ERR, "VMDQ config: %u, max support:%u",
5213                         conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools,
5214                         pf->max_nb_vmdq_vsi);
5215                 return -ENOTSUP;
5216         }
5217
5218         if (pf->vmdq != NULL) {
5219                 PMD_INIT_LOG(INFO, "VMDQ already configured");
5220                 return 0;
5221         }
5222
5223         pf->vmdq = rte_zmalloc("vmdq_info_struct",
5224                                 sizeof(*vmdq_info) * conf_vsis, 0);
5225
5226         if (pf->vmdq == NULL) {
5227                 PMD_INIT_LOG(ERR, "Failed to allocate memory");
5228                 return -ENOMEM;
5229         }
5230
5231         vmdq_conf = &conf->rx_adv_conf.vmdq_rx_conf;
5232
5233         /* Create VMDQ VSI */
5234         for (i = 0; i < conf_vsis; i++) {
5235                 vsi = i40e_vsi_setup(pf, I40E_VSI_VMDQ2, pf->main_vsi,
5236                                 vmdq_conf->enable_loop_back);
5237                 if (vsi == NULL) {
5238                         PMD_INIT_LOG(ERR, "Failed to create VMDQ VSI");
5239                         err = -1;
5240                         goto err_vsi_setup;
5241                 }
5242                 vmdq_info = &pf->vmdq[i];
5243                 vmdq_info->pf = pf;
5244                 vmdq_info->vsi = vsi;
5245         }
5246         pf->nb_cfg_vmdq_vsi = conf_vsis;
5247
5248         /* Configure Vlan */
5249         loop = sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT;
5250         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
5251                 for (j = 0; j < loop && j < pf->nb_cfg_vmdq_vsi; j++) {
5252                         if (vmdq_conf->pool_map[i].pools & (1UL << j)) {
5253                                 PMD_INIT_LOG(INFO, "Add vlan %u to vmdq pool %u",
5254                                         vmdq_conf->pool_map[i].vlan_id, j);
5255
5256                                 err = i40e_vsi_add_vlan(pf->vmdq[j].vsi,
5257                                                 vmdq_conf->pool_map[i].vlan_id);
5258                                 if (err) {
5259                                         PMD_INIT_LOG(ERR, "Failed to add vlan");
5260                                         err = -1;
5261                                         goto err_vsi_setup;
5262                                 }
5263                         }
5264                 }
5265         }
5266
5267         i40e_pf_enable_irq0(hw);
5268
5269         return 0;
5270
5271 err_vsi_setup:
5272         for (i = 0; i < conf_vsis; i++)
5273                 if (pf->vmdq[i].vsi == NULL)
5274                         break;
5275                 else
5276                         i40e_vsi_release(pf->vmdq[i].vsi);
5277
5278         rte_free(pf->vmdq);
5279         pf->vmdq = NULL;
5280         i40e_pf_enable_irq0(hw);
5281         return err;
5282 }
5283
5284 static void
5285 i40e_stat_update_32(struct i40e_hw *hw,
5286                    uint32_t reg,
5287                    bool offset_loaded,
5288                    uint64_t *offset,
5289                    uint64_t *stat)
5290 {
5291         uint64_t new_data;
5292
5293         new_data = (uint64_t)I40E_READ_REG(hw, reg);
5294         if (!offset_loaded)
5295                 *offset = new_data;
5296
5297         if (new_data >= *offset)
5298                 *stat = (uint64_t)(new_data - *offset);
5299         else
5300                 *stat = (uint64_t)((new_data +
5301                         ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset);
5302 }
5303
5304 static void
5305 i40e_stat_update_48(struct i40e_hw *hw,
5306                    uint32_t hireg,
5307                    uint32_t loreg,
5308                    bool offset_loaded,
5309                    uint64_t *offset,
5310                    uint64_t *stat)
5311 {
5312         uint64_t new_data;
5313
5314         new_data = (uint64_t)I40E_READ_REG(hw, loreg);
5315         new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
5316                         I40E_16_BIT_MASK)) << I40E_32_BIT_WIDTH;
5317
5318         if (!offset_loaded)
5319                 *offset = new_data;
5320
5321         if (new_data >= *offset)
5322                 *stat = new_data - *offset;
5323         else
5324                 *stat = (uint64_t)((new_data +
5325                         ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset);
5326
5327         *stat &= I40E_48_BIT_MASK;
5328 }
5329
5330 /* Disable IRQ0 */
5331 void
5332 i40e_pf_disable_irq0(struct i40e_hw *hw)
5333 {
5334         /* Disable all interrupt types */
5335         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
5336         I40E_WRITE_FLUSH(hw);
5337 }
5338
5339 /* Enable IRQ0 */
5340 void
5341 i40e_pf_enable_irq0(struct i40e_hw *hw)
5342 {
5343         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
5344                 I40E_PFINT_DYN_CTL0_INTENA_MASK |
5345                 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
5346                 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
5347         I40E_WRITE_FLUSH(hw);
5348 }
5349
5350 static void
5351 i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue)
5352 {
5353         /* read pending request and disable first */
5354         i40e_pf_disable_irq0(hw);
5355         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, I40E_PFINT_ICR0_ENA_MASK);
5356         I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
5357                 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
5358
5359         if (no_queue)
5360                 /* Link no queues with irq0 */
5361                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
5362                                I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
5363 }
5364
5365 static void
5366 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
5367 {
5368         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5369         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5370         int i;
5371         uint16_t abs_vf_id;
5372         uint32_t index, offset, val;
5373
5374         if (!pf->vfs)
5375                 return;
5376         /**
5377          * Try to find which VF trigger a reset, use absolute VF id to access
5378          * since the reg is global register.
5379          */
5380         for (i = 0; i < pf->vf_num; i++) {
5381                 abs_vf_id = hw->func_caps.vf_base_id + i;
5382                 index = abs_vf_id / I40E_UINT32_BIT_SIZE;
5383                 offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
5384                 val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
5385                 /* VFR event occured */
5386                 if (val & (0x1 << offset)) {
5387                         int ret;
5388
5389                         /* Clear the event first */
5390                         I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
5391                                                         (0x1 << offset));
5392                         PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id);
5393                         /**
5394                          * Only notify a VF reset event occured,
5395                          * don't trigger another SW reset
5396                          */
5397                         ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
5398                         if (ret != I40E_SUCCESS)
5399                                 PMD_DRV_LOG(ERR, "Failed to do VF reset");
5400                 }
5401         }
5402 }
5403
5404 static void
5405 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
5406 {
5407         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5408         struct i40e_arq_event_info info;
5409         uint16_t pending, opcode;
5410         int ret;
5411
5412         info.buf_len = I40E_AQ_BUF_SZ;
5413         info.msg_buf = rte_zmalloc("msg_buffer", info.buf_len, 0);
5414         if (!info.msg_buf) {
5415                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
5416                 return;
5417         }
5418
5419         pending = 1;
5420         while (pending) {
5421                 ret = i40e_clean_arq_element(hw, &info, &pending);
5422
5423                 if (ret != I40E_SUCCESS) {
5424                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
5425                                     "aq_err: %u", hw->aq.asq_last_status);
5426                         break;
5427                 }
5428                 opcode = rte_le_to_cpu_16(info.desc.opcode);
5429
5430                 switch (opcode) {
5431                 case i40e_aqc_opc_send_msg_to_pf:
5432                         /* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
5433                         i40e_pf_host_handle_vf_msg(dev,
5434                                         rte_le_to_cpu_16(info.desc.retval),
5435                                         rte_le_to_cpu_32(info.desc.cookie_high),
5436                                         rte_le_to_cpu_32(info.desc.cookie_low),
5437                                         info.msg_buf,
5438                                         info.msg_len);
5439                         break;
5440                 default:
5441                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
5442                                     opcode);
5443                         break;
5444                 }
5445         }
5446         rte_free(info.msg_buf);
5447 }
5448
5449 /*
5450  * Interrupt handler is registered as the alarm callback for handling LSC
5451  * interrupt in a definite of time, in order to wait the NIC into a stable
5452  * state. Currently it waits 1 sec in i40e for the link up interrupt, and
5453  * no need for link down interrupt.
5454  */
5455 static void
5456 i40e_dev_interrupt_delayed_handler(void *param)
5457 {
5458         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
5459         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5460         uint32_t icr0;
5461
5462         /* read interrupt causes again */
5463         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
5464
5465 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
5466         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
5467                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error\n");
5468         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
5469                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected\n");
5470         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
5471                 PMD_DRV_LOG(INFO, "ICR0: global reset requested\n");
5472         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
5473                 PMD_DRV_LOG(INFO, "ICR0: PCI exception\n activated\n");
5474         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
5475                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control "
5476                                                                 "state\n");
5477         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
5478                 PMD_DRV_LOG(ERR, "ICR0: HMC error\n");
5479         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
5480                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error\n");
5481 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
5482
5483         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
5484                 PMD_DRV_LOG(INFO, "INT:VF reset detected\n");
5485                 i40e_dev_handle_vfr_event(dev);
5486         }
5487         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
5488                 PMD_DRV_LOG(INFO, "INT:ADMINQ event\n");
5489                 i40e_dev_handle_aq_msg(dev);
5490         }
5491
5492         /* handle the link up interrupt in an alarm callback */
5493         i40e_dev_link_update(dev, 0);
5494         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
5495
5496         i40e_pf_enable_irq0(hw);
5497         rte_intr_enable(&(dev->pci_dev->intr_handle));
5498 }
5499
5500 /**
5501  * Interrupt handler triggered by NIC  for handling
5502  * specific interrupt.
5503  *
5504  * @param handle
5505  *  Pointer to interrupt handle.
5506  * @param param
5507  *  The address of parameter (struct rte_eth_dev *) regsitered before.
5508  *
5509  * @return
5510  *  void
5511  */
5512 static void
5513 i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
5514                            void *param)
5515 {
5516         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
5517         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5518         uint32_t icr0;
5519
5520         /* Disable interrupt */
5521         i40e_pf_disable_irq0(hw);
5522
5523         /* read out interrupt causes */
5524         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
5525
5526         /* No interrupt event indicated */
5527         if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK)) {
5528                 PMD_DRV_LOG(INFO, "No interrupt event");
5529                 goto done;
5530         }
5531 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
5532         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
5533                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
5534         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
5535                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
5536         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
5537                 PMD_DRV_LOG(INFO, "ICR0: global reset requested");
5538         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
5539                 PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
5540         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
5541                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
5542         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
5543                 PMD_DRV_LOG(ERR, "ICR0: HMC error");
5544         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
5545                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
5546 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
5547
5548         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
5549                 PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
5550                 i40e_dev_handle_vfr_event(dev);
5551         }
5552         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
5553                 PMD_DRV_LOG(INFO, "ICR0: adminq event");
5554                 i40e_dev_handle_aq_msg(dev);
5555         }
5556
5557         /* Link Status Change interrupt */
5558         if (icr0 & I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK) {
5559 #define I40E_US_PER_SECOND 1000000
5560                 struct rte_eth_link link;
5561
5562                 PMD_DRV_LOG(INFO, "ICR0: link status changed\n");
5563                 memset(&link, 0, sizeof(link));
5564                 rte_i40e_dev_atomic_read_link_status(dev, &link);
5565                 i40e_dev_link_update(dev, 0);
5566
5567                 /*
5568                  * For link up interrupt, it needs to wait 1 second to let the
5569                  * hardware be a stable state. Otherwise several consecutive
5570                  * interrupts can be observed.
5571                  * For link down interrupt, no need to wait.
5572                  */
5573                 if (!link.link_status && rte_eal_alarm_set(I40E_US_PER_SECOND,
5574                         i40e_dev_interrupt_delayed_handler, (void *)dev) >= 0)
5575                         return;
5576                 else
5577                         _rte_eth_dev_callback_process(dev,
5578                                 RTE_ETH_EVENT_INTR_LSC);
5579         }
5580
5581 done:
5582         /* Enable interrupt */
5583         i40e_pf_enable_irq0(hw);
5584         rte_intr_enable(&(dev->pci_dev->intr_handle));
5585 }
5586
5587 static int
5588 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
5589                          struct i40e_macvlan_filter *filter,
5590                          int total)
5591 {
5592         int ele_num, ele_buff_size;
5593         int num, actual_num, i;
5594         uint16_t flags;
5595         int ret = I40E_SUCCESS;
5596         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5597         struct i40e_aqc_add_macvlan_element_data *req_list;
5598
5599         if (filter == NULL  || total == 0)
5600                 return I40E_ERR_PARAM;
5601         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5602         ele_buff_size = hw->aq.asq_buf_size;
5603
5604         req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
5605         if (req_list == NULL) {
5606                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5607                 return I40E_ERR_NO_MEMORY;
5608         }
5609
5610         num = 0;
5611         do {
5612                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5613                 memset(req_list, 0, ele_buff_size);
5614
5615                 for (i = 0; i < actual_num; i++) {
5616                         (void)rte_memcpy(req_list[i].mac_addr,
5617                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5618                         req_list[i].vlan_tag =
5619                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5620
5621                         switch (filter[num + i].filter_type) {
5622                         case RTE_MAC_PERFECT_MATCH:
5623                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH |
5624                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5625                                 break;
5626                         case RTE_MACVLAN_PERFECT_MATCH:
5627                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
5628                                 break;
5629                         case RTE_MAC_HASH_MATCH:
5630                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH |
5631                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5632                                 break;
5633                         case RTE_MACVLAN_HASH_MATCH:
5634                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH;
5635                                 break;
5636                         default:
5637                                 PMD_DRV_LOG(ERR, "Invalid MAC match type\n");
5638                                 ret = I40E_ERR_PARAM;
5639                                 goto DONE;
5640                         }
5641
5642                         req_list[i].queue_number = 0;
5643
5644                         req_list[i].flags = rte_cpu_to_le_16(flags);
5645                 }
5646
5647                 ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
5648                                                 actual_num, NULL);
5649                 if (ret != I40E_SUCCESS) {
5650                         PMD_DRV_LOG(ERR, "Failed to add macvlan filter");
5651                         goto DONE;
5652                 }
5653                 num += actual_num;
5654         } while (num < total);
5655
5656 DONE:
5657         rte_free(req_list);
5658         return ret;
5659 }
5660
5661 static int
5662 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
5663                             struct i40e_macvlan_filter *filter,
5664                             int total)
5665 {
5666         int ele_num, ele_buff_size;
5667         int num, actual_num, i;
5668         uint16_t flags;
5669         int ret = I40E_SUCCESS;
5670         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5671         struct i40e_aqc_remove_macvlan_element_data *req_list;
5672
5673         if (filter == NULL  || total == 0)
5674                 return I40E_ERR_PARAM;
5675
5676         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5677         ele_buff_size = hw->aq.asq_buf_size;
5678
5679         req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
5680         if (req_list == NULL) {
5681                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5682                 return I40E_ERR_NO_MEMORY;
5683         }
5684
5685         num = 0;
5686         do {
5687                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5688                 memset(req_list, 0, ele_buff_size);
5689
5690                 for (i = 0; i < actual_num; i++) {
5691                         (void)rte_memcpy(req_list[i].mac_addr,
5692                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5693                         req_list[i].vlan_tag =
5694                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5695
5696                         switch (filter[num + i].filter_type) {
5697                         case RTE_MAC_PERFECT_MATCH:
5698                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
5699                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5700                                 break;
5701                         case RTE_MACVLAN_PERFECT_MATCH:
5702                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
5703                                 break;
5704                         case RTE_MAC_HASH_MATCH:
5705                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH |
5706                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5707                                 break;
5708                         case RTE_MACVLAN_HASH_MATCH:
5709                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH;
5710                                 break;
5711                         default:
5712                                 PMD_DRV_LOG(ERR, "Invalid MAC filter type\n");
5713                                 ret = I40E_ERR_PARAM;
5714                                 goto DONE;
5715                         }
5716                         req_list[i].flags = rte_cpu_to_le_16(flags);
5717                 }
5718
5719                 ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
5720                                                 actual_num, NULL);
5721                 if (ret != I40E_SUCCESS) {
5722                         PMD_DRV_LOG(ERR, "Failed to remove macvlan filter");
5723                         goto DONE;
5724                 }
5725                 num += actual_num;
5726         } while (num < total);
5727
5728 DONE:
5729         rte_free(req_list);
5730         return ret;
5731 }
5732
5733 /* Find out specific MAC filter */
5734 static struct i40e_mac_filter *
5735 i40e_find_mac_filter(struct i40e_vsi *vsi,
5736                          struct ether_addr *macaddr)
5737 {
5738         struct i40e_mac_filter *f;
5739
5740         TAILQ_FOREACH(f, &vsi->mac_list, next) {
5741                 if (is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
5742                         return f;
5743         }
5744
5745         return NULL;
5746 }
5747
5748 static bool
5749 i40e_find_vlan_filter(struct i40e_vsi *vsi,
5750                          uint16_t vlan_id)
5751 {
5752         uint32_t vid_idx, vid_bit;
5753
5754         if (vlan_id > ETH_VLAN_ID_MAX)
5755                 return 0;
5756
5757         vid_idx = I40E_VFTA_IDX(vlan_id);
5758         vid_bit = I40E_VFTA_BIT(vlan_id);
5759
5760         if (vsi->vfta[vid_idx] & vid_bit)
5761                 return 1;
5762         else
5763                 return 0;
5764 }
5765
5766 static void
5767 i40e_set_vlan_filter(struct i40e_vsi *vsi,
5768                          uint16_t vlan_id, bool on)
5769 {
5770         uint32_t vid_idx, vid_bit;
5771
5772         if (vlan_id > ETH_VLAN_ID_MAX)
5773                 return;
5774
5775         vid_idx = I40E_VFTA_IDX(vlan_id);
5776         vid_bit = I40E_VFTA_BIT(vlan_id);
5777
5778         if (on)
5779                 vsi->vfta[vid_idx] |= vid_bit;
5780         else
5781                 vsi->vfta[vid_idx] &= ~vid_bit;
5782 }
5783
5784 /**
5785  * Find all vlan options for specific mac addr,
5786  * return with actual vlan found.
5787  */
5788 static inline int
5789 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
5790                            struct i40e_macvlan_filter *mv_f,
5791                            int num, struct ether_addr *addr)
5792 {
5793         int i;
5794         uint32_t j, k;
5795
5796         /**
5797          * Not to use i40e_find_vlan_filter to decrease the loop time,
5798          * although the code looks complex.
5799           */
5800         if (num < vsi->vlan_num)
5801                 return I40E_ERR_PARAM;
5802
5803         i = 0;
5804         for (j = 0; j < I40E_VFTA_SIZE; j++) {
5805                 if (vsi->vfta[j]) {
5806                         for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
5807                                 if (vsi->vfta[j] & (1 << k)) {
5808                                         if (i > num - 1) {
5809                                                 PMD_DRV_LOG(ERR, "vlan number "
5810                                                             "not match");
5811                                                 return I40E_ERR_PARAM;
5812                                         }
5813                                         (void)rte_memcpy(&mv_f[i].macaddr,
5814                                                         addr, ETH_ADDR_LEN);
5815                                         mv_f[i].vlan_id =
5816                                                 j * I40E_UINT32_BIT_SIZE + k;
5817                                         i++;
5818                                 }
5819                         }
5820                 }
5821         }
5822         return I40E_SUCCESS;
5823 }
5824
5825 static inline int
5826 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
5827                            struct i40e_macvlan_filter *mv_f,
5828                            int num,
5829                            uint16_t vlan)
5830 {
5831         int i = 0;
5832         struct i40e_mac_filter *f;
5833
5834         if (num < vsi->mac_num)
5835                 return I40E_ERR_PARAM;
5836
5837         TAILQ_FOREACH(f, &vsi->mac_list, next) {
5838                 if (i > num - 1) {
5839                         PMD_DRV_LOG(ERR, "buffer number not match");
5840                         return I40E_ERR_PARAM;
5841                 }
5842                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
5843                                 ETH_ADDR_LEN);
5844                 mv_f[i].vlan_id = vlan;
5845                 mv_f[i].filter_type = f->mac_info.filter_type;
5846                 i++;
5847         }
5848
5849         return I40E_SUCCESS;
5850 }
5851
5852 static int
5853 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
5854 {
5855         int i, num;
5856         struct i40e_mac_filter *f;
5857         struct i40e_macvlan_filter *mv_f;
5858         int ret = I40E_SUCCESS;
5859
5860         if (vsi == NULL || vsi->mac_num == 0)
5861                 return I40E_ERR_PARAM;
5862
5863         /* Case that no vlan is set */
5864         if (vsi->vlan_num == 0)
5865                 num = vsi->mac_num;
5866         else
5867                 num = vsi->mac_num * vsi->vlan_num;
5868
5869         mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
5870         if (mv_f == NULL) {
5871                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5872                 return I40E_ERR_NO_MEMORY;
5873         }
5874
5875         i = 0;
5876         if (vsi->vlan_num == 0) {
5877                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
5878                         (void)rte_memcpy(&mv_f[i].macaddr,
5879                                 &f->mac_info.mac_addr, ETH_ADDR_LEN);
5880                         mv_f[i].vlan_id = 0;
5881                         i++;
5882                 }
5883         } else {
5884                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
5885                         ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
5886                                         vsi->vlan_num, &f->mac_info.mac_addr);
5887                         if (ret != I40E_SUCCESS)
5888                                 goto DONE;
5889                         i += vsi->vlan_num;
5890                 }
5891         }
5892
5893         ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
5894 DONE:
5895         rte_free(mv_f);
5896
5897         return ret;
5898 }
5899
5900 int
5901 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
5902 {
5903         struct i40e_macvlan_filter *mv_f;
5904         int mac_num;
5905         int ret = I40E_SUCCESS;
5906
5907         if (!vsi || vlan > ETHER_MAX_VLAN_ID)
5908                 return I40E_ERR_PARAM;
5909
5910         /* If it's already set, just return */
5911         if (i40e_find_vlan_filter(vsi,vlan))
5912                 return I40E_SUCCESS;
5913
5914         mac_num = vsi->mac_num;
5915
5916         if (mac_num == 0) {
5917                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
5918                 return I40E_ERR_PARAM;
5919         }
5920
5921         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
5922
5923         if (mv_f == NULL) {
5924                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5925                 return I40E_ERR_NO_MEMORY;
5926         }
5927
5928         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
5929
5930         if (ret != I40E_SUCCESS)
5931                 goto DONE;
5932
5933         ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
5934
5935         if (ret != I40E_SUCCESS)
5936                 goto DONE;
5937
5938         i40e_set_vlan_filter(vsi, vlan, 1);
5939
5940         vsi->vlan_num++;
5941         ret = I40E_SUCCESS;
5942 DONE:
5943         rte_free(mv_f);
5944         return ret;
5945 }
5946
5947 int
5948 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
5949 {
5950         struct i40e_macvlan_filter *mv_f;
5951         int mac_num;
5952         int ret = I40E_SUCCESS;
5953
5954         /**
5955          * Vlan 0 is the generic filter for untagged packets
5956          * and can't be removed.
5957          */
5958         if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
5959                 return I40E_ERR_PARAM;
5960
5961         /* If can't find it, just return */
5962         if (!i40e_find_vlan_filter(vsi, vlan))
5963                 return I40E_ERR_PARAM;
5964
5965         mac_num = vsi->mac_num;
5966
5967         if (mac_num == 0) {
5968                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
5969                 return I40E_ERR_PARAM;
5970         }
5971
5972         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
5973
5974         if (mv_f == NULL) {
5975                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5976                 return I40E_ERR_NO_MEMORY;
5977         }
5978
5979         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
5980
5981         if (ret != I40E_SUCCESS)
5982                 goto DONE;
5983
5984         ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
5985
5986         if (ret != I40E_SUCCESS)
5987                 goto DONE;
5988
5989         /* This is last vlan to remove, replace all mac filter with vlan 0 */
5990         if (vsi->vlan_num == 1) {
5991                 ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
5992                 if (ret != I40E_SUCCESS)
5993                         goto DONE;
5994
5995                 ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
5996                 if (ret != I40E_SUCCESS)
5997                         goto DONE;
5998         }
5999
6000         i40e_set_vlan_filter(vsi, vlan, 0);
6001
6002         vsi->vlan_num--;
6003         ret = I40E_SUCCESS;
6004 DONE:
6005         rte_free(mv_f);
6006         return ret;
6007 }
6008
6009 int
6010 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter)
6011 {
6012         struct i40e_mac_filter *f;
6013         struct i40e_macvlan_filter *mv_f;
6014         int i, vlan_num = 0;
6015         int ret = I40E_SUCCESS;
6016
6017         /* If it's add and we've config it, return */
6018         f = i40e_find_mac_filter(vsi, &mac_filter->mac_addr);
6019         if (f != NULL)
6020                 return I40E_SUCCESS;
6021         if ((mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH) ||
6022                 (mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH)) {
6023
6024                 /**
6025                  * If vlan_num is 0, that's the first time to add mac,
6026                  * set mask for vlan_id 0.
6027                  */
6028                 if (vsi->vlan_num == 0) {
6029                         i40e_set_vlan_filter(vsi, 0, 1);
6030                         vsi->vlan_num = 1;
6031                 }
6032                 vlan_num = vsi->vlan_num;
6033         } else if ((mac_filter->filter_type == RTE_MAC_PERFECT_MATCH) ||
6034                         (mac_filter->filter_type == RTE_MAC_HASH_MATCH))
6035                 vlan_num = 1;
6036
6037         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
6038         if (mv_f == NULL) {
6039                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6040                 return I40E_ERR_NO_MEMORY;
6041         }
6042
6043         for (i = 0; i < vlan_num; i++) {
6044                 mv_f[i].filter_type = mac_filter->filter_type;
6045                 (void)rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr,
6046                                 ETH_ADDR_LEN);
6047         }
6048
6049         if (mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6050                 mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH) {
6051                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num,
6052                                         &mac_filter->mac_addr);
6053                 if (ret != I40E_SUCCESS)
6054                         goto DONE;
6055         }
6056
6057         ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
6058         if (ret != I40E_SUCCESS)
6059                 goto DONE;
6060
6061         /* Add the mac addr into mac list */
6062         f = rte_zmalloc("macv_filter", sizeof(*f), 0);
6063         if (f == NULL) {
6064                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6065                 ret = I40E_ERR_NO_MEMORY;
6066                 goto DONE;
6067         }
6068         (void)rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr,
6069                         ETH_ADDR_LEN);
6070         f->mac_info.filter_type = mac_filter->filter_type;
6071         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
6072         vsi->mac_num++;
6073
6074         ret = I40E_SUCCESS;
6075 DONE:
6076         rte_free(mv_f);
6077
6078         return ret;
6079 }
6080
6081 int
6082 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
6083 {
6084         struct i40e_mac_filter *f;
6085         struct i40e_macvlan_filter *mv_f;
6086         int i, vlan_num;
6087         enum rte_mac_filter_type filter_type;
6088         int ret = I40E_SUCCESS;
6089
6090         /* Can't find it, return an error */
6091         f = i40e_find_mac_filter(vsi, addr);
6092         if (f == NULL)
6093                 return I40E_ERR_PARAM;
6094
6095         vlan_num = vsi->vlan_num;
6096         filter_type = f->mac_info.filter_type;
6097         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6098                 filter_type == RTE_MACVLAN_HASH_MATCH) {
6099                 if (vlan_num == 0) {
6100                         PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0\n");
6101                         return I40E_ERR_PARAM;
6102                 }
6103         } else if (filter_type == RTE_MAC_PERFECT_MATCH ||
6104                         filter_type == RTE_MAC_HASH_MATCH)
6105                 vlan_num = 1;
6106
6107         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
6108         if (mv_f == NULL) {
6109                 PMD_DRV_LOG(ERR, "failed to allocate memory");
6110                 return I40E_ERR_NO_MEMORY;
6111         }
6112
6113         for (i = 0; i < vlan_num; i++) {
6114                 mv_f[i].filter_type = filter_type;
6115                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
6116                                 ETH_ADDR_LEN);
6117         }
6118         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
6119                         filter_type == RTE_MACVLAN_HASH_MATCH) {
6120                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
6121                 if (ret != I40E_SUCCESS)
6122                         goto DONE;
6123         }
6124
6125         ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
6126         if (ret != I40E_SUCCESS)
6127                 goto DONE;
6128
6129         /* Remove the mac addr into mac list */
6130         TAILQ_REMOVE(&vsi->mac_list, f, next);
6131         rte_free(f);
6132         vsi->mac_num--;
6133
6134         ret = I40E_SUCCESS;
6135 DONE:
6136         rte_free(mv_f);
6137         return ret;
6138 }
6139
6140 /* Configure hash enable flags for RSS */
6141 uint64_t
6142 i40e_config_hena(uint64_t flags)
6143 {
6144         uint64_t hena = 0;
6145
6146         if (!flags)
6147                 return hena;
6148
6149         if (flags & ETH_RSS_FRAG_IPV4)
6150                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4;
6151         if (flags & ETH_RSS_NONFRAG_IPV4_TCP)
6152 #ifdef X722_SUPPORT
6153                 hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
6154                         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
6155 #else
6156                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
6157 #endif
6158         if (flags & ETH_RSS_NONFRAG_IPV4_UDP)
6159 #ifdef X722_SUPPORT
6160                 hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
6161                         (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
6162                         (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
6163 #else
6164                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
6165 #endif
6166         if (flags & ETH_RSS_NONFRAG_IPV4_SCTP)
6167                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP;
6168         if (flags & ETH_RSS_NONFRAG_IPV4_OTHER)
6169                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
6170         if (flags & ETH_RSS_FRAG_IPV6)
6171                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6;
6172         if (flags & ETH_RSS_NONFRAG_IPV6_TCP)
6173 #ifdef X722_SUPPORT
6174                 hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
6175                         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
6176 #else
6177                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
6178 #endif
6179         if (flags & ETH_RSS_NONFRAG_IPV6_UDP)
6180 #ifdef X722_SUPPORT
6181                 hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
6182                         (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
6183                         (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
6184 #else
6185                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
6186 #endif
6187         if (flags & ETH_RSS_NONFRAG_IPV6_SCTP)
6188                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP;
6189         if (flags & ETH_RSS_NONFRAG_IPV6_OTHER)
6190                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
6191         if (flags & ETH_RSS_L2_PAYLOAD)
6192                 hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD;
6193
6194         return hena;
6195 }
6196
6197 /* Parse the hash enable flags */
6198 uint64_t
6199 i40e_parse_hena(uint64_t flags)
6200 {
6201         uint64_t rss_hf = 0;
6202
6203         if (!flags)
6204                 return rss_hf;
6205         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4))
6206                 rss_hf |= ETH_RSS_FRAG_IPV4;
6207         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP))
6208                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
6209 #ifdef X722_SUPPORT
6210         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK))
6211                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
6212 #endif
6213         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP))
6214                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
6215 #ifdef X722_SUPPORT
6216         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP))
6217                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
6218         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP))
6219                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
6220 #endif
6221         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP))
6222                 rss_hf |= ETH_RSS_NONFRAG_IPV4_SCTP;
6223         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER))
6224                 rss_hf |= ETH_RSS_NONFRAG_IPV4_OTHER;
6225         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6))
6226                 rss_hf |= ETH_RSS_FRAG_IPV6;
6227         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP))
6228                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
6229 #ifdef X722_SUPPORT
6230         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK))
6231                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
6232 #endif
6233         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP))
6234                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
6235 #ifdef X722_SUPPORT
6236         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP))
6237                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
6238         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP))
6239                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
6240 #endif
6241         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP))
6242                 rss_hf |= ETH_RSS_NONFRAG_IPV6_SCTP;
6243         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER))
6244                 rss_hf |= ETH_RSS_NONFRAG_IPV6_OTHER;
6245         if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
6246                 rss_hf |= ETH_RSS_L2_PAYLOAD;
6247
6248         return rss_hf;
6249 }
6250
6251 /* Disable RSS */
6252 static void
6253 i40e_pf_disable_rss(struct i40e_pf *pf)
6254 {
6255         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6256         uint64_t hena;
6257
6258         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6259         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6260         hena &= ~I40E_RSS_HENA_ALL;
6261         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
6262         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
6263         I40E_WRITE_FLUSH(hw);
6264 }
6265
6266 static int
6267 i40e_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
6268 {
6269         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
6270         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6271         int ret = 0;
6272
6273         if (!key || key_len == 0) {
6274                 PMD_DRV_LOG(DEBUG, "No key to be configured");
6275                 return 0;
6276         } else if (key_len != (I40E_PFQF_HKEY_MAX_INDEX + 1) *
6277                 sizeof(uint32_t)) {
6278                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
6279                 return -EINVAL;
6280         }
6281
6282         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
6283                 struct i40e_aqc_get_set_rss_key_data *key_dw =
6284                         (struct i40e_aqc_get_set_rss_key_data *)key;
6285
6286                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
6287                 if (ret)
6288                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
6289                                      "via AQ");
6290         } else {
6291                 uint32_t *hash_key = (uint32_t *)key;
6292                 uint16_t i;
6293
6294                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6295                         i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), hash_key[i]);
6296                 I40E_WRITE_FLUSH(hw);
6297         }
6298
6299         return ret;
6300 }
6301
6302 static int
6303 i40e_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
6304 {
6305         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
6306         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
6307         int ret;
6308
6309         if (!key || !key_len)
6310                 return -EINVAL;
6311
6312         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
6313                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
6314                         (struct i40e_aqc_get_set_rss_key_data *)key);
6315                 if (ret) {
6316                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
6317                         return ret;
6318                 }
6319         } else {
6320                 uint32_t *key_dw = (uint32_t *)key;
6321                 uint16_t i;
6322
6323                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
6324                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
6325         }
6326         *key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
6327
6328         return 0;
6329 }
6330
6331 static int
6332 i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf)
6333 {
6334         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6335         uint64_t rss_hf;
6336         uint64_t hena;
6337         int ret;
6338
6339         ret = i40e_set_rss_key(pf->main_vsi, rss_conf->rss_key,
6340                                rss_conf->rss_key_len);
6341         if (ret)
6342                 return ret;
6343
6344         rss_hf = rss_conf->rss_hf;
6345         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6346         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6347         hena &= ~I40E_RSS_HENA_ALL;
6348         hena |= i40e_config_hena(rss_hf);
6349         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
6350         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
6351         I40E_WRITE_FLUSH(hw);
6352
6353         return 0;
6354 }
6355
6356 static int
6357 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
6358                          struct rte_eth_rss_conf *rss_conf)
6359 {
6360         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6361         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6362         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
6363         uint64_t hena;
6364
6365         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6366         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6367         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
6368                 if (rss_hf != 0) /* Enable RSS */
6369                         return -EINVAL;
6370                 return 0; /* Nothing to do */
6371         }
6372         /* RSS enabled */
6373         if (rss_hf == 0) /* Disable RSS */
6374                 return -EINVAL;
6375
6376         return i40e_hw_rss_hash_set(pf, rss_conf);
6377 }
6378
6379 static int
6380 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
6381                            struct rte_eth_rss_conf *rss_conf)
6382 {
6383         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6384         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6385         uint64_t hena;
6386
6387         i40e_get_rss_key(pf->main_vsi, rss_conf->rss_key,
6388                          &rss_conf->rss_key_len);
6389
6390         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
6391         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
6392         rss_conf->rss_hf = i40e_parse_hena(hena);
6393
6394         return 0;
6395 }
6396
6397 static int
6398 i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag)
6399 {
6400         switch (filter_type) {
6401         case RTE_TUNNEL_FILTER_IMAC_IVLAN:
6402                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN;
6403                 break;
6404         case RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID:
6405                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID;
6406                 break;
6407         case RTE_TUNNEL_FILTER_IMAC_TENID:
6408                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID;
6409                 break;
6410         case RTE_TUNNEL_FILTER_OMAC_TENID_IMAC:
6411                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC;
6412                 break;
6413         case ETH_TUNNEL_FILTER_IMAC:
6414                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC;
6415                 break;
6416         case ETH_TUNNEL_FILTER_OIP:
6417                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OIP;
6418                 break;
6419         case ETH_TUNNEL_FILTER_IIP:
6420                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IIP;
6421                 break;
6422         default:
6423                 PMD_DRV_LOG(ERR, "invalid tunnel filter type");
6424                 return -EINVAL;
6425         }
6426
6427         return 0;
6428 }
6429
6430 static int
6431 i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
6432                         struct rte_eth_tunnel_filter_conf *tunnel_filter,
6433                         uint8_t add)
6434 {
6435         uint16_t ip_type;
6436         uint32_t ipv4_addr;
6437         uint8_t i, tun_type = 0;
6438         /* internal varialbe to convert ipv6 byte order */
6439         uint32_t convert_ipv6[4];
6440         int val, ret = 0;
6441         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6442         struct i40e_vsi *vsi = pf->main_vsi;
6443         struct i40e_aqc_add_remove_cloud_filters_element_data  *cld_filter;
6444         struct i40e_aqc_add_remove_cloud_filters_element_data  *pfilter;
6445
6446         cld_filter = rte_zmalloc("tunnel_filter",
6447                 sizeof(struct i40e_aqc_add_remove_cloud_filters_element_data),
6448                 0);
6449
6450         if (NULL == cld_filter) {
6451                 PMD_DRV_LOG(ERR, "Failed to alloc memory.");
6452                 return -EINVAL;
6453         }
6454         pfilter = cld_filter;
6455
6456         ether_addr_copy(&tunnel_filter->outer_mac, (struct ether_addr*)&pfilter->outer_mac);
6457         ether_addr_copy(&tunnel_filter->inner_mac, (struct ether_addr*)&pfilter->inner_mac);
6458
6459         pfilter->inner_vlan = rte_cpu_to_le_16(tunnel_filter->inner_vlan);
6460         if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) {
6461                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4;
6462                 ipv4_addr = rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv4_addr);
6463                 rte_memcpy(&pfilter->ipaddr.v4.data,
6464                                 &rte_cpu_to_le_32(ipv4_addr),
6465                                 sizeof(pfilter->ipaddr.v4.data));
6466         } else {
6467                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6;
6468                 for (i = 0; i < 4; i++) {
6469                         convert_ipv6[i] =
6470                         rte_cpu_to_le_32(rte_be_to_cpu_32(tunnel_filter->ip_addr.ipv6_addr[i]));
6471                 }
6472                 rte_memcpy(&pfilter->ipaddr.v6.data, &convert_ipv6,
6473                                 sizeof(pfilter->ipaddr.v6.data));
6474         }
6475
6476         /* check tunneled type */
6477         switch (tunnel_filter->tunnel_type) {
6478         case RTE_TUNNEL_TYPE_VXLAN:
6479                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN;
6480                 break;
6481         case RTE_TUNNEL_TYPE_NVGRE:
6482                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC;
6483                 break;
6484         case RTE_TUNNEL_TYPE_IP_IN_GRE:
6485                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP;
6486                 break;
6487         default:
6488                 /* Other tunnel types is not supported. */
6489                 PMD_DRV_LOG(ERR, "tunnel type is not supported.");
6490                 rte_free(cld_filter);
6491                 return -EINVAL;
6492         }
6493
6494         val = i40e_dev_get_filter_type(tunnel_filter->filter_type,
6495                                                 &pfilter->flags);
6496         if (val < 0) {
6497                 rte_free(cld_filter);
6498                 return -EINVAL;
6499         }
6500
6501         pfilter->flags |= rte_cpu_to_le_16(
6502                 I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE |
6503                 ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT));
6504         pfilter->tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id);
6505         pfilter->queue_number = rte_cpu_to_le_16(tunnel_filter->queue_id);
6506
6507         if (add)
6508                 ret = i40e_aq_add_cloud_filters(hw, vsi->seid, cld_filter, 1);
6509         else
6510                 ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
6511                                                 cld_filter, 1);
6512
6513         rte_free(cld_filter);
6514         return ret;
6515 }
6516
6517 static int
6518 i40e_get_vxlan_port_idx(struct i40e_pf *pf, uint16_t port)
6519 {
6520         uint8_t i;
6521
6522         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
6523                 if (pf->vxlan_ports[i] == port)
6524                         return i;
6525         }
6526
6527         return -1;
6528 }
6529
6530 static int
6531 i40e_add_vxlan_port(struct i40e_pf *pf, uint16_t port)
6532 {
6533         int  idx, ret;
6534         uint8_t filter_idx;
6535         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6536
6537         idx = i40e_get_vxlan_port_idx(pf, port);
6538
6539         /* Check if port already exists */
6540         if (idx >= 0) {
6541                 PMD_DRV_LOG(ERR, "Port %d already offloaded", port);
6542                 return -EINVAL;
6543         }
6544
6545         /* Now check if there is space to add the new port */
6546         idx = i40e_get_vxlan_port_idx(pf, 0);
6547         if (idx < 0) {
6548                 PMD_DRV_LOG(ERR, "Maximum number of UDP ports reached,"
6549                         "not adding port %d", port);
6550                 return -ENOSPC;
6551         }
6552
6553         ret =  i40e_aq_add_udp_tunnel(hw, port, I40E_AQC_TUNNEL_TYPE_VXLAN,
6554                                         &filter_idx, NULL);
6555         if (ret < 0) {
6556                 PMD_DRV_LOG(ERR, "Failed to add VXLAN UDP port %d", port);
6557                 return -1;
6558         }
6559
6560         PMD_DRV_LOG(INFO, "Added port %d with AQ command with index %d",
6561                          port,  filter_idx);
6562
6563         /* New port: add it and mark its index in the bitmap */
6564         pf->vxlan_ports[idx] = port;
6565         pf->vxlan_bitmap |= (1 << idx);
6566
6567         if (!(pf->flags & I40E_FLAG_VXLAN))
6568                 pf->flags |= I40E_FLAG_VXLAN;
6569
6570         return 0;
6571 }
6572
6573 static int
6574 i40e_del_vxlan_port(struct i40e_pf *pf, uint16_t port)
6575 {
6576         int idx;
6577         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6578
6579         if (!(pf->flags & I40E_FLAG_VXLAN)) {
6580                 PMD_DRV_LOG(ERR, "VXLAN UDP port was not configured.");
6581                 return -EINVAL;
6582         }
6583
6584         idx = i40e_get_vxlan_port_idx(pf, port);
6585
6586         if (idx < 0) {
6587                 PMD_DRV_LOG(ERR, "Port %d doesn't exist", port);
6588                 return -EINVAL;
6589         }
6590
6591         if (i40e_aq_del_udp_tunnel(hw, idx, NULL) < 0) {
6592                 PMD_DRV_LOG(ERR, "Failed to delete VXLAN UDP port %d", port);
6593                 return -1;
6594         }
6595
6596         PMD_DRV_LOG(INFO, "Deleted port %d with AQ command with index %d",
6597                         port, idx);
6598
6599         pf->vxlan_ports[idx] = 0;
6600         pf->vxlan_bitmap &= ~(1 << idx);
6601
6602         if (!pf->vxlan_bitmap)
6603                 pf->flags &= ~I40E_FLAG_VXLAN;
6604
6605         return 0;
6606 }
6607
6608 /* Add UDP tunneling port */
6609 static int
6610 i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
6611                              struct rte_eth_udp_tunnel *udp_tunnel)
6612 {
6613         int ret = 0;
6614         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6615
6616         if (udp_tunnel == NULL)
6617                 return -EINVAL;
6618
6619         switch (udp_tunnel->prot_type) {
6620         case RTE_TUNNEL_TYPE_VXLAN:
6621                 ret = i40e_add_vxlan_port(pf, udp_tunnel->udp_port);
6622                 break;
6623
6624         case RTE_TUNNEL_TYPE_GENEVE:
6625         case RTE_TUNNEL_TYPE_TEREDO:
6626                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6627                 ret = -1;
6628                 break;
6629
6630         default:
6631                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6632                 ret = -1;
6633                 break;
6634         }
6635
6636         return ret;
6637 }
6638
6639 /* Remove UDP tunneling port */
6640 static int
6641 i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
6642                              struct rte_eth_udp_tunnel *udp_tunnel)
6643 {
6644         int ret = 0;
6645         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6646
6647         if (udp_tunnel == NULL)
6648                 return -EINVAL;
6649
6650         switch (udp_tunnel->prot_type) {
6651         case RTE_TUNNEL_TYPE_VXLAN:
6652                 ret = i40e_del_vxlan_port(pf, udp_tunnel->udp_port);
6653                 break;
6654         case RTE_TUNNEL_TYPE_GENEVE:
6655         case RTE_TUNNEL_TYPE_TEREDO:
6656                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6657                 ret = -1;
6658                 break;
6659         default:
6660                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6661                 ret = -1;
6662                 break;
6663         }
6664
6665         return ret;
6666 }
6667
6668 /* Calculate the maximum number of contiguous PF queues that are configured */
6669 static int
6670 i40e_pf_calc_configured_queues_num(struct i40e_pf *pf)
6671 {
6672         struct rte_eth_dev_data *data = pf->dev_data;
6673         int i, num;
6674         struct i40e_rx_queue *rxq;
6675
6676         num = 0;
6677         for (i = 0; i < pf->lan_nb_qps; i++) {
6678                 rxq = data->rx_queues[i];
6679                 if (rxq && rxq->q_set)
6680                         num++;
6681                 else
6682                         break;
6683         }
6684
6685         return num;
6686 }
6687
6688 /* Configure RSS */
6689 static int
6690 i40e_pf_config_rss(struct i40e_pf *pf)
6691 {
6692         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6693         struct rte_eth_rss_conf rss_conf;
6694         uint32_t i, lut = 0;
6695         uint16_t j, num;
6696
6697         /*
6698          * If both VMDQ and RSS enabled, not all of PF queues are configured.
6699          * It's necessary to calulate the actual PF queues that are configured.
6700          */
6701         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
6702                 num = i40e_pf_calc_configured_queues_num(pf);
6703         else
6704                 num = pf->dev_data->nb_rx_queues;
6705
6706         num = RTE_MIN(num, I40E_MAX_Q_PER_TC);
6707         PMD_INIT_LOG(INFO, "Max of contiguous %u PF queues are configured",
6708                         num);
6709
6710         if (num == 0) {
6711                 PMD_INIT_LOG(ERR, "No PF queues are configured to enable RSS");
6712                 return -ENOTSUP;
6713         }
6714
6715         for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
6716                 if (j == num)
6717                         j = 0;
6718                 lut = (lut << 8) | (j & ((0x1 <<
6719                         hw->func_caps.rss_table_entry_width) - 1));
6720                 if ((i & 3) == 3)
6721                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
6722         }
6723
6724         rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
6725         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
6726                 i40e_pf_disable_rss(pf);
6727                 return 0;
6728         }
6729         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
6730                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
6731                 /* Random default keys */
6732                 static uint32_t rss_key_default[] = {0x6b793944,
6733                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
6734                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
6735                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
6736
6737                 rss_conf.rss_key = (uint8_t *)rss_key_default;
6738                 rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
6739                                                         sizeof(uint32_t);
6740         }
6741
6742         return i40e_hw_rss_hash_set(pf, &rss_conf);
6743 }
6744
6745 static int
6746 i40e_tunnel_filter_param_check(struct i40e_pf *pf,
6747                                struct rte_eth_tunnel_filter_conf *filter)
6748 {
6749         if (pf == NULL || filter == NULL) {
6750                 PMD_DRV_LOG(ERR, "Invalid parameter");
6751                 return -EINVAL;
6752         }
6753
6754         if (filter->queue_id >= pf->dev_data->nb_rx_queues) {
6755                 PMD_DRV_LOG(ERR, "Invalid queue ID");
6756                 return -EINVAL;
6757         }
6758
6759         if (filter->inner_vlan > ETHER_MAX_VLAN_ID) {
6760                 PMD_DRV_LOG(ERR, "Invalid inner VLAN ID");
6761                 return -EINVAL;
6762         }
6763
6764         if ((filter->filter_type & ETH_TUNNEL_FILTER_OMAC) &&
6765                 (is_zero_ether_addr(&filter->outer_mac))) {
6766                 PMD_DRV_LOG(ERR, "Cannot add NULL outer MAC address");
6767                 return -EINVAL;
6768         }
6769
6770         if ((filter->filter_type & ETH_TUNNEL_FILTER_IMAC) &&
6771                 (is_zero_ether_addr(&filter->inner_mac))) {
6772                 PMD_DRV_LOG(ERR, "Cannot add NULL inner MAC address");
6773                 return -EINVAL;
6774         }
6775
6776         return 0;
6777 }
6778
6779 #define I40E_GL_PRS_FVBM_MSK_ENA 0x80000000
6780 #define I40E_GL_PRS_FVBM(_i)     (0x00269760 + ((_i) * 4))
6781 static int
6782 i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len)
6783 {
6784         uint32_t val, reg;
6785         int ret = -EINVAL;
6786
6787         val = I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2));
6788         PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x\n", val);
6789
6790         if (len == 3) {
6791                 reg = val | I40E_GL_PRS_FVBM_MSK_ENA;
6792         } else if (len == 4) {
6793                 reg = val & ~I40E_GL_PRS_FVBM_MSK_ENA;
6794         } else {
6795                 PMD_DRV_LOG(ERR, "Unsupported GRE key length of %u", len);
6796                 return ret;
6797         }
6798
6799         if (reg != val) {
6800                 ret = i40e_aq_debug_write_register(hw, I40E_GL_PRS_FVBM(2),
6801                                                    reg, NULL);
6802                 if (ret != 0)
6803                         return ret;
6804         } else {
6805                 ret = 0;
6806         }
6807         PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x\n",
6808                     I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2)));
6809
6810         return ret;
6811 }
6812
6813 static int
6814 i40e_dev_global_config_set(struct i40e_hw *hw, struct rte_eth_global_cfg *cfg)
6815 {
6816         int ret = -EINVAL;
6817
6818         if (!hw || !cfg)
6819                 return -EINVAL;
6820
6821         switch (cfg->cfg_type) {
6822         case RTE_ETH_GLOBAL_CFG_TYPE_GRE_KEY_LEN:
6823                 ret = i40e_dev_set_gre_key_len(hw, cfg->cfg.gre_key_len);
6824                 break;
6825         default:
6826                 PMD_DRV_LOG(ERR, "Unknown config type %u", cfg->cfg_type);
6827                 break;
6828         }
6829
6830         return ret;
6831 }
6832
6833 static int
6834 i40e_filter_ctrl_global_config(struct rte_eth_dev *dev,
6835                                enum rte_filter_op filter_op,
6836                                void *arg)
6837 {
6838         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6839         int ret = I40E_ERR_PARAM;
6840
6841         switch (filter_op) {
6842         case RTE_ETH_FILTER_SET:
6843                 ret = i40e_dev_global_config_set(hw,
6844                         (struct rte_eth_global_cfg *)arg);
6845                 break;
6846         default:
6847                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
6848                 break;
6849         }
6850
6851         return ret;
6852 }
6853
6854 static int
6855 i40e_tunnel_filter_handle(struct rte_eth_dev *dev,
6856                           enum rte_filter_op filter_op,
6857                           void *arg)
6858 {
6859         struct rte_eth_tunnel_filter_conf *filter;
6860         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6861         int ret = I40E_SUCCESS;
6862
6863         filter = (struct rte_eth_tunnel_filter_conf *)(arg);
6864
6865         if (i40e_tunnel_filter_param_check(pf, filter) < 0)
6866                 return I40E_ERR_PARAM;
6867
6868         switch (filter_op) {
6869         case RTE_ETH_FILTER_NOP:
6870                 if (!(pf->flags & I40E_FLAG_VXLAN))
6871                         ret = I40E_NOT_SUPPORTED;
6872                 break;
6873         case RTE_ETH_FILTER_ADD:
6874                 ret = i40e_dev_tunnel_filter_set(pf, filter, 1);
6875                 break;
6876         case RTE_ETH_FILTER_DELETE:
6877                 ret = i40e_dev_tunnel_filter_set(pf, filter, 0);
6878                 break;
6879         default:
6880                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
6881                 ret = I40E_ERR_PARAM;
6882                 break;
6883         }
6884
6885         return ret;
6886 }
6887
6888 static int
6889 i40e_pf_config_mq_rx(struct i40e_pf *pf)
6890 {
6891         int ret = 0;
6892         enum rte_eth_rx_mq_mode mq_mode = pf->dev_data->dev_conf.rxmode.mq_mode;
6893
6894         /* RSS setup */
6895         if (mq_mode & ETH_MQ_RX_RSS_FLAG)
6896                 ret = i40e_pf_config_rss(pf);
6897         else
6898                 i40e_pf_disable_rss(pf);
6899
6900         return ret;
6901 }
6902
6903 /* Get the symmetric hash enable configurations per port */
6904 static void
6905 i40e_get_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t *enable)
6906 {
6907         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
6908
6909         *enable = reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK ? 1 : 0;
6910 }
6911
6912 /* Set the symmetric hash enable configurations per port */
6913 static void
6914 i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable)
6915 {
6916         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
6917
6918         if (enable > 0) {
6919                 if (reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK) {
6920                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
6921                                                         "been enabled");
6922                         return;
6923                 }
6924                 reg |= I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
6925         } else {
6926                 if (!(reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK)) {
6927                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
6928                                                         "been disabled");
6929                         return;
6930                 }
6931                 reg &= ~I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
6932         }
6933         i40e_write_rx_ctl(hw, I40E_PRTQF_CTL_0, reg);
6934         I40E_WRITE_FLUSH(hw);
6935 }
6936
6937 /*
6938  * Get global configurations of hash function type and symmetric hash enable
6939  * per flow type (pctype). Note that global configuration means it affects all
6940  * the ports on the same NIC.
6941  */
6942 static int
6943 i40e_get_hash_filter_global_config(struct i40e_hw *hw,
6944                                    struct rte_eth_hash_global_conf *g_cfg)
6945 {
6946         uint32_t reg, mask = I40E_FLOW_TYPES;
6947         uint16_t i;
6948         enum i40e_filter_pctype pctype;
6949
6950         memset(g_cfg, 0, sizeof(*g_cfg));
6951         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
6952         if (reg & I40E_GLQF_CTL_HTOEP_MASK)
6953                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ;
6954         else
6955                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
6956         PMD_DRV_LOG(DEBUG, "Hash function is %s",
6957                 (reg & I40E_GLQF_CTL_HTOEP_MASK) ? "Toeplitz" : "Simple XOR");
6958
6959         for (i = 0; mask && i < RTE_ETH_FLOW_MAX; i++) {
6960                 if (!(mask & (1UL << i)))
6961                         continue;
6962                 mask &= ~(1UL << i);
6963                 /* Bit set indicats the coresponding flow type is supported */
6964                 g_cfg->valid_bit_mask[0] |= (1UL << i);
6965                 /* if flowtype is invalid, continue */
6966                 if (!I40E_VALID_FLOW(i))
6967                         continue;
6968                 pctype = i40e_flowtype_to_pctype(i);
6969                 reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
6970                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
6971                         g_cfg->sym_hash_enable_mask[0] |= (1UL << i);
6972         }
6973
6974         return 0;
6975 }
6976
6977 static int
6978 i40e_hash_global_config_check(struct rte_eth_hash_global_conf *g_cfg)
6979 {
6980         uint32_t i;
6981         uint32_t mask0, i40e_mask = I40E_FLOW_TYPES;
6982
6983         if (g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_TOEPLITZ &&
6984                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR &&
6985                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
6986                 PMD_DRV_LOG(ERR, "Unsupported hash function type %d",
6987                                                 g_cfg->hash_func);
6988                 return -EINVAL;
6989         }
6990
6991         /*
6992          * As i40e supports less than 32 flow types, only first 32 bits need to
6993          * be checked.
6994          */
6995         mask0 = g_cfg->valid_bit_mask[0];
6996         for (i = 0; i < RTE_SYM_HASH_MASK_ARRAY_SIZE; i++) {
6997                 if (i == 0) {
6998                         /* Check if any unsupported flow type configured */
6999                         if ((mask0 | i40e_mask) ^ i40e_mask)
7000                                 goto mask_err;
7001                 } else {
7002                         if (g_cfg->valid_bit_mask[i])
7003                                 goto mask_err;
7004                 }
7005         }
7006
7007         return 0;
7008
7009 mask_err:
7010         PMD_DRV_LOG(ERR, "i40e unsupported flow type bit(s) configured");
7011
7012         return -EINVAL;
7013 }
7014
7015 /*
7016  * Set global configurations of hash function type and symmetric hash enable
7017  * per flow type (pctype). Note any modifying global configuration will affect
7018  * all the ports on the same NIC.
7019  */
7020 static int
7021 i40e_set_hash_filter_global_config(struct i40e_hw *hw,
7022                                    struct rte_eth_hash_global_conf *g_cfg)
7023 {
7024         int ret;
7025         uint16_t i;
7026         uint32_t reg;
7027         uint32_t mask0 = g_cfg->valid_bit_mask[0];
7028         enum i40e_filter_pctype pctype;
7029
7030         /* Check the input parameters */
7031         ret = i40e_hash_global_config_check(g_cfg);
7032         if (ret < 0)
7033                 return ret;
7034
7035         for (i = 0; mask0 && i < UINT32_BIT; i++) {
7036                 if (!(mask0 & (1UL << i)))
7037                         continue;
7038                 mask0 &= ~(1UL << i);
7039                 /* if flowtype is invalid, continue */
7040                 if (!I40E_VALID_FLOW(i))
7041                         continue;
7042                 pctype = i40e_flowtype_to_pctype(i);
7043                 reg = (g_cfg->sym_hash_enable_mask[0] & (1UL << i)) ?
7044                                 I40E_GLQF_HSYM_SYMH_ENA_MASK : 0;
7045                 i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
7046         }
7047
7048         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
7049         if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
7050                 /* Toeplitz */
7051                 if (reg & I40E_GLQF_CTL_HTOEP_MASK) {
7052                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
7053                                                                 "Toeplitz");
7054                         goto out;
7055                 }
7056                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
7057         } else if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
7058                 /* Simple XOR */
7059                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK)) {
7060                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
7061                                                         "Simple XOR");
7062                         goto out;
7063                 }
7064                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
7065         } else
7066                 /* Use the default, and keep it as it is */
7067                 goto out;
7068
7069         i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
7070
7071 out:
7072         I40E_WRITE_FLUSH(hw);
7073
7074         return 0;
7075 }
7076
7077 /**
7078  * Valid input sets for hash and flow director filters per PCTYPE
7079  */
7080 static uint64_t
7081 i40e_get_valid_input_set(enum i40e_filter_pctype pctype,
7082                 enum rte_filter_type filter)
7083 {
7084         uint64_t valid;
7085
7086         static const uint64_t valid_hash_inset_table[] = {
7087                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7088                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7089                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7090                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_SRC |
7091                         I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS |
7092                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7093                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7094                         I40E_INSET_FLEX_PAYLOAD,
7095                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7096                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7097                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7098                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7099                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7100                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7101                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7102                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7103                         I40E_INSET_FLEX_PAYLOAD,
7104 #ifdef X722_SUPPORT
7105                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
7106                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7107                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7108                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7109                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7110                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7111                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7112                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7113                         I40E_INSET_FLEX_PAYLOAD,
7114                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
7115                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7116                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7117                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7118                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7119                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7120                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7121                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7122                         I40E_INSET_FLEX_PAYLOAD,
7123 #endif
7124                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7125                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7126                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7127                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7128                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7129                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7130                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7131                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7132                         I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
7133 #ifdef X722_SUPPORT
7134                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
7135                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7136                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7137                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7138                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7139                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7140                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7141                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7142                         I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
7143 #endif
7144                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7145                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7146                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7147                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7148                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7149                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7150                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7151                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7152                         I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD,
7153                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7154                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7155                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7156                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
7157                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
7158                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
7159                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7160                         I40E_INSET_FLEX_PAYLOAD,
7161                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7162                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7163                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7164                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7165                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7166                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_TUNNEL_DMAC |
7167                         I40E_INSET_TUNNEL_ID | I40E_INSET_IPV6_SRC |
7168                         I40E_INSET_IPV6_DST | I40E_INSET_FLEX_PAYLOAD,
7169                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7170                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7171                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7172                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7173                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7174                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7175                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7176                         I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD,
7177 #ifdef X722_SUPPORT
7178                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
7179                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7180                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7181                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7182                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7183                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7184                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7185                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7186                         I40E_INSET_FLEX_PAYLOAD,
7187                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
7188                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7189                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7190                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7191                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7192                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7193                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7194                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7195                         I40E_INSET_FLEX_PAYLOAD,
7196 #endif
7197                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7198                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7199                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7200                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7201                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7202                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7203                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7204                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7205                         I40E_INSET_FLEX_PAYLOAD,
7206 #ifdef X722_SUPPORT
7207                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
7208                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7209                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7210                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7211                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7212                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7213                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7214                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
7215                         I40E_INSET_FLEX_PAYLOAD,
7216 #endif
7217                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7218                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7219                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7220                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7221                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7222                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7223                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
7224                         I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT |
7225                         I40E_INSET_FLEX_PAYLOAD,
7226                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7227                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7228                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7229                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
7230                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
7231                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
7232                         I40E_INSET_IPV6_DST | I40E_INSET_TUNNEL_ID |
7233                         I40E_INSET_FLEX_PAYLOAD,
7234                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7235                         I40E_INSET_DMAC | I40E_INSET_SMAC |
7236                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7237                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_LAST_ETHER_TYPE |
7238                         I40E_INSET_FLEX_PAYLOAD,
7239         };
7240
7241         /**
7242          * Flow director supports only fields defined in
7243          * union rte_eth_fdir_flow.
7244          */
7245         static const uint64_t valid_fdir_inset_table[] = {
7246                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7247                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7248                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7249                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
7250                 I40E_INSET_IPV4_TTL,
7251                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7252                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7253                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7254                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7255                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7256 #ifdef X722_SUPPORT
7257                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
7258                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7259                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7260                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7261                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7262                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
7263                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7264                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7265                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7266                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7267 #endif
7268                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7269                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7270                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7271                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7272                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7273 #ifdef X722_SUPPORT
7274                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
7275                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7276                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7277                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7278                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7279 #endif
7280                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7281                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7282                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7283                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_TTL |
7284                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7285                 I40E_INSET_SCTP_VT,
7286                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7287                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7288                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7289                 I40E_INSET_IPV4_TOS | I40E_INSET_IPV4_PROTO |
7290                 I40E_INSET_IPV4_TTL,
7291                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7292                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7293                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7294                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
7295                 I40E_INSET_IPV6_HOP_LIMIT,
7296                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7297                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7298                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7299                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7300                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7301 #ifdef X722_SUPPORT
7302                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
7303                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7304                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7305                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7306                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7307                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
7308                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7309                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7310                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7311                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7312 #endif
7313                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7314                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7315                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7316                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7317                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7318 #ifdef X722_SUPPORT
7319                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
7320                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7321                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7322                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7323                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7324 #endif
7325                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7326                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7327                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7328                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_HOP_LIMIT |
7329                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7330                 I40E_INSET_SCTP_VT,
7331                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7332                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7333                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7334                 I40E_INSET_IPV6_TC | I40E_INSET_IPV6_NEXT_HDR |
7335                 I40E_INSET_IPV6_HOP_LIMIT,
7336                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7337                 I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
7338                 I40E_INSET_LAST_ETHER_TYPE,
7339         };
7340
7341         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
7342                 return 0;
7343         if (filter == RTE_ETH_FILTER_HASH)
7344                 valid = valid_hash_inset_table[pctype];
7345         else
7346                 valid = valid_fdir_inset_table[pctype];
7347
7348         return valid;
7349 }
7350
7351 /**
7352  * Validate if the input set is allowed for a specific PCTYPE
7353  */
7354 static int
7355 i40e_validate_input_set(enum i40e_filter_pctype pctype,
7356                 enum rte_filter_type filter, uint64_t inset)
7357 {
7358         uint64_t valid;
7359
7360         valid = i40e_get_valid_input_set(pctype, filter);
7361         if (inset & (~valid))
7362                 return -EINVAL;
7363
7364         return 0;
7365 }
7366
7367 /* default input set fields combination per pctype */
7368 static uint64_t
7369 i40e_get_default_input_set(uint16_t pctype)
7370 {
7371         static const uint64_t default_inset_table[] = {
7372                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
7373                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
7374                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7375                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7376                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7377 #ifdef X722_SUPPORT
7378                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
7379                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7380                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7381                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
7382                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7383                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7384 #endif
7385                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7386                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7387                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7388 #ifdef X722_SUPPORT
7389                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
7390                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7391                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7392 #endif
7393                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7394                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
7395                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7396                         I40E_INSET_SCTP_VT,
7397                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7398                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
7399                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
7400                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
7401                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7402                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7403                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7404 #ifdef X722_SUPPORT
7405                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
7406                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7407                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7408                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
7409                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7410                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7411 #endif
7412                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7413                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7414                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7415 #ifdef X722_SUPPORT
7416                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
7417                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7418                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
7419 #endif
7420                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7421                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
7422                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
7423                         I40E_INSET_SCTP_VT,
7424                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7425                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
7426                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
7427                         I40E_INSET_LAST_ETHER_TYPE,
7428         };
7429
7430         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
7431                 return 0;
7432
7433         return default_inset_table[pctype];
7434 }
7435
7436 /**
7437  * Parse the input set from index to logical bit masks
7438  */
7439 static int
7440 i40e_parse_input_set(uint64_t *inset,
7441                      enum i40e_filter_pctype pctype,
7442                      enum rte_eth_input_set_field *field,
7443                      uint16_t size)
7444 {
7445         uint16_t i, j;
7446         int ret = -EINVAL;
7447
7448         static const struct {
7449                 enum rte_eth_input_set_field field;
7450                 uint64_t inset;
7451         } inset_convert_table[] = {
7452                 {RTE_ETH_INPUT_SET_NONE, I40E_INSET_NONE},
7453                 {RTE_ETH_INPUT_SET_L2_SRC_MAC, I40E_INSET_SMAC},
7454                 {RTE_ETH_INPUT_SET_L2_DST_MAC, I40E_INSET_DMAC},
7455                 {RTE_ETH_INPUT_SET_L2_OUTER_VLAN, I40E_INSET_VLAN_OUTER},
7456                 {RTE_ETH_INPUT_SET_L2_INNER_VLAN, I40E_INSET_VLAN_INNER},
7457                 {RTE_ETH_INPUT_SET_L2_ETHERTYPE, I40E_INSET_LAST_ETHER_TYPE},
7458                 {RTE_ETH_INPUT_SET_L3_SRC_IP4, I40E_INSET_IPV4_SRC},
7459                 {RTE_ETH_INPUT_SET_L3_DST_IP4, I40E_INSET_IPV4_DST},
7460                 {RTE_ETH_INPUT_SET_L3_IP4_TOS, I40E_INSET_IPV4_TOS},
7461                 {RTE_ETH_INPUT_SET_L3_IP4_PROTO, I40E_INSET_IPV4_PROTO},
7462                 {RTE_ETH_INPUT_SET_L3_IP4_TTL, I40E_INSET_IPV4_TTL},
7463                 {RTE_ETH_INPUT_SET_L3_SRC_IP6, I40E_INSET_IPV6_SRC},
7464                 {RTE_ETH_INPUT_SET_L3_DST_IP6, I40E_INSET_IPV6_DST},
7465                 {RTE_ETH_INPUT_SET_L3_IP6_TC, I40E_INSET_IPV6_TC},
7466                 {RTE_ETH_INPUT_SET_L3_IP6_NEXT_HEADER,
7467                         I40E_INSET_IPV6_NEXT_HDR},
7468                 {RTE_ETH_INPUT_SET_L3_IP6_HOP_LIMITS,
7469                         I40E_INSET_IPV6_HOP_LIMIT},
7470                 {RTE_ETH_INPUT_SET_L4_UDP_SRC_PORT, I40E_INSET_SRC_PORT},
7471                 {RTE_ETH_INPUT_SET_L4_TCP_SRC_PORT, I40E_INSET_SRC_PORT},
7472                 {RTE_ETH_INPUT_SET_L4_SCTP_SRC_PORT, I40E_INSET_SRC_PORT},
7473                 {RTE_ETH_INPUT_SET_L4_UDP_DST_PORT, I40E_INSET_DST_PORT},
7474                 {RTE_ETH_INPUT_SET_L4_TCP_DST_PORT, I40E_INSET_DST_PORT},
7475                 {RTE_ETH_INPUT_SET_L4_SCTP_DST_PORT, I40E_INSET_DST_PORT},
7476                 {RTE_ETH_INPUT_SET_L4_SCTP_VERIFICATION_TAG,
7477                         I40E_INSET_SCTP_VT},
7478                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_DST_MAC,
7479                         I40E_INSET_TUNNEL_DMAC},
7480                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_VLAN,
7481                         I40E_INSET_VLAN_TUNNEL},
7482                 {RTE_ETH_INPUT_SET_TUNNEL_L4_UDP_KEY,
7483                         I40E_INSET_TUNNEL_ID},
7484                 {RTE_ETH_INPUT_SET_TUNNEL_GRE_KEY, I40E_INSET_TUNNEL_ID},
7485                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_1ST_WORD,
7486                         I40E_INSET_FLEX_PAYLOAD_W1},
7487                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_2ND_WORD,
7488                         I40E_INSET_FLEX_PAYLOAD_W2},
7489                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_3RD_WORD,
7490                         I40E_INSET_FLEX_PAYLOAD_W3},
7491                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_4TH_WORD,
7492                         I40E_INSET_FLEX_PAYLOAD_W4},
7493                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_5TH_WORD,
7494                         I40E_INSET_FLEX_PAYLOAD_W5},
7495                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_6TH_WORD,
7496                         I40E_INSET_FLEX_PAYLOAD_W6},
7497                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_7TH_WORD,
7498                         I40E_INSET_FLEX_PAYLOAD_W7},
7499                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_8TH_WORD,
7500                         I40E_INSET_FLEX_PAYLOAD_W8},
7501         };
7502
7503         if (!inset || !field || size > RTE_ETH_INSET_SIZE_MAX)
7504                 return ret;
7505
7506         /* Only one item allowed for default or all */
7507         if (size == 1) {
7508                 if (field[0] == RTE_ETH_INPUT_SET_DEFAULT) {
7509                         *inset = i40e_get_default_input_set(pctype);
7510                         return 0;
7511                 } else if (field[0] == RTE_ETH_INPUT_SET_NONE) {
7512                         *inset = I40E_INSET_NONE;
7513                         return 0;
7514                 }
7515         }
7516
7517         for (i = 0, *inset = 0; i < size; i++) {
7518                 for (j = 0; j < RTE_DIM(inset_convert_table); j++) {
7519                         if (field[i] == inset_convert_table[j].field) {
7520                                 *inset |= inset_convert_table[j].inset;
7521                                 break;
7522                         }
7523                 }
7524
7525                 /* It contains unsupported input set, return immediately */
7526                 if (j == RTE_DIM(inset_convert_table))
7527                         return ret;
7528         }
7529
7530         return 0;
7531 }
7532
7533 /**
7534  * Translate the input set from bit masks to register aware bit masks
7535  * and vice versa
7536  */
7537 static uint64_t
7538 i40e_translate_input_set_reg(uint64_t input)
7539 {
7540         uint64_t val = 0;
7541         uint16_t i;
7542
7543         static const struct {
7544                 uint64_t inset;
7545                 uint64_t inset_reg;
7546         } inset_map[] = {
7547                 {I40E_INSET_DMAC, I40E_REG_INSET_L2_DMAC},
7548                 {I40E_INSET_SMAC, I40E_REG_INSET_L2_SMAC},
7549                 {I40E_INSET_VLAN_OUTER, I40E_REG_INSET_L2_OUTER_VLAN},
7550                 {I40E_INSET_VLAN_INNER, I40E_REG_INSET_L2_INNER_VLAN},
7551                 {I40E_INSET_LAST_ETHER_TYPE, I40E_REG_INSET_LAST_ETHER_TYPE},
7552                 {I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4},
7553                 {I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4},
7554                 {I40E_INSET_IPV4_TOS, I40E_REG_INSET_L3_IP4_TOS},
7555                 {I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO},
7556                 {I40E_INSET_IPV4_TTL, I40E_REG_INSET_L3_IP4_TTL},
7557                 {I40E_INSET_IPV6_SRC, I40E_REG_INSET_L3_SRC_IP6},
7558                 {I40E_INSET_IPV6_DST, I40E_REG_INSET_L3_DST_IP6},
7559                 {I40E_INSET_IPV6_TC, I40E_REG_INSET_L3_IP6_TC},
7560                 {I40E_INSET_IPV6_NEXT_HDR, I40E_REG_INSET_L3_IP6_NEXT_HDR},
7561                 {I40E_INSET_IPV6_HOP_LIMIT, I40E_REG_INSET_L3_IP6_HOP_LIMIT},
7562                 {I40E_INSET_SRC_PORT, I40E_REG_INSET_L4_SRC_PORT},
7563                 {I40E_INSET_DST_PORT, I40E_REG_INSET_L4_DST_PORT},
7564                 {I40E_INSET_SCTP_VT, I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG},
7565                 {I40E_INSET_TUNNEL_ID, I40E_REG_INSET_TUNNEL_ID},
7566                 {I40E_INSET_TUNNEL_DMAC,
7567                         I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC},
7568                 {I40E_INSET_TUNNEL_IPV4_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP4},
7569                 {I40E_INSET_TUNNEL_IPV6_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP6},
7570                 {I40E_INSET_TUNNEL_SRC_PORT,
7571                         I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT},
7572                 {I40E_INSET_TUNNEL_DST_PORT,
7573                         I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT},
7574                 {I40E_INSET_VLAN_TUNNEL, I40E_REG_INSET_TUNNEL_VLAN},
7575                 {I40E_INSET_FLEX_PAYLOAD_W1, I40E_REG_INSET_FLEX_PAYLOAD_WORD1},
7576                 {I40E_INSET_FLEX_PAYLOAD_W2, I40E_REG_INSET_FLEX_PAYLOAD_WORD2},
7577                 {I40E_INSET_FLEX_PAYLOAD_W3, I40E_REG_INSET_FLEX_PAYLOAD_WORD3},
7578                 {I40E_INSET_FLEX_PAYLOAD_W4, I40E_REG_INSET_FLEX_PAYLOAD_WORD4},
7579                 {I40E_INSET_FLEX_PAYLOAD_W5, I40E_REG_INSET_FLEX_PAYLOAD_WORD5},
7580                 {I40E_INSET_FLEX_PAYLOAD_W6, I40E_REG_INSET_FLEX_PAYLOAD_WORD6},
7581                 {I40E_INSET_FLEX_PAYLOAD_W7, I40E_REG_INSET_FLEX_PAYLOAD_WORD7},
7582                 {I40E_INSET_FLEX_PAYLOAD_W8, I40E_REG_INSET_FLEX_PAYLOAD_WORD8},
7583         };
7584
7585         if (input == 0)
7586                 return val;
7587
7588         /* Translate input set to register aware inset */
7589         for (i = 0; i < RTE_DIM(inset_map); i++) {
7590                 if (input & inset_map[i].inset)
7591                         val |= inset_map[i].inset_reg;
7592         }
7593
7594         return val;
7595 }
7596
7597 static int
7598 i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem)
7599 {
7600         uint8_t i, idx = 0;
7601         uint64_t inset_need_mask = inset;
7602
7603         static const struct {
7604                 uint64_t inset;
7605                 uint32_t mask;
7606         } inset_mask_map[] = {
7607                 {I40E_INSET_IPV4_TOS, I40E_INSET_IPV4_TOS_MASK},
7608                 {I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL, 0},
7609                 {I40E_INSET_IPV4_PROTO, I40E_INSET_IPV4_PROTO_MASK},
7610                 {I40E_INSET_IPV4_TTL, I40E_INSET_IPv4_TTL_MASK},
7611                 {I40E_INSET_IPV6_TC, I40E_INSET_IPV6_TC_MASK},
7612                 {I40E_INSET_IPV6_NEXT_HDR | I40E_INSET_IPV6_HOP_LIMIT, 0},
7613                 {I40E_INSET_IPV6_NEXT_HDR, I40E_INSET_IPV6_NEXT_HDR_MASK},
7614                 {I40E_INSET_IPV6_HOP_LIMIT, I40E_INSET_IPV6_HOP_LIMIT_MASK},
7615         };
7616
7617         if (!inset || !mask || !nb_elem)
7618                 return 0;
7619
7620         for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
7621                 /* Clear the inset bit, if no MASK is required,
7622                  * for example proto + ttl
7623                  */
7624                 if ((inset & inset_mask_map[i].inset) ==
7625                      inset_mask_map[i].inset && inset_mask_map[i].mask == 0)
7626                         inset_need_mask &= ~inset_mask_map[i].inset;
7627                 if (!inset_need_mask)
7628                         return 0;
7629         }
7630         for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
7631                 if ((inset_need_mask & inset_mask_map[i].inset) ==
7632                     inset_mask_map[i].inset) {
7633                         if (idx >= nb_elem) {
7634                                 PMD_DRV_LOG(ERR, "exceed maximal number of bitmasks");
7635                                 return -EINVAL;
7636                         }
7637                         mask[idx] = inset_mask_map[i].mask;
7638                         idx++;
7639                 }
7640         }
7641
7642         return idx;
7643 }
7644
7645 static void
7646 i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
7647 {
7648         uint32_t reg = i40e_read_rx_ctl(hw, addr);
7649
7650         PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x\n", addr, reg);
7651         if (reg != val)
7652                 i40e_write_rx_ctl(hw, addr, val);
7653         PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x\n", addr,
7654                     (uint32_t)i40e_read_rx_ctl(hw, addr));
7655 }
7656
7657 static void
7658 i40e_filter_input_set_init(struct i40e_pf *pf)
7659 {
7660         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7661         enum i40e_filter_pctype pctype;
7662         uint64_t input_set, inset_reg;
7663         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
7664         int num, i;
7665
7666         for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
7667              pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++) {
7668                 if (!I40E_VALID_PCTYPE(pctype))
7669                         continue;
7670                 input_set = i40e_get_default_input_set(pctype);
7671
7672                 num = i40e_generate_inset_mask_reg(input_set, mask_reg,
7673                                                    I40E_INSET_MASK_NUM_REG);
7674                 if (num < 0)
7675                         return;
7676                 inset_reg = i40e_translate_input_set_reg(input_set);
7677
7678                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
7679                                       (uint32_t)(inset_reg & UINT32_MAX));
7680                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
7681                                      (uint32_t)((inset_reg >>
7682                                      I40E_32_BIT_WIDTH) & UINT32_MAX));
7683                 i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype),
7684                                       (uint32_t)(inset_reg & UINT32_MAX));
7685                 i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
7686                                      (uint32_t)((inset_reg >>
7687                                      I40E_32_BIT_WIDTH) & UINT32_MAX));
7688
7689                 for (i = 0; i < num; i++) {
7690                         i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7691                                              mask_reg[i]);
7692                         i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7693                                              mask_reg[i]);
7694                 }
7695                 /*clear unused mask registers of the pctype */
7696                 for (i = num; i < I40E_INSET_MASK_NUM_REG; i++) {
7697                         i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7698                                              0);
7699                         i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7700                                              0);
7701                 }
7702                 I40E_WRITE_FLUSH(hw);
7703
7704                 /* store the default input set */
7705                 pf->hash_input_set[pctype] = input_set;
7706                 pf->fdir.input_set[pctype] = input_set;
7707         }
7708 }
7709
7710 int
7711 i40e_hash_filter_inset_select(struct i40e_hw *hw,
7712                          struct rte_eth_input_set_conf *conf)
7713 {
7714         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
7715         enum i40e_filter_pctype pctype;
7716         uint64_t input_set, inset_reg = 0;
7717         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
7718         int ret, i, num;
7719
7720         if (!conf) {
7721                 PMD_DRV_LOG(ERR, "Invalid pointer");
7722                 return -EFAULT;
7723         }
7724         if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
7725             conf->op != RTE_ETH_INPUT_SET_ADD) {
7726                 PMD_DRV_LOG(ERR, "Unsupported input set operation");
7727                 return -EINVAL;
7728         }
7729
7730         if (!I40E_VALID_FLOW(conf->flow_type)) {
7731                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
7732                 return -EINVAL;
7733         }
7734
7735 #ifdef X722_SUPPORT
7736         /* get translated pctype value in fd pctype register */
7737         pctype = (enum i40e_filter_pctype)i40e_read_rx_ctl(hw,
7738                 I40E_GLQF_FD_PCTYPES((int)i40e_flowtype_to_pctype(
7739                 conf->flow_type)));
7740 #else
7741         pctype = i40e_flowtype_to_pctype(conf->flow_type);
7742 #endif
7743
7744         ret = i40e_parse_input_set(&input_set, pctype, conf->field,
7745                                    conf->inset_size);
7746         if (ret) {
7747                 PMD_DRV_LOG(ERR, "Failed to parse input set");
7748                 return -EINVAL;
7749         }
7750         if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_HASH,
7751                                     input_set) != 0) {
7752                 PMD_DRV_LOG(ERR, "Invalid input set");
7753                 return -EINVAL;
7754         }
7755         if (conf->op == RTE_ETH_INPUT_SET_ADD) {
7756                 /* get inset value in register */
7757                 inset_reg = i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, pctype));
7758                 inset_reg <<= I40E_32_BIT_WIDTH;
7759                 inset_reg |= i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, pctype));
7760                 input_set |= pf->hash_input_set[pctype];
7761         }
7762         num = i40e_generate_inset_mask_reg(input_set, mask_reg,
7763                                            I40E_INSET_MASK_NUM_REG);
7764         if (num < 0)
7765                 return -EINVAL;
7766
7767         inset_reg |= i40e_translate_input_set_reg(input_set);
7768
7769         i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype),
7770                               (uint32_t)(inset_reg & UINT32_MAX));
7771         i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
7772                              (uint32_t)((inset_reg >>
7773                              I40E_32_BIT_WIDTH) & UINT32_MAX));
7774
7775         for (i = 0; i < num; i++)
7776                 i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7777                                      mask_reg[i]);
7778         /*clear unused mask registers of the pctype */
7779         for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
7780                 i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7781                                      0);
7782         I40E_WRITE_FLUSH(hw);
7783
7784         pf->hash_input_set[pctype] = input_set;
7785         return 0;
7786 }
7787
7788 int
7789 i40e_fdir_filter_inset_select(struct i40e_pf *pf,
7790                          struct rte_eth_input_set_conf *conf)
7791 {
7792         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7793         enum i40e_filter_pctype pctype;
7794         uint64_t input_set, inset_reg = 0;
7795         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG] = {0};
7796         int ret, i, num;
7797
7798         if (!hw || !conf) {
7799                 PMD_DRV_LOG(ERR, "Invalid pointer");
7800                 return -EFAULT;
7801         }
7802         if (conf->op != RTE_ETH_INPUT_SET_SELECT &&
7803             conf->op != RTE_ETH_INPUT_SET_ADD) {
7804                 PMD_DRV_LOG(ERR, "Unsupported input set operation");
7805                 return -EINVAL;
7806         }
7807
7808         if (!I40E_VALID_FLOW(conf->flow_type)) {
7809                 PMD_DRV_LOG(ERR, "invalid flow_type input.");
7810                 return -EINVAL;
7811         }
7812
7813         pctype = i40e_flowtype_to_pctype(conf->flow_type);
7814
7815         ret = i40e_parse_input_set(&input_set, pctype, conf->field,
7816                                    conf->inset_size);
7817         if (ret) {
7818                 PMD_DRV_LOG(ERR, "Failed to parse input set");
7819                 return -EINVAL;
7820         }
7821         if (i40e_validate_input_set(pctype, RTE_ETH_FILTER_FDIR,
7822                                     input_set) != 0) {
7823                 PMD_DRV_LOG(ERR, "Invalid input set");
7824                 return -EINVAL;
7825         }
7826
7827         /* get inset value in register */
7828         inset_reg = i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 1));
7829         inset_reg <<= I40E_32_BIT_WIDTH;
7830         inset_reg |= i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 0));
7831
7832         /* Can not change the inset reg for flex payload for fdir,
7833          * it is done by writing I40E_PRTQF_FD_FLXINSET
7834          * in i40e_set_flex_mask_on_pctype.
7835          */
7836         if (conf->op == RTE_ETH_INPUT_SET_SELECT)
7837                 inset_reg &= I40E_REG_INSET_FLEX_PAYLOAD_WORDS;
7838         else
7839                 input_set |= pf->fdir.input_set[pctype];
7840         num = i40e_generate_inset_mask_reg(input_set, mask_reg,
7841                                            I40E_INSET_MASK_NUM_REG);
7842         if (num < 0)
7843                 return -EINVAL;
7844
7845         inset_reg |= i40e_translate_input_set_reg(input_set);
7846
7847         i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
7848                               (uint32_t)(inset_reg & UINT32_MAX));
7849         i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
7850                              (uint32_t)((inset_reg >>
7851                              I40E_32_BIT_WIDTH) & UINT32_MAX));
7852
7853         for (i = 0; i < num; i++)
7854                 i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7855                                      mask_reg[i]);
7856         /*clear unused mask registers of the pctype */
7857         for (i = num; i < I40E_INSET_MASK_NUM_REG; i++)
7858                 i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7859                                      0);
7860         I40E_WRITE_FLUSH(hw);
7861
7862         pf->fdir.input_set[pctype] = input_set;
7863         return 0;
7864 }
7865
7866 static int
7867 i40e_hash_filter_get(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
7868 {
7869         int ret = 0;
7870
7871         if (!hw || !info) {
7872                 PMD_DRV_LOG(ERR, "Invalid pointer");
7873                 return -EFAULT;
7874         }
7875
7876         switch (info->info_type) {
7877         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
7878                 i40e_get_symmetric_hash_enable_per_port(hw,
7879                                         &(info->info.enable));
7880                 break;
7881         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
7882                 ret = i40e_get_hash_filter_global_config(hw,
7883                                 &(info->info.global_conf));
7884                 break;
7885         default:
7886                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
7887                                                         info->info_type);
7888                 ret = -EINVAL;
7889                 break;
7890         }
7891
7892         return ret;
7893 }
7894
7895 static int
7896 i40e_hash_filter_set(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
7897 {
7898         int ret = 0;
7899
7900         if (!hw || !info) {
7901                 PMD_DRV_LOG(ERR, "Invalid pointer");
7902                 return -EFAULT;
7903         }
7904
7905         switch (info->info_type) {
7906         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
7907                 i40e_set_symmetric_hash_enable_per_port(hw, info->info.enable);
7908                 break;
7909         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
7910                 ret = i40e_set_hash_filter_global_config(hw,
7911                                 &(info->info.global_conf));
7912                 break;
7913         case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT:
7914                 ret = i40e_hash_filter_inset_select(hw,
7915                                                &(info->info.input_set_conf));
7916                 break;
7917
7918         default:
7919                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
7920                                                         info->info_type);
7921                 ret = -EINVAL;
7922                 break;
7923         }
7924
7925         return ret;
7926 }
7927
7928 /* Operations for hash function */
7929 static int
7930 i40e_hash_filter_ctrl(struct rte_eth_dev *dev,
7931                       enum rte_filter_op filter_op,
7932                       void *arg)
7933 {
7934         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7935         int ret = 0;
7936
7937         switch (filter_op) {
7938         case RTE_ETH_FILTER_NOP:
7939                 break;
7940         case RTE_ETH_FILTER_GET:
7941                 ret = i40e_hash_filter_get(hw,
7942                         (struct rte_eth_hash_filter_info *)arg);
7943                 break;
7944         case RTE_ETH_FILTER_SET:
7945                 ret = i40e_hash_filter_set(hw,
7946                         (struct rte_eth_hash_filter_info *)arg);
7947                 break;
7948         default:
7949                 PMD_DRV_LOG(WARNING, "Filter operation (%d) not supported",
7950                                                                 filter_op);
7951                 ret = -ENOTSUP;
7952                 break;
7953         }
7954
7955         return ret;
7956 }
7957
7958 /*
7959  * Configure ethertype filter, which can director packet by filtering
7960  * with mac address and ether_type or only ether_type
7961  */
7962 static int
7963 i40e_ethertype_filter_set(struct i40e_pf *pf,
7964                         struct rte_eth_ethertype_filter *filter,
7965                         bool add)
7966 {
7967         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7968         struct i40e_control_filter_stats stats;
7969         uint16_t flags = 0;
7970         int ret;
7971
7972         if (filter->queue >= pf->dev_data->nb_rx_queues) {
7973                 PMD_DRV_LOG(ERR, "Invalid queue ID");
7974                 return -EINVAL;
7975         }
7976         if (filter->ether_type == ETHER_TYPE_IPv4 ||
7977                 filter->ether_type == ETHER_TYPE_IPv6) {
7978                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
7979                         " control packet filter.", filter->ether_type);
7980                 return -EINVAL;
7981         }
7982         if (filter->ether_type == ETHER_TYPE_VLAN)
7983                 PMD_DRV_LOG(WARNING, "filter vlan ether_type in first tag is"
7984                         " not supported.");
7985
7986         if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC))
7987                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC;
7988         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP)
7989                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP;
7990         flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE;
7991
7992         memset(&stats, 0, sizeof(stats));
7993         ret = i40e_aq_add_rem_control_packet_filter(hw,
7994                         filter->mac_addr.addr_bytes,
7995                         filter->ether_type, flags,
7996                         pf->main_vsi->seid,
7997                         filter->queue, add, &stats, NULL);
7998
7999         PMD_DRV_LOG(INFO, "add/rem control packet filter, return %d,"
8000                          " mac_etype_used = %u, etype_used = %u,"
8001                          " mac_etype_free = %u, etype_free = %u\n",
8002                          ret, stats.mac_etype_used, stats.etype_used,
8003                          stats.mac_etype_free, stats.etype_free);
8004         if (ret < 0)
8005                 return -ENOSYS;
8006         return 0;
8007 }
8008
8009 /*
8010  * Handle operations for ethertype filter.
8011  */
8012 static int
8013 i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
8014                                 enum rte_filter_op filter_op,
8015                                 void *arg)
8016 {
8017         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8018         int ret = 0;
8019
8020         if (filter_op == RTE_ETH_FILTER_NOP)
8021                 return ret;
8022
8023         if (arg == NULL) {
8024                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
8025                             filter_op);
8026                 return -EINVAL;
8027         }
8028
8029         switch (filter_op) {
8030         case RTE_ETH_FILTER_ADD:
8031                 ret = i40e_ethertype_filter_set(pf,
8032                         (struct rte_eth_ethertype_filter *)arg,
8033                         TRUE);
8034                 break;
8035         case RTE_ETH_FILTER_DELETE:
8036                 ret = i40e_ethertype_filter_set(pf,
8037                         (struct rte_eth_ethertype_filter *)arg,
8038                         FALSE);
8039                 break;
8040         default:
8041                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
8042                 ret = -ENOSYS;
8043                 break;
8044         }
8045         return ret;
8046 }
8047
8048 static int
8049 i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
8050                      enum rte_filter_type filter_type,
8051                      enum rte_filter_op filter_op,
8052                      void *arg)
8053 {
8054         int ret = 0;
8055
8056         if (dev == NULL)
8057                 return -EINVAL;
8058
8059         switch (filter_type) {
8060         case RTE_ETH_FILTER_NONE:
8061                 /* For global configuration */
8062                 ret = i40e_filter_ctrl_global_config(dev, filter_op, arg);
8063                 break;
8064         case RTE_ETH_FILTER_HASH:
8065                 ret = i40e_hash_filter_ctrl(dev, filter_op, arg);
8066                 break;
8067         case RTE_ETH_FILTER_MACVLAN:
8068                 ret = i40e_mac_filter_handle(dev, filter_op, arg);
8069                 break;
8070         case RTE_ETH_FILTER_ETHERTYPE:
8071                 ret = i40e_ethertype_filter_handle(dev, filter_op, arg);
8072                 break;
8073         case RTE_ETH_FILTER_TUNNEL:
8074                 ret = i40e_tunnel_filter_handle(dev, filter_op, arg);
8075                 break;
8076         case RTE_ETH_FILTER_FDIR:
8077                 ret = i40e_fdir_ctrl_func(dev, filter_op, arg);
8078                 break;
8079         default:
8080                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
8081                                                         filter_type);
8082                 ret = -EINVAL;
8083                 break;
8084         }
8085
8086         return ret;
8087 }
8088
8089 /*
8090  * Check and enable Extended Tag.
8091  * Enabling Extended Tag is important for 40G performance.
8092  */
8093 static void
8094 i40e_enable_extended_tag(struct rte_eth_dev *dev)
8095 {
8096         uint32_t buf = 0;
8097         int ret;
8098
8099         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
8100                                       PCI_DEV_CAP_REG);
8101         if (ret < 0) {
8102                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
8103                             PCI_DEV_CAP_REG);
8104                 return;
8105         }
8106         if (!(buf & PCI_DEV_CAP_EXT_TAG_MASK)) {
8107                 PMD_DRV_LOG(ERR, "Does not support Extended Tag");
8108                 return;
8109         }
8110
8111         buf = 0;
8112         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
8113                                       PCI_DEV_CTRL_REG);
8114         if (ret < 0) {
8115                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
8116                             PCI_DEV_CTRL_REG);
8117                 return;
8118         }
8119         if (buf & PCI_DEV_CTRL_EXT_TAG_MASK) {
8120                 PMD_DRV_LOG(DEBUG, "Extended Tag has already been enabled");
8121                 return;
8122         }
8123         buf |= PCI_DEV_CTRL_EXT_TAG_MASK;
8124         ret = rte_eal_pci_write_config(dev->pci_dev, &buf, sizeof(buf),
8125                                        PCI_DEV_CTRL_REG);
8126         if (ret < 0) {
8127                 PMD_DRV_LOG(ERR, "Failed to write PCI offset 0x%x",
8128                             PCI_DEV_CTRL_REG);
8129                 return;
8130         }
8131 }
8132
8133 /*
8134  * As some registers wouldn't be reset unless a global hardware reset,
8135  * hardware initialization is needed to put those registers into an
8136  * expected initial state.
8137  */
8138 static void
8139 i40e_hw_init(struct rte_eth_dev *dev)
8140 {
8141         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8142
8143         i40e_enable_extended_tag(dev);
8144
8145         /* clear the PF Queue Filter control register */
8146         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, 0);
8147
8148         /* Disable symmetric hash per port */
8149         i40e_set_symmetric_hash_enable_per_port(hw, 0);
8150 }
8151
8152 enum i40e_filter_pctype
8153 i40e_flowtype_to_pctype(uint16_t flow_type)
8154 {
8155         static const enum i40e_filter_pctype pctype_table[] = {
8156                 [RTE_ETH_FLOW_FRAG_IPV4] = I40E_FILTER_PCTYPE_FRAG_IPV4,
8157                 [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] =
8158                         I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8159                 [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] =
8160                         I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8161                 [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] =
8162                         I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8163                 [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] =
8164                         I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8165                 [RTE_ETH_FLOW_FRAG_IPV6] = I40E_FILTER_PCTYPE_FRAG_IPV6,
8166                 [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] =
8167                         I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
8168                 [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] =
8169                         I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
8170                 [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] =
8171                         I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
8172                 [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] =
8173                         I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
8174                 [RTE_ETH_FLOW_L2_PAYLOAD] = I40E_FILTER_PCTYPE_L2_PAYLOAD,
8175         };
8176
8177         return pctype_table[flow_type];
8178 }
8179
8180 uint16_t
8181 i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype)
8182 {
8183         static const uint16_t flowtype_table[] = {
8184                 [I40E_FILTER_PCTYPE_FRAG_IPV4] = RTE_ETH_FLOW_FRAG_IPV4,
8185                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
8186                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
8187 #ifdef X722_SUPPORT
8188                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] =
8189                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
8190                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] =
8191                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
8192 #endif
8193                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
8194                         RTE_ETH_FLOW_NONFRAG_IPV4_TCP,
8195 #ifdef X722_SUPPORT
8196                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] =
8197                         RTE_ETH_FLOW_NONFRAG_IPV4_TCP,
8198 #endif
8199                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
8200                         RTE_ETH_FLOW_NONFRAG_IPV4_SCTP,
8201                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
8202                         RTE_ETH_FLOW_NONFRAG_IPV4_OTHER,
8203                 [I40E_FILTER_PCTYPE_FRAG_IPV6] = RTE_ETH_FLOW_FRAG_IPV6,
8204                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
8205                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
8206 #ifdef X722_SUPPORT
8207                 [I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] =
8208                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
8209                 [I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] =
8210                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
8211 #endif
8212                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
8213                         RTE_ETH_FLOW_NONFRAG_IPV6_TCP,
8214 #ifdef X722_SUPPORT
8215                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] =
8216                         RTE_ETH_FLOW_NONFRAG_IPV6_TCP,
8217 #endif
8218                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
8219                         RTE_ETH_FLOW_NONFRAG_IPV6_SCTP,
8220                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
8221                         RTE_ETH_FLOW_NONFRAG_IPV6_OTHER,
8222                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] = RTE_ETH_FLOW_L2_PAYLOAD,
8223         };
8224
8225         return flowtype_table[pctype];
8226 }
8227
8228 /*
8229  * On X710, performance number is far from the expectation on recent firmware
8230  * versions; on XL710, performance number is also far from the expectation on
8231  * recent firmware versions, if promiscuous mode is disabled, or promiscuous
8232  * mode is enabled and port MAC address is equal to the packet destination MAC
8233  * address. The fix for this issue may not be integrated in the following
8234  * firmware version. So the workaround in software driver is needed. It needs
8235  * to modify the initial values of 3 internal only registers for both X710 and
8236  * XL710. Note that the values for X710 or XL710 could be different, and the
8237  * workaround can be removed when it is fixed in firmware in the future.
8238  */
8239
8240 /* For both X710 and XL710 */
8241 #define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x10000200
8242 #define I40E_GL_SWR_PRI_JOIN_MAP_0       0x26CE00
8243
8244 #define I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x011f0200
8245 #define I40E_GL_SWR_PRI_JOIN_MAP_2       0x26CE08
8246
8247 /* For X710 */
8248 #define I40E_GL_SWR_PM_UP_THR_EF_VALUE   0x03030303
8249 /* For XL710 */
8250 #define I40E_GL_SWR_PM_UP_THR_SF_VALUE   0x06060606
8251 #define I40E_GL_SWR_PM_UP_THR            0x269FBC
8252
8253 static void
8254 i40e_configure_registers(struct i40e_hw *hw)
8255 {
8256         static struct {
8257                 uint32_t addr;
8258                 uint64_t val;
8259         } reg_table[] = {
8260                 {I40E_GL_SWR_PRI_JOIN_MAP_0, I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE},
8261                 {I40E_GL_SWR_PRI_JOIN_MAP_2, I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE},
8262                 {I40E_GL_SWR_PM_UP_THR, 0}, /* Compute value dynamically */
8263         };
8264         uint64_t reg;
8265         uint32_t i;
8266         int ret;
8267
8268         for (i = 0; i < RTE_DIM(reg_table); i++) {
8269                 if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) {
8270                         if (i40e_is_40G_device(hw->device_id)) /* For XL710 */
8271                                 reg_table[i].val =
8272                                         I40E_GL_SWR_PM_UP_THR_SF_VALUE;
8273                         else /* For X710 */
8274                                 reg_table[i].val =
8275                                         I40E_GL_SWR_PM_UP_THR_EF_VALUE;
8276                 }
8277
8278                 ret = i40e_aq_debug_read_register(hw, reg_table[i].addr,
8279                                                         &reg, NULL);
8280                 if (ret < 0) {
8281                         PMD_DRV_LOG(ERR, "Failed to read from 0x%"PRIx32,
8282                                                         reg_table[i].addr);
8283                         break;
8284                 }
8285                 PMD_DRV_LOG(DEBUG, "Read from 0x%"PRIx32": 0x%"PRIx64,
8286                                                 reg_table[i].addr, reg);
8287                 if (reg == reg_table[i].val)
8288                         continue;
8289
8290                 ret = i40e_aq_debug_write_register(hw, reg_table[i].addr,
8291                                                 reg_table[i].val, NULL);
8292                 if (ret < 0) {
8293                         PMD_DRV_LOG(ERR, "Failed to write 0x%"PRIx64" to the "
8294                                 "address of 0x%"PRIx32, reg_table[i].val,
8295                                                         reg_table[i].addr);
8296                         break;
8297                 }
8298                 PMD_DRV_LOG(DEBUG, "Write 0x%"PRIx64" to the address of "
8299                         "0x%"PRIx32, reg_table[i].val, reg_table[i].addr);
8300         }
8301 }
8302
8303 #define I40E_VSI_TSR(_i)            (0x00050800 + ((_i) * 4))
8304 #define I40E_VSI_TSR_QINQ_CONFIG    0xc030
8305 #define I40E_VSI_L2TAGSTXVALID(_i)  (0x00042800 + ((_i) * 4))
8306 #define I40E_VSI_L2TAGSTXVALID_QINQ 0xab
8307 static int
8308 i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi)
8309 {
8310         uint32_t reg;
8311         int ret;
8312
8313         if (vsi->vsi_id >= I40E_MAX_NUM_VSIS) {
8314                 PMD_DRV_LOG(ERR, "VSI ID exceeds the maximum");
8315                 return -EINVAL;
8316         }
8317
8318         /* Configure for double VLAN RX stripping */
8319         reg = I40E_READ_REG(hw, I40E_VSI_TSR(vsi->vsi_id));
8320         if ((reg & I40E_VSI_TSR_QINQ_CONFIG) != I40E_VSI_TSR_QINQ_CONFIG) {
8321                 reg |= I40E_VSI_TSR_QINQ_CONFIG;
8322                 ret = i40e_aq_debug_write_register(hw,
8323                                                    I40E_VSI_TSR(vsi->vsi_id),
8324                                                    reg, NULL);
8325                 if (ret < 0) {
8326                         PMD_DRV_LOG(ERR, "Failed to update VSI_TSR[%d]",
8327                                     vsi->vsi_id);
8328                         return I40E_ERR_CONFIG;
8329                 }
8330         }
8331
8332         /* Configure for double VLAN TX insertion */
8333         reg = I40E_READ_REG(hw, I40E_VSI_L2TAGSTXVALID(vsi->vsi_id));
8334         if ((reg & 0xff) != I40E_VSI_L2TAGSTXVALID_QINQ) {
8335                 reg = I40E_VSI_L2TAGSTXVALID_QINQ;
8336                 ret = i40e_aq_debug_write_register(hw,
8337                                                    I40E_VSI_L2TAGSTXVALID(
8338                                                    vsi->vsi_id), reg, NULL);
8339                 if (ret < 0) {
8340                         PMD_DRV_LOG(ERR, "Failed to update "
8341                                 "VSI_L2TAGSTXVALID[%d]", vsi->vsi_id);
8342                         return I40E_ERR_CONFIG;
8343                 }
8344         }
8345
8346         return 0;
8347 }
8348
8349 /**
8350  * i40e_aq_add_mirror_rule
8351  * @hw: pointer to the hardware structure
8352  * @seid: VEB seid to add mirror rule to
8353  * @dst_id: destination vsi seid
8354  * @entries: Buffer which contains the entities to be mirrored
8355  * @count: number of entities contained in the buffer
8356  * @rule_id:the rule_id of the rule to be added
8357  *
8358  * Add a mirror rule for a given veb.
8359  *
8360  **/
8361 static enum i40e_status_code
8362 i40e_aq_add_mirror_rule(struct i40e_hw *hw,
8363                         uint16_t seid, uint16_t dst_id,
8364                         uint16_t rule_type, uint16_t *entries,
8365                         uint16_t count, uint16_t *rule_id)
8366 {
8367         struct i40e_aq_desc desc;
8368         struct i40e_aqc_add_delete_mirror_rule cmd;
8369         struct i40e_aqc_add_delete_mirror_rule_completion *resp =
8370                 (struct i40e_aqc_add_delete_mirror_rule_completion *)
8371                 &desc.params.raw;
8372         uint16_t buff_len;
8373         enum i40e_status_code status;
8374
8375         i40e_fill_default_direct_cmd_desc(&desc,
8376                                           i40e_aqc_opc_add_mirror_rule);
8377         memset(&cmd, 0, sizeof(cmd));
8378
8379         buff_len = sizeof(uint16_t) * count;
8380         desc.datalen = rte_cpu_to_le_16(buff_len);
8381         if (buff_len > 0)
8382                 desc.flags |= rte_cpu_to_le_16(
8383                         (uint16_t)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD));
8384         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
8385                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
8386         cmd.num_entries = rte_cpu_to_le_16(count);
8387         cmd.seid = rte_cpu_to_le_16(seid);
8388         cmd.destination = rte_cpu_to_le_16(dst_id);
8389
8390         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
8391         status = i40e_asq_send_command(hw, &desc, entries, buff_len, NULL);
8392         PMD_DRV_LOG(INFO, "i40e_aq_add_mirror_rule, aq_status %d,"
8393                          "rule_id = %u"
8394                          " mirror_rules_used = %u, mirror_rules_free = %u,",
8395                          hw->aq.asq_last_status, resp->rule_id,
8396                          resp->mirror_rules_used, resp->mirror_rules_free);
8397         *rule_id = rte_le_to_cpu_16(resp->rule_id);
8398
8399         return status;
8400 }
8401
8402 /**
8403  * i40e_aq_del_mirror_rule
8404  * @hw: pointer to the hardware structure
8405  * @seid: VEB seid to add mirror rule to
8406  * @entries: Buffer which contains the entities to be mirrored
8407  * @count: number of entities contained in the buffer
8408  * @rule_id:the rule_id of the rule to be delete
8409  *
8410  * Delete a mirror rule for a given veb.
8411  *
8412  **/
8413 static enum i40e_status_code
8414 i40e_aq_del_mirror_rule(struct i40e_hw *hw,
8415                 uint16_t seid, uint16_t rule_type, uint16_t *entries,
8416                 uint16_t count, uint16_t rule_id)
8417 {
8418         struct i40e_aq_desc desc;
8419         struct i40e_aqc_add_delete_mirror_rule cmd;
8420         uint16_t buff_len = 0;
8421         enum i40e_status_code status;
8422         void *buff = NULL;
8423
8424         i40e_fill_default_direct_cmd_desc(&desc,
8425                                           i40e_aqc_opc_delete_mirror_rule);
8426         memset(&cmd, 0, sizeof(cmd));
8427         if (rule_type == I40E_AQC_MIRROR_RULE_TYPE_VLAN) {
8428                 desc.flags |= rte_cpu_to_le_16((uint16_t)(I40E_AQ_FLAG_BUF |
8429                                                           I40E_AQ_FLAG_RD));
8430                 cmd.num_entries = count;
8431                 buff_len = sizeof(uint16_t) * count;
8432                 desc.datalen = rte_cpu_to_le_16(buff_len);
8433                 buff = (void *)entries;
8434         } else
8435                 /* rule id is filled in destination field for deleting mirror rule */
8436                 cmd.destination = rte_cpu_to_le_16(rule_id);
8437
8438         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
8439                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
8440         cmd.seid = rte_cpu_to_le_16(seid);
8441
8442         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
8443         status = i40e_asq_send_command(hw, &desc, buff, buff_len, NULL);
8444
8445         return status;
8446 }
8447
8448 /**
8449  * i40e_mirror_rule_set
8450  * @dev: pointer to the hardware structure
8451  * @mirror_conf: mirror rule info
8452  * @sw_id: mirror rule's sw_id
8453  * @on: enable/disable
8454  *
8455  * set a mirror rule.
8456  *
8457  **/
8458 static int
8459 i40e_mirror_rule_set(struct rte_eth_dev *dev,
8460                         struct rte_eth_mirror_conf *mirror_conf,
8461                         uint8_t sw_id, uint8_t on)
8462 {
8463         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8464         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8465         struct i40e_mirror_rule *it, *mirr_rule = NULL;
8466         struct i40e_mirror_rule *parent = NULL;
8467         uint16_t seid, dst_seid, rule_id;
8468         uint16_t i, j = 0;
8469         int ret;
8470
8471         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_set: sw_id = %d.", sw_id);
8472
8473         if (pf->main_vsi->veb == NULL || pf->vfs == NULL) {
8474                 PMD_DRV_LOG(ERR, "mirror rule can not be configured"
8475                         " without veb or vfs.");
8476                 return -ENOSYS;
8477         }
8478         if (pf->nb_mirror_rule > I40E_MAX_MIRROR_RULES) {
8479                 PMD_DRV_LOG(ERR, "mirror table is full.");
8480                 return -ENOSPC;
8481         }
8482         if (mirror_conf->dst_pool > pf->vf_num) {
8483                 PMD_DRV_LOG(ERR, "invalid destination pool %u.",
8484                                  mirror_conf->dst_pool);
8485                 return -EINVAL;
8486         }
8487
8488         seid = pf->main_vsi->veb->seid;
8489
8490         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
8491                 if (sw_id <= it->index) {
8492                         mirr_rule = it;
8493                         break;
8494                 }
8495                 parent = it;
8496         }
8497         if (mirr_rule && sw_id == mirr_rule->index) {
8498                 if (on) {
8499                         PMD_DRV_LOG(ERR, "mirror rule exists.");
8500                         return -EEXIST;
8501                 } else {
8502                         ret = i40e_aq_del_mirror_rule(hw, seid,
8503                                         mirr_rule->rule_type,
8504                                         mirr_rule->entries,
8505                                         mirr_rule->num_entries, mirr_rule->id);
8506                         if (ret < 0) {
8507                                 PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
8508                                                    " ret = %d, aq_err = %d.",
8509                                                    ret, hw->aq.asq_last_status);
8510                                 return -ENOSYS;
8511                         }
8512                         TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
8513                         rte_free(mirr_rule);
8514                         pf->nb_mirror_rule--;
8515                         return 0;
8516                 }
8517         } else if (!on) {
8518                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
8519                 return -ENOENT;
8520         }
8521
8522         mirr_rule = rte_zmalloc("i40e_mirror_rule",
8523                                 sizeof(struct i40e_mirror_rule) , 0);
8524         if (!mirr_rule) {
8525                 PMD_DRV_LOG(ERR, "failed to allocate memory");
8526                 return I40E_ERR_NO_MEMORY;
8527         }
8528         switch (mirror_conf->rule_type) {
8529         case ETH_MIRROR_VLAN:
8530                 for (i = 0, j = 0; i < ETH_MIRROR_MAX_VLANS; i++) {
8531                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
8532                                 mirr_rule->entries[j] =
8533                                         mirror_conf->vlan.vlan_id[i];
8534                                 j++;
8535                         }
8536                 }
8537                 if (j == 0) {
8538                         PMD_DRV_LOG(ERR, "vlan is not specified.");
8539                         rte_free(mirr_rule);
8540                         return -EINVAL;
8541                 }
8542                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_VLAN;
8543                 break;
8544         case ETH_MIRROR_VIRTUAL_POOL_UP:
8545         case ETH_MIRROR_VIRTUAL_POOL_DOWN:
8546                 /* check if the specified pool bit is out of range */
8547                 if (mirror_conf->pool_mask > (uint64_t)(1ULL << (pf->vf_num + 1))) {
8548                         PMD_DRV_LOG(ERR, "pool mask is out of range.");
8549                         rte_free(mirr_rule);
8550                         return -EINVAL;
8551                 }
8552                 for (i = 0, j = 0; i < pf->vf_num; i++) {
8553                         if (mirror_conf->pool_mask & (1ULL << i)) {
8554                                 mirr_rule->entries[j] = pf->vfs[i].vsi->seid;
8555                                 j++;
8556                         }
8557                 }
8558                 if (mirror_conf->pool_mask & (1ULL << pf->vf_num)) {
8559                         /* add pf vsi to entries */
8560                         mirr_rule->entries[j] = pf->main_vsi_seid;
8561                         j++;
8562                 }
8563                 if (j == 0) {
8564                         PMD_DRV_LOG(ERR, "pool is not specified.");
8565                         rte_free(mirr_rule);
8566                         return -EINVAL;
8567                 }
8568                 /* egress and ingress in aq commands means from switch but not port */
8569                 mirr_rule->rule_type =
8570                         (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) ?
8571                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_EGRESS :
8572                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_INGRESS;
8573                 break;
8574         case ETH_MIRROR_UPLINK_PORT:
8575                 /* egress and ingress in aq commands means from switch but not port*/
8576                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_EGRESS;
8577                 break;
8578         case ETH_MIRROR_DOWNLINK_PORT:
8579                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_INGRESS;
8580                 break;
8581         default:
8582                 PMD_DRV_LOG(ERR, "unsupported mirror type %d.",
8583                         mirror_conf->rule_type);
8584                 rte_free(mirr_rule);
8585                 return -EINVAL;
8586         }
8587
8588         /* If the dst_pool is equal to vf_num, consider it as PF */
8589         if (mirror_conf->dst_pool == pf->vf_num)
8590                 dst_seid = pf->main_vsi_seid;
8591         else
8592                 dst_seid = pf->vfs[mirror_conf->dst_pool].vsi->seid;
8593
8594         ret = i40e_aq_add_mirror_rule(hw, seid, dst_seid,
8595                                       mirr_rule->rule_type, mirr_rule->entries,
8596                                       j, &rule_id);
8597         if (ret < 0) {
8598                 PMD_DRV_LOG(ERR, "failed to add mirror rule:"
8599                                    " ret = %d, aq_err = %d.",
8600                                    ret, hw->aq.asq_last_status);
8601                 rte_free(mirr_rule);
8602                 return -ENOSYS;
8603         }
8604
8605         mirr_rule->index = sw_id;
8606         mirr_rule->num_entries = j;
8607         mirr_rule->id = rule_id;
8608         mirr_rule->dst_vsi_seid = dst_seid;
8609
8610         if (parent)
8611                 TAILQ_INSERT_AFTER(&pf->mirror_list, parent, mirr_rule, rules);
8612         else
8613                 TAILQ_INSERT_HEAD(&pf->mirror_list, mirr_rule, rules);
8614
8615         pf->nb_mirror_rule++;
8616         return 0;
8617 }
8618
8619 /**
8620  * i40e_mirror_rule_reset
8621  * @dev: pointer to the device
8622  * @sw_id: mirror rule's sw_id
8623  *
8624  * reset a mirror rule.
8625  *
8626  **/
8627 static int
8628 i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id)
8629 {
8630         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8631         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8632         struct i40e_mirror_rule *it, *mirr_rule = NULL;
8633         uint16_t seid;
8634         int ret;
8635
8636         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_reset: sw_id = %d.", sw_id);
8637
8638         seid = pf->main_vsi->veb->seid;
8639
8640         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
8641                 if (sw_id == it->index) {
8642                         mirr_rule = it;
8643                         break;
8644                 }
8645         }
8646         if (mirr_rule) {
8647                 ret = i40e_aq_del_mirror_rule(hw, seid,
8648                                 mirr_rule->rule_type,
8649                                 mirr_rule->entries,
8650                                 mirr_rule->num_entries, mirr_rule->id);
8651                 if (ret < 0) {
8652                         PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
8653                                            " status = %d, aq_err = %d.",
8654                                            ret, hw->aq.asq_last_status);
8655                         return -ENOSYS;
8656                 }
8657                 TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
8658                 rte_free(mirr_rule);
8659                 pf->nb_mirror_rule--;
8660         } else {
8661                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
8662                 return -ENOENT;
8663         }
8664         return 0;
8665 }
8666
8667 static uint64_t
8668 i40e_read_systime_cyclecounter(struct rte_eth_dev *dev)
8669 {
8670         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8671         uint64_t systim_cycles;
8672
8673         systim_cycles = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_L);
8674         systim_cycles |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_H)
8675                         << 32;
8676
8677         return systim_cycles;
8678 }
8679
8680 static uint64_t
8681 i40e_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev, uint8_t index)
8682 {
8683         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8684         uint64_t rx_tstamp;
8685
8686         rx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(index));
8687         rx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(index))
8688                         << 32;
8689
8690         return rx_tstamp;
8691 }
8692
8693 static uint64_t
8694 i40e_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
8695 {
8696         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8697         uint64_t tx_tstamp;
8698
8699         tx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_L);
8700         tx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H)
8701                         << 32;
8702
8703         return tx_tstamp;
8704 }
8705
8706 static void
8707 i40e_start_timecounters(struct rte_eth_dev *dev)
8708 {
8709         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8710         struct i40e_adapter *adapter =
8711                         (struct i40e_adapter *)dev->data->dev_private;
8712         struct rte_eth_link link;
8713         uint32_t tsync_inc_l;
8714         uint32_t tsync_inc_h;
8715
8716         /* Get current link speed. */
8717         memset(&link, 0, sizeof(link));
8718         i40e_dev_link_update(dev, 1);
8719         rte_i40e_dev_atomic_read_link_status(dev, &link);
8720
8721         switch (link.link_speed) {
8722         case ETH_SPEED_NUM_40G:
8723                 tsync_inc_l = I40E_PTP_40GB_INCVAL & 0xFFFFFFFF;
8724                 tsync_inc_h = I40E_PTP_40GB_INCVAL >> 32;
8725                 break;
8726         case ETH_SPEED_NUM_10G:
8727                 tsync_inc_l = I40E_PTP_10GB_INCVAL & 0xFFFFFFFF;
8728                 tsync_inc_h = I40E_PTP_10GB_INCVAL >> 32;
8729                 break;
8730         case ETH_SPEED_NUM_1G:
8731                 tsync_inc_l = I40E_PTP_1GB_INCVAL & 0xFFFFFFFF;
8732                 tsync_inc_h = I40E_PTP_1GB_INCVAL >> 32;
8733                 break;
8734         default:
8735                 tsync_inc_l = 0x0;
8736                 tsync_inc_h = 0x0;
8737         }
8738
8739         /* Set the timesync increment value. */
8740         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, tsync_inc_l);
8741         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, tsync_inc_h);
8742
8743         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
8744         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
8745         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
8746
8747         adapter->systime_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8748         adapter->systime_tc.cc_shift = 0;
8749         adapter->systime_tc.nsec_mask = 0;
8750
8751         adapter->rx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8752         adapter->rx_tstamp_tc.cc_shift = 0;
8753         adapter->rx_tstamp_tc.nsec_mask = 0;
8754
8755         adapter->tx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8756         adapter->tx_tstamp_tc.cc_shift = 0;
8757         adapter->tx_tstamp_tc.nsec_mask = 0;
8758 }
8759
8760 static int
8761 i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
8762 {
8763         struct i40e_adapter *adapter =
8764                         (struct i40e_adapter *)dev->data->dev_private;
8765
8766         adapter->systime_tc.nsec += delta;
8767         adapter->rx_tstamp_tc.nsec += delta;
8768         adapter->tx_tstamp_tc.nsec += delta;
8769
8770         return 0;
8771 }
8772
8773 static int
8774 i40e_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
8775 {
8776         uint64_t ns;
8777         struct i40e_adapter *adapter =
8778                         (struct i40e_adapter *)dev->data->dev_private;
8779
8780         ns = rte_timespec_to_ns(ts);
8781
8782         /* Set the timecounters to a new value. */
8783         adapter->systime_tc.nsec = ns;
8784         adapter->rx_tstamp_tc.nsec = ns;
8785         adapter->tx_tstamp_tc.nsec = ns;
8786
8787         return 0;
8788 }
8789
8790 static int
8791 i40e_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
8792 {
8793         uint64_t ns, systime_cycles;
8794         struct i40e_adapter *adapter =
8795                         (struct i40e_adapter *)dev->data->dev_private;
8796
8797         systime_cycles = i40e_read_systime_cyclecounter(dev);
8798         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
8799         *ts = rte_ns_to_timespec(ns);
8800
8801         return 0;
8802 }
8803
8804 static int
8805 i40e_timesync_enable(struct rte_eth_dev *dev)
8806 {
8807         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8808         uint32_t tsync_ctl_l;
8809         uint32_t tsync_ctl_h;
8810
8811         /* Stop the timesync system time. */
8812         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
8813         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
8814         /* Reset the timesync system time value. */
8815         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_L, 0x0);
8816         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_H, 0x0);
8817
8818         i40e_start_timecounters(dev);
8819
8820         /* Clear timesync registers. */
8821         I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
8822         I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H);
8823         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(0));
8824         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(1));
8825         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(2));
8826         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(3));
8827
8828         /* Enable timestamping of PTP packets. */
8829         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
8830         tsync_ctl_l |= I40E_PRTTSYN_TSYNENA;
8831
8832         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
8833         tsync_ctl_h |= I40E_PRTTSYN_TSYNENA;
8834         tsync_ctl_h |= I40E_PRTTSYN_TSYNTYPE;
8835
8836         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
8837         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
8838
8839         return 0;
8840 }
8841
8842 static int
8843 i40e_timesync_disable(struct rte_eth_dev *dev)
8844 {
8845         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8846         uint32_t tsync_ctl_l;
8847         uint32_t tsync_ctl_h;
8848
8849         /* Disable timestamping of transmitted PTP packets. */
8850         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
8851         tsync_ctl_l &= ~I40E_PRTTSYN_TSYNENA;
8852
8853         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
8854         tsync_ctl_h &= ~I40E_PRTTSYN_TSYNENA;
8855
8856         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
8857         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
8858
8859         /* Reset the timesync increment value. */
8860         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
8861         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
8862
8863         return 0;
8864 }
8865
8866 static int
8867 i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
8868                                 struct timespec *timestamp, uint32_t flags)
8869 {
8870         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8871         struct i40e_adapter *adapter =
8872                 (struct i40e_adapter *)dev->data->dev_private;
8873
8874         uint32_t sync_status;
8875         uint32_t index = flags & 0x03;
8876         uint64_t rx_tstamp_cycles;
8877         uint64_t ns;
8878
8879         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_1);
8880         if ((sync_status & (1 << index)) == 0)
8881                 return -EINVAL;
8882
8883         rx_tstamp_cycles = i40e_read_rx_tstamp_cyclecounter(dev, index);
8884         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
8885         *timestamp = rte_ns_to_timespec(ns);
8886
8887         return 0;
8888 }
8889
8890 static int
8891 i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
8892                                 struct timespec *timestamp)
8893 {
8894         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8895         struct i40e_adapter *adapter =
8896                 (struct i40e_adapter *)dev->data->dev_private;
8897
8898         uint32_t sync_status;
8899         uint64_t tx_tstamp_cycles;
8900         uint64_t ns;
8901
8902         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
8903         if ((sync_status & I40E_PRTTSYN_STAT_0_TXTIME_MASK) == 0)
8904                 return -EINVAL;
8905
8906         tx_tstamp_cycles = i40e_read_tx_tstamp_cyclecounter(dev);
8907         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
8908         *timestamp = rte_ns_to_timespec(ns);
8909
8910         return 0;
8911 }
8912
8913 /*
8914  * i40e_parse_dcb_configure - parse dcb configure from user
8915  * @dev: the device being configured
8916  * @dcb_cfg: pointer of the result of parse
8917  * @*tc_map: bit map of enabled traffic classes
8918  *
8919  * Returns 0 on success, negative value on failure
8920  */
8921 static int
8922 i40e_parse_dcb_configure(struct rte_eth_dev *dev,
8923                          struct i40e_dcbx_config *dcb_cfg,
8924                          uint8_t *tc_map)
8925 {
8926         struct rte_eth_dcb_rx_conf *dcb_rx_conf;
8927         uint8_t i, tc_bw, bw_lf;
8928
8929         memset(dcb_cfg, 0, sizeof(struct i40e_dcbx_config));
8930
8931         dcb_rx_conf = &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
8932         if (dcb_rx_conf->nb_tcs > I40E_MAX_TRAFFIC_CLASS) {
8933                 PMD_INIT_LOG(ERR, "number of tc exceeds max.");
8934                 return -EINVAL;
8935         }
8936
8937         /* assume each tc has the same bw */
8938         tc_bw = I40E_MAX_PERCENT / dcb_rx_conf->nb_tcs;
8939         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
8940                 dcb_cfg->etscfg.tcbwtable[i] = tc_bw;
8941         /* to ensure the sum of tcbw is equal to 100 */
8942         bw_lf = I40E_MAX_PERCENT % dcb_rx_conf->nb_tcs;
8943         for (i = 0; i < bw_lf; i++)
8944                 dcb_cfg->etscfg.tcbwtable[i]++;
8945
8946         /* assume each tc has the same Transmission Selection Algorithm */
8947         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
8948                 dcb_cfg->etscfg.tsatable[i] = I40E_IEEE_TSA_ETS;
8949
8950         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
8951                 dcb_cfg->etscfg.prioritytable[i] =
8952                                 dcb_rx_conf->dcb_tc[i];
8953
8954         /* FW needs one App to configure HW */
8955         dcb_cfg->numapps = I40E_DEFAULT_DCB_APP_NUM;
8956         dcb_cfg->app[0].selector = I40E_APP_SEL_ETHTYPE;
8957         dcb_cfg->app[0].priority = I40E_DEFAULT_DCB_APP_PRIO;
8958         dcb_cfg->app[0].protocolid = I40E_APP_PROTOID_FCOE;
8959
8960         if (dcb_rx_conf->nb_tcs == 0)
8961                 *tc_map = 1; /* tc0 only */
8962         else
8963                 *tc_map = RTE_LEN2MASK(dcb_rx_conf->nb_tcs, uint8_t);
8964
8965         if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
8966                 dcb_cfg->pfc.willing = 0;
8967                 dcb_cfg->pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
8968                 dcb_cfg->pfc.pfcenable = *tc_map;
8969         }
8970         return 0;
8971 }
8972
8973
8974 static enum i40e_status_code
8975 i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi,
8976                               struct i40e_aqc_vsi_properties_data *info,
8977                               uint8_t enabled_tcmap)
8978 {
8979         enum i40e_status_code ret;
8980         int i, total_tc = 0;
8981         uint16_t qpnum_per_tc, bsf, qp_idx;
8982         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
8983         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
8984         uint16_t used_queues;
8985
8986         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
8987         if (ret != I40E_SUCCESS)
8988                 return ret;
8989
8990         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8991                 if (enabled_tcmap & (1 << i))
8992                         total_tc++;
8993         }
8994         if (total_tc == 0)
8995                 total_tc = 1;
8996         vsi->enabled_tc = enabled_tcmap;
8997
8998         /* different VSI has different queues assigned */
8999         if (vsi->type == I40E_VSI_MAIN)
9000                 used_queues = dev_data->nb_rx_queues -
9001                         pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
9002         else if (vsi->type == I40E_VSI_VMDQ2)
9003                 used_queues = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
9004         else {
9005                 PMD_INIT_LOG(ERR, "unsupported VSI type.");
9006                 return I40E_ERR_NO_AVAILABLE_VSI;
9007         }
9008
9009         qpnum_per_tc = used_queues / total_tc;
9010         /* Number of queues per enabled TC */
9011         if (qpnum_per_tc == 0) {
9012                 PMD_INIT_LOG(ERR, " number of queues is less that tcs.");
9013                 return I40E_ERR_INVALID_QP_ID;
9014         }
9015         qpnum_per_tc = RTE_MIN(i40e_align_floor(qpnum_per_tc),
9016                                 I40E_MAX_Q_PER_TC);
9017         bsf = rte_bsf32(qpnum_per_tc);
9018
9019         /**
9020          * Configure TC and queue mapping parameters, for enabled TC,
9021          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
9022          * default queue will serve it.
9023          */
9024         qp_idx = 0;
9025         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9026                 if (vsi->enabled_tc & (1 << i)) {
9027                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
9028                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
9029                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
9030                         qp_idx += qpnum_per_tc;
9031                 } else
9032                         info->tc_mapping[i] = 0;
9033         }
9034
9035         /* Associate queue number with VSI, Keep vsi->nb_qps unchanged */
9036         if (vsi->type == I40E_VSI_SRIOV) {
9037                 info->mapping_flags |=
9038                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
9039                 for (i = 0; i < vsi->nb_qps; i++)
9040                         info->queue_mapping[i] =
9041                                 rte_cpu_to_le_16(vsi->base_queue + i);
9042         } else {
9043                 info->mapping_flags |=
9044                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
9045                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
9046         }
9047         info->valid_sections |=
9048                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
9049
9050         return I40E_SUCCESS;
9051 }
9052
9053 /*
9054  * i40e_config_switch_comp_tc - Configure VEB tc setting for given TC map
9055  * @veb: VEB to be configured
9056  * @tc_map: enabled TC bitmap
9057  *
9058  * Returns 0 on success, negative value on failure
9059  */
9060 static enum i40e_status_code
9061 i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map)
9062 {
9063         struct i40e_aqc_configure_switching_comp_bw_config_data veb_bw;
9064         struct i40e_aqc_query_switching_comp_bw_config_resp bw_query;
9065         struct i40e_aqc_query_switching_comp_ets_config_resp ets_query;
9066         struct i40e_hw *hw = I40E_VSI_TO_HW(veb->associate_vsi);
9067         enum i40e_status_code ret = I40E_SUCCESS;
9068         int i;
9069         uint32_t bw_max;
9070
9071         /* Check if enabled_tc is same as existing or new TCs */
9072         if (veb->enabled_tc == tc_map)
9073                 return ret;
9074
9075         /* configure tc bandwidth */
9076         memset(&veb_bw, 0, sizeof(veb_bw));
9077         veb_bw.tc_valid_bits = tc_map;
9078         /* Enable ETS TCs with equal BW Share for now across all VSIs */
9079         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9080                 if (tc_map & BIT_ULL(i))
9081                         veb_bw.tc_bw_share_credits[i] = 1;
9082         }
9083         ret = i40e_aq_config_switch_comp_bw_config(hw, veb->seid,
9084                                                    &veb_bw, NULL);
9085         if (ret) {
9086                 PMD_INIT_LOG(ERR, "AQ command Config switch_comp BW allocation"
9087                                   " per TC failed = %d",
9088                                   hw->aq.asq_last_status);
9089                 return ret;
9090         }
9091
9092         memset(&ets_query, 0, sizeof(ets_query));
9093         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
9094                                                    &ets_query, NULL);
9095         if (ret != I40E_SUCCESS) {
9096                 PMD_DRV_LOG(ERR, "Failed to get switch_comp ETS"
9097                                  " configuration %u", hw->aq.asq_last_status);
9098                 return ret;
9099         }
9100         memset(&bw_query, 0, sizeof(bw_query));
9101         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
9102                                                   &bw_query, NULL);
9103         if (ret != I40E_SUCCESS) {
9104                 PMD_DRV_LOG(ERR, "Failed to get switch_comp bandwidth"
9105                                  " configuration %u", hw->aq.asq_last_status);
9106                 return ret;
9107         }
9108
9109         /* store and print out BW info */
9110         veb->bw_info.bw_limit = rte_le_to_cpu_16(ets_query.port_bw_limit);
9111         veb->bw_info.bw_max = ets_query.tc_bw_max;
9112         PMD_DRV_LOG(DEBUG, "switch_comp bw limit:%u", veb->bw_info.bw_limit);
9113         PMD_DRV_LOG(DEBUG, "switch_comp max_bw:%u", veb->bw_info.bw_max);
9114         bw_max = rte_le_to_cpu_16(bw_query.tc_bw_max[0]) |
9115                     (rte_le_to_cpu_16(bw_query.tc_bw_max[1]) <<
9116                      I40E_16_BIT_WIDTH);
9117         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9118                 veb->bw_info.bw_ets_share_credits[i] =
9119                                 bw_query.tc_bw_share_credits[i];
9120                 veb->bw_info.bw_ets_credits[i] =
9121                                 rte_le_to_cpu_16(bw_query.tc_bw_limits[i]);
9122                 /* 4 bits per TC, 4th bit is reserved */
9123                 veb->bw_info.bw_ets_max[i] =
9124                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
9125                                   RTE_LEN2MASK(3, uint8_t));
9126                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:share credits %u", i,
9127                             veb->bw_info.bw_ets_share_credits[i]);
9128                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:credits %u", i,
9129                             veb->bw_info.bw_ets_credits[i]);
9130                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u: max credits: %u", i,
9131                             veb->bw_info.bw_ets_max[i]);
9132         }
9133
9134         veb->enabled_tc = tc_map;
9135
9136         return ret;
9137 }
9138
9139
9140 /*
9141  * i40e_vsi_config_tc - Configure VSI tc setting for given TC map
9142  * @vsi: VSI to be configured
9143  * @tc_map: enabled TC bitmap
9144  *
9145  * Returns 0 on success, negative value on failure
9146  */
9147 static enum i40e_status_code
9148 i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map)
9149 {
9150         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
9151         struct i40e_vsi_context ctxt;
9152         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
9153         enum i40e_status_code ret = I40E_SUCCESS;
9154         int i;
9155
9156         /* Check if enabled_tc is same as existing or new TCs */
9157         if (vsi->enabled_tc == tc_map)
9158                 return ret;
9159
9160         /* configure tc bandwidth */
9161         memset(&bw_data, 0, sizeof(bw_data));
9162         bw_data.tc_valid_bits = tc_map;
9163         /* Enable ETS TCs with equal BW Share for now across all VSIs */
9164         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9165                 if (tc_map & BIT_ULL(i))
9166                         bw_data.tc_bw_credits[i] = 1;
9167         }
9168         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &bw_data, NULL);
9169         if (ret) {
9170                 PMD_INIT_LOG(ERR, "AQ command Config VSI BW allocation"
9171                         " per TC failed = %d",
9172                         hw->aq.asq_last_status);
9173                 goto out;
9174         }
9175         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
9176                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
9177
9178         /* Update Queue Pairs Mapping for currently enabled UPs */
9179         ctxt.seid = vsi->seid;
9180         ctxt.pf_num = hw->pf_id;
9181         ctxt.vf_num = 0;
9182         ctxt.uplink_seid = vsi->uplink_seid;
9183         ctxt.info = vsi->info;
9184         i40e_get_cap(hw);
9185         ret = i40e_vsi_update_queue_mapping(vsi, &ctxt.info, tc_map);
9186         if (ret)
9187                 goto out;
9188
9189         /* Update the VSI after updating the VSI queue-mapping information */
9190         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9191         if (ret) {
9192                 PMD_INIT_LOG(ERR, "Failed to configure "
9193                             "TC queue mapping = %d",
9194                             hw->aq.asq_last_status);
9195                 goto out;
9196         }
9197         /* update the local VSI info with updated queue map */
9198         (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
9199                                         sizeof(vsi->info.tc_mapping));
9200         (void)rte_memcpy(&vsi->info.queue_mapping,
9201                         &ctxt.info.queue_mapping,
9202                 sizeof(vsi->info.queue_mapping));
9203         vsi->info.mapping_flags = ctxt.info.mapping_flags;
9204         vsi->info.valid_sections = 0;
9205
9206         /* query and update current VSI BW information */
9207         ret = i40e_vsi_get_bw_config(vsi);
9208         if (ret) {
9209                 PMD_INIT_LOG(ERR,
9210                          "Failed updating vsi bw info, err %s aq_err %s",
9211                          i40e_stat_str(hw, ret),
9212                          i40e_aq_str(hw, hw->aq.asq_last_status));
9213                 goto out;
9214         }
9215
9216         vsi->enabled_tc = tc_map;
9217
9218 out:
9219         return ret;
9220 }
9221
9222 /*
9223  * i40e_dcb_hw_configure - program the dcb setting to hw
9224  * @pf: pf the configuration is taken on
9225  * @new_cfg: new configuration
9226  * @tc_map: enabled TC bitmap
9227  *
9228  * Returns 0 on success, negative value on failure
9229  */
9230 static enum i40e_status_code
9231 i40e_dcb_hw_configure(struct i40e_pf *pf,
9232                       struct i40e_dcbx_config *new_cfg,
9233                       uint8_t tc_map)
9234 {
9235         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
9236         struct i40e_dcbx_config *old_cfg = &hw->local_dcbx_config;
9237         struct i40e_vsi *main_vsi = pf->main_vsi;
9238         struct i40e_vsi_list *vsi_list;
9239         enum i40e_status_code ret;
9240         int i;
9241         uint32_t val;
9242
9243         /* Use the FW API if FW > v4.4*/
9244         if (!(((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4)) ||
9245               (hw->aq.fw_maj_ver >= 5))) {
9246                 PMD_INIT_LOG(ERR, "FW < v4.4, can not use FW LLDP API"
9247                                   " to configure DCB");
9248                 return I40E_ERR_FIRMWARE_API_VERSION;
9249         }
9250
9251         /* Check if need reconfiguration */
9252         if (!memcmp(new_cfg, old_cfg, sizeof(struct i40e_dcbx_config))) {
9253                 PMD_INIT_LOG(ERR, "No Change in DCB Config required.");
9254                 return I40E_SUCCESS;
9255         }
9256
9257         /* Copy the new config to the current config */
9258         *old_cfg = *new_cfg;
9259         old_cfg->etsrec = old_cfg->etscfg;
9260         ret = i40e_set_dcb_config(hw);
9261         if (ret) {
9262                 PMD_INIT_LOG(ERR,
9263                          "Set DCB Config failed, err %s aq_err %s\n",
9264                          i40e_stat_str(hw, ret),
9265                          i40e_aq_str(hw, hw->aq.asq_last_status));
9266                 return ret;
9267         }
9268         /* set receive Arbiter to RR mode and ETS scheme by default */
9269         for (i = 0; i <= I40E_PRTDCB_RETSTCC_MAX_INDEX; i++) {
9270                 val = I40E_READ_REG(hw, I40E_PRTDCB_RETSTCC(i));
9271                 val &= ~(I40E_PRTDCB_RETSTCC_BWSHARE_MASK     |
9272                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK |
9273                          I40E_PRTDCB_RETSTCC_ETSTC_SHIFT);
9274                 val |= ((uint32_t)old_cfg->etscfg.tcbwtable[i] <<
9275                         I40E_PRTDCB_RETSTCC_BWSHARE_SHIFT) &
9276                          I40E_PRTDCB_RETSTCC_BWSHARE_MASK;
9277                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_UPINTC_MODE_SHIFT) &
9278                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK;
9279                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_ETSTC_SHIFT) &
9280                          I40E_PRTDCB_RETSTCC_ETSTC_MASK;
9281                 I40E_WRITE_REG(hw, I40E_PRTDCB_RETSTCC(i), val);
9282         }
9283         /* get local mib to check whether it is configured correctly */
9284         /* IEEE mode */
9285         hw->local_dcbx_config.dcbx_mode = I40E_DCBX_MODE_IEEE;
9286         /* Get Local DCB Config */
9287         i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_LOCAL, 0,
9288                                      &hw->local_dcbx_config);
9289
9290         /* if Veb is created, need to update TC of it at first */
9291         if (main_vsi->veb) {
9292                 ret = i40e_config_switch_comp_tc(main_vsi->veb, tc_map);
9293                 if (ret)
9294                         PMD_INIT_LOG(WARNING,
9295                                  "Failed configuring TC for VEB seid=%d\n",
9296                                  main_vsi->veb->seid);
9297         }
9298         /* Update each VSI */
9299         i40e_vsi_config_tc(main_vsi, tc_map);
9300         if (main_vsi->veb) {
9301                 TAILQ_FOREACH(vsi_list, &main_vsi->veb->head, list) {
9302                         /* Beside main VSI and VMDQ VSIs, only enable default
9303                          * TC for other VSIs
9304                          */
9305                         if (vsi_list->vsi->type == I40E_VSI_VMDQ2)
9306                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
9307                                                          tc_map);
9308                         else
9309                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
9310                                                          I40E_DEFAULT_TCMAP);
9311                         if (ret)
9312                                 PMD_INIT_LOG(WARNING,
9313                                          "Failed configuring TC for VSI seid=%d\n",
9314                                          vsi_list->vsi->seid);
9315                         /* continue */
9316                 }
9317         }
9318         return I40E_SUCCESS;
9319 }
9320
9321 /*
9322  * i40e_dcb_init_configure - initial dcb config
9323  * @dev: device being configured
9324  * @sw_dcb: indicate whether dcb is sw configured or hw offload
9325  *
9326  * Returns 0 on success, negative value on failure
9327  */
9328 static int
9329 i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb)
9330 {
9331         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9332         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9333         int ret = 0;
9334
9335         if ((pf->flags & I40E_FLAG_DCB) == 0) {
9336                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
9337                 return -ENOTSUP;
9338         }
9339
9340         /* DCB initialization:
9341          * Update DCB configuration from the Firmware and configure
9342          * LLDP MIB change event.
9343          */
9344         if (sw_dcb == TRUE) {
9345                 ret = i40e_aq_stop_lldp(hw, TRUE, NULL);
9346                 if (ret != I40E_SUCCESS)
9347                         PMD_INIT_LOG(DEBUG, "Failed to stop lldp");
9348
9349                 ret = i40e_init_dcb(hw);
9350                 /* if sw_dcb, lldp agent is stopped, the return from
9351                  * i40e_init_dcb we expect is failure with I40E_AQ_RC_EPERM
9352                  * adminq status.
9353                  */
9354                 if (ret != I40E_SUCCESS &&
9355                     hw->aq.asq_last_status == I40E_AQ_RC_EPERM) {
9356                         memset(&hw->local_dcbx_config, 0,
9357                                 sizeof(struct i40e_dcbx_config));
9358                         /* set dcb default configuration */
9359                         hw->local_dcbx_config.etscfg.willing = 0;
9360                         hw->local_dcbx_config.etscfg.maxtcs = 0;
9361                         hw->local_dcbx_config.etscfg.tcbwtable[0] = 100;
9362                         hw->local_dcbx_config.etscfg.tsatable[0] =
9363                                                 I40E_IEEE_TSA_ETS;
9364                         hw->local_dcbx_config.etsrec =
9365                                 hw->local_dcbx_config.etscfg;
9366                         hw->local_dcbx_config.pfc.willing = 0;
9367                         hw->local_dcbx_config.pfc.pfccap =
9368                                                 I40E_MAX_TRAFFIC_CLASS;
9369                         /* FW needs one App to configure HW */
9370                         hw->local_dcbx_config.numapps = 1;
9371                         hw->local_dcbx_config.app[0].selector =
9372                                                 I40E_APP_SEL_ETHTYPE;
9373                         hw->local_dcbx_config.app[0].priority = 3;
9374                         hw->local_dcbx_config.app[0].protocolid =
9375                                                 I40E_APP_PROTOID_FCOE;
9376                         ret = i40e_set_dcb_config(hw);
9377                         if (ret) {
9378                                 PMD_INIT_LOG(ERR, "default dcb config fails."
9379                                         " err = %d, aq_err = %d.", ret,
9380                                           hw->aq.asq_last_status);
9381                                 return -ENOSYS;
9382                         }
9383                 } else {
9384                         PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d,"
9385                                           " aq_err = %d.", ret,
9386                                           hw->aq.asq_last_status);
9387                         return -ENOTSUP;
9388                 }
9389         } else {
9390                 ret = i40e_aq_start_lldp(hw, NULL);
9391                 if (ret != I40E_SUCCESS)
9392                         PMD_INIT_LOG(DEBUG, "Failed to start lldp");
9393
9394                 ret = i40e_init_dcb(hw);
9395                 if (!ret) {
9396                         if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) {
9397                                 PMD_INIT_LOG(ERR, "HW doesn't support"
9398                                                   " DCBX offload.");
9399                                 return -ENOTSUP;
9400                         }
9401                 } else {
9402                         PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d,"
9403                                           " aq_err = %d.", ret,
9404                                           hw->aq.asq_last_status);
9405                         return -ENOTSUP;
9406                 }
9407         }
9408         return 0;
9409 }
9410
9411 /*
9412  * i40e_dcb_setup - setup dcb related config
9413  * @dev: device being configured
9414  *
9415  * Returns 0 on success, negative value on failure
9416  */
9417 static int
9418 i40e_dcb_setup(struct rte_eth_dev *dev)
9419 {
9420         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9421         struct i40e_dcbx_config dcb_cfg;
9422         uint8_t tc_map = 0;
9423         int ret = 0;
9424
9425         if ((pf->flags & I40E_FLAG_DCB) == 0) {
9426                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
9427                 return -ENOTSUP;
9428         }
9429
9430         if (pf->vf_num != 0)
9431                 PMD_INIT_LOG(DEBUG, " DCB only works on pf and vmdq vsis.");
9432
9433         ret = i40e_parse_dcb_configure(dev, &dcb_cfg, &tc_map);
9434         if (ret) {
9435                 PMD_INIT_LOG(ERR, "invalid dcb config");
9436                 return -EINVAL;
9437         }
9438         ret = i40e_dcb_hw_configure(pf, &dcb_cfg, tc_map);
9439         if (ret) {
9440                 PMD_INIT_LOG(ERR, "dcb sw configure fails");
9441                 return -ENOSYS;
9442         }
9443
9444         return 0;
9445 }
9446
9447 static int
9448 i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
9449                       struct rte_eth_dcb_info *dcb_info)
9450 {
9451         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9452         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9453         struct i40e_vsi *vsi = pf->main_vsi;
9454         struct i40e_dcbx_config *dcb_cfg = &hw->local_dcbx_config;
9455         uint16_t bsf, tc_mapping;
9456         int i, j = 0;
9457
9458         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
9459                 dcb_info->nb_tcs = rte_bsf32(vsi->enabled_tc + 1);
9460         else
9461                 dcb_info->nb_tcs = 1;
9462         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
9463                 dcb_info->prio_tc[i] = dcb_cfg->etscfg.prioritytable[i];
9464         for (i = 0; i < dcb_info->nb_tcs; i++)
9465                 dcb_info->tc_bws[i] = dcb_cfg->etscfg.tcbwtable[i];
9466
9467         /* get queue mapping if vmdq is disabled */
9468         if (!pf->nb_cfg_vmdq_vsi) {
9469                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9470                         if (!(vsi->enabled_tc & (1 << i)))
9471                                 continue;
9472                         tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
9473                         dcb_info->tc_queue.tc_rxq[j][i].base =
9474                                 (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
9475                                 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
9476                         dcb_info->tc_queue.tc_txq[j][i].base =
9477                                 dcb_info->tc_queue.tc_rxq[j][i].base;
9478                         bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
9479                                 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
9480                         dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
9481                         dcb_info->tc_queue.tc_txq[j][i].nb_queue =
9482                                 dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
9483                 }
9484                 return 0;
9485         }
9486
9487         /* get queue mapping if vmdq is enabled */
9488         do {
9489                 vsi = pf->vmdq[j].vsi;
9490                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9491                         if (!(vsi->enabled_tc & (1 << i)))
9492                                 continue;
9493                         tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
9494                         dcb_info->tc_queue.tc_rxq[j][i].base =
9495                                 (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
9496                                 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
9497                         dcb_info->tc_queue.tc_txq[j][i].base =
9498                                 dcb_info->tc_queue.tc_rxq[j][i].base;
9499                         bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
9500                                 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
9501                         dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
9502                         dcb_info->tc_queue.tc_txq[j][i].nb_queue =
9503                                 dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
9504                 }
9505                 j++;
9506         } while (j < RTE_MIN(pf->nb_cfg_vmdq_vsi, ETH_MAX_VMDQ_POOL));
9507         return 0;
9508 }
9509
9510 static int
9511 i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
9512 {
9513         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
9514         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9515         uint16_t interval =
9516                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
9517         uint16_t msix_intr;
9518
9519         msix_intr = intr_handle->intr_vec[queue_id];
9520         if (msix_intr == I40E_MISC_VEC_ID)
9521                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
9522                                I40E_PFINT_DYN_CTLN_INTENA_MASK |
9523                                I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
9524                                (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
9525                                (interval <<
9526                                 I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
9527         else
9528                 I40E_WRITE_REG(hw,
9529                                I40E_PFINT_DYN_CTLN(msix_intr -
9530                                                    I40E_RX_VEC_START),
9531                                I40E_PFINT_DYN_CTLN_INTENA_MASK |
9532                                I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
9533                                (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
9534                                (interval <<
9535                                 I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
9536
9537         I40E_WRITE_FLUSH(hw);
9538         rte_intr_enable(&dev->pci_dev->intr_handle);
9539
9540         return 0;
9541 }
9542
9543 static int
9544 i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
9545 {
9546         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
9547         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9548         uint16_t msix_intr;
9549
9550         msix_intr = intr_handle->intr_vec[queue_id];
9551         if (msix_intr == I40E_MISC_VEC_ID)
9552                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
9553         else
9554                 I40E_WRITE_REG(hw,
9555                                I40E_PFINT_DYN_CTLN(msix_intr -
9556                                                    I40E_RX_VEC_START),
9557                                0);
9558         I40E_WRITE_FLUSH(hw);
9559
9560         return 0;
9561 }
9562
9563 static int i40e_get_regs(struct rte_eth_dev *dev,
9564                          struct rte_dev_reg_info *regs)
9565 {
9566         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9567         uint32_t *ptr_data = regs->data;
9568         uint32_t reg_idx, arr_idx, arr_idx2, reg_offset;
9569         const struct i40e_reg_info *reg_info;
9570
9571         if (ptr_data == NULL) {
9572                 regs->length = I40E_GLGEN_STAT_CLEAR + 4;
9573                 regs->width = sizeof(uint32_t);
9574                 return 0;
9575         }
9576
9577         /* The first few registers have to be read using AQ operations */
9578         reg_idx = 0;
9579         while (i40e_regs_adminq[reg_idx].name) {
9580                 reg_info = &i40e_regs_adminq[reg_idx++];
9581                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
9582                         for (arr_idx2 = 0;
9583                                         arr_idx2 <= reg_info->count2;
9584                                         arr_idx2++) {
9585                                 reg_offset = arr_idx * reg_info->stride1 +
9586                                         arr_idx2 * reg_info->stride2;
9587                                 reg_offset += reg_info->base_addr;
9588                                 ptr_data[reg_offset >> 2] =
9589                                         i40e_read_rx_ctl(hw, reg_offset);
9590                         }
9591         }
9592
9593         /* The remaining registers can be read using primitives */
9594         reg_idx = 0;
9595         while (i40e_regs_others[reg_idx].name) {
9596                 reg_info = &i40e_regs_others[reg_idx++];
9597                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
9598                         for (arr_idx2 = 0;
9599                                         arr_idx2 <= reg_info->count2;
9600                                         arr_idx2++) {
9601                                 reg_offset = arr_idx * reg_info->stride1 +
9602                                         arr_idx2 * reg_info->stride2;
9603                                 reg_offset += reg_info->base_addr;
9604                                 ptr_data[reg_offset >> 2] =
9605                                         I40E_READ_REG(hw, reg_offset);
9606                         }
9607         }
9608
9609         return 0;
9610 }
9611
9612 static int i40e_get_eeprom_length(struct rte_eth_dev *dev)
9613 {
9614         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9615
9616         /* Convert word count to byte count */
9617         return hw->nvm.sr_size << 1;
9618 }
9619
9620 static int i40e_get_eeprom(struct rte_eth_dev *dev,
9621                            struct rte_dev_eeprom_info *eeprom)
9622 {
9623         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9624         uint16_t *data = eeprom->data;
9625         uint16_t offset, length, cnt_words;
9626         int ret_code;
9627
9628         offset = eeprom->offset >> 1;
9629         length = eeprom->length >> 1;
9630         cnt_words = length;
9631
9632         if (offset > hw->nvm.sr_size ||
9633                 offset + length > hw->nvm.sr_size) {
9634                 PMD_DRV_LOG(ERR, "Requested EEPROM bytes out of range.");
9635                 return -EINVAL;
9636         }
9637
9638         eeprom->magic = hw->vendor_id | (hw->device_id << 16);
9639
9640         ret_code = i40e_read_nvm_buffer(hw, offset, &cnt_words, data);
9641         if (ret_code != I40E_SUCCESS || cnt_words != length) {
9642                 PMD_DRV_LOG(ERR, "EEPROM read failed.");
9643                 return -EIO;
9644         }
9645
9646         return 0;
9647 }
9648
9649 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
9650                                       struct ether_addr *mac_addr)
9651 {
9652         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9653
9654         if (!is_valid_assigned_ether_addr(mac_addr)) {
9655                 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
9656                 return;
9657         }
9658
9659         /* Flags: 0x3 updates port address */
9660         i40e_aq_mac_address_write(hw, 0x3, mac_addr->addr_bytes, NULL);
9661 }
9662
9663 static int
9664 i40e_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
9665 {
9666         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
9667         struct rte_eth_dev_data *dev_data = pf->dev_data;
9668         uint32_t frame_size = mtu + ETHER_HDR_LEN
9669                               + ETHER_CRC_LEN + I40E_VLAN_TAG_SIZE;
9670         int ret = 0;
9671
9672         /* check if mtu is within the allowed range */
9673         if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX))
9674                 return -EINVAL;
9675
9676         /* mtu setting is forbidden if port is start */
9677         if (dev_data->dev_started) {
9678                 PMD_DRV_LOG(ERR,
9679                             "port %d must be stopped before configuration\n",
9680                             dev_data->port_id);
9681                 return -EBUSY;
9682         }
9683
9684         if (frame_size > ETHER_MAX_LEN)
9685                 dev_data->dev_conf.rxmode.jumbo_frame = 1;
9686         else
9687                 dev_data->dev_conf.rxmode.jumbo_frame = 0;
9688
9689         dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
9690
9691         return ret;
9692 }