drivers/net/i40e/i40e_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <assert.h>
  43
  44 #include <rte_string_fns.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memzone.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_alarm.h>
  52 #include <rte_dev.h>
  53 #include <rte_eth_ctrl.h>
  54
  55 #include "i40e_logs.h"
  56 #include "base/i40e_prototype.h"
  57 #include "base/i40e_adminq_cmd.h"
  58 #include "base/i40e_type.h"
  59 #include "base/i40e_register.h"
  60 #include "base/i40e_dcb.h"
  61 #include "i40e_ethdev.h"
  62 #include "i40e_rxtx.h"
  63 #include "i40e_pf.h"
  64 #include "i40e_regs.h"
  65
  66 #define I40E_CLEAR_PXE_WAIT_MS     200
  67
  68 /* Maximun number of capability elements */
  69 #define I40E_MAX_CAP_ELE_NUM       128
  70
  71 /* Wait count and inteval */
  72 #define I40E_CHK_Q_ENA_COUNT       1000
  73 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
  74
  75 /* Maximun number of VSI */
  76 #define I40E_MAX_NUM_VSIS          (384UL)
  77
  78 #define I40E_PRE_TX_Q_CFG_WAIT_US       10 /* 10 us */
  79
  80 /* Flow control default timer */
  81 #define I40E_DEFAULT_PAUSE_TIME 0xFFFFU
  82
  83 /* Flow control default high water */
  84 #define I40E_DEFAULT_HIGH_WATER (0x1C40/1024)
  85
  86 /* Flow control default low water */
  87 #define I40E_DEFAULT_LOW_WATER  (0x1A40/1024)
  88
  89 /* Flow control enable fwd bit */
  90 #define I40E_PRTMAC_FWD_CTRL   0x00000001
  91
  92 /* Receive Packet Buffer size */
  93 #define I40E_RXPBSIZE (968 * 1024)
  94
  95 /* Kilobytes shift */
  96 #define I40E_KILOSHIFT 10
  97
  98 /* Receive Average Packet Size in Byte*/
  99 #define I40E_PACKET_AVERAGE_SIZE 128
 100
 101 /* Mask of PF interrupt causes */
 102 #define I40E_PFINT_ICR0_ENA_MASK ( \
 103                 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | \
 104                 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | \
 105                 I40E_PFINT_ICR0_ENA_GRST_MASK | \
 106                 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | \
 107                 I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK | \
 108                 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK | \
 109                 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | \
 110                 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | \
 111                 I40E_PFINT_ICR0_ENA_VFLR_MASK | \
 112                 I40E_PFINT_ICR0_ENA_ADMINQ_MASK)
 113
 114 #define I40E_FLOW_TYPES ( \
 115         (1UL << RTE_ETH_FLOW_FRAG_IPV4) | \
 116         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_TCP) | \
 117         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_UDP) | \
 118         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_SCTP) | \
 119         (1UL << RTE_ETH_FLOW_NONFRAG_IPV4_OTHER) | \
 120         (1UL << RTE_ETH_FLOW_FRAG_IPV6) | \
 121         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_TCP) | \
 122         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_UDP) | \
 123         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_SCTP) | \
 124         (1UL << RTE_ETH_FLOW_NONFRAG_IPV6_OTHER) | \
 125         (1UL << RTE_ETH_FLOW_L2_PAYLOAD))
 126
 127 /* Additional timesync values. */
 128 #define I40E_PTP_40GB_INCVAL     0x0199999999ULL
 129 #define I40E_PTP_10GB_INCVAL     0x0333333333ULL
 130 #define I40E_PTP_1GB_INCVAL      0x2000000000ULL
 131 #define I40E_PRTTSYN_TSYNENA     0x80000000
 132 #define I40E_PRTTSYN_TSYNTYPE    0x0e000000
 133 #define I40E_CYCLECOUNTER_MASK   0xffffffffffffffffULL
 134
 135 #define I40E_MAX_PERCENT            100
 136 #define I40E_DEFAULT_DCB_APP_NUM    1
 137 #define I40E_DEFAULT_DCB_APP_PRIO   3
 138
 139 #define I40E_INSET_NONE            0x00000000000000000ULL
 140
 141 /* bit0 ~ bit 7 */
 142 #define I40E_INSET_DMAC            0x0000000000000001ULL
 143 #define I40E_INSET_SMAC            0x0000000000000002ULL
 144 #define I40E_INSET_VLAN_OUTER      0x0000000000000004ULL
 145 #define I40E_INSET_VLAN_INNER      0x0000000000000008ULL
 146 #define I40E_INSET_VLAN_TUNNEL     0x0000000000000010ULL
 147
 148 /* bit 8 ~ bit 15 */
 149 #define I40E_INSET_IPV4_SRC        0x0000000000000100ULL
 150 #define I40E_INSET_IPV4_DST        0x0000000000000200ULL
 151 #define I40E_INSET_IPV6_SRC        0x0000000000000400ULL
 152 #define I40E_INSET_IPV6_DST        0x0000000000000800ULL
 153 #define I40E_INSET_SRC_PORT        0x0000000000001000ULL
 154 #define I40E_INSET_DST_PORT        0x0000000000002000ULL
 155 #define I40E_INSET_SCTP_VT         0x0000000000004000ULL
 156
 157 /* bit 16 ~ bit 31 */
 158 #define I40E_INSET_IPV4_TOS        0x0000000000010000ULL
 159 #define I40E_INSET_IPV4_PROTO      0x0000000000020000ULL
 160 #define I40E_INSET_IPV4_TTL        0x0000000000040000ULL
 161 #define I40E_INSET_IPV6_TC         0x0000000000080000ULL
 162 #define I40E_INSET_IPV6_FLOW       0x0000000000100000ULL
 163 #define I40E_INSET_IPV6_NEXT_HDR   0x0000000000200000ULL
 164 #define I40E_INSET_IPV6_HOP_LIMIT  0x0000000000400000ULL
 165 #define I40E_INSET_TCP_FLAGS       0x0000000000800000ULL
 166
 167 /* bit 32 ~ bit 47, tunnel fields */
 168 #define I40E_INSET_TUNNEL_IPV4_DST       0x0000000100000000ULL
 169 #define I40E_INSET_TUNNEL_IPV6_DST       0x0000000200000000ULL
 170 #define I40E_INSET_TUNNEL_DMAC           0x0000000400000000ULL
 171 #define I40E_INSET_TUNNEL_SRC_PORT       0x0000000800000000ULL
 172 #define I40E_INSET_TUNNEL_DST_PORT       0x0000001000000000ULL
 173 #define I40E_INSET_TUNNEL_ID             0x0000002000000000ULL
 174
 175 /* bit 48 ~ bit 55 */
 176 #define I40E_INSET_LAST_ETHER_TYPE 0x0001000000000000ULL
 177
 178 /* bit 56 ~ bit 63, Flex Payload */
 179 #define I40E_INSET_FLEX_PAYLOAD_W1 0x0100000000000000ULL
 180 #define I40E_INSET_FLEX_PAYLOAD_W2 0x0200000000000000ULL
 181 #define I40E_INSET_FLEX_PAYLOAD_W3 0x0400000000000000ULL
 182 #define I40E_INSET_FLEX_PAYLOAD_W4 0x0800000000000000ULL
 183 #define I40E_INSET_FLEX_PAYLOAD_W5 0x1000000000000000ULL
 184 #define I40E_INSET_FLEX_PAYLOAD_W6 0x2000000000000000ULL
 185 #define I40E_INSET_FLEX_PAYLOAD_W7 0x4000000000000000ULL
 186 #define I40E_INSET_FLEX_PAYLOAD_W8 0x8000000000000000ULL
 187 #define I40E_INSET_FLEX_PAYLOAD \
 188         (I40E_INSET_FLEX_PAYLOAD_W1 | I40E_INSET_FLEX_PAYLOAD_W2 | \
 189         I40E_INSET_FLEX_PAYLOAD_W3 | I40E_INSET_FLEX_PAYLOAD_W4 | \
 190         I40E_INSET_FLEX_PAYLOAD_W5 | I40E_INSET_FLEX_PAYLOAD_W6 | \
 191         I40E_INSET_FLEX_PAYLOAD_W7 | I40E_INSET_FLEX_PAYLOAD_W8)
 192
 193 /**
 194  * Below are values for writing un-exposed registers suggested
 195  * by silicon experts
 196  */
 197 /* Destination MAC address */
 198 #define I40E_REG_INSET_L2_DMAC                   0xE000000000000000ULL
 199 /* Source MAC address */
 200 #define I40E_REG_INSET_L2_SMAC                   0x1C00000000000000ULL
 201 /* VLAN tag in the outer L2 header */
 202 #define I40E_REG_INSET_L2_OUTER_VLAN             0x0080000000000000ULL
 203 /* VLAN tag in the inner L2 header */
 204 #define I40E_REG_INSET_L2_INNER_VLAN             0x0100000000000000ULL
 205 /* Source IPv4 address */
 206 #define I40E_REG_INSET_L3_SRC_IP4                0x0001800000000000ULL
 207 /* Destination IPv4 address */
 208 #define I40E_REG_INSET_L3_DST_IP4                0x0000001800000000ULL
 209 /* IPv4 Type of Service (TOS) */
 210 #define I40E_REG_INSET_L3_IP4_TOS                0x0040000000000000ULL
 211 /* IPv4 Protocol */
 212 #define I40E_REG_INSET_L3_IP4_PROTO              0x0004000000000000ULL
 213 /* Source IPv6 address */
 214 #define I40E_REG_INSET_L3_SRC_IP6                0x0007F80000000000ULL
 215 /* Destination IPv6 address */
 216 #define I40E_REG_INSET_L3_DST_IP6                0x000007F800000000ULL
 217 /* IPv6 Traffic Class (TC) */
 218 #define I40E_REG_INSET_L3_IP6_TC                 0x0040000000000000ULL
 219 /* IPv6 Next Header */
 220 #define I40E_REG_INSET_L3_IP6_NEXT_HDR           0x0008000000000000ULL
 221 /* Source L4 port */
 222 #define I40E_REG_INSET_L4_SRC_PORT               0x0000000400000000ULL
 223 /* Destination L4 port */
 224 #define I40E_REG_INSET_L4_DST_PORT               0x0000000200000000ULL
 225 /* SCTP verification tag */
 226 #define I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG  0x0000000180000000ULL
 227 /* Inner destination MAC address (MAC-in-UDP/MAC-in-GRE)*/
 228 #define I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC   0x0000000001C00000ULL
 229 /* Source port of tunneling UDP */
 230 #define I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT    0x0000000000200000ULL
 231 /* Destination port of tunneling UDP */
 232 #define I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT    0x0000000000100000ULL
 233 /* UDP Tunneling ID, NVGRE/GRE key */
 234 #define I40E_REG_INSET_TUNNEL_ID                 0x00000000000C0000ULL
 235 /* Last ether type */
 236 #define I40E_REG_INSET_LAST_ETHER_TYPE           0x0000000000004000ULL
 237 /* Tunneling outer destination IPv4 address */
 238 #define I40E_REG_INSET_TUNNEL_L3_DST_IP4         0x00000000000000C0ULL
 239 /* Tunneling outer destination IPv6 address */
 240 #define I40E_REG_INSET_TUNNEL_L3_DST_IP6         0x0000000000003FC0ULL
 241 /* 1st word of flex payload */
 242 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD1        0x0000000000002000ULL
 243 /* 2nd word of flex payload */
 244 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD2        0x0000000000001000ULL
 245 /* 3rd word of flex payload */
 246 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD3        0x0000000000000800ULL
 247 /* 4th word of flex payload */
 248 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD4        0x0000000000000400ULL
 249 /* 5th word of flex payload */
 250 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD5        0x0000000000000200ULL
 251 /* 6th word of flex payload */
 252 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD6        0x0000000000000100ULL
 253 /* 7th word of flex payload */
 254 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD7        0x0000000000000080ULL
 255 /* 8th word of flex payload */
 256 #define I40E_REG_INSET_FLEX_PAYLOAD_WORD8        0x0000000000000040ULL
 257
 258 #define I40E_REG_INSET_MASK_DEFAULT              0x0000000000000000ULL
 259
 260 #define I40E_TRANSLATE_INSET 0
 261 #define I40E_TRANSLATE_REG   1
 262
 263 #define I40E_INSET_IPV4_TOS_MASK      0x0009FF00UL
 264 #define I40E_INSET_IPV4_PROTO_MASK    0x000DFF00UL
 265 #define I40E_INSET_IPV6_TC_MASK       0x0009F00FUL
 266 #define I40E_INSET_IPV6_NEXT_HDR_MASK 0x000C00FFUL
 267
 268 #define I40E_GL_SWT_L2TAGCTRL(_i)             (0x001C0A70 + ((_i) * 4))
 269 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT 16
 270 #define I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK  \
 271         I40E_MASK(0xFFFF, I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT)
 272
 273 /* PCI offset for querying capability */
 274 #define PCI_DEV_CAP_REG            0xA4
 275 /* PCI offset for enabling/disabling Extended Tag */
 276 #define PCI_DEV_CTRL_REG           0xA8
 277 /* Bit mask of Extended Tag capability */
 278 #define PCI_DEV_CAP_EXT_TAG_MASK   0x20
 279 /* Bit shift of Extended Tag enable/disable */
 280 #define PCI_DEV_CTRL_EXT_TAG_SHIFT 8
 281 /* Bit mask of Extended Tag enable/disable */
 282 #define PCI_DEV_CTRL_EXT_TAG_MASK  (1 << PCI_DEV_CTRL_EXT_TAG_SHIFT)
 283
 284 static int eth_i40e_dev_init(struct rte_eth_dev *eth_dev);
 285 static int eth_i40e_dev_uninit(struct rte_eth_dev *eth_dev);
 286 static int i40e_dev_configure(struct rte_eth_dev *dev);
 287 static int i40e_dev_start(struct rte_eth_dev *dev);
 288 static void i40e_dev_stop(struct rte_eth_dev *dev);
 289 static void i40e_dev_close(struct rte_eth_dev *dev);
 290 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
 291 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
 292 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
 293 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
 294 static int i40e_dev_set_link_up(struct rte_eth_dev *dev);
 295 static int i40e_dev_set_link_down(struct rte_eth_dev *dev);
 296 static void i40e_dev_stats_get(struct rte_eth_dev *dev,
 297                                struct rte_eth_stats *stats);
 298 static int i40e_dev_xstats_get(struct rte_eth_dev *dev,
 299                                struct rte_eth_xstats *xstats, unsigned n);
 300 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
 301 static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev,
 302                                             uint16_t queue_id,
 303                                             uint8_t stat_idx,
 304                                             uint8_t is_rx);
 305 static void i40e_dev_info_get(struct rte_eth_dev *dev,
 306                               struct rte_eth_dev_info *dev_info);
 307 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
 308                                 uint16_t vlan_id,
 309                                 int on);
 310 static int i40e_vlan_tpid_set(struct rte_eth_dev *dev,
 311                               enum rte_vlan_type vlan_type,
 312                               uint16_t tpid);
 313 static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 314 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
 315                                       uint16_t queue,
 316                                       int on);
 317 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
 318 static int i40e_dev_led_on(struct rte_eth_dev *dev);
 319 static int i40e_dev_led_off(struct rte_eth_dev *dev);
 320 static int i40e_flow_ctrl_get(struct rte_eth_dev *dev,
 321                               struct rte_eth_fc_conf *fc_conf);
 322 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
 323                               struct rte_eth_fc_conf *fc_conf);
 324 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 325                                        struct rte_eth_pfc_conf *pfc_conf);
 326 static void i40e_macaddr_add(struct rte_eth_dev *dev,
 327                           struct ether_addr *mac_addr,
 328                           uint32_t index,
 329                           uint32_t pool);
 330 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 331 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
 332                                     struct rte_eth_rss_reta_entry64 *reta_conf,
 333                                     uint16_t reta_size);
 334 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
 335                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 336                                    uint16_t reta_size);
 337
 338 static int i40e_get_cap(struct i40e_hw *hw);
 339 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
 340 static int i40e_pf_setup(struct i40e_pf *pf);
 341 static int i40e_dev_rxtx_init(struct i40e_pf *pf);
 342 static int i40e_vmdq_setup(struct rte_eth_dev *dev);
 343 static int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb);
 344 static int i40e_dcb_setup(struct rte_eth_dev *dev);
 345 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
 346                 bool offset_loaded, uint64_t *offset, uint64_t *stat);
 347 static void i40e_stat_update_48(struct i40e_hw *hw,
 348                                uint32_t hireg,
 349                                uint32_t loreg,
 350                                bool offset_loaded,
 351                                uint64_t *offset,
 352                                uint64_t *stat);
 353 static void i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue);
 354 static void i40e_dev_interrupt_handler(
 355                 __rte_unused struct rte_intr_handle *handle, void *param);
 356 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 357                                 uint32_t base, uint32_t num);
 358 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
 359 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
 360                         uint32_t base);
 361 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
 362                         uint16_t num);
 363 static int i40e_dev_init_vlan(struct rte_eth_dev *dev);
 364 static int i40e_veb_release(struct i40e_veb *veb);
 365 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
 366                                                 struct i40e_vsi *vsi);
 367 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
 368 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
 369 static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
 370                                              struct i40e_macvlan_filter *mv_f,
 371                                              int num,
 372                                              struct ether_addr *addr);
 373 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
 374                                              struct i40e_macvlan_filter *mv_f,
 375                                              int num,
 376                                              uint16_t vlan);
 377 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
 378 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
 379                                     struct rte_eth_rss_conf *rss_conf);
 380 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 381                                       struct rte_eth_rss_conf *rss_conf);
 382 static int i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
 383                                         struct rte_eth_udp_tunnel *udp_tunnel);
 384 static int i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
 385                                         struct rte_eth_udp_tunnel *udp_tunnel);
 386 static int i40e_ethertype_filter_set(struct i40e_pf *pf,
 387                         struct rte_eth_ethertype_filter *filter,
 388                         bool add);
 389 static int i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
 390                                 enum rte_filter_op filter_op,
 391                                 void *arg);
 392 static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
 393                                 enum rte_filter_type filter_type,
 394                                 enum rte_filter_op filter_op,
 395                                 void *arg);
 396 static int i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
 397                                   struct rte_eth_dcb_info *dcb_info);
 398 static void i40e_configure_registers(struct i40e_hw *hw);
 399 static void i40e_hw_init(struct rte_eth_dev *dev);
 400 static int i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi);
 401 static int i40e_mirror_rule_set(struct rte_eth_dev *dev,
 402                         struct rte_eth_mirror_conf *mirror_conf,
 403                         uint8_t sw_id, uint8_t on);
 404 static int i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id);
 405
 406 static int i40e_timesync_enable(struct rte_eth_dev *dev);
 407 static int i40e_timesync_disable(struct rte_eth_dev *dev);
 408 static int i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 409                                            struct timespec *timestamp,
 410                                            uint32_t flags);
 411 static int i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 412                                            struct timespec *timestamp);
 413 static void i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw);
 414
 415 static int i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 416
 417 static int i40e_timesync_read_time(struct rte_eth_dev *dev,
 418                                    struct timespec *timestamp);
 419 static int i40e_timesync_write_time(struct rte_eth_dev *dev,
 420                                     const struct timespec *timestamp);
 421
 422 static int i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 423                                          uint16_t queue_id);
 424 static int i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 425                                           uint16_t queue_id);
 426
 427 static int i40e_get_reg_length(struct rte_eth_dev *dev);
 428
 429 static int i40e_get_regs(struct rte_eth_dev *dev,
 430                          struct rte_dev_reg_info *regs);
 431
 432 static int i40e_get_eeprom_length(struct rte_eth_dev *dev);
 433
 434 static int i40e_get_eeprom(struct rte_eth_dev *dev,
 435                            struct rte_dev_eeprom_info *eeprom);
 436
 437 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
 438                                       struct ether_addr *mac_addr);
 439
 440 static const struct rte_pci_id pci_id_i40e_map[] = {
 441 #define RTE_PCI_DEV_ID_DECL_I40E(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 442 #include "rte_pci_dev_ids.h"
 443 { .vendor_id = 0, /* sentinel */ },
 444 };
 445
 446 static const struct eth_dev_ops i40e_eth_dev_ops = {
 447         .dev_configure                = i40e_dev_configure,
 448         .dev_start                    = i40e_dev_start,
 449         .dev_stop                     = i40e_dev_stop,
 450         .dev_close                    = i40e_dev_close,
 451         .promiscuous_enable           = i40e_dev_promiscuous_enable,
 452         .promiscuous_disable          = i40e_dev_promiscuous_disable,
 453         .allmulticast_enable          = i40e_dev_allmulticast_enable,
 454         .allmulticast_disable         = i40e_dev_allmulticast_disable,
 455         .dev_set_link_up              = i40e_dev_set_link_up,
 456         .dev_set_link_down            = i40e_dev_set_link_down,
 457         .link_update                  = i40e_dev_link_update,
 458         .stats_get                    = i40e_dev_stats_get,
 459         .xstats_get                   = i40e_dev_xstats_get,
 460         .stats_reset                  = i40e_dev_stats_reset,
 461         .xstats_reset                 = i40e_dev_stats_reset,
 462         .queue_stats_mapping_set      = i40e_dev_queue_stats_mapping_set,
 463         .dev_infos_get                = i40e_dev_info_get,
 464         .vlan_filter_set              = i40e_vlan_filter_set,
 465         .vlan_tpid_set                = i40e_vlan_tpid_set,
 466         .vlan_offload_set             = i40e_vlan_offload_set,
 467         .vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
 468         .vlan_pvid_set                = i40e_vlan_pvid_set,
 469         .rx_queue_start               = i40e_dev_rx_queue_start,
 470         .rx_queue_stop                = i40e_dev_rx_queue_stop,
 471         .tx_queue_start               = i40e_dev_tx_queue_start,
 472         .tx_queue_stop                = i40e_dev_tx_queue_stop,
 473         .rx_queue_setup               = i40e_dev_rx_queue_setup,
 474         .rx_queue_intr_enable         = i40e_dev_rx_queue_intr_enable,
 475         .rx_queue_intr_disable        = i40e_dev_rx_queue_intr_disable,
 476         .rx_queue_release             = i40e_dev_rx_queue_release,
 477         .rx_queue_count               = i40e_dev_rx_queue_count,
 478         .rx_descriptor_done           = i40e_dev_rx_descriptor_done,
 479         .tx_queue_setup               = i40e_dev_tx_queue_setup,
 480         .tx_queue_release             = i40e_dev_tx_queue_release,
 481         .dev_led_on                   = i40e_dev_led_on,
 482         .dev_led_off                  = i40e_dev_led_off,
 483         .flow_ctrl_get                = i40e_flow_ctrl_get,
 484         .flow_ctrl_set                = i40e_flow_ctrl_set,
 485         .priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
 486         .mac_addr_add                 = i40e_macaddr_add,
 487         .mac_addr_remove              = i40e_macaddr_remove,
 488         .reta_update                  = i40e_dev_rss_reta_update,
 489         .reta_query                   = i40e_dev_rss_reta_query,
 490         .rss_hash_update              = i40e_dev_rss_hash_update,
 491         .rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
 492         .udp_tunnel_port_add          = i40e_dev_udp_tunnel_port_add,
 493         .udp_tunnel_port_del          = i40e_dev_udp_tunnel_port_del,
 494         .filter_ctrl                  = i40e_dev_filter_ctrl,
 495         .rxq_info_get                 = i40e_rxq_info_get,
 496         .txq_info_get                 = i40e_txq_info_get,
 497         .mirror_rule_set              = i40e_mirror_rule_set,
 498         .mirror_rule_reset            = i40e_mirror_rule_reset,
 499         .timesync_enable              = i40e_timesync_enable,
 500         .timesync_disable             = i40e_timesync_disable,
 501         .timesync_read_rx_timestamp   = i40e_timesync_read_rx_timestamp,
 502         .timesync_read_tx_timestamp   = i40e_timesync_read_tx_timestamp,
 503         .get_dcb_info                 = i40e_dev_get_dcb_info,
 504         .timesync_adjust_time         = i40e_timesync_adjust_time,
 505         .timesync_read_time           = i40e_timesync_read_time,
 506         .timesync_write_time          = i40e_timesync_write_time,
 507         .get_reg_length               = i40e_get_reg_length,
 508         .get_reg                      = i40e_get_regs,
 509         .get_eeprom_length            = i40e_get_eeprom_length,
 510         .get_eeprom                   = i40e_get_eeprom,
 511         .mac_addr_set                 = i40e_set_default_mac_addr,
 512 };
 513
 514 /* store statistics names and its offset in stats structure */
 515 struct rte_i40e_xstats_name_off {
 516         char name[RTE_ETH_XSTATS_NAME_SIZE];
 517         unsigned offset;
 518 };
 519
 520 static const struct rte_i40e_xstats_name_off rte_i40e_stats_strings[] = {
 521         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 522         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 523         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 524         {"rx_dropped", offsetof(struct i40e_eth_stats, rx_discards)},
 525         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 526                 rx_unknown_protocol)},
 527         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)},
 528         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)},
 529         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)},
 530         {"tx_dropped", offsetof(struct i40e_eth_stats, tx_discards)},
 531 };
 532
 533 #define I40E_NB_ETH_XSTATS (sizeof(rte_i40e_stats_strings) / \
 534                 sizeof(rte_i40e_stats_strings[0]))
 535
 536 static const struct rte_i40e_xstats_name_off rte_i40e_hw_port_strings[] = {
 537         {"tx_link_down_dropped", offsetof(struct i40e_hw_port_stats,
 538                 tx_dropped_link_down)},
 539         {"rx_crc_errors", offsetof(struct i40e_hw_port_stats, crc_errors)},
 540         {"rx_illegal_byte_errors", offsetof(struct i40e_hw_port_stats,
 541                 illegal_bytes)},
 542         {"rx_error_bytes", offsetof(struct i40e_hw_port_stats, error_bytes)},
 543         {"mac_local_errors", offsetof(struct i40e_hw_port_stats,
 544                 mac_local_faults)},
 545         {"mac_remote_errors", offsetof(struct i40e_hw_port_stats,
 546                 mac_remote_faults)},
 547         {"rx_length_errors", offsetof(struct i40e_hw_port_stats,
 548                 rx_length_errors)},
 549         {"tx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_tx)},
 550         {"rx_xon_packets", offsetof(struct i40e_hw_port_stats, link_xon_rx)},
 551         {"tx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_tx)},
 552         {"rx_xoff_packets", offsetof(struct i40e_hw_port_stats, link_xoff_rx)},
 553         {"rx_size_64_packets", offsetof(struct i40e_hw_port_stats, rx_size_64)},
 554         {"rx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 555                 rx_size_127)},
 556         {"rx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 557                 rx_size_255)},
 558         {"rx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 559                 rx_size_511)},
 560         {"rx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 561                 rx_size_1023)},
 562         {"rx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 563                 rx_size_1522)},
 564         {"rx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 565                 rx_size_big)},
 566         {"rx_undersized_errors", offsetof(struct i40e_hw_port_stats,
 567                 rx_undersize)},
 568         {"rx_oversize_errors", offsetof(struct i40e_hw_port_stats,
 569                 rx_oversize)},
 570         {"rx_mac_short_dropped", offsetof(struct i40e_hw_port_stats,
 571                 mac_short_packet_dropped)},
 572         {"rx_fragmented_errors", offsetof(struct i40e_hw_port_stats,
 573                 rx_fragments)},
 574         {"rx_jabber_errors", offsetof(struct i40e_hw_port_stats, rx_jabber)},
 575         {"tx_size_64_packets", offsetof(struct i40e_hw_port_stats, tx_size_64)},
 576         {"tx_size_65_to_127_packets", offsetof(struct i40e_hw_port_stats,
 577                 tx_size_127)},
 578         {"tx_size_128_to_255_packets", offsetof(struct i40e_hw_port_stats,
 579                 tx_size_255)},
 580         {"tx_size_256_to_511_packets", offsetof(struct i40e_hw_port_stats,
 581                 tx_size_511)},
 582         {"tx_size_512_to_1023_packets", offsetof(struct i40e_hw_port_stats,
 583                 tx_size_1023)},
 584         {"tx_size_1024_to_1522_packets", offsetof(struct i40e_hw_port_stats,
 585                 tx_size_1522)},
 586         {"tx_size_1523_to_max_packets", offsetof(struct i40e_hw_port_stats,
 587                 tx_size_big)},
 588         {"rx_flow_director_atr_match_packets",
 589                 offsetof(struct i40e_hw_port_stats, fd_atr_match)},
 590         {"rx_flow_director_sb_match_packets",
 591                 offsetof(struct i40e_hw_port_stats, fd_sb_match)},
 592         {"tx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 593                 tx_lpi_status)},
 594         {"rx_low_power_idle_status", offsetof(struct i40e_hw_port_stats,
 595                 rx_lpi_status)},
 596         {"tx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 597                 tx_lpi_count)},
 598         {"rx_low_power_idle_count", offsetof(struct i40e_hw_port_stats,
 599                 rx_lpi_count)},
 600 };
 601
 602 #define I40E_NB_HW_PORT_XSTATS (sizeof(rte_i40e_hw_port_strings) / \
 603                 sizeof(rte_i40e_hw_port_strings[0]))
 604
 605 static const struct rte_i40e_xstats_name_off rte_i40e_rxq_prio_strings[] = {
 606         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 607                 priority_xon_rx)},
 608         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 609                 priority_xoff_rx)},
 610 };
 611
 612 #define I40E_NB_RXQ_PRIO_XSTATS (sizeof(rte_i40e_rxq_prio_strings) / \
 613                 sizeof(rte_i40e_rxq_prio_strings[0]))
 614
 615 static const struct rte_i40e_xstats_name_off rte_i40e_txq_prio_strings[] = {
 616         {"xon_packets", offsetof(struct i40e_hw_port_stats,
 617                 priority_xon_tx)},
 618         {"xoff_packets", offsetof(struct i40e_hw_port_stats,
 619                 priority_xoff_tx)},
 620         {"xon_to_xoff_packets", offsetof(struct i40e_hw_port_stats,
 621                 priority_xon_2_xoff)},
 622 };
 623
 624 #define I40E_NB_TXQ_PRIO_XSTATS (sizeof(rte_i40e_txq_prio_strings) / \
 625                 sizeof(rte_i40e_txq_prio_strings[0]))
 626
 627 static struct eth_driver rte_i40e_pmd = {
 628         .pci_drv = {
 629                 .name = "rte_i40e_pmd",
 630                 .id_table = pci_id_i40e_map,
 631                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 632                         RTE_PCI_DRV_DETACHABLE,
 633         },
 634         .eth_dev_init = eth_i40e_dev_init,
 635         .eth_dev_uninit = eth_i40e_dev_uninit,
 636         .dev_private_size = sizeof(struct i40e_adapter),
 637 };
 638
 639 static inline int
 640 rte_i40e_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 641                                      struct rte_eth_link *link)
 642 {
 643         struct rte_eth_link *dst = link;
 644         struct rte_eth_link *src = &(dev->data->dev_link);
 645
 646         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 647                                         *(uint64_t *)src) == 0)
 648                 return -1;
 649
 650         return 0;
 651 }
 652
 653 static inline int
 654 rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 655                                       struct rte_eth_link *link)
 656 {
 657         struct rte_eth_link *dst = &(dev->data->dev_link);
 658         struct rte_eth_link *src = link;
 659
 660         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 661                                         *(uint64_t *)src) == 0)
 662                 return -1;
 663
 664         return 0;
 665 }
 666
 667 /*
 668  * Driver initialization routine.
 669  * Invoked once at EAL init time.
 670  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 671  */
 672 static int
 673 rte_i40e_pmd_init(const char *name __rte_unused,
 674                   const char *params __rte_unused)
 675 {
 676         PMD_INIT_FUNC_TRACE();
 677         rte_eth_driver_register(&rte_i40e_pmd);
 678
 679         return 0;
 680 }
 681
 682 static struct rte_driver rte_i40e_driver = {
 683         .type = PMD_PDEV,
 684         .init = rte_i40e_pmd_init,
 685 };
 686
 687 PMD_REGISTER_DRIVER(rte_i40e_driver);
 688
 689 /*
 690  * Initialize registers for flexible payload, which should be set by NVM.
 691  * This should be removed from code once it is fixed in NVM.
 692  */
 693 #ifndef I40E_GLQF_ORT
 694 #define I40E_GLQF_ORT(_i)    (0x00268900 + ((_i) * 4))
 695 #endif
 696 #ifndef I40E_GLQF_PIT
 697 #define I40E_GLQF_PIT(_i)    (0x00268C80 + ((_i) * 4))
 698 #endif
 699
 700 static inline void i40e_flex_payload_reg_init(struct i40e_hw *hw)
 701 {
 702         I40E_WRITE_REG(hw, I40E_GLQF_ORT(18), 0x00000030);
 703         I40E_WRITE_REG(hw, I40E_GLQF_ORT(19), 0x00000030);
 704         I40E_WRITE_REG(hw, I40E_GLQF_ORT(26), 0x0000002B);
 705         I40E_WRITE_REG(hw, I40E_GLQF_ORT(30), 0x0000002B);
 706         I40E_WRITE_REG(hw, I40E_GLQF_ORT(33), 0x000000E0);
 707         I40E_WRITE_REG(hw, I40E_GLQF_ORT(34), 0x000000E3);
 708         I40E_WRITE_REG(hw, I40E_GLQF_ORT(35), 0x000000E6);
 709         I40E_WRITE_REG(hw, I40E_GLQF_ORT(20), 0x00000031);
 710         I40E_WRITE_REG(hw, I40E_GLQF_ORT(23), 0x00000031);
 711         I40E_WRITE_REG(hw, I40E_GLQF_ORT(63), 0x0000002D);
 712
 713         /* GLQF_PIT Registers */
 714         I40E_WRITE_REG(hw, I40E_GLQF_PIT(16), 0x00007480);
 715         I40E_WRITE_REG(hw, I40E_GLQF_PIT(17), 0x00007440);
 716 }
 717
 718 #define I40E_FLOW_CONTROL_ETHERTYPE  0x8808
 719
 720 /*
 721  * Add a ethertype filter to drop all flow control frames transmitted
 722  * from VSIs.
 723 */
 724 static void
 725 i40e_add_tx_flow_control_drop_filter(struct i40e_pf *pf)
 726 {
 727         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
 728         uint16_t flags = I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC |
 729                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP |
 730                         I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX;
 731         int ret;
 732
 733         ret = i40e_aq_add_rem_control_packet_filter(hw, NULL,
 734                                 I40E_FLOW_CONTROL_ETHERTYPE, flags,
 735                                 pf->main_vsi_seid, 0,
 736                                 TRUE, NULL, NULL);
 737         if (ret)
 738                 PMD_INIT_LOG(ERR, "Failed to add filter to drop flow control "
 739                                   " frames from VSIs.");
 740 }
 741
 742 static int
 743 eth_i40e_dev_init(struct rte_eth_dev *dev)
 744 {
 745         struct rte_pci_device *pci_dev;
 746         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 747         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 748         struct i40e_vsi *vsi;
 749         int ret;
 750         uint32_t len;
 751         uint8_t aq_fail = 0;
 752
 753         PMD_INIT_FUNC_TRACE();
 754
 755         dev->dev_ops = &i40e_eth_dev_ops;
 756         dev->rx_pkt_burst = i40e_recv_pkts;
 757         dev->tx_pkt_burst = i40e_xmit_pkts;
 758
 759         /* for secondary processes, we don't initialise any further as primary
 760          * has already done this work. Only check we don't need a different
 761          * RX function */
 762         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 763                 i40e_set_rx_function(dev);
 764                 i40e_set_tx_function(dev);
 765                 return 0;
 766         }
 767         pci_dev = dev->pci_dev;
 768
 769         rte_eth_copy_pci_info(dev, pci_dev);
 770
 771         pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 772         pf->adapter->eth_dev = dev;
 773         pf->dev_data = dev->data;
 774
 775         hw->back = I40E_PF_TO_ADAPTER(pf);
 776         hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
 777         if (!hw->hw_addr) {
 778                 PMD_INIT_LOG(ERR, "Hardware is not available, "
 779                              "as address is NULL");
 780                 return -ENODEV;
 781         }
 782
 783         hw->vendor_id = pci_dev->id.vendor_id;
 784         hw->device_id = pci_dev->id.device_id;
 785         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 786         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 787         hw->bus.device = pci_dev->addr.devid;
 788         hw->bus.func = pci_dev->addr.function;
 789         hw->adapter_stopped = 0;
 790
 791         /* Make sure all is clean before doing PF reset */
 792         i40e_clear_hw(hw);
 793
 794         /* Initialize the hardware */
 795         i40e_hw_init(dev);
 796
 797         /* Reset here to make sure all is clean for each PF */
 798         ret = i40e_pf_reset(hw);
 799         if (ret) {
 800                 PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
 801                 return ret;
 802         }
 803
 804         /* Initialize the shared code (base driver) */
 805         ret = i40e_init_shared_code(hw);
 806         if (ret) {
 807                 PMD_INIT_LOG(ERR, "Failed to init shared code (base driver): %d", ret);
 808                 return ret;
 809         }
 810
 811         /*
 812          * To work around the NVM issue,initialize registers
 813          * for flexible payload by software.
 814          * It should be removed once issues are fixed in NVM.
 815          */
 816         i40e_flex_payload_reg_init(hw);
 817
 818         /* Initialize the parameters for adminq */
 819         i40e_init_adminq_parameter(hw);
 820         ret = i40e_init_adminq(hw);
 821         if (ret != I40E_SUCCESS) {
 822                 PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
 823                 return -EIO;
 824         }
 825         PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
 826                      hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
 827                      hw->aq.api_maj_ver, hw->aq.api_min_ver,
 828                      ((hw->nvm.version >> 12) & 0xf),
 829                      ((hw->nvm.version >> 4) & 0xff),
 830                      (hw->nvm.version & 0xf), hw->nvm.eetrack);
 831
 832         /* Clear PXE mode */
 833         i40e_clear_pxe_mode(hw);
 834
 835         /*
 836          * On X710, performance number is far from the expectation on recent
 837          * firmware versions. The fix for this issue may not be integrated in
 838          * the following firmware version. So the workaround in software driver
 839          * is needed. It needs to modify the initial values of 3 internal only
 840          * registers. Note that the workaround can be removed when it is fixed
 841          * in firmware in the future.
 842          */
 843         i40e_configure_registers(hw);
 844
 845         /* Get hw capabilities */
 846         ret = i40e_get_cap(hw);
 847         if (ret != I40E_SUCCESS) {
 848                 PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
 849                 goto err_get_capabilities;
 850         }
 851
 852         /* Initialize parameters for PF */
 853         ret = i40e_pf_parameter_init(dev);
 854         if (ret != 0) {
 855                 PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
 856                 goto err_parameter_init;
 857         }
 858
 859         /* Initialize the queue management */
 860         ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
 861         if (ret < 0) {
 862                 PMD_INIT_LOG(ERR, "Failed to init queue pool");
 863                 goto err_qp_pool_init;
 864         }
 865         ret = i40e_res_pool_init(&pf->msix_pool, 1,
 866                                 hw->func_caps.num_msix_vectors - 1);
 867         if (ret < 0) {
 868                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
 869                 goto err_msix_pool_init;
 870         }
 871
 872         /* Initialize lan hmc */
 873         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 874                                 hw->func_caps.num_rx_qp, 0, 0);
 875         if (ret != I40E_SUCCESS) {
 876                 PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
 877                 goto err_init_lan_hmc;
 878         }
 879
 880         /* Configure lan hmc */
 881         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 882         if (ret != I40E_SUCCESS) {
 883                 PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
 884                 goto err_configure_lan_hmc;
 885         }
 886
 887         /* Get and check the mac address */
 888         i40e_get_mac_addr(hw, hw->mac.addr);
 889         if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
 890                 PMD_INIT_LOG(ERR, "mac address is not valid");
 891                 ret = -EIO;
 892                 goto err_get_mac_addr;
 893         }
 894         /* Copy the permanent MAC address */
 895         ether_addr_copy((struct ether_addr *) hw->mac.addr,
 896                         (struct ether_addr *) hw->mac.perm_addr);
 897
 898         /* Disable flow control */
 899         hw->fc.requested_mode = I40E_FC_NONE;
 900         i40e_set_fc(hw, &aq_fail, TRUE);
 901
 902         /* Set the global registers with default ether type value */
 903         ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_OUTER, ETHER_TYPE_VLAN);
 904         if (ret != I40E_SUCCESS) {
 905                 PMD_INIT_LOG(ERR, "Failed to set the default outer "
 906                              "VLAN ether type");
 907                 goto err_setup_pf_switch;
 908         }
 909         ret = i40e_vlan_tpid_set(dev, ETH_VLAN_TYPE_INNER, ETHER_TYPE_VLAN);
 910         if (ret != I40E_SUCCESS) {
 911                 PMD_INIT_LOG(ERR, "Failed to set the default outer "
 912                              "VLAN ether type");
 913                 goto err_setup_pf_switch;
 914         }
 915
 916         /* PF setup, which includes VSI setup */
 917         ret = i40e_pf_setup(pf);
 918         if (ret) {
 919                 PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
 920                 goto err_setup_pf_switch;
 921         }
 922
 923         vsi = pf->main_vsi;
 924
 925         /* Disable double vlan by default */
 926         i40e_vsi_config_double_vlan(vsi, FALSE);
 927
 928         if (!vsi->max_macaddrs)
 929                 len = ETHER_ADDR_LEN;
 930         else
 931                 len = ETHER_ADDR_LEN * vsi->max_macaddrs;
 932
 933         /* Should be after VSI initialized */
 934         dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
 935         if (!dev->data->mac_addrs) {
 936                 PMD_INIT_LOG(ERR, "Failed to allocated memory "
 937                                         "for storing mac address");
 938                 goto err_mac_alloc;
 939         }
 940         ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
 941                                         &dev->data->mac_addrs[0]);
 942
 943         /* initialize pf host driver to setup SRIOV resource if applicable */
 944         i40e_pf_host_init(dev);
 945
 946         /* register callback func to eal lib */
 947         rte_intr_callback_register(&(pci_dev->intr_handle),
 948                 i40e_dev_interrupt_handler, (void *)dev);
 949
 950         /* configure and enable device interrupt */
 951         i40e_pf_config_irq0(hw, TRUE);
 952         i40e_pf_enable_irq0(hw);
 953
 954         /* enable uio intr after callback register */
 955         rte_intr_enable(&(pci_dev->intr_handle));
 956         /*
 957          * Add an ethertype filter to drop all flow control frames transmitted
 958          * from VSIs. By doing so, we stop VF from sending out PAUSE or PFC
 959          * frames to wire.
 960          */
 961         i40e_add_tx_flow_control_drop_filter(pf);
 962
 963         /* Set the max frame size to 0x2600 by default,
 964          * in case other drivers changed the default value.
 965          */
 966         i40e_aq_set_mac_config(hw, I40E_FRAME_SIZE_MAX, TRUE, 0, NULL);
 967
 968         /* initialize mirror rule list */
 969         TAILQ_INIT(&pf->mirror_list);
 970
 971         /* Init dcb to sw mode by default */
 972         ret = i40e_dcb_init_configure(dev, TRUE);
 973         if (ret != I40E_SUCCESS) {
 974                 PMD_INIT_LOG(INFO, "Failed to init dcb.");
 975                 pf->flags &= ~I40E_FLAG_DCB;
 976         }
 977
 978         return 0;
 979
 980 err_mac_alloc:
 981         i40e_vsi_release(pf->main_vsi);
 982 err_setup_pf_switch:
 983 err_get_mac_addr:
 984 err_configure_lan_hmc:
 985         (void)i40e_shutdown_lan_hmc(hw);
 986 err_init_lan_hmc:
 987         i40e_res_pool_destroy(&pf->msix_pool);
 988 err_msix_pool_init:
 989         i40e_res_pool_destroy(&pf->qp_pool);
 990 err_qp_pool_init:
 991 err_parameter_init:
 992 err_get_capabilities:
 993         (void)i40e_shutdown_adminq(hw);
 994
 995         return ret;
 996 }
 997
 998 static int
 999 eth_i40e_dev_uninit(struct rte_eth_dev *dev)
1000 {
1001         struct rte_pci_device *pci_dev;
1002         struct i40e_hw *hw;
1003         struct i40e_filter_control_settings settings;
1004         int ret;
1005         uint8_t aq_fail = 0;
1006
1007         PMD_INIT_FUNC_TRACE();
1008
1009         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1010                 return 0;
1011
1012         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1013         pci_dev = dev->pci_dev;
1014
1015         if (hw->adapter_stopped == 0)
1016                 i40e_dev_close(dev);
1017
1018         dev->dev_ops = NULL;
1019         dev->rx_pkt_burst = NULL;
1020         dev->tx_pkt_burst = NULL;
1021
1022         /* Disable LLDP */
1023         ret = i40e_aq_stop_lldp(hw, true, NULL);
1024         if (ret != I40E_SUCCESS) /* Its failure can be ignored */
1025                 PMD_INIT_LOG(INFO, "Failed to stop lldp");
1026
1027         /* Clear PXE mode */
1028         i40e_clear_pxe_mode(hw);
1029
1030         /* Unconfigure filter control */
1031         memset(&settings, 0, sizeof(settings));
1032         ret = i40e_set_filter_control(hw, &settings);
1033         if (ret)
1034                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
1035                                         ret);
1036
1037         /* Disable flow control */
1038         hw->fc.requested_mode = I40E_FC_NONE;
1039         i40e_set_fc(hw, &aq_fail, TRUE);
1040
1041         /* uninitialize pf host driver */
1042         i40e_pf_host_uninit(dev);
1043
1044         rte_free(dev->data->mac_addrs);
1045         dev->data->mac_addrs = NULL;
1046
1047         /* disable uio intr before callback unregister */
1048         rte_intr_disable(&(pci_dev->intr_handle));
1049
1050         /* register callback func to eal lib */
1051         rte_intr_callback_unregister(&(pci_dev->intr_handle),
1052                 i40e_dev_interrupt_handler, (void *)dev);
1053
1054         return 0;
1055 }
1056
1057 static int
1058 i40e_dev_configure(struct rte_eth_dev *dev)
1059 {
1060         struct i40e_adapter *ad =
1061                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1062         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1063         enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1064         int i, ret;
1065
1066         /* Initialize to TRUE. If any of Rx queues doesn't meet the
1067          * bulk allocation or vector Rx preconditions we will reset it.
1068          */
1069         ad->rx_bulk_alloc_allowed = true;
1070         ad->rx_vec_allowed = true;
1071         ad->tx_simple_allowed = true;
1072         ad->tx_vec_allowed = true;
1073
1074         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
1075                 ret = i40e_fdir_setup(pf);
1076                 if (ret != I40E_SUCCESS) {
1077                         PMD_DRV_LOG(ERR, "Failed to setup flow director.");
1078                         return -ENOTSUP;
1079                 }
1080                 ret = i40e_fdir_configure(dev);
1081                 if (ret < 0) {
1082                         PMD_DRV_LOG(ERR, "failed to configure fdir.");
1083                         goto err;
1084                 }
1085         } else
1086                 i40e_fdir_teardown(pf);
1087
1088         ret = i40e_dev_init_vlan(dev);
1089         if (ret < 0)
1090                 goto err;
1091
1092         /* VMDQ setup.
1093          *  Needs to move VMDQ setting out of i40e_pf_config_mq_rx() as VMDQ and
1094          *  RSS setting have different requirements.
1095          *  General PMD driver call sequence are NIC init, configure,
1096          *  rx/tx_queue_setup and dev_start. In rx/tx_queue_setup() function, it
1097          *  will try to lookup the VSI that specific queue belongs to if VMDQ
1098          *  applicable. So, VMDQ setting has to be done before
1099          *  rx/tx_queue_setup(). This function is good  to place vmdq_setup.
1100          *  For RSS setting, it will try to calculate actual configured RX queue
1101          *  number, which will be available after rx_queue_setup(). dev_start()
1102          *  function is good to place RSS setup.
1103          */
1104         if (mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
1105                 ret = i40e_vmdq_setup(dev);
1106                 if (ret)
1107                         goto err;
1108         }
1109
1110         if (mq_mode & ETH_MQ_RX_DCB_FLAG) {
1111                 ret = i40e_dcb_setup(dev);
1112                 if (ret) {
1113                         PMD_DRV_LOG(ERR, "failed to configure DCB.");
1114                         goto err_dcb;
1115                 }
1116         }
1117
1118         return 0;
1119
1120 err_dcb:
1121         /* need to release vmdq resource if exists */
1122         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1123                 i40e_vsi_release(pf->vmdq[i].vsi);
1124                 pf->vmdq[i].vsi = NULL;
1125         }
1126         rte_free(pf->vmdq);
1127         pf->vmdq = NULL;
1128 err:
1129         /* need to release fdir resource if exists */
1130         i40e_fdir_teardown(pf);
1131         return ret;
1132 }
1133
1134 void
1135 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
1136 {
1137         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1138         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1139         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1140         uint16_t msix_vect = vsi->msix_intr;
1141         uint16_t i;
1142
1143         for (i = 0; i < vsi->nb_qps; i++) {
1144                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1145                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1146                 rte_wmb();
1147         }
1148
1149         if (vsi->type != I40E_VSI_SRIOV) {
1150                 if (!rte_intr_allow_others(intr_handle)) {
1151                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1152                                        I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
1153                         I40E_WRITE_REG(hw,
1154                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1155                                        0);
1156                 } else {
1157                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1158                                        I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK);
1159                         I40E_WRITE_REG(hw,
1160                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1161                                                        msix_vect - 1), 0);
1162                 }
1163         } else {
1164                 uint32_t reg;
1165                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1166                         vsi->user_param + (msix_vect - 1);
1167
1168                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1169                                I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
1170         }
1171         I40E_WRITE_FLUSH(hw);
1172 }
1173
1174 static void
1175 __vsi_queues_bind_intr(struct i40e_vsi *vsi, uint16_t msix_vect,
1176                        int base_queue, int nb_queue)
1177 {
1178         int i;
1179         uint32_t val;
1180         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1181
1182         /* Bind all RX queues to allocated MSIX interrupt */
1183         for (i = 0; i < nb_queue; i++) {
1184                 val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
1185                         I40E_QINT_RQCTL_ITR_INDX_MASK |
1186                         ((base_queue + i + 1) <<
1187                          I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
1188                         (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
1189                         I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1190
1191                 if (i == nb_queue - 1)
1192                         val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
1193                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(base_queue + i), val);
1194         }
1195
1196         /* Write first RX queue to Link list register as the head element */
1197         if (vsi->type != I40E_VSI_SRIOV) {
1198                 uint16_t interval =
1199                         i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
1200
1201                 if (msix_vect == I40E_MISC_VEC_ID) {
1202                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
1203                                        (base_queue <<
1204                                         I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1205                                        (0x0 <<
1206                                         I40E_PFINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1207                         I40E_WRITE_REG(hw,
1208                                        I40E_PFINT_ITR0(I40E_ITR_INDEX_DEFAULT),
1209                                        interval);
1210                 } else {
1211                         I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
1212                                        (base_queue <<
1213                                         I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1214                                        (0x0 <<
1215                                         I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1216                         I40E_WRITE_REG(hw,
1217                                        I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
1218                                                        msix_vect - 1),
1219                                        interval);
1220                 }
1221         } else {
1222                 uint32_t reg;
1223
1224                 if (msix_vect == I40E_MISC_VEC_ID) {
1225                         I40E_WRITE_REG(hw,
1226                                        I40E_VPINT_LNKLST0(vsi->user_param),
1227                                        (base_queue <<
1228                                         I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
1229                                        (0x0 <<
1230                                         I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
1231                 } else {
1232                         /* num_msix_vectors_vf needs to minus irq0 */
1233                         reg = (hw->func_caps.num_msix_vectors_vf - 1) *
1234                                 vsi->user_param + (msix_vect - 1);
1235
1236                         I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
1237                                        (base_queue <<
1238                                         I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
1239                                        (0x0 <<
1240                                         I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
1241                 }
1242         }
1243
1244         I40E_WRITE_FLUSH(hw);
1245 }
1246
1247 void
1248 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi)
1249 {
1250         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1251         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1252         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1253         uint16_t msix_vect = vsi->msix_intr;
1254         uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd);
1255         uint16_t queue_idx = 0;
1256         int record = 0;
1257         uint32_t val;
1258         int i;
1259
1260         for (i = 0; i < vsi->nb_qps; i++) {
1261                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
1262                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
1263         }
1264
1265         /* INTENA flag is not auto-cleared for interrupt */
1266         val = I40E_READ_REG(hw, I40E_GLINT_CTL);
1267         val |= I40E_GLINT_CTL_DIS_AUTOMASK_PF0_MASK |
1268                 I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK |
1269                 I40E_GLINT_CTL_DIS_AUTOMASK_VF0_MASK;
1270         I40E_WRITE_REG(hw, I40E_GLINT_CTL, val);
1271
1272         /* VF bind interrupt */
1273         if (vsi->type == I40E_VSI_SRIOV) {
1274                 __vsi_queues_bind_intr(vsi, msix_vect,
1275                                        vsi->base_queue, vsi->nb_qps);
1276                 return;
1277         }
1278
1279         /* PF & VMDq bind interrupt */
1280         if (rte_intr_dp_is_en(intr_handle)) {
1281                 if (vsi->type == I40E_VSI_MAIN) {
1282                         queue_idx = 0;
1283                         record = 1;
1284                 } else if (vsi->type == I40E_VSI_VMDQ2) {
1285                         struct i40e_vsi *main_vsi =
1286                                 I40E_DEV_PRIVATE_TO_MAIN_VSI(vsi->adapter);
1287                         queue_idx = vsi->base_queue - main_vsi->nb_qps;
1288                         record = 1;
1289                 }
1290         }
1291
1292         for (i = 0; i < vsi->nb_used_qps; i++) {
1293                 if (nb_msix <= 1) {
1294                         if (!rte_intr_allow_others(intr_handle))
1295                                 /* allow to share MISC_VEC_ID */
1296                                 msix_vect = I40E_MISC_VEC_ID;
1297
1298                         /* no enough msix_vect, map all to one */
1299                         __vsi_queues_bind_intr(vsi, msix_vect,
1300                                                vsi->base_queue + i,
1301                                                vsi->nb_used_qps - i);
1302                         for (; !!record && i < vsi->nb_used_qps; i++)
1303                                 intr_handle->intr_vec[queue_idx + i] =
1304                                         msix_vect;
1305                         break;
1306                 }
1307                 /* 1:1 queue/msix_vect mapping */
1308                 __vsi_queues_bind_intr(vsi, msix_vect,
1309                                        vsi->base_queue + i, 1);
1310                 if (!!record)
1311                         intr_handle->intr_vec[queue_idx + i] = msix_vect;
1312
1313                 msix_vect++;
1314                 nb_msix--;
1315         }
1316 }
1317
1318 static void
1319 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
1320 {
1321         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1322         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1323         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1324         uint16_t interval = i40e_calc_itr_interval(\
1325                 RTE_LIBRTE_I40E_ITR_INTERVAL);
1326         uint16_t msix_intr, i;
1327
1328         if (rte_intr_allow_others(intr_handle))
1329                 for (i = 0; i < vsi->nb_msix; i++) {
1330                         msix_intr = vsi->msix_intr + i;
1331                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1332                                 I40E_PFINT_DYN_CTLN_INTENA_MASK |
1333                                 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
1334                                 (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
1335                                 (interval <<
1336                                  I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
1337                 }
1338         else
1339                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
1340                                I40E_PFINT_DYN_CTL0_INTENA_MASK |
1341                                I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
1342                                (0 << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT) |
1343                                (interval <<
1344                                 I40E_PFINT_DYN_CTL0_INTERVAL_SHIFT));
1345
1346         I40E_WRITE_FLUSH(hw);
1347 }
1348
1349 static void
1350 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
1351 {
1352         struct rte_eth_dev *dev = vsi->adapter->eth_dev;
1353         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1354         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1355         uint16_t msix_intr, i;
1356
1357         if (rte_intr_allow_others(intr_handle))
1358                 for (i = 0; i < vsi->nb_msix; i++) {
1359                         msix_intr = vsi->msix_intr + i;
1360                         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(msix_intr - 1),
1361                                        0);
1362                 }
1363         else
1364                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
1365
1366         I40E_WRITE_FLUSH(hw);
1367 }
1368
1369 static inline uint8_t
1370 i40e_parse_link_speed(uint16_t eth_link_speed)
1371 {
1372         uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN;
1373
1374         switch (eth_link_speed) {
1375         case ETH_LINK_SPEED_40G:
1376                 link_speed = I40E_LINK_SPEED_40GB;
1377                 break;
1378         case ETH_LINK_SPEED_20G:
1379                 link_speed = I40E_LINK_SPEED_20GB;
1380                 break;
1381         case ETH_LINK_SPEED_10G:
1382                 link_speed = I40E_LINK_SPEED_10GB;
1383                 break;
1384         case ETH_LINK_SPEED_1000:
1385                 link_speed = I40E_LINK_SPEED_1GB;
1386                 break;
1387         case ETH_LINK_SPEED_100:
1388                 link_speed = I40E_LINK_SPEED_100MB;
1389                 break;
1390         }
1391
1392         return link_speed;
1393 }
1394
1395 static int
1396 i40e_phy_conf_link(__rte_unused struct i40e_hw *hw,
1397                    __rte_unused uint8_t abilities,
1398                    __rte_unused uint8_t force_speed)
1399 {
1400         /* Skip any phy config on both 10G and 40G interfaces, as a workaround
1401          * for the link control limitation of that all link control should be
1402          * handled by firmware. It should follow up if link control will be
1403          * opened to software driver in future firmware versions.
1404          */
1405         return I40E_SUCCESS;
1406 }
1407
1408 static int
1409 i40e_apply_link_speed(struct rte_eth_dev *dev)
1410 {
1411         uint8_t speed;
1412         uint8_t abilities = 0;
1413         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1414         struct rte_eth_conf *conf = &dev->data->dev_conf;
1415
1416         speed = i40e_parse_link_speed(conf->link_speed);
1417         abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
1418         if (conf->link_speed == ETH_LINK_SPEED_AUTONEG)
1419                 abilities |= I40E_AQ_PHY_AN_ENABLED;
1420         else
1421                 abilities |= I40E_AQ_PHY_LINK_ENABLED;
1422
1423         return i40e_phy_conf_link(hw, abilities, speed);
1424 }
1425
1426 static int
1427 i40e_dev_start(struct rte_eth_dev *dev)
1428 {
1429         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1430         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1431         struct i40e_vsi *main_vsi = pf->main_vsi;
1432         int ret, i;
1433         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1434         uint32_t intr_vector = 0;
1435
1436         hw->adapter_stopped = 0;
1437
1438         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1439                 (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1440                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
1441                              dev->data->dev_conf.link_duplex,
1442                              dev->data->port_id);
1443                 return -EINVAL;
1444         }
1445
1446         rte_intr_disable(intr_handle);
1447
1448         if ((rte_intr_cap_multiple(intr_handle) ||
1449              !RTE_ETH_DEV_SRIOV(dev).active) &&
1450             dev->data->dev_conf.intr_conf.rxq != 0) {
1451                 intr_vector = dev->data->nb_rx_queues;
1452                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1453                         return -1;
1454         }
1455
1456         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1457                 intr_handle->intr_vec =
1458                         rte_zmalloc("intr_vec",
1459                                     dev->data->nb_rx_queues * sizeof(int),
1460                                     0);
1461                 if (!intr_handle->intr_vec) {
1462                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1463                                      " intr_vec\n", dev->data->nb_rx_queues);
1464                         return -ENOMEM;
1465                 }
1466         }
1467
1468         /* Initialize VSI */
1469         ret = i40e_dev_rxtx_init(pf);
1470         if (ret != I40E_SUCCESS) {
1471                 PMD_DRV_LOG(ERR, "Failed to init rx/tx queues");
1472                 goto err_up;
1473         }
1474
1475         /* Map queues with MSIX interrupt */
1476         main_vsi->nb_used_qps = dev->data->nb_rx_queues -
1477                 pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1478         i40e_vsi_queues_bind_intr(main_vsi);
1479         i40e_vsi_enable_queues_intr(main_vsi);
1480
1481         /* Map VMDQ VSI queues with MSIX interrupt */
1482         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1483                 pf->vmdq[i].vsi->nb_used_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
1484                 i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi);
1485                 i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi);
1486         }
1487
1488         /* enable FDIR MSIX interrupt */
1489         if (pf->fdir.fdir_vsi) {
1490                 i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi);
1491                 i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi);
1492         }
1493
1494         /* Enable all queues which have been configured */
1495         ret = i40e_dev_switch_queues(pf, TRUE);
1496         if (ret != I40E_SUCCESS) {
1497                 PMD_DRV_LOG(ERR, "Failed to enable VSI");
1498                 goto err_up;
1499         }
1500
1501         /* Enable receiving broadcast packets */
1502         ret = i40e_aq_set_vsi_broadcast(hw, main_vsi->seid, true, NULL);
1503         if (ret != I40E_SUCCESS)
1504                 PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1505
1506         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1507                 ret = i40e_aq_set_vsi_broadcast(hw, pf->vmdq[i].vsi->seid,
1508                                                 true, NULL);
1509                 if (ret != I40E_SUCCESS)
1510                         PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
1511         }
1512
1513         /* Apply link configure */
1514         ret = i40e_apply_link_speed(dev);
1515         if (I40E_SUCCESS != ret) {
1516                 PMD_DRV_LOG(ERR, "Fail to apply link setting");
1517                 goto err_up;
1518         }
1519
1520         if (!rte_intr_allow_others(intr_handle)) {
1521                 rte_intr_callback_unregister(intr_handle,
1522                                              i40e_dev_interrupt_handler,
1523                                              (void *)dev);
1524                 /* configure and enable device interrupt */
1525                 i40e_pf_config_irq0(hw, FALSE);
1526                 i40e_pf_enable_irq0(hw);
1527
1528                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1529                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1530                                      " no intr multiplex\n");
1531         }
1532
1533         /* enable uio intr after callback register */
1534         rte_intr_enable(intr_handle);
1535
1536         return I40E_SUCCESS;
1537
1538 err_up:
1539         i40e_dev_switch_queues(pf, FALSE);
1540         i40e_dev_clear_queues(dev);
1541
1542         return ret;
1543 }
1544
1545 static void
1546 i40e_dev_stop(struct rte_eth_dev *dev)
1547 {
1548         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1549         struct i40e_vsi *main_vsi = pf->main_vsi;
1550         struct i40e_mirror_rule *p_mirror;
1551         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1552         int i;
1553
1554         /* Disable all queues */
1555         i40e_dev_switch_queues(pf, FALSE);
1556
1557         /* un-map queues with interrupt registers */
1558         i40e_vsi_disable_queues_intr(main_vsi);
1559         i40e_vsi_queues_unbind_intr(main_vsi);
1560
1561         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1562                 i40e_vsi_disable_queues_intr(pf->vmdq[i].vsi);
1563                 i40e_vsi_queues_unbind_intr(pf->vmdq[i].vsi);
1564         }
1565
1566         if (pf->fdir.fdir_vsi) {
1567                 i40e_vsi_queues_unbind_intr(pf->fdir.fdir_vsi);
1568                 i40e_vsi_disable_queues_intr(pf->fdir.fdir_vsi);
1569         }
1570         /* Clear all queues and release memory */
1571         i40e_dev_clear_queues(dev);
1572
1573         /* Set link down */
1574         i40e_dev_set_link_down(dev);
1575
1576         /* Remove all mirror rules */
1577         while ((p_mirror = TAILQ_FIRST(&pf->mirror_list))) {
1578                 TAILQ_REMOVE(&pf->mirror_list, p_mirror, rules);
1579                 rte_free(p_mirror);
1580         }
1581         pf->nb_mirror_rule = 0;
1582
1583         if (!rte_intr_allow_others(intr_handle))
1584                 /* resume to the default handler */
1585                 rte_intr_callback_register(intr_handle,
1586                                            i40e_dev_interrupt_handler,
1587                                            (void *)dev);
1588
1589         /* Clean datapath event and queue/vec mapping */
1590         rte_intr_efd_disable(intr_handle);
1591         if (intr_handle->intr_vec) {
1592                 rte_free(intr_handle->intr_vec);
1593                 intr_handle->intr_vec = NULL;
1594         }
1595 }
1596
1597 static void
1598 i40e_dev_close(struct rte_eth_dev *dev)
1599 {
1600         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1601         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1602         uint32_t reg;
1603         int i;
1604
1605         PMD_INIT_FUNC_TRACE();
1606
1607         i40e_dev_stop(dev);
1608         hw->adapter_stopped = 1;
1609         i40e_dev_free_queues(dev);
1610
1611         /* Disable interrupt */
1612         i40e_pf_disable_irq0(hw);
1613         rte_intr_disable(&(dev->pci_dev->intr_handle));
1614
1615         /* shutdown and destroy the HMC */
1616         i40e_shutdown_lan_hmc(hw);
1617
1618         /* release all the existing VSIs and VEBs */
1619         i40e_fdir_teardown(pf);
1620         i40e_vsi_release(pf->main_vsi);
1621
1622         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
1623                 i40e_vsi_release(pf->vmdq[i].vsi);
1624                 pf->vmdq[i].vsi = NULL;
1625         }
1626
1627         rte_free(pf->vmdq);
1628         pf->vmdq = NULL;
1629
1630         /* shutdown the adminq */
1631         i40e_aq_queue_shutdown(hw, true);
1632         i40e_shutdown_adminq(hw);
1633
1634         i40e_res_pool_destroy(&pf->qp_pool);
1635         i40e_res_pool_destroy(&pf->msix_pool);
1636
1637         /* force a PF reset to clean anything leftover */
1638         reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
1639         I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
1640                         (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
1641         I40E_WRITE_FLUSH(hw);
1642 }
1643
1644 static void
1645 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
1646 {
1647         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1648         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1649         struct i40e_vsi *vsi = pf->main_vsi;
1650         int status;
1651
1652         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
1653                                                         true, NULL);
1654         if (status != I40E_SUCCESS)
1655                 PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous");
1656
1657         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
1658                                                         TRUE, NULL);
1659         if (status != I40E_SUCCESS)
1660                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
1661
1662 }
1663
1664 static void
1665 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
1666 {
1667         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1668         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1669         struct i40e_vsi *vsi = pf->main_vsi;
1670         int status;
1671
1672         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
1673                                                         false, NULL);
1674         if (status != I40E_SUCCESS)
1675                 PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
1676
1677         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
1678                                                         false, NULL);
1679         if (status != I40E_SUCCESS)
1680                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1681 }
1682
1683 static void
1684 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
1685 {
1686         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1687         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1688         struct i40e_vsi *vsi = pf->main_vsi;
1689         int ret;
1690
1691         ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
1692         if (ret != I40E_SUCCESS)
1693                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
1694 }
1695
1696 static void
1697 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
1698 {
1699         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1700         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1701         struct i40e_vsi *vsi = pf->main_vsi;
1702         int ret;
1703
1704         if (dev->data->promiscuous == 1)
1705                 return; /* must remain in all_multicast mode */
1706
1707         ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
1708                                 vsi->seid, FALSE, NULL);
1709         if (ret != I40E_SUCCESS)
1710                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1711 }
1712
1713 /*
1714  * Set device link up.
1715  */
1716 static int
1717 i40e_dev_set_link_up(struct rte_eth_dev *dev)
1718 {
1719         /* re-apply link speed setting */
1720         return i40e_apply_link_speed(dev);
1721 }
1722
1723 /*
1724  * Set device link down.
1725  */
1726 static int
1727 i40e_dev_set_link_down(__rte_unused struct rte_eth_dev *dev)
1728 {
1729         uint8_t speed = I40E_LINK_SPEED_UNKNOWN;
1730         uint8_t abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
1731         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1732
1733         return i40e_phy_conf_link(hw, abilities, speed);
1734 }
1735
1736 int
1737 i40e_dev_link_update(struct rte_eth_dev *dev,
1738                      int wait_to_complete)
1739 {
1740 #define CHECK_INTERVAL 100  /* 100ms */
1741 #define MAX_REPEAT_TIME 10  /* 1s (10 * 100ms) in total */
1742         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1743         struct i40e_link_status link_status;
1744         struct rte_eth_link link, old;
1745         int status;
1746         unsigned rep_cnt = MAX_REPEAT_TIME;
1747
1748         memset(&link, 0, sizeof(link));
1749         memset(&old, 0, sizeof(old));
1750         memset(&link_status, 0, sizeof(link_status));
1751         rte_i40e_dev_atomic_read_link_status(dev, &old);
1752
1753         do {
1754                 /* Get link status information from hardware */
1755                 status = i40e_aq_get_link_info(hw, false, &link_status, NULL);
1756                 if (status != I40E_SUCCESS) {
1757                         link.link_speed = ETH_LINK_SPEED_100;
1758                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1759                         PMD_DRV_LOG(ERR, "Failed to get link info");
1760                         goto out;
1761                 }
1762
1763                 link.link_status = link_status.link_info & I40E_AQ_LINK_UP;
1764                 if (!wait_to_complete)
1765                         break;
1766
1767                 rte_delay_ms(CHECK_INTERVAL);
1768         } while (!link.link_status && rep_cnt--);
1769
1770         if (!link.link_status)
1771                 goto out;
1772
1773         /* i40e uses full duplex only */
1774         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1775
1776         /* Parse the link status */
1777         switch (link_status.link_speed) {
1778         case I40E_LINK_SPEED_100MB:
1779                 link.link_speed = ETH_LINK_SPEED_100;
1780                 break;
1781         case I40E_LINK_SPEED_1GB:
1782                 link.link_speed = ETH_LINK_SPEED_1000;
1783                 break;
1784         case I40E_LINK_SPEED_10GB:
1785                 link.link_speed = ETH_LINK_SPEED_10G;
1786                 break;
1787         case I40E_LINK_SPEED_20GB:
1788                 link.link_speed = ETH_LINK_SPEED_20G;
1789                 break;
1790         case I40E_LINK_SPEED_40GB:
1791                 link.link_speed = ETH_LINK_SPEED_40G;
1792                 break;
1793         default:
1794                 link.link_speed = ETH_LINK_SPEED_100;
1795                 break;
1796         }
1797
1798 out:
1799         rte_i40e_dev_atomic_write_link_status(dev, &link);
1800         if (link.link_status == old.link_status)
1801                 return -1;
1802
1803         return 0;
1804 }
1805
1806 /* Get all the statistics of a VSI */
1807 void
1808 i40e_update_vsi_stats(struct i40e_vsi *vsi)
1809 {
1810         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
1811         struct i40e_eth_stats *nes = &vsi->eth_stats;
1812         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1813         int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
1814
1815         i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
1816                             vsi->offset_loaded, &oes->rx_bytes,
1817                             &nes->rx_bytes);
1818         i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
1819                             vsi->offset_loaded, &oes->rx_unicast,
1820                             &nes->rx_unicast);
1821         i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
1822                             vsi->offset_loaded, &oes->rx_multicast,
1823                             &nes->rx_multicast);
1824         i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
1825                             vsi->offset_loaded, &oes->rx_broadcast,
1826                             &nes->rx_broadcast);
1827         i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
1828                             &oes->rx_discards, &nes->rx_discards);
1829         /* GLV_REPC not supported */
1830         /* GLV_RMPC not supported */
1831         i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
1832                             &oes->rx_unknown_protocol,
1833                             &nes->rx_unknown_protocol);
1834         i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
1835                             vsi->offset_loaded, &oes->tx_bytes,
1836                             &nes->tx_bytes);
1837         i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
1838                             vsi->offset_loaded, &oes->tx_unicast,
1839                             &nes->tx_unicast);
1840         i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
1841                             vsi->offset_loaded, &oes->tx_multicast,
1842                             &nes->tx_multicast);
1843         i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
1844                             vsi->offset_loaded,  &oes->tx_broadcast,
1845                             &nes->tx_broadcast);
1846         /* GLV_TDPC not supported */
1847         i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
1848                             &oes->tx_errors, &nes->tx_errors);
1849         vsi->offset_loaded = true;
1850
1851         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats start *******************",
1852                     vsi->vsi_id);
1853         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", nes->rx_bytes);
1854         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", nes->rx_unicast);
1855         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", nes->rx_multicast);
1856         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", nes->rx_broadcast);
1857         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", nes->rx_discards);
1858         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
1859                     nes->rx_unknown_protocol);
1860         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", nes->tx_bytes);
1861         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", nes->tx_unicast);
1862         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", nes->tx_multicast);
1863         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", nes->tx_broadcast);
1864         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", nes->tx_discards);
1865         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", nes->tx_errors);
1866         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats end *******************",
1867                     vsi->vsi_id);
1868 }
1869
1870 static void
1871 i40e_read_stats_registers(struct i40e_pf *pf, struct i40e_hw *hw)
1872 {
1873         unsigned int i;
1874         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
1875         struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
1876
1877         /* Get statistics of struct i40e_eth_stats */
1878         i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
1879                             I40E_GLPRT_GORCL(hw->port),
1880                             pf->offset_loaded, &os->eth.rx_bytes,
1881                             &ns->eth.rx_bytes);
1882         i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
1883                             I40E_GLPRT_UPRCL(hw->port),
1884                             pf->offset_loaded, &os->eth.rx_unicast,
1885                             &ns->eth.rx_unicast);
1886         i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
1887                             I40E_GLPRT_MPRCL(hw->port),
1888                             pf->offset_loaded, &os->eth.rx_multicast,
1889                             &ns->eth.rx_multicast);
1890         i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
1891                             I40E_GLPRT_BPRCL(hw->port),
1892                             pf->offset_loaded, &os->eth.rx_broadcast,
1893                             &ns->eth.rx_broadcast);
1894         /* Workaround: CRC size should not be included in byte statistics,
1895          * so subtract ETHER_CRC_LEN from the byte counter for each rx packet.
1896          */
1897         ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast +
1898                 ns->eth.rx_broadcast) * ETHER_CRC_LEN;
1899
1900         i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
1901                             pf->offset_loaded, &os->eth.rx_discards,
1902                             &ns->eth.rx_discards);
1903         /* GLPRT_REPC not supported */
1904         /* GLPRT_RMPC not supported */
1905         i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
1906                             pf->offset_loaded,
1907                             &os->eth.rx_unknown_protocol,
1908                             &ns->eth.rx_unknown_protocol);
1909         i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
1910                             I40E_GLPRT_GOTCL(hw->port),
1911                             pf->offset_loaded, &os->eth.tx_bytes,
1912                             &ns->eth.tx_bytes);
1913         i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
1914                             I40E_GLPRT_UPTCL(hw->port),
1915                             pf->offset_loaded, &os->eth.tx_unicast,
1916                             &ns->eth.tx_unicast);
1917         i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
1918                             I40E_GLPRT_MPTCL(hw->port),
1919                             pf->offset_loaded, &os->eth.tx_multicast,
1920                             &ns->eth.tx_multicast);
1921         i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
1922                             I40E_GLPRT_BPTCL(hw->port),
1923                             pf->offset_loaded, &os->eth.tx_broadcast,
1924                             &ns->eth.tx_broadcast);
1925         ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast +
1926                 ns->eth.tx_broadcast) * ETHER_CRC_LEN;
1927         /* GLPRT_TEPC not supported */
1928
1929         /* additional port specific stats */
1930         i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
1931                             pf->offset_loaded, &os->tx_dropped_link_down,
1932                             &ns->tx_dropped_link_down);
1933         i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
1934                             pf->offset_loaded, &os->crc_errors,
1935                             &ns->crc_errors);
1936         i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
1937                             pf->offset_loaded, &os->illegal_bytes,
1938                             &ns->illegal_bytes);
1939         /* GLPRT_ERRBC not supported */
1940         i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
1941                             pf->offset_loaded, &os->mac_local_faults,
1942                             &ns->mac_local_faults);
1943         i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
1944                             pf->offset_loaded, &os->mac_remote_faults,
1945                             &ns->mac_remote_faults);
1946         i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
1947                             pf->offset_loaded, &os->rx_length_errors,
1948                             &ns->rx_length_errors);
1949         i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
1950                             pf->offset_loaded, &os->link_xon_rx,
1951                             &ns->link_xon_rx);
1952         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1953                             pf->offset_loaded, &os->link_xoff_rx,
1954                             &ns->link_xoff_rx);
1955         for (i = 0; i < 8; i++) {
1956                 i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1957                                     pf->offset_loaded,
1958                                     &os->priority_xon_rx[i],
1959                                     &ns->priority_xon_rx[i]);
1960                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1961                                     pf->offset_loaded,
1962                                     &os->priority_xoff_rx[i],
1963                                     &ns->priority_xoff_rx[i]);
1964         }
1965         i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
1966                             pf->offset_loaded, &os->link_xon_tx,
1967                             &ns->link_xon_tx);
1968         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1969                             pf->offset_loaded, &os->link_xoff_tx,
1970                             &ns->link_xoff_tx);
1971         for (i = 0; i < 8; i++) {
1972                 i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1973                                     pf->offset_loaded,
1974                                     &os->priority_xon_tx[i],
1975                                     &ns->priority_xon_tx[i]);
1976                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1977                                     pf->offset_loaded,
1978                                     &os->priority_xoff_tx[i],
1979                                     &ns->priority_xoff_tx[i]);
1980                 i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1981                                     pf->offset_loaded,
1982                                     &os->priority_xon_2_xoff[i],
1983                                     &ns->priority_xon_2_xoff[i]);
1984         }
1985         i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
1986                             I40E_GLPRT_PRC64L(hw->port),
1987                             pf->offset_loaded, &os->rx_size_64,
1988                             &ns->rx_size_64);
1989         i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
1990                             I40E_GLPRT_PRC127L(hw->port),
1991                             pf->offset_loaded, &os->rx_size_127,
1992                             &ns->rx_size_127);
1993         i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
1994                             I40E_GLPRT_PRC255L(hw->port),
1995                             pf->offset_loaded, &os->rx_size_255,
1996                             &ns->rx_size_255);
1997         i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
1998                             I40E_GLPRT_PRC511L(hw->port),
1999                             pf->offset_loaded, &os->rx_size_511,
2000                             &ns->rx_size_511);
2001         i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
2002                             I40E_GLPRT_PRC1023L(hw->port),
2003                             pf->offset_loaded, &os->rx_size_1023,
2004                             &ns->rx_size_1023);
2005         i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
2006                             I40E_GLPRT_PRC1522L(hw->port),
2007                             pf->offset_loaded, &os->rx_size_1522,
2008                             &ns->rx_size_1522);
2009         i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
2010                             I40E_GLPRT_PRC9522L(hw->port),
2011                             pf->offset_loaded, &os->rx_size_big,
2012                             &ns->rx_size_big);
2013         i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
2014                             pf->offset_loaded, &os->rx_undersize,
2015                             &ns->rx_undersize);
2016         i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
2017                             pf->offset_loaded, &os->rx_fragments,
2018                             &ns->rx_fragments);
2019         i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
2020                             pf->offset_loaded, &os->rx_oversize,
2021                             &ns->rx_oversize);
2022         i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
2023                             pf->offset_loaded, &os->rx_jabber,
2024                             &ns->rx_jabber);
2025         i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
2026                             I40E_GLPRT_PTC64L(hw->port),
2027                             pf->offset_loaded, &os->tx_size_64,
2028                             &ns->tx_size_64);
2029         i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
2030                             I40E_GLPRT_PTC127L(hw->port),
2031                             pf->offset_loaded, &os->tx_size_127,
2032                             &ns->tx_size_127);
2033         i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
2034                             I40E_GLPRT_PTC255L(hw->port),
2035                             pf->offset_loaded, &os->tx_size_255,
2036                             &ns->tx_size_255);
2037         i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
2038                             I40E_GLPRT_PTC511L(hw->port),
2039                             pf->offset_loaded, &os->tx_size_511,
2040                             &ns->tx_size_511);
2041         i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
2042                             I40E_GLPRT_PTC1023L(hw->port),
2043                             pf->offset_loaded, &os->tx_size_1023,
2044                             &ns->tx_size_1023);
2045         i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
2046                             I40E_GLPRT_PTC1522L(hw->port),
2047                             pf->offset_loaded, &os->tx_size_1522,
2048                             &ns->tx_size_1522);
2049         i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
2050                             I40E_GLPRT_PTC9522L(hw->port),
2051                             pf->offset_loaded, &os->tx_size_big,
2052                             &ns->tx_size_big);
2053         i40e_stat_update_32(hw, I40E_GLQF_PCNT(pf->fdir.match_counter_index),
2054                            pf->offset_loaded,
2055                            &os->fd_sb_match, &ns->fd_sb_match);
2056         /* GLPRT_MSPDC not supported */
2057         /* GLPRT_XEC not supported */
2058
2059         pf->offset_loaded = true;
2060
2061         if (pf->main_vsi)
2062                 i40e_update_vsi_stats(pf->main_vsi);
2063 }
2064
2065 /* Get all statistics of a port */
2066 static void
2067 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2068 {
2069         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2070         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2071         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
2072         unsigned i;
2073
2074         /* call read registers - updates values, now write them to struct */
2075         i40e_read_stats_registers(pf, hw);
2076
2077         stats->ipackets = pf->main_vsi->eth_stats.rx_unicast +
2078                         pf->main_vsi->eth_stats.rx_multicast +
2079                         pf->main_vsi->eth_stats.rx_broadcast -
2080                         pf->main_vsi->eth_stats.rx_discards;
2081         stats->opackets = pf->main_vsi->eth_stats.tx_unicast +
2082                         pf->main_vsi->eth_stats.tx_multicast +
2083                         pf->main_vsi->eth_stats.tx_broadcast;
2084         stats->ibytes   = ns->eth.rx_bytes;
2085         stats->obytes   = ns->eth.tx_bytes;
2086         stats->oerrors  = ns->eth.tx_errors +
2087                         pf->main_vsi->eth_stats.tx_errors;
2088         stats->imcasts  = pf->main_vsi->eth_stats.rx_multicast;
2089
2090         /* Rx Errors */
2091         stats->imissed  = ns->eth.rx_discards +
2092                         pf->main_vsi->eth_stats.rx_discards;
2093         stats->ierrors  = ns->crc_errors +
2094                         ns->rx_length_errors + ns->rx_undersize +
2095                         ns->rx_oversize + ns->rx_fragments + ns->rx_jabber +
2096                         stats->imissed;
2097
2098         PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************");
2099         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", ns->eth.rx_bytes);
2100         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", ns->eth.rx_unicast);
2101         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", ns->eth.rx_multicast);
2102         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", ns->eth.rx_broadcast);
2103         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", ns->eth.rx_discards);
2104         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
2105                     ns->eth.rx_unknown_protocol);
2106         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", ns->eth.tx_bytes);
2107         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", ns->eth.tx_unicast);
2108         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", ns->eth.tx_multicast);
2109         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", ns->eth.tx_broadcast);
2110         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", ns->eth.tx_discards);
2111         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", ns->eth.tx_errors);
2112
2113         PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:     %"PRIu64"",
2114                     ns->tx_dropped_link_down);
2115         PMD_DRV_LOG(DEBUG, "crc_errors:               %"PRIu64"", ns->crc_errors);
2116         PMD_DRV_LOG(DEBUG, "illegal_bytes:            %"PRIu64"",
2117                     ns->illegal_bytes);
2118         PMD_DRV_LOG(DEBUG, "error_bytes:              %"PRIu64"", ns->error_bytes);
2119         PMD_DRV_LOG(DEBUG, "mac_local_faults:         %"PRIu64"",
2120                     ns->mac_local_faults);
2121         PMD_DRV_LOG(DEBUG, "mac_remote_faults:        %"PRIu64"",
2122                     ns->mac_remote_faults);
2123         PMD_DRV_LOG(DEBUG, "rx_length_errors:         %"PRIu64"",
2124                     ns->rx_length_errors);
2125         PMD_DRV_LOG(DEBUG, "link_xon_rx:              %"PRIu64"", ns->link_xon_rx);
2126         PMD_DRV_LOG(DEBUG, "link_xoff_rx:             %"PRIu64"", ns->link_xoff_rx);
2127         for (i = 0; i < 8; i++) {
2128                 PMD_DRV_LOG(DEBUG, "priority_xon_rx[%d]:      %"PRIu64"",
2129                                 i, ns->priority_xon_rx[i]);
2130                 PMD_DRV_LOG(DEBUG, "priority_xoff_rx[%d]:     %"PRIu64"",
2131                                 i, ns->priority_xoff_rx[i]);
2132         }
2133         PMD_DRV_LOG(DEBUG, "link_xon_tx:              %"PRIu64"", ns->link_xon_tx);
2134         PMD_DRV_LOG(DEBUG, "link_xoff_tx:             %"PRIu64"", ns->link_xoff_tx);
2135         for (i = 0; i < 8; i++) {
2136                 PMD_DRV_LOG(DEBUG, "priority_xon_tx[%d]:      %"PRIu64"",
2137                                 i, ns->priority_xon_tx[i]);
2138                 PMD_DRV_LOG(DEBUG, "priority_xoff_tx[%d]:     %"PRIu64"",
2139                                 i, ns->priority_xoff_tx[i]);
2140                 PMD_DRV_LOG(DEBUG, "priority_xon_2_xoff[%d]:  %"PRIu64"",
2141                                 i, ns->priority_xon_2_xoff[i]);
2142         }
2143         PMD_DRV_LOG(DEBUG, "rx_size_64:               %"PRIu64"", ns->rx_size_64);
2144         PMD_DRV_LOG(DEBUG, "rx_size_127:              %"PRIu64"", ns->rx_size_127);
2145         PMD_DRV_LOG(DEBUG, "rx_size_255:              %"PRIu64"", ns->rx_size_255);
2146         PMD_DRV_LOG(DEBUG, "rx_size_511:              %"PRIu64"", ns->rx_size_511);
2147         PMD_DRV_LOG(DEBUG, "rx_size_1023:             %"PRIu64"", ns->rx_size_1023);
2148         PMD_DRV_LOG(DEBUG, "rx_size_1522:             %"PRIu64"", ns->rx_size_1522);
2149         PMD_DRV_LOG(DEBUG, "rx_size_big:              %"PRIu64"", ns->rx_size_big);
2150         PMD_DRV_LOG(DEBUG, "rx_undersize:             %"PRIu64"", ns->rx_undersize);
2151         PMD_DRV_LOG(DEBUG, "rx_fragments:             %"PRIu64"", ns->rx_fragments);
2152         PMD_DRV_LOG(DEBUG, "rx_oversize:              %"PRIu64"", ns->rx_oversize);
2153         PMD_DRV_LOG(DEBUG, "rx_jabber:                %"PRIu64"", ns->rx_jabber);
2154         PMD_DRV_LOG(DEBUG, "tx_size_64:               %"PRIu64"", ns->tx_size_64);
2155         PMD_DRV_LOG(DEBUG, "tx_size_127:              %"PRIu64"", ns->tx_size_127);
2156         PMD_DRV_LOG(DEBUG, "tx_size_255:              %"PRIu64"", ns->tx_size_255);
2157         PMD_DRV_LOG(DEBUG, "tx_size_511:              %"PRIu64"", ns->tx_size_511);
2158         PMD_DRV_LOG(DEBUG, "tx_size_1023:             %"PRIu64"", ns->tx_size_1023);
2159         PMD_DRV_LOG(DEBUG, "tx_size_1522:             %"PRIu64"", ns->tx_size_1522);
2160         PMD_DRV_LOG(DEBUG, "tx_size_big:              %"PRIu64"", ns->tx_size_big);
2161         PMD_DRV_LOG(DEBUG, "mac_short_packet_dropped: %"PRIu64"",
2162                         ns->mac_short_packet_dropped);
2163         PMD_DRV_LOG(DEBUG, "checksum_error:           %"PRIu64"",
2164                     ns->checksum_error);
2165         PMD_DRV_LOG(DEBUG, "fdir_match:               %"PRIu64"", ns->fd_sb_match);
2166         PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************");
2167 }
2168
2169 /* Reset the statistics */
2170 static void
2171 i40e_dev_stats_reset(struct rte_eth_dev *dev)
2172 {
2173         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2174         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2175
2176         /* Mark PF and VSI stats to update the offset, aka "reset" */
2177         pf->offset_loaded = false;
2178         if (pf->main_vsi)
2179                 pf->main_vsi->offset_loaded = false;
2180
2181         /* read the stats, reading current register values into offset */
2182         i40e_read_stats_registers(pf, hw);
2183 }
2184
2185 static uint32_t
2186 i40e_xstats_calc_num(void)
2187 {
2188         return I40E_NB_ETH_XSTATS + I40E_NB_HW_PORT_XSTATS +
2189                 (I40E_NB_RXQ_PRIO_XSTATS * 8) +
2190                 (I40E_NB_TXQ_PRIO_XSTATS * 8);
2191 }
2192
2193 static int
2194 i40e_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
2195                     unsigned n)
2196 {
2197         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2198         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2199         unsigned i, count, prio;
2200         struct i40e_hw_port_stats *hw_stats = &pf->stats;
2201
2202         count = i40e_xstats_calc_num();
2203         if (n < count)
2204                 return count;
2205
2206         i40e_read_stats_registers(pf, hw);
2207
2208         if (xstats == NULL)
2209                 return 0;
2210
2211         count = 0;
2212
2213         /* Get stats from i40e_eth_stats struct */
2214         for (i = 0; i < I40E_NB_ETH_XSTATS; i++) {
2215                 snprintf(xstats[count].name, sizeof(xstats[count].name),
2216                          "%s", rte_i40e_stats_strings[i].name);
2217                 xstats[count].value = *(uint64_t *)(((char *)&hw_stats->eth) +
2218                         rte_i40e_stats_strings[i].offset);
2219                 count++;
2220         }
2221
2222         /* Get individiual stats from i40e_hw_port struct */
2223         for (i = 0; i < I40E_NB_HW_PORT_XSTATS; i++) {
2224                 snprintf(xstats[count].name, sizeof(xstats[count].name),
2225                          "%s", rte_i40e_hw_port_strings[i].name);
2226                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2227                                 rte_i40e_hw_port_strings[i].offset);
2228                 count++;
2229         }
2230
2231         for (i = 0; i < I40E_NB_RXQ_PRIO_XSTATS; i++) {
2232                 for (prio = 0; prio < 8; prio++) {
2233                         snprintf(xstats[count].name,
2234                                  sizeof(xstats[count].name),
2235                                  "rx_priority%u_%s", prio,
2236                                  rte_i40e_rxq_prio_strings[i].name);
2237                         xstats[count].value =
2238                                 *(uint64_t *)(((char *)hw_stats) +
2239                                 rte_i40e_rxq_prio_strings[i].offset +
2240                                 (sizeof(uint64_t) * prio));
2241                         count++;
2242                 }
2243         }
2244
2245         for (i = 0; i < I40E_NB_TXQ_PRIO_XSTATS; i++) {
2246                 for (prio = 0; prio < 8; prio++) {
2247                         snprintf(xstats[count].name,
2248                                  sizeof(xstats[count].name),
2249                                  "tx_priority%u_%s", prio,
2250                                  rte_i40e_txq_prio_strings[i].name);
2251                         xstats[count].value =
2252                                 *(uint64_t *)(((char *)hw_stats) +
2253                                 rte_i40e_txq_prio_strings[i].offset +
2254                                 (sizeof(uint64_t) * prio));
2255                         count++;
2256                 }
2257         }
2258
2259         return count;
2260 }
2261
2262 static int
2263 i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev,
2264                                  __rte_unused uint16_t queue_id,
2265                                  __rte_unused uint8_t stat_idx,
2266                                  __rte_unused uint8_t is_rx)
2267 {
2268         PMD_INIT_FUNC_TRACE();
2269
2270         return -ENOSYS;
2271 }
2272
2273 static void
2274 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2275 {
2276         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2277         struct i40e_vsi *vsi = pf->main_vsi;
2278
2279         dev_info->max_rx_queues = vsi->nb_qps;
2280         dev_info->max_tx_queues = vsi->nb_qps;
2281         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2282         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2283         dev_info->max_mac_addrs = vsi->max_macaddrs;
2284         dev_info->max_vfs = dev->pci_dev->max_vfs;
2285         dev_info->rx_offload_capa =
2286                 DEV_RX_OFFLOAD_VLAN_STRIP |
2287                 DEV_RX_OFFLOAD_QINQ_STRIP |
2288                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2289                 DEV_RX_OFFLOAD_UDP_CKSUM |
2290                 DEV_RX_OFFLOAD_TCP_CKSUM;
2291         dev_info->tx_offload_capa =
2292                 DEV_TX_OFFLOAD_VLAN_INSERT |
2293                 DEV_TX_OFFLOAD_QINQ_INSERT |
2294                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2295                 DEV_TX_OFFLOAD_UDP_CKSUM |
2296                 DEV_TX_OFFLOAD_TCP_CKSUM |
2297                 DEV_TX_OFFLOAD_SCTP_CKSUM |
2298                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
2299                 DEV_TX_OFFLOAD_TCP_TSO;
2300         dev_info->hash_key_size = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
2301                                                 sizeof(uint32_t);
2302         dev_info->reta_size = pf->hash_lut_size;
2303         dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2304
2305         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2306                 .rx_thresh = {
2307                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2308                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2309                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2310                 },
2311                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2312                 .rx_drop_en = 0,
2313         };
2314
2315         dev_info->default_txconf = (struct rte_eth_txconf) {
2316                 .tx_thresh = {
2317                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2318                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2319                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2320                 },
2321                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2322                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2323                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2324                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2325         };
2326
2327         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2328                 .nb_max = I40E_MAX_RING_DESC,
2329                 .nb_min = I40E_MIN_RING_DESC,
2330                 .nb_align = I40E_ALIGN_RING_DESC,
2331         };
2332
2333         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2334                 .nb_max = I40E_MAX_RING_DESC,
2335                 .nb_min = I40E_MIN_RING_DESC,
2336                 .nb_align = I40E_ALIGN_RING_DESC,
2337         };
2338
2339         if (pf->flags & I40E_FLAG_VMDQ) {
2340                 dev_info->max_vmdq_pools = pf->max_nb_vmdq_vsi;
2341                 dev_info->vmdq_queue_base = dev_info->max_rx_queues;
2342                 dev_info->vmdq_queue_num = pf->vmdq_nb_qps *
2343                                                 pf->max_nb_vmdq_vsi;
2344                 dev_info->vmdq_pool_base = I40E_VMDQ_POOL_BASE;
2345                 dev_info->max_rx_queues += dev_info->vmdq_queue_num;
2346                 dev_info->max_tx_queues += dev_info->vmdq_queue_num;
2347         }
2348 }
2349
2350 static int
2351 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2352 {
2353         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2354         struct i40e_vsi *vsi = pf->main_vsi;
2355         PMD_INIT_FUNC_TRACE();
2356
2357         if (on)
2358                 return i40e_vsi_add_vlan(vsi, vlan_id);
2359         else
2360                 return i40e_vsi_delete_vlan(vsi, vlan_id);
2361 }
2362
2363 static int
2364 i40e_vlan_tpid_set(struct rte_eth_dev *dev,
2365                    enum rte_vlan_type vlan_type,
2366                    uint16_t tpid)
2367 {
2368         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2369         uint64_t reg_r = 0, reg_w = 0;
2370         uint16_t reg_id = 0;
2371         int ret = 0;
2372
2373         switch (vlan_type) {
2374         case ETH_VLAN_TYPE_OUTER:
2375                 reg_id = 2;
2376                 break;
2377         case ETH_VLAN_TYPE_INNER:
2378                 reg_id = 3;
2379                 break;
2380         default:
2381                 ret = -EINVAL;
2382                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d", vlan_type);
2383                 return ret;
2384         }
2385         ret = i40e_aq_debug_read_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2386                                           &reg_r, NULL);
2387         if (ret != I40E_SUCCESS) {
2388                 PMD_DRV_LOG(ERR, "Fail to debug read from "
2389                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2390                 ret = -EIO;
2391                 return ret;
2392         }
2393         PMD_DRV_LOG(DEBUG, "Debug read from I40E_GL_SWT_L2TAGCTRL[%d]: "
2394                     "0x%08"PRIx64"", reg_id, reg_r);
2395
2396         reg_w = reg_r & (~(I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_MASK));
2397         reg_w |= ((uint64_t)tpid << I40E_GL_SWT_L2TAGCTRL_ETHERTYPE_SHIFT);
2398         if (reg_r == reg_w) {
2399                 ret = 0;
2400                 PMD_DRV_LOG(DEBUG, "No need to write");
2401                 return ret;
2402         }
2403
2404         ret = i40e_aq_debug_write_register(hw, I40E_GL_SWT_L2TAGCTRL(reg_id),
2405                                            reg_w, NULL);
2406         if (ret != I40E_SUCCESS) {
2407                 ret = -EIO;
2408                 PMD_DRV_LOG(ERR, "Fail to debug write to "
2409                             "I40E_GL_SWT_L2TAGCTRL[%d]", reg_id);
2410                 return ret;
2411         }
2412         PMD_DRV_LOG(DEBUG, "Debug write 0x%08"PRIx64" to "
2413                     "I40E_GL_SWT_L2TAGCTRL[%d]", reg_w, reg_id);
2414
2415         return ret;
2416 }
2417
2418 static void
2419 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2420 {
2421         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2422         struct i40e_vsi *vsi = pf->main_vsi;
2423
2424         if (mask & ETH_VLAN_FILTER_MASK) {
2425                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2426                         i40e_vsi_config_vlan_filter(vsi, TRUE);
2427                 else
2428                         i40e_vsi_config_vlan_filter(vsi, FALSE);
2429         }
2430
2431         if (mask & ETH_VLAN_STRIP_MASK) {
2432                 /* Enable or disable VLAN stripping */
2433                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2434                         i40e_vsi_config_vlan_stripping(vsi, TRUE);
2435                 else
2436                         i40e_vsi_config_vlan_stripping(vsi, FALSE);
2437         }
2438
2439         if (mask & ETH_VLAN_EXTEND_MASK) {
2440                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2441                         i40e_vsi_config_double_vlan(vsi, TRUE);
2442                 else
2443                         i40e_vsi_config_double_vlan(vsi, FALSE);
2444         }
2445 }
2446
2447 static void
2448 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
2449                           __rte_unused uint16_t queue,
2450                           __rte_unused int on)
2451 {
2452         PMD_INIT_FUNC_TRACE();
2453 }
2454
2455 static int
2456 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
2457 {
2458         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2459         struct i40e_vsi *vsi = pf->main_vsi;
2460         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
2461         struct i40e_vsi_vlan_pvid_info info;
2462
2463         memset(&info, 0, sizeof(info));
2464         info.on = on;
2465         if (info.on)
2466                 info.config.pvid = pvid;
2467         else {
2468                 info.config.reject.tagged =
2469                                 data->dev_conf.txmode.hw_vlan_reject_tagged;
2470                 info.config.reject.untagged =
2471                                 data->dev_conf.txmode.hw_vlan_reject_untagged;
2472         }
2473
2474         return i40e_vsi_vlan_pvid_set(vsi, &info);
2475 }
2476
2477 static int
2478 i40e_dev_led_on(struct rte_eth_dev *dev)
2479 {
2480         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2481         uint32_t mode = i40e_led_get(hw);
2482
2483         if (mode == 0)
2484                 i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
2485
2486         return 0;
2487 }
2488
2489 static int
2490 i40e_dev_led_off(struct rte_eth_dev *dev)
2491 {
2492         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2493         uint32_t mode = i40e_led_get(hw);
2494
2495         if (mode != 0)
2496                 i40e_led_set(hw, 0, false);
2497
2498         return 0;
2499 }
2500
2501 static int
2502 i40e_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2503 {
2504         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2505         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2506
2507         fc_conf->pause_time = pf->fc_conf.pause_time;
2508         fc_conf->high_water =  pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS];
2509         fc_conf->low_water = pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS];
2510
2511          /* Return current mode according to actual setting*/
2512         switch (hw->fc.current_mode) {
2513         case I40E_FC_FULL:
2514                 fc_conf->mode = RTE_FC_FULL;
2515                 break;
2516         case I40E_FC_TX_PAUSE:
2517                 fc_conf->mode = RTE_FC_TX_PAUSE;
2518                 break;
2519         case I40E_FC_RX_PAUSE:
2520                 fc_conf->mode = RTE_FC_RX_PAUSE;
2521                 break;
2522         case I40E_FC_NONE:
2523         default:
2524                 fc_conf->mode = RTE_FC_NONE;
2525         };
2526
2527         return 0;
2528 }
2529
2530 static int
2531 i40e_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2532 {
2533         uint32_t mflcn_reg, fctrl_reg, reg;
2534         uint32_t max_high_water;
2535         uint8_t i, aq_failure;
2536         int err;
2537         struct i40e_hw *hw;
2538         struct i40e_pf *pf;
2539         enum i40e_fc_mode rte_fcmode_2_i40e_fcmode[] = {
2540                 [RTE_FC_NONE] = I40E_FC_NONE,
2541                 [RTE_FC_RX_PAUSE] = I40E_FC_RX_PAUSE,
2542                 [RTE_FC_TX_PAUSE] = I40E_FC_TX_PAUSE,
2543                 [RTE_FC_FULL] = I40E_FC_FULL
2544         };
2545
2546         /* high_water field in the rte_eth_fc_conf using the kilobytes unit */
2547
2548         max_high_water = I40E_RXPBSIZE >> I40E_KILOSHIFT;
2549         if ((fc_conf->high_water > max_high_water) ||
2550                         (fc_conf->high_water < fc_conf->low_water)) {
2551                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB, "
2552                         "High_water must <= %d.", max_high_water);
2553                 return -EINVAL;
2554         }
2555
2556         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2557         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2558         hw->fc.requested_mode = rte_fcmode_2_i40e_fcmode[fc_conf->mode];
2559
2560         pf->fc_conf.pause_time = fc_conf->pause_time;
2561         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->high_water;
2562         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = fc_conf->low_water;
2563
2564         PMD_INIT_FUNC_TRACE();
2565
2566         /* All the link flow control related enable/disable register
2567          * configuration is handle by the F/W
2568          */
2569         err = i40e_set_fc(hw, &aq_failure, true);
2570         if (err < 0)
2571                 return -ENOSYS;
2572
2573         if (i40e_is_40G_device(hw->device_id)) {
2574                 /* Configure flow control refresh threshold,
2575                  * the value for stat_tx_pause_refresh_timer[8]
2576                  * is used for global pause operation.
2577                  */
2578
2579                 I40E_WRITE_REG(hw,
2580                                I40E_PRTMAC_HSEC_CTL_TX_PAUSE_REFRESH_TIMER(8),
2581                                pf->fc_conf.pause_time);
2582
2583                 /* configure the timer value included in transmitted pause
2584                  * frame,
2585                  * the value for stat_tx_pause_quanta[8] is used for global
2586                  * pause operation
2587                  */
2588                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_TX_PAUSE_QUANTA(8),
2589                                pf->fc_conf.pause_time);
2590
2591                 fctrl_reg = I40E_READ_REG(hw,
2592                                           I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL);
2593
2594                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2595                         fctrl_reg |= I40E_PRTMAC_FWD_CTRL;
2596                 else
2597                         fctrl_reg &= ~I40E_PRTMAC_FWD_CTRL;
2598
2599                 I40E_WRITE_REG(hw, I40E_PRTMAC_HSEC_CTL_RX_FORWARD_CONTROL,
2600                                fctrl_reg);
2601         } else {
2602                 /* Configure pause time (2 TCs per register) */
2603                 reg = (uint32_t)pf->fc_conf.pause_time * (uint32_t)0x00010001;
2604                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS / 2; i++)
2605                         I40E_WRITE_REG(hw, I40E_PRTDCB_FCTTVN(i), reg);
2606
2607                 /* Configure flow control refresh threshold value */
2608                 I40E_WRITE_REG(hw, I40E_PRTDCB_FCRTV,
2609                                pf->fc_conf.pause_time / 2);
2610
2611                 mflcn_reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
2612
2613                 /* set or clear MFLCN.PMCF & MFLCN.DPF bits
2614                  *depending on configuration
2615                  */
2616                 if (fc_conf->mac_ctrl_frame_fwd != 0) {
2617                         mflcn_reg |= I40E_PRTDCB_MFLCN_PMCF_MASK;
2618                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_DPF_MASK;
2619                 } else {
2620                         mflcn_reg &= ~I40E_PRTDCB_MFLCN_PMCF_MASK;
2621                         mflcn_reg |= I40E_PRTDCB_MFLCN_DPF_MASK;
2622                 }
2623
2624                 I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, mflcn_reg);
2625         }
2626
2627         /* config the water marker both based on the packets and bytes */
2628         I40E_WRITE_REG(hw, I40E_GLRPB_PHW,
2629                        (pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
2630                        << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
2631         I40E_WRITE_REG(hw, I40E_GLRPB_PLW,
2632                        (pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
2633                        << I40E_KILOSHIFT) / I40E_PACKET_AVERAGE_SIZE);
2634         I40E_WRITE_REG(hw, I40E_GLRPB_GHW,
2635                        pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS]
2636                        << I40E_KILOSHIFT);
2637         I40E_WRITE_REG(hw, I40E_GLRPB_GLW,
2638                        pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS]
2639                        << I40E_KILOSHIFT);
2640
2641         I40E_WRITE_FLUSH(hw);
2642
2643         return 0;
2644 }
2645
2646 static int
2647 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
2648                             __rte_unused struct rte_eth_pfc_conf *pfc_conf)
2649 {
2650         PMD_INIT_FUNC_TRACE();
2651
2652         return -ENOSYS;
2653 }
2654
2655 /* Add a MAC address, and update filters */
2656 static void
2657 i40e_macaddr_add(struct rte_eth_dev *dev,
2658                  struct ether_addr *mac_addr,
2659                  __rte_unused uint32_t index,
2660                  uint32_t pool)
2661 {
2662         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2663         struct i40e_mac_filter_info mac_filter;
2664         struct i40e_vsi *vsi;
2665         int ret;
2666
2667         /* If VMDQ not enabled or configured, return */
2668         if (pool != 0 && (!(pf->flags | I40E_FLAG_VMDQ) || !pf->nb_cfg_vmdq_vsi)) {
2669                 PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u",
2670                         pf->flags | I40E_FLAG_VMDQ ? "configured" : "enabled",
2671                         pool);
2672                 return;
2673         }
2674
2675         if (pool > pf->nb_cfg_vmdq_vsi) {
2676                 PMD_DRV_LOG(ERR, "Pool number %u invalid. Max pool is %u",
2677                                 pool, pf->nb_cfg_vmdq_vsi);
2678                 return;
2679         }
2680
2681         (void)rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN);
2682         if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2683                 mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
2684         else
2685                 mac_filter.filter_type = RTE_MAC_PERFECT_MATCH;
2686
2687         if (pool == 0)
2688                 vsi = pf->main_vsi;
2689         else
2690                 vsi = pf->vmdq[pool - 1].vsi;
2691
2692         ret = i40e_vsi_add_mac(vsi, &mac_filter);
2693         if (ret != I40E_SUCCESS) {
2694                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
2695                 return;
2696         }
2697 }
2698
2699 /* Remove a MAC address, and update filters */
2700 static void
2701 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
2702 {
2703         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2704         struct i40e_vsi *vsi;
2705         struct rte_eth_dev_data *data = dev->data;
2706         struct ether_addr *macaddr;
2707         int ret;
2708         uint32_t i;
2709         uint64_t pool_sel;
2710
2711         macaddr = &(data->mac_addrs[index]);
2712
2713         pool_sel = dev->data->mac_pool_sel[index];
2714
2715         for (i = 0; i < sizeof(pool_sel) * CHAR_BIT; i++) {
2716                 if (pool_sel & (1ULL << i)) {
2717                         if (i == 0)
2718                                 vsi = pf->main_vsi;
2719                         else {
2720                                 /* No VMDQ pool enabled or configured */
2721                                 if (!(pf->flags | I40E_FLAG_VMDQ) ||
2722                                         (i > pf->nb_cfg_vmdq_vsi)) {
2723                                         PMD_DRV_LOG(ERR, "No VMDQ pool enabled"
2724                                                         "/configured");
2725                                         return;
2726                                 }
2727                                 vsi = pf->vmdq[i - 1].vsi;
2728                         }
2729                         ret = i40e_vsi_delete_mac(vsi, macaddr);
2730
2731                         if (ret) {
2732                                 PMD_DRV_LOG(ERR, "Failed to remove MACVLAN filter");
2733                                 return;
2734                         }
2735                 }
2736         }
2737 }
2738
2739 /* Set perfect match or hash match of MAC and VLAN for a VF */
2740 static int
2741 i40e_vf_mac_filter_set(struct i40e_pf *pf,
2742                  struct rte_eth_mac_filter *filter,
2743                  bool add)
2744 {
2745         struct i40e_hw *hw;
2746         struct i40e_mac_filter_info mac_filter;
2747         struct ether_addr old_mac;
2748         struct ether_addr *new_mac;
2749         struct i40e_pf_vf *vf = NULL;
2750         uint16_t vf_id;
2751         int ret;
2752
2753         if (pf == NULL) {
2754                 PMD_DRV_LOG(ERR, "Invalid PF argument.");
2755                 return -EINVAL;
2756         }
2757         hw = I40E_PF_TO_HW(pf);
2758
2759         if (filter == NULL) {
2760                 PMD_DRV_LOG(ERR, "Invalid mac filter argument.");
2761                 return -EINVAL;
2762         }
2763
2764         new_mac = &filter->mac_addr;
2765
2766         if (is_zero_ether_addr(new_mac)) {
2767                 PMD_DRV_LOG(ERR, "Invalid ethernet address.");
2768                 return -EINVAL;
2769         }
2770
2771         vf_id = filter->dst_id;
2772
2773         if (vf_id > pf->vf_num - 1 || !pf->vfs) {
2774                 PMD_DRV_LOG(ERR, "Invalid argument.");
2775                 return -EINVAL;
2776         }
2777         vf = &pf->vfs[vf_id];
2778
2779         if (add && is_same_ether_addr(new_mac, &(pf->dev_addr))) {
2780                 PMD_DRV_LOG(INFO, "Ignore adding permanent MAC address.");
2781                 return -EINVAL;
2782         }
2783
2784         if (add) {
2785                 (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
2786                 (void)rte_memcpy(hw->mac.addr, new_mac->addr_bytes,
2787                                 ETHER_ADDR_LEN);
2788                 (void)rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr,
2789                                  ETHER_ADDR_LEN);
2790
2791                 mac_filter.filter_type = filter->filter_type;
2792                 ret = i40e_vsi_add_mac(vf->vsi, &mac_filter);
2793                 if (ret != I40E_SUCCESS) {
2794                         PMD_DRV_LOG(ERR, "Failed to add MAC filter.");
2795                         return -1;
2796                 }
2797                 ether_addr_copy(new_mac, &pf->dev_addr);
2798         } else {
2799                 (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr,
2800                                 ETHER_ADDR_LEN);
2801                 ret = i40e_vsi_delete_mac(vf->vsi, &filter->mac_addr);
2802                 if (ret != I40E_SUCCESS) {
2803                         PMD_DRV_LOG(ERR, "Failed to delete MAC filter.");
2804                         return -1;
2805                 }
2806
2807                 /* Clear device address as it has been removed */
2808                 if (is_same_ether_addr(&(pf->dev_addr), new_mac))
2809                         memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
2810         }
2811
2812         return 0;
2813 }
2814
2815 /* MAC filter handle */
2816 static int
2817 i40e_mac_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
2818                 void *arg)
2819 {
2820         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2821         struct rte_eth_mac_filter *filter;
2822         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2823         int ret = I40E_NOT_SUPPORTED;
2824
2825         filter = (struct rte_eth_mac_filter *)(arg);
2826
2827         switch (filter_op) {
2828         case RTE_ETH_FILTER_NOP:
2829                 ret = I40E_SUCCESS;
2830                 break;
2831         case RTE_ETH_FILTER_ADD:
2832                 i40e_pf_disable_irq0(hw);
2833                 if (filter->is_vf)
2834                         ret = i40e_vf_mac_filter_set(pf, filter, 1);
2835                 i40e_pf_enable_irq0(hw);
2836                 break;
2837         case RTE_ETH_FILTER_DELETE:
2838                 i40e_pf_disable_irq0(hw);
2839                 if (filter->is_vf)
2840                         ret = i40e_vf_mac_filter_set(pf, filter, 0);
2841                 i40e_pf_enable_irq0(hw);
2842                 break;
2843         default:
2844                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
2845                 ret = I40E_ERR_PARAM;
2846                 break;
2847         }
2848
2849         return ret;
2850 }
2851
2852 static int
2853 i40e_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2854 {
2855         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
2856         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2857         int ret;
2858
2859         if (!lut)
2860                 return -EINVAL;
2861
2862         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2863                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, TRUE,
2864                                           lut, lut_size);
2865                 if (ret) {
2866                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
2867                         return ret;
2868                 }
2869         } else {
2870                 uint32_t *lut_dw = (uint32_t *)lut;
2871                 uint16_t i, lut_size_dw = lut_size / 4;
2872
2873                 for (i = 0; i < lut_size_dw; i++)
2874                         lut_dw[i] = I40E_READ_REG(hw, I40E_PFQF_HLUT(i));
2875         }
2876
2877         return 0;
2878 }
2879
2880 static int
2881 i40e_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2882 {
2883         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
2884         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2885         int ret;
2886
2887         if (!vsi || !lut)
2888                 return -EINVAL;
2889
2890         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2891                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, TRUE,
2892                                           lut, lut_size);
2893                 if (ret) {
2894                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
2895                         return ret;
2896                 }
2897         } else {
2898                 uint32_t *lut_dw = (uint32_t *)lut;
2899                 uint16_t i, lut_size_dw = lut_size / 4;
2900
2901                 for (i = 0; i < lut_size_dw; i++)
2902                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
2903                 I40E_WRITE_FLUSH(hw);
2904         }
2905
2906         return 0;
2907 }
2908
2909 static int
2910 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
2911                          struct rte_eth_rss_reta_entry64 *reta_conf,
2912                          uint16_t reta_size)
2913 {
2914         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2915         uint16_t i, lut_size = pf->hash_lut_size;
2916         uint16_t idx, shift;
2917         uint8_t *lut;
2918         int ret;
2919
2920         if (reta_size != lut_size ||
2921                 reta_size > ETH_RSS_RETA_SIZE_512) {
2922                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2923                         "(%d) doesn't match the number hardware can supported "
2924                                         "(%d)\n", reta_size, lut_size);
2925                 return -EINVAL;
2926         }
2927
2928         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2929         if (!lut) {
2930                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2931                 return -ENOMEM;
2932         }
2933         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
2934         if (ret)
2935                 goto out;
2936         for (i = 0; i < reta_size; i++) {
2937                 idx = i / RTE_RETA_GROUP_SIZE;
2938                 shift = i % RTE_RETA_GROUP_SIZE;
2939                 if (reta_conf[idx].mask & (1ULL << shift))
2940                         lut[i] = reta_conf[idx].reta[shift];
2941         }
2942         ret = i40e_set_rss_lut(pf->main_vsi, lut, reta_size);
2943
2944 out:
2945         rte_free(lut);
2946
2947         return ret;
2948 }
2949
2950 static int
2951 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
2952                         struct rte_eth_rss_reta_entry64 *reta_conf,
2953                         uint16_t reta_size)
2954 {
2955         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2956         uint16_t i, lut_size = pf->hash_lut_size;
2957         uint16_t idx, shift;
2958         uint8_t *lut;
2959         int ret;
2960
2961         if (reta_size != lut_size ||
2962                 reta_size > ETH_RSS_RETA_SIZE_512) {
2963                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2964                         "(%d) doesn't match the number hardware can supported "
2965                                         "(%d)\n", reta_size, lut_size);
2966                 return -EINVAL;
2967         }
2968
2969         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2970         if (!lut) {
2971                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2972                 return -ENOMEM;
2973         }
2974
2975         ret = i40e_get_rss_lut(pf->main_vsi, lut, reta_size);
2976         if (ret)
2977                 goto out;
2978         for (i = 0; i < reta_size; i++) {
2979                 idx = i / RTE_RETA_GROUP_SIZE;
2980                 shift = i % RTE_RETA_GROUP_SIZE;
2981                 if (reta_conf[idx].mask & (1ULL << shift))
2982                         reta_conf[idx].reta[shift] = lut[i];
2983         }
2984
2985 out:
2986         rte_free(lut);
2987
2988         return ret;
2989 }
2990
2991 /**
2992  * i40e_allocate_dma_mem_d - specific memory alloc for shared code (base driver)
2993  * @hw:   pointer to the HW structure
2994  * @mem:  pointer to mem struct to fill out
2995  * @size: size of memory requested
2996  * @alignment: what to align the allocation to
2997  **/
2998 enum i40e_status_code
2999 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3000                         struct i40e_dma_mem *mem,
3001                         u64 size,
3002                         u32 alignment)
3003 {
3004         const struct rte_memzone *mz = NULL;
3005         char z_name[RTE_MEMZONE_NAMESIZE];
3006
3007         if (!mem)
3008                 return I40E_ERR_PARAM;
3009
3010         snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand());
3011         mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
3012                                          alignment, RTE_PGSIZE_2M);
3013         if (!mz)
3014                 return I40E_ERR_NO_MEMORY;
3015
3016         mem->size = size;
3017         mem->va = mz->addr;
3018         mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
3019         mem->zone = (const void *)mz;
3020         PMD_DRV_LOG(DEBUG, "memzone %s allocated with physical address: "
3021                     "%"PRIu64, mz->name, mem->pa);
3022
3023         return I40E_SUCCESS;
3024 }
3025
3026 /**
3027  * i40e_free_dma_mem_d - specific memory free for shared code (base driver)
3028  * @hw:   pointer to the HW structure
3029  * @mem:  ptr to mem struct to free
3030  **/
3031 enum i40e_status_code
3032 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3033                     struct i40e_dma_mem *mem)
3034 {
3035         if (!mem)
3036                 return I40E_ERR_PARAM;
3037
3038         PMD_DRV_LOG(DEBUG, "memzone %s to be freed with physical address: "
3039                     "%"PRIu64, ((const struct rte_memzone *)mem->zone)->name,
3040                     mem->pa);
3041         rte_memzone_free((const struct rte_memzone *)mem->zone);
3042         mem->zone = NULL;
3043         mem->va = NULL;
3044         mem->pa = (u64)0;
3045
3046         return I40E_SUCCESS;
3047 }
3048
3049 /**
3050  * i40e_allocate_virt_mem_d - specific memory alloc for shared code (base driver)
3051  * @hw:   pointer to the HW structure
3052  * @mem:  pointer to mem struct to fill out
3053  * @size: size of memory requested
3054  **/
3055 enum i40e_status_code
3056 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3057                          struct i40e_virt_mem *mem,
3058                          u32 size)
3059 {
3060         if (!mem)
3061                 return I40E_ERR_PARAM;
3062
3063         mem->size = size;
3064         mem->va = rte_zmalloc("i40e", size, 0);
3065
3066         if (mem->va)
3067                 return I40E_SUCCESS;
3068         else
3069                 return I40E_ERR_NO_MEMORY;
3070 }
3071
3072 /**
3073  * i40e_free_virt_mem_d - specific memory free for shared code (base driver)
3074  * @hw:   pointer to the HW structure
3075  * @mem:  pointer to mem struct to free
3076  **/
3077 enum i40e_status_code
3078 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
3079                      struct i40e_virt_mem *mem)
3080 {
3081         if (!mem)
3082                 return I40E_ERR_PARAM;
3083
3084         rte_free(mem->va);
3085         mem->va = NULL;
3086
3087         return I40E_SUCCESS;
3088 }
3089
3090 void
3091 i40e_init_spinlock_d(struct i40e_spinlock *sp)
3092 {
3093         rte_spinlock_init(&sp->spinlock);
3094 }
3095
3096 void
3097 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
3098 {
3099         rte_spinlock_lock(&sp->spinlock);
3100 }
3101
3102 void
3103 i40e_release_spinlock_d(struct i40e_spinlock *sp)
3104 {
3105         rte_spinlock_unlock(&sp->spinlock);
3106 }
3107
3108 void
3109 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
3110 {
3111         return;
3112 }
3113
3114 /**
3115  * Get the hardware capabilities, which will be parsed
3116  * and saved into struct i40e_hw.
3117  */
3118 static int
3119 i40e_get_cap(struct i40e_hw *hw)
3120 {
3121         struct i40e_aqc_list_capabilities_element_resp *buf;
3122         uint16_t len, size = 0;
3123         int ret;
3124
3125         /* Calculate a huge enough buff for saving response data temporarily */
3126         len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
3127                                                 I40E_MAX_CAP_ELE_NUM;
3128         buf = rte_zmalloc("i40e", len, 0);
3129         if (!buf) {
3130                 PMD_DRV_LOG(ERR, "Failed to allocate memory");
3131                 return I40E_ERR_NO_MEMORY;
3132         }
3133
3134         /* Get, parse the capabilities and save it to hw */
3135         ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
3136                         i40e_aqc_opc_list_func_capabilities, NULL);
3137         if (ret != I40E_SUCCESS)
3138                 PMD_DRV_LOG(ERR, "Failed to discover capabilities");
3139
3140         /* Free the temporary buffer after being used */
3141         rte_free(buf);
3142
3143         return ret;
3144 }
3145
3146 static int
3147 i40e_pf_parameter_init(struct rte_eth_dev *dev)
3148 {
3149         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3150         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3151         uint16_t qp_count = 0, vsi_count = 0;
3152
3153         if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
3154                 PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV");
3155                 return -EINVAL;
3156         }
3157         /* Add the parameter init for LFC */
3158         pf->fc_conf.pause_time = I40E_DEFAULT_PAUSE_TIME;
3159         pf->fc_conf.high_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_HIGH_WATER;
3160         pf->fc_conf.low_water[I40E_MAX_TRAFFIC_CLASS] = I40E_DEFAULT_LOW_WATER;
3161
3162         pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
3163         pf->max_num_vsi = hw->func_caps.num_vsis;
3164         pf->lan_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_PF;
3165         pf->vmdq_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
3166         pf->vf_nb_qp_max = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3167
3168         /* FDir queue/VSI allocation */
3169         pf->fdir_qp_offset = 0;
3170         if (hw->func_caps.fd) {
3171                 pf->flags |= I40E_FLAG_FDIR;
3172                 pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
3173         } else {
3174                 pf->fdir_nb_qps = 0;
3175         }
3176         qp_count += pf->fdir_nb_qps;
3177         vsi_count += 1;
3178
3179         /* LAN queue/VSI allocation */
3180         pf->lan_qp_offset = pf->fdir_qp_offset + pf->fdir_nb_qps;
3181         if (!hw->func_caps.rss) {
3182                 pf->lan_nb_qps = 1;
3183         } else {
3184                 pf->flags |= I40E_FLAG_RSS;
3185                 if (hw->mac.type == I40E_MAC_X722)
3186                         pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE;
3187                 pf->lan_nb_qps = pf->lan_nb_qp_max;
3188         }
3189         qp_count += pf->lan_nb_qps;
3190         vsi_count += 1;
3191
3192         /* VF queue/VSI allocation */
3193         pf->vf_qp_offset = pf->lan_qp_offset + pf->lan_nb_qps;
3194         if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) {
3195                 pf->flags |= I40E_FLAG_SRIOV;
3196                 pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
3197                 pf->vf_num = dev->pci_dev->max_vfs;
3198                 PMD_DRV_LOG(DEBUG, "%u VF VSIs, %u queues per VF VSI, "
3199                             "in total %u queues", pf->vf_num, pf->vf_nb_qps,
3200                             pf->vf_nb_qps * pf->vf_num);
3201         } else {
3202                 pf->vf_nb_qps = 0;
3203                 pf->vf_num = 0;
3204         }
3205         qp_count += pf->vf_nb_qps * pf->vf_num;
3206         vsi_count += pf->vf_num;
3207
3208         /* VMDq queue/VSI allocation */
3209         pf->vmdq_qp_offset = pf->vf_qp_offset + pf->vf_nb_qps * pf->vf_num;
3210         pf->vmdq_nb_qps = 0;
3211         pf->max_nb_vmdq_vsi = 0;
3212         if (hw->func_caps.vmdq) {
3213                 if (qp_count < hw->func_caps.num_tx_qp &&
3214                         vsi_count < hw->func_caps.num_vsis) {
3215                         pf->max_nb_vmdq_vsi = (hw->func_caps.num_tx_qp -
3216                                 qp_count) / pf->vmdq_nb_qp_max;
3217
3218                         /* Limit the maximum number of VMDq vsi to the maximum
3219                          * ethdev can support
3220                          */
3221                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3222                                 hw->func_caps.num_vsis - vsi_count);
3223                         pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
3224                                 ETH_64_POOLS);
3225                         if (pf->max_nb_vmdq_vsi) {
3226                                 pf->flags |= I40E_FLAG_VMDQ;
3227                                 pf->vmdq_nb_qps = pf->vmdq_nb_qp_max;
3228                                 PMD_DRV_LOG(DEBUG, "%u VMDQ VSIs, %u queues "
3229                                             "per VMDQ VSI, in total %u queues",
3230                                             pf->max_nb_vmdq_vsi,
3231                                             pf->vmdq_nb_qps, pf->vmdq_nb_qps *
3232                                             pf->max_nb_vmdq_vsi);
3233                         } else {
3234                                 PMD_DRV_LOG(INFO, "No enough queues left for "
3235                                             "VMDq");
3236                         }
3237                 } else {
3238                         PMD_DRV_LOG(INFO, "No queue or VSI left for VMDq");
3239                 }
3240         }
3241         qp_count += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi;
3242         vsi_count += pf->max_nb_vmdq_vsi;
3243
3244         if (hw->func_caps.dcb)
3245                 pf->flags |= I40E_FLAG_DCB;
3246
3247         if (qp_count > hw->func_caps.num_tx_qp) {
3248                 PMD_DRV_LOG(ERR, "Failed to allocate %u queues, which exceeds "
3249                             "the hardware maximum %u", qp_count,
3250                             hw->func_caps.num_tx_qp);
3251                 return -EINVAL;
3252         }
3253         if (vsi_count > hw->func_caps.num_vsis) {
3254                 PMD_DRV_LOG(ERR, "Failed to allocate %u VSIs, which exceeds "
3255                             "the hardware maximum %u", vsi_count,
3256                             hw->func_caps.num_vsis);
3257                 return -EINVAL;
3258         }
3259
3260         return 0;
3261 }
3262
3263 static int
3264 i40e_pf_get_switch_config(struct i40e_pf *pf)
3265 {
3266         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3267         struct i40e_aqc_get_switch_config_resp *switch_config;
3268         struct i40e_aqc_switch_config_element_resp *element;
3269         uint16_t start_seid = 0, num_reported;
3270         int ret;
3271
3272         switch_config = (struct i40e_aqc_get_switch_config_resp *)\
3273                         rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
3274         if (!switch_config) {
3275                 PMD_DRV_LOG(ERR, "Failed to allocated memory");
3276                 return -ENOMEM;
3277         }
3278
3279         /* Get the switch configurations */
3280         ret = i40e_aq_get_switch_config(hw, switch_config,
3281                 I40E_AQ_LARGE_BUF, &start_seid, NULL);
3282         if (ret != I40E_SUCCESS) {
3283                 PMD_DRV_LOG(ERR, "Failed to get switch configurations");
3284                 goto fail;
3285         }
3286         num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
3287         if (num_reported != 1) { /* The number should be 1 */
3288                 PMD_DRV_LOG(ERR, "Wrong number of switch config reported");
3289                 goto fail;
3290         }
3291
3292         /* Parse the switch configuration elements */
3293         element = &(switch_config->element[0]);
3294         if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
3295                 pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
3296                 pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
3297         } else
3298                 PMD_DRV_LOG(INFO, "Unknown element type");
3299
3300 fail:
3301         rte_free(switch_config);
3302
3303         return ret;
3304 }
3305
3306 static int
3307 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
3308                         uint32_t num)
3309 {
3310         struct pool_entry *entry;
3311
3312         if (pool == NULL || num == 0)
3313                 return -EINVAL;
3314
3315         entry = rte_zmalloc("i40e", sizeof(*entry), 0);
3316         if (entry == NULL) {
3317                 PMD_DRV_LOG(ERR, "Failed to allocate memory for resource pool");
3318                 return -ENOMEM;
3319         }
3320
3321         /* queue heap initialize */
3322         pool->num_free = num;
3323         pool->num_alloc = 0;
3324         pool->base = base;
3325         LIST_INIT(&pool->alloc_list);
3326         LIST_INIT(&pool->free_list);
3327
3328         /* Initialize element  */
3329         entry->base = 0;
3330         entry->len = num;
3331
3332         LIST_INSERT_HEAD(&pool->free_list, entry, next);
3333         return 0;
3334 }
3335
3336 static void
3337 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
3338 {
3339         struct pool_entry *entry;
3340
3341         if (pool == NULL)
3342                 return;
3343
3344         LIST_FOREACH(entry, &pool->alloc_list, next) {
3345                 LIST_REMOVE(entry, next);
3346                 rte_free(entry);
3347         }
3348
3349         LIST_FOREACH(entry, &pool->free_list, next) {
3350                 LIST_REMOVE(entry, next);
3351                 rte_free(entry);
3352         }
3353
3354         pool->num_free = 0;
3355         pool->num_alloc = 0;
3356         pool->base = 0;
3357         LIST_INIT(&pool->alloc_list);
3358         LIST_INIT(&pool->free_list);
3359 }
3360
3361 static int
3362 i40e_res_pool_free(struct i40e_res_pool_info *pool,
3363                        uint32_t base)
3364 {
3365         struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
3366         uint32_t pool_offset;
3367         int insert;
3368
3369         if (pool == NULL) {
3370                 PMD_DRV_LOG(ERR, "Invalid parameter");
3371                 return -EINVAL;
3372         }
3373
3374         pool_offset = base - pool->base;
3375         /* Lookup in alloc list */
3376         LIST_FOREACH(entry, &pool->alloc_list, next) {
3377                 if (entry->base == pool_offset) {
3378                         valid_entry = entry;
3379                         LIST_REMOVE(entry, next);
3380                         break;
3381                 }
3382         }
3383
3384         /* Not find, return */
3385         if (valid_entry == NULL) {
3386                 PMD_DRV_LOG(ERR, "Failed to find entry");
3387                 return -EINVAL;
3388         }
3389
3390         /**
3391          * Found it, move it to free list  and try to merge.
3392          * In order to make merge easier, always sort it by qbase.
3393          * Find adjacent prev and last entries.
3394          */
3395         prev = next = NULL;
3396         LIST_FOREACH(entry, &pool->free_list, next) {
3397                 if (entry->base > valid_entry->base) {
3398                         next = entry;
3399                         break;
3400                 }
3401                 prev = entry;
3402         }
3403
3404         insert = 0;
3405         /* Try to merge with next one*/
3406         if (next != NULL) {
3407                 /* Merge with next one */
3408                 if (valid_entry->base + valid_entry->len == next->base) {
3409                         next->base = valid_entry->base;
3410                         next->len += valid_entry->len;
3411                         rte_free(valid_entry);
3412                         valid_entry = next;
3413                         insert = 1;
3414                 }
3415         }
3416
3417         if (prev != NULL) {
3418                 /* Merge with previous one */
3419                 if (prev->base + prev->len == valid_entry->base) {
3420                         prev->len += valid_entry->len;
3421                         /* If it merge with next one, remove next node */
3422                         if (insert == 1) {
3423                                 LIST_REMOVE(valid_entry, next);
3424                                 rte_free(valid_entry);
3425                         } else {
3426                                 rte_free(valid_entry);
3427                                 insert = 1;
3428                         }
3429                 }
3430         }
3431
3432         /* Not find any entry to merge, insert */
3433         if (insert == 0) {
3434                 if (prev != NULL)
3435                         LIST_INSERT_AFTER(prev, valid_entry, next);
3436                 else if (next != NULL)
3437                         LIST_INSERT_BEFORE(next, valid_entry, next);
3438                 else /* It's empty list, insert to head */
3439                         LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
3440         }
3441
3442         pool->num_free += valid_entry->len;
3443         pool->num_alloc -= valid_entry->len;
3444
3445         return 0;
3446 }
3447
3448 static int
3449 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
3450                        uint16_t num)
3451 {
3452         struct pool_entry *entry, *valid_entry;
3453
3454         if (pool == NULL || num == 0) {
3455                 PMD_DRV_LOG(ERR, "Invalid parameter");
3456                 return -EINVAL;
3457         }
3458
3459         if (pool->num_free < num) {
3460                 PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u",
3461                             num, pool->num_free);
3462                 return -ENOMEM;
3463         }
3464
3465         valid_entry = NULL;
3466         /* Lookup  in free list and find most fit one */
3467         LIST_FOREACH(entry, &pool->free_list, next) {
3468                 if (entry->len >= num) {
3469                         /* Find best one */
3470                         if (entry->len == num) {
3471                                 valid_entry = entry;
3472                                 break;
3473                         }
3474                         if (valid_entry == NULL || valid_entry->len > entry->len)
3475                                 valid_entry = entry;
3476                 }
3477         }
3478
3479         /* Not find one to satisfy the request, return */
3480         if (valid_entry == NULL) {
3481                 PMD_DRV_LOG(ERR, "No valid entry found");
3482                 return -ENOMEM;
3483         }
3484         /**
3485          * The entry have equal queue number as requested,
3486          * remove it from alloc_list.
3487          */
3488         if (valid_entry->len == num) {
3489                 LIST_REMOVE(valid_entry, next);
3490         } else {
3491                 /**
3492                  * The entry have more numbers than requested,
3493                  * create a new entry for alloc_list and minus its
3494                  * queue base and number in free_list.
3495                  */
3496                 entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
3497                 if (entry == NULL) {
3498                         PMD_DRV_LOG(ERR, "Failed to allocate memory for "
3499                                     "resource pool");
3500                         return -ENOMEM;
3501                 }
3502                 entry->base = valid_entry->base;
3503                 entry->len = num;
3504                 valid_entry->base += num;
3505                 valid_entry->len -= num;
3506                 valid_entry = entry;
3507         }
3508
3509         /* Insert it into alloc list, not sorted */
3510         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
3511
3512         pool->num_free -= valid_entry->len;
3513         pool->num_alloc += valid_entry->len;
3514
3515         return valid_entry->base + pool->base;
3516 }
3517
3518 /**
3519  * bitmap_is_subset - Check whether src2 is subset of src1
3520  **/
3521 static inline int
3522 bitmap_is_subset(uint8_t src1, uint8_t src2)
3523 {
3524         return !((src1 ^ src2) & src2);
3525 }
3526
3527 static enum i40e_status_code
3528 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
3529 {
3530         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3531
3532         /* If DCB is not supported, only default TC is supported */
3533         if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
3534                 PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported");
3535                 return I40E_NOT_SUPPORTED;
3536         }
3537
3538         if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
3539                 PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to "
3540                             "HW support 0x%x", hw->func_caps.enabled_tcmap,
3541                             enabled_tcmap);
3542                 return I40E_NOT_SUPPORTED;
3543         }
3544         return I40E_SUCCESS;
3545 }
3546
3547 int
3548 i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi,
3549                                 struct i40e_vsi_vlan_pvid_info *info)
3550 {
3551         struct i40e_hw *hw;
3552         struct i40e_vsi_context ctxt;
3553         uint8_t vlan_flags = 0;
3554         int ret;
3555
3556         if (vsi == NULL || info == NULL) {
3557                 PMD_DRV_LOG(ERR, "invalid parameters");
3558                 return I40E_ERR_PARAM;
3559         }
3560
3561         if (info->on) {
3562                 vsi->info.pvid = info->config.pvid;
3563                 /**
3564                  * If insert pvid is enabled, only tagged pkts are
3565                  * allowed to be sent out.
3566                  */
3567                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
3568                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
3569         } else {
3570                 vsi->info.pvid = 0;
3571                 if (info->config.reject.tagged == 0)
3572                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
3573
3574                 if (info->config.reject.untagged == 0)
3575                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
3576         }
3577         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
3578                                         I40E_AQ_VSI_PVLAN_MODE_MASK);
3579         vsi->info.port_vlan_flags |= vlan_flags;
3580         vsi->info.valid_sections =
3581                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
3582         memset(&ctxt, 0, sizeof(ctxt));
3583         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3584         ctxt.seid = vsi->seid;
3585
3586         hw = I40E_VSI_TO_HW(vsi);
3587         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3588         if (ret != I40E_SUCCESS)
3589                 PMD_DRV_LOG(ERR, "Failed to update VSI params");
3590
3591         return ret;
3592 }
3593
3594 static int
3595 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
3596 {
3597         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3598         int i, ret;
3599         struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
3600
3601         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
3602         if (ret != I40E_SUCCESS)
3603                 return ret;
3604
3605         if (!vsi->seid) {
3606                 PMD_DRV_LOG(ERR, "seid not valid");
3607                 return -EINVAL;
3608         }
3609
3610         memset(&tc_bw_data, 0, sizeof(tc_bw_data));
3611         tc_bw_data.tc_valid_bits = enabled_tcmap;
3612         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3613                 tc_bw_data.tc_bw_credits[i] =
3614                         (enabled_tcmap & (1 << i)) ? 1 : 0;
3615
3616         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
3617         if (ret != I40E_SUCCESS) {
3618                 PMD_DRV_LOG(ERR, "Failed to configure TC BW");
3619                 return ret;
3620         }
3621
3622         (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
3623                                         sizeof(vsi->info.qs_handle));
3624         return I40E_SUCCESS;
3625 }
3626
3627 static enum i40e_status_code
3628 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
3629                                  struct i40e_aqc_vsi_properties_data *info,
3630                                  uint8_t enabled_tcmap)
3631 {
3632         enum i40e_status_code ret;
3633         int i, total_tc = 0;
3634         uint16_t qpnum_per_tc, bsf, qp_idx;
3635
3636         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
3637         if (ret != I40E_SUCCESS)
3638                 return ret;
3639
3640         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3641                 if (enabled_tcmap & (1 << i))
3642                         total_tc++;
3643         vsi->enabled_tc = enabled_tcmap;
3644
3645         /* Number of queues per enabled TC */
3646         qpnum_per_tc = i40e_align_floor(vsi->nb_qps / total_tc);
3647         qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
3648         bsf = rte_bsf32(qpnum_per_tc);
3649
3650         /* Adjust the queue number to actual queues that can be applied */
3651         if (!(vsi->type == I40E_VSI_MAIN && total_tc == 1))
3652                 vsi->nb_qps = qpnum_per_tc * total_tc;
3653
3654         /**
3655          * Configure TC and queue mapping parameters, for enabled TC,
3656          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
3657          * default queue will serve it.
3658          */
3659         qp_idx = 0;
3660         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3661                 if (vsi->enabled_tc & (1 << i)) {
3662                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
3663                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
3664                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
3665                         qp_idx += qpnum_per_tc;
3666                 } else
3667                         info->tc_mapping[i] = 0;
3668         }
3669
3670         /* Associate queue number with VSI */
3671         if (vsi->type == I40E_VSI_SRIOV) {
3672                 info->mapping_flags |=
3673                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
3674                 for (i = 0; i < vsi->nb_qps; i++)
3675                         info->queue_mapping[i] =
3676                                 rte_cpu_to_le_16(vsi->base_queue + i);
3677         } else {
3678                 info->mapping_flags |=
3679                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
3680                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
3681         }
3682         info->valid_sections |=
3683                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
3684
3685         return I40E_SUCCESS;
3686 }
3687
3688 static int
3689 i40e_veb_release(struct i40e_veb *veb)
3690 {
3691         struct i40e_vsi *vsi;
3692         struct i40e_hw *hw;
3693
3694         if (veb == NULL || veb->associate_vsi == NULL)
3695                 return -EINVAL;
3696
3697         if (!TAILQ_EMPTY(&veb->head)) {
3698                 PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove");
3699                 return -EACCES;
3700         }
3701
3702         vsi = veb->associate_vsi;
3703         hw = I40E_VSI_TO_HW(vsi);
3704
3705         vsi->uplink_seid = veb->uplink_seid;
3706         i40e_aq_delete_element(hw, veb->seid, NULL);
3707         rte_free(veb);
3708         vsi->veb = NULL;
3709         return I40E_SUCCESS;
3710 }
3711
3712 /* Setup a veb */
3713 static struct i40e_veb *
3714 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
3715 {
3716         struct i40e_veb *veb;
3717         int ret;
3718         struct i40e_hw *hw;
3719
3720         if (NULL == pf || vsi == NULL) {
3721                 PMD_DRV_LOG(ERR, "veb setup failed, "
3722                             "associated VSI shouldn't null");
3723                 return NULL;
3724         }
3725         hw = I40E_PF_TO_HW(pf);
3726
3727         veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
3728         if (!veb) {
3729                 PMD_DRV_LOG(ERR, "Failed to allocate memory for veb");
3730                 goto fail;
3731         }
3732
3733         veb->associate_vsi = vsi;
3734         TAILQ_INIT(&veb->head);
3735         veb->uplink_seid = vsi->uplink_seid;
3736
3737         ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
3738                 I40E_DEFAULT_TCMAP, false, &veb->seid, false, NULL);
3739
3740         if (ret != I40E_SUCCESS) {
3741                 PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d",
3742                             hw->aq.asq_last_status);
3743                 goto fail;
3744         }
3745
3746         /* get statistics index */
3747         ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
3748                                 &veb->stats_idx, NULL, NULL, NULL);
3749         if (ret != I40E_SUCCESS) {
3750                 PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d",
3751                             hw->aq.asq_last_status);
3752                 goto fail;
3753         }
3754
3755         /* Get VEB bandwidth, to be implemented */
3756         /* Now associated vsi binding to the VEB, set uplink to this VEB */
3757         vsi->uplink_seid = veb->seid;
3758
3759         return veb;
3760 fail:
3761         rte_free(veb);
3762         return NULL;
3763 }
3764
3765 int
3766 i40e_vsi_release(struct i40e_vsi *vsi)
3767 {
3768         struct i40e_pf *pf;
3769         struct i40e_hw *hw;
3770         struct i40e_vsi_list *vsi_list;
3771         int ret;
3772         struct i40e_mac_filter *f;
3773
3774         if (!vsi)
3775                 return I40E_SUCCESS;
3776
3777         pf = I40E_VSI_TO_PF(vsi);
3778         hw = I40E_VSI_TO_HW(vsi);
3779
3780         /* VSI has child to attach, release child first */
3781         if (vsi->veb) {
3782                 TAILQ_FOREACH(vsi_list, &vsi->veb->head, list) {
3783                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
3784                                 return -1;
3785                         TAILQ_REMOVE(&vsi->veb->head, vsi_list, list);
3786                 }
3787                 i40e_veb_release(vsi->veb);
3788         }
3789
3790         /* Remove all macvlan filters of the VSI */
3791         i40e_vsi_remove_all_macvlan_filter(vsi);
3792         TAILQ_FOREACH(f, &vsi->mac_list, next)
3793                 rte_free(f);
3794
3795         if (vsi->type != I40E_VSI_MAIN) {
3796                 /* Remove vsi from parent's sibling list */
3797                 if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
3798                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
3799                         return I40E_ERR_PARAM;
3800                 }
3801                 TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
3802                                 &vsi->sib_vsi_list, list);
3803
3804                 /* Remove all switch element of the VSI */
3805                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
3806                 if (ret != I40E_SUCCESS)
3807                         PMD_DRV_LOG(ERR, "Failed to delete element");
3808         }
3809         i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
3810
3811         if (vsi->type != I40E_VSI_SRIOV)
3812                 i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
3813         rte_free(vsi);
3814
3815         return I40E_SUCCESS;
3816 }
3817
3818 static int
3819 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
3820 {
3821         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3822         struct i40e_aqc_remove_macvlan_element_data def_filter;
3823         struct i40e_mac_filter_info filter;
3824         int ret;
3825
3826         if (vsi->type != I40E_VSI_MAIN)
3827                 return I40E_ERR_CONFIG;
3828         memset(&def_filter, 0, sizeof(def_filter));
3829         (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
3830                                         ETH_ADDR_LEN);
3831         def_filter.vlan_tag = 0;
3832         def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
3833                                 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
3834         ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
3835         if (ret != I40E_SUCCESS) {
3836                 struct i40e_mac_filter *f;
3837                 struct ether_addr *mac;
3838
3839                 PMD_DRV_LOG(WARNING, "Cannot remove the default "
3840                             "macvlan filter");
3841                 /* It needs to add the permanent mac into mac list */
3842                 f = rte_zmalloc("macv_filter", sizeof(*f), 0);
3843                 if (f == NULL) {
3844                         PMD_DRV_LOG(ERR, "failed to allocate memory");
3845                         return I40E_ERR_NO_MEMORY;
3846                 }
3847                 mac = &f->mac_info.mac_addr;
3848                 (void)rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr,
3849                                 ETH_ADDR_LEN);
3850                 f->mac_info.filter_type = RTE_MACVLAN_PERFECT_MATCH;
3851                 TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
3852                 vsi->mac_num++;
3853
3854                 return ret;
3855         }
3856         (void)rte_memcpy(&filter.mac_addr,
3857                 (struct ether_addr *)(hw->mac.perm_addr), ETH_ADDR_LEN);
3858         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
3859         return i40e_vsi_add_mac(vsi, &filter);
3860 }
3861
3862 /*
3863  * i40e_vsi_get_bw_config - Query VSI BW Information
3864  * @vsi: the VSI to be queried
3865  *
3866  * Returns 0 on success, negative value on failure
3867  */
3868 static enum i40e_status_code
3869 i40e_vsi_get_bw_config(struct i40e_vsi *vsi)
3870 {
3871         struct i40e_aqc_query_vsi_bw_config_resp bw_config;
3872         struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
3873         struct i40e_hw *hw = &vsi->adapter->hw;
3874         i40e_status ret;
3875         int i;
3876         uint32_t bw_max;
3877
3878         memset(&bw_config, 0, sizeof(bw_config));
3879         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
3880         if (ret != I40E_SUCCESS) {
3881                 PMD_DRV_LOG(ERR, "VSI failed to get bandwidth configuration %u",
3882                             hw->aq.asq_last_status);
3883                 return ret;
3884         }
3885
3886         memset(&ets_sla_config, 0, sizeof(ets_sla_config));
3887         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
3888                                         &ets_sla_config, NULL);
3889         if (ret != I40E_SUCCESS) {
3890                 PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith "
3891                             "configuration %u", hw->aq.asq_last_status);
3892                 return ret;
3893         }
3894
3895         /* store and print out BW info */
3896         vsi->bw_info.bw_limit = rte_le_to_cpu_16(bw_config.port_bw_limit);
3897         vsi->bw_info.bw_max = bw_config.max_bw;
3898         PMD_DRV_LOG(DEBUG, "VSI bw limit:%u", vsi->bw_info.bw_limit);
3899         PMD_DRV_LOG(DEBUG, "VSI max_bw:%u", vsi->bw_info.bw_max);
3900         bw_max = rte_le_to_cpu_16(ets_sla_config.tc_bw_max[0]) |
3901                     (rte_le_to_cpu_16(ets_sla_config.tc_bw_max[1]) <<
3902                      I40E_16_BIT_WIDTH);
3903         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3904                 vsi->bw_info.bw_ets_share_credits[i] =
3905                                 ets_sla_config.share_credits[i];
3906                 vsi->bw_info.bw_ets_credits[i] =
3907                                 rte_le_to_cpu_16(ets_sla_config.credits[i]);
3908                 /* 4 bits per TC, 4th bit is reserved */
3909                 vsi->bw_info.bw_ets_max[i] =
3910                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
3911                                   RTE_LEN2MASK(3, uint8_t));
3912                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:share credits %u", i,
3913                             vsi->bw_info.bw_ets_share_credits[i]);
3914                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u:credits %u", i,
3915                             vsi->bw_info.bw_ets_credits[i]);
3916                 PMD_DRV_LOG(DEBUG, "\tVSI TC%u: max credits: %u", i,
3917                             vsi->bw_info.bw_ets_max[i]);
3918         }
3919
3920         return I40E_SUCCESS;
3921 }
3922
3923 /* i40e_enable_pf_lb
3924  * @pf: pointer to the pf structure
3925  *
3926  * allow loopback on pf
3927  */
3928 static inline void
3929 i40e_enable_pf_lb(struct i40e_pf *pf)
3930 {
3931         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3932         struct i40e_vsi_context ctxt;
3933         int ret;
3934
3935         /* Use the FW API if FW >= v5.0 */
3936         if (hw->aq.fw_maj_ver < 5) {
3937                 PMD_INIT_LOG(ERR, "FW < v5.0, cannot enable loopback");
3938                 return;
3939         }
3940
3941         memset(&ctxt, 0, sizeof(ctxt));
3942         ctxt.seid = pf->main_vsi_seid;
3943         ctxt.pf_num = hw->pf_id;
3944         ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
3945         if (ret) {
3946                 PMD_DRV_LOG(ERR, "cannot get pf vsi config, err %d, aq_err %d",
3947                             ret, hw->aq.asq_last_status);
3948                 return;
3949         }
3950         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
3951         ctxt.info.valid_sections =
3952                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
3953         ctxt.info.switch_id |=
3954                 rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
3955
3956         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3957         if (ret)
3958                 PMD_DRV_LOG(ERR, "update vsi switch failed, aq_err=%d\n",
3959                             hw->aq.asq_last_status);
3960 }
3961
3962 /* Setup a VSI */
3963 struct i40e_vsi *
3964 i40e_vsi_setup(struct i40e_pf *pf,
3965                enum i40e_vsi_type type,
3966                struct i40e_vsi *uplink_vsi,
3967                uint16_t user_param)
3968 {
3969         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3970         struct i40e_vsi *vsi;
3971         struct i40e_mac_filter_info filter;
3972         int ret;
3973         struct i40e_vsi_context ctxt;
3974         struct ether_addr broadcast =
3975                 {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
3976
3977         if (type != I40E_VSI_MAIN && uplink_vsi == NULL) {
3978                 PMD_DRV_LOG(ERR, "VSI setup failed, "
3979                             "VSI link shouldn't be NULL");
3980                 return NULL;
3981         }
3982
3983         if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
3984                 PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI "
3985                             "uplink VSI should be NULL");
3986                 return NULL;
3987         }
3988
3989         /* If uplink vsi didn't setup VEB, create one first */
3990         if (type != I40E_VSI_MAIN && uplink_vsi->veb == NULL) {
3991                 uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
3992
3993                 if (NULL == uplink_vsi->veb) {
3994                         PMD_DRV_LOG(ERR, "VEB setup failed");
3995                         return NULL;
3996                 }
3997                 /* set ALLOWLOOPBACk on pf, when veb is created */
3998                 i40e_enable_pf_lb(pf);
3999         }
4000
4001         vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
4002         if (!vsi) {
4003                 PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi");
4004                 return NULL;
4005         }
4006         TAILQ_INIT(&vsi->mac_list);
4007         vsi->type = type;
4008         vsi->adapter = I40E_PF_TO_ADAPTER(pf);
4009         vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
4010         vsi->parent_vsi = uplink_vsi;
4011         vsi->user_param = user_param;
4012         /* Allocate queues */
4013         switch (vsi->type) {
4014         case I40E_VSI_MAIN  :
4015                 vsi->nb_qps = pf->lan_nb_qps;
4016                 break;
4017         case I40E_VSI_SRIOV :
4018                 vsi->nb_qps = pf->vf_nb_qps;
4019                 break;
4020         case I40E_VSI_VMDQ2:
4021                 vsi->nb_qps = pf->vmdq_nb_qps;
4022                 break;
4023         case I40E_VSI_FDIR:
4024                 vsi->nb_qps = pf->fdir_nb_qps;
4025                 break;
4026         default:
4027                 goto fail_mem;
4028         }
4029         /*
4030          * The filter status descriptor is reported in rx queue 0,
4031          * while the tx queue for fdir filter programming has no
4032          * such constraints, can be non-zero queues.
4033          * To simplify it, choose FDIR vsi use queue 0 pair.
4034          * To make sure it will use queue 0 pair, queue allocation
4035          * need be done before this function is called
4036          */
4037         if (type != I40E_VSI_FDIR) {
4038                 ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
4039                         if (ret < 0) {
4040                                 PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
4041                                                 vsi->seid, ret);
4042                                 goto fail_mem;
4043                         }
4044                         vsi->base_queue = ret;
4045         } else
4046                 vsi->base_queue = I40E_FDIR_QUEUE_ID;
4047
4048         /* VF has MSIX interrupt in VF range, don't allocate here */
4049         if (type == I40E_VSI_MAIN) {
4050                 ret = i40e_res_pool_alloc(&pf->msix_pool,
4051                                           RTE_MIN(vsi->nb_qps,
4052                                                   RTE_MAX_RXTX_INTR_VEC_ID));
4053                 if (ret < 0) {
4054                         PMD_DRV_LOG(ERR, "VSI MAIN %d get heap failed %d",
4055                                     vsi->seid, ret);
4056                         goto fail_queue_alloc;
4057                 }
4058                 vsi->msix_intr = ret;
4059                 vsi->nb_msix = RTE_MIN(vsi->nb_qps, RTE_MAX_RXTX_INTR_VEC_ID);
4060         } else if (type != I40E_VSI_SRIOV) {
4061                 ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
4062                 if (ret < 0) {
4063                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
4064                         goto fail_queue_alloc;
4065                 }
4066                 vsi->msix_intr = ret;
4067                 vsi->nb_msix = 1;
4068         } else {
4069                 vsi->msix_intr = 0;
4070                 vsi->nb_msix = 0;
4071         }
4072
4073         /* Add VSI */
4074         if (type == I40E_VSI_MAIN) {
4075                 /* For main VSI, no need to add since it's default one */
4076                 vsi->uplink_seid = pf->mac_seid;
4077                 vsi->seid = pf->main_vsi_seid;
4078                 /* Bind queues with specific MSIX interrupt */
4079                 /**
4080                  * Needs 2 interrupt at least, one for misc cause which will
4081                  * enabled from OS side, Another for queues binding the
4082                  * interrupt from device side only.
4083                  */
4084
4085                 /* Get default VSI parameters from hardware */
4086                 memset(&ctxt, 0, sizeof(ctxt));
4087                 ctxt.seid = vsi->seid;
4088                 ctxt.pf_num = hw->pf_id;
4089                 ctxt.uplink_seid = vsi->uplink_seid;
4090                 ctxt.vf_num = 0;
4091                 ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
4092                 if (ret != I40E_SUCCESS) {
4093                         PMD_DRV_LOG(ERR, "Failed to get VSI params");
4094                         goto fail_msix_alloc;
4095                 }
4096                 (void)rte_memcpy(&vsi->info, &ctxt.info,
4097                         sizeof(struct i40e_aqc_vsi_properties_data));
4098                 vsi->vsi_id = ctxt.vsi_number;
4099                 vsi->info.valid_sections = 0;
4100
4101                 /* Configure tc, enabled TC0 only */
4102                 if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
4103                         I40E_SUCCESS) {
4104                         PMD_DRV_LOG(ERR, "Failed to update TC bandwidth");
4105                         goto fail_msix_alloc;
4106                 }
4107
4108                 /* TC, queue mapping */
4109                 memset(&ctxt, 0, sizeof(ctxt));
4110                 vsi->info.valid_sections |=
4111                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4112                 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
4113                                         I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
4114                 (void)rte_memcpy(&ctxt.info, &vsi->info,
4115                         sizeof(struct i40e_aqc_vsi_properties_data));
4116                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4117                                                 I40E_DEFAULT_TCMAP);
4118                 if (ret != I40E_SUCCESS) {
4119                         PMD_DRV_LOG(ERR, "Failed to configure "
4120                                     "TC queue mapping");
4121                         goto fail_msix_alloc;
4122                 }
4123                 ctxt.seid = vsi->seid;
4124                 ctxt.pf_num = hw->pf_id;
4125                 ctxt.uplink_seid = vsi->uplink_seid;
4126                 ctxt.vf_num = 0;
4127
4128                 /* Update VSI parameters */
4129                 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4130                 if (ret != I40E_SUCCESS) {
4131                         PMD_DRV_LOG(ERR, "Failed to update VSI params");
4132                         goto fail_msix_alloc;
4133                 }
4134
4135                 (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
4136                                                 sizeof(vsi->info.tc_mapping));
4137                 (void)rte_memcpy(&vsi->info.queue_mapping,
4138                                 &ctxt.info.queue_mapping,
4139                         sizeof(vsi->info.queue_mapping));
4140                 vsi->info.mapping_flags = ctxt.info.mapping_flags;
4141                 vsi->info.valid_sections = 0;
4142
4143                 (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
4144                                 ETH_ADDR_LEN);
4145
4146                 /**
4147                  * Updating default filter settings are necessary to prevent
4148                  * reception of tagged packets.
4149                  * Some old firmware configurations load a default macvlan
4150                  * filter which accepts both tagged and untagged packets.
4151                  * The updating is to use a normal filter instead if needed.
4152                  * For NVM 4.2.2 or after, the updating is not needed anymore.
4153                  * The firmware with correct configurations load the default
4154                  * macvlan filter which is expected and cannot be removed.
4155                  */
4156                 i40e_update_default_filter_setting(vsi);
4157                 i40e_config_qinq(hw, vsi);
4158         } else if (type == I40E_VSI_SRIOV) {
4159                 memset(&ctxt, 0, sizeof(ctxt));
4160                 /**
4161                  * For other VSI, the uplink_seid equals to uplink VSI's
4162                  * uplink_seid since they share same VEB
4163                  */
4164                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4165                 ctxt.pf_num = hw->pf_id;
4166                 ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
4167                 ctxt.uplink_seid = vsi->uplink_seid;
4168                 ctxt.connection_type = 0x1;
4169                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
4170
4171                 /* Use the VEB configuration if FW >= v5.0 */
4172                 if (hw->aq.fw_maj_ver >= 5) {
4173                         /* Configure switch ID */
4174                         ctxt.info.valid_sections |=
4175                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4176                         ctxt.info.switch_id =
4177                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4178                 }
4179
4180                 /* Configure port/vlan */
4181                 ctxt.info.valid_sections |=
4182                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4183                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4184                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4185                                                 I40E_DEFAULT_TCMAP);
4186                 if (ret != I40E_SUCCESS) {
4187                         PMD_DRV_LOG(ERR, "Failed to configure "
4188                                     "TC queue mapping");
4189                         goto fail_msix_alloc;
4190                 }
4191                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4192                 ctxt.info.valid_sections |=
4193                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4194                 /**
4195                  * Since VSI is not created yet, only configure parameter,
4196                  * will add vsi below.
4197                  */
4198
4199                 i40e_config_qinq(hw, vsi);
4200         } else if (type == I40E_VSI_VMDQ2) {
4201                 memset(&ctxt, 0, sizeof(ctxt));
4202                 /*
4203                  * For other VSI, the uplink_seid equals to uplink VSI's
4204                  * uplink_seid since they share same VEB
4205                  */
4206                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4207                 ctxt.pf_num = hw->pf_id;
4208                 ctxt.vf_num = 0;
4209                 ctxt.uplink_seid = vsi->uplink_seid;
4210                 ctxt.connection_type = 0x1;
4211                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
4212
4213                 ctxt.info.valid_sections |=
4214                                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
4215                 /* user_param carries flag to enable loop back */
4216                 if (user_param) {
4217                         ctxt.info.switch_id =
4218                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
4219                         ctxt.info.switch_id |=
4220                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
4221                 }
4222
4223                 /* Configure port/vlan */
4224                 ctxt.info.valid_sections |=
4225                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4226                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
4227                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4228                                                 I40E_DEFAULT_TCMAP);
4229                 if (ret != I40E_SUCCESS) {
4230                         PMD_DRV_LOG(ERR, "Failed to configure "
4231                                         "TC queue mapping");
4232                         goto fail_msix_alloc;
4233                 }
4234                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4235                 ctxt.info.valid_sections |=
4236                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4237         } else if (type == I40E_VSI_FDIR) {
4238                 memset(&ctxt, 0, sizeof(ctxt));
4239                 vsi->uplink_seid = uplink_vsi->uplink_seid;
4240                 ctxt.pf_num = hw->pf_id;
4241                 ctxt.vf_num = 0;
4242                 ctxt.uplink_seid = vsi->uplink_seid;
4243                 ctxt.connection_type = 0x1;     /* regular data port */
4244                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
4245                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
4246                                                 I40E_DEFAULT_TCMAP);
4247                 if (ret != I40E_SUCCESS) {
4248                         PMD_DRV_LOG(ERR, "Failed to configure "
4249                                         "TC queue mapping.");
4250                         goto fail_msix_alloc;
4251                 }
4252                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
4253                 ctxt.info.valid_sections |=
4254                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
4255         } else {
4256                 PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet");
4257                 goto fail_msix_alloc;
4258         }
4259
4260         if (vsi->type != I40E_VSI_MAIN) {
4261                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
4262                 if (ret != I40E_SUCCESS) {
4263                         PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d",
4264                                     hw->aq.asq_last_status);
4265                         goto fail_msix_alloc;
4266                 }
4267                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
4268                 vsi->info.valid_sections = 0;
4269                 vsi->seid = ctxt.seid;
4270                 vsi->vsi_id = ctxt.vsi_number;
4271                 vsi->sib_vsi_list.vsi = vsi;
4272                 TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
4273                                 &vsi->sib_vsi_list, list);
4274         }
4275
4276         /* MAC/VLAN configuration */
4277         (void)rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN);
4278         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
4279
4280         ret = i40e_vsi_add_mac(vsi, &filter);
4281         if (ret != I40E_SUCCESS) {
4282                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
4283                 goto fail_msix_alloc;
4284         }
4285
4286         /* Get VSI BW information */
4287         i40e_vsi_get_bw_config(vsi);
4288         return vsi;
4289 fail_msix_alloc:
4290         i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
4291 fail_queue_alloc:
4292         i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
4293 fail_mem:
4294         rte_free(vsi);
4295         return NULL;
4296 }
4297
4298 /* Configure vlan filter on or off */
4299 int
4300 i40e_vsi_config_vlan_filter(struct i40e_vsi *vsi, bool on)
4301 {
4302         int i, num;
4303         struct i40e_mac_filter *f;
4304         struct i40e_mac_filter_info *mac_filter;
4305         enum rte_mac_filter_type desired_filter;
4306         int ret = I40E_SUCCESS;
4307
4308         if (on) {
4309                 /* Filter to match MAC and VLAN */
4310                 desired_filter = RTE_MACVLAN_PERFECT_MATCH;
4311         } else {
4312                 /* Filter to match only MAC */
4313                 desired_filter = RTE_MAC_PERFECT_MATCH;
4314         }
4315
4316         num = vsi->mac_num;
4317
4318         mac_filter = rte_zmalloc("mac_filter_info_data",
4319                                  num * sizeof(*mac_filter), 0);
4320         if (mac_filter == NULL) {
4321                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4322                 return I40E_ERR_NO_MEMORY;
4323         }
4324
4325         i = 0;
4326
4327         /* Remove all existing mac */
4328         TAILQ_FOREACH(f, &vsi->mac_list, next) {
4329                 mac_filter[i] = f->mac_info;
4330                 ret = i40e_vsi_delete_mac(vsi, &f->mac_info.mac_addr);
4331                 if (ret) {
4332                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
4333                                     on ? "enable" : "disable");
4334                         goto DONE;
4335                 }
4336                 i++;
4337         }
4338
4339         /* Override with new filter */
4340         for (i = 0; i < num; i++) {
4341                 mac_filter[i].filter_type = desired_filter;
4342                 ret = i40e_vsi_add_mac(vsi, &mac_filter[i]);
4343                 if (ret) {
4344                         PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan filter",
4345                                     on ? "enable" : "disable");
4346                         goto DONE;
4347                 }
4348         }
4349
4350 DONE:
4351         rte_free(mac_filter);
4352         return ret;
4353 }
4354
4355 /* Configure vlan stripping on or off */
4356 int
4357 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
4358 {
4359         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4360         struct i40e_vsi_context ctxt;
4361         uint8_t vlan_flags;
4362         int ret = I40E_SUCCESS;
4363
4364         /* Check if it has been already on or off */
4365         if (vsi->info.valid_sections &
4366                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
4367                 if (on) {
4368                         if ((vsi->info.port_vlan_flags &
4369                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
4370                                 return 0; /* already on */
4371                 } else {
4372                         if ((vsi->info.port_vlan_flags &
4373                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
4374                                 I40E_AQ_VSI_PVLAN_EMOD_MASK)
4375                                 return 0; /* already off */
4376                 }
4377         }
4378
4379         if (on)
4380                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
4381         else
4382                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
4383         vsi->info.valid_sections =
4384                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
4385         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
4386         vsi->info.port_vlan_flags |= vlan_flags;
4387         ctxt.seid = vsi->seid;
4388         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
4389         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
4390         if (ret)
4391                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping",
4392                             on ? "enable" : "disable");
4393
4394         return ret;
4395 }
4396
4397 static int
4398 i40e_dev_init_vlan(struct rte_eth_dev *dev)
4399 {
4400         struct rte_eth_dev_data *data = dev->data;
4401         int ret;
4402         int mask = 0;
4403
4404         /* Apply vlan offload setting */
4405         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK;
4406         i40e_vlan_offload_set(dev, mask);
4407
4408         /* Apply double-vlan setting, not implemented yet */
4409
4410         /* Apply pvid setting */
4411         ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
4412                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
4413         if (ret)
4414                 PMD_DRV_LOG(INFO, "Failed to update VSI params");
4415
4416         return ret;
4417 }
4418
4419 static int
4420 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
4421 {
4422         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4423
4424         return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
4425 }
4426
4427 static int
4428 i40e_update_flow_control(struct i40e_hw *hw)
4429 {
4430 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
4431         struct i40e_link_status link_status;
4432         uint32_t rxfc = 0, txfc = 0, reg;
4433         uint8_t an_info;
4434         int ret;
4435
4436         memset(&link_status, 0, sizeof(link_status));
4437         ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
4438         if (ret != I40E_SUCCESS) {
4439                 PMD_DRV_LOG(ERR, "Failed to get link status information");
4440                 goto write_reg; /* Disable flow control */
4441         }
4442
4443         an_info = hw->phy.link_info.an_info;
4444         if (!(an_info & I40E_AQ_AN_COMPLETED)) {
4445                 PMD_DRV_LOG(INFO, "Link auto negotiation not completed");
4446                 ret = I40E_ERR_NOT_READY;
4447                 goto write_reg; /* Disable flow control */
4448         }
4449         /**
4450          * If link auto negotiation is enabled, flow control needs to
4451          * be configured according to it
4452          */
4453         switch (an_info & I40E_LINK_PAUSE_RXTX) {
4454         case I40E_LINK_PAUSE_RXTX:
4455                 rxfc = 1;
4456                 txfc = 1;
4457                 hw->fc.current_mode = I40E_FC_FULL;
4458                 break;
4459         case I40E_AQ_LINK_PAUSE_RX:
4460                 rxfc = 1;
4461                 hw->fc.current_mode = I40E_FC_RX_PAUSE;
4462                 break;
4463         case I40E_AQ_LINK_PAUSE_TX:
4464                 txfc = 1;
4465                 hw->fc.current_mode = I40E_FC_TX_PAUSE;
4466                 break;
4467         default:
4468                 hw->fc.current_mode = I40E_FC_NONE;
4469                 break;
4470         }
4471
4472 write_reg:
4473         I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
4474                 txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
4475         reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
4476         reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
4477         reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
4478         I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
4479
4480         return ret;
4481 }
4482
4483 /* PF setup */
4484 static int
4485 i40e_pf_setup(struct i40e_pf *pf)
4486 {
4487         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4488         struct i40e_filter_control_settings settings;
4489         struct i40e_vsi *vsi;
4490         int ret;
4491
4492         /* Clear all stats counters */
4493         pf->offset_loaded = FALSE;
4494         memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
4495         memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
4496
4497         ret = i40e_pf_get_switch_config(pf);
4498         if (ret != I40E_SUCCESS) {
4499                 PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
4500                 return ret;
4501         }
4502         if (pf->flags & I40E_FLAG_FDIR) {
4503                 /* make queue allocated first, let FDIR use queue pair 0*/
4504                 ret = i40e_res_pool_alloc(&pf->qp_pool, I40E_DEFAULT_QP_NUM_FDIR);
4505                 if (ret != I40E_FDIR_QUEUE_ID) {
4506                         PMD_DRV_LOG(ERR, "queue allocation fails for FDIR :"
4507                                     " ret =%d", ret);
4508                         pf->flags &= ~I40E_FLAG_FDIR;
4509                 }
4510         }
4511         /*  main VSI setup */
4512         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
4513         if (!vsi) {
4514                 PMD_DRV_LOG(ERR, "Setup of main vsi failed");
4515                 return I40E_ERR_NOT_READY;
4516         }
4517         pf->main_vsi = vsi;
4518
4519         /* Configure filter control */
4520         memset(&settings, 0, sizeof(settings));
4521         if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_128)
4522                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
4523         else if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_512)
4524                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_512;
4525         else {
4526                 PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported\n",
4527                                                 hw->func_caps.rss_table_size);
4528                 return I40E_ERR_PARAM;
4529         }
4530         PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table "
4531                         "size: %u\n", hw->func_caps.rss_table_size);
4532         pf->hash_lut_size = hw->func_caps.rss_table_size;
4533
4534         /* Enable ethtype and macvlan filters */
4535         settings.enable_ethtype = TRUE;
4536         settings.enable_macvlan = TRUE;
4537         ret = i40e_set_filter_control(hw, &settings);
4538         if (ret)
4539                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
4540                                                                 ret);
4541
4542         /* Update flow control according to the auto negotiation */
4543         i40e_update_flow_control(hw);
4544
4545         return I40E_SUCCESS;
4546 }
4547
4548 int
4549 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
4550 {
4551         uint32_t reg;
4552         uint16_t j;
4553
4554         /**
4555          * Set or clear TX Queue Disable flags,
4556          * which is required by hardware.
4557          */
4558         i40e_pre_tx_queue_cfg(hw, q_idx, on);
4559         rte_delay_us(I40E_PRE_TX_Q_CFG_WAIT_US);
4560
4561         /* Wait until the request is finished */
4562         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4563                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4564                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
4565                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
4566                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
4567                                                         & 0x1))) {
4568                         break;
4569                 }
4570         }
4571         if (on) {
4572                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
4573                         return I40E_SUCCESS; /* already on, skip next steps */
4574
4575                 I40E_WRITE_REG(hw, I40E_QTX_HEAD(q_idx), 0);
4576                 reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4577         } else {
4578                 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4579                         return I40E_SUCCESS; /* already off, skip next steps */
4580                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4581         }
4582         /* Write the register */
4583         I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
4584         /* Check the result */
4585         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4586                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4587                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
4588                 if (on) {
4589                         if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
4590                                 (reg & I40E_QTX_ENA_QENA_STAT_MASK))
4591                                 break;
4592                 } else {
4593                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
4594                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
4595                                 break;
4596                 }
4597         }
4598         /* Check if it is timeout */
4599         if (j >= I40E_CHK_Q_ENA_COUNT) {
4600                 PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]",
4601                             (on ? "enable" : "disable"), q_idx);
4602                 return I40E_ERR_TIMEOUT;
4603         }
4604
4605         return I40E_SUCCESS;
4606 }
4607
4608 /* Swith on or off the tx queues */
4609 static int
4610 i40e_dev_switch_tx_queues(struct i40e_pf *pf, bool on)
4611 {
4612         struct rte_eth_dev_data *dev_data = pf->dev_data;
4613         struct i40e_tx_queue *txq;
4614         struct rte_eth_dev *dev = pf->adapter->eth_dev;
4615         uint16_t i;
4616         int ret;
4617
4618         for (i = 0; i < dev_data->nb_tx_queues; i++) {
4619                 txq = dev_data->tx_queues[i];
4620                 /* Don't operate the queue if not configured or
4621                  * if starting only per queue */
4622                 if (!txq || !txq->q_set || (on && txq->tx_deferred_start))
4623                         continue;
4624                 if (on)
4625                         ret = i40e_dev_tx_queue_start(dev, i);
4626                 else
4627                         ret = i40e_dev_tx_queue_stop(dev, i);
4628                 if ( ret != I40E_SUCCESS)
4629                         return ret;
4630         }
4631
4632         return I40E_SUCCESS;
4633 }
4634
4635 int
4636 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
4637 {
4638         uint32_t reg;
4639         uint16_t j;
4640
4641         /* Wait until the request is finished */
4642         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4643                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4644                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
4645                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
4646                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
4647                         break;
4648         }
4649
4650         if (on) {
4651                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
4652                         return I40E_SUCCESS; /* Already on, skip next steps */
4653                 reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4654         } else {
4655                 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
4656                         return I40E_SUCCESS; /* Already off, skip next steps */
4657                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4658         }
4659
4660         /* Write the register */
4661         I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
4662         /* Check the result */
4663         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
4664                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
4665                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
4666                 if (on) {
4667                         if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
4668                                 (reg & I40E_QRX_ENA_QENA_STAT_MASK))
4669                                 break;
4670                 } else {
4671                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
4672                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
4673                                 break;
4674                 }
4675         }
4676
4677         /* Check if it is timeout */
4678         if (j >= I40E_CHK_Q_ENA_COUNT) {
4679                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
4680                             (on ? "enable" : "disable"), q_idx);
4681                 return I40E_ERR_TIMEOUT;
4682         }
4683
4684         return I40E_SUCCESS;
4685 }
4686 /* Switch on or off the rx queues */
4687 static int
4688 i40e_dev_switch_rx_queues(struct i40e_pf *pf, bool on)
4689 {
4690         struct rte_eth_dev_data *dev_data = pf->dev_data;
4691         struct i40e_rx_queue *rxq;
4692         struct rte_eth_dev *dev = pf->adapter->eth_dev;
4693         uint16_t i;
4694         int ret;
4695
4696         for (i = 0; i < dev_data->nb_rx_queues; i++) {
4697                 rxq = dev_data->rx_queues[i];
4698                 /* Don't operate the queue if not configured or
4699                  * if starting only per queue */
4700                 if (!rxq || !rxq->q_set || (on && rxq->rx_deferred_start))
4701                         continue;
4702                 if (on)
4703                         ret = i40e_dev_rx_queue_start(dev, i);
4704                 else
4705                         ret = i40e_dev_rx_queue_stop(dev, i);
4706                 if (ret != I40E_SUCCESS)
4707                         return ret;
4708         }
4709
4710         return I40E_SUCCESS;
4711 }
4712
4713 /* Switch on or off all the rx/tx queues */
4714 int
4715 i40e_dev_switch_queues(struct i40e_pf *pf, bool on)
4716 {
4717         int ret;
4718
4719         if (on) {
4720                 /* enable rx queues before enabling tx queues */
4721                 ret = i40e_dev_switch_rx_queues(pf, on);
4722                 if (ret) {
4723                         PMD_DRV_LOG(ERR, "Failed to switch rx queues");
4724                         return ret;
4725                 }
4726                 ret = i40e_dev_switch_tx_queues(pf, on);
4727         } else {
4728                 /* Stop tx queues before stopping rx queues */
4729                 ret = i40e_dev_switch_tx_queues(pf, on);
4730                 if (ret) {
4731                         PMD_DRV_LOG(ERR, "Failed to switch tx queues");
4732                         return ret;
4733                 }
4734                 ret = i40e_dev_switch_rx_queues(pf, on);
4735         }
4736
4737         return ret;
4738 }
4739
4740 /* Initialize VSI for TX */
4741 static int
4742 i40e_dev_tx_init(struct i40e_pf *pf)
4743 {
4744         struct rte_eth_dev_data *data = pf->dev_data;
4745         uint16_t i;
4746         uint32_t ret = I40E_SUCCESS;
4747         struct i40e_tx_queue *txq;
4748
4749         for (i = 0; i < data->nb_tx_queues; i++) {
4750                 txq = data->tx_queues[i];
4751                 if (!txq || !txq->q_set)
4752                         continue;
4753                 ret = i40e_tx_queue_init(txq);
4754                 if (ret != I40E_SUCCESS)
4755                         break;
4756         }
4757         if (ret == I40E_SUCCESS)
4758                 i40e_set_tx_function(container_of(pf, struct i40e_adapter, pf)
4759                                      ->eth_dev);
4760
4761         return ret;
4762 }
4763
4764 /* Initialize VSI for RX */
4765 static int
4766 i40e_dev_rx_init(struct i40e_pf *pf)
4767 {
4768         struct rte_eth_dev_data *data = pf->dev_data;
4769         int ret = I40E_SUCCESS;
4770         uint16_t i;
4771         struct i40e_rx_queue *rxq;
4772
4773         i40e_pf_config_mq_rx(pf);
4774         for (i = 0; i < data->nb_rx_queues; i++) {
4775                 rxq = data->rx_queues[i];
4776                 if (!rxq || !rxq->q_set)
4777                         continue;
4778
4779                 ret = i40e_rx_queue_init(rxq);
4780                 if (ret != I40E_SUCCESS) {
4781                         PMD_DRV_LOG(ERR, "Failed to do RX queue "
4782                                     "initialization");
4783                         break;
4784                 }
4785         }
4786         if (ret == I40E_SUCCESS)
4787                 i40e_set_rx_function(container_of(pf, struct i40e_adapter, pf)
4788                                      ->eth_dev);
4789
4790         return ret;
4791 }
4792
4793 static int
4794 i40e_dev_rxtx_init(struct i40e_pf *pf)
4795 {
4796         int err;
4797
4798         err = i40e_dev_tx_init(pf);
4799         if (err) {
4800                 PMD_DRV_LOG(ERR, "Failed to do TX initialization");
4801                 return err;
4802         }
4803         err = i40e_dev_rx_init(pf);
4804         if (err) {
4805                 PMD_DRV_LOG(ERR, "Failed to do RX initialization");
4806                 return err;
4807         }
4808
4809         return err;
4810 }
4811
4812 static int
4813 i40e_vmdq_setup(struct rte_eth_dev *dev)
4814 {
4815         struct rte_eth_conf *conf = &dev->data->dev_conf;
4816         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4817         int i, err, conf_vsis, j, loop;
4818         struct i40e_vsi *vsi;
4819         struct i40e_vmdq_info *vmdq_info;
4820         struct rte_eth_vmdq_rx_conf *vmdq_conf;
4821         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4822
4823         /*
4824          * Disable interrupt to avoid message from VF. Furthermore, it will
4825          * avoid race condition in VSI creation/destroy.
4826          */
4827         i40e_pf_disable_irq0(hw);
4828
4829         if ((pf->flags & I40E_FLAG_VMDQ) == 0) {
4830                 PMD_INIT_LOG(ERR, "FW doesn't support VMDQ");
4831                 return -ENOTSUP;
4832         }
4833
4834         conf_vsis = conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
4835         if (conf_vsis > pf->max_nb_vmdq_vsi) {
4836                 PMD_INIT_LOG(ERR, "VMDQ config: %u, max support:%u",
4837                         conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools,
4838                         pf->max_nb_vmdq_vsi);
4839                 return -ENOTSUP;
4840         }
4841
4842         if (pf->vmdq != NULL) {
4843                 PMD_INIT_LOG(INFO, "VMDQ already configured");
4844                 return 0;
4845         }
4846
4847         pf->vmdq = rte_zmalloc("vmdq_info_struct",
4848                                 sizeof(*vmdq_info) * conf_vsis, 0);
4849
4850         if (pf->vmdq == NULL) {
4851                 PMD_INIT_LOG(ERR, "Failed to allocate memory");
4852                 return -ENOMEM;
4853         }
4854
4855         vmdq_conf = &conf->rx_adv_conf.vmdq_rx_conf;
4856
4857         /* Create VMDQ VSI */
4858         for (i = 0; i < conf_vsis; i++) {
4859                 vsi = i40e_vsi_setup(pf, I40E_VSI_VMDQ2, pf->main_vsi,
4860                                 vmdq_conf->enable_loop_back);
4861                 if (vsi == NULL) {
4862                         PMD_INIT_LOG(ERR, "Failed to create VMDQ VSI");
4863                         err = -1;
4864                         goto err_vsi_setup;
4865                 }
4866                 vmdq_info = &pf->vmdq[i];
4867                 vmdq_info->pf = pf;
4868                 vmdq_info->vsi = vsi;
4869         }
4870         pf->nb_cfg_vmdq_vsi = conf_vsis;
4871
4872         /* Configure Vlan */
4873         loop = sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT;
4874         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
4875                 for (j = 0; j < loop && j < pf->nb_cfg_vmdq_vsi; j++) {
4876                         if (vmdq_conf->pool_map[i].pools & (1UL << j)) {
4877                                 PMD_INIT_LOG(INFO, "Add vlan %u to vmdq pool %u",
4878                                         vmdq_conf->pool_map[i].vlan_id, j);
4879
4880                                 err = i40e_vsi_add_vlan(pf->vmdq[j].vsi,
4881                                                 vmdq_conf->pool_map[i].vlan_id);
4882                                 if (err) {
4883                                         PMD_INIT_LOG(ERR, "Failed to add vlan");
4884                                         err = -1;
4885                                         goto err_vsi_setup;
4886                                 }
4887                         }
4888                 }
4889         }
4890
4891         i40e_pf_enable_irq0(hw);
4892
4893         return 0;
4894
4895 err_vsi_setup:
4896         for (i = 0; i < conf_vsis; i++)
4897                 if (pf->vmdq[i].vsi == NULL)
4898                         break;
4899                 else
4900                         i40e_vsi_release(pf->vmdq[i].vsi);
4901
4902         rte_free(pf->vmdq);
4903         pf->vmdq = NULL;
4904         i40e_pf_enable_irq0(hw);
4905         return err;
4906 }
4907
4908 static void
4909 i40e_stat_update_32(struct i40e_hw *hw,
4910                    uint32_t reg,
4911                    bool offset_loaded,
4912                    uint64_t *offset,
4913                    uint64_t *stat)
4914 {
4915         uint64_t new_data;
4916
4917         new_data = (uint64_t)I40E_READ_REG(hw, reg);
4918         if (!offset_loaded)
4919                 *offset = new_data;
4920
4921         if (new_data >= *offset)
4922                 *stat = (uint64_t)(new_data - *offset);
4923         else
4924                 *stat = (uint64_t)((new_data +
4925                         ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset);
4926 }
4927
4928 static void
4929 i40e_stat_update_48(struct i40e_hw *hw,
4930                    uint32_t hireg,
4931                    uint32_t loreg,
4932                    bool offset_loaded,
4933                    uint64_t *offset,
4934                    uint64_t *stat)
4935 {
4936         uint64_t new_data;
4937
4938         new_data = (uint64_t)I40E_READ_REG(hw, loreg);
4939         new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
4940                         I40E_16_BIT_MASK)) << I40E_32_BIT_WIDTH;
4941
4942         if (!offset_loaded)
4943                 *offset = new_data;
4944
4945         if (new_data >= *offset)
4946                 *stat = new_data - *offset;
4947         else
4948                 *stat = (uint64_t)((new_data +
4949                         ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset);
4950
4951         *stat &= I40E_48_BIT_MASK;
4952 }
4953
4954 /* Disable IRQ0 */
4955 void
4956 i40e_pf_disable_irq0(struct i40e_hw *hw)
4957 {
4958         /* Disable all interrupt types */
4959         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
4960         I40E_WRITE_FLUSH(hw);
4961 }
4962
4963 /* Enable IRQ0 */
4964 void
4965 i40e_pf_enable_irq0(struct i40e_hw *hw)
4966 {
4967         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
4968                 I40E_PFINT_DYN_CTL0_INTENA_MASK |
4969                 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4970                 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
4971         I40E_WRITE_FLUSH(hw);
4972 }
4973
4974 static void
4975 i40e_pf_config_irq0(struct i40e_hw *hw, bool no_queue)
4976 {
4977         /* read pending request and disable first */
4978         i40e_pf_disable_irq0(hw);
4979         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, I40E_PFINT_ICR0_ENA_MASK);
4980         I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
4981                 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
4982
4983         if (no_queue)
4984                 /* Link no queues with irq0 */
4985                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
4986                                I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
4987 }
4988
4989 static void
4990 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
4991 {
4992         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4993         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4994         int i;
4995         uint16_t abs_vf_id;
4996         uint32_t index, offset, val;
4997
4998         if (!pf->vfs)
4999                 return;
5000         /**
5001          * Try to find which VF trigger a reset, use absolute VF id to access
5002          * since the reg is global register.
5003          */
5004         for (i = 0; i < pf->vf_num; i++) {
5005                 abs_vf_id = hw->func_caps.vf_base_id + i;
5006                 index = abs_vf_id / I40E_UINT32_BIT_SIZE;
5007                 offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
5008                 val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
5009                 /* VFR event occured */
5010                 if (val & (0x1 << offset)) {
5011                         int ret;
5012
5013                         /* Clear the event first */
5014                         I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
5015                                                         (0x1 << offset));
5016                         PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id);
5017                         /**
5018                          * Only notify a VF reset event occured,
5019                          * don't trigger another SW reset
5020                          */
5021                         ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
5022                         if (ret != I40E_SUCCESS)
5023                                 PMD_DRV_LOG(ERR, "Failed to do VF reset");
5024                 }
5025         }
5026 }
5027
5028 static void
5029 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
5030 {
5031         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5032         struct i40e_arq_event_info info;
5033         uint16_t pending, opcode;
5034         int ret;
5035
5036         info.buf_len = I40E_AQ_BUF_SZ;
5037         info.msg_buf = rte_zmalloc("msg_buffer", info.buf_len, 0);
5038         if (!info.msg_buf) {
5039                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
5040                 return;
5041         }
5042
5043         pending = 1;
5044         while (pending) {
5045                 ret = i40e_clean_arq_element(hw, &info, &pending);
5046
5047                 if (ret != I40E_SUCCESS) {
5048                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
5049                                     "aq_err: %u", hw->aq.asq_last_status);
5050                         break;
5051                 }
5052                 opcode = rte_le_to_cpu_16(info.desc.opcode);
5053
5054                 switch (opcode) {
5055                 case i40e_aqc_opc_send_msg_to_pf:
5056                         /* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
5057                         i40e_pf_host_handle_vf_msg(dev,
5058                                         rte_le_to_cpu_16(info.desc.retval),
5059                                         rte_le_to_cpu_32(info.desc.cookie_high),
5060                                         rte_le_to_cpu_32(info.desc.cookie_low),
5061                                         info.msg_buf,
5062                                         info.msg_len);
5063                         break;
5064                 default:
5065                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
5066                                     opcode);
5067                         break;
5068                 }
5069         }
5070         rte_free(info.msg_buf);
5071 }
5072
5073 /*
5074  * Interrupt handler is registered as the alarm callback for handling LSC
5075  * interrupt in a definite of time, in order to wait the NIC into a stable
5076  * state. Currently it waits 1 sec in i40e for the link up interrupt, and
5077  * no need for link down interrupt.
5078  */
5079 static void
5080 i40e_dev_interrupt_delayed_handler(void *param)
5081 {
5082         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
5083         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5084         uint32_t icr0;
5085
5086         /* read interrupt causes again */
5087         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
5088
5089 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
5090         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
5091                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error\n");
5092         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
5093                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected\n");
5094         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
5095                 PMD_DRV_LOG(INFO, "ICR0: global reset requested\n");
5096         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
5097                 PMD_DRV_LOG(INFO, "ICR0: PCI exception\n activated\n");
5098         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
5099                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control "
5100                                                                 "state\n");
5101         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
5102                 PMD_DRV_LOG(ERR, "ICR0: HMC error\n");
5103         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
5104                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error\n");
5105 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
5106
5107         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
5108                 PMD_DRV_LOG(INFO, "INT:VF reset detected\n");
5109                 i40e_dev_handle_vfr_event(dev);
5110         }
5111         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
5112                 PMD_DRV_LOG(INFO, "INT:ADMINQ event\n");
5113                 i40e_dev_handle_aq_msg(dev);
5114         }
5115
5116         /* handle the link up interrupt in an alarm callback */
5117         i40e_dev_link_update(dev, 0);
5118         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
5119
5120         i40e_pf_enable_irq0(hw);
5121         rte_intr_enable(&(dev->pci_dev->intr_handle));
5122 }
5123
5124 /**
5125  * Interrupt handler triggered by NIC  for handling
5126  * specific interrupt.
5127  *
5128  * @param handle
5129  *  Pointer to interrupt handle.
5130  * @param param
5131  *  The address of parameter (struct rte_eth_dev *) regsitered before.
5132  *
5133  * @return
5134  *  void
5135  */
5136 static void
5137 i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
5138                            void *param)
5139 {
5140         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
5141         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5142         uint32_t icr0;
5143
5144         /* Disable interrupt */
5145         i40e_pf_disable_irq0(hw);
5146
5147         /* read out interrupt causes */
5148         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
5149
5150         /* No interrupt event indicated */
5151         if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK)) {
5152                 PMD_DRV_LOG(INFO, "No interrupt event");
5153                 goto done;
5154         }
5155 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
5156         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
5157                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
5158         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
5159                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
5160         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
5161                 PMD_DRV_LOG(INFO, "ICR0: global reset requested");
5162         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
5163                 PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
5164         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
5165                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
5166         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
5167                 PMD_DRV_LOG(ERR, "ICR0: HMC error");
5168         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
5169                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
5170 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
5171
5172         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
5173                 PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
5174                 i40e_dev_handle_vfr_event(dev);
5175         }
5176         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
5177                 PMD_DRV_LOG(INFO, "ICR0: adminq event");
5178                 i40e_dev_handle_aq_msg(dev);
5179         }
5180
5181         /* Link Status Change interrupt */
5182         if (icr0 & I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK) {
5183 #define I40E_US_PER_SECOND 1000000
5184                 struct rte_eth_link link;
5185
5186                 PMD_DRV_LOG(INFO, "ICR0: link status changed\n");
5187                 memset(&link, 0, sizeof(link));
5188                 rte_i40e_dev_atomic_read_link_status(dev, &link);
5189                 i40e_dev_link_update(dev, 0);
5190
5191                 /*
5192                  * For link up interrupt, it needs to wait 1 second to let the
5193                  * hardware be a stable state. Otherwise several consecutive
5194                  * interrupts can be observed.
5195                  * For link down interrupt, no need to wait.
5196                  */
5197                 if (!link.link_status && rte_eal_alarm_set(I40E_US_PER_SECOND,
5198                         i40e_dev_interrupt_delayed_handler, (void *)dev) >= 0)
5199                         return;
5200                 else
5201                         _rte_eth_dev_callback_process(dev,
5202                                 RTE_ETH_EVENT_INTR_LSC);
5203         }
5204
5205 done:
5206         /* Enable interrupt */
5207         i40e_pf_enable_irq0(hw);
5208         rte_intr_enable(&(dev->pci_dev->intr_handle));
5209 }
5210
5211 static int
5212 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
5213                          struct i40e_macvlan_filter *filter,
5214                          int total)
5215 {
5216         int ele_num, ele_buff_size;
5217         int num, actual_num, i;
5218         uint16_t flags;
5219         int ret = I40E_SUCCESS;
5220         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5221         struct i40e_aqc_add_macvlan_element_data *req_list;
5222
5223         if (filter == NULL  || total == 0)
5224                 return I40E_ERR_PARAM;
5225         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5226         ele_buff_size = hw->aq.asq_buf_size;
5227
5228         req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
5229         if (req_list == NULL) {
5230                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5231                 return I40E_ERR_NO_MEMORY;
5232         }
5233
5234         num = 0;
5235         do {
5236                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5237                 memset(req_list, 0, ele_buff_size);
5238
5239                 for (i = 0; i < actual_num; i++) {
5240                         (void)rte_memcpy(req_list[i].mac_addr,
5241                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5242                         req_list[i].vlan_tag =
5243                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5244
5245                         switch (filter[num + i].filter_type) {
5246                         case RTE_MAC_PERFECT_MATCH:
5247                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH |
5248                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5249                                 break;
5250                         case RTE_MACVLAN_PERFECT_MATCH:
5251                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
5252                                 break;
5253                         case RTE_MAC_HASH_MATCH:
5254                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH |
5255                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
5256                                 break;
5257                         case RTE_MACVLAN_HASH_MATCH:
5258                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH;
5259                                 break;
5260                         default:
5261                                 PMD_DRV_LOG(ERR, "Invalid MAC match type\n");
5262                                 ret = I40E_ERR_PARAM;
5263                                 goto DONE;
5264                         }
5265
5266                         req_list[i].queue_number = 0;
5267
5268                         req_list[i].flags = rte_cpu_to_le_16(flags);
5269                 }
5270
5271                 ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
5272                                                 actual_num, NULL);
5273                 if (ret != I40E_SUCCESS) {
5274                         PMD_DRV_LOG(ERR, "Failed to add macvlan filter");
5275                         goto DONE;
5276                 }
5277                 num += actual_num;
5278         } while (num < total);
5279
5280 DONE:
5281         rte_free(req_list);
5282         return ret;
5283 }
5284
5285 static int
5286 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
5287                             struct i40e_macvlan_filter *filter,
5288                             int total)
5289 {
5290         int ele_num, ele_buff_size;
5291         int num, actual_num, i;
5292         uint16_t flags;
5293         int ret = I40E_SUCCESS;
5294         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5295         struct i40e_aqc_remove_macvlan_element_data *req_list;
5296
5297         if (filter == NULL  || total == 0)
5298                 return I40E_ERR_PARAM;
5299
5300         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
5301         ele_buff_size = hw->aq.asq_buf_size;
5302
5303         req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
5304         if (req_list == NULL) {
5305                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
5306                 return I40E_ERR_NO_MEMORY;
5307         }
5308
5309         num = 0;
5310         do {
5311                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
5312                 memset(req_list, 0, ele_buff_size);
5313
5314                 for (i = 0; i < actual_num; i++) {
5315                         (void)rte_memcpy(req_list[i].mac_addr,
5316                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
5317                         req_list[i].vlan_tag =
5318                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
5319
5320                         switch (filter[num + i].filter_type) {
5321                         case RTE_MAC_PERFECT_MATCH:
5322                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
5323                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5324                                 break;
5325                         case RTE_MACVLAN_PERFECT_MATCH:
5326                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
5327                                 break;
5328                         case RTE_MAC_HASH_MATCH:
5329                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH |
5330                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
5331                                 break;
5332                         case RTE_MACVLAN_HASH_MATCH:
5333                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH;
5334                                 break;
5335                         default:
5336                                 PMD_DRV_LOG(ERR, "Invalid MAC filter type\n");
5337                                 ret = I40E_ERR_PARAM;
5338                                 goto DONE;
5339                         }
5340                         req_list[i].flags = rte_cpu_to_le_16(flags);
5341                 }
5342
5343                 ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
5344                                                 actual_num, NULL);
5345                 if (ret != I40E_SUCCESS) {
5346                         PMD_DRV_LOG(ERR, "Failed to remove macvlan filter");
5347                         goto DONE;
5348                 }
5349                 num += actual_num;
5350         } while (num < total);
5351
5352 DONE:
5353         rte_free(req_list);
5354         return ret;
5355 }
5356
5357 /* Find out specific MAC filter */
5358 static struct i40e_mac_filter *
5359 i40e_find_mac_filter(struct i40e_vsi *vsi,
5360                          struct ether_addr *macaddr)
5361 {
5362         struct i40e_mac_filter *f;
5363
5364         TAILQ_FOREACH(f, &vsi->mac_list, next) {
5365                 if (is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
5366                         return f;
5367         }
5368
5369         return NULL;
5370 }
5371
5372 static bool
5373 i40e_find_vlan_filter(struct i40e_vsi *vsi,
5374                          uint16_t vlan_id)
5375 {
5376         uint32_t vid_idx, vid_bit;
5377
5378         if (vlan_id > ETH_VLAN_ID_MAX)
5379                 return 0;
5380
5381         vid_idx = I40E_VFTA_IDX(vlan_id);
5382         vid_bit = I40E_VFTA_BIT(vlan_id);
5383
5384         if (vsi->vfta[vid_idx] & vid_bit)
5385                 return 1;
5386         else
5387                 return 0;
5388 }
5389
5390 static void
5391 i40e_set_vlan_filter(struct i40e_vsi *vsi,
5392                          uint16_t vlan_id, bool on)
5393 {
5394         uint32_t vid_idx, vid_bit;
5395
5396         if (vlan_id > ETH_VLAN_ID_MAX)
5397                 return;
5398
5399         vid_idx = I40E_VFTA_IDX(vlan_id);
5400         vid_bit = I40E_VFTA_BIT(vlan_id);
5401
5402         if (on)
5403                 vsi->vfta[vid_idx] |= vid_bit;
5404         else
5405                 vsi->vfta[vid_idx] &= ~vid_bit;
5406 }
5407
5408 /**
5409  * Find all vlan options for specific mac addr,
5410  * return with actual vlan found.
5411  */
5412 static inline int
5413 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
5414                            struct i40e_macvlan_filter *mv_f,
5415                            int num, struct ether_addr *addr)
5416 {
5417         int i;
5418         uint32_t j, k;
5419
5420         /**
5421          * Not to use i40e_find_vlan_filter to decrease the loop time,
5422          * although the code looks complex.
5423           */
5424         if (num < vsi->vlan_num)
5425                 return I40E_ERR_PARAM;
5426
5427         i = 0;
5428         for (j = 0; j < I40E_VFTA_SIZE; j++) {
5429                 if (vsi->vfta[j]) {
5430                         for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
5431                                 if (vsi->vfta[j] & (1 << k)) {
5432                                         if (i > num - 1) {
5433                                                 PMD_DRV_LOG(ERR, "vlan number "
5434                                                             "not match");
5435                                                 return I40E_ERR_PARAM;
5436                                         }
5437                                         (void)rte_memcpy(&mv_f[i].macaddr,
5438                                                         addr, ETH_ADDR_LEN);
5439                                         mv_f[i].vlan_id =
5440                                                 j * I40E_UINT32_BIT_SIZE + k;
5441                                         i++;
5442                                 }
5443                         }
5444                 }
5445         }
5446         return I40E_SUCCESS;
5447 }
5448
5449 static inline int
5450 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
5451                            struct i40e_macvlan_filter *mv_f,
5452                            int num,
5453                            uint16_t vlan)
5454 {
5455         int i = 0;
5456         struct i40e_mac_filter *f;
5457
5458         if (num < vsi->mac_num)
5459                 return I40E_ERR_PARAM;
5460
5461         TAILQ_FOREACH(f, &vsi->mac_list, next) {
5462                 if (i > num - 1) {
5463                         PMD_DRV_LOG(ERR, "buffer number not match");
5464                         return I40E_ERR_PARAM;
5465                 }
5466                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
5467                                 ETH_ADDR_LEN);
5468                 mv_f[i].vlan_id = vlan;
5469                 mv_f[i].filter_type = f->mac_info.filter_type;
5470                 i++;
5471         }
5472
5473         return I40E_SUCCESS;
5474 }
5475
5476 static int
5477 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
5478 {
5479         int i, num;
5480         struct i40e_mac_filter *f;
5481         struct i40e_macvlan_filter *mv_f;
5482         int ret = I40E_SUCCESS;
5483
5484         if (vsi == NULL || vsi->mac_num == 0)
5485                 return I40E_ERR_PARAM;
5486
5487         /* Case that no vlan is set */
5488         if (vsi->vlan_num == 0)
5489                 num = vsi->mac_num;
5490         else
5491                 num = vsi->mac_num * vsi->vlan_num;
5492
5493         mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
5494         if (mv_f == NULL) {
5495                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5496                 return I40E_ERR_NO_MEMORY;
5497         }
5498
5499         i = 0;
5500         if (vsi->vlan_num == 0) {
5501                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
5502                         (void)rte_memcpy(&mv_f[i].macaddr,
5503                                 &f->mac_info.mac_addr, ETH_ADDR_LEN);
5504                         mv_f[i].vlan_id = 0;
5505                         i++;
5506                 }
5507         } else {
5508                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
5509                         ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
5510                                         vsi->vlan_num, &f->mac_info.mac_addr);
5511                         if (ret != I40E_SUCCESS)
5512                                 goto DONE;
5513                         i += vsi->vlan_num;
5514                 }
5515         }
5516
5517         ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
5518 DONE:
5519         rte_free(mv_f);
5520
5521         return ret;
5522 }
5523
5524 int
5525 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
5526 {
5527         struct i40e_macvlan_filter *mv_f;
5528         int mac_num;
5529         int ret = I40E_SUCCESS;
5530
5531         if (!vsi || vlan > ETHER_MAX_VLAN_ID)
5532                 return I40E_ERR_PARAM;
5533
5534         /* If it's already set, just return */
5535         if (i40e_find_vlan_filter(vsi,vlan))
5536                 return I40E_SUCCESS;
5537
5538         mac_num = vsi->mac_num;
5539
5540         if (mac_num == 0) {
5541                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
5542                 return I40E_ERR_PARAM;
5543         }
5544
5545         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
5546
5547         if (mv_f == NULL) {
5548                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5549                 return I40E_ERR_NO_MEMORY;
5550         }
5551
5552         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
5553
5554         if (ret != I40E_SUCCESS)
5555                 goto DONE;
5556
5557         ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
5558
5559         if (ret != I40E_SUCCESS)
5560                 goto DONE;
5561
5562         i40e_set_vlan_filter(vsi, vlan, 1);
5563
5564         vsi->vlan_num++;
5565         ret = I40E_SUCCESS;
5566 DONE:
5567         rte_free(mv_f);
5568         return ret;
5569 }
5570
5571 int
5572 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
5573 {
5574         struct i40e_macvlan_filter *mv_f;
5575         int mac_num;
5576         int ret = I40E_SUCCESS;
5577
5578         /**
5579          * Vlan 0 is the generic filter for untagged packets
5580          * and can't be removed.
5581          */
5582         if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
5583                 return I40E_ERR_PARAM;
5584
5585         /* If can't find it, just return */
5586         if (!i40e_find_vlan_filter(vsi, vlan))
5587                 return I40E_ERR_PARAM;
5588
5589         mac_num = vsi->mac_num;
5590
5591         if (mac_num == 0) {
5592                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
5593                 return I40E_ERR_PARAM;
5594         }
5595
5596         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
5597
5598         if (mv_f == NULL) {
5599                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5600                 return I40E_ERR_NO_MEMORY;
5601         }
5602
5603         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
5604
5605         if (ret != I40E_SUCCESS)
5606                 goto DONE;
5607
5608         ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
5609
5610         if (ret != I40E_SUCCESS)
5611                 goto DONE;
5612
5613         /* This is last vlan to remove, replace all mac filter with vlan 0 */
5614         if (vsi->vlan_num == 1) {
5615                 ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
5616                 if (ret != I40E_SUCCESS)
5617                         goto DONE;
5618
5619                 ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
5620                 if (ret != I40E_SUCCESS)
5621                         goto DONE;
5622         }
5623
5624         i40e_set_vlan_filter(vsi, vlan, 0);
5625
5626         vsi->vlan_num--;
5627         ret = I40E_SUCCESS;
5628 DONE:
5629         rte_free(mv_f);
5630         return ret;
5631 }
5632
5633 int
5634 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter)
5635 {
5636         struct i40e_mac_filter *f;
5637         struct i40e_macvlan_filter *mv_f;
5638         int i, vlan_num = 0;
5639         int ret = I40E_SUCCESS;
5640
5641         /* If it's add and we've config it, return */
5642         f = i40e_find_mac_filter(vsi, &mac_filter->mac_addr);
5643         if (f != NULL)
5644                 return I40E_SUCCESS;
5645         if ((mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH) ||
5646                 (mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH)) {
5647
5648                 /**
5649                  * If vlan_num is 0, that's the first time to add mac,
5650                  * set mask for vlan_id 0.
5651                  */
5652                 if (vsi->vlan_num == 0) {
5653                         i40e_set_vlan_filter(vsi, 0, 1);
5654                         vsi->vlan_num = 1;
5655                 }
5656                 vlan_num = vsi->vlan_num;
5657         } else if ((mac_filter->filter_type == RTE_MAC_PERFECT_MATCH) ||
5658                         (mac_filter->filter_type == RTE_MAC_HASH_MATCH))
5659                 vlan_num = 1;
5660
5661         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
5662         if (mv_f == NULL) {
5663                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5664                 return I40E_ERR_NO_MEMORY;
5665         }
5666
5667         for (i = 0; i < vlan_num; i++) {
5668                 mv_f[i].filter_type = mac_filter->filter_type;
5669                 (void)rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr,
5670                                 ETH_ADDR_LEN);
5671         }
5672
5673         if (mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH ||
5674                 mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH) {
5675                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num,
5676                                         &mac_filter->mac_addr);
5677                 if (ret != I40E_SUCCESS)
5678                         goto DONE;
5679         }
5680
5681         ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
5682         if (ret != I40E_SUCCESS)
5683                 goto DONE;
5684
5685         /* Add the mac addr into mac list */
5686         f = rte_zmalloc("macv_filter", sizeof(*f), 0);
5687         if (f == NULL) {
5688                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5689                 ret = I40E_ERR_NO_MEMORY;
5690                 goto DONE;
5691         }
5692         (void)rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr,
5693                         ETH_ADDR_LEN);
5694         f->mac_info.filter_type = mac_filter->filter_type;
5695         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
5696         vsi->mac_num++;
5697
5698         ret = I40E_SUCCESS;
5699 DONE:
5700         rte_free(mv_f);
5701
5702         return ret;
5703 }
5704
5705 int
5706 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
5707 {
5708         struct i40e_mac_filter *f;
5709         struct i40e_macvlan_filter *mv_f;
5710         int i, vlan_num;
5711         enum rte_mac_filter_type filter_type;
5712         int ret = I40E_SUCCESS;
5713
5714         /* Can't find it, return an error */
5715         f = i40e_find_mac_filter(vsi, addr);
5716         if (f == NULL)
5717                 return I40E_ERR_PARAM;
5718
5719         vlan_num = vsi->vlan_num;
5720         filter_type = f->mac_info.filter_type;
5721         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
5722                 filter_type == RTE_MACVLAN_HASH_MATCH) {
5723                 if (vlan_num == 0) {
5724                         PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0\n");
5725                         return I40E_ERR_PARAM;
5726                 }
5727         } else if (filter_type == RTE_MAC_PERFECT_MATCH ||
5728                         filter_type == RTE_MAC_HASH_MATCH)
5729                 vlan_num = 1;
5730
5731         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
5732         if (mv_f == NULL) {
5733                 PMD_DRV_LOG(ERR, "failed to allocate memory");
5734                 return I40E_ERR_NO_MEMORY;
5735         }
5736
5737         for (i = 0; i < vlan_num; i++) {
5738                 mv_f[i].filter_type = filter_type;
5739                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
5740                                 ETH_ADDR_LEN);
5741         }
5742         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
5743                         filter_type == RTE_MACVLAN_HASH_MATCH) {
5744                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
5745                 if (ret != I40E_SUCCESS)
5746                         goto DONE;
5747         }
5748
5749         ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
5750         if (ret != I40E_SUCCESS)
5751                 goto DONE;
5752
5753         /* Remove the mac addr into mac list */
5754         TAILQ_REMOVE(&vsi->mac_list, f, next);
5755         rte_free(f);
5756         vsi->mac_num--;
5757
5758         ret = I40E_SUCCESS;
5759 DONE:
5760         rte_free(mv_f);
5761         return ret;
5762 }
5763
5764 /* Configure hash enable flags for RSS */
5765 uint64_t
5766 i40e_config_hena(uint64_t flags)
5767 {
5768         uint64_t hena = 0;
5769
5770         if (!flags)
5771                 return hena;
5772
5773         if (flags & ETH_RSS_FRAG_IPV4)
5774                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4;
5775         if (flags & ETH_RSS_NONFRAG_IPV4_TCP)
5776                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
5777         if (flags & ETH_RSS_NONFRAG_IPV4_UDP)
5778                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
5779         if (flags & ETH_RSS_NONFRAG_IPV4_SCTP)
5780                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP;
5781         if (flags & ETH_RSS_NONFRAG_IPV4_OTHER)
5782                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
5783         if (flags & ETH_RSS_FRAG_IPV6)
5784                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6;
5785         if (flags & ETH_RSS_NONFRAG_IPV6_TCP)
5786                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
5787         if (flags & ETH_RSS_NONFRAG_IPV6_UDP)
5788                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
5789         if (flags & ETH_RSS_NONFRAG_IPV6_SCTP)
5790                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP;
5791         if (flags & ETH_RSS_NONFRAG_IPV6_OTHER)
5792                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
5793         if (flags & ETH_RSS_L2_PAYLOAD)
5794                 hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD;
5795
5796         return hena;
5797 }
5798
5799 /* Parse the hash enable flags */
5800 uint64_t
5801 i40e_parse_hena(uint64_t flags)
5802 {
5803         uint64_t rss_hf = 0;
5804
5805         if (!flags)
5806                 return rss_hf;
5807         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4))
5808                 rss_hf |= ETH_RSS_FRAG_IPV4;
5809         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP))
5810                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
5811         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP))
5812                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
5813         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP))
5814                 rss_hf |= ETH_RSS_NONFRAG_IPV4_SCTP;
5815         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER))
5816                 rss_hf |= ETH_RSS_NONFRAG_IPV4_OTHER;
5817         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6))
5818                 rss_hf |= ETH_RSS_FRAG_IPV6;
5819         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP))
5820                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
5821         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP))
5822                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
5823         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP))
5824                 rss_hf |= ETH_RSS_NONFRAG_IPV6_SCTP;
5825         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER))
5826                 rss_hf |= ETH_RSS_NONFRAG_IPV6_OTHER;
5827         if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
5828                 rss_hf |= ETH_RSS_L2_PAYLOAD;
5829
5830         return rss_hf;
5831 }
5832
5833 /* Disable RSS */
5834 static void
5835 i40e_pf_disable_rss(struct i40e_pf *pf)
5836 {
5837         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5838         uint64_t hena;
5839
5840         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
5841         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
5842         hena &= ~I40E_RSS_HENA_ALL;
5843         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
5844         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
5845         I40E_WRITE_FLUSH(hw);
5846 }
5847
5848 static int
5849 i40e_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
5850 {
5851         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
5852         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5853         int ret = 0;
5854
5855         if (!key || key_len == 0) {
5856                 PMD_DRV_LOG(DEBUG, "No key to be configured");
5857                 return 0;
5858         } else if (key_len != (I40E_PFQF_HKEY_MAX_INDEX + 1) *
5859                 sizeof(uint32_t)) {
5860                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
5861                 return -EINVAL;
5862         }
5863
5864         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
5865                 struct i40e_aqc_get_set_rss_key_data *key_dw =
5866                         (struct i40e_aqc_get_set_rss_key_data *)key;
5867
5868                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
5869                 if (ret)
5870                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
5871                                      "via AQ");
5872         } else {
5873                 uint32_t *hash_key = (uint32_t *)key;
5874                 uint16_t i;
5875
5876                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
5877                         i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), hash_key[i]);
5878                 I40E_WRITE_FLUSH(hw);
5879         }
5880
5881         return ret;
5882 }
5883
5884 static int
5885 i40e_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
5886 {
5887         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
5888         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
5889         int ret;
5890
5891         if (!key || !key_len)
5892                 return -EINVAL;
5893
5894         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
5895                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
5896                         (struct i40e_aqc_get_set_rss_key_data *)key);
5897                 if (ret) {
5898                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
5899                         return ret;
5900                 }
5901         } else {
5902                 uint32_t *key_dw = (uint32_t *)key;
5903                 uint16_t i;
5904
5905                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
5906                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
5907         }
5908         *key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
5909
5910         return 0;
5911 }
5912
5913 static int
5914 i40e_hw_rss_hash_set(struct i40e_pf *pf, struct rte_eth_rss_conf *rss_conf)
5915 {
5916         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5917         uint64_t rss_hf;
5918         uint64_t hena;
5919         int ret;
5920
5921         ret = i40e_set_rss_key(pf->main_vsi, rss_conf->rss_key,
5922                                rss_conf->rss_key_len);
5923         if (ret)
5924                 return ret;
5925
5926         rss_hf = rss_conf->rss_hf;
5927         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
5928         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
5929         hena &= ~I40E_RSS_HENA_ALL;
5930         hena |= i40e_config_hena(rss_hf);
5931         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
5932         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
5933         I40E_WRITE_FLUSH(hw);
5934
5935         return 0;
5936 }
5937
5938 static int
5939 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
5940                          struct rte_eth_rss_conf *rss_conf)
5941 {
5942         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5943         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5944         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
5945         uint64_t hena;
5946
5947         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
5948         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
5949         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
5950                 if (rss_hf != 0) /* Enable RSS */
5951                         return -EINVAL;
5952                 return 0; /* Nothing to do */
5953         }
5954         /* RSS enabled */
5955         if (rss_hf == 0) /* Disable RSS */
5956                 return -EINVAL;
5957
5958         return i40e_hw_rss_hash_set(pf, rss_conf);
5959 }
5960
5961 static int
5962 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
5963                            struct rte_eth_rss_conf *rss_conf)
5964 {
5965         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5966         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5967         uint64_t hena;
5968
5969         i40e_get_rss_key(pf->main_vsi, rss_conf->rss_key,
5970                          &rss_conf->rss_key_len);
5971
5972         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0));
5973         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1))) << 32;
5974         rss_conf->rss_hf = i40e_parse_hena(hena);
5975
5976         return 0;
5977 }
5978
5979 static int
5980 i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag)
5981 {
5982         switch (filter_type) {
5983         case RTE_TUNNEL_FILTER_IMAC_IVLAN:
5984                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN;
5985                 break;
5986         case RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID:
5987                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID;
5988                 break;
5989         case RTE_TUNNEL_FILTER_IMAC_TENID:
5990                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID;
5991                 break;
5992         case RTE_TUNNEL_FILTER_OMAC_TENID_IMAC:
5993                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC;
5994                 break;
5995         case ETH_TUNNEL_FILTER_IMAC:
5996                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC;
5997                 break;
5998         case ETH_TUNNEL_FILTER_OIP:
5999                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OIP;
6000                 break;
6001         case ETH_TUNNEL_FILTER_IIP:
6002                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IIP;
6003                 break;
6004         default:
6005                 PMD_DRV_LOG(ERR, "invalid tunnel filter type");
6006                 return -EINVAL;
6007         }
6008
6009         return 0;
6010 }
6011
6012 static int
6013 i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
6014                         struct rte_eth_tunnel_filter_conf *tunnel_filter,
6015                         uint8_t add)
6016 {
6017         uint16_t ip_type;
6018         uint8_t i, tun_type = 0;
6019         /* internal varialbe to convert ipv6 byte order */
6020         uint32_t convert_ipv6[4];
6021         int val, ret = 0;
6022         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6023         struct i40e_vsi *vsi = pf->main_vsi;
6024         struct i40e_aqc_add_remove_cloud_filters_element_data  *cld_filter;
6025         struct i40e_aqc_add_remove_cloud_filters_element_data  *pfilter;
6026
6027         cld_filter = rte_zmalloc("tunnel_filter",
6028                 sizeof(struct i40e_aqc_add_remove_cloud_filters_element_data),
6029                 0);
6030
6031         if (NULL == cld_filter) {
6032                 PMD_DRV_LOG(ERR, "Failed to alloc memory.");
6033                 return -EINVAL;
6034         }
6035         pfilter = cld_filter;
6036
6037         ether_addr_copy(&tunnel_filter->outer_mac, (struct ether_addr*)&pfilter->outer_mac);
6038         ether_addr_copy(&tunnel_filter->inner_mac, (struct ether_addr*)&pfilter->inner_mac);
6039
6040         pfilter->inner_vlan = rte_cpu_to_le_16(tunnel_filter->inner_vlan);
6041         if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) {
6042                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4;
6043                 rte_memcpy(&pfilter->ipaddr.v4.data,
6044                                 &rte_cpu_to_le_32(tunnel_filter->ip_addr.ipv4_addr),
6045                                 sizeof(pfilter->ipaddr.v4.data));
6046         } else {
6047                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6;
6048                 for (i = 0; i < 4; i++) {
6049                         convert_ipv6[i] =
6050                         rte_cpu_to_le_32(tunnel_filter->ip_addr.ipv6_addr[i]);
6051                 }
6052                 rte_memcpy(&pfilter->ipaddr.v6.data, &convert_ipv6,
6053                                 sizeof(pfilter->ipaddr.v6.data));
6054         }
6055
6056         /* check tunneled type */
6057         switch (tunnel_filter->tunnel_type) {
6058         case RTE_TUNNEL_TYPE_VXLAN:
6059                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_VXLAN;
6060                 break;
6061         case RTE_TUNNEL_TYPE_NVGRE:
6062                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_NVGRE_OMAC;
6063                 break;
6064         case RTE_TUNNEL_TYPE_IP_IN_GRE:
6065                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_IP;
6066                 break;
6067         default:
6068                 /* Other tunnel types is not supported. */
6069                 PMD_DRV_LOG(ERR, "tunnel type is not supported.");
6070                 rte_free(cld_filter);
6071                 return -EINVAL;
6072         }
6073
6074         val = i40e_dev_get_filter_type(tunnel_filter->filter_type,
6075                                                 &pfilter->flags);
6076         if (val < 0) {
6077                 rte_free(cld_filter);
6078                 return -EINVAL;
6079         }
6080
6081         pfilter->flags |= rte_cpu_to_le_16(
6082                 I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE |
6083                 ip_type | (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT));
6084         pfilter->tenant_id = rte_cpu_to_le_32(tunnel_filter->tenant_id);
6085         pfilter->queue_number = rte_cpu_to_le_16(tunnel_filter->queue_id);
6086
6087         if (add)
6088                 ret = i40e_aq_add_cloud_filters(hw, vsi->seid, cld_filter, 1);
6089         else
6090                 ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
6091                                                 cld_filter, 1);
6092
6093         rte_free(cld_filter);
6094         return ret;
6095 }
6096
6097 static int
6098 i40e_get_vxlan_port_idx(struct i40e_pf *pf, uint16_t port)
6099 {
6100         uint8_t i;
6101
6102         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
6103                 if (pf->vxlan_ports[i] == port)
6104                         return i;
6105         }
6106
6107         return -1;
6108 }
6109
6110 static int
6111 i40e_add_vxlan_port(struct i40e_pf *pf, uint16_t port)
6112 {
6113         int  idx, ret;
6114         uint8_t filter_idx;
6115         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6116
6117         idx = i40e_get_vxlan_port_idx(pf, port);
6118
6119         /* Check if port already exists */
6120         if (idx >= 0) {
6121                 PMD_DRV_LOG(ERR, "Port %d already offloaded", port);
6122                 return -EINVAL;
6123         }
6124
6125         /* Now check if there is space to add the new port */
6126         idx = i40e_get_vxlan_port_idx(pf, 0);
6127         if (idx < 0) {
6128                 PMD_DRV_LOG(ERR, "Maximum number of UDP ports reached,"
6129                         "not adding port %d", port);
6130                 return -ENOSPC;
6131         }
6132
6133         ret =  i40e_aq_add_udp_tunnel(hw, port, I40E_AQC_TUNNEL_TYPE_VXLAN,
6134                                         &filter_idx, NULL);
6135         if (ret < 0) {
6136                 PMD_DRV_LOG(ERR, "Failed to add VXLAN UDP port %d", port);
6137                 return -1;
6138         }
6139
6140         PMD_DRV_LOG(INFO, "Added port %d with AQ command with index %d",
6141                          port,  filter_idx);
6142
6143         /* New port: add it and mark its index in the bitmap */
6144         pf->vxlan_ports[idx] = port;
6145         pf->vxlan_bitmap |= (1 << idx);
6146
6147         if (!(pf->flags & I40E_FLAG_VXLAN))
6148                 pf->flags |= I40E_FLAG_VXLAN;
6149
6150         return 0;
6151 }
6152
6153 static int
6154 i40e_del_vxlan_port(struct i40e_pf *pf, uint16_t port)
6155 {
6156         int idx;
6157         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6158
6159         if (!(pf->flags & I40E_FLAG_VXLAN)) {
6160                 PMD_DRV_LOG(ERR, "VXLAN UDP port was not configured.");
6161                 return -EINVAL;
6162         }
6163
6164         idx = i40e_get_vxlan_port_idx(pf, port);
6165
6166         if (idx < 0) {
6167                 PMD_DRV_LOG(ERR, "Port %d doesn't exist", port);
6168                 return -EINVAL;
6169         }
6170
6171         if (i40e_aq_del_udp_tunnel(hw, idx, NULL) < 0) {
6172                 PMD_DRV_LOG(ERR, "Failed to delete VXLAN UDP port %d", port);
6173                 return -1;
6174         }
6175
6176         PMD_DRV_LOG(INFO, "Deleted port %d with AQ command with index %d",
6177                         port, idx);
6178
6179         pf->vxlan_ports[idx] = 0;
6180         pf->vxlan_bitmap &= ~(1 << idx);
6181
6182         if (!pf->vxlan_bitmap)
6183                 pf->flags &= ~I40E_FLAG_VXLAN;
6184
6185         return 0;
6186 }
6187
6188 /* Add UDP tunneling port */
6189 static int
6190 i40e_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
6191                              struct rte_eth_udp_tunnel *udp_tunnel)
6192 {
6193         int ret = 0;
6194         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6195
6196         if (udp_tunnel == NULL)
6197                 return -EINVAL;
6198
6199         switch (udp_tunnel->prot_type) {
6200         case RTE_TUNNEL_TYPE_VXLAN:
6201                 ret = i40e_add_vxlan_port(pf, udp_tunnel->udp_port);
6202                 break;
6203
6204         case RTE_TUNNEL_TYPE_GENEVE:
6205         case RTE_TUNNEL_TYPE_TEREDO:
6206                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6207                 ret = -1;
6208                 break;
6209
6210         default:
6211                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6212                 ret = -1;
6213                 break;
6214         }
6215
6216         return ret;
6217 }
6218
6219 /* Remove UDP tunneling port */
6220 static int
6221 i40e_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
6222                              struct rte_eth_udp_tunnel *udp_tunnel)
6223 {
6224         int ret = 0;
6225         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6226
6227         if (udp_tunnel == NULL)
6228                 return -EINVAL;
6229
6230         switch (udp_tunnel->prot_type) {
6231         case RTE_TUNNEL_TYPE_VXLAN:
6232                 ret = i40e_del_vxlan_port(pf, udp_tunnel->udp_port);
6233                 break;
6234         case RTE_TUNNEL_TYPE_GENEVE:
6235         case RTE_TUNNEL_TYPE_TEREDO:
6236                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
6237                 ret = -1;
6238                 break;
6239         default:
6240                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6241                 ret = -1;
6242                 break;
6243         }
6244
6245         return ret;
6246 }
6247
6248 /* Calculate the maximum number of contiguous PF queues that are configured */
6249 static int
6250 i40e_pf_calc_configured_queues_num(struct i40e_pf *pf)
6251 {
6252         struct rte_eth_dev_data *data = pf->dev_data;
6253         int i, num;
6254         struct i40e_rx_queue *rxq;
6255
6256         num = 0;
6257         for (i = 0; i < pf->lan_nb_qps; i++) {
6258                 rxq = data->rx_queues[i];
6259                 if (rxq && rxq->q_set)
6260                         num++;
6261                 else
6262                         break;
6263         }
6264
6265         return num;
6266 }
6267
6268 /* Configure RSS */
6269 static int
6270 i40e_pf_config_rss(struct i40e_pf *pf)
6271 {
6272         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
6273         struct rte_eth_rss_conf rss_conf;
6274         uint32_t i, lut = 0;
6275         uint16_t j, num;
6276
6277         /*
6278          * If both VMDQ and RSS enabled, not all of PF queues are configured.
6279          * It's necessary to calulate the actual PF queues that are configured.
6280          */
6281         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
6282                 num = i40e_pf_calc_configured_queues_num(pf);
6283         else
6284                 num = pf->dev_data->nb_rx_queues;
6285
6286         num = RTE_MIN(num, I40E_MAX_Q_PER_TC);
6287         PMD_INIT_LOG(INFO, "Max of contiguous %u PF queues are configured",
6288                         num);
6289
6290         if (num == 0) {
6291                 PMD_INIT_LOG(ERR, "No PF queues are configured to enable RSS");
6292                 return -ENOTSUP;
6293         }
6294
6295         for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
6296                 if (j == num)
6297                         j = 0;
6298                 lut = (lut << 8) | (j & ((0x1 <<
6299                         hw->func_caps.rss_table_entry_width) - 1));
6300                 if ((i & 3) == 3)
6301                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
6302         }
6303
6304         rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
6305         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
6306                 i40e_pf_disable_rss(pf);
6307                 return 0;
6308         }
6309         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
6310                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
6311                 /* Random default keys */
6312                 static uint32_t rss_key_default[] = {0x6b793944,
6313                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
6314                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
6315                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
6316
6317                 rss_conf.rss_key = (uint8_t *)rss_key_default;
6318                 rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
6319                                                         sizeof(uint32_t);
6320         }
6321
6322         return i40e_hw_rss_hash_set(pf, &rss_conf);
6323 }
6324
6325 static int
6326 i40e_tunnel_filter_param_check(struct i40e_pf *pf,
6327                                struct rte_eth_tunnel_filter_conf *filter)
6328 {
6329         if (pf == NULL || filter == NULL) {
6330                 PMD_DRV_LOG(ERR, "Invalid parameter");
6331                 return -EINVAL;
6332         }
6333
6334         if (filter->queue_id >= pf->dev_data->nb_rx_queues) {
6335                 PMD_DRV_LOG(ERR, "Invalid queue ID");
6336                 return -EINVAL;
6337         }
6338
6339         if (filter->inner_vlan > ETHER_MAX_VLAN_ID) {
6340                 PMD_DRV_LOG(ERR, "Invalid inner VLAN ID");
6341                 return -EINVAL;
6342         }
6343
6344         if ((filter->filter_type & ETH_TUNNEL_FILTER_OMAC) &&
6345                 (is_zero_ether_addr(&filter->outer_mac))) {
6346                 PMD_DRV_LOG(ERR, "Cannot add NULL outer MAC address");
6347                 return -EINVAL;
6348         }
6349
6350         if ((filter->filter_type & ETH_TUNNEL_FILTER_IMAC) &&
6351                 (is_zero_ether_addr(&filter->inner_mac))) {
6352                 PMD_DRV_LOG(ERR, "Cannot add NULL inner MAC address");
6353                 return -EINVAL;
6354         }
6355
6356         return 0;
6357 }
6358
6359 #define I40E_GL_PRS_FVBM_MSK_ENA 0x80000000
6360 #define I40E_GL_PRS_FVBM(_i)     (0x00269760 + ((_i) * 4))
6361 static int
6362 i40e_dev_set_gre_key_len(struct i40e_hw *hw, uint8_t len)
6363 {
6364         uint32_t val, reg;
6365         int ret = -EINVAL;
6366
6367         val = I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2));
6368         PMD_DRV_LOG(DEBUG, "Read original GL_PRS_FVBM with 0x%08x\n", val);
6369
6370         if (len == 3) {
6371                 reg = val | I40E_GL_PRS_FVBM_MSK_ENA;
6372         } else if (len == 4) {
6373                 reg = val & ~I40E_GL_PRS_FVBM_MSK_ENA;
6374         } else {
6375                 PMD_DRV_LOG(ERR, "Unsupported GRE key length of %u", len);
6376                 return ret;
6377         }
6378
6379         if (reg != val) {
6380                 ret = i40e_aq_debug_write_register(hw, I40E_GL_PRS_FVBM(2),
6381                                                    reg, NULL);
6382                 if (ret != 0)
6383                         return ret;
6384         } else {
6385                 ret = 0;
6386         }
6387         PMD_DRV_LOG(DEBUG, "Read modified GL_PRS_FVBM with 0x%08x\n",
6388                     I40E_READ_REG(hw, I40E_GL_PRS_FVBM(2)));
6389
6390         return ret;
6391 }
6392
6393 static int
6394 i40e_dev_global_config_set(struct i40e_hw *hw, struct rte_eth_global_cfg *cfg)
6395 {
6396         int ret = -EINVAL;
6397
6398         if (!hw || !cfg)
6399                 return -EINVAL;
6400
6401         switch (cfg->cfg_type) {
6402         case RTE_ETH_GLOBAL_CFG_TYPE_GRE_KEY_LEN:
6403                 ret = i40e_dev_set_gre_key_len(hw, cfg->cfg.gre_key_len);
6404                 break;
6405         default:
6406                 PMD_DRV_LOG(ERR, "Unknown config type %u", cfg->cfg_type);
6407                 break;
6408         }
6409
6410         return ret;
6411 }
6412
6413 static int
6414 i40e_filter_ctrl_global_config(struct rte_eth_dev *dev,
6415                                enum rte_filter_op filter_op,
6416                                void *arg)
6417 {
6418         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6419         int ret = I40E_ERR_PARAM;
6420
6421         switch (filter_op) {
6422         case RTE_ETH_FILTER_SET:
6423                 ret = i40e_dev_global_config_set(hw,
6424                         (struct rte_eth_global_cfg *)arg);
6425                 break;
6426         default:
6427                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
6428                 break;
6429         }
6430
6431         return ret;
6432 }
6433
6434 static int
6435 i40e_tunnel_filter_handle(struct rte_eth_dev *dev,
6436                           enum rte_filter_op filter_op,
6437                           void *arg)
6438 {
6439         struct rte_eth_tunnel_filter_conf *filter;
6440         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
6441         int ret = I40E_SUCCESS;
6442
6443         filter = (struct rte_eth_tunnel_filter_conf *)(arg);
6444
6445         if (i40e_tunnel_filter_param_check(pf, filter) < 0)
6446                 return I40E_ERR_PARAM;
6447
6448         switch (filter_op) {
6449         case RTE_ETH_FILTER_NOP:
6450                 if (!(pf->flags & I40E_FLAG_VXLAN))
6451                         ret = I40E_NOT_SUPPORTED;
6452                 break;
6453         case RTE_ETH_FILTER_ADD:
6454                 ret = i40e_dev_tunnel_filter_set(pf, filter, 1);
6455                 break;
6456         case RTE_ETH_FILTER_DELETE:
6457                 ret = i40e_dev_tunnel_filter_set(pf, filter, 0);
6458                 break;
6459         default:
6460                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
6461                 ret = I40E_ERR_PARAM;
6462                 break;
6463         }
6464
6465         return ret;
6466 }
6467
6468 static int
6469 i40e_pf_config_mq_rx(struct i40e_pf *pf)
6470 {
6471         int ret = 0;
6472         enum rte_eth_rx_mq_mode mq_mode = pf->dev_data->dev_conf.rxmode.mq_mode;
6473
6474         /* RSS setup */
6475         if (mq_mode & ETH_MQ_RX_RSS_FLAG)
6476                 ret = i40e_pf_config_rss(pf);
6477         else
6478                 i40e_pf_disable_rss(pf);
6479
6480         return ret;
6481 }
6482
6483 /* Get the symmetric hash enable configurations per port */
6484 static void
6485 i40e_get_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t *enable)
6486 {
6487         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
6488
6489         *enable = reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK ? 1 : 0;
6490 }
6491
6492 /* Set the symmetric hash enable configurations per port */
6493 static void
6494 i40e_set_symmetric_hash_enable_per_port(struct i40e_hw *hw, uint8_t enable)
6495 {
6496         uint32_t reg = i40e_read_rx_ctl(hw, I40E_PRTQF_CTL_0);
6497
6498         if (enable > 0) {
6499                 if (reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK) {
6500                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
6501                                                         "been enabled");
6502                         return;
6503                 }
6504                 reg |= I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
6505         } else {
6506                 if (!(reg & I40E_PRTQF_CTL_0_HSYM_ENA_MASK)) {
6507                         PMD_DRV_LOG(INFO, "Symmetric hash has already "
6508                                                         "been disabled");
6509                         return;
6510                 }
6511                 reg &= ~I40E_PRTQF_CTL_0_HSYM_ENA_MASK;
6512         }
6513         i40e_write_rx_ctl(hw, I40E_PRTQF_CTL_0, reg);
6514         I40E_WRITE_FLUSH(hw);
6515 }
6516
6517 /*
6518  * Get global configurations of hash function type and symmetric hash enable
6519  * per flow type (pctype). Note that global configuration means it affects all
6520  * the ports on the same NIC.
6521  */
6522 static int
6523 i40e_get_hash_filter_global_config(struct i40e_hw *hw,
6524                                    struct rte_eth_hash_global_conf *g_cfg)
6525 {
6526         uint32_t reg, mask = I40E_FLOW_TYPES;
6527         uint16_t i;
6528         enum i40e_filter_pctype pctype;
6529
6530         memset(g_cfg, 0, sizeof(*g_cfg));
6531         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
6532         if (reg & I40E_GLQF_CTL_HTOEP_MASK)
6533                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ;
6534         else
6535                 g_cfg->hash_func = RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
6536         PMD_DRV_LOG(DEBUG, "Hash function is %s",
6537                 (reg & I40E_GLQF_CTL_HTOEP_MASK) ? "Toeplitz" : "Simple XOR");
6538
6539         for (i = 0; mask && i < RTE_ETH_FLOW_MAX; i++) {
6540                 if (!(mask & (1UL << i)))
6541                         continue;
6542                 mask &= ~(1UL << i);
6543                 /* Bit set indicats the coresponding flow type is supported */
6544                 g_cfg->valid_bit_mask[0] |= (1UL << i);
6545                 pctype = i40e_flowtype_to_pctype(i);
6546                 reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
6547                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
6548                         g_cfg->sym_hash_enable_mask[0] |= (1UL << i);
6549         }
6550
6551         return 0;
6552 }
6553
6554 static int
6555 i40e_hash_global_config_check(struct rte_eth_hash_global_conf *g_cfg)
6556 {
6557         uint32_t i;
6558         uint32_t mask0, i40e_mask = I40E_FLOW_TYPES;
6559
6560         if (g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_TOEPLITZ &&
6561                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR &&
6562                 g_cfg->hash_func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
6563                 PMD_DRV_LOG(ERR, "Unsupported hash function type %d",
6564                                                 g_cfg->hash_func);
6565                 return -EINVAL;
6566         }
6567
6568         /*
6569          * As i40e supports less than 32 flow types, only first 32 bits need to
6570          * be checked.
6571          */
6572         mask0 = g_cfg->valid_bit_mask[0];
6573         for (i = 0; i < RTE_SYM_HASH_MASK_ARRAY_SIZE; i++) {
6574                 if (i == 0) {
6575                         /* Check if any unsupported flow type configured */
6576                         if ((mask0 | i40e_mask) ^ i40e_mask)
6577                                 goto mask_err;
6578                 } else {
6579                         if (g_cfg->valid_bit_mask[i])
6580                                 goto mask_err;
6581                 }
6582         }
6583
6584         return 0;
6585
6586 mask_err:
6587         PMD_DRV_LOG(ERR, "i40e unsupported flow type bit(s) configured");
6588
6589         return -EINVAL;
6590 }
6591
6592 /*
6593  * Set global configurations of hash function type and symmetric hash enable
6594  * per flow type (pctype). Note any modifying global configuration will affect
6595  * all the ports on the same NIC.
6596  */
6597 static int
6598 i40e_set_hash_filter_global_config(struct i40e_hw *hw,
6599                                    struct rte_eth_hash_global_conf *g_cfg)
6600 {
6601         int ret;
6602         uint16_t i;
6603         uint32_t reg;
6604         uint32_t mask0 = g_cfg->valid_bit_mask[0];
6605         enum i40e_filter_pctype pctype;
6606
6607         /* Check the input parameters */
6608         ret = i40e_hash_global_config_check(g_cfg);
6609         if (ret < 0)
6610                 return ret;
6611
6612         for (i = 0; mask0 && i < UINT32_BIT; i++) {
6613                 if (!(mask0 & (1UL << i)))
6614                         continue;
6615                 mask0 &= ~(1UL << i);
6616                 pctype = i40e_flowtype_to_pctype(i);
6617                 reg = (g_cfg->sym_hash_enable_mask[0] & (1UL << i)) ?
6618                                 I40E_GLQF_HSYM_SYMH_ENA_MASK : 0;
6619                 i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
6620         }
6621
6622         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
6623         if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
6624                 /* Toeplitz */
6625                 if (reg & I40E_GLQF_CTL_HTOEP_MASK) {
6626                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
6627                                                                 "Toeplitz");
6628                         goto out;
6629                 }
6630                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
6631         } else if (g_cfg->hash_func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
6632                 /* Simple XOR */
6633                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK)) {
6634                         PMD_DRV_LOG(DEBUG, "Hash function already set to "
6635                                                         "Simple XOR");
6636                         goto out;
6637                 }
6638                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
6639         } else
6640                 /* Use the default, and keep it as it is */
6641                 goto out;
6642
6643         i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
6644
6645 out:
6646         I40E_WRITE_FLUSH(hw);
6647
6648         return 0;
6649 }
6650
6651 /**
6652  * Valid input sets for hash and flow director filters per PCTYPE
6653  */
6654 static uint64_t
6655 i40e_get_valid_input_set(enum i40e_filter_pctype pctype,
6656                 enum rte_filter_type filter)
6657 {
6658         uint64_t valid;
6659
6660         static const uint64_t valid_hash_inset_table[] = {
6661                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
6662                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6663                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6664                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_SRC |
6665                         I40E_INSET_IPV4_DST | I40E_INSET_IPV4_TOS |
6666                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
6667                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
6668                         I40E_INSET_FLEX_PAYLOAD,
6669                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
6670                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6671                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6672                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
6673                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
6674                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
6675                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6676                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6677                         I40E_INSET_FLEX_PAYLOAD,
6678                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
6679                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6680                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6681                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
6682                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
6683                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
6684                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6685                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6686                         I40E_INSET_TCP_FLAGS | I40E_INSET_FLEX_PAYLOAD,
6687                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
6688                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6689                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6690                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
6691                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
6692                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
6693                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6694                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6695                         I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD,
6696                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
6697                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6698                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6699                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV4_TOS |
6700                         I40E_INSET_IPV4_PROTO | I40E_INSET_IPV4_TTL |
6701                         I40E_INSET_TUNNEL_DMAC | I40E_INSET_TUNNEL_ID |
6702                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6703                         I40E_INSET_FLEX_PAYLOAD,
6704                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
6705                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6706                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6707                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
6708                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
6709                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_TUNNEL_DMAC |
6710                         I40E_INSET_TUNNEL_ID | I40E_INSET_IPV6_SRC |
6711                         I40E_INSET_IPV6_DST | I40E_INSET_FLEX_PAYLOAD,
6712                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
6713                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6714                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6715                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
6716                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
6717                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
6718                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
6719                         I40E_INSET_DST_PORT | I40E_INSET_FLEX_PAYLOAD,
6720                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
6721                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6722                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6723                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
6724                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
6725                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
6726                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
6727                         I40E_INSET_DST_PORT | I40E_INSET_TCP_FLAGS |
6728                         I40E_INSET_FLEX_PAYLOAD,
6729                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
6730                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6731                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6732                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
6733                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
6734                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
6735                         I40E_INSET_IPV6_DST | I40E_INSET_SRC_PORT |
6736                         I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT |
6737                         I40E_INSET_FLEX_PAYLOAD,
6738                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
6739                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6740                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6741                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_IPV6_TC |
6742                         I40E_INSET_IPV6_FLOW | I40E_INSET_IPV6_NEXT_HDR |
6743                         I40E_INSET_IPV6_HOP_LIMIT | I40E_INSET_IPV6_SRC |
6744                         I40E_INSET_IPV6_DST | I40E_INSET_TUNNEL_ID |
6745                         I40E_INSET_FLEX_PAYLOAD,
6746                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
6747                         I40E_INSET_DMAC | I40E_INSET_SMAC |
6748                         I40E_INSET_VLAN_OUTER | I40E_INSET_VLAN_INNER |
6749                         I40E_INSET_VLAN_TUNNEL | I40E_INSET_LAST_ETHER_TYPE |
6750                         I40E_INSET_FLEX_PAYLOAD,
6751         };
6752
6753         /**
6754          * Flow director supports only fields defined in
6755          * union rte_eth_fdir_flow.
6756          */
6757         static const uint64_t valid_fdir_inset_table[] = {
6758                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
6759                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6760                 I40E_INSET_FLEX_PAYLOAD,
6761                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
6762                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6763                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6764                 I40E_INSET_FLEX_PAYLOAD,
6765                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
6766                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6767                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6768                 I40E_INSET_FLEX_PAYLOAD,
6769                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
6770                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6771                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6772                 I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD,
6773                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
6774                 I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6775                 I40E_INSET_FLEX_PAYLOAD,
6776                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
6777                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6778                 I40E_INSET_FLEX_PAYLOAD,
6779                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
6780                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6781                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6782                 I40E_INSET_FLEX_PAYLOAD,
6783                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
6784                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6785                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6786                 I40E_INSET_FLEX_PAYLOAD,
6787                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
6788                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6789                 I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6790                 I40E_INSET_SCTP_VT | I40E_INSET_FLEX_PAYLOAD,
6791                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
6792                 I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6793                 I40E_INSET_FLEX_PAYLOAD,
6794                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
6795                 I40E_INSET_LAST_ETHER_TYPE | I40E_INSET_FLEX_PAYLOAD,
6796         };
6797
6798         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
6799                 return 0;
6800         if (filter == RTE_ETH_FILTER_HASH)
6801                 valid = valid_hash_inset_table[pctype];
6802         else
6803                 valid = valid_fdir_inset_table[pctype];
6804
6805         return valid;
6806 }
6807
6808 /**
6809  * Validate if the input set is allowed for a specific PCTYPE
6810  */
6811 static int
6812 i40e_validate_input_set(enum i40e_filter_pctype pctype,
6813                 enum rte_filter_type filter, uint64_t inset)
6814 {
6815         uint64_t valid;
6816
6817         valid = i40e_get_valid_input_set(pctype, filter);
6818         if (inset & (~valid))
6819                 return -EINVAL;
6820
6821         return 0;
6822 }
6823
6824 /* default input set fields combination per pctype */
6825 static uint64_t
6826 i40e_get_default_input_set(uint16_t pctype)
6827 {
6828         static const uint64_t default_inset_table[] = {
6829                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
6830                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
6831                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
6832                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6833                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
6834                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
6835                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6836                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
6837                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
6838                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
6839                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6840                         I40E_INSET_SCTP_VT,
6841                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
6842                         I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST,
6843                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
6844                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
6845                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
6846                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6847                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
6848                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
6849                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6850                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT,
6851                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
6852                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
6853                         I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT |
6854                         I40E_INSET_SCTP_VT,
6855                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
6856                         I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST,
6857                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] =
6858                         I40E_INSET_LAST_ETHER_TYPE,
6859         };
6860
6861         if (pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD)
6862                 return 0;
6863
6864         return default_inset_table[pctype];
6865 }
6866
6867 /**
6868  * Parse the input set from index to logical bit masks
6869  */
6870 static int
6871 i40e_parse_input_set(uint64_t *inset,
6872                      enum i40e_filter_pctype pctype,
6873                      enum rte_eth_input_set_field *field,
6874                      uint16_t size)
6875 {
6876         uint16_t i, j;
6877         int ret = -EINVAL;
6878
6879         static const struct {
6880                 enum rte_eth_input_set_field field;
6881                 uint64_t inset;
6882         } inset_convert_table[] = {
6883                 {RTE_ETH_INPUT_SET_NONE, I40E_INSET_NONE},
6884                 {RTE_ETH_INPUT_SET_L2_SRC_MAC, I40E_INSET_SMAC},
6885                 {RTE_ETH_INPUT_SET_L2_DST_MAC, I40E_INSET_DMAC},
6886                 {RTE_ETH_INPUT_SET_L2_OUTER_VLAN, I40E_INSET_VLAN_OUTER},
6887                 {RTE_ETH_INPUT_SET_L2_INNER_VLAN, I40E_INSET_VLAN_INNER},
6888                 {RTE_ETH_INPUT_SET_L2_ETHERTYPE, I40E_INSET_LAST_ETHER_TYPE},
6889                 {RTE_ETH_INPUT_SET_L3_SRC_IP4, I40E_INSET_IPV4_SRC},
6890                 {RTE_ETH_INPUT_SET_L3_DST_IP4, I40E_INSET_IPV4_DST},
6891                 {RTE_ETH_INPUT_SET_L3_IP4_TOS, I40E_INSET_IPV4_TOS},
6892                 {RTE_ETH_INPUT_SET_L3_IP4_PROTO, I40E_INSET_IPV4_PROTO},
6893                 {RTE_ETH_INPUT_SET_L3_SRC_IP6, I40E_INSET_IPV6_SRC},
6894                 {RTE_ETH_INPUT_SET_L3_DST_IP6, I40E_INSET_IPV6_DST},
6895                 {RTE_ETH_INPUT_SET_L3_IP6_TC, I40E_INSET_IPV6_TC},
6896                 {RTE_ETH_INPUT_SET_L3_IP6_NEXT_HEADER,
6897                         I40E_INSET_IPV6_NEXT_HDR},
6898                 {RTE_ETH_INPUT_SET_L4_UDP_SRC_PORT, I40E_INSET_SRC_PORT},
6899                 {RTE_ETH_INPUT_SET_L4_TCP_SRC_PORT, I40E_INSET_SRC_PORT},
6900                 {RTE_ETH_INPUT_SET_L4_SCTP_SRC_PORT, I40E_INSET_SRC_PORT},
6901                 {RTE_ETH_INPUT_SET_L4_UDP_DST_PORT, I40E_INSET_DST_PORT},
6902                 {RTE_ETH_INPUT_SET_L4_TCP_DST_PORT, I40E_INSET_DST_PORT},
6903                 {RTE_ETH_INPUT_SET_L4_SCTP_DST_PORT, I40E_INSET_DST_PORT},
6904                 {RTE_ETH_INPUT_SET_L4_SCTP_VERIFICATION_TAG,
6905                         I40E_INSET_SCTP_VT},
6906                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_DST_MAC,
6907                         I40E_INSET_TUNNEL_DMAC},
6908                 {RTE_ETH_INPUT_SET_TUNNEL_L2_INNER_VLAN,
6909                         I40E_INSET_VLAN_TUNNEL},
6910                 {RTE_ETH_INPUT_SET_TUNNEL_L4_UDP_KEY,
6911                         I40E_INSET_TUNNEL_ID},
6912                 {RTE_ETH_INPUT_SET_TUNNEL_GRE_KEY, I40E_INSET_TUNNEL_ID},
6913                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_1ST_WORD,
6914                         I40E_INSET_FLEX_PAYLOAD_W1},
6915                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_2ND_WORD,
6916                         I40E_INSET_FLEX_PAYLOAD_W2},
6917                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_3RD_WORD,
6918                         I40E_INSET_FLEX_PAYLOAD_W3},
6919                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_4TH_WORD,
6920                         I40E_INSET_FLEX_PAYLOAD_W4},
6921                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_5TH_WORD,
6922                         I40E_INSET_FLEX_PAYLOAD_W5},
6923                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_6TH_WORD,
6924                         I40E_INSET_FLEX_PAYLOAD_W6},
6925                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_7TH_WORD,
6926                         I40E_INSET_FLEX_PAYLOAD_W7},
6927                 {RTE_ETH_INPUT_SET_FLEX_PAYLOAD_8TH_WORD,
6928                         I40E_INSET_FLEX_PAYLOAD_W8},
6929         };
6930
6931         if (!inset || !field || size > RTE_ETH_INSET_SIZE_MAX)
6932                 return ret;
6933
6934         /* Only one item allowed for default or all */
6935         if (size == 1) {
6936                 if (field[0] == RTE_ETH_INPUT_SET_DEFAULT) {
6937                         *inset = i40e_get_default_input_set(pctype);
6938                         return 0;
6939                 } else if (field[0] == RTE_ETH_INPUT_SET_NONE) {
6940                         *inset = I40E_INSET_NONE;
6941                         return 0;
6942                 }
6943         }
6944
6945         for (i = 0, *inset = 0; i < size; i++) {
6946                 for (j = 0; j < RTE_DIM(inset_convert_table); j++) {
6947                         if (field[i] == inset_convert_table[j].field) {
6948                                 *inset |= inset_convert_table[j].inset;
6949                                 break;
6950                         }
6951                 }
6952
6953                 /* It contains unsupported input set, return immediately */
6954                 if (j == RTE_DIM(inset_convert_table))
6955                         return ret;
6956         }
6957
6958         return 0;
6959 }
6960
6961 /**
6962  * Translate the input set from bit masks to register aware bit masks
6963  * and vice versa
6964  */
6965 static uint64_t
6966 i40e_translate_input_set_reg(uint64_t input)
6967 {
6968         uint64_t val = 0;
6969         uint16_t i;
6970
6971         static const struct {
6972                 uint64_t inset;
6973                 uint64_t inset_reg;
6974         } inset_map[] = {
6975                 {I40E_INSET_DMAC, I40E_REG_INSET_L2_DMAC},
6976                 {I40E_INSET_SMAC, I40E_REG_INSET_L2_SMAC},
6977                 {I40E_INSET_VLAN_OUTER, I40E_REG_INSET_L2_OUTER_VLAN},
6978                 {I40E_INSET_VLAN_INNER, I40E_REG_INSET_L2_INNER_VLAN},
6979                 {I40E_INSET_LAST_ETHER_TYPE, I40E_REG_INSET_LAST_ETHER_TYPE},
6980                 {I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4},
6981                 {I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4},
6982                 {I40E_INSET_IPV4_TOS, I40E_REG_INSET_L3_IP4_TOS},
6983                 {I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO},
6984                 {I40E_INSET_IPV6_SRC, I40E_REG_INSET_L3_SRC_IP6},
6985                 {I40E_INSET_IPV6_DST, I40E_REG_INSET_L3_DST_IP6},
6986                 {I40E_INSET_IPV6_TC, I40E_REG_INSET_L3_IP6_TC},
6987                 {I40E_INSET_IPV6_NEXT_HDR, I40E_REG_INSET_L3_IP6_NEXT_HDR},
6988                 {I40E_INSET_SRC_PORT, I40E_REG_INSET_L4_SRC_PORT},
6989                 {I40E_INSET_DST_PORT, I40E_REG_INSET_L4_DST_PORT},
6990                 {I40E_INSET_SCTP_VT, I40E_REG_INSET_L4_SCTP_VERIFICATION_TAG},
6991                 {I40E_INSET_TUNNEL_ID, I40E_REG_INSET_TUNNEL_ID},
6992                 {I40E_INSET_TUNNEL_DMAC,
6993                         I40E_REG_INSET_TUNNEL_L2_INNER_DST_MAC},
6994                 {I40E_INSET_TUNNEL_IPV4_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP4},
6995                 {I40E_INSET_TUNNEL_IPV6_DST, I40E_REG_INSET_TUNNEL_L3_DST_IP6},
6996                 {I40E_INSET_TUNNEL_SRC_PORT,
6997                         I40E_REG_INSET_TUNNEL_L4_UDP_SRC_PORT},
6998                 {I40E_INSET_TUNNEL_DST_PORT,
6999                         I40E_REG_INSET_TUNNEL_L4_UDP_DST_PORT},
7000                 {I40E_INSET_TUNNEL_ID, I40E_REG_INSET_TUNNEL_ID},
7001                 {I40E_INSET_FLEX_PAYLOAD_W1, I40E_REG_INSET_FLEX_PAYLOAD_WORD1},
7002                 {I40E_INSET_FLEX_PAYLOAD_W2, I40E_REG_INSET_FLEX_PAYLOAD_WORD2},
7003                 {I40E_INSET_FLEX_PAYLOAD_W3, I40E_REG_INSET_FLEX_PAYLOAD_WORD3},
7004                 {I40E_INSET_FLEX_PAYLOAD_W4, I40E_REG_INSET_FLEX_PAYLOAD_WORD4},
7005                 {I40E_INSET_FLEX_PAYLOAD_W5, I40E_REG_INSET_FLEX_PAYLOAD_WORD5},
7006                 {I40E_INSET_FLEX_PAYLOAD_W6, I40E_REG_INSET_FLEX_PAYLOAD_WORD6},
7007                 {I40E_INSET_FLEX_PAYLOAD_W7, I40E_REG_INSET_FLEX_PAYLOAD_WORD7},
7008                 {I40E_INSET_FLEX_PAYLOAD_W8, I40E_REG_INSET_FLEX_PAYLOAD_WORD8},
7009         };
7010
7011         if (input == 0)
7012                 return val;
7013
7014         /* Translate input set to register aware inset */
7015         for (i = 0; i < RTE_DIM(inset_map); i++) {
7016                 if (input & inset_map[i].inset)
7017                         val |= inset_map[i].inset_reg;
7018         }
7019
7020         return val;
7021 }
7022
7023 static uint8_t
7024 i40e_generate_inset_mask_reg(uint64_t inset, uint32_t *mask, uint8_t nb_elem)
7025 {
7026         uint8_t i, idx = 0;
7027
7028         static const struct {
7029                 uint64_t inset;
7030                 uint32_t mask;
7031         } inset_mask_map[] = {
7032                 {I40E_INSET_IPV4_TOS, I40E_INSET_IPV4_TOS_MASK},
7033                 {I40E_INSET_IPV4_PROTO, I40E_INSET_IPV4_PROTO_MASK},
7034                 {I40E_INSET_IPV6_TC, I40E_INSET_IPV6_TC_MASK},
7035                 {I40E_INSET_IPV6_NEXT_HDR, I40E_INSET_IPV6_NEXT_HDR_MASK},
7036         };
7037
7038         if (!inset || !mask || !nb_elem)
7039                 return 0;
7040
7041         if (!inset && nb_elem >= I40E_INSET_MASK_NUM_REG) {
7042                 for (i = 0; i < I40E_INSET_MASK_NUM_REG; i++)
7043                         mask[i] = 0;
7044                 return I40E_INSET_MASK_NUM_REG;
7045         }
7046
7047         for (i = 0, idx = 0; i < RTE_DIM(inset_mask_map); i++) {
7048                 if (idx >= nb_elem)
7049                         break;
7050                 if (inset & inset_mask_map[i].inset) {
7051                         mask[idx] = inset_mask_map[i].mask;
7052                         idx++;
7053                 }
7054         }
7055
7056         return idx;
7057 }
7058
7059 static uint64_t
7060 i40e_get_reg_inset(struct i40e_hw *hw, enum rte_filter_type filter,
7061                             enum i40e_filter_pctype pctype)
7062 {
7063         uint64_t reg = 0;
7064
7065         if (filter == RTE_ETH_FILTER_HASH) {
7066                 reg = i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, pctype));
7067                 reg <<= I40E_32_BIT_WIDTH;
7068                 reg |= i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, pctype));
7069         } else if (filter == RTE_ETH_FILTER_FDIR) {
7070                 reg = i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 1));
7071                 reg <<= I40E_32_BIT_WIDTH;
7072                 reg |= i40e_read_rx_ctl(hw, I40E_PRTQF_FD_INSET(pctype, 0));
7073         }
7074
7075         return reg;
7076 }
7077
7078 static void
7079 i40e_check_write_reg(struct i40e_hw *hw, uint32_t addr, uint32_t val)
7080 {
7081         uint32_t reg = i40e_read_rx_ctl(hw, addr);
7082
7083         PMD_DRV_LOG(DEBUG, "[0x%08x] original: 0x%08x\n", addr, reg);
7084         if (reg != val)
7085                 i40e_write_rx_ctl(hw, addr, val);
7086         PMD_DRV_LOG(DEBUG, "[0x%08x] after: 0x%08x\n", addr,
7087                     (uint32_t)i40e_read_rx_ctl(hw, addr));
7088 }
7089
7090 static int
7091 i40e_set_hash_inset_mask(struct i40e_hw *hw,
7092                          enum i40e_filter_pctype pctype,
7093                          enum rte_filter_input_set_op op,
7094                          uint32_t *mask_reg,
7095                          uint8_t num)
7096 {
7097         uint32_t reg;
7098         uint8_t i;
7099
7100         if (!mask_reg || num > RTE_ETH_INPUT_SET_SELECT)
7101                 return -EINVAL;
7102
7103         if (op == RTE_ETH_INPUT_SET_SELECT) {
7104                 for (i = 0; i < I40E_INSET_MASK_NUM_REG; i++) {
7105                         i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7106                                              0);
7107                         if (i >= num)
7108                                 continue;
7109                         i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7110                                              mask_reg[i]);
7111                 }
7112         } else if (op == RTE_ETH_INPUT_SET_ADD) {
7113                 uint8_t j, count = 0;
7114
7115                 for (i = 0; i < I40E_INSET_MASK_NUM_REG; i++) {
7116                         reg = i40e_read_rx_ctl(hw,
7117                                                I40E_GLQF_HASH_MSK(i, pctype));
7118                         if (reg & I40E_GLQF_HASH_MSK_MASK_MASK)
7119                                 count++;
7120                 }
7121                 if (count + num > I40E_INSET_MASK_NUM_REG)
7122                         return -EINVAL;
7123
7124                 for (i = count, j = 0; i < I40E_INSET_MASK_NUM_REG; i++, j++)
7125                         i40e_check_write_reg(hw, I40E_GLQF_HASH_MSK(i, pctype),
7126                                              mask_reg[j]);
7127         }
7128
7129         return 0;
7130 }
7131
7132 static int
7133 i40e_set_fd_inset_mask(struct i40e_hw *hw,
7134                        enum i40e_filter_pctype pctype,
7135                        enum rte_filter_input_set_op op,
7136                        uint32_t *mask_reg,
7137                        uint8_t num)
7138 {
7139         uint32_t reg;
7140         uint8_t i;
7141
7142         if (!mask_reg || num > RTE_ETH_INPUT_SET_SELECT)
7143                 return -EINVAL;
7144
7145         if (op == RTE_ETH_INPUT_SET_SELECT) {
7146                 for (i = 0; i < I40E_INSET_MASK_NUM_REG; i++) {
7147                         i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7148                                              0);
7149                         if (i >= num)
7150                                 continue;
7151                         i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7152                                              mask_reg[i]);
7153                 }
7154         } else if (op == RTE_ETH_INPUT_SET_ADD) {
7155                 uint8_t j, count = 0;
7156
7157                 for (i = 0; i < I40E_INSET_MASK_NUM_REG; i++) {
7158                         reg = i40e_read_rx_ctl(hw,
7159                                                I40E_GLQF_FD_MSK(i, pctype));
7160                         if (reg & I40E_GLQF_FD_MSK_MASK_MASK)
7161                                 count++;
7162                 }
7163                 if (count + num > I40E_INSET_MASK_NUM_REG)
7164                         return -EINVAL;
7165
7166                 for (i = count, j = 0; i < I40E_INSET_MASK_NUM_REG; i++, j++)
7167                         i40e_check_write_reg(hw, I40E_GLQF_FD_MSK(i, pctype),
7168                                              mask_reg[j]);
7169         }
7170
7171         return 0;
7172 }
7173
7174 int
7175 i40e_filter_inset_select(struct i40e_hw *hw,
7176                          struct rte_eth_input_set_conf *conf,
7177                          enum rte_filter_type filter)
7178 {
7179         enum i40e_filter_pctype pctype;
7180         uint64_t inset_reg = 0, input_set;
7181         uint32_t mask_reg[I40E_INSET_MASK_NUM_REG];
7182         uint8_t num;
7183         int ret;
7184
7185         if (!hw || !conf) {
7186                 PMD_DRV_LOG(ERR, "Invalid pointer");
7187                 return -EFAULT;
7188         }
7189
7190         pctype = i40e_flowtype_to_pctype(conf->flow_type);
7191         if (pctype == 0 || pctype > I40E_FILTER_PCTYPE_L2_PAYLOAD) {
7192                 PMD_DRV_LOG(ERR, "Not supported flow type (%u)",
7193                             conf->flow_type);
7194                 return -EINVAL;
7195         }
7196         if (filter != RTE_ETH_FILTER_HASH && filter != RTE_ETH_FILTER_FDIR) {
7197                 PMD_DRV_LOG(ERR, "Not supported filter type (%u)", filter);
7198                 return -EINVAL;
7199         }
7200
7201         ret = i40e_parse_input_set(&input_set, pctype, conf->field,
7202                                    conf->inset_size);
7203         if (ret) {
7204                 PMD_DRV_LOG(ERR, "Failed to parse input set");
7205                 return -EINVAL;
7206         }
7207         if (i40e_validate_input_set(pctype, filter, input_set) != 0) {
7208                 PMD_DRV_LOG(ERR, "Invalid input set");
7209                 return -EINVAL;
7210         }
7211
7212         if (conf->op == RTE_ETH_INPUT_SET_ADD) {
7213                 inset_reg |= i40e_get_reg_inset(hw, filter, pctype);
7214         } else if (conf->op != RTE_ETH_INPUT_SET_SELECT) {
7215                 PMD_DRV_LOG(ERR, "Unsupported input set operation");
7216                 return -EINVAL;
7217         }
7218         num = i40e_generate_inset_mask_reg(input_set, mask_reg,
7219                                            I40E_INSET_MASK_NUM_REG);
7220         inset_reg |= i40e_translate_input_set_reg(input_set);
7221
7222         if (filter == RTE_ETH_FILTER_HASH) {
7223                 ret = i40e_set_hash_inset_mask(hw, pctype, conf->op, mask_reg,
7224                                                num);
7225                 if (ret)
7226                         return -EINVAL;
7227
7228                 i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype),
7229                                       (uint32_t)(inset_reg & UINT32_MAX));
7230                 i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(1, pctype),
7231                                      (uint32_t)((inset_reg >>
7232                                      I40E_32_BIT_WIDTH) & UINT32_MAX));
7233         } else if (filter == RTE_ETH_FILTER_FDIR) {
7234                 ret = i40e_set_fd_inset_mask(hw, pctype, conf->op, mask_reg,
7235                                              num);
7236                 if (ret)
7237                         return -EINVAL;
7238
7239                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0),
7240                                       (uint32_t)(inset_reg & UINT32_MAX));
7241                 i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 1),
7242                                      (uint32_t)((inset_reg >>
7243                                      I40E_32_BIT_WIDTH) & UINT32_MAX));
7244         } else {
7245                 PMD_DRV_LOG(ERR, "Not supported filter type (%u)", filter);
7246                 return -EINVAL;
7247         }
7248         I40E_WRITE_FLUSH(hw);
7249
7250         return 0;
7251 }
7252
7253 static int
7254 i40e_hash_filter_get(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
7255 {
7256         int ret = 0;
7257
7258         if (!hw || !info) {
7259                 PMD_DRV_LOG(ERR, "Invalid pointer");
7260                 return -EFAULT;
7261         }
7262
7263         switch (info->info_type) {
7264         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
7265                 i40e_get_symmetric_hash_enable_per_port(hw,
7266                                         &(info->info.enable));
7267                 break;
7268         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
7269                 ret = i40e_get_hash_filter_global_config(hw,
7270                                 &(info->info.global_conf));
7271                 break;
7272         default:
7273                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
7274                                                         info->info_type);
7275                 ret = -EINVAL;
7276                 break;
7277         }
7278
7279         return ret;
7280 }
7281
7282 static int
7283 i40e_hash_filter_set(struct i40e_hw *hw, struct rte_eth_hash_filter_info *info)
7284 {
7285         int ret = 0;
7286
7287         if (!hw || !info) {
7288                 PMD_DRV_LOG(ERR, "Invalid pointer");
7289                 return -EFAULT;
7290         }
7291
7292         switch (info->info_type) {
7293         case RTE_ETH_HASH_FILTER_SYM_HASH_ENA_PER_PORT:
7294                 i40e_set_symmetric_hash_enable_per_port(hw, info->info.enable);
7295                 break;
7296         case RTE_ETH_HASH_FILTER_GLOBAL_CONFIG:
7297                 ret = i40e_set_hash_filter_global_config(hw,
7298                                 &(info->info.global_conf));
7299                 break;
7300         case RTE_ETH_HASH_FILTER_INPUT_SET_SELECT:
7301                 ret = i40e_filter_inset_select(hw,
7302                                                &(info->info.input_set_conf),
7303                                                RTE_ETH_FILTER_HASH);
7304                 break;
7305
7306         default:
7307                 PMD_DRV_LOG(ERR, "Hash filter info type (%d) not supported",
7308                                                         info->info_type);
7309                 ret = -EINVAL;
7310                 break;
7311         }
7312
7313         return ret;
7314 }
7315
7316 /* Operations for hash function */
7317 static int
7318 i40e_hash_filter_ctrl(struct rte_eth_dev *dev,
7319                       enum rte_filter_op filter_op,
7320                       void *arg)
7321 {
7322         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7323         int ret = 0;
7324
7325         switch (filter_op) {
7326         case RTE_ETH_FILTER_NOP:
7327                 break;
7328         case RTE_ETH_FILTER_GET:
7329                 ret = i40e_hash_filter_get(hw,
7330                         (struct rte_eth_hash_filter_info *)arg);
7331                 break;
7332         case RTE_ETH_FILTER_SET:
7333                 ret = i40e_hash_filter_set(hw,
7334                         (struct rte_eth_hash_filter_info *)arg);
7335                 break;
7336         default:
7337                 PMD_DRV_LOG(WARNING, "Filter operation (%d) not supported",
7338                                                                 filter_op);
7339                 ret = -ENOTSUP;
7340                 break;
7341         }
7342
7343         return ret;
7344 }
7345
7346 /*
7347  * Configure ethertype filter, which can director packet by filtering
7348  * with mac address and ether_type or only ether_type
7349  */
7350 static int
7351 i40e_ethertype_filter_set(struct i40e_pf *pf,
7352                         struct rte_eth_ethertype_filter *filter,
7353                         bool add)
7354 {
7355         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
7356         struct i40e_control_filter_stats stats;
7357         uint16_t flags = 0;
7358         int ret;
7359
7360         if (filter->queue >= pf->dev_data->nb_rx_queues) {
7361                 PMD_DRV_LOG(ERR, "Invalid queue ID");
7362                 return -EINVAL;
7363         }
7364         if (filter->ether_type == ETHER_TYPE_IPv4 ||
7365                 filter->ether_type == ETHER_TYPE_IPv6) {
7366                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
7367                         " control packet filter.", filter->ether_type);
7368                 return -EINVAL;
7369         }
7370         if (filter->ether_type == ETHER_TYPE_VLAN)
7371                 PMD_DRV_LOG(WARNING, "filter vlan ether_type in first tag is"
7372                         " not supported.");
7373
7374         if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC))
7375                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC;
7376         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP)
7377                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP;
7378         flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE;
7379
7380         memset(&stats, 0, sizeof(stats));
7381         ret = i40e_aq_add_rem_control_packet_filter(hw,
7382                         filter->mac_addr.addr_bytes,
7383                         filter->ether_type, flags,
7384                         pf->main_vsi->seid,
7385                         filter->queue, add, &stats, NULL);
7386
7387         PMD_DRV_LOG(INFO, "add/rem control packet filter, return %d,"
7388                          " mac_etype_used = %u, etype_used = %u,"
7389                          " mac_etype_free = %u, etype_free = %u\n",
7390                          ret, stats.mac_etype_used, stats.etype_used,
7391                          stats.mac_etype_free, stats.etype_free);
7392         if (ret < 0)
7393                 return -ENOSYS;
7394         return 0;
7395 }
7396
7397 /*
7398  * Handle operations for ethertype filter.
7399  */
7400 static int
7401 i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
7402                                 enum rte_filter_op filter_op,
7403                                 void *arg)
7404 {
7405         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
7406         int ret = 0;
7407
7408         if (filter_op == RTE_ETH_FILTER_NOP)
7409                 return ret;
7410
7411         if (arg == NULL) {
7412                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
7413                             filter_op);
7414                 return -EINVAL;
7415         }
7416
7417         switch (filter_op) {
7418         case RTE_ETH_FILTER_ADD:
7419                 ret = i40e_ethertype_filter_set(pf,
7420                         (struct rte_eth_ethertype_filter *)arg,
7421                         TRUE);
7422                 break;
7423         case RTE_ETH_FILTER_DELETE:
7424                 ret = i40e_ethertype_filter_set(pf,
7425                         (struct rte_eth_ethertype_filter *)arg,
7426                         FALSE);
7427                 break;
7428         default:
7429                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
7430                 ret = -ENOSYS;
7431                 break;
7432         }
7433         return ret;
7434 }
7435
7436 static int
7437 i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
7438                      enum rte_filter_type filter_type,
7439                      enum rte_filter_op filter_op,
7440                      void *arg)
7441 {
7442         int ret = 0;
7443
7444         if (dev == NULL)
7445                 return -EINVAL;
7446
7447         switch (filter_type) {
7448         case RTE_ETH_FILTER_NONE:
7449                 /* For global configuration */
7450                 ret = i40e_filter_ctrl_global_config(dev, filter_op, arg);
7451                 break;
7452         case RTE_ETH_FILTER_HASH:
7453                 ret = i40e_hash_filter_ctrl(dev, filter_op, arg);
7454                 break;
7455         case RTE_ETH_FILTER_MACVLAN:
7456                 ret = i40e_mac_filter_handle(dev, filter_op, arg);
7457                 break;
7458         case RTE_ETH_FILTER_ETHERTYPE:
7459                 ret = i40e_ethertype_filter_handle(dev, filter_op, arg);
7460                 break;
7461         case RTE_ETH_FILTER_TUNNEL:
7462                 ret = i40e_tunnel_filter_handle(dev, filter_op, arg);
7463                 break;
7464         case RTE_ETH_FILTER_FDIR:
7465                 ret = i40e_fdir_ctrl_func(dev, filter_op, arg);
7466                 break;
7467         default:
7468                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
7469                                                         filter_type);
7470                 ret = -EINVAL;
7471                 break;
7472         }
7473
7474         return ret;
7475 }
7476
7477 /*
7478  * Check and enable Extended Tag.
7479  * Enabling Extended Tag is important for 40G performance.
7480  */
7481 static void
7482 i40e_enable_extended_tag(struct rte_eth_dev *dev)
7483 {
7484         uint32_t buf = 0;
7485         int ret;
7486
7487         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
7488                                       PCI_DEV_CAP_REG);
7489         if (ret < 0) {
7490                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
7491                             PCI_DEV_CAP_REG);
7492                 return;
7493         }
7494         if (!(buf & PCI_DEV_CAP_EXT_TAG_MASK)) {
7495                 PMD_DRV_LOG(ERR, "Does not support Extended Tag");
7496                 return;
7497         }
7498
7499         buf = 0;
7500         ret = rte_eal_pci_read_config(dev->pci_dev, &buf, sizeof(buf),
7501                                       PCI_DEV_CTRL_REG);
7502         if (ret < 0) {
7503                 PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x",
7504                             PCI_DEV_CTRL_REG);
7505                 return;
7506         }
7507         if (buf & PCI_DEV_CTRL_EXT_TAG_MASK) {
7508                 PMD_DRV_LOG(DEBUG, "Extended Tag has already been enabled");
7509                 return;
7510         }
7511         buf |= PCI_DEV_CTRL_EXT_TAG_MASK;
7512         ret = rte_eal_pci_write_config(dev->pci_dev, &buf, sizeof(buf),
7513                                        PCI_DEV_CTRL_REG);
7514         if (ret < 0) {
7515                 PMD_DRV_LOG(ERR, "Failed to write PCI offset 0x%x",
7516                             PCI_DEV_CTRL_REG);
7517                 return;
7518         }
7519 }
7520
7521 /*
7522  * As some registers wouldn't be reset unless a global hardware reset,
7523  * hardware initialization is needed to put those registers into an
7524  * expected initial state.
7525  */
7526 static void
7527 i40e_hw_init(struct rte_eth_dev *dev)
7528 {
7529         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7530
7531         i40e_enable_extended_tag(dev);
7532
7533         /* clear the PF Queue Filter control register */
7534         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, 0);
7535
7536         /* Disable symmetric hash per port */
7537         i40e_set_symmetric_hash_enable_per_port(hw, 0);
7538 }
7539
7540 enum i40e_filter_pctype
7541 i40e_flowtype_to_pctype(uint16_t flow_type)
7542 {
7543         static const enum i40e_filter_pctype pctype_table[] = {
7544                 [RTE_ETH_FLOW_FRAG_IPV4] = I40E_FILTER_PCTYPE_FRAG_IPV4,
7545                 [RTE_ETH_FLOW_NONFRAG_IPV4_UDP] =
7546                         I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
7547                 [RTE_ETH_FLOW_NONFRAG_IPV4_TCP] =
7548                         I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
7549                 [RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] =
7550                         I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
7551                 [RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] =
7552                         I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
7553                 [RTE_ETH_FLOW_FRAG_IPV6] = I40E_FILTER_PCTYPE_FRAG_IPV6,
7554                 [RTE_ETH_FLOW_NONFRAG_IPV6_UDP] =
7555                         I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
7556                 [RTE_ETH_FLOW_NONFRAG_IPV6_TCP] =
7557                         I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
7558                 [RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] =
7559                         I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
7560                 [RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] =
7561                         I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
7562                 [RTE_ETH_FLOW_L2_PAYLOAD] = I40E_FILTER_PCTYPE_L2_PAYLOAD,
7563         };
7564
7565         return pctype_table[flow_type];
7566 }
7567
7568 uint16_t
7569 i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype)
7570 {
7571         static const uint16_t flowtype_table[] = {
7572                 [I40E_FILTER_PCTYPE_FRAG_IPV4] = RTE_ETH_FLOW_FRAG_IPV4,
7573                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] =
7574                         RTE_ETH_FLOW_NONFRAG_IPV4_UDP,
7575                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] =
7576                         RTE_ETH_FLOW_NONFRAG_IPV4_TCP,
7577                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] =
7578                         RTE_ETH_FLOW_NONFRAG_IPV4_SCTP,
7579                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
7580                         RTE_ETH_FLOW_NONFRAG_IPV4_OTHER,
7581                 [I40E_FILTER_PCTYPE_FRAG_IPV6] = RTE_ETH_FLOW_FRAG_IPV6,
7582                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] =
7583                         RTE_ETH_FLOW_NONFRAG_IPV6_UDP,
7584                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] =
7585                         RTE_ETH_FLOW_NONFRAG_IPV6_TCP,
7586                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] =
7587                         RTE_ETH_FLOW_NONFRAG_IPV6_SCTP,
7588                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
7589                         RTE_ETH_FLOW_NONFRAG_IPV6_OTHER,
7590                 [I40E_FILTER_PCTYPE_L2_PAYLOAD] = RTE_ETH_FLOW_L2_PAYLOAD,
7591         };
7592
7593         return flowtype_table[pctype];
7594 }
7595
7596 /*
7597  * On X710, performance number is far from the expectation on recent firmware
7598  * versions; on XL710, performance number is also far from the expectation on
7599  * recent firmware versions, if promiscuous mode is disabled, or promiscuous
7600  * mode is enabled and port MAC address is equal to the packet destination MAC
7601  * address. The fix for this issue may not be integrated in the following
7602  * firmware version. So the workaround in software driver is needed. It needs
7603  * to modify the initial values of 3 internal only registers for both X710 and
7604  * XL710. Note that the values for X710 or XL710 could be different, and the
7605  * workaround can be removed when it is fixed in firmware in the future.
7606  */
7607
7608 /* For both X710 and XL710 */
7609 #define I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE 0x10000200
7610 #define I40E_GL_SWR_PRI_JOIN_MAP_0       0x26CE00
7611
7612 #define I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE 0x011f0200
7613 #define I40E_GL_SWR_PRI_JOIN_MAP_2       0x26CE08
7614
7615 /* For X710 */
7616 #define I40E_GL_SWR_PM_UP_THR_EF_VALUE   0x03030303
7617 /* For XL710 */
7618 #define I40E_GL_SWR_PM_UP_THR_SF_VALUE   0x06060606
7619 #define I40E_GL_SWR_PM_UP_THR            0x269FBC
7620
7621 static void
7622 i40e_configure_registers(struct i40e_hw *hw)
7623 {
7624         static struct {
7625                 uint32_t addr;
7626                 uint64_t val;
7627         } reg_table[] = {
7628                 {I40E_GL_SWR_PRI_JOIN_MAP_0, I40E_GL_SWR_PRI_JOIN_MAP_0_VALUE},
7629                 {I40E_GL_SWR_PRI_JOIN_MAP_2, I40E_GL_SWR_PRI_JOIN_MAP_2_VALUE},
7630                 {I40E_GL_SWR_PM_UP_THR, 0}, /* Compute value dynamically */
7631         };
7632         uint64_t reg;
7633         uint32_t i;
7634         int ret;
7635
7636         for (i = 0; i < RTE_DIM(reg_table); i++) {
7637                 if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) {
7638                         if (i40e_is_40G_device(hw->device_id)) /* For XL710 */
7639                                 reg_table[i].val =
7640                                         I40E_GL_SWR_PM_UP_THR_SF_VALUE;
7641                         else /* For X710 */
7642                                 reg_table[i].val =
7643                                         I40E_GL_SWR_PM_UP_THR_EF_VALUE;
7644                 }
7645
7646                 ret = i40e_aq_debug_read_register(hw, reg_table[i].addr,
7647                                                         &reg, NULL);
7648                 if (ret < 0) {
7649                         PMD_DRV_LOG(ERR, "Failed to read from 0x%"PRIx32,
7650                                                         reg_table[i].addr);
7651                         break;
7652                 }
7653                 PMD_DRV_LOG(DEBUG, "Read from 0x%"PRIx32": 0x%"PRIx64,
7654                                                 reg_table[i].addr, reg);
7655                 if (reg == reg_table[i].val)
7656                         continue;
7657
7658                 ret = i40e_aq_debug_write_register(hw, reg_table[i].addr,
7659                                                 reg_table[i].val, NULL);
7660                 if (ret < 0) {
7661                         PMD_DRV_LOG(ERR, "Failed to write 0x%"PRIx64" to the "
7662                                 "address of 0x%"PRIx32, reg_table[i].val,
7663                                                         reg_table[i].addr);
7664                         break;
7665                 }
7666                 PMD_DRV_LOG(DEBUG, "Write 0x%"PRIx64" to the address of "
7667                         "0x%"PRIx32, reg_table[i].val, reg_table[i].addr);
7668         }
7669 }
7670
7671 #define I40E_VSI_TSR(_i)            (0x00050800 + ((_i) * 4))
7672 #define I40E_VSI_TSR_QINQ_CONFIG    0xc030
7673 #define I40E_VSI_L2TAGSTXVALID(_i)  (0x00042800 + ((_i) * 4))
7674 #define I40E_VSI_L2TAGSTXVALID_QINQ 0xab
7675 static int
7676 i40e_config_qinq(struct i40e_hw *hw, struct i40e_vsi *vsi)
7677 {
7678         uint32_t reg;
7679         int ret;
7680
7681         if (vsi->vsi_id >= I40E_MAX_NUM_VSIS) {
7682                 PMD_DRV_LOG(ERR, "VSI ID exceeds the maximum");
7683                 return -EINVAL;
7684         }
7685
7686         /* Configure for double VLAN RX stripping */
7687         reg = I40E_READ_REG(hw, I40E_VSI_TSR(vsi->vsi_id));
7688         if ((reg & I40E_VSI_TSR_QINQ_CONFIG) != I40E_VSI_TSR_QINQ_CONFIG) {
7689                 reg |= I40E_VSI_TSR_QINQ_CONFIG;
7690                 ret = i40e_aq_debug_write_register(hw,
7691                                                    I40E_VSI_TSR(vsi->vsi_id),
7692                                                    reg, NULL);
7693                 if (ret < 0) {
7694                         PMD_DRV_LOG(ERR, "Failed to update VSI_TSR[%d]",
7695                                     vsi->vsi_id);
7696                         return I40E_ERR_CONFIG;
7697                 }
7698         }
7699
7700         /* Configure for double VLAN TX insertion */
7701         reg = I40E_READ_REG(hw, I40E_VSI_L2TAGSTXVALID(vsi->vsi_id));
7702         if ((reg & 0xff) != I40E_VSI_L2TAGSTXVALID_QINQ) {
7703                 reg = I40E_VSI_L2TAGSTXVALID_QINQ;
7704                 ret = i40e_aq_debug_write_register(hw,
7705                                                    I40E_VSI_L2TAGSTXVALID(
7706                                                    vsi->vsi_id), reg, NULL);
7707                 if (ret < 0) {
7708                         PMD_DRV_LOG(ERR, "Failed to update "
7709                                 "VSI_L2TAGSTXVALID[%d]", vsi->vsi_id);
7710                         return I40E_ERR_CONFIG;
7711                 }
7712         }
7713
7714         return 0;
7715 }
7716
7717 /**
7718  * i40e_aq_add_mirror_rule
7719  * @hw: pointer to the hardware structure
7720  * @seid: VEB seid to add mirror rule to
7721  * @dst_id: destination vsi seid
7722  * @entries: Buffer which contains the entities to be mirrored
7723  * @count: number of entities contained in the buffer
7724  * @rule_id:the rule_id of the rule to be added
7725  *
7726  * Add a mirror rule for a given veb.
7727  *
7728  **/
7729 static enum i40e_status_code
7730 i40e_aq_add_mirror_rule(struct i40e_hw *hw,
7731                         uint16_t seid, uint16_t dst_id,
7732                         uint16_t rule_type, uint16_t *entries,
7733                         uint16_t count, uint16_t *rule_id)
7734 {
7735         struct i40e_aq_desc desc;
7736         struct i40e_aqc_add_delete_mirror_rule cmd;
7737         struct i40e_aqc_add_delete_mirror_rule_completion *resp =
7738                 (struct i40e_aqc_add_delete_mirror_rule_completion *)
7739                 &desc.params.raw;
7740         uint16_t buff_len;
7741         enum i40e_status_code status;
7742
7743         i40e_fill_default_direct_cmd_desc(&desc,
7744                                           i40e_aqc_opc_add_mirror_rule);
7745         memset(&cmd, 0, sizeof(cmd));
7746
7747         buff_len = sizeof(uint16_t) * count;
7748         desc.datalen = rte_cpu_to_le_16(buff_len);
7749         if (buff_len > 0)
7750                 desc.flags |= rte_cpu_to_le_16(
7751                         (uint16_t)(I40E_AQ_FLAG_BUF | I40E_AQ_FLAG_RD));
7752         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
7753                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
7754         cmd.num_entries = rte_cpu_to_le_16(count);
7755         cmd.seid = rte_cpu_to_le_16(seid);
7756         cmd.destination = rte_cpu_to_le_16(dst_id);
7757
7758         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
7759         status = i40e_asq_send_command(hw, &desc, entries, buff_len, NULL);
7760         PMD_DRV_LOG(INFO, "i40e_aq_add_mirror_rule, aq_status %d,"
7761                          "rule_id = %u"
7762                          " mirror_rules_used = %u, mirror_rules_free = %u,",
7763                          hw->aq.asq_last_status, resp->rule_id,
7764                          resp->mirror_rules_used, resp->mirror_rules_free);
7765         *rule_id = rte_le_to_cpu_16(resp->rule_id);
7766
7767         return status;
7768 }
7769
7770 /**
7771  * i40e_aq_del_mirror_rule
7772  * @hw: pointer to the hardware structure
7773  * @seid: VEB seid to add mirror rule to
7774  * @entries: Buffer which contains the entities to be mirrored
7775  * @count: number of entities contained in the buffer
7776  * @rule_id:the rule_id of the rule to be delete
7777  *
7778  * Delete a mirror rule for a given veb.
7779  *
7780  **/
7781 static enum i40e_status_code
7782 i40e_aq_del_mirror_rule(struct i40e_hw *hw,
7783                 uint16_t seid, uint16_t rule_type, uint16_t *entries,
7784                 uint16_t count, uint16_t rule_id)
7785 {
7786         struct i40e_aq_desc desc;
7787         struct i40e_aqc_add_delete_mirror_rule cmd;
7788         uint16_t buff_len = 0;
7789         enum i40e_status_code status;
7790         void *buff = NULL;
7791
7792         i40e_fill_default_direct_cmd_desc(&desc,
7793                                           i40e_aqc_opc_delete_mirror_rule);
7794         memset(&cmd, 0, sizeof(cmd));
7795         if (rule_type == I40E_AQC_MIRROR_RULE_TYPE_VLAN) {
7796                 desc.flags |= rte_cpu_to_le_16((uint16_t)(I40E_AQ_FLAG_BUF |
7797                                                           I40E_AQ_FLAG_RD));
7798                 cmd.num_entries = count;
7799                 buff_len = sizeof(uint16_t) * count;
7800                 desc.datalen = rte_cpu_to_le_16(buff_len);
7801                 buff = (void *)entries;
7802         } else
7803                 /* rule id is filled in destination field for deleting mirror rule */
7804                 cmd.destination = rte_cpu_to_le_16(rule_id);
7805
7806         cmd.rule_type = rte_cpu_to_le_16(rule_type <<
7807                                 I40E_AQC_MIRROR_RULE_TYPE_SHIFT);
7808         cmd.seid = rte_cpu_to_le_16(seid);
7809
7810         rte_memcpy(&desc.params.raw, &cmd, sizeof(cmd));
7811         status = i40e_asq_send_command(hw, &desc, buff, buff_len, NULL);
7812
7813         return status;
7814 }
7815
7816 /**
7817  * i40e_mirror_rule_set
7818  * @dev: pointer to the hardware structure
7819  * @mirror_conf: mirror rule info
7820  * @sw_id: mirror rule's sw_id
7821  * @on: enable/disable
7822  *
7823  * set a mirror rule.
7824  *
7825  **/
7826 static int
7827 i40e_mirror_rule_set(struct rte_eth_dev *dev,
7828                         struct rte_eth_mirror_conf *mirror_conf,
7829                         uint8_t sw_id, uint8_t on)
7830 {
7831         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
7832         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7833         struct i40e_mirror_rule *it, *mirr_rule = NULL;
7834         struct i40e_mirror_rule *parent = NULL;
7835         uint16_t seid, dst_seid, rule_id;
7836         uint16_t i, j = 0;
7837         int ret;
7838
7839         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_set: sw_id = %d.", sw_id);
7840
7841         if (pf->main_vsi->veb == NULL || pf->vfs == NULL) {
7842                 PMD_DRV_LOG(ERR, "mirror rule can not be configured"
7843                         " without veb or vfs.");
7844                 return -ENOSYS;
7845         }
7846         if (pf->nb_mirror_rule > I40E_MAX_MIRROR_RULES) {
7847                 PMD_DRV_LOG(ERR, "mirror table is full.");
7848                 return -ENOSPC;
7849         }
7850         if (mirror_conf->dst_pool > pf->vf_num) {
7851                 PMD_DRV_LOG(ERR, "invalid destination pool %u.",
7852                                  mirror_conf->dst_pool);
7853                 return -EINVAL;
7854         }
7855
7856         seid = pf->main_vsi->veb->seid;
7857
7858         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
7859                 if (sw_id <= it->index) {
7860                         mirr_rule = it;
7861                         break;
7862                 }
7863                 parent = it;
7864         }
7865         if (mirr_rule && sw_id == mirr_rule->index) {
7866                 if (on) {
7867                         PMD_DRV_LOG(ERR, "mirror rule exists.");
7868                         return -EEXIST;
7869                 } else {
7870                         ret = i40e_aq_del_mirror_rule(hw, seid,
7871                                         mirr_rule->rule_type,
7872                                         mirr_rule->entries,
7873                                         mirr_rule->num_entries, mirr_rule->id);
7874                         if (ret < 0) {
7875                                 PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
7876                                                    " ret = %d, aq_err = %d.",
7877                                                    ret, hw->aq.asq_last_status);
7878                                 return -ENOSYS;
7879                         }
7880                         TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
7881                         rte_free(mirr_rule);
7882                         pf->nb_mirror_rule--;
7883                         return 0;
7884                 }
7885         } else if (!on) {
7886                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
7887                 return -ENOENT;
7888         }
7889
7890         mirr_rule = rte_zmalloc("i40e_mirror_rule",
7891                                 sizeof(struct i40e_mirror_rule) , 0);
7892         if (!mirr_rule) {
7893                 PMD_DRV_LOG(ERR, "failed to allocate memory");
7894                 return I40E_ERR_NO_MEMORY;
7895         }
7896         switch (mirror_conf->rule_type) {
7897         case ETH_MIRROR_VLAN:
7898                 for (i = 0, j = 0; i < ETH_MIRROR_MAX_VLANS; i++) {
7899                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
7900                                 mirr_rule->entries[j] =
7901                                         mirror_conf->vlan.vlan_id[i];
7902                                 j++;
7903                         }
7904                 }
7905                 if (j == 0) {
7906                         PMD_DRV_LOG(ERR, "vlan is not specified.");
7907                         rte_free(mirr_rule);
7908                         return -EINVAL;
7909                 }
7910                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_VLAN;
7911                 break;
7912         case ETH_MIRROR_VIRTUAL_POOL_UP:
7913         case ETH_MIRROR_VIRTUAL_POOL_DOWN:
7914                 /* check if the specified pool bit is out of range */
7915                 if (mirror_conf->pool_mask > (uint64_t)(1ULL << (pf->vf_num + 1))) {
7916                         PMD_DRV_LOG(ERR, "pool mask is out of range.");
7917                         rte_free(mirr_rule);
7918                         return -EINVAL;
7919                 }
7920                 for (i = 0, j = 0; i < pf->vf_num; i++) {
7921                         if (mirror_conf->pool_mask & (1ULL << i)) {
7922                                 mirr_rule->entries[j] = pf->vfs[i].vsi->seid;
7923                                 j++;
7924                         }
7925                 }
7926                 if (mirror_conf->pool_mask & (1ULL << pf->vf_num)) {
7927                         /* add pf vsi to entries */
7928                         mirr_rule->entries[j] = pf->main_vsi_seid;
7929                         j++;
7930                 }
7931                 if (j == 0) {
7932                         PMD_DRV_LOG(ERR, "pool is not specified.");
7933                         rte_free(mirr_rule);
7934                         return -EINVAL;
7935                 }
7936                 /* egress and ingress in aq commands means from switch but not port */
7937                 mirr_rule->rule_type =
7938                         (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) ?
7939                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_EGRESS :
7940                         I40E_AQC_MIRROR_RULE_TYPE_VPORT_INGRESS;
7941                 break;
7942         case ETH_MIRROR_UPLINK_PORT:
7943                 /* egress and ingress in aq commands means from switch but not port*/
7944                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_EGRESS;
7945                 break;
7946         case ETH_MIRROR_DOWNLINK_PORT:
7947                 mirr_rule->rule_type = I40E_AQC_MIRROR_RULE_TYPE_ALL_INGRESS;
7948                 break;
7949         default:
7950                 PMD_DRV_LOG(ERR, "unsupported mirror type %d.",
7951                         mirror_conf->rule_type);
7952                 rte_free(mirr_rule);
7953                 return -EINVAL;
7954         }
7955
7956         /* If the dst_pool is equal to vf_num, consider it as PF */
7957         if (mirror_conf->dst_pool == pf->vf_num)
7958                 dst_seid = pf->main_vsi_seid;
7959         else
7960                 dst_seid = pf->vfs[mirror_conf->dst_pool].vsi->seid;
7961
7962         ret = i40e_aq_add_mirror_rule(hw, seid, dst_seid,
7963                                       mirr_rule->rule_type, mirr_rule->entries,
7964                                       j, &rule_id);
7965         if (ret < 0) {
7966                 PMD_DRV_LOG(ERR, "failed to add mirror rule:"
7967                                    " ret = %d, aq_err = %d.",
7968                                    ret, hw->aq.asq_last_status);
7969                 rte_free(mirr_rule);
7970                 return -ENOSYS;
7971         }
7972
7973         mirr_rule->index = sw_id;
7974         mirr_rule->num_entries = j;
7975         mirr_rule->id = rule_id;
7976         mirr_rule->dst_vsi_seid = dst_seid;
7977
7978         if (parent)
7979                 TAILQ_INSERT_AFTER(&pf->mirror_list, parent, mirr_rule, rules);
7980         else
7981                 TAILQ_INSERT_HEAD(&pf->mirror_list, mirr_rule, rules);
7982
7983         pf->nb_mirror_rule++;
7984         return 0;
7985 }
7986
7987 /**
7988  * i40e_mirror_rule_reset
7989  * @dev: pointer to the device
7990  * @sw_id: mirror rule's sw_id
7991  *
7992  * reset a mirror rule.
7993  *
7994  **/
7995 static int
7996 i40e_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t sw_id)
7997 {
7998         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
7999         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8000         struct i40e_mirror_rule *it, *mirr_rule = NULL;
8001         uint16_t seid;
8002         int ret;
8003
8004         PMD_DRV_LOG(DEBUG, "i40e_mirror_rule_reset: sw_id = %d.", sw_id);
8005
8006         seid = pf->main_vsi->veb->seid;
8007
8008         TAILQ_FOREACH(it, &pf->mirror_list, rules) {
8009                 if (sw_id == it->index) {
8010                         mirr_rule = it;
8011                         break;
8012                 }
8013         }
8014         if (mirr_rule) {
8015                 ret = i40e_aq_del_mirror_rule(hw, seid,
8016                                 mirr_rule->rule_type,
8017                                 mirr_rule->entries,
8018                                 mirr_rule->num_entries, mirr_rule->id);
8019                 if (ret < 0) {
8020                         PMD_DRV_LOG(ERR, "failed to remove mirror rule:"
8021                                            " status = %d, aq_err = %d.",
8022                                            ret, hw->aq.asq_last_status);
8023                         return -ENOSYS;
8024                 }
8025                 TAILQ_REMOVE(&pf->mirror_list, mirr_rule, rules);
8026                 rte_free(mirr_rule);
8027                 pf->nb_mirror_rule--;
8028         } else {
8029                 PMD_DRV_LOG(ERR, "mirror rule doesn't exist.");
8030                 return -ENOENT;
8031         }
8032         return 0;
8033 }
8034
8035 static uint64_t
8036 i40e_read_systime_cyclecounter(struct rte_eth_dev *dev)
8037 {
8038         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8039         uint64_t systim_cycles;
8040
8041         systim_cycles = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_L);
8042         systim_cycles |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TIME_H)
8043                         << 32;
8044
8045         return systim_cycles;
8046 }
8047
8048 static uint64_t
8049 i40e_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev, uint8_t index)
8050 {
8051         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8052         uint64_t rx_tstamp;
8053
8054         rx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_L(index));
8055         rx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(index))
8056                         << 32;
8057
8058         return rx_tstamp;
8059 }
8060
8061 static uint64_t
8062 i40e_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
8063 {
8064         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8065         uint64_t tx_tstamp;
8066
8067         tx_tstamp = (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_L);
8068         tx_tstamp |= (uint64_t)I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H)
8069                         << 32;
8070
8071         return tx_tstamp;
8072 }
8073
8074 static void
8075 i40e_start_timecounters(struct rte_eth_dev *dev)
8076 {
8077         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8078         struct i40e_adapter *adapter =
8079                         (struct i40e_adapter *)dev->data->dev_private;
8080         struct rte_eth_link link;
8081         uint32_t tsync_inc_l;
8082         uint32_t tsync_inc_h;
8083
8084         /* Get current link speed. */
8085         memset(&link, 0, sizeof(link));
8086         i40e_dev_link_update(dev, 1);
8087         rte_i40e_dev_atomic_read_link_status(dev, &link);
8088
8089         switch (link.link_speed) {
8090         case ETH_LINK_SPEED_40G:
8091                 tsync_inc_l = I40E_PTP_40GB_INCVAL & 0xFFFFFFFF;
8092                 tsync_inc_h = I40E_PTP_40GB_INCVAL >> 32;
8093                 break;
8094         case ETH_LINK_SPEED_10G:
8095                 tsync_inc_l = I40E_PTP_10GB_INCVAL & 0xFFFFFFFF;
8096                 tsync_inc_h = I40E_PTP_10GB_INCVAL >> 32;
8097                 break;
8098         case ETH_LINK_SPEED_1000:
8099                 tsync_inc_l = I40E_PTP_1GB_INCVAL & 0xFFFFFFFF;
8100                 tsync_inc_h = I40E_PTP_1GB_INCVAL >> 32;
8101                 break;
8102         default:
8103                 tsync_inc_l = 0x0;
8104                 tsync_inc_h = 0x0;
8105         }
8106
8107         /* Set the timesync increment value. */
8108         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, tsync_inc_l);
8109         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, tsync_inc_h);
8110
8111         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
8112         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
8113         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
8114
8115         adapter->systime_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8116         adapter->systime_tc.cc_shift = 0;
8117         adapter->systime_tc.nsec_mask = 0;
8118
8119         adapter->rx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8120         adapter->rx_tstamp_tc.cc_shift = 0;
8121         adapter->rx_tstamp_tc.nsec_mask = 0;
8122
8123         adapter->tx_tstamp_tc.cc_mask = I40E_CYCLECOUNTER_MASK;
8124         adapter->tx_tstamp_tc.cc_shift = 0;
8125         adapter->tx_tstamp_tc.nsec_mask = 0;
8126 }
8127
8128 static int
8129 i40e_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
8130 {
8131         struct i40e_adapter *adapter =
8132                         (struct i40e_adapter *)dev->data->dev_private;
8133
8134         adapter->systime_tc.nsec += delta;
8135         adapter->rx_tstamp_tc.nsec += delta;
8136         adapter->tx_tstamp_tc.nsec += delta;
8137
8138         return 0;
8139 }
8140
8141 static int
8142 i40e_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
8143 {
8144         uint64_t ns;
8145         struct i40e_adapter *adapter =
8146                         (struct i40e_adapter *)dev->data->dev_private;
8147
8148         ns = rte_timespec_to_ns(ts);
8149
8150         /* Set the timecounters to a new value. */
8151         adapter->systime_tc.nsec = ns;
8152         adapter->rx_tstamp_tc.nsec = ns;
8153         adapter->tx_tstamp_tc.nsec = ns;
8154
8155         return 0;
8156 }
8157
8158 static int
8159 i40e_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
8160 {
8161         uint64_t ns, systime_cycles;
8162         struct i40e_adapter *adapter =
8163                         (struct i40e_adapter *)dev->data->dev_private;
8164
8165         systime_cycles = i40e_read_systime_cyclecounter(dev);
8166         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
8167         *ts = rte_ns_to_timespec(ns);
8168
8169         return 0;
8170 }
8171
8172 static int
8173 i40e_timesync_enable(struct rte_eth_dev *dev)
8174 {
8175         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8176         uint32_t tsync_ctl_l;
8177         uint32_t tsync_ctl_h;
8178
8179         /* Stop the timesync system time. */
8180         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
8181         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
8182         /* Reset the timesync system time value. */
8183         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_L, 0x0);
8184         I40E_WRITE_REG(hw, I40E_PRTTSYN_TIME_H, 0x0);
8185
8186         i40e_start_timecounters(dev);
8187
8188         /* Clear timesync registers. */
8189         I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
8190         I40E_READ_REG(hw, I40E_PRTTSYN_TXTIME_H);
8191         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(0));
8192         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(1));
8193         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(2));
8194         I40E_READ_REG(hw, I40E_PRTTSYN_RXTIME_H(3));
8195
8196         /* Enable timestamping of PTP packets. */
8197         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
8198         tsync_ctl_l |= I40E_PRTTSYN_TSYNENA;
8199
8200         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
8201         tsync_ctl_h |= I40E_PRTTSYN_TSYNENA;
8202         tsync_ctl_h |= I40E_PRTTSYN_TSYNTYPE;
8203
8204         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
8205         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
8206
8207         return 0;
8208 }
8209
8210 static int
8211 i40e_timesync_disable(struct rte_eth_dev *dev)
8212 {
8213         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8214         uint32_t tsync_ctl_l;
8215         uint32_t tsync_ctl_h;
8216
8217         /* Disable timestamping of transmitted PTP packets. */
8218         tsync_ctl_l = I40E_READ_REG(hw, I40E_PRTTSYN_CTL0);
8219         tsync_ctl_l &= ~I40E_PRTTSYN_TSYNENA;
8220
8221         tsync_ctl_h = I40E_READ_REG(hw, I40E_PRTTSYN_CTL1);
8222         tsync_ctl_h &= ~I40E_PRTTSYN_TSYNENA;
8223
8224         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL0, tsync_ctl_l);
8225         I40E_WRITE_REG(hw, I40E_PRTTSYN_CTL1, tsync_ctl_h);
8226
8227         /* Reset the timesync increment value. */
8228         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_L, 0x0);
8229         I40E_WRITE_REG(hw, I40E_PRTTSYN_INC_H, 0x0);
8230
8231         return 0;
8232 }
8233
8234 static int
8235 i40e_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
8236                                 struct timespec *timestamp, uint32_t flags)
8237 {
8238         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8239         struct i40e_adapter *adapter =
8240                 (struct i40e_adapter *)dev->data->dev_private;
8241
8242         uint32_t sync_status;
8243         uint32_t index = flags & 0x03;
8244         uint64_t rx_tstamp_cycles;
8245         uint64_t ns;
8246
8247         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_1);
8248         if ((sync_status & (1 << index)) == 0)
8249                 return -EINVAL;
8250
8251         rx_tstamp_cycles = i40e_read_rx_tstamp_cyclecounter(dev, index);
8252         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
8253         *timestamp = rte_ns_to_timespec(ns);
8254
8255         return 0;
8256 }
8257
8258 static int
8259 i40e_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
8260                                 struct timespec *timestamp)
8261 {
8262         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8263         struct i40e_adapter *adapter =
8264                 (struct i40e_adapter *)dev->data->dev_private;
8265
8266         uint32_t sync_status;
8267         uint64_t tx_tstamp_cycles;
8268         uint64_t ns;
8269
8270         sync_status = I40E_READ_REG(hw, I40E_PRTTSYN_STAT_0);
8271         if ((sync_status & I40E_PRTTSYN_STAT_0_TXTIME_MASK) == 0)
8272                 return -EINVAL;
8273
8274         tx_tstamp_cycles = i40e_read_tx_tstamp_cyclecounter(dev);
8275         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
8276         *timestamp = rte_ns_to_timespec(ns);
8277
8278         return 0;
8279 }
8280
8281 /*
8282  * i40e_parse_dcb_configure - parse dcb configure from user
8283  * @dev: the device being configured
8284  * @dcb_cfg: pointer of the result of parse
8285  * @*tc_map: bit map of enabled traffic classes
8286  *
8287  * Returns 0 on success, negative value on failure
8288  */
8289 static int
8290 i40e_parse_dcb_configure(struct rte_eth_dev *dev,
8291                          struct i40e_dcbx_config *dcb_cfg,
8292                          uint8_t *tc_map)
8293 {
8294         struct rte_eth_dcb_rx_conf *dcb_rx_conf;
8295         uint8_t i, tc_bw, bw_lf;
8296
8297         memset(dcb_cfg, 0, sizeof(struct i40e_dcbx_config));
8298
8299         dcb_rx_conf = &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
8300         if (dcb_rx_conf->nb_tcs > I40E_MAX_TRAFFIC_CLASS) {
8301                 PMD_INIT_LOG(ERR, "number of tc exceeds max.");
8302                 return -EINVAL;
8303         }
8304
8305         /* assume each tc has the same bw */
8306         tc_bw = I40E_MAX_PERCENT / dcb_rx_conf->nb_tcs;
8307         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
8308                 dcb_cfg->etscfg.tcbwtable[i] = tc_bw;
8309         /* to ensure the sum of tcbw is equal to 100 */
8310         bw_lf = I40E_MAX_PERCENT % dcb_rx_conf->nb_tcs;
8311         for (i = 0; i < bw_lf; i++)
8312                 dcb_cfg->etscfg.tcbwtable[i]++;
8313
8314         /* assume each tc has the same Transmission Selection Algorithm */
8315         for (i = 0; i < dcb_rx_conf->nb_tcs; i++)
8316                 dcb_cfg->etscfg.tsatable[i] = I40E_IEEE_TSA_ETS;
8317
8318         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
8319                 dcb_cfg->etscfg.prioritytable[i] =
8320                                 dcb_rx_conf->dcb_tc[i];
8321
8322         /* FW needs one App to configure HW */
8323         dcb_cfg->numapps = I40E_DEFAULT_DCB_APP_NUM;
8324         dcb_cfg->app[0].selector = I40E_APP_SEL_ETHTYPE;
8325         dcb_cfg->app[0].priority = I40E_DEFAULT_DCB_APP_PRIO;
8326         dcb_cfg->app[0].protocolid = I40E_APP_PROTOID_FCOE;
8327
8328         if (dcb_rx_conf->nb_tcs == 0)
8329                 *tc_map = 1; /* tc0 only */
8330         else
8331                 *tc_map = RTE_LEN2MASK(dcb_rx_conf->nb_tcs, uint8_t);
8332
8333         if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
8334                 dcb_cfg->pfc.willing = 0;
8335                 dcb_cfg->pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
8336                 dcb_cfg->pfc.pfcenable = *tc_map;
8337         }
8338         return 0;
8339 }
8340
8341
8342 static enum i40e_status_code
8343 i40e_vsi_update_queue_mapping(struct i40e_vsi *vsi,
8344                               struct i40e_aqc_vsi_properties_data *info,
8345                               uint8_t enabled_tcmap)
8346 {
8347         enum i40e_status_code ret;
8348         int i, total_tc = 0;
8349         uint16_t qpnum_per_tc, bsf, qp_idx;
8350         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
8351         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
8352         uint16_t used_queues;
8353
8354         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
8355         if (ret != I40E_SUCCESS)
8356                 return ret;
8357
8358         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8359                 if (enabled_tcmap & (1 << i))
8360                         total_tc++;
8361         }
8362         if (total_tc == 0)
8363                 total_tc = 1;
8364         vsi->enabled_tc = enabled_tcmap;
8365
8366         /* different VSI has different queues assigned */
8367         if (vsi->type == I40E_VSI_MAIN)
8368                 used_queues = dev_data->nb_rx_queues -
8369                         pf->nb_cfg_vmdq_vsi * RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
8370         else if (vsi->type == I40E_VSI_VMDQ2)
8371                 used_queues = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
8372         else {
8373                 PMD_INIT_LOG(ERR, "unsupported VSI type.");
8374                 return I40E_ERR_NO_AVAILABLE_VSI;
8375         }
8376
8377         qpnum_per_tc = used_queues / total_tc;
8378         /* Number of queues per enabled TC */
8379         if (qpnum_per_tc == 0) {
8380                 PMD_INIT_LOG(ERR, " number of queues is less that tcs.");
8381                 return I40E_ERR_INVALID_QP_ID;
8382         }
8383         qpnum_per_tc = RTE_MIN(i40e_align_floor(qpnum_per_tc),
8384                                 I40E_MAX_Q_PER_TC);
8385         bsf = rte_bsf32(qpnum_per_tc);
8386
8387         /**
8388          * Configure TC and queue mapping parameters, for enabled TC,
8389          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
8390          * default queue will serve it.
8391          */
8392         qp_idx = 0;
8393         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8394                 if (vsi->enabled_tc & (1 << i)) {
8395                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
8396                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
8397                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
8398                         qp_idx += qpnum_per_tc;
8399                 } else
8400                         info->tc_mapping[i] = 0;
8401         }
8402
8403         /* Associate queue number with VSI, Keep vsi->nb_qps unchanged */
8404         if (vsi->type == I40E_VSI_SRIOV) {
8405                 info->mapping_flags |=
8406                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
8407                 for (i = 0; i < vsi->nb_qps; i++)
8408                         info->queue_mapping[i] =
8409                                 rte_cpu_to_le_16(vsi->base_queue + i);
8410         } else {
8411                 info->mapping_flags |=
8412                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
8413                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
8414         }
8415         info->valid_sections |=
8416                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
8417
8418         return I40E_SUCCESS;
8419 }
8420
8421 /*
8422  * i40e_config_switch_comp_tc - Configure VEB tc setting for given TC map
8423  * @veb: VEB to be configured
8424  * @tc_map: enabled TC bitmap
8425  *
8426  * Returns 0 on success, negative value on failure
8427  */
8428 static enum i40e_status_code
8429 i40e_config_switch_comp_tc(struct i40e_veb *veb, uint8_t tc_map)
8430 {
8431         struct i40e_aqc_configure_switching_comp_bw_config_data veb_bw;
8432         struct i40e_aqc_query_switching_comp_bw_config_resp bw_query;
8433         struct i40e_aqc_query_switching_comp_ets_config_resp ets_query;
8434         struct i40e_hw *hw = I40E_VSI_TO_HW(veb->associate_vsi);
8435         enum i40e_status_code ret = I40E_SUCCESS;
8436         int i;
8437         uint32_t bw_max;
8438
8439         /* Check if enabled_tc is same as existing or new TCs */
8440         if (veb->enabled_tc == tc_map)
8441                 return ret;
8442
8443         /* configure tc bandwidth */
8444         memset(&veb_bw, 0, sizeof(veb_bw));
8445         veb_bw.tc_valid_bits = tc_map;
8446         /* Enable ETS TCs with equal BW Share for now across all VSIs */
8447         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8448                 if (tc_map & BIT_ULL(i))
8449                         veb_bw.tc_bw_share_credits[i] = 1;
8450         }
8451         ret = i40e_aq_config_switch_comp_bw_config(hw, veb->seid,
8452                                                    &veb_bw, NULL);
8453         if (ret) {
8454                 PMD_INIT_LOG(ERR, "AQ command Config switch_comp BW allocation"
8455                                   " per TC failed = %d",
8456                                   hw->aq.asq_last_status);
8457                 return ret;
8458         }
8459
8460         memset(&ets_query, 0, sizeof(ets_query));
8461         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
8462                                                    &ets_query, NULL);
8463         if (ret != I40E_SUCCESS) {
8464                 PMD_DRV_LOG(ERR, "Failed to get switch_comp ETS"
8465                                  " configuration %u", hw->aq.asq_last_status);
8466                 return ret;
8467         }
8468         memset(&bw_query, 0, sizeof(bw_query));
8469         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
8470                                                   &bw_query, NULL);
8471         if (ret != I40E_SUCCESS) {
8472                 PMD_DRV_LOG(ERR, "Failed to get switch_comp bandwidth"
8473                                  " configuration %u", hw->aq.asq_last_status);
8474                 return ret;
8475         }
8476
8477         /* store and print out BW info */
8478         veb->bw_info.bw_limit = rte_le_to_cpu_16(ets_query.port_bw_limit);
8479         veb->bw_info.bw_max = ets_query.tc_bw_max;
8480         PMD_DRV_LOG(DEBUG, "switch_comp bw limit:%u", veb->bw_info.bw_limit);
8481         PMD_DRV_LOG(DEBUG, "switch_comp max_bw:%u", veb->bw_info.bw_max);
8482         bw_max = rte_le_to_cpu_16(bw_query.tc_bw_max[0]) |
8483                     (rte_le_to_cpu_16(bw_query.tc_bw_max[1]) <<
8484                      I40E_16_BIT_WIDTH);
8485         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8486                 veb->bw_info.bw_ets_share_credits[i] =
8487                                 bw_query.tc_bw_share_credits[i];
8488                 veb->bw_info.bw_ets_credits[i] =
8489                                 rte_le_to_cpu_16(bw_query.tc_bw_limits[i]);
8490                 /* 4 bits per TC, 4th bit is reserved */
8491                 veb->bw_info.bw_ets_max[i] =
8492                         (uint8_t)((bw_max >> (i * I40E_4_BIT_WIDTH)) &
8493                                   RTE_LEN2MASK(3, uint8_t));
8494                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:share credits %u", i,
8495                             veb->bw_info.bw_ets_share_credits[i]);
8496                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u:credits %u", i,
8497                             veb->bw_info.bw_ets_credits[i]);
8498                 PMD_DRV_LOG(DEBUG, "\tVEB TC%u: max credits: %u", i,
8499                             veb->bw_info.bw_ets_max[i]);
8500         }
8501
8502         veb->enabled_tc = tc_map;
8503
8504         return ret;
8505 }
8506
8507
8508 /*
8509  * i40e_vsi_config_tc - Configure VSI tc setting for given TC map
8510  * @vsi: VSI to be configured
8511  * @tc_map: enabled TC bitmap
8512  *
8513  * Returns 0 on success, negative value on failure
8514  */
8515 static enum i40e_status_code
8516 i40e_vsi_config_tc(struct i40e_vsi *vsi, uint8_t tc_map)
8517 {
8518         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
8519         struct i40e_vsi_context ctxt;
8520         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
8521         enum i40e_status_code ret = I40E_SUCCESS;
8522         int i;
8523
8524         /* Check if enabled_tc is same as existing or new TCs */
8525         if (vsi->enabled_tc == tc_map)
8526                 return ret;
8527
8528         /* configure tc bandwidth */
8529         memset(&bw_data, 0, sizeof(bw_data));
8530         bw_data.tc_valid_bits = tc_map;
8531         /* Enable ETS TCs with equal BW Share for now across all VSIs */
8532         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8533                 if (tc_map & BIT_ULL(i))
8534                         bw_data.tc_bw_credits[i] = 1;
8535         }
8536         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &bw_data, NULL);
8537         if (ret) {
8538                 PMD_INIT_LOG(ERR, "AQ command Config VSI BW allocation"
8539                         " per TC failed = %d",
8540                         hw->aq.asq_last_status);
8541                 goto out;
8542         }
8543         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
8544                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
8545
8546         /* Update Queue Pairs Mapping for currently enabled UPs */
8547         ctxt.seid = vsi->seid;
8548         ctxt.pf_num = hw->pf_id;
8549         ctxt.vf_num = 0;
8550         ctxt.uplink_seid = vsi->uplink_seid;
8551         ctxt.info = vsi->info;
8552         i40e_get_cap(hw);
8553         ret = i40e_vsi_update_queue_mapping(vsi, &ctxt.info, tc_map);
8554         if (ret)
8555                 goto out;
8556
8557         /* Update the VSI after updating the VSI queue-mapping information */
8558         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
8559         if (ret) {
8560                 PMD_INIT_LOG(ERR, "Failed to configure "
8561                             "TC queue mapping = %d",
8562                             hw->aq.asq_last_status);
8563                 goto out;
8564         }
8565         /* update the local VSI info with updated queue map */
8566         (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
8567                                         sizeof(vsi->info.tc_mapping));
8568         (void)rte_memcpy(&vsi->info.queue_mapping,
8569                         &ctxt.info.queue_mapping,
8570                 sizeof(vsi->info.queue_mapping));
8571         vsi->info.mapping_flags = ctxt.info.mapping_flags;
8572         vsi->info.valid_sections = 0;
8573
8574         /* query and update current VSI BW information */
8575         ret = i40e_vsi_get_bw_config(vsi);
8576         if (ret) {
8577                 PMD_INIT_LOG(ERR,
8578                          "Failed updating vsi bw info, err %s aq_err %s",
8579                          i40e_stat_str(hw, ret),
8580                          i40e_aq_str(hw, hw->aq.asq_last_status));
8581                 goto out;
8582         }
8583
8584         vsi->enabled_tc = tc_map;
8585
8586 out:
8587         return ret;
8588 }
8589
8590 /*
8591  * i40e_dcb_hw_configure - program the dcb setting to hw
8592  * @pf: pf the configuration is taken on
8593  * @new_cfg: new configuration
8594  * @tc_map: enabled TC bitmap
8595  *
8596  * Returns 0 on success, negative value on failure
8597  */
8598 static enum i40e_status_code
8599 i40e_dcb_hw_configure(struct i40e_pf *pf,
8600                       struct i40e_dcbx_config *new_cfg,
8601                       uint8_t tc_map)
8602 {
8603         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
8604         struct i40e_dcbx_config *old_cfg = &hw->local_dcbx_config;
8605         struct i40e_vsi *main_vsi = pf->main_vsi;
8606         struct i40e_vsi_list *vsi_list;
8607         enum i40e_status_code ret;
8608         int i;
8609         uint32_t val;
8610
8611         /* Use the FW API if FW > v4.4*/
8612         if (!(((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver >= 4)) ||
8613               (hw->aq.fw_maj_ver >= 5))) {
8614                 PMD_INIT_LOG(ERR, "FW < v4.4, can not use FW LLDP API"
8615                                   " to configure DCB");
8616                 return I40E_ERR_FIRMWARE_API_VERSION;
8617         }
8618
8619         /* Check if need reconfiguration */
8620         if (!memcmp(new_cfg, old_cfg, sizeof(struct i40e_dcbx_config))) {
8621                 PMD_INIT_LOG(ERR, "No Change in DCB Config required.");
8622                 return I40E_SUCCESS;
8623         }
8624
8625         /* Copy the new config to the current config */
8626         *old_cfg = *new_cfg;
8627         old_cfg->etsrec = old_cfg->etscfg;
8628         ret = i40e_set_dcb_config(hw);
8629         if (ret) {
8630                 PMD_INIT_LOG(ERR,
8631                          "Set DCB Config failed, err %s aq_err %s\n",
8632                          i40e_stat_str(hw, ret),
8633                          i40e_aq_str(hw, hw->aq.asq_last_status));
8634                 return ret;
8635         }
8636         /* set receive Arbiter to RR mode and ETS scheme by default */
8637         for (i = 0; i <= I40E_PRTDCB_RETSTCC_MAX_INDEX; i++) {
8638                 val = I40E_READ_REG(hw, I40E_PRTDCB_RETSTCC(i));
8639                 val &= ~(I40E_PRTDCB_RETSTCC_BWSHARE_MASK     |
8640                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK |
8641                          I40E_PRTDCB_RETSTCC_ETSTC_SHIFT);
8642                 val |= ((uint32_t)old_cfg->etscfg.tcbwtable[i] <<
8643                         I40E_PRTDCB_RETSTCC_BWSHARE_SHIFT) &
8644                          I40E_PRTDCB_RETSTCC_BWSHARE_MASK;
8645                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_UPINTC_MODE_SHIFT) &
8646                          I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK;
8647                 val |= ((uint32_t)1 << I40E_PRTDCB_RETSTCC_ETSTC_SHIFT) &
8648                          I40E_PRTDCB_RETSTCC_ETSTC_MASK;
8649                 I40E_WRITE_REG(hw, I40E_PRTDCB_RETSTCC(i), val);
8650         }
8651         /* get local mib to check whether it is configured correctly */
8652         /* IEEE mode */
8653         hw->local_dcbx_config.dcbx_mode = I40E_DCBX_MODE_IEEE;
8654         /* Get Local DCB Config */
8655         i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_LOCAL, 0,
8656                                      &hw->local_dcbx_config);
8657
8658         /* if Veb is created, need to update TC of it at first */
8659         if (main_vsi->veb) {
8660                 ret = i40e_config_switch_comp_tc(main_vsi->veb, tc_map);
8661                 if (ret)
8662                         PMD_INIT_LOG(WARNING,
8663                                  "Failed configuring TC for VEB seid=%d\n",
8664                                  main_vsi->veb->seid);
8665         }
8666         /* Update each VSI */
8667         i40e_vsi_config_tc(main_vsi, tc_map);
8668         if (main_vsi->veb) {
8669                 TAILQ_FOREACH(vsi_list, &main_vsi->veb->head, list) {
8670                         /* Beside main VSI and VMDQ VSIs, only enable default
8671                          * TC for other VSIs
8672                          */
8673                         if (vsi_list->vsi->type == I40E_VSI_VMDQ2)
8674                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
8675                                                          tc_map);
8676                         else
8677                                 ret = i40e_vsi_config_tc(vsi_list->vsi,
8678                                                          I40E_DEFAULT_TCMAP);
8679                         if (ret)
8680                                 PMD_INIT_LOG(WARNING,
8681                                          "Failed configuring TC for VSI seid=%d\n",
8682                                          vsi_list->vsi->seid);
8683                         /* continue */
8684                 }
8685         }
8686         return I40E_SUCCESS;
8687 }
8688
8689 /*
8690  * i40e_dcb_init_configure - initial dcb config
8691  * @dev: device being configured
8692  * @sw_dcb: indicate whether dcb is sw configured or hw offload
8693  *
8694  * Returns 0 on success, negative value on failure
8695  */
8696 static int
8697 i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb)
8698 {
8699         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8700         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8701         int ret = 0;
8702
8703         if ((pf->flags & I40E_FLAG_DCB) == 0) {
8704                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
8705                 return -ENOTSUP;
8706         }
8707
8708         /* DCB initialization:
8709          * Update DCB configuration from the Firmware and configure
8710          * LLDP MIB change event.
8711          */
8712         if (sw_dcb == TRUE) {
8713                 ret = i40e_aq_stop_lldp(hw, TRUE, NULL);
8714                 if (ret != I40E_SUCCESS)
8715                         PMD_INIT_LOG(DEBUG, "Failed to stop lldp");
8716
8717                 ret = i40e_init_dcb(hw);
8718                 /* if sw_dcb, lldp agent is stopped, the return from
8719                  * i40e_init_dcb we expect is failure with I40E_AQ_RC_EPERM
8720                  * adminq status.
8721                  */
8722                 if (ret != I40E_SUCCESS &&
8723                     hw->aq.asq_last_status == I40E_AQ_RC_EPERM) {
8724                         memset(&hw->local_dcbx_config, 0,
8725                                 sizeof(struct i40e_dcbx_config));
8726                         /* set dcb default configuration */
8727                         hw->local_dcbx_config.etscfg.willing = 0;
8728                         hw->local_dcbx_config.etscfg.maxtcs = 0;
8729                         hw->local_dcbx_config.etscfg.tcbwtable[0] = 100;
8730                         hw->local_dcbx_config.etscfg.tsatable[0] =
8731                                                 I40E_IEEE_TSA_ETS;
8732                         hw->local_dcbx_config.etsrec =
8733                                 hw->local_dcbx_config.etscfg;
8734                         hw->local_dcbx_config.pfc.willing = 0;
8735                         hw->local_dcbx_config.pfc.pfccap =
8736                                                 I40E_MAX_TRAFFIC_CLASS;
8737                         /* FW needs one App to configure HW */
8738                         hw->local_dcbx_config.numapps = 1;
8739                         hw->local_dcbx_config.app[0].selector =
8740                                                 I40E_APP_SEL_ETHTYPE;
8741                         hw->local_dcbx_config.app[0].priority = 3;
8742                         hw->local_dcbx_config.app[0].protocolid =
8743                                                 I40E_APP_PROTOID_FCOE;
8744                         ret = i40e_set_dcb_config(hw);
8745                         if (ret) {
8746                                 PMD_INIT_LOG(ERR, "default dcb config fails."
8747                                         " err = %d, aq_err = %d.", ret,
8748                                           hw->aq.asq_last_status);
8749                                 return -ENOSYS;
8750                         }
8751                 } else {
8752                         PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d,"
8753                                           " aq_err = %d.", ret,
8754                                           hw->aq.asq_last_status);
8755                         return -ENOTSUP;
8756                 }
8757         } else {
8758                 ret = i40e_aq_start_lldp(hw, NULL);
8759                 if (ret != I40E_SUCCESS)
8760                         PMD_INIT_LOG(DEBUG, "Failed to start lldp");
8761
8762                 ret = i40e_init_dcb(hw);
8763                 if (!ret) {
8764                         if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) {
8765                                 PMD_INIT_LOG(ERR, "HW doesn't support"
8766                                                   " DCBX offload.");
8767                                 return -ENOTSUP;
8768                         }
8769                 } else {
8770                         PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d,"
8771                                           " aq_err = %d.", ret,
8772                                           hw->aq.asq_last_status);
8773                         return -ENOTSUP;
8774                 }
8775         }
8776         return 0;
8777 }
8778
8779 /*
8780  * i40e_dcb_setup - setup dcb related config
8781  * @dev: device being configured
8782  *
8783  * Returns 0 on success, negative value on failure
8784  */
8785 static int
8786 i40e_dcb_setup(struct rte_eth_dev *dev)
8787 {
8788         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8789         struct i40e_dcbx_config dcb_cfg;
8790         uint8_t tc_map = 0;
8791         int ret = 0;
8792
8793         if ((pf->flags & I40E_FLAG_DCB) == 0) {
8794                 PMD_INIT_LOG(ERR, "HW doesn't support DCB");
8795                 return -ENOTSUP;
8796         }
8797
8798         if (pf->vf_num != 0)
8799                 PMD_INIT_LOG(DEBUG, " DCB only works on pf and vmdq vsis.");
8800
8801         ret = i40e_parse_dcb_configure(dev, &dcb_cfg, &tc_map);
8802         if (ret) {
8803                 PMD_INIT_LOG(ERR, "invalid dcb config");
8804                 return -EINVAL;
8805         }
8806         ret = i40e_dcb_hw_configure(pf, &dcb_cfg, tc_map);
8807         if (ret) {
8808                 PMD_INIT_LOG(ERR, "dcb sw configure fails");
8809                 return -ENOSYS;
8810         }
8811
8812         return 0;
8813 }
8814
8815 static int
8816 i40e_dev_get_dcb_info(struct rte_eth_dev *dev,
8817                       struct rte_eth_dcb_info *dcb_info)
8818 {
8819         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
8820         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8821         struct i40e_vsi *vsi = pf->main_vsi;
8822         struct i40e_dcbx_config *dcb_cfg = &hw->local_dcbx_config;
8823         uint16_t bsf, tc_mapping;
8824         int i, j;
8825
8826         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
8827                 dcb_info->nb_tcs = rte_bsf32(vsi->enabled_tc + 1);
8828         else
8829                 dcb_info->nb_tcs = 1;
8830         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
8831                 dcb_info->prio_tc[i] = dcb_cfg->etscfg.prioritytable[i];
8832         for (i = 0; i < dcb_info->nb_tcs; i++)
8833                 dcb_info->tc_bws[i] = dcb_cfg->etscfg.tcbwtable[i];
8834
8835         j = 0;
8836         do {
8837                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
8838                         if (!(vsi->enabled_tc & (1 << i)))
8839                                 continue;
8840                         tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
8841                         /* only main vsi support multi TCs */
8842                         dcb_info->tc_queue.tc_rxq[j][i].base =
8843                                 (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
8844                                 I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
8845                         dcb_info->tc_queue.tc_txq[j][i].base =
8846                                 dcb_info->tc_queue.tc_rxq[j][i].base;
8847                         bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
8848                                 I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
8849                         dcb_info->tc_queue.tc_rxq[j][i].nb_queue = 1 << bsf;
8850                         dcb_info->tc_queue.tc_txq[j][i].nb_queue =
8851                                 dcb_info->tc_queue.tc_rxq[j][i].nb_queue;
8852                 }
8853                 vsi = pf->vmdq[j].vsi;
8854                 j++;
8855         } while (j < RTE_MIN(pf->nb_cfg_vmdq_vsi, ETH_MAX_VMDQ_POOL));
8856         return 0;
8857 }
8858
8859 static int
8860 i40e_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
8861 {
8862         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
8863         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8864         uint16_t interval =
8865                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
8866         uint16_t msix_intr;
8867
8868         msix_intr = intr_handle->intr_vec[queue_id];
8869         if (msix_intr == I40E_MISC_VEC_ID)
8870                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
8871                                I40E_PFINT_DYN_CTLN_INTENA_MASK |
8872                                I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
8873                                (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
8874                                (interval <<
8875                                 I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
8876         else
8877                 I40E_WRITE_REG(hw,
8878                                I40E_PFINT_DYN_CTLN(msix_intr -
8879                                                    I40E_RX_VEC_START),
8880                                I40E_PFINT_DYN_CTLN_INTENA_MASK |
8881                                I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
8882                                (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
8883                                (interval <<
8884                                 I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
8885
8886         I40E_WRITE_FLUSH(hw);
8887         rte_intr_enable(&dev->pci_dev->intr_handle);
8888
8889         return 0;
8890 }
8891
8892 static int
8893 i40e_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
8894 {
8895         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
8896         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8897         uint16_t msix_intr;
8898
8899         msix_intr = intr_handle->intr_vec[queue_id];
8900         if (msix_intr == I40E_MISC_VEC_ID)
8901                 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
8902         else
8903                 I40E_WRITE_REG(hw,
8904                                I40E_PFINT_DYN_CTLN(msix_intr -
8905                                                    I40E_RX_VEC_START),
8906                                0);
8907         I40E_WRITE_FLUSH(hw);
8908
8909         return 0;
8910 }
8911
8912 static int i40e_get_reg_length(__rte_unused struct rte_eth_dev *dev)
8913 {
8914         /* Highest base addr + 32-bit word */
8915         return I40E_GLGEN_STAT_CLEAR + 4;
8916 }
8917
8918 static int i40e_get_regs(struct rte_eth_dev *dev,
8919                          struct rte_dev_reg_info *regs)
8920 {
8921         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8922         uint32_t *ptr_data = regs->data;
8923         uint32_t reg_idx, arr_idx, arr_idx2, reg_offset;
8924         const struct i40e_reg_info *reg_info;
8925
8926         /* The first few registers have to be read using AQ operations */
8927         reg_idx = 0;
8928         while (i40e_regs_adminq[reg_idx].name) {
8929                 reg_info = &i40e_regs_adminq[reg_idx++];
8930                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
8931                         for (arr_idx2 = 0;
8932                                         arr_idx2 <= reg_info->count2;
8933                                         arr_idx2++) {
8934                                 reg_offset = arr_idx * reg_info->stride1 +
8935                                         arr_idx2 * reg_info->stride2;
8936                                 ptr_data[reg_offset >> 2] =
8937                                         i40e_read_rx_ctl(hw, reg_offset);
8938                         }
8939         }
8940
8941         /* The remaining registers can be read using primitives */
8942         reg_idx = 0;
8943         while (i40e_regs_others[reg_idx].name) {
8944                 reg_info = &i40e_regs_others[reg_idx++];
8945                 for (arr_idx = 0; arr_idx <= reg_info->count1; arr_idx++)
8946                         for (arr_idx2 = 0;
8947                                         arr_idx2 <= reg_info->count2;
8948                                         arr_idx2++) {
8949                                 reg_offset = arr_idx * reg_info->stride1 +
8950                                         arr_idx2 * reg_info->stride2;
8951                                 ptr_data[reg_offset >> 2] =
8952                                         I40E_READ_REG(hw, reg_offset);
8953                         }
8954         }
8955
8956         return 0;
8957 }
8958
8959 static int i40e_get_eeprom_length(struct rte_eth_dev *dev)
8960 {
8961         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8962
8963         /* Convert word count to byte count */
8964         return hw->nvm.sr_size << 1;
8965 }
8966
8967 static int i40e_get_eeprom(struct rte_eth_dev *dev,
8968                            struct rte_dev_eeprom_info *eeprom)
8969 {
8970         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
8971         uint16_t *data = eeprom->data;
8972         uint16_t offset, length, cnt_words;
8973         int ret_code;
8974
8975         offset = eeprom->offset >> 1;
8976         length = eeprom->length >> 1;
8977         cnt_words = length;
8978
8979         if (offset > hw->nvm.sr_size ||
8980                 offset + length > hw->nvm.sr_size) {
8981                 PMD_DRV_LOG(ERR, "Requested EEPROM bytes out of range.");
8982                 return -EINVAL;
8983         }
8984
8985         eeprom->magic = hw->vendor_id | (hw->device_id << 16);
8986
8987         ret_code = i40e_read_nvm_buffer(hw, offset, &cnt_words, data);
8988         if (ret_code != I40E_SUCCESS || cnt_words != length) {
8989                 PMD_DRV_LOG(ERR, "EEPROM read failed.");
8990                 return -EIO;
8991         }
8992
8993         return 0;
8994 }
8995
8996 static void i40e_set_default_mac_addr(struct rte_eth_dev *dev,
8997                                       struct ether_addr *mac_addr)
8998 {
8999         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
9000
9001         if (!is_valid_assigned_ether_addr(mac_addr)) {
9002                 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
9003                 return;
9004         }
9005
9006         /* Flags: 0x3 updates port address */
9007         i40e_aq_mac_address_write(hw, 0x3, mac_addr->addr_bytes, NULL);
9008 }