drivers/net/i40e/i40e_ethdev_vf.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_eal.h>
  55 #include <rte_alarm.h>
  56 #include <rte_ether.h>
  57 #include <rte_ethdev.h>
  58 #include <rte_atomic.h>
  59 #include <rte_malloc.h>
  60 #include <rte_dev.h>
  61
  62 #include "i40e_logs.h"
  63 #include "base/i40e_prototype.h"
  64 #include "base/i40e_adminq_cmd.h"
  65 #include "base/i40e_type.h"
  66
  67 #include "i40e_rxtx.h"
  68 #include "i40e_ethdev.h"
  69 #include "i40e_pf.h"
  70 #define I40EVF_VSI_DEFAULT_MSIX_INTR     1
  71 #define I40EVF_VSI_DEFAULT_MSIX_INTR_LNX 0
  72
  73 /* busy wait delay in msec */
  74 #define I40EVF_BUSY_WAIT_DELAY 10
  75 #define I40EVF_BUSY_WAIT_COUNT 50
  76 #define MAX_RESET_WAIT_CNT     20
  77
  78 struct i40evf_arq_msg_info {
  79         enum i40e_virtchnl_ops ops;
  80         enum i40e_status_code result;
  81         uint16_t buf_len;
  82         uint16_t msg_len;
  83         uint8_t *msg;
  84 };
  85
  86 struct vf_cmd_info {
  87         enum i40e_virtchnl_ops ops;
  88         uint8_t *in_args;
  89         uint32_t in_args_size;
  90         uint8_t *out_buffer;
  91         /* Input & output type. pass in buffer size and pass out
  92          * actual return result
  93          */
  94         uint32_t out_size;
  95 };
  96
  97 enum i40evf_aq_result {
  98         I40EVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
  99         I40EVF_MSG_NON,      /* Read nothing from admin queue */
 100         I40EVF_MSG_SYS,      /* Read system msg from admin queue */
 101         I40EVF_MSG_CMD,      /* Read async command result */
 102 };
 103
 104 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 105 static int i40evf_dev_start(struct rte_eth_dev *dev);
 106 static void i40evf_dev_stop(struct rte_eth_dev *dev);
 107 static void i40evf_dev_info_get(struct rte_eth_dev *dev,
 108                                 struct rte_eth_dev_info *dev_info);
 109 static int i40evf_dev_link_update(struct rte_eth_dev *dev,
 110                                   __rte_unused int wait_to_complete);
 111 static void i40evf_dev_stats_get(struct rte_eth_dev *dev,
 112                                 struct rte_eth_stats *stats);
 113 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
 114                                  struct rte_eth_xstats *xstats, unsigned n);
 115 static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev);
 116 static int i40evf_vlan_filter_set(struct rte_eth_dev *dev,
 117                                   uint16_t vlan_id, int on);
 118 static void i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static int i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid,
 120                                 int on);
 121 static void i40evf_dev_close(struct rte_eth_dev *dev);
 122 static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev);
 123 static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev);
 124 static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev);
 125 static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev);
 126 static int i40evf_get_link_status(struct rte_eth_dev *dev,
 127                                   struct rte_eth_link *link);
 128 static int i40evf_init_vlan(struct rte_eth_dev *dev);
 129 static int i40evf_dev_rx_queue_start(struct rte_eth_dev *dev,
 130                                      uint16_t rx_queue_id);
 131 static int i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev,
 132                                     uint16_t rx_queue_id);
 133 static int i40evf_dev_tx_queue_start(struct rte_eth_dev *dev,
 134                                      uint16_t tx_queue_id);
 135 static int i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev,
 136                                     uint16_t tx_queue_id);
 137 static void i40evf_add_mac_addr(struct rte_eth_dev *dev,
 138                                 struct ether_addr *addr,
 139                                 uint32_t index,
 140                                 uint32_t pool);
 141 static void i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 142 static int i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
 143                         struct rte_eth_rss_reta_entry64 *reta_conf,
 144                         uint16_t reta_size);
 145 static int i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
 146                         struct rte_eth_rss_reta_entry64 *reta_conf,
 147                         uint16_t reta_size);
 148 static int i40evf_config_rss(struct i40e_vf *vf);
 149 static int i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
 150                                       struct rte_eth_rss_conf *rss_conf);
 151 static int i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 152                                         struct rte_eth_rss_conf *rss_conf);
 153 static int
 154 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 155 static int
 156 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 157 static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
 158                                    uint8_t *msg,
 159                                    uint16_t msglen);
 160
 161 /* Default hash key buffer for RSS */
 162 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
 163
 164 struct rte_i40evf_xstats_name_off {
 165         char name[RTE_ETH_XSTATS_NAME_SIZE];
 166         unsigned offset;
 167 };
 168
 169 static const struct rte_i40evf_xstats_name_off rte_i40evf_stats_strings[] = {
 170         {"rx_bytes", offsetof(struct i40e_eth_stats, rx_bytes)},
 171         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 172         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 173         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 174         {"rx_dropped_packets", offsetof(struct i40e_eth_stats, rx_discards)},
 175         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 176                 rx_unknown_protocol)},
 177         {"tx_bytes", offsetof(struct i40e_eth_stats, tx_bytes)},
 178         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 179         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 180         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 181         {"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 182         {"tx_error_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 183 };
 184
 185 #define I40EVF_NB_XSTATS (sizeof(rte_i40evf_stats_strings) / \
 186                 sizeof(rte_i40evf_stats_strings[0]))
 187
 188 static const struct eth_dev_ops i40evf_eth_dev_ops = {
 189         .dev_configure        = i40evf_dev_configure,
 190         .dev_start            = i40evf_dev_start,
 191         .dev_stop             = i40evf_dev_stop,
 192         .promiscuous_enable   = i40evf_dev_promiscuous_enable,
 193         .promiscuous_disable  = i40evf_dev_promiscuous_disable,
 194         .allmulticast_enable  = i40evf_dev_allmulticast_enable,
 195         .allmulticast_disable = i40evf_dev_allmulticast_disable,
 196         .link_update          = i40evf_dev_link_update,
 197         .stats_get            = i40evf_dev_stats_get,
 198         .xstats_get           = i40evf_dev_xstats_get,
 199         .xstats_reset         = i40evf_dev_xstats_reset,
 200         .dev_close            = i40evf_dev_close,
 201         .dev_infos_get        = i40evf_dev_info_get,
 202         .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get,
 203         .vlan_filter_set      = i40evf_vlan_filter_set,
 204         .vlan_offload_set     = i40evf_vlan_offload_set,
 205         .vlan_pvid_set        = i40evf_vlan_pvid_set,
 206         .rx_queue_start       = i40evf_dev_rx_queue_start,
 207         .rx_queue_stop        = i40evf_dev_rx_queue_stop,
 208         .tx_queue_start       = i40evf_dev_tx_queue_start,
 209         .tx_queue_stop        = i40evf_dev_tx_queue_stop,
 210         .rx_queue_setup       = i40e_dev_rx_queue_setup,
 211         .rx_queue_release     = i40e_dev_rx_queue_release,
 212         .rx_queue_intr_enable = i40evf_dev_rx_queue_intr_enable,
 213         .rx_queue_intr_disable = i40evf_dev_rx_queue_intr_disable,
 214         .rx_descriptor_done   = i40e_dev_rx_descriptor_done,
 215         .tx_queue_setup       = i40e_dev_tx_queue_setup,
 216         .tx_queue_release     = i40e_dev_tx_queue_release,
 217         .mac_addr_add         = i40evf_add_mac_addr,
 218         .mac_addr_remove      = i40evf_del_mac_addr,
 219         .reta_update          = i40evf_dev_rss_reta_update,
 220         .reta_query           = i40evf_dev_rss_reta_query,
 221         .rss_hash_update      = i40evf_dev_rss_hash_update,
 222         .rss_hash_conf_get    = i40evf_dev_rss_hash_conf_get,
 223 };
 224
 225 /*
 226  * Read data in admin queue to get msg from pf driver
 227  */
 228 static enum i40evf_aq_result
 229 i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 230 {
 231         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 232         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 233         struct i40e_arq_event_info event;
 234         enum i40e_virtchnl_ops opcode;
 235         enum i40e_status_code retval;
 236         int ret;
 237         enum i40evf_aq_result result = I40EVF_MSG_NON;
 238
 239         event.buf_len = data->buf_len;
 240         event.msg_buf = data->msg;
 241         ret = i40e_clean_arq_element(hw, &event, NULL);
 242         /* Can't read any msg from adminQ */
 243         if (ret) {
 244                 if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
 245                         result = I40EVF_MSG_ERR;
 246                 return result;
 247         }
 248
 249         opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
 250         retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 251         /* pf sys event */
 252         if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 253                 struct i40e_virtchnl_pf_event *vpe =
 254                         (struct i40e_virtchnl_pf_event *)event.msg_buf;
 255
 256                 result = I40EVF_MSG_SYS;
 257                 switch (vpe->event) {
 258                 case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 259                         vf->link_up =
 260                                 vpe->event_data.link_event.link_status;
 261                         vf->link_speed =
 262                                 vpe->event_data.link_event.link_speed;
 263                         vf->pend_msg |= PFMSG_LINK_CHANGE;
 264                         PMD_DRV_LOG(INFO, "Link status update:%s",
 265                                     vf->link_up ? "up" : "down");
 266                         break;
 267                 case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
 268                         vf->vf_reset = true;
 269                         vf->pend_msg |= PFMSG_RESET_IMPENDING;
 270                         PMD_DRV_LOG(INFO, "vf is reseting");
 271                         break;
 272                 case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
 273                         vf->dev_closed = true;
 274                         vf->pend_msg |= PFMSG_DRIVER_CLOSE;
 275                         PMD_DRV_LOG(INFO, "PF driver closed");
 276                         break;
 277                 default:
 278                         PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
 279                                     __func__, vpe->event);
 280                 }
 281         } else {
 282                 /* async reply msg on command issued by vf previously */
 283                 result = I40EVF_MSG_CMD;
 284                 /* Actual data length read from PF */
 285                 data->msg_len = event.msg_len;
 286         }
 287
 288         data->result = retval;
 289         data->ops = opcode;
 290
 291         return result;
 292 }
 293
 294 /**
 295  * clear current command. Only call in case execute
 296  * _atomic_set_cmd successfully.
 297  */
 298 static inline void
 299 _clear_cmd(struct i40e_vf *vf)
 300 {
 301         rte_wmb();
 302         vf->pend_cmd = I40E_VIRTCHNL_OP_UNKNOWN;
 303 }
 304
 305 /*
 306  * Check there is pending cmd in execution. If none, set new command.
 307  */
 308 static inline int
 309 _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 310 {
 311         int ret = rte_atomic32_cmpset(&vf->pend_cmd,
 312                         I40E_VIRTCHNL_OP_UNKNOWN, ops);
 313
 314         if (!ret)
 315                 PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd);
 316
 317         return !ret;
 318 }
 319
 320 #define MAX_TRY_TIMES 200
 321 #define ASQ_DELAY_MS  10
 322
 323 static int
 324 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 325 {
 326         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 327         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 328         struct i40evf_arq_msg_info info;
 329         enum i40evf_aq_result ret;
 330         int err = -1;
 331         int i = 0;
 332
 333         if (_atomic_set_cmd(vf, args->ops))
 334                 return -1;
 335
 336         info.msg = args->out_buffer;
 337         info.buf_len = args->out_size;
 338         info.ops = I40E_VIRTCHNL_OP_UNKNOWN;
 339         info.result = I40E_SUCCESS;
 340
 341         err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS,
 342                      args->in_args, args->in_args_size, NULL);
 343         if (err) {
 344                 PMD_DRV_LOG(ERR, "fail to send cmd %d", args->ops);
 345                 _clear_cmd(vf);
 346                 return err;
 347         }
 348
 349         switch (args->ops) {
 350         case I40E_VIRTCHNL_OP_RESET_VF:
 351                 /*no need to process in this function */
 352                 break;
 353         case I40E_VIRTCHNL_OP_VERSION:
 354         case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
 355                 /* for init adminq commands, need to poll the response */
 356                 do {
 357                         ret = i40evf_read_pfmsg(dev, &info);
 358                         if (ret == I40EVF_MSG_CMD) {
 359                                 err = 0;
 360                                 break;
 361                         } else if (ret == I40EVF_MSG_ERR) {
 362                                 err = -1;
 363                                 break;
 364                         }
 365                         rte_delay_ms(ASQ_DELAY_MS);
 366                         /* If don't read msg or read sys event, continue */
 367                 } while (i++ < MAX_TRY_TIMES);
 368                 _clear_cmd(vf);
 369                 break;
 370
 371         default:
 372                 /* for other adminq in running time, waiting the cmd done flag */
 373                 do {
 374                         if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
 375                                 err = 0;
 376                                 break;
 377                         }
 378                         rte_delay_ms(ASQ_DELAY_MS);
 379                         /* If don't read msg or read sys event, continue */
 380                 } while (i++ < MAX_TRY_TIMES);
 381                 break;
 382         }
 383
 384         return err | vf->cmd_retval;
 385 }
 386
 387 /*
 388  * Check API version with sync wait until version read or fail from admin queue
 389  */
 390 static int
 391 i40evf_check_api_version(struct rte_eth_dev *dev)
 392 {
 393         struct i40e_virtchnl_version_info version, *pver;
 394         int err;
 395         struct vf_cmd_info args;
 396         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 397
 398         version.major = I40E_VIRTCHNL_VERSION_MAJOR;
 399         version.minor = I40E_VIRTCHNL_VERSION_MINOR;
 400
 401         args.ops = I40E_VIRTCHNL_OP_VERSION;
 402         args.in_args = (uint8_t *)&version;
 403         args.in_args_size = sizeof(version);
 404         args.out_buffer = vf->aq_resp;
 405         args.out_size = I40E_AQ_BUF_SZ;
 406
 407         err = i40evf_execute_vf_cmd(dev, &args);
 408         if (err) {
 409                 PMD_INIT_LOG(ERR, "fail to execute command OP_VERSION");
 410                 return err;
 411         }
 412
 413         pver = (struct i40e_virtchnl_version_info *)args.out_buffer;
 414         vf->version_major = pver->major;
 415         vf->version_minor = pver->minor;
 416         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 417                 PMD_DRV_LOG(INFO, "Peer is DPDK PF host");
 418         else if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) &&
 419                 (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR))
 420                 PMD_DRV_LOG(INFO, "Peer is Linux PF host");
 421         else {
 422                 PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)",
 423                                         vf->version_major, vf->version_minor,
 424                                                 I40E_VIRTCHNL_VERSION_MAJOR,
 425                                                 I40E_VIRTCHNL_VERSION_MINOR);
 426                 return -1;
 427         }
 428
 429         return 0;
 430 }
 431
 432 static int
 433 i40evf_get_vf_resource(struct rte_eth_dev *dev)
 434 {
 435         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 436         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 437         int err;
 438         struct vf_cmd_info args;
 439         uint32_t caps, len;
 440
 441         args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
 442         args.out_buffer = vf->aq_resp;
 443         args.out_size = I40E_AQ_BUF_SZ;
 444         if (PF_IS_V11(vf)) {
 445                 caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
 446                        I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ |
 447                        I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
 448                        I40E_VIRTCHNL_VF_OFFLOAD_VLAN |
 449                        I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING;
 450                 args.in_args = (uint8_t *)&caps;
 451                 args.in_args_size = sizeof(caps);
 452         } else {
 453                 args.in_args = NULL;
 454                 args.in_args_size = 0;
 455         }
 456         err = i40evf_execute_vf_cmd(dev, &args);
 457
 458         if (err) {
 459                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_VF_RESOURCE");
 460                 return err;
 461         }
 462
 463         len =  sizeof(struct i40e_virtchnl_vf_resource) +
 464                 I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource);
 465
 466         (void)rte_memcpy(vf->vf_res, args.out_buffer,
 467                         RTE_MIN(args.out_size, len));
 468         i40e_vf_parse_hw_config(hw, vf->vf_res);
 469
 470         return 0;
 471 }
 472
 473 static int
 474 i40evf_config_promisc(struct rte_eth_dev *dev,
 475                       bool enable_unicast,
 476                       bool enable_multicast)
 477 {
 478         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 479         int err;
 480         struct vf_cmd_info args;
 481         struct i40e_virtchnl_promisc_info promisc;
 482
 483         promisc.flags = 0;
 484         promisc.vsi_id = vf->vsi_res->vsi_id;
 485
 486         if (enable_unicast)
 487                 promisc.flags |= I40E_FLAG_VF_UNICAST_PROMISC;
 488
 489         if (enable_multicast)
 490                 promisc.flags |= I40E_FLAG_VF_MULTICAST_PROMISC;
 491
 492         args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 493         args.in_args = (uint8_t *)&promisc;
 494         args.in_args_size = sizeof(promisc);
 495         args.out_buffer = vf->aq_resp;
 496         args.out_size = I40E_AQ_BUF_SZ;
 497
 498         err = i40evf_execute_vf_cmd(dev, &args);
 499
 500         if (err)
 501                 PMD_DRV_LOG(ERR, "fail to execute command "
 502                             "CONFIG_PROMISCUOUS_MODE");
 503         return err;
 504 }
 505
 506 /* Configure vlan and double vlan offload. Use flag to specify which part to configure */
 507 static int
 508 i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 509                                 bool enable_vlan_strip)
 510 {
 511         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 512         int err;
 513         struct vf_cmd_info args;
 514         struct i40e_virtchnl_vlan_offload_info offload;
 515
 516         offload.vsi_id = vf->vsi_res->vsi_id;
 517         offload.enable_vlan_strip = enable_vlan_strip;
 518
 519         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 520         args.in_args = (uint8_t *)&offload;
 521         args.in_args_size = sizeof(offload);
 522         args.out_buffer = vf->aq_resp;
 523         args.out_size = I40E_AQ_BUF_SZ;
 524
 525         err = i40evf_execute_vf_cmd(dev, &args);
 526         if (err)
 527                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_OFFLOAD");
 528
 529         return err;
 530 }
 531
 532 static int
 533 i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 534                                 struct i40e_vsi_vlan_pvid_info *info)
 535 {
 536         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 537         int err;
 538         struct vf_cmd_info args;
 539         struct i40e_virtchnl_pvid_info tpid_info;
 540
 541         if (info == NULL) {
 542                 PMD_DRV_LOG(ERR, "invalid parameters");
 543                 return I40E_ERR_PARAM;
 544         }
 545
 546         memset(&tpid_info, 0, sizeof(tpid_info));
 547         tpid_info.vsi_id = vf->vsi_res->vsi_id;
 548         (void)rte_memcpy(&tpid_info.info, info, sizeof(*info));
 549
 550         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 551         args.in_args = (uint8_t *)&tpid_info;
 552         args.in_args_size = sizeof(tpid_info);
 553         args.out_buffer = vf->aq_resp;
 554         args.out_size = I40E_AQ_BUF_SZ;
 555
 556         err = i40evf_execute_vf_cmd(dev, &args);
 557         if (err)
 558                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_PVID");
 559
 560         return err;
 561 }
 562
 563 static void
 564 i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info,
 565                                   uint16_t vsi_id,
 566                                   uint16_t queue_id,
 567                                   uint16_t nb_txq,
 568                                   struct i40e_tx_queue *txq)
 569 {
 570         txq_info->vsi_id = vsi_id;
 571         txq_info->queue_id = queue_id;
 572         if (queue_id < nb_txq) {
 573                 txq_info->ring_len = txq->nb_tx_desc;
 574                 txq_info->dma_ring_addr = txq->tx_ring_phys_addr;
 575         }
 576 }
 577
 578 static void
 579 i40evf_fill_virtchnl_vsi_rxq_info(struct i40e_virtchnl_rxq_info *rxq_info,
 580                                   uint16_t vsi_id,
 581                                   uint16_t queue_id,
 582                                   uint16_t nb_rxq,
 583                                   uint32_t max_pkt_size,
 584                                   struct i40e_rx_queue *rxq)
 585 {
 586         rxq_info->vsi_id = vsi_id;
 587         rxq_info->queue_id = queue_id;
 588         rxq_info->max_pkt_size = max_pkt_size;
 589         if (queue_id < nb_rxq) {
 590                 rxq_info->ring_len = rxq->nb_rx_desc;
 591                 rxq_info->dma_ring_addr = rxq->rx_ring_phys_addr;
 592                 rxq_info->databuffer_size =
 593                         (rte_pktmbuf_data_room_size(rxq->mp) -
 594                                 RTE_PKTMBUF_HEADROOM);
 595         }
 596 }
 597
 598 /* It configures VSI queues to co-work with Linux PF host */
 599 static int
 600 i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 601 {
 602         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 603         struct i40e_rx_queue **rxq =
 604                 (struct i40e_rx_queue **)dev->data->rx_queues;
 605         struct i40e_tx_queue **txq =
 606                 (struct i40e_tx_queue **)dev->data->tx_queues;
 607         struct i40e_virtchnl_vsi_queue_config_info *vc_vqci;
 608         struct i40e_virtchnl_queue_pair_info *vc_qpi;
 609         struct vf_cmd_info args;
 610         uint16_t i, nb_qp = vf->num_queue_pairs;
 611         const uint32_t size =
 612                 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqci, nb_qp);
 613         uint8_t buff[size];
 614         int ret;
 615
 616         memset(buff, 0, sizeof(buff));
 617         vc_vqci = (struct i40e_virtchnl_vsi_queue_config_info *)buff;
 618         vc_vqci->vsi_id = vf->vsi_res->vsi_id;
 619         vc_vqci->num_queue_pairs = nb_qp;
 620
 621         for (i = 0, vc_qpi = vc_vqci->qpair; i < nb_qp; i++, vc_qpi++) {
 622                 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpi->txq,
 623                         vc_vqci->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
 624                 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpi->rxq,
 625                         vc_vqci->vsi_id, i, dev->data->nb_rx_queues,
 626                                         vf->max_pkt_len, rxq[i]);
 627         }
 628         memset(&args, 0, sizeof(args));
 629         args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 630         args.in_args = (uint8_t *)vc_vqci;
 631         args.in_args_size = size;
 632         args.out_buffer = vf->aq_resp;
 633         args.out_size = I40E_AQ_BUF_SZ;
 634         ret = i40evf_execute_vf_cmd(dev, &args);
 635         if (ret)
 636                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 637                         "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES\n");
 638
 639         return ret;
 640 }
 641
 642 /* It configures VSI queues to co-work with DPDK PF host */
 643 static int
 644 i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 645 {
 646         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 647         struct i40e_rx_queue **rxq =
 648                 (struct i40e_rx_queue **)dev->data->rx_queues;
 649         struct i40e_tx_queue **txq =
 650                 (struct i40e_tx_queue **)dev->data->tx_queues;
 651         struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei;
 652         struct i40e_virtchnl_queue_pair_ext_info *vc_qpei;
 653         struct vf_cmd_info args;
 654         uint16_t i, nb_qp = vf->num_queue_pairs;
 655         const uint32_t size =
 656                 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqcei, nb_qp);
 657         uint8_t buff[size];
 658         int ret;
 659
 660         memset(buff, 0, sizeof(buff));
 661         vc_vqcei = (struct i40e_virtchnl_vsi_queue_config_ext_info *)buff;
 662         vc_vqcei->vsi_id = vf->vsi_res->vsi_id;
 663         vc_vqcei->num_queue_pairs = nb_qp;
 664         vc_qpei = vc_vqcei->qpair;
 665         for (i = 0; i < nb_qp; i++, vc_qpei++) {
 666                 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpei->txq,
 667                         vc_vqcei->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
 668                 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpei->rxq,
 669                         vc_vqcei->vsi_id, i, dev->data->nb_rx_queues,
 670                                         vf->max_pkt_len, rxq[i]);
 671                 if (i < dev->data->nb_rx_queues)
 672                         /*
 673                          * It adds extra info for configuring VSI queues, which
 674                          * is needed to enable the configurable crc stripping
 675                          * in VF.
 676                          */
 677                         vc_qpei->rxq_ext.crcstrip =
 678                                 dev->data->dev_conf.rxmode.hw_strip_crc;
 679         }
 680         memset(&args, 0, sizeof(args));
 681         args.ops =
 682                 (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 683         args.in_args = (uint8_t *)vc_vqcei;
 684         args.in_args_size = size;
 685         args.out_buffer = vf->aq_resp;
 686         args.out_size = I40E_AQ_BUF_SZ;
 687         ret = i40evf_execute_vf_cmd(dev, &args);
 688         if (ret)
 689                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 690                         "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT\n");
 691
 692         return ret;
 693 }
 694
 695 static int
 696 i40evf_configure_queues(struct rte_eth_dev *dev)
 697 {
 698         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 699
 700         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 701                 /* To support DPDK PF host */
 702                 return i40evf_configure_vsi_queues_ext(dev);
 703         else
 704                 /* To support Linux PF host */
 705                 return i40evf_configure_vsi_queues(dev);
 706 }
 707
 708 static int
 709 i40evf_config_irq_map(struct rte_eth_dev *dev)
 710 {
 711         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 712         struct vf_cmd_info args;
 713         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_irq_map_info) + \
 714                 sizeof(struct i40e_virtchnl_vector_map)];
 715         struct i40e_virtchnl_irq_map_info *map_info;
 716         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 717         uint32_t vector_id;
 718         int i, err;
 719
 720         if (rte_intr_allow_others(intr_handle)) {
 721                 if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 722                         vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR;
 723                 else
 724                         vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR_LNX;
 725         } else {
 726                 vector_id = I40E_MISC_VEC_ID;
 727         }
 728
 729         map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 730         map_info->num_vectors = 1;
 731         map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 732         map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 733         /* Alway use default dynamic MSIX interrupt */
 734         map_info->vecmap[0].vector_id = vector_id;
 735         /* Don't map any tx queue */
 736         map_info->vecmap[0].txq_map = 0;
 737         map_info->vecmap[0].rxq_map = 0;
 738         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 739                 map_info->vecmap[0].rxq_map |= 1 << i;
 740                 if (rte_intr_dp_is_en(intr_handle))
 741                         intr_handle->intr_vec[i] = vector_id;
 742         }
 743
 744         args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 745         args.in_args = (u8 *)cmd_buffer;
 746         args.in_args_size = sizeof(cmd_buffer);
 747         args.out_buffer = vf->aq_resp;
 748         args.out_size = I40E_AQ_BUF_SZ;
 749         err = i40evf_execute_vf_cmd(dev, &args);
 750         if (err)
 751                 PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES");
 752
 753         return err;
 754 }
 755
 756 static int
 757 i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 758                                 bool on)
 759 {
 760         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 761         struct i40e_virtchnl_queue_select queue_select;
 762         int err;
 763         struct vf_cmd_info args;
 764         memset(&queue_select, 0, sizeof(queue_select));
 765         queue_select.vsi_id = vf->vsi_res->vsi_id;
 766
 767         if (isrx)
 768                 queue_select.rx_queues |= 1 << qid;
 769         else
 770                 queue_select.tx_queues |= 1 << qid;
 771
 772         if (on)
 773                 args.ops = I40E_VIRTCHNL_OP_ENABLE_QUEUES;
 774         else
 775                 args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 776         args.in_args = (u8 *)&queue_select;
 777         args.in_args_size = sizeof(queue_select);
 778         args.out_buffer = vf->aq_resp;
 779         args.out_size = I40E_AQ_BUF_SZ;
 780         err = i40evf_execute_vf_cmd(dev, &args);
 781         if (err)
 782                 PMD_DRV_LOG(ERR, "fail to switch %s %u %s",
 783                             isrx ? "RX" : "TX", qid, on ? "on" : "off");
 784
 785         return err;
 786 }
 787
 788 static int
 789 i40evf_start_queues(struct rte_eth_dev *dev)
 790 {
 791         struct rte_eth_dev_data *dev_data = dev->data;
 792         int i;
 793         struct i40e_rx_queue *rxq;
 794         struct i40e_tx_queue *txq;
 795
 796         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 797                 rxq = dev_data->rx_queues[i];
 798                 if (rxq->rx_deferred_start)
 799                         continue;
 800                 if (i40evf_dev_rx_queue_start(dev, i) != 0) {
 801                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 802                         return -1;
 803                 }
 804         }
 805
 806         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 807                 txq = dev_data->tx_queues[i];
 808                 if (txq->tx_deferred_start)
 809                         continue;
 810                 if (i40evf_dev_tx_queue_start(dev, i) != 0) {
 811                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 812                         return -1;
 813                 }
 814         }
 815
 816         return 0;
 817 }
 818
 819 static int
 820 i40evf_stop_queues(struct rte_eth_dev *dev)
 821 {
 822         int i;
 823
 824         /* Stop TX queues first */
 825         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 826                 if (i40evf_dev_tx_queue_stop(dev, i) != 0) {
 827                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 828                         return -1;
 829                 }
 830         }
 831
 832         /* Then stop RX queues */
 833         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 834                 if (i40evf_dev_rx_queue_stop(dev, i) != 0) {
 835                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 836                         return -1;
 837                 }
 838         }
 839
 840         return 0;
 841 }
 842
 843 static void
 844 i40evf_add_mac_addr(struct rte_eth_dev *dev,
 845                     struct ether_addr *addr,
 846                     __rte_unused uint32_t index,
 847                     __rte_unused uint32_t pool)
 848 {
 849         struct i40e_virtchnl_ether_addr_list *list;
 850         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 851         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 852                         sizeof(struct i40e_virtchnl_ether_addr)];
 853         int err;
 854         struct vf_cmd_info args;
 855
 856         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 857                 PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x",
 858                             addr->addr_bytes[0], addr->addr_bytes[1],
 859                             addr->addr_bytes[2], addr->addr_bytes[3],
 860                             addr->addr_bytes[4], addr->addr_bytes[5]);
 861                 return;
 862         }
 863
 864         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 865         list->vsi_id = vf->vsi_res->vsi_id;
 866         list->num_elements = 1;
 867         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 868                                         sizeof(addr->addr_bytes));
 869
 870         args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 871         args.in_args = cmd_buffer;
 872         args.in_args_size = sizeof(cmd_buffer);
 873         args.out_buffer = vf->aq_resp;
 874         args.out_size = I40E_AQ_BUF_SZ;
 875         err = i40evf_execute_vf_cmd(dev, &args);
 876         if (err)
 877                 PMD_DRV_LOG(ERR, "fail to execute command "
 878                             "OP_ADD_ETHER_ADDRESS");
 879
 880         return;
 881 }
 882
 883 static void
 884 i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index)
 885 {
 886         struct i40e_virtchnl_ether_addr_list *list;
 887         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 888         struct rte_eth_dev_data *data = dev->data;
 889         struct ether_addr *addr;
 890         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 891                         sizeof(struct i40e_virtchnl_ether_addr)];
 892         int err;
 893         struct vf_cmd_info args;
 894
 895         addr = &(data->mac_addrs[index]);
 896
 897         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 898                 PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x",
 899                             addr->addr_bytes[0], addr->addr_bytes[1],
 900                             addr->addr_bytes[2], addr->addr_bytes[3],
 901                             addr->addr_bytes[4], addr->addr_bytes[5]);
 902                 return;
 903         }
 904
 905         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 906         list->vsi_id = vf->vsi_res->vsi_id;
 907         list->num_elements = 1;
 908         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 909                         sizeof(addr->addr_bytes));
 910
 911         args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 912         args.in_args = cmd_buffer;
 913         args.in_args_size = sizeof(cmd_buffer);
 914         args.out_buffer = vf->aq_resp;
 915         args.out_size = I40E_AQ_BUF_SZ;
 916         err = i40evf_execute_vf_cmd(dev, &args);
 917         if (err)
 918                 PMD_DRV_LOG(ERR, "fail to execute command "
 919                             "OP_DEL_ETHER_ADDRESS");
 920         return;
 921 }
 922
 923 static int
 924 i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 925 {
 926         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 927         struct i40e_virtchnl_queue_select q_stats;
 928         int err;
 929         struct vf_cmd_info args;
 930
 931         memset(&q_stats, 0, sizeof(q_stats));
 932         q_stats.vsi_id = vf->vsi_res->vsi_id;
 933         args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 934         args.in_args = (u8 *)&q_stats;
 935         args.in_args_size = sizeof(q_stats);
 936         args.out_buffer = vf->aq_resp;
 937         args.out_size = I40E_AQ_BUF_SZ;
 938
 939         err = i40evf_execute_vf_cmd(dev, &args);
 940         if (err) {
 941                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS");
 942                 *pstats = NULL;
 943                 return err;
 944         }
 945         *pstats = (struct i40e_eth_stats *)args.out_buffer;
 946         return 0;
 947 }
 948
 949 static int
 950 i40evf_get_statics(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 951 {
 952         int ret;
 953         struct i40e_eth_stats *pstats = NULL;
 954
 955         ret = i40evf_update_stats(dev, &pstats);
 956         if (ret != 0)
 957                 return 0;
 958
 959         stats->ipackets = pstats->rx_unicast + pstats->rx_multicast +
 960                                                 pstats->rx_broadcast;
 961         stats->opackets = pstats->tx_broadcast + pstats->tx_multicast +
 962                                                 pstats->tx_unicast;
 963         stats->ierrors = pstats->rx_discards;
 964         stats->oerrors = pstats->tx_errors + pstats->tx_discards;
 965         stats->ibytes = pstats->rx_bytes;
 966         stats->obytes = pstats->tx_bytes;
 967
 968         return 0;
 969 }
 970
 971 static void
 972 i40evf_dev_xstats_reset(struct rte_eth_dev *dev)
 973 {
 974         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 975         struct i40e_eth_stats *pstats = NULL;
 976
 977         /* read stat values to clear hardware registers */
 978         i40evf_update_stats(dev, &pstats);
 979
 980         /* set stats offset base on current values */
 981         vf->vsi.eth_stats_offset = vf->vsi.eth_stats;
 982 }
 983
 984 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
 985                                  struct rte_eth_xstats *xstats, unsigned n)
 986 {
 987         int ret;
 988         unsigned i;
 989         struct i40e_eth_stats *pstats = NULL;
 990
 991         if (n < I40EVF_NB_XSTATS)
 992                 return I40EVF_NB_XSTATS;
 993
 994         ret = i40evf_update_stats(dev, &pstats);
 995         if (ret != 0)
 996                 return 0;
 997
 998         if (!xstats)
 999                 return 0;
1000
1001         /* loop over xstats array and values from pstats */
1002         for (i = 0; i < I40EVF_NB_XSTATS; i++) {
1003                 snprintf(xstats[i].name, sizeof(xstats[i].name),
1004                          "%s", rte_i40evf_stats_strings[i].name);
1005                 xstats[i].value = *(uint64_t *)(((char *)pstats) +
1006                         rte_i40evf_stats_strings[i].offset);
1007         }
1008
1009         return I40EVF_NB_XSTATS;
1010 }
1011
1012 static int
1013 i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1014 {
1015         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1016         struct i40e_virtchnl_vlan_filter_list *vlan_list;
1017         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
1018                                                         sizeof(uint16_t)];
1019         int err;
1020         struct vf_cmd_info args;
1021
1022         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
1023         vlan_list->vsi_id = vf->vsi_res->vsi_id;
1024         vlan_list->num_elements = 1;
1025         vlan_list->vlan_id[0] = vlanid;
1026
1027         args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
1028         args.in_args = (u8 *)&cmd_buffer;
1029         args.in_args_size = sizeof(cmd_buffer);
1030         args.out_buffer = vf->aq_resp;
1031         args.out_size = I40E_AQ_BUF_SZ;
1032         err = i40evf_execute_vf_cmd(dev, &args);
1033         if (err)
1034                 PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_VLAN");
1035
1036         return err;
1037 }
1038
1039 static int
1040 i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1041 {
1042         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1043         struct i40e_virtchnl_vlan_filter_list *vlan_list;
1044         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
1045                                                         sizeof(uint16_t)];
1046         int err;
1047         struct vf_cmd_info args;
1048
1049         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
1050         vlan_list->vsi_id = vf->vsi_res->vsi_id;
1051         vlan_list->num_elements = 1;
1052         vlan_list->vlan_id[0] = vlanid;
1053
1054         args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
1055         args.in_args = (u8 *)&cmd_buffer;
1056         args.in_args_size = sizeof(cmd_buffer);
1057         args.out_buffer = vf->aq_resp;
1058         args.out_size = I40E_AQ_BUF_SZ;
1059         err = i40evf_execute_vf_cmd(dev, &args);
1060         if (err)
1061                 PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_VLAN");
1062
1063         return err;
1064 }
1065
1066 static int
1067 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
1068 {
1069         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1070         int err;
1071         struct vf_cmd_info args;
1072         struct rte_eth_link *new_link;
1073
1074         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
1075         args.in_args = NULL;
1076         args.in_args_size = 0;
1077         args.out_buffer = vf->aq_resp;
1078         args.out_size = I40E_AQ_BUF_SZ;
1079         err = i40evf_execute_vf_cmd(dev, &args);
1080         if (err) {
1081                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_LINK_STAT");
1082                 return err;
1083         }
1084
1085         new_link = (struct rte_eth_link *)args.out_buffer;
1086         (void)rte_memcpy(link, new_link, sizeof(*link));
1087
1088         return 0;
1089 }
1090
1091 static const struct rte_pci_id pci_id_i40evf_map[] = {
1092 #define RTE_PCI_DEV_ID_DECL_I40EVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
1093 #include "rte_pci_dev_ids.h"
1094 { .vendor_id = 0, /* sentinel */ },
1095 };
1096
1097 static inline int
1098 i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
1099                                     struct rte_eth_link *link)
1100 {
1101         struct rte_eth_link *dst = &(dev->data->dev_link);
1102         struct rte_eth_link *src = link;
1103
1104         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1105                                         *(uint64_t *)src) == 0)
1106                 return -1;
1107
1108         return 0;
1109 }
1110
1111 /* Disable IRQ0 */
1112 static inline void
1113 i40evf_disable_irq0(struct i40e_hw *hw)
1114 {
1115         /* Disable all interrupt types */
1116         I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
1117         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1118                        I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1119         I40EVF_WRITE_FLUSH(hw);
1120 }
1121
1122 /* Enable IRQ0 */
1123 static inline void
1124 i40evf_enable_irq0(struct i40e_hw *hw)
1125 {
1126         /* Enable admin queue interrupt trigger */
1127         uint32_t val;
1128
1129         i40evf_disable_irq0(hw);
1130         val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
1131         val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
1132                 I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
1133         I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
1134
1135         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1136                 I40E_VFINT_DYN_CTL01_INTENA_MASK |
1137                 I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1138                 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1139
1140         I40EVF_WRITE_FLUSH(hw);
1141 }
1142
1143 static int
1144 i40evf_reset_vf(struct i40e_hw *hw)
1145 {
1146         int i, reset;
1147
1148         if (i40e_vf_reset(hw) != I40E_SUCCESS) {
1149                 PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1150                 return -1;
1151         }
1152         /**
1153           * After issuing vf reset command to pf, pf won't necessarily
1154           * reset vf, it depends on what state it exactly is. If it's not
1155           * initialized yet, it won't have vf reset since it's in a certain
1156           * state. If not, it will try to reset. Even vf is reset, pf will
1157           * set I40E_VFGEN_RSTAT to COMPLETE first, then wait 10ms and set
1158           * it to ACTIVE. In this duration, vf may not catch the moment that
1159           * COMPLETE is set. So, for vf, we'll try to wait a long time.
1160           */
1161         rte_delay_ms(200);
1162
1163         for (i = 0; i < MAX_RESET_WAIT_CNT; i++) {
1164                 reset = rd32(hw, I40E_VFGEN_RSTAT) &
1165                         I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1166                 reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT;
1167                 if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset)
1168                         break;
1169                 else
1170                         rte_delay_ms(50);
1171         }
1172
1173         if (i >= MAX_RESET_WAIT_CNT) {
1174                 PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1175                 return -1;
1176         }
1177
1178         return 0;
1179 }
1180
1181 static int
1182 i40evf_init_vf(struct rte_eth_dev *dev)
1183 {
1184         int i, err, bufsz;
1185         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1186         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1187         struct ether_addr *p_mac_addr;
1188         uint16_t interval =
1189                 i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
1190
1191         vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1192         vf->dev_data = dev->data;
1193         err = i40e_set_mac_type(hw);
1194         if (err) {
1195                 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err);
1196                 goto err;
1197         }
1198
1199         i40e_init_adminq_parameter(hw);
1200         err = i40e_init_adminq(hw);
1201         if (err) {
1202                 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err);
1203                 goto err;
1204         }
1205
1206         /* Reset VF and wait until it's complete */
1207         if (i40evf_reset_vf(hw)) {
1208                 PMD_INIT_LOG(ERR, "reset NIC failed");
1209                 goto err_aq;
1210         }
1211
1212         /* VF reset, shutdown admin queue and initialize again */
1213         if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) {
1214                 PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed");
1215                 return -1;
1216         }
1217
1218         i40e_init_adminq_parameter(hw);
1219         if (i40e_init_adminq(hw) != I40E_SUCCESS) {
1220                 PMD_INIT_LOG(ERR, "init_adminq failed");
1221                 return -1;
1222         }
1223         vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
1224         if (!vf->aq_resp) {
1225                 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
1226                         goto err_aq;
1227         }
1228         if (i40evf_check_api_version(dev) != 0) {
1229                 PMD_INIT_LOG(ERR, "check_api version failed");
1230                 goto err_aq;
1231         }
1232         bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
1233                 (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource));
1234         vf->vf_res = rte_zmalloc("vf_res", bufsz, 0);
1235         if (!vf->vf_res) {
1236                 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory");
1237                         goto err_aq;
1238         }
1239
1240         if (i40evf_get_vf_resource(dev) != 0) {
1241                 PMD_INIT_LOG(ERR, "i40evf_get_vf_config failed");
1242                 goto err_alloc;
1243         }
1244
1245         /* got VF config message back from PF, now we can parse it */
1246         for (i = 0; i < vf->vf_res->num_vsis; i++) {
1247                 if (vf->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
1248                         vf->vsi_res = &vf->vf_res->vsi_res[i];
1249         }
1250
1251         if (!vf->vsi_res) {
1252                 PMD_INIT_LOG(ERR, "no LAN VSI found");
1253                 goto err_alloc;
1254         }
1255
1256         if (hw->mac.type == I40E_MAC_X722_VF)
1257                 vf->flags = I40E_FLAG_RSS_AQ_CAPABLE;
1258         vf->vsi.vsi_id = vf->vsi_res->vsi_id;
1259         vf->vsi.type = vf->vsi_res->vsi_type;
1260         vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs;
1261         vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1262
1263         /* Store the MAC address configured by host, or generate random one */
1264         p_mac_addr = (struct ether_addr *)(vf->vsi_res->default_mac_addr);
1265         if (is_valid_assigned_ether_addr(p_mac_addr)) /* Configured by host */
1266                 ether_addr_copy(p_mac_addr, (struct ether_addr *)hw->mac.addr);
1267         else
1268                 eth_random_addr(hw->mac.addr); /* Generate a random one */
1269
1270         /* If the PF host is not DPDK, set the interval of ITR0 to max*/
1271         if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
1272                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1273                                (I40E_ITR_INDEX_DEFAULT <<
1274                                 I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
1275                                (interval <<
1276                                 I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
1277                 I40EVF_WRITE_FLUSH(hw);
1278         }
1279
1280         return 0;
1281
1282 err_alloc:
1283         rte_free(vf->vf_res);
1284 err_aq:
1285         i40e_shutdown_adminq(hw); /* ignore error */
1286 err:
1287         return -1;
1288 }
1289
1290 static int
1291 i40evf_uninit_vf(struct rte_eth_dev *dev)
1292 {
1293         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1294         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1295
1296         PMD_INIT_FUNC_TRACE();
1297
1298         if (hw->adapter_stopped == 0)
1299                 i40evf_dev_close(dev);
1300         rte_free(vf->vf_res);
1301         vf->vf_res = NULL;
1302         rte_free(vf->aq_resp);
1303         vf->aq_resp = NULL;
1304
1305         return 0;
1306 }
1307
1308 static void
1309 i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
1310                            uint8_t *msg,
1311                            __rte_unused uint16_t msglen)
1312 {
1313         struct i40e_virtchnl_pf_event *pf_msg =
1314                         (struct i40e_virtchnl_pf_event *)msg;
1315         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1316
1317         switch (pf_msg->event) {
1318         case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
1319                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
1320                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
1321                 break;
1322         case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
1323                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
1324                 vf->link_up = pf_msg->event_data.link_event.link_status;
1325                 vf->link_speed = pf_msg->event_data.link_event.link_speed;
1326                 break;
1327         case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
1328                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
1329                 break;
1330         default:
1331                 PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
1332                 break;
1333         }
1334 }
1335
1336 static void
1337 i40evf_handle_aq_msg(struct rte_eth_dev *dev)
1338 {
1339         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1340         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1341         struct i40e_arq_event_info info;
1342         struct i40e_virtchnl_msg *v_msg;
1343         uint16_t pending, opcode;
1344         int ret;
1345
1346         info.buf_len = I40E_AQ_BUF_SZ;
1347         if (!vf->aq_resp) {
1348                 PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
1349                 return;
1350         }
1351         info.msg_buf = vf->aq_resp;
1352         v_msg = (struct i40e_virtchnl_msg *)&info.desc;
1353
1354         pending = 1;
1355         while (pending) {
1356                 ret = i40e_clean_arq_element(hw, &info, &pending);
1357
1358                 if (ret != I40E_SUCCESS) {
1359                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
1360                                     "ret: %d", ret);
1361                         break;
1362                 }
1363                 opcode = rte_le_to_cpu_16(info.desc.opcode);
1364
1365                 switch (opcode) {
1366                 case i40e_aqc_opc_send_msg_to_vf:
1367                         if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
1368                                 /* process event*/
1369                                 i40evf_handle_pf_event(dev, info.msg_buf,
1370                                                        info.msg_len);
1371                         else {
1372                                 /* read message and it's expected one */
1373                                 if (v_msg->v_opcode == vf->pend_cmd) {
1374                                         vf->cmd_retval = v_msg->v_retval;
1375                                         /* prevent compiler reordering */
1376                                         rte_compiler_barrier();
1377                                         _clear_cmd(vf);
1378                                 } else
1379                                         PMD_DRV_LOG(ERR, "command mismatch,"
1380                                                 "expect %u, get %u",
1381                                                 vf->pend_cmd, v_msg->v_opcode);
1382                                 PMD_DRV_LOG(DEBUG, "adminq response is received,"
1383                                              " opcode = %d\n", v_msg->v_opcode);
1384                         }
1385                         break;
1386                 default:
1387                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
1388                                     opcode);
1389                         break;
1390                 }
1391         }
1392 }
1393
1394 /**
1395  * Interrupt handler triggered by NIC  for handling
1396  * specific interrupt. Only adminq interrupt is processed in VF.
1397  *
1398  * @param handle
1399  *  Pointer to interrupt handle.
1400  * @param param
1401  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1402  *
1403  * @return
1404  *  void
1405  */
1406 static void
1407 i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1408                              void *param)
1409 {
1410         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1411         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1412         uint32_t icr0;
1413
1414         i40evf_disable_irq0(hw);
1415
1416         /* read out interrupt causes */
1417         icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
1418
1419         /* No interrupt event indicated */
1420         if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
1421                 PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
1422                 goto done;
1423         }
1424
1425         if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
1426                 PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
1427                 i40evf_handle_aq_msg(dev);
1428         }
1429
1430         /* Link Status Change interrupt */
1431         if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
1432                 PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
1433                                    " do nothing\n");
1434
1435 done:
1436         i40evf_enable_irq0(hw);
1437         rte_intr_enable(&dev->pci_dev->intr_handle);
1438 }
1439
1440 static int
1441 i40evf_dev_init(struct rte_eth_dev *eth_dev)
1442 {
1443         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
1444                         eth_dev->data->dev_private);
1445         struct rte_pci_device *pci_dev = eth_dev->pci_dev;
1446
1447         PMD_INIT_FUNC_TRACE();
1448
1449         /* assign ops func pointer */
1450         eth_dev->dev_ops = &i40evf_eth_dev_ops;
1451         eth_dev->rx_pkt_burst = &i40e_recv_pkts;
1452         eth_dev->tx_pkt_burst = &i40e_xmit_pkts;
1453
1454         /*
1455          * For secondary processes, we don't initialise any further as primary
1456          * has already done this work.
1457          */
1458         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1459                 i40e_set_rx_function(eth_dev);
1460                 i40e_set_tx_function(eth_dev);
1461                 return 0;
1462         }
1463
1464         rte_eth_copy_pci_info(eth_dev, eth_dev->pci_dev);
1465
1466         hw->vendor_id = eth_dev->pci_dev->id.vendor_id;
1467         hw->device_id = eth_dev->pci_dev->id.device_id;
1468         hw->subsystem_vendor_id = eth_dev->pci_dev->id.subsystem_vendor_id;
1469         hw->subsystem_device_id = eth_dev->pci_dev->id.subsystem_device_id;
1470         hw->bus.device = eth_dev->pci_dev->addr.devid;
1471         hw->bus.func = eth_dev->pci_dev->addr.function;
1472         hw->hw_addr = (void *)eth_dev->pci_dev->mem_resource[0].addr;
1473         hw->adapter_stopped = 0;
1474
1475         if(i40evf_init_vf(eth_dev) != 0) {
1476                 PMD_INIT_LOG(ERR, "Init vf failed");
1477                 return -1;
1478         }
1479
1480         /* register callback func to eal lib */
1481         rte_intr_callback_register(&pci_dev->intr_handle,
1482                 i40evf_dev_interrupt_handler, (void *)eth_dev);
1483
1484         /* enable uio intr after callback register */
1485         rte_intr_enable(&pci_dev->intr_handle);
1486
1487         /* configure and enable device interrupt */
1488         i40evf_enable_irq0(hw);
1489
1490         /* copy mac addr */
1491         eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
1492                                         ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX,
1493                                         0);
1494         if (eth_dev->data->mac_addrs == NULL) {
1495                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to"
1496                                 " store MAC addresses",
1497                                 ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX);
1498                 return -ENOMEM;
1499         }
1500         ether_addr_copy((struct ether_addr *)hw->mac.addr,
1501                         &eth_dev->data->mac_addrs[0]);
1502
1503         return 0;
1504 }
1505
1506 static int
1507 i40evf_dev_uninit(struct rte_eth_dev *eth_dev)
1508 {
1509         PMD_INIT_FUNC_TRACE();
1510
1511         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1512                 return -EPERM;
1513
1514         eth_dev->dev_ops = NULL;
1515         eth_dev->rx_pkt_burst = NULL;
1516         eth_dev->tx_pkt_burst = NULL;
1517
1518         if (i40evf_uninit_vf(eth_dev) != 0) {
1519                 PMD_INIT_LOG(ERR, "i40evf_uninit_vf failed");
1520                 return -1;
1521         }
1522
1523         rte_free(eth_dev->data->mac_addrs);
1524         eth_dev->data->mac_addrs = NULL;
1525
1526         return 0;
1527 }
1528 /*
1529  * virtual function driver struct
1530  */
1531 static struct eth_driver rte_i40evf_pmd = {
1532         .pci_drv = {
1533                 .name = "rte_i40evf_pmd",
1534                 .id_table = pci_id_i40evf_map,
1535                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1536         },
1537         .eth_dev_init = i40evf_dev_init,
1538         .eth_dev_uninit = i40evf_dev_uninit,
1539         .dev_private_size = sizeof(struct i40e_adapter),
1540 };
1541
1542 /*
1543  * VF Driver initialization routine.
1544  * Invoked one at EAL init time.
1545  * Register itself as the [Virtual Poll Mode] Driver of PCI Fortville devices.
1546  */
1547 static int
1548 rte_i40evf_pmd_init(const char *name __rte_unused,
1549                     const char *params __rte_unused)
1550 {
1551         PMD_INIT_FUNC_TRACE();
1552
1553         rte_eth_driver_register(&rte_i40evf_pmd);
1554
1555         return 0;
1556 }
1557
1558 static struct rte_driver rte_i40evf_driver = {
1559         .type = PMD_PDEV,
1560         .init = rte_i40evf_pmd_init,
1561 };
1562
1563 PMD_REGISTER_DRIVER(rte_i40evf_driver);
1564
1565 static int
1566 i40evf_dev_configure(struct rte_eth_dev *dev)
1567 {
1568         struct i40e_adapter *ad =
1569                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1570         struct rte_eth_conf *conf = &dev->data->dev_conf;
1571         struct i40e_vf *vf;
1572
1573         /* Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1574          * allocation or vector Rx preconditions we will reset it.
1575          */
1576         ad->rx_bulk_alloc_allowed = true;
1577         ad->rx_vec_allowed = true;
1578         ad->tx_simple_allowed = true;
1579         ad->tx_vec_allowed = true;
1580
1581         /* For non-DPDK PF drivers, VF has no ability to disable HW
1582          * CRC strip, and is implicitly enabled by the PF.
1583          */
1584         if (!conf->rxmode.hw_strip_crc) {
1585                 vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1586                 if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) &&
1587                     (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR)) {
1588                         /* Peer is running non-DPDK PF driver. */
1589                         PMD_INIT_LOG(ERR, "VF can't disable HW CRC Strip");
1590                         return -EINVAL;
1591                 }
1592         }
1593
1594         return i40evf_init_vlan(dev);
1595 }
1596
1597 static int
1598 i40evf_init_vlan(struct rte_eth_dev *dev)
1599 {
1600         struct rte_eth_dev_data *data = dev->data;
1601         int ret;
1602
1603         /* Apply vlan offload setting */
1604         i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
1605
1606         /* Apply pvid setting */
1607         ret = i40evf_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
1608                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
1609         return ret;
1610 }
1611
1612 static void
1613 i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1614 {
1615         bool enable_vlan_strip = 0;
1616         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1617         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1618
1619         /* Linux pf host doesn't support vlan offload yet */
1620         if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1621                 /* Vlan stripping setting */
1622                 if (mask & ETH_VLAN_STRIP_MASK) {
1623                         /* Enable or disable VLAN stripping */
1624                         if (dev_conf->rxmode.hw_vlan_strip)
1625                                 enable_vlan_strip = 1;
1626                         else
1627                                 enable_vlan_strip = 0;
1628
1629                         i40evf_config_vlan_offload(dev, enable_vlan_strip);
1630                 }
1631         }
1632 }
1633
1634 static int
1635 i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1636 {
1637         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1638         struct i40e_vsi_vlan_pvid_info info;
1639         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1640
1641         memset(&info, 0, sizeof(info));
1642         info.on = on;
1643
1644         /* Linux pf host don't support vlan offload yet */
1645         if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1646                 if (info.on)
1647                         info.config.pvid = pvid;
1648                 else {
1649                         info.config.reject.tagged =
1650                                 dev_conf->txmode.hw_vlan_reject_tagged;
1651                         info.config.reject.untagged =
1652                                 dev_conf->txmode.hw_vlan_reject_untagged;
1653                 }
1654                 return i40evf_config_vlan_pvid(dev, &info);
1655         }
1656
1657         return 0;
1658 }
1659
1660 static int
1661 i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1662 {
1663         struct i40e_rx_queue *rxq;
1664         int err = 0;
1665         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1666
1667         PMD_INIT_FUNC_TRACE();
1668
1669         if (rx_queue_id < dev->data->nb_rx_queues) {
1670                 rxq = dev->data->rx_queues[rx_queue_id];
1671
1672                 err = i40e_alloc_rx_queue_mbufs(rxq);
1673                 if (err) {
1674                         PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1675                         return err;
1676                 }
1677
1678                 rte_wmb();
1679
1680                 /* Init the RX tail register. */
1681                 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1682                 I40EVF_WRITE_FLUSH(hw);
1683
1684                 /* Ready to switch the queue on */
1685                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE);
1686
1687                 if (err)
1688                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
1689                                     rx_queue_id);
1690                 else
1691                         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1692         }
1693
1694         return err;
1695 }
1696
1697 static int
1698 i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1699 {
1700         struct i40e_rx_queue *rxq;
1701         int err;
1702
1703         if (rx_queue_id < dev->data->nb_rx_queues) {
1704                 rxq = dev->data->rx_queues[rx_queue_id];
1705
1706                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE);
1707
1708                 if (err) {
1709                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
1710                                     rx_queue_id);
1711                         return err;
1712                 }
1713
1714                 i40e_rx_queue_release_mbufs(rxq);
1715                 i40e_reset_rx_queue(rxq);
1716                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1717         }
1718
1719         return 0;
1720 }
1721
1722 static int
1723 i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1724 {
1725         int err = 0;
1726
1727         PMD_INIT_FUNC_TRACE();
1728
1729         if (tx_queue_id < dev->data->nb_tx_queues) {
1730
1731                 /* Ready to switch the queue on */
1732                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE);
1733
1734                 if (err)
1735                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
1736                                     tx_queue_id);
1737                 else
1738                         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1739         }
1740
1741         return err;
1742 }
1743
1744 static int
1745 i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1746 {
1747         struct i40e_tx_queue *txq;
1748         int err;
1749
1750         if (tx_queue_id < dev->data->nb_tx_queues) {
1751                 txq = dev->data->tx_queues[tx_queue_id];
1752
1753                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE);
1754
1755                 if (err) {
1756                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
1757                                     tx_queue_id);
1758                         return err;
1759                 }
1760
1761                 i40e_tx_queue_release_mbufs(txq);
1762                 i40e_reset_tx_queue(txq);
1763                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1764         }
1765
1766         return 0;
1767 }
1768
1769 static int
1770 i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1771 {
1772         int ret;
1773
1774         if (on)
1775                 ret = i40evf_add_vlan(dev, vlan_id);
1776         else
1777                 ret = i40evf_del_vlan(dev,vlan_id);
1778
1779         return ret;
1780 }
1781
1782 static int
1783 i40evf_rxq_init(struct rte_eth_dev *dev, struct i40e_rx_queue *rxq)
1784 {
1785         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1786         struct rte_eth_dev_data *dev_data = dev->data;
1787         struct rte_pktmbuf_pool_private *mbp_priv;
1788         uint16_t buf_size, len;
1789
1790         rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL1(rxq->queue_id);
1791         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1792         I40EVF_WRITE_FLUSH(hw);
1793
1794         /* Calculate the maximum packet length allowed */
1795         mbp_priv = rte_mempool_get_priv(rxq->mp);
1796         buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
1797                                         RTE_PKTMBUF_HEADROOM);
1798         rxq->hs_mode = i40e_header_split_none;
1799         rxq->rx_hdr_len = 0;
1800         rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
1801         len = rxq->rx_buf_len * I40E_MAX_CHAINED_RX_BUFFERS;
1802         rxq->max_pkt_len = RTE_MIN(len,
1803                 dev_data->dev_conf.rxmode.max_rx_pkt_len);
1804
1805         /**
1806          * Check if the jumbo frame and maximum packet length are set correctly
1807          */
1808         if (dev_data->dev_conf.rxmode.jumbo_frame == 1) {
1809                 if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
1810                     rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
1811                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1812                                 "larger than %u and smaller than %u, as jumbo "
1813                                 "frame is enabled", (uint32_t)ETHER_MAX_LEN,
1814                                         (uint32_t)I40E_FRAME_SIZE_MAX);
1815                         return I40E_ERR_CONFIG;
1816                 }
1817         } else {
1818                 if (rxq->max_pkt_len < ETHER_MIN_LEN ||
1819                     rxq->max_pkt_len > ETHER_MAX_LEN) {
1820                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1821                                 "larger than %u and smaller than %u, as jumbo "
1822                                 "frame is disabled", (uint32_t)ETHER_MIN_LEN,
1823                                                 (uint32_t)ETHER_MAX_LEN);
1824                         return I40E_ERR_CONFIG;
1825                 }
1826         }
1827
1828         if (dev_data->dev_conf.rxmode.enable_scatter ||
1829             (rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
1830                 dev_data->scattered_rx = 1;
1831         }
1832
1833         return 0;
1834 }
1835
1836 static int
1837 i40evf_rx_init(struct rte_eth_dev *dev)
1838 {
1839         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1840         uint16_t i;
1841         int ret = I40E_SUCCESS;
1842         struct i40e_rx_queue **rxq =
1843                 (struct i40e_rx_queue **)dev->data->rx_queues;
1844
1845         i40evf_config_rss(vf);
1846         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1847                 if (!rxq[i] || !rxq[i]->q_set)
1848                         continue;
1849                 ret = i40evf_rxq_init(dev, rxq[i]);
1850                 if (ret != I40E_SUCCESS)
1851                         break;
1852         }
1853         if (ret == I40E_SUCCESS)
1854                 i40e_set_rx_function(dev);
1855
1856         return ret;
1857 }
1858
1859 static void
1860 i40evf_tx_init(struct rte_eth_dev *dev)
1861 {
1862         uint16_t i;
1863         struct i40e_tx_queue **txq =
1864                 (struct i40e_tx_queue **)dev->data->tx_queues;
1865         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1866
1867         for (i = 0; i < dev->data->nb_tx_queues; i++)
1868                 txq[i]->qtx_tail = hw->hw_addr + I40E_QTX_TAIL1(i);
1869
1870         i40e_set_tx_function(dev);
1871 }
1872
1873 static inline void
1874 i40evf_enable_queues_intr(struct rte_eth_dev *dev)
1875 {
1876         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1877         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1878         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1879
1880         if (!rte_intr_allow_others(intr_handle)) {
1881                 I40E_WRITE_REG(hw,
1882                                I40E_VFINT_DYN_CTL01,
1883                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1884                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1885                                I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1886                 I40EVF_WRITE_FLUSH(hw);
1887                 return;
1888         }
1889
1890         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1891                 /* To support DPDK PF host */
1892                 I40E_WRITE_REG(hw,
1893                         I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
1894                         I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1895                         I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
1896         /* If host driver is kernel driver, do nothing.
1897          * Interrupt 0 is used for rx packets, but don't set
1898          * I40E_VFINT_DYN_CTL01,
1899          * because it is already done in i40evf_enable_irq0.
1900          */
1901
1902         I40EVF_WRITE_FLUSH(hw);
1903 }
1904
1905 static inline void
1906 i40evf_disable_queues_intr(struct rte_eth_dev *dev)
1907 {
1908         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1909         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1910         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1911
1912         if (!rte_intr_allow_others(intr_handle)) {
1913                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1914                                I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1915                 I40EVF_WRITE_FLUSH(hw);
1916                 return;
1917         }
1918
1919         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1920                 I40E_WRITE_REG(hw,
1921                                I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
1922                                                     - 1),
1923                                0);
1924         /* If host driver is kernel driver, do nothing.
1925          * Interrupt 0 is used for rx packets, but don't zero
1926          * I40E_VFINT_DYN_CTL01,
1927          * because interrupt 0 is also used for adminq processing.
1928          */
1929
1930         I40EVF_WRITE_FLUSH(hw);
1931 }
1932
1933 static int
1934 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1935 {
1936         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1937         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1938         uint16_t interval =
1939                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
1940         uint16_t msix_intr;
1941
1942         msix_intr = intr_handle->intr_vec[queue_id];
1943         if (msix_intr == I40E_MISC_VEC_ID)
1944                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1945                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1946                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1947                                (0 << I40E_VFINT_DYN_CTL01_ITR_INDX_SHIFT) |
1948                                (interval <<
1949                                 I40E_VFINT_DYN_CTL01_INTERVAL_SHIFT));
1950         else
1951                 I40E_WRITE_REG(hw,
1952                                I40E_VFINT_DYN_CTLN1(msix_intr -
1953                                                     I40E_RX_VEC_START),
1954                                I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1955                                I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
1956                                (0 << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1957                                (interval <<
1958                                 I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT));
1959
1960         I40EVF_WRITE_FLUSH(hw);
1961
1962         rte_intr_enable(&dev->pci_dev->intr_handle);
1963
1964         return 0;
1965 }
1966
1967 static int
1968 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1969 {
1970         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1971         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1972         uint16_t msix_intr;
1973
1974         msix_intr = intr_handle->intr_vec[queue_id];
1975         if (msix_intr == I40E_MISC_VEC_ID)
1976                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
1977         else
1978                 I40E_WRITE_REG(hw,
1979                                I40E_VFINT_DYN_CTLN1(msix_intr -
1980                                                     I40E_RX_VEC_START),
1981                                0);
1982
1983         I40EVF_WRITE_FLUSH(hw);
1984
1985         return 0;
1986 }
1987
1988 static void
1989 i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add)
1990 {
1991         struct i40e_virtchnl_ether_addr_list *list;
1992         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1993         int err, i, j;
1994         int next_begin = 0;
1995         int begin = 0;
1996         uint32_t len;
1997         struct ether_addr *addr;
1998         struct vf_cmd_info args;
1999
2000         do {
2001                 j = 0;
2002                 len = sizeof(struct i40e_virtchnl_ether_addr_list);
2003                 for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) {
2004                         if (is_zero_ether_addr(&dev->data->mac_addrs[i]))
2005                                 continue;
2006                         len += sizeof(struct i40e_virtchnl_ether_addr);
2007                         if (len >= I40E_AQ_BUF_SZ) {
2008                                 next_begin = i + 1;
2009                                 break;
2010                         }
2011                 }
2012
2013                 list = rte_zmalloc("i40evf_del_mac_buffer", len, 0);
2014
2015                 for (i = begin; i < next_begin; i++) {
2016                         addr = &dev->data->mac_addrs[i];
2017                         if (is_zero_ether_addr(addr))
2018                                 continue;
2019                         (void)rte_memcpy(list->list[j].addr, addr->addr_bytes,
2020                                          sizeof(addr->addr_bytes));
2021                         PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x",
2022                                     addr->addr_bytes[0], addr->addr_bytes[1],
2023                                     addr->addr_bytes[2], addr->addr_bytes[3],
2024                                     addr->addr_bytes[4], addr->addr_bytes[5]);
2025                         j++;
2026                 }
2027                 list->vsi_id = vf->vsi_res->vsi_id;
2028                 list->num_elements = j;
2029                 args.ops = add ? I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS :
2030                            I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
2031                 args.in_args = (uint8_t *)list;
2032                 args.in_args_size = len;
2033                 args.out_buffer = vf->aq_resp;
2034                 args.out_size = I40E_AQ_BUF_SZ;
2035                 err = i40evf_execute_vf_cmd(dev, &args);
2036                 if (err)
2037                         PMD_DRV_LOG(ERR, "fail to execute command %s",
2038                                     add ? "OP_ADD_ETHER_ADDRESS" :
2039                                     "OP_DEL_ETHER_ADDRESS");
2040                 rte_free(list);
2041                 begin = next_begin;
2042         } while (begin < I40E_NUM_MACADDR_MAX);
2043 }
2044
2045 static int
2046 i40evf_dev_start(struct rte_eth_dev *dev)
2047 {
2048         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2049         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2050         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2051         uint32_t intr_vector = 0;
2052
2053         PMD_INIT_FUNC_TRACE();
2054
2055         hw->adapter_stopped = 0;
2056
2057         vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
2058         vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
2059                                         dev->data->nb_tx_queues);
2060
2061         /* check and configure queue intr-vector mapping */
2062         if (dev->data->dev_conf.intr_conf.rxq != 0) {
2063                 intr_vector = dev->data->nb_rx_queues;
2064                 if (rte_intr_efd_enable(intr_handle, intr_vector))
2065                         return -1;
2066         }
2067
2068         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
2069                 intr_handle->intr_vec =
2070                         rte_zmalloc("intr_vec",
2071                                     dev->data->nb_rx_queues * sizeof(int), 0);
2072                 if (!intr_handle->intr_vec) {
2073                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2074                                      " intr_vec\n", dev->data->nb_rx_queues);
2075                         return -ENOMEM;
2076                 }
2077         }
2078
2079         if (i40evf_rx_init(dev) != 0){
2080                 PMD_DRV_LOG(ERR, "failed to do RX init");
2081                 return -1;
2082         }
2083
2084         i40evf_tx_init(dev);
2085
2086         if (i40evf_configure_queues(dev) != 0) {
2087                 PMD_DRV_LOG(ERR, "configure queues failed");
2088                 goto err_queue;
2089         }
2090         if (i40evf_config_irq_map(dev)) {
2091                 PMD_DRV_LOG(ERR, "config_irq_map failed");
2092                 goto err_queue;
2093         }
2094
2095         /* Set all mac addrs */
2096         i40evf_add_del_all_mac_addr(dev, TRUE);
2097
2098         if (i40evf_start_queues(dev) != 0) {
2099                 PMD_DRV_LOG(ERR, "enable queues failed");
2100                 goto err_mac;
2101         }
2102
2103         i40evf_enable_queues_intr(dev);
2104         return 0;
2105
2106 err_mac:
2107         i40evf_add_del_all_mac_addr(dev, FALSE);
2108 err_queue:
2109         return -1;
2110 }
2111
2112 static void
2113 i40evf_dev_stop(struct rte_eth_dev *dev)
2114 {
2115         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2116
2117         PMD_INIT_FUNC_TRACE();
2118
2119         i40evf_stop_queues(dev);
2120         i40evf_disable_queues_intr(dev);
2121         i40e_dev_clear_queues(dev);
2122
2123         /* Clean datapath event and queue/vec mapping */
2124         rte_intr_efd_disable(intr_handle);
2125         if (intr_handle->intr_vec) {
2126                 rte_free(intr_handle->intr_vec);
2127                 intr_handle->intr_vec = NULL;
2128         }
2129         /* remove all mac addrs */
2130         i40evf_add_del_all_mac_addr(dev, FALSE);
2131
2132 }
2133
2134 static int
2135 i40evf_dev_link_update(struct rte_eth_dev *dev,
2136                        __rte_unused int wait_to_complete)
2137 {
2138         struct rte_eth_link new_link;
2139         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2140         /*
2141          * DPDK pf host provide interfacet to acquire link status
2142          * while Linux driver does not
2143          */
2144         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
2145                 i40evf_get_link_status(dev, &new_link);
2146         else {
2147                 /* Linux driver PF host */
2148                 switch (vf->link_speed) {
2149                 case I40E_LINK_SPEED_100MB:
2150                         new_link.link_speed = ETH_SPEED_NUM_100M;
2151                         break;
2152                 case I40E_LINK_SPEED_1GB:
2153                         new_link.link_speed = ETH_SPEED_NUM_1G;
2154                         break;
2155                 case I40E_LINK_SPEED_10GB:
2156                         new_link.link_speed = ETH_SPEED_NUM_10G;
2157                         break;
2158                 case I40E_LINK_SPEED_20GB:
2159                         new_link.link_speed = ETH_SPEED_NUM_20G;
2160                         break;
2161                 case I40E_LINK_SPEED_40GB:
2162                         new_link.link_speed = ETH_SPEED_NUM_40G;
2163                         break;
2164                 default:
2165                         new_link.link_speed = ETH_SPEED_NUM_100M;
2166                         break;
2167                 }
2168                 /* full duplex only */
2169                 new_link.link_duplex = ETH_LINK_FULL_DUPLEX;
2170                 new_link.link_status = vf->link_up ? ETH_LINK_UP :
2171                                                      ETH_LINK_DOWN;
2172         }
2173         i40evf_dev_atomic_write_link_status(dev, &new_link);
2174
2175         return 0;
2176 }
2177
2178 static void
2179 i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev)
2180 {
2181         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2182         int ret;
2183
2184         /* If enabled, just return */
2185         if (vf->promisc_unicast_enabled)
2186                 return;
2187
2188         ret = i40evf_config_promisc(dev, 1, vf->promisc_multicast_enabled);
2189         if (ret == 0)
2190                 vf->promisc_unicast_enabled = TRUE;
2191 }
2192
2193 static void
2194 i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev)
2195 {
2196         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2197         int ret;
2198
2199         /* If disabled, just return */
2200         if (!vf->promisc_unicast_enabled)
2201                 return;
2202
2203         ret = i40evf_config_promisc(dev, 0, vf->promisc_multicast_enabled);
2204         if (ret == 0)
2205                 vf->promisc_unicast_enabled = FALSE;
2206 }
2207
2208 static void
2209 i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev)
2210 {
2211         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2212         int ret;
2213
2214         /* If enabled, just return */
2215         if (vf->promisc_multicast_enabled)
2216                 return;
2217
2218         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 1);
2219         if (ret == 0)
2220                 vf->promisc_multicast_enabled = TRUE;
2221 }
2222
2223 static void
2224 i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev)
2225 {
2226         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2227         int ret;
2228
2229         /* If enabled, just return */
2230         if (!vf->promisc_multicast_enabled)
2231                 return;
2232
2233         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 0);
2234         if (ret == 0)
2235                 vf->promisc_multicast_enabled = FALSE;
2236 }
2237
2238 static void
2239 i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2240 {
2241         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2242
2243         memset(dev_info, 0, sizeof(*dev_info));
2244         dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
2245         dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
2246         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2247         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2248         dev_info->hash_key_size = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2249         dev_info->reta_size = ETH_RSS_RETA_SIZE_64;
2250         dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2251         dev_info->max_mac_addrs = I40E_NUM_MACADDR_MAX;
2252         dev_info->rx_offload_capa =
2253                 DEV_RX_OFFLOAD_VLAN_STRIP |
2254                 DEV_RX_OFFLOAD_QINQ_STRIP |
2255                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2256                 DEV_RX_OFFLOAD_UDP_CKSUM |
2257                 DEV_RX_OFFLOAD_TCP_CKSUM;
2258         dev_info->tx_offload_capa =
2259                 DEV_TX_OFFLOAD_VLAN_INSERT |
2260                 DEV_TX_OFFLOAD_QINQ_INSERT |
2261                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2262                 DEV_TX_OFFLOAD_UDP_CKSUM |
2263                 DEV_TX_OFFLOAD_TCP_CKSUM |
2264                 DEV_TX_OFFLOAD_SCTP_CKSUM;
2265
2266         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2267                 .rx_thresh = {
2268                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2269                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2270                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2271                 },
2272                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2273                 .rx_drop_en = 0,
2274         };
2275
2276         dev_info->default_txconf = (struct rte_eth_txconf) {
2277                 .tx_thresh = {
2278                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2279                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2280                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2281                 },
2282                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2283                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2284                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2285                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2286         };
2287
2288         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2289                 .nb_max = I40E_MAX_RING_DESC,
2290                 .nb_min = I40E_MIN_RING_DESC,
2291                 .nb_align = I40E_ALIGN_RING_DESC,
2292         };
2293
2294         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2295                 .nb_max = I40E_MAX_RING_DESC,
2296                 .nb_min = I40E_MIN_RING_DESC,
2297                 .nb_align = I40E_ALIGN_RING_DESC,
2298         };
2299 }
2300
2301 static void
2302 i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2303 {
2304         if (i40evf_get_statics(dev, stats))
2305                 PMD_DRV_LOG(ERR, "Get statics failed");
2306 }
2307
2308 static void
2309 i40evf_dev_close(struct rte_eth_dev *dev)
2310 {
2311         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2312         struct rte_pci_device *pci_dev = dev->pci_dev;
2313
2314         i40evf_dev_stop(dev);
2315         hw->adapter_stopped = 1;
2316         i40e_dev_free_queues(dev);
2317         i40evf_reset_vf(hw);
2318         i40e_shutdown_adminq(hw);
2319         /* disable uio intr before callback unregister */
2320         rte_intr_disable(&pci_dev->intr_handle);
2321
2322         /* unregister callback func from eal lib */
2323         rte_intr_callback_unregister(&pci_dev->intr_handle,
2324                 i40evf_dev_interrupt_handler, (void *)dev);
2325         i40evf_disable_irq0(hw);
2326 }
2327
2328 static int
2329 i40evf_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2330 {
2331         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2332         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2333         int ret;
2334
2335         if (!lut)
2336                 return -EINVAL;
2337
2338         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2339                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, FALSE,
2340                                           lut, lut_size);
2341                 if (ret) {
2342                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
2343                         return ret;
2344                 }
2345         } else {
2346                 uint32_t *lut_dw = (uint32_t *)lut;
2347                 uint16_t i, lut_size_dw = lut_size / 4;
2348
2349                 for (i = 0; i < lut_size_dw; i++)
2350                         lut_dw[i] = I40E_READ_REG(hw, I40E_VFQF_HLUT(i));
2351         }
2352
2353         return 0;
2354 }
2355
2356 static int
2357 i40evf_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2358 {
2359         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2360         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2361         int ret;
2362
2363         if (!vsi || !lut)
2364                 return -EINVAL;
2365
2366         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2367                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, FALSE,
2368                                           lut, lut_size);
2369                 if (ret) {
2370                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
2371                         return ret;
2372                 }
2373         } else {
2374                 uint32_t *lut_dw = (uint32_t *)lut;
2375                 uint16_t i, lut_size_dw = lut_size / 4;
2376
2377                 for (i = 0; i < lut_size_dw; i++)
2378                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i), lut_dw[i]);
2379                 I40EVF_WRITE_FLUSH(hw);
2380         }
2381
2382         return 0;
2383 }
2384
2385 static int
2386 i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
2387                            struct rte_eth_rss_reta_entry64 *reta_conf,
2388                            uint16_t reta_size)
2389 {
2390         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2391         uint8_t *lut;
2392         uint16_t i, idx, shift;
2393         int ret;
2394
2395         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2396                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2397                         "(%d) doesn't match the number of hardware can "
2398                         "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2399                 return -EINVAL;
2400         }
2401
2402         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2403         if (!lut) {
2404                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2405                 return -ENOMEM;
2406         }
2407         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2408         if (ret)
2409                 goto out;
2410         for (i = 0; i < reta_size; i++) {
2411                 idx = i / RTE_RETA_GROUP_SIZE;
2412                 shift = i % RTE_RETA_GROUP_SIZE;
2413                 if (reta_conf[idx].mask & (1ULL << shift))
2414                         lut[i] = reta_conf[idx].reta[shift];
2415         }
2416         ret = i40evf_set_rss_lut(&vf->vsi, lut, reta_size);
2417
2418 out:
2419         rte_free(lut);
2420
2421         return ret;
2422 }
2423
2424 static int
2425 i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
2426                           struct rte_eth_rss_reta_entry64 *reta_conf,
2427                           uint16_t reta_size)
2428 {
2429         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2430         uint16_t i, idx, shift;
2431         uint8_t *lut;
2432         int ret;
2433
2434         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2435                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2436                         "(%d) doesn't match the number of hardware can "
2437                         "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2438                 return -EINVAL;
2439         }
2440
2441         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2442         if (!lut) {
2443                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2444                 return -ENOMEM;
2445         }
2446
2447         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2448         if (ret)
2449                 goto out;
2450         for (i = 0; i < reta_size; i++) {
2451                 idx = i / RTE_RETA_GROUP_SIZE;
2452                 shift = i % RTE_RETA_GROUP_SIZE;
2453                 if (reta_conf[idx].mask & (1ULL << shift))
2454                         reta_conf[idx].reta[shift] = lut[i];
2455         }
2456
2457 out:
2458         rte_free(lut);
2459
2460         return ret;
2461 }
2462
2463 static int
2464 i40evf_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
2465 {
2466         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2467         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2468         int ret = 0;
2469
2470         if (!key || key_len == 0) {
2471                 PMD_DRV_LOG(DEBUG, "No key to be configured");
2472                 return 0;
2473         } else if (key_len != (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2474                 sizeof(uint32_t)) {
2475                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
2476                 return -EINVAL;
2477         }
2478
2479         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2480                 struct i40e_aqc_get_set_rss_key_data *key_dw =
2481                         (struct i40e_aqc_get_set_rss_key_data *)key;
2482
2483                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
2484                 if (ret)
2485                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
2486                                      "via AQ");
2487         } else {
2488                 uint32_t *hash_key = (uint32_t *)key;
2489                 uint16_t i;
2490
2491                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2492                         i40e_write_rx_ctl(hw, I40E_VFQF_HKEY(i), hash_key[i]);
2493                 I40EVF_WRITE_FLUSH(hw);
2494         }
2495
2496         return ret;
2497 }
2498
2499 static int
2500 i40evf_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
2501 {
2502         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2503         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2504         int ret;
2505
2506         if (!key || !key_len)
2507                 return -EINVAL;
2508
2509         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2510                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
2511                         (struct i40e_aqc_get_set_rss_key_data *)key);
2512                 if (ret) {
2513                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
2514                         return ret;
2515                 }
2516         } else {
2517                 uint32_t *key_dw = (uint32_t *)key;
2518                 uint16_t i;
2519
2520                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2521                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_VFQF_HKEY(i));
2522         }
2523         *key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2524
2525         return 0;
2526 }
2527
2528 static int
2529 i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf)
2530 {
2531         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2532         uint64_t rss_hf, hena;
2533         int ret;
2534
2535         ret = i40evf_set_rss_key(&vf->vsi, rss_conf->rss_key,
2536                                  rss_conf->rss_key_len);
2537         if (ret)
2538                 return ret;
2539
2540         rss_hf = rss_conf->rss_hf;
2541         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2542         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2543         hena &= ~I40E_RSS_HENA_ALL;
2544         hena |= i40e_config_hena(rss_hf);
2545         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2546         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2547         I40EVF_WRITE_FLUSH(hw);
2548
2549         return 0;
2550 }
2551
2552 static void
2553 i40evf_disable_rss(struct i40e_vf *vf)
2554 {
2555         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2556         uint64_t hena;
2557
2558         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2559         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2560         hena &= ~I40E_RSS_HENA_ALL;
2561         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2562         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2563         I40EVF_WRITE_FLUSH(hw);
2564 }
2565
2566 static int
2567 i40evf_config_rss(struct i40e_vf *vf)
2568 {
2569         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2570         struct rte_eth_rss_conf rss_conf;
2571         uint32_t i, j, lut = 0, nb_q = (I40E_VFQF_HLUT_MAX_INDEX + 1) * 4;
2572         uint16_t num;
2573
2574         if (vf->dev_data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
2575                 i40evf_disable_rss(vf);
2576                 PMD_DRV_LOG(DEBUG, "RSS not configured\n");
2577                 return 0;
2578         }
2579
2580         num = RTE_MIN(vf->dev_data->nb_rx_queues, I40E_MAX_QP_NUM_PER_VF);
2581         /* Fill out the look up table */
2582         for (i = 0, j = 0; i < nb_q; i++, j++) {
2583                 if (j >= num)
2584                         j = 0;
2585                 lut = (lut << 8) | j;
2586                 if ((i & 3) == 3)
2587                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i >> 2), lut);
2588         }
2589
2590         rss_conf = vf->dev_data->dev_conf.rx_adv_conf.rss_conf;
2591         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
2592                 i40evf_disable_rss(vf);
2593                 PMD_DRV_LOG(DEBUG, "No hash flag is set\n");
2594                 return 0;
2595         }
2596
2597         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
2598                 (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
2599                 /* Calculate the default hash key */
2600                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2601                         rss_key_default[i] = (uint32_t)rte_rand();
2602                 rss_conf.rss_key = (uint8_t *)rss_key_default;
2603                 rss_conf.rss_key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2604                         sizeof(uint32_t);
2605         }
2606
2607         return i40evf_hw_rss_hash_set(vf, &rss_conf);
2608 }
2609
2610 static int
2611 i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
2612                            struct rte_eth_rss_conf *rss_conf)
2613 {
2614         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2615         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2616         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
2617         uint64_t hena;
2618
2619         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2620         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2621         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
2622                 if (rss_hf != 0) /* Enable RSS */
2623                         return -EINVAL;
2624                 return 0;
2625         }
2626
2627         /* RSS enabled */
2628         if (rss_hf == 0) /* Disable RSS */
2629                 return -EINVAL;
2630
2631         return i40evf_hw_rss_hash_set(vf, rss_conf);
2632 }
2633
2634 static int
2635 i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2636                              struct rte_eth_rss_conf *rss_conf)
2637 {
2638         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2639         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2640         uint64_t hena;
2641
2642         i40evf_get_rss_key(&vf->vsi, rss_conf->rss_key,
2643                            &rss_conf->rss_key_len);
2644
2645         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2646         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2647         rss_conf->rss_hf = i40e_parse_hena(hena);
2648
2649         return 0;
2650 }