drivers/net/i40e/i40e_ethdev_vf.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_eal.h>
  55 #include <rte_alarm.h>
  56 #include <rte_ether.h>
  57 #include <rte_ethdev.h>
  58 #include <rte_atomic.h>
  59 #include <rte_malloc.h>
  60 #include <rte_dev.h>
  61
  62 #include "i40e_logs.h"
  63 #include "base/i40e_prototype.h"
  64 #include "base/i40e_adminq_cmd.h"
  65 #include "base/i40e_type.h"
  66
  67 #include "i40e_rxtx.h"
  68 #include "i40e_ethdev.h"
  69 #include "i40e_pf.h"
  70 #define I40EVF_VSI_DEFAULT_MSIX_INTR     1
  71 #define I40EVF_VSI_DEFAULT_MSIX_INTR_LNX 0
  72
  73 /* busy wait delay in msec */
  74 #define I40EVF_BUSY_WAIT_DELAY 10
  75 #define I40EVF_BUSY_WAIT_COUNT 50
  76 #define MAX_RESET_WAIT_CNT     20
  77 /*ITR index for NOITR*/
  78 #define I40E_QINT_RQCTL_MSIX_INDX_NOITR     3
  79
  80 struct i40evf_arq_msg_info {
  81         enum i40e_virtchnl_ops ops;
  82         enum i40e_status_code result;
  83         uint16_t buf_len;
  84         uint16_t msg_len;
  85         uint8_t *msg;
  86 };
  87
  88 struct vf_cmd_info {
  89         enum i40e_virtchnl_ops ops;
  90         uint8_t *in_args;
  91         uint32_t in_args_size;
  92         uint8_t *out_buffer;
  93         /* Input & output type. pass in buffer size and pass out
  94          * actual return result
  95          */
  96         uint32_t out_size;
  97 };
  98
  99 enum i40evf_aq_result {
 100         I40EVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
 101         I40EVF_MSG_NON,      /* Read nothing from admin queue */
 102         I40EVF_MSG_SYS,      /* Read system msg from admin queue */
 103         I40EVF_MSG_CMD,      /* Read async command result */
 104 };
 105
 106 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 107 static int i40evf_dev_start(struct rte_eth_dev *dev);
 108 static void i40evf_dev_stop(struct rte_eth_dev *dev);
 109 static void i40evf_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int i40evf_dev_link_update(struct rte_eth_dev *dev,
 112                                   __rte_unused int wait_to_complete);
 113 static void i40evf_dev_stats_get(struct rte_eth_dev *dev,
 114                                 struct rte_eth_stats *stats);
 115 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
 116                                  struct rte_eth_xstats *xstats, unsigned n);
 117 static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev);
 118 static int i40evf_vlan_filter_set(struct rte_eth_dev *dev,
 119                                   uint16_t vlan_id, int on);
 120 static void i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 121 static int i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid,
 122                                 int on);
 123 static void i40evf_dev_close(struct rte_eth_dev *dev);
 124 static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev);
 125 static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev);
 126 static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev);
 127 static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev);
 128 static int i40evf_get_link_status(struct rte_eth_dev *dev,
 129                                   struct rte_eth_link *link);
 130 static int i40evf_init_vlan(struct rte_eth_dev *dev);
 131 static int i40evf_dev_rx_queue_start(struct rte_eth_dev *dev,
 132                                      uint16_t rx_queue_id);
 133 static int i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev,
 134                                     uint16_t rx_queue_id);
 135 static int i40evf_dev_tx_queue_start(struct rte_eth_dev *dev,
 136                                      uint16_t tx_queue_id);
 137 static int i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev,
 138                                     uint16_t tx_queue_id);
 139 static void i40evf_add_mac_addr(struct rte_eth_dev *dev,
 140                                 struct ether_addr *addr,
 141                                 uint32_t index,
 142                                 uint32_t pool);
 143 static void i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 144 static int i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
 145                         struct rte_eth_rss_reta_entry64 *reta_conf,
 146                         uint16_t reta_size);
 147 static int i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
 148                         struct rte_eth_rss_reta_entry64 *reta_conf,
 149                         uint16_t reta_size);
 150 static int i40evf_config_rss(struct i40e_vf *vf);
 151 static int i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
 152                                       struct rte_eth_rss_conf *rss_conf);
 153 static int i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 154                                         struct rte_eth_rss_conf *rss_conf);
 155 static int
 156 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 157 static int
 158 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 159
 160 /* Default hash key buffer for RSS */
 161 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
 162
 163 struct rte_i40evf_xstats_name_off {
 164         char name[RTE_ETH_XSTATS_NAME_SIZE];
 165         unsigned offset;
 166 };
 167
 168 static const struct rte_i40evf_xstats_name_off rte_i40evf_stats_strings[] = {
 169         {"rx_bytes", offsetof(struct i40e_eth_stats, rx_bytes)},
 170         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 171         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 172         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 173         {"rx_dropped_packets", offsetof(struct i40e_eth_stats, rx_discards)},
 174         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 175                 rx_unknown_protocol)},
 176         {"tx_bytes", offsetof(struct i40e_eth_stats, tx_bytes)},
 177         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 178         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 179         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 180         {"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 181         {"tx_error_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
 182 };
 183
 184 #define I40EVF_NB_XSTATS (sizeof(rte_i40evf_stats_strings) / \
 185                 sizeof(rte_i40evf_stats_strings[0]))
 186
 187 static const struct eth_dev_ops i40evf_eth_dev_ops = {
 188         .dev_configure        = i40evf_dev_configure,
 189         .dev_start            = i40evf_dev_start,
 190         .dev_stop             = i40evf_dev_stop,
 191         .promiscuous_enable   = i40evf_dev_promiscuous_enable,
 192         .promiscuous_disable  = i40evf_dev_promiscuous_disable,
 193         .allmulticast_enable  = i40evf_dev_allmulticast_enable,
 194         .allmulticast_disable = i40evf_dev_allmulticast_disable,
 195         .link_update          = i40evf_dev_link_update,
 196         .stats_get            = i40evf_dev_stats_get,
 197         .xstats_get           = i40evf_dev_xstats_get,
 198         .xstats_reset         = i40evf_dev_xstats_reset,
 199         .dev_close            = i40evf_dev_close,
 200         .dev_infos_get        = i40evf_dev_info_get,
 201         .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get,
 202         .vlan_filter_set      = i40evf_vlan_filter_set,
 203         .vlan_offload_set     = i40evf_vlan_offload_set,
 204         .vlan_pvid_set        = i40evf_vlan_pvid_set,
 205         .rx_queue_start       = i40evf_dev_rx_queue_start,
 206         .rx_queue_stop        = i40evf_dev_rx_queue_stop,
 207         .tx_queue_start       = i40evf_dev_tx_queue_start,
 208         .tx_queue_stop        = i40evf_dev_tx_queue_stop,
 209         .rx_queue_setup       = i40e_dev_rx_queue_setup,
 210         .rx_queue_release     = i40e_dev_rx_queue_release,
 211         .rx_queue_intr_enable = i40evf_dev_rx_queue_intr_enable,
 212         .rx_queue_intr_disable = i40evf_dev_rx_queue_intr_disable,
 213         .rx_descriptor_done   = i40e_dev_rx_descriptor_done,
 214         .tx_queue_setup       = i40e_dev_tx_queue_setup,
 215         .tx_queue_release     = i40e_dev_tx_queue_release,
 216         .mac_addr_add         = i40evf_add_mac_addr,
 217         .mac_addr_remove      = i40evf_del_mac_addr,
 218         .reta_update          = i40evf_dev_rss_reta_update,
 219         .reta_query           = i40evf_dev_rss_reta_query,
 220         .rss_hash_update      = i40evf_dev_rss_hash_update,
 221         .rss_hash_conf_get    = i40evf_dev_rss_hash_conf_get,
 222 };
 223
 224 /*
 225  * Read data in admin queue to get msg from pf driver
 226  */
 227 static enum i40evf_aq_result
 228 i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 229 {
 230         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 231         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 232         struct i40e_arq_event_info event;
 233         enum i40e_virtchnl_ops opcode;
 234         enum i40e_status_code retval;
 235         int ret;
 236         enum i40evf_aq_result result = I40EVF_MSG_NON;
 237
 238         event.buf_len = data->buf_len;
 239         event.msg_buf = data->msg;
 240         ret = i40e_clean_arq_element(hw, &event, NULL);
 241         /* Can't read any msg from adminQ */
 242         if (ret) {
 243                 if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
 244                         result = I40EVF_MSG_ERR;
 245                 return result;
 246         }
 247
 248         opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
 249         retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 250         /* pf sys event */
 251         if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 252                 struct i40e_virtchnl_pf_event *vpe =
 253                         (struct i40e_virtchnl_pf_event *)event.msg_buf;
 254
 255                 result = I40EVF_MSG_SYS;
 256                 switch (vpe->event) {
 257                 case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 258                         vf->link_up =
 259                                 vpe->event_data.link_event.link_status;
 260                         vf->pend_msg |= PFMSG_LINK_CHANGE;
 261                         PMD_DRV_LOG(INFO, "Link status update:%s",
 262                                     vf->link_up ? "up" : "down");
 263                         break;
 264                 case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
 265                         vf->vf_reset = true;
 266                         vf->pend_msg |= PFMSG_RESET_IMPENDING;
 267                         PMD_DRV_LOG(INFO, "vf is reseting");
 268                         break;
 269                 case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
 270                         vf->dev_closed = true;
 271                         vf->pend_msg |= PFMSG_DRIVER_CLOSE;
 272                         PMD_DRV_LOG(INFO, "PF driver closed");
 273                         break;
 274                 default:
 275                         PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
 276                                     __func__, vpe->event);
 277                 }
 278         } else {
 279                 /* async reply msg on command issued by vf previously */
 280                 result = I40EVF_MSG_CMD;
 281                 /* Actual data length read from PF */
 282                 data->msg_len = event.msg_len;
 283         }
 284
 285         data->result = retval;
 286         data->ops = opcode;
 287
 288         return result;
 289 }
 290
 291 /**
 292  * clear current command. Only call in case execute
 293  * _atomic_set_cmd successfully.
 294  */
 295 static inline void
 296 _clear_cmd(struct i40e_vf *vf)
 297 {
 298         rte_wmb();
 299         vf->pend_cmd = I40E_VIRTCHNL_OP_UNKNOWN;
 300 }
 301
 302 /*
 303  * Check there is pending cmd in execution. If none, set new command.
 304  */
 305 static inline int
 306 _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 307 {
 308         int ret = rte_atomic32_cmpset(&vf->pend_cmd,
 309                         I40E_VIRTCHNL_OP_UNKNOWN, ops);
 310
 311         if (!ret)
 312                 PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd);
 313
 314         return !ret;
 315 }
 316
 317 #define MAX_TRY_TIMES 200
 318 #define ASQ_DELAY_MS  10
 319
 320 static int
 321 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 322 {
 323         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 324         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 325         struct i40evf_arq_msg_info info;
 326         enum i40evf_aq_result ret;
 327         int err = -1;
 328         int i = 0;
 329
 330         if (_atomic_set_cmd(vf, args->ops))
 331                 return -1;
 332
 333         info.msg = args->out_buffer;
 334         info.buf_len = args->out_size;
 335         info.ops = I40E_VIRTCHNL_OP_UNKNOWN;
 336         info.result = I40E_SUCCESS;
 337
 338         err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS,
 339                      args->in_args, args->in_args_size, NULL);
 340         if (err) {
 341                 PMD_DRV_LOG(ERR, "fail to send cmd %d", args->ops);
 342                 _clear_cmd(vf);
 343                 return err;
 344         }
 345
 346         do {
 347                 ret = i40evf_read_pfmsg(dev, &info);
 348                 if (ret == I40EVF_MSG_CMD) {
 349                         err = 0;
 350                         break;
 351                 } else if (ret == I40EVF_MSG_ERR) {
 352                         err = -1;
 353                         break;
 354                 }
 355                 rte_delay_ms(ASQ_DELAY_MS);
 356                 /* If don't read msg or read sys event, continue */
 357         } while (i++ < MAX_TRY_TIMES);
 358         _clear_cmd(vf);
 359
 360         return err | vf->cmd_retval;
 361 }
 362
 363 /*
 364  * Check API version with sync wait until version read or fail from admin queue
 365  */
 366 static int
 367 i40evf_check_api_version(struct rte_eth_dev *dev)
 368 {
 369         struct i40e_virtchnl_version_info version, *pver;
 370         int err;
 371         struct vf_cmd_info args;
 372         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 373
 374         version.major = I40E_VIRTCHNL_VERSION_MAJOR;
 375         version.minor = I40E_VIRTCHNL_VERSION_MINOR;
 376
 377         args.ops = I40E_VIRTCHNL_OP_VERSION;
 378         args.in_args = (uint8_t *)&version;
 379         args.in_args_size = sizeof(version);
 380         args.out_buffer = vf->aq_resp;
 381         args.out_size = I40E_AQ_BUF_SZ;
 382
 383         err = i40evf_execute_vf_cmd(dev, &args);
 384         if (err) {
 385                 PMD_INIT_LOG(ERR, "fail to execute command OP_VERSION");
 386                 return err;
 387         }
 388
 389         pver = (struct i40e_virtchnl_version_info *)args.out_buffer;
 390         vf->version_major = pver->major;
 391         vf->version_minor = pver->minor;
 392         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 393                 PMD_DRV_LOG(INFO, "Peer is DPDK PF host");
 394         else if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) &&
 395                 (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR))
 396                 PMD_DRV_LOG(INFO, "Peer is Linux PF host");
 397         else {
 398                 PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)",
 399                                         vf->version_major, vf->version_minor,
 400                                                 I40E_VIRTCHNL_VERSION_MAJOR,
 401                                                 I40E_VIRTCHNL_VERSION_MINOR);
 402                 return -1;
 403         }
 404
 405         return 0;
 406 }
 407
 408 static int
 409 i40evf_get_vf_resource(struct rte_eth_dev *dev)
 410 {
 411         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 412         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 413         int err;
 414         struct vf_cmd_info args;
 415         uint32_t caps, len;
 416
 417         args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
 418         args.out_buffer = vf->aq_resp;
 419         args.out_size = I40E_AQ_BUF_SZ;
 420         if (PF_IS_V11(vf)) {
 421                 caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
 422                        I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ |
 423                        I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
 424                        I40E_VIRTCHNL_VF_OFFLOAD_VLAN |
 425                        I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING;
 426                 args.in_args = (uint8_t *)&caps;
 427                 args.in_args_size = sizeof(caps);
 428         } else {
 429                 args.in_args = NULL;
 430                 args.in_args_size = 0;
 431         }
 432         err = i40evf_execute_vf_cmd(dev, &args);
 433
 434         if (err) {
 435                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_VF_RESOURCE");
 436                 return err;
 437         }
 438
 439         len =  sizeof(struct i40e_virtchnl_vf_resource) +
 440                 I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource);
 441
 442         (void)rte_memcpy(vf->vf_res, args.out_buffer,
 443                         RTE_MIN(args.out_size, len));
 444         i40e_vf_parse_hw_config(hw, vf->vf_res);
 445
 446         return 0;
 447 }
 448
 449 static int
 450 i40evf_config_promisc(struct rte_eth_dev *dev,
 451                       bool enable_unicast,
 452                       bool enable_multicast)
 453 {
 454         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 455         int err;
 456         struct vf_cmd_info args;
 457         struct i40e_virtchnl_promisc_info promisc;
 458
 459         promisc.flags = 0;
 460         promisc.vsi_id = vf->vsi_res->vsi_id;
 461
 462         if (enable_unicast)
 463                 promisc.flags |= I40E_FLAG_VF_UNICAST_PROMISC;
 464
 465         if (enable_multicast)
 466                 promisc.flags |= I40E_FLAG_VF_MULTICAST_PROMISC;
 467
 468         args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 469         args.in_args = (uint8_t *)&promisc;
 470         args.in_args_size = sizeof(promisc);
 471         args.out_buffer = vf->aq_resp;
 472         args.out_size = I40E_AQ_BUF_SZ;
 473
 474         err = i40evf_execute_vf_cmd(dev, &args);
 475
 476         if (err)
 477                 PMD_DRV_LOG(ERR, "fail to execute command "
 478                             "CONFIG_PROMISCUOUS_MODE");
 479         return err;
 480 }
 481
 482 /* Configure vlan and double vlan offload. Use flag to specify which part to configure */
 483 static int
 484 i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 485                                 bool enable_vlan_strip)
 486 {
 487         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 488         int err;
 489         struct vf_cmd_info args;
 490         struct i40e_virtchnl_vlan_offload_info offload;
 491
 492         offload.vsi_id = vf->vsi_res->vsi_id;
 493         offload.enable_vlan_strip = enable_vlan_strip;
 494
 495         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 496         args.in_args = (uint8_t *)&offload;
 497         args.in_args_size = sizeof(offload);
 498         args.out_buffer = vf->aq_resp;
 499         args.out_size = I40E_AQ_BUF_SZ;
 500
 501         err = i40evf_execute_vf_cmd(dev, &args);
 502         if (err)
 503                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_OFFLOAD");
 504
 505         return err;
 506 }
 507
 508 static int
 509 i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 510                                 struct i40e_vsi_vlan_pvid_info *info)
 511 {
 512         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 513         int err;
 514         struct vf_cmd_info args;
 515         struct i40e_virtchnl_pvid_info tpid_info;
 516
 517         if (dev == NULL || info == NULL) {
 518                 PMD_DRV_LOG(ERR, "invalid parameters");
 519                 return I40E_ERR_PARAM;
 520         }
 521
 522         memset(&tpid_info, 0, sizeof(tpid_info));
 523         tpid_info.vsi_id = vf->vsi_res->vsi_id;
 524         (void)rte_memcpy(&tpid_info.info, info, sizeof(*info));
 525
 526         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 527         args.in_args = (uint8_t *)&tpid_info;
 528         args.in_args_size = sizeof(tpid_info);
 529         args.out_buffer = vf->aq_resp;
 530         args.out_size = I40E_AQ_BUF_SZ;
 531
 532         err = i40evf_execute_vf_cmd(dev, &args);
 533         if (err)
 534                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_PVID");
 535
 536         return err;
 537 }
 538
 539 static void
 540 i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info,
 541                                   uint16_t vsi_id,
 542                                   uint16_t queue_id,
 543                                   uint16_t nb_txq,
 544                                   struct i40e_tx_queue *txq)
 545 {
 546         txq_info->vsi_id = vsi_id;
 547         txq_info->queue_id = queue_id;
 548         if (queue_id < nb_txq) {
 549                 txq_info->ring_len = txq->nb_tx_desc;
 550                 txq_info->dma_ring_addr = txq->tx_ring_phys_addr;
 551         }
 552 }
 553
 554 static void
 555 i40evf_fill_virtchnl_vsi_rxq_info(struct i40e_virtchnl_rxq_info *rxq_info,
 556                                   uint16_t vsi_id,
 557                                   uint16_t queue_id,
 558                                   uint16_t nb_rxq,
 559                                   uint32_t max_pkt_size,
 560                                   struct i40e_rx_queue *rxq)
 561 {
 562         rxq_info->vsi_id = vsi_id;
 563         rxq_info->queue_id = queue_id;
 564         rxq_info->max_pkt_size = max_pkt_size;
 565         if (queue_id < nb_rxq) {
 566                 rxq_info->ring_len = rxq->nb_rx_desc;
 567                 rxq_info->dma_ring_addr = rxq->rx_ring_phys_addr;
 568                 rxq_info->databuffer_size =
 569                         (rte_pktmbuf_data_room_size(rxq->mp) -
 570                                 RTE_PKTMBUF_HEADROOM);
 571         }
 572 }
 573
 574 /* It configures VSI queues to co-work with Linux PF host */
 575 static int
 576 i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 577 {
 578         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 579         struct i40e_rx_queue **rxq =
 580                 (struct i40e_rx_queue **)dev->data->rx_queues;
 581         struct i40e_tx_queue **txq =
 582                 (struct i40e_tx_queue **)dev->data->tx_queues;
 583         struct i40e_virtchnl_vsi_queue_config_info *vc_vqci;
 584         struct i40e_virtchnl_queue_pair_info *vc_qpi;
 585         struct vf_cmd_info args;
 586         uint16_t i, nb_qp = vf->num_queue_pairs;
 587         const uint32_t size =
 588                 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqci, nb_qp);
 589         uint8_t buff[size];
 590         int ret;
 591
 592         memset(buff, 0, sizeof(buff));
 593         vc_vqci = (struct i40e_virtchnl_vsi_queue_config_info *)buff;
 594         vc_vqci->vsi_id = vf->vsi_res->vsi_id;
 595         vc_vqci->num_queue_pairs = nb_qp;
 596
 597         for (i = 0, vc_qpi = vc_vqci->qpair; i < nb_qp; i++, vc_qpi++) {
 598                 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpi->txq,
 599                         vc_vqci->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
 600                 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpi->rxq,
 601                         vc_vqci->vsi_id, i, dev->data->nb_rx_queues,
 602                                         vf->max_pkt_len, rxq[i]);
 603         }
 604         memset(&args, 0, sizeof(args));
 605         args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 606         args.in_args = (uint8_t *)vc_vqci;
 607         args.in_args_size = size;
 608         args.out_buffer = vf->aq_resp;
 609         args.out_size = I40E_AQ_BUF_SZ;
 610         ret = i40evf_execute_vf_cmd(dev, &args);
 611         if (ret)
 612                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 613                         "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES\n");
 614
 615         return ret;
 616 }
 617
 618 /* It configures VSI queues to co-work with DPDK PF host */
 619 static int
 620 i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
 621 {
 622         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 623         struct i40e_rx_queue **rxq =
 624                 (struct i40e_rx_queue **)dev->data->rx_queues;
 625         struct i40e_tx_queue **txq =
 626                 (struct i40e_tx_queue **)dev->data->tx_queues;
 627         struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei;
 628         struct i40e_virtchnl_queue_pair_ext_info *vc_qpei;
 629         struct vf_cmd_info args;
 630         uint16_t i, nb_qp = vf->num_queue_pairs;
 631         const uint32_t size =
 632                 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqcei, nb_qp);
 633         uint8_t buff[size];
 634         int ret;
 635
 636         memset(buff, 0, sizeof(buff));
 637         vc_vqcei = (struct i40e_virtchnl_vsi_queue_config_ext_info *)buff;
 638         vc_vqcei->vsi_id = vf->vsi_res->vsi_id;
 639         vc_vqcei->num_queue_pairs = nb_qp;
 640         vc_qpei = vc_vqcei->qpair;
 641         for (i = 0; i < nb_qp; i++, vc_qpei++) {
 642                 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpei->txq,
 643                         vc_vqcei->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
 644                 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpei->rxq,
 645                         vc_vqcei->vsi_id, i, dev->data->nb_rx_queues,
 646                                         vf->max_pkt_len, rxq[i]);
 647                 if (i < dev->data->nb_rx_queues)
 648                         /*
 649                          * It adds extra info for configuring VSI queues, which
 650                          * is needed to enable the configurable crc stripping
 651                          * in VF.
 652                          */
 653                         vc_qpei->rxq_ext.crcstrip =
 654                                 dev->data->dev_conf.rxmode.hw_strip_crc;
 655         }
 656         memset(&args, 0, sizeof(args));
 657         args.ops =
 658                 (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
 659         args.in_args = (uint8_t *)vc_vqcei;
 660         args.in_args_size = size;
 661         args.out_buffer = vf->aq_resp;
 662         args.out_size = I40E_AQ_BUF_SZ;
 663         ret = i40evf_execute_vf_cmd(dev, &args);
 664         if (ret)
 665                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 666                         "I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT\n");
 667
 668         return ret;
 669 }
 670
 671 static int
 672 i40evf_configure_queues(struct rte_eth_dev *dev)
 673 {
 674         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 675
 676         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 677                 /* To support DPDK PF host */
 678                 return i40evf_configure_vsi_queues_ext(dev);
 679         else
 680                 /* To support Linux PF host */
 681                 return i40evf_configure_vsi_queues(dev);
 682 }
 683
 684 static int
 685 i40evf_config_irq_map(struct rte_eth_dev *dev)
 686 {
 687         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 688         struct vf_cmd_info args;
 689         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_irq_map_info) + \
 690                 sizeof(struct i40e_virtchnl_vector_map)];
 691         struct i40e_virtchnl_irq_map_info *map_info;
 692         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 693         uint32_t vector_id;
 694         int i, err;
 695
 696         if (rte_intr_allow_others(intr_handle)) {
 697                 if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
 698                         vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR;
 699                 else
 700                         vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR_LNX;
 701         } else {
 702                 vector_id = I40E_MISC_VEC_ID;
 703         }
 704
 705         map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 706         map_info->num_vectors = 1;
 707         map_info->vecmap[0].rxitr_idx = I40E_QINT_RQCTL_MSIX_INDX_NOITR;
 708         map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 709         /* Alway use default dynamic MSIX interrupt */
 710         map_info->vecmap[0].vector_id = vector_id;
 711         /* Don't map any tx queue */
 712         map_info->vecmap[0].txq_map = 0;
 713         map_info->vecmap[0].rxq_map = 0;
 714         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 715                 map_info->vecmap[0].rxq_map |= 1 << i;
 716                 if (rte_intr_dp_is_en(intr_handle))
 717                         intr_handle->intr_vec[i] = vector_id;
 718         }
 719
 720         args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 721         args.in_args = (u8 *)cmd_buffer;
 722         args.in_args_size = sizeof(cmd_buffer);
 723         args.out_buffer = vf->aq_resp;
 724         args.out_size = I40E_AQ_BUF_SZ;
 725         err = i40evf_execute_vf_cmd(dev, &args);
 726         if (err)
 727                 PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES");
 728
 729         return err;
 730 }
 731
 732 static int
 733 i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 734                                 bool on)
 735 {
 736         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 737         struct i40e_virtchnl_queue_select queue_select;
 738         int err;
 739         struct vf_cmd_info args;
 740         memset(&queue_select, 0, sizeof(queue_select));
 741         queue_select.vsi_id = vf->vsi_res->vsi_id;
 742
 743         if (isrx)
 744                 queue_select.rx_queues |= 1 << qid;
 745         else
 746                 queue_select.tx_queues |= 1 << qid;
 747
 748         if (on)
 749                 args.ops = I40E_VIRTCHNL_OP_ENABLE_QUEUES;
 750         else
 751                 args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 752         args.in_args = (u8 *)&queue_select;
 753         args.in_args_size = sizeof(queue_select);
 754         args.out_buffer = vf->aq_resp;
 755         args.out_size = I40E_AQ_BUF_SZ;
 756         err = i40evf_execute_vf_cmd(dev, &args);
 757         if (err)
 758                 PMD_DRV_LOG(ERR, "fail to switch %s %u %s",
 759                             isrx ? "RX" : "TX", qid, on ? "on" : "off");
 760
 761         return err;
 762 }
 763
 764 static int
 765 i40evf_start_queues(struct rte_eth_dev *dev)
 766 {
 767         struct rte_eth_dev_data *dev_data = dev->data;
 768         int i;
 769         struct i40e_rx_queue *rxq;
 770         struct i40e_tx_queue *txq;
 771
 772         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 773                 rxq = dev_data->rx_queues[i];
 774                 if (rxq->rx_deferred_start)
 775                         continue;
 776                 if (i40evf_dev_rx_queue_start(dev, i) != 0) {
 777                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 778                         return -1;
 779                 }
 780         }
 781
 782         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 783                 txq = dev_data->tx_queues[i];
 784                 if (txq->tx_deferred_start)
 785                         continue;
 786                 if (i40evf_dev_tx_queue_start(dev, i) != 0) {
 787                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 788                         return -1;
 789                 }
 790         }
 791
 792         return 0;
 793 }
 794
 795 static int
 796 i40evf_stop_queues(struct rte_eth_dev *dev)
 797 {
 798         int i;
 799
 800         /* Stop TX queues first */
 801         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 802                 if (i40evf_dev_tx_queue_stop(dev, i) != 0) {
 803                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 804                         return -1;
 805                 }
 806         }
 807
 808         /* Then stop RX queues */
 809         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 810                 if (i40evf_dev_rx_queue_stop(dev, i) != 0) {
 811                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 812                         return -1;
 813                 }
 814         }
 815
 816         return 0;
 817 }
 818
 819 static void
 820 i40evf_add_mac_addr(struct rte_eth_dev *dev,
 821                     struct ether_addr *addr,
 822                     __rte_unused uint32_t index,
 823                     __rte_unused uint32_t pool)
 824 {
 825         struct i40e_virtchnl_ether_addr_list *list;
 826         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 827         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 828                         sizeof(struct i40e_virtchnl_ether_addr)];
 829         int err;
 830         struct vf_cmd_info args;
 831
 832         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 833                 PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x",
 834                             addr->addr_bytes[0], addr->addr_bytes[1],
 835                             addr->addr_bytes[2], addr->addr_bytes[3],
 836                             addr->addr_bytes[4], addr->addr_bytes[5]);
 837                 return;
 838         }
 839
 840         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 841         list->vsi_id = vf->vsi_res->vsi_id;
 842         list->num_elements = 1;
 843         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 844                                         sizeof(addr->addr_bytes));
 845
 846         args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 847         args.in_args = cmd_buffer;
 848         args.in_args_size = sizeof(cmd_buffer);
 849         args.out_buffer = vf->aq_resp;
 850         args.out_size = I40E_AQ_BUF_SZ;
 851         err = i40evf_execute_vf_cmd(dev, &args);
 852         if (err)
 853                 PMD_DRV_LOG(ERR, "fail to execute command "
 854                             "OP_ADD_ETHER_ADDRESS");
 855
 856         return;
 857 }
 858
 859 static void
 860 i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index)
 861 {
 862         struct i40e_virtchnl_ether_addr_list *list;
 863         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 864         struct rte_eth_dev_data *data = dev->data;
 865         struct ether_addr *addr;
 866         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 867                         sizeof(struct i40e_virtchnl_ether_addr)];
 868         int err;
 869         struct vf_cmd_info args;
 870
 871         addr = &(data->mac_addrs[index]);
 872
 873         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 874                 PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x",
 875                             addr->addr_bytes[0], addr->addr_bytes[1],
 876                             addr->addr_bytes[2], addr->addr_bytes[3],
 877                             addr->addr_bytes[4], addr->addr_bytes[5]);
 878                 return;
 879         }
 880
 881         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 882         list->vsi_id = vf->vsi_res->vsi_id;
 883         list->num_elements = 1;
 884         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 885                         sizeof(addr->addr_bytes));
 886
 887         args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 888         args.in_args = cmd_buffer;
 889         args.in_args_size = sizeof(cmd_buffer);
 890         args.out_buffer = vf->aq_resp;
 891         args.out_size = I40E_AQ_BUF_SZ;
 892         err = i40evf_execute_vf_cmd(dev, &args);
 893         if (err)
 894                 PMD_DRV_LOG(ERR, "fail to execute command "
 895                             "OP_DEL_ETHER_ADDRESS");
 896         return;
 897 }
 898
 899 static int
 900 i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 901 {
 902         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 903         struct i40e_virtchnl_queue_select q_stats;
 904         int err;
 905         struct vf_cmd_info args;
 906
 907         memset(&q_stats, 0, sizeof(q_stats));
 908         q_stats.vsi_id = vf->vsi_res->vsi_id;
 909         args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 910         args.in_args = (u8 *)&q_stats;
 911         args.in_args_size = sizeof(q_stats);
 912         args.out_buffer = vf->aq_resp;
 913         args.out_size = I40E_AQ_BUF_SZ;
 914
 915         err = i40evf_execute_vf_cmd(dev, &args);
 916         if (err) {
 917                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS");
 918                 *pstats = NULL;
 919                 return err;
 920         }
 921         *pstats = (struct i40e_eth_stats *)args.out_buffer;
 922         return 0;
 923 }
 924
 925 static int
 926 i40evf_get_statics(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 927 {
 928         int ret;
 929         struct i40e_eth_stats *pstats = NULL;
 930
 931         ret = i40evf_update_stats(dev, &pstats);
 932         if (ret != 0)
 933                 return 0;
 934
 935         stats->ipackets = pstats->rx_unicast + pstats->rx_multicast +
 936                                                 pstats->rx_broadcast;
 937         stats->opackets = pstats->tx_broadcast + pstats->tx_multicast +
 938                                                 pstats->tx_unicast;
 939         stats->ierrors = pstats->rx_discards;
 940         stats->oerrors = pstats->tx_errors + pstats->tx_discards;
 941         stats->ibytes = pstats->rx_bytes;
 942         stats->obytes = pstats->tx_bytes;
 943
 944         return 0;
 945 }
 946
 947 static void
 948 i40evf_dev_xstats_reset(struct rte_eth_dev *dev)
 949 {
 950         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 951         struct i40e_eth_stats *pstats = NULL;
 952
 953         /* read stat values to clear hardware registers */
 954         i40evf_update_stats(dev, &pstats);
 955
 956         /* set stats offset base on current values */
 957         vf->vsi.eth_stats_offset = vf->vsi.eth_stats;
 958 }
 959
 960 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
 961                                  struct rte_eth_xstats *xstats, unsigned n)
 962 {
 963         int ret;
 964         unsigned i;
 965         struct i40e_eth_stats *pstats = NULL;
 966
 967         if (n < I40EVF_NB_XSTATS)
 968                 return I40EVF_NB_XSTATS;
 969
 970         ret = i40evf_update_stats(dev, &pstats);
 971         if (ret != 0)
 972                 return 0;
 973
 974         if (!xstats)
 975                 return 0;
 976
 977         /* loop over xstats array and values from pstats */
 978         for (i = 0; i < I40EVF_NB_XSTATS; i++) {
 979                 snprintf(xstats[i].name, sizeof(xstats[i].name),
 980                          "%s", rte_i40evf_stats_strings[i].name);
 981                 xstats[i].value = *(uint64_t *)(((char *)pstats) +
 982                         rte_i40evf_stats_strings[i].offset);
 983         }
 984
 985         return I40EVF_NB_XSTATS;
 986 }
 987
 988 static int
 989 i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 990 {
 991         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 992         struct i40e_virtchnl_vlan_filter_list *vlan_list;
 993         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
 994                                                         sizeof(uint16_t)];
 995         int err;
 996         struct vf_cmd_info args;
 997
 998         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
 999         vlan_list->vsi_id = vf->vsi_res->vsi_id;
1000         vlan_list->num_elements = 1;
1001         vlan_list->vlan_id[0] = vlanid;
1002
1003         args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
1004         args.in_args = (u8 *)&cmd_buffer;
1005         args.in_args_size = sizeof(cmd_buffer);
1006         args.out_buffer = vf->aq_resp;
1007         args.out_size = I40E_AQ_BUF_SZ;
1008         err = i40evf_execute_vf_cmd(dev, &args);
1009         if (err)
1010                 PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_VLAN");
1011
1012         return err;
1013 }
1014
1015 static int
1016 i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1017 {
1018         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1019         struct i40e_virtchnl_vlan_filter_list *vlan_list;
1020         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
1021                                                         sizeof(uint16_t)];
1022         int err;
1023         struct vf_cmd_info args;
1024
1025         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
1026         vlan_list->vsi_id = vf->vsi_res->vsi_id;
1027         vlan_list->num_elements = 1;
1028         vlan_list->vlan_id[0] = vlanid;
1029
1030         args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
1031         args.in_args = (u8 *)&cmd_buffer;
1032         args.in_args_size = sizeof(cmd_buffer);
1033         args.out_buffer = vf->aq_resp;
1034         args.out_size = I40E_AQ_BUF_SZ;
1035         err = i40evf_execute_vf_cmd(dev, &args);
1036         if (err)
1037                 PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_VLAN");
1038
1039         return err;
1040 }
1041
1042 static int
1043 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
1044 {
1045         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1046         int err;
1047         struct vf_cmd_info args;
1048         struct rte_eth_link *new_link;
1049
1050         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
1051         args.in_args = NULL;
1052         args.in_args_size = 0;
1053         args.out_buffer = vf->aq_resp;
1054         args.out_size = I40E_AQ_BUF_SZ;
1055         err = i40evf_execute_vf_cmd(dev, &args);
1056         if (err) {
1057                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_LINK_STAT");
1058                 return err;
1059         }
1060
1061         new_link = (struct rte_eth_link *)args.out_buffer;
1062         (void)rte_memcpy(link, new_link, sizeof(*link));
1063
1064         return 0;
1065 }
1066
1067 static const struct rte_pci_id pci_id_i40evf_map[] = {
1068 #define RTE_PCI_DEV_ID_DECL_I40EVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
1069 #include "rte_pci_dev_ids.h"
1070 { .vendor_id = 0, /* sentinel */ },
1071 };
1072
1073 static inline int
1074 i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
1075                                     struct rte_eth_link *link)
1076 {
1077         struct rte_eth_link *dst = &(dev->data->dev_link);
1078         struct rte_eth_link *src = link;
1079
1080         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1081                                         *(uint64_t *)src) == 0)
1082                 return -1;
1083
1084         return 0;
1085 }
1086
1087 static int
1088 i40evf_reset_vf(struct i40e_hw *hw)
1089 {
1090         int i, reset;
1091
1092         if (i40e_vf_reset(hw) != I40E_SUCCESS) {
1093                 PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1094                 return -1;
1095         }
1096         /**
1097           * After issuing vf reset command to pf, pf won't necessarily
1098           * reset vf, it depends on what state it exactly is. If it's not
1099           * initialized yet, it won't have vf reset since it's in a certain
1100           * state. If not, it will try to reset. Even vf is reset, pf will
1101           * set I40E_VFGEN_RSTAT to COMPLETE first, then wait 10ms and set
1102           * it to ACTIVE. In this duration, vf may not catch the moment that
1103           * COMPLETE is set. So, for vf, we'll try to wait a long time.
1104           */
1105         rte_delay_ms(200);
1106
1107         for (i = 0; i < MAX_RESET_WAIT_CNT; i++) {
1108                 reset = rd32(hw, I40E_VFGEN_RSTAT) &
1109                         I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1110                 reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT;
1111                 if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset)
1112                         break;
1113                 else
1114                         rte_delay_ms(50);
1115         }
1116
1117         if (i >= MAX_RESET_WAIT_CNT) {
1118                 PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1119                 return -1;
1120         }
1121
1122         return 0;
1123 }
1124
1125 static int
1126 i40evf_init_vf(struct rte_eth_dev *dev)
1127 {
1128         int i, err, bufsz;
1129         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1130         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1131         struct ether_addr *p_mac_addr;
1132
1133         vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1134         vf->dev_data = dev->data;
1135         err = i40e_set_mac_type(hw);
1136         if (err) {
1137                 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err);
1138                 goto err;
1139         }
1140
1141         i40e_init_adminq_parameter(hw);
1142         err = i40e_init_adminq(hw);
1143         if (err) {
1144                 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err);
1145                 goto err;
1146         }
1147
1148         /* Reset VF and wait until it's complete */
1149         if (i40evf_reset_vf(hw)) {
1150                 PMD_INIT_LOG(ERR, "reset NIC failed");
1151                 goto err_aq;
1152         }
1153
1154         /* VF reset, shutdown admin queue and initialize again */
1155         if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) {
1156                 PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed");
1157                 return -1;
1158         }
1159
1160         i40e_init_adminq_parameter(hw);
1161         if (i40e_init_adminq(hw) != I40E_SUCCESS) {
1162                 PMD_INIT_LOG(ERR, "init_adminq failed");
1163                 return -1;
1164         }
1165         vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
1166         if (!vf->aq_resp) {
1167                 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
1168                         goto err_aq;
1169         }
1170         if (i40evf_check_api_version(dev) != 0) {
1171                 PMD_INIT_LOG(ERR, "check_api version failed");
1172                 goto err_aq;
1173         }
1174         bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
1175                 (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource));
1176         vf->vf_res = rte_zmalloc("vf_res", bufsz, 0);
1177         if (!vf->vf_res) {
1178                 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory");
1179                         goto err_aq;
1180         }
1181
1182         if (i40evf_get_vf_resource(dev) != 0) {
1183                 PMD_INIT_LOG(ERR, "i40evf_get_vf_config failed");
1184                 goto err_alloc;
1185         }
1186
1187         /* got VF config message back from PF, now we can parse it */
1188         for (i = 0; i < vf->vf_res->num_vsis; i++) {
1189                 if (vf->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
1190                         vf->vsi_res = &vf->vf_res->vsi_res[i];
1191         }
1192
1193         if (!vf->vsi_res) {
1194                 PMD_INIT_LOG(ERR, "no LAN VSI found");
1195                 goto err_alloc;
1196         }
1197
1198         if (hw->mac.type == I40E_MAC_X722_VF)
1199                 vf->flags = I40E_FLAG_RSS_AQ_CAPABLE;
1200         vf->vsi.vsi_id = vf->vsi_res->vsi_id;
1201         vf->vsi.type = vf->vsi_res->vsi_type;
1202         vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs;
1203         vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1204
1205         /* Store the MAC address configured by host, or generate random one */
1206         p_mac_addr = (struct ether_addr *)(vf->vsi_res->default_mac_addr);
1207         if (is_valid_assigned_ether_addr(p_mac_addr)) /* Configured by host */
1208                 ether_addr_copy(p_mac_addr, (struct ether_addr *)hw->mac.addr);
1209         else
1210                 eth_random_addr(hw->mac.addr); /* Generate a random one */
1211
1212         return 0;
1213
1214 err_alloc:
1215         rte_free(vf->vf_res);
1216 err_aq:
1217         i40e_shutdown_adminq(hw); /* ignore error */
1218 err:
1219         return -1;
1220 }
1221
1222 static int
1223 i40evf_uninit_vf(struct rte_eth_dev *dev)
1224 {
1225         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1226         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1227
1228         PMD_INIT_FUNC_TRACE();
1229
1230         if (hw->adapter_stopped == 0)
1231                 i40evf_dev_close(dev);
1232         rte_free(vf->vf_res);
1233         vf->vf_res = NULL;
1234         rte_free(vf->aq_resp);
1235         vf->aq_resp = NULL;
1236
1237         return 0;
1238 }
1239
1240 static int
1241 i40evf_dev_init(struct rte_eth_dev *eth_dev)
1242 {
1243         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
1244                         eth_dev->data->dev_private);
1245
1246         PMD_INIT_FUNC_TRACE();
1247
1248         /* assign ops func pointer */
1249         eth_dev->dev_ops = &i40evf_eth_dev_ops;
1250         eth_dev->rx_pkt_burst = &i40e_recv_pkts;
1251         eth_dev->tx_pkt_burst = &i40e_xmit_pkts;
1252
1253         /*
1254          * For secondary processes, we don't initialise any further as primary
1255          * has already done this work.
1256          */
1257         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1258                 i40e_set_rx_function(eth_dev);
1259                 i40e_set_tx_function(eth_dev);
1260                 return 0;
1261         }
1262
1263         rte_eth_copy_pci_info(eth_dev, eth_dev->pci_dev);
1264
1265         hw->vendor_id = eth_dev->pci_dev->id.vendor_id;
1266         hw->device_id = eth_dev->pci_dev->id.device_id;
1267         hw->subsystem_vendor_id = eth_dev->pci_dev->id.subsystem_vendor_id;
1268         hw->subsystem_device_id = eth_dev->pci_dev->id.subsystem_device_id;
1269         hw->bus.device = eth_dev->pci_dev->addr.devid;
1270         hw->bus.func = eth_dev->pci_dev->addr.function;
1271         hw->hw_addr = (void *)eth_dev->pci_dev->mem_resource[0].addr;
1272         hw->adapter_stopped = 0;
1273
1274         if(i40evf_init_vf(eth_dev) != 0) {
1275                 PMD_INIT_LOG(ERR, "Init vf failed");
1276                 return -1;
1277         }
1278
1279         /* allocate memory for mac addr storage */
1280         eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
1281                                         ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX,
1282                                         0);
1283         if (eth_dev->data->mac_addrs == NULL) {
1284                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to"
1285                                 " store MAC addresses",
1286                                 ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX);
1287                 return -ENOMEM;
1288         }
1289         ether_addr_copy((struct ether_addr *)hw->mac.addr,
1290                         &eth_dev->data->mac_addrs[0]);
1291
1292         return 0;
1293 }
1294
1295 static int
1296 i40evf_dev_uninit(struct rte_eth_dev *eth_dev)
1297 {
1298         PMD_INIT_FUNC_TRACE();
1299
1300         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1301                 return -EPERM;
1302
1303         eth_dev->dev_ops = NULL;
1304         eth_dev->rx_pkt_burst = NULL;
1305         eth_dev->tx_pkt_burst = NULL;
1306
1307         if (i40evf_uninit_vf(eth_dev) != 0) {
1308                 PMD_INIT_LOG(ERR, "i40evf_uninit_vf failed");
1309                 return -1;
1310         }
1311
1312         rte_free(eth_dev->data->mac_addrs);
1313         eth_dev->data->mac_addrs = NULL;
1314
1315         return 0;
1316 }
1317 /*
1318  * virtual function driver struct
1319  */
1320 static struct eth_driver rte_i40evf_pmd = {
1321         .pci_drv = {
1322                 .name = "rte_i40evf_pmd",
1323                 .id_table = pci_id_i40evf_map,
1324                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1325         },
1326         .eth_dev_init = i40evf_dev_init,
1327         .eth_dev_uninit = i40evf_dev_uninit,
1328         .dev_private_size = sizeof(struct i40e_adapter),
1329 };
1330
1331 /*
1332  * VF Driver initialization routine.
1333  * Invoked one at EAL init time.
1334  * Register itself as the [Virtual Poll Mode] Driver of PCI Fortville devices.
1335  */
1336 static int
1337 rte_i40evf_pmd_init(const char *name __rte_unused,
1338                     const char *params __rte_unused)
1339 {
1340         PMD_INIT_FUNC_TRACE();
1341
1342         rte_eth_driver_register(&rte_i40evf_pmd);
1343
1344         return 0;
1345 }
1346
1347 static struct rte_driver rte_i40evf_driver = {
1348         .type = PMD_PDEV,
1349         .init = rte_i40evf_pmd_init,
1350 };
1351
1352 PMD_REGISTER_DRIVER(rte_i40evf_driver);
1353
1354 static int
1355 i40evf_dev_configure(struct rte_eth_dev *dev)
1356 {
1357         struct i40e_adapter *ad =
1358                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1359
1360         /* Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1361          * allocation or vector Rx preconditions we will reset it.
1362          */
1363         ad->rx_bulk_alloc_allowed = true;
1364         ad->rx_vec_allowed = true;
1365         ad->tx_simple_allowed = true;
1366         ad->tx_vec_allowed = true;
1367
1368         return i40evf_init_vlan(dev);
1369 }
1370
1371 static int
1372 i40evf_init_vlan(struct rte_eth_dev *dev)
1373 {
1374         struct rte_eth_dev_data *data = dev->data;
1375         int ret;
1376
1377         /* Apply vlan offload setting */
1378         i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
1379
1380         /* Apply pvid setting */
1381         ret = i40evf_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
1382                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
1383         return ret;
1384 }
1385
1386 static void
1387 i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1388 {
1389         bool enable_vlan_strip = 0;
1390         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1391         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1392
1393         /* Linux pf host doesn't support vlan offload yet */
1394         if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1395                 /* Vlan stripping setting */
1396                 if (mask & ETH_VLAN_STRIP_MASK) {
1397                         /* Enable or disable VLAN stripping */
1398                         if (dev_conf->rxmode.hw_vlan_strip)
1399                                 enable_vlan_strip = 1;
1400                         else
1401                                 enable_vlan_strip = 0;
1402
1403                         i40evf_config_vlan_offload(dev, enable_vlan_strip);
1404                 }
1405         }
1406 }
1407
1408 static int
1409 i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1410 {
1411         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1412         struct i40e_vsi_vlan_pvid_info info;
1413         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1414
1415         memset(&info, 0, sizeof(info));
1416         info.on = on;
1417
1418         /* Linux pf host don't support vlan offload yet */
1419         if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1420                 if (info.on)
1421                         info.config.pvid = pvid;
1422                 else {
1423                         info.config.reject.tagged =
1424                                 dev_conf->txmode.hw_vlan_reject_tagged;
1425                         info.config.reject.untagged =
1426                                 dev_conf->txmode.hw_vlan_reject_untagged;
1427                 }
1428                 return i40evf_config_vlan_pvid(dev, &info);
1429         }
1430
1431         return 0;
1432 }
1433
1434 static int
1435 i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1436 {
1437         struct i40e_rx_queue *rxq;
1438         int err = 0;
1439         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1440
1441         PMD_INIT_FUNC_TRACE();
1442
1443         if (rx_queue_id < dev->data->nb_rx_queues) {
1444                 rxq = dev->data->rx_queues[rx_queue_id];
1445
1446                 err = i40e_alloc_rx_queue_mbufs(rxq);
1447                 if (err) {
1448                         PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1449                         return err;
1450                 }
1451
1452                 rte_wmb();
1453
1454                 /* Init the RX tail register. */
1455                 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1456                 I40EVF_WRITE_FLUSH(hw);
1457
1458                 /* Ready to switch the queue on */
1459                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE);
1460
1461                 if (err)
1462                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
1463                                     rx_queue_id);
1464                 else
1465                         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1466         }
1467
1468         return err;
1469 }
1470
1471 static int
1472 i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1473 {
1474         struct i40e_rx_queue *rxq;
1475         int err;
1476
1477         if (rx_queue_id < dev->data->nb_rx_queues) {
1478                 rxq = dev->data->rx_queues[rx_queue_id];
1479
1480                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE);
1481
1482                 if (err) {
1483                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
1484                                     rx_queue_id);
1485                         return err;
1486                 }
1487
1488                 i40e_rx_queue_release_mbufs(rxq);
1489                 i40e_reset_rx_queue(rxq);
1490                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1491         }
1492
1493         return 0;
1494 }
1495
1496 static int
1497 i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1498 {
1499         int err = 0;
1500
1501         PMD_INIT_FUNC_TRACE();
1502
1503         if (tx_queue_id < dev->data->nb_tx_queues) {
1504
1505                 /* Ready to switch the queue on */
1506                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE);
1507
1508                 if (err)
1509                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
1510                                     tx_queue_id);
1511                 else
1512                         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1513         }
1514
1515         return err;
1516 }
1517
1518 static int
1519 i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1520 {
1521         struct i40e_tx_queue *txq;
1522         int err;
1523
1524         if (tx_queue_id < dev->data->nb_tx_queues) {
1525                 txq = dev->data->tx_queues[tx_queue_id];
1526
1527                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE);
1528
1529                 if (err) {
1530                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
1531                                     tx_queue_id);
1532                         return err;
1533                 }
1534
1535                 i40e_tx_queue_release_mbufs(txq);
1536                 i40e_reset_tx_queue(txq);
1537                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1538         }
1539
1540         return 0;
1541 }
1542
1543 static int
1544 i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1545 {
1546         int ret;
1547
1548         if (on)
1549                 ret = i40evf_add_vlan(dev, vlan_id);
1550         else
1551                 ret = i40evf_del_vlan(dev,vlan_id);
1552
1553         return ret;
1554 }
1555
1556 static int
1557 i40evf_rxq_init(struct rte_eth_dev *dev, struct i40e_rx_queue *rxq)
1558 {
1559         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1560         struct rte_eth_dev_data *dev_data = dev->data;
1561         struct rte_pktmbuf_pool_private *mbp_priv;
1562         uint16_t buf_size, len;
1563
1564         rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL1(rxq->queue_id);
1565         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1566         I40EVF_WRITE_FLUSH(hw);
1567
1568         /* Calculate the maximum packet length allowed */
1569         mbp_priv = rte_mempool_get_priv(rxq->mp);
1570         buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
1571                                         RTE_PKTMBUF_HEADROOM);
1572         rxq->hs_mode = i40e_header_split_none;
1573         rxq->rx_hdr_len = 0;
1574         rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
1575         len = rxq->rx_buf_len * I40E_MAX_CHAINED_RX_BUFFERS;
1576         rxq->max_pkt_len = RTE_MIN(len,
1577                 dev_data->dev_conf.rxmode.max_rx_pkt_len);
1578
1579         /**
1580          * Check if the jumbo frame and maximum packet length are set correctly
1581          */
1582         if (dev_data->dev_conf.rxmode.jumbo_frame == 1) {
1583                 if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
1584                     rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
1585                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1586                                 "larger than %u and smaller than %u, as jumbo "
1587                                 "frame is enabled", (uint32_t)ETHER_MAX_LEN,
1588                                         (uint32_t)I40E_FRAME_SIZE_MAX);
1589                         return I40E_ERR_CONFIG;
1590                 }
1591         } else {
1592                 if (rxq->max_pkt_len < ETHER_MIN_LEN ||
1593                     rxq->max_pkt_len > ETHER_MAX_LEN) {
1594                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1595                                 "larger than %u and smaller than %u, as jumbo "
1596                                 "frame is disabled", (uint32_t)ETHER_MIN_LEN,
1597                                                 (uint32_t)ETHER_MAX_LEN);
1598                         return I40E_ERR_CONFIG;
1599                 }
1600         }
1601
1602         if (dev_data->dev_conf.rxmode.enable_scatter ||
1603             (rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
1604                 dev_data->scattered_rx = 1;
1605         }
1606
1607         return 0;
1608 }
1609
1610 static int
1611 i40evf_rx_init(struct rte_eth_dev *dev)
1612 {
1613         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1614         uint16_t i;
1615         int ret = I40E_SUCCESS;
1616         struct i40e_rx_queue **rxq =
1617                 (struct i40e_rx_queue **)dev->data->rx_queues;
1618
1619         i40evf_config_rss(vf);
1620         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1621                 if (!rxq[i] || !rxq[i]->q_set)
1622                         continue;
1623                 ret = i40evf_rxq_init(dev, rxq[i]);
1624                 if (ret != I40E_SUCCESS)
1625                         break;
1626         }
1627         if (ret == I40E_SUCCESS)
1628                 i40e_set_rx_function(dev);
1629
1630         return ret;
1631 }
1632
1633 static void
1634 i40evf_tx_init(struct rte_eth_dev *dev)
1635 {
1636         uint16_t i;
1637         struct i40e_tx_queue **txq =
1638                 (struct i40e_tx_queue **)dev->data->tx_queues;
1639         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1640
1641         for (i = 0; i < dev->data->nb_tx_queues; i++)
1642                 txq[i]->qtx_tail = hw->hw_addr + I40E_QTX_TAIL1(i);
1643
1644         i40e_set_tx_function(dev);
1645 }
1646
1647 static inline void
1648 i40evf_enable_queues_intr(struct rte_eth_dev *dev)
1649 {
1650         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1651         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1652         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1653
1654         if (!rte_intr_allow_others(intr_handle)) {
1655                 I40E_WRITE_REG(hw,
1656                                I40E_VFINT_DYN_CTL01,
1657                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1658                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
1659                 I40EVF_WRITE_FLUSH(hw);
1660                 return;
1661         }
1662
1663         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1664                 /* To support DPDK PF host */
1665                 I40E_WRITE_REG(hw,
1666                         I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
1667                         I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1668                         I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
1669         else
1670                 /* To support Linux PF host */
1671                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1672                                 I40E_VFINT_DYN_CTL01_INTENA_MASK |
1673                                 I40E_VFINT_DYN_CTL01_CLEARPBA_MASK);
1674
1675         I40EVF_WRITE_FLUSH(hw);
1676 }
1677
1678 static inline void
1679 i40evf_disable_queues_intr(struct rte_eth_dev *dev)
1680 {
1681         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1682         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1683         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1684
1685         if (!rte_intr_allow_others(intr_handle)) {
1686                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
1687                 I40EVF_WRITE_FLUSH(hw);
1688                 return;
1689         }
1690
1691         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1692                 I40E_WRITE_REG(hw,
1693                                I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
1694                                                     - 1),
1695                                0);
1696         else
1697                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
1698
1699         I40EVF_WRITE_FLUSH(hw);
1700 }
1701
1702 static int
1703 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1704 {
1705         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1706         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1707         uint16_t interval =
1708                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
1709         uint16_t msix_intr;
1710
1711         msix_intr = intr_handle->intr_vec[queue_id];
1712         if (msix_intr == I40E_MISC_VEC_ID)
1713                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1714                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1715                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1716                                (0 << I40E_VFINT_DYN_CTL01_ITR_INDX_SHIFT) |
1717                                (interval <<
1718                                 I40E_VFINT_DYN_CTL01_INTERVAL_SHIFT));
1719         else
1720                 I40E_WRITE_REG(hw,
1721                                I40E_VFINT_DYN_CTLN1(msix_intr -
1722                                                     I40E_RX_VEC_START),
1723                                I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1724                                I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
1725                                (0 << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1726                                (interval <<
1727                                 I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT));
1728
1729         I40EVF_WRITE_FLUSH(hw);
1730
1731         rte_intr_enable(&dev->pci_dev->intr_handle);
1732
1733         return 0;
1734 }
1735
1736 static int
1737 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1738 {
1739         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1740         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1741         uint16_t msix_intr;
1742
1743         msix_intr = intr_handle->intr_vec[queue_id];
1744         if (msix_intr == I40E_MISC_VEC_ID)
1745                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
1746         else
1747                 I40E_WRITE_REG(hw,
1748                                I40E_VFINT_DYN_CTLN1(msix_intr -
1749                                                     I40E_RX_VEC_START),
1750                                0);
1751
1752         I40EVF_WRITE_FLUSH(hw);
1753
1754         return 0;
1755 }
1756
1757 static void
1758 i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add)
1759 {
1760         struct i40e_virtchnl_ether_addr_list *list;
1761         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1762         int err, i, j;
1763         int next_begin = 0;
1764         int begin = 0;
1765         uint32_t len;
1766         struct ether_addr *addr;
1767         struct vf_cmd_info args;
1768
1769         do {
1770                 j = 0;
1771                 len = sizeof(struct i40e_virtchnl_ether_addr_list);
1772                 for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) {
1773                         if (is_zero_ether_addr(&dev->data->mac_addrs[i]))
1774                                 continue;
1775                         len += sizeof(struct i40e_virtchnl_ether_addr);
1776                         if (len >= I40E_AQ_BUF_SZ) {
1777                                 next_begin = i + 1;
1778                                 break;
1779                         }
1780                 }
1781
1782                 list = rte_zmalloc("i40evf_del_mac_buffer", len, 0);
1783
1784                 for (i = begin; i < next_begin; i++) {
1785                         addr = &dev->data->mac_addrs[i];
1786                         if (is_zero_ether_addr(addr))
1787                                 continue;
1788                         (void)rte_memcpy(list->list[j].addr, addr->addr_bytes,
1789                                          sizeof(addr->addr_bytes));
1790                         PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x",
1791                                     addr->addr_bytes[0], addr->addr_bytes[1],
1792                                     addr->addr_bytes[2], addr->addr_bytes[3],
1793                                     addr->addr_bytes[4], addr->addr_bytes[5]);
1794                         j++;
1795                 }
1796                 list->vsi_id = vf->vsi_res->vsi_id;
1797                 list->num_elements = j;
1798                 args.ops = add ? I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS :
1799                            I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
1800                 args.in_args = (uint8_t *)list;
1801                 args.in_args_size = len;
1802                 args.out_buffer = vf->aq_resp;
1803                 args.out_size = I40E_AQ_BUF_SZ;
1804                 err = i40evf_execute_vf_cmd(dev, &args);
1805                 if (err)
1806                         PMD_DRV_LOG(ERR, "fail to execute command %s",
1807                                     add ? "OP_ADD_ETHER_ADDRESS" :
1808                                     "OP_DEL_ETHER_ADDRESS");
1809                 rte_free(list);
1810                 begin = next_begin;
1811         } while (begin < I40E_NUM_MACADDR_MAX);
1812 }
1813
1814 static int
1815 i40evf_dev_start(struct rte_eth_dev *dev)
1816 {
1817         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1818         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1819         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1820         uint32_t intr_vector = 0;
1821
1822         PMD_INIT_FUNC_TRACE();
1823
1824         hw->adapter_stopped = 0;
1825
1826         vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1827         vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
1828                                         dev->data->nb_tx_queues);
1829
1830         /* check and configure queue intr-vector mapping */
1831         if (dev->data->dev_conf.intr_conf.rxq != 0) {
1832                 intr_vector = dev->data->nb_rx_queues;
1833                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1834                         return -1;
1835         }
1836
1837         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1838                 intr_handle->intr_vec =
1839                         rte_zmalloc("intr_vec",
1840                                     dev->data->nb_rx_queues * sizeof(int), 0);
1841                 if (!intr_handle->intr_vec) {
1842                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1843                                      " intr_vec\n", dev->data->nb_rx_queues);
1844                         return -ENOMEM;
1845                 }
1846         }
1847
1848         if (i40evf_rx_init(dev) != 0){
1849                 PMD_DRV_LOG(ERR, "failed to do RX init");
1850                 return -1;
1851         }
1852
1853         i40evf_tx_init(dev);
1854
1855         if (i40evf_configure_queues(dev) != 0) {
1856                 PMD_DRV_LOG(ERR, "configure queues failed");
1857                 goto err_queue;
1858         }
1859         if (i40evf_config_irq_map(dev)) {
1860                 PMD_DRV_LOG(ERR, "config_irq_map failed");
1861                 goto err_queue;
1862         }
1863
1864         /* Set all mac addrs */
1865         i40evf_add_del_all_mac_addr(dev, TRUE);
1866
1867         if (i40evf_start_queues(dev) != 0) {
1868                 PMD_DRV_LOG(ERR, "enable queues failed");
1869                 goto err_mac;
1870         }
1871
1872         /* vf don't allow intr except for rxq intr */
1873         if (dev->data->dev_conf.intr_conf.rxq != 0)
1874                 rte_intr_enable(intr_handle);
1875
1876         i40evf_enable_queues_intr(dev);
1877         return 0;
1878
1879 err_mac:
1880         i40evf_add_del_all_mac_addr(dev, FALSE);
1881 err_queue:
1882         return -1;
1883 }
1884
1885 static void
1886 i40evf_dev_stop(struct rte_eth_dev *dev)
1887 {
1888         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1889
1890         PMD_INIT_FUNC_TRACE();
1891
1892         i40evf_stop_queues(dev);
1893         i40evf_disable_queues_intr(dev);
1894         i40e_dev_clear_queues(dev);
1895
1896         /* Clean datapath event and queue/vec mapping */
1897         rte_intr_efd_disable(intr_handle);
1898         if (intr_handle->intr_vec) {
1899                 rte_free(intr_handle->intr_vec);
1900                 intr_handle->intr_vec = NULL;
1901         }
1902         /* remove all mac addrs */
1903         i40evf_add_del_all_mac_addr(dev, FALSE);
1904
1905 }
1906
1907 static int
1908 i40evf_dev_link_update(struct rte_eth_dev *dev,
1909                        __rte_unused int wait_to_complete)
1910 {
1911         struct rte_eth_link new_link;
1912         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1913         /*
1914          * DPDK pf host provide interfacet to acquire link status
1915          * while Linux driver does not
1916          */
1917         if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1918                 i40evf_get_link_status(dev, &new_link);
1919         else {
1920                 /* Always assume it's up, for Linux driver PF host */
1921                 new_link.link_duplex = ETH_LINK_AUTONEG_DUPLEX;
1922                 new_link.link_speed  = ETH_LINK_SPEED_10000;
1923                 new_link.link_status = 1;
1924         }
1925         i40evf_dev_atomic_write_link_status(dev, &new_link);
1926
1927         return 0;
1928 }
1929
1930 static void
1931 i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev)
1932 {
1933         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1934         int ret;
1935
1936         /* If enabled, just return */
1937         if (vf->promisc_unicast_enabled)
1938                 return;
1939
1940         ret = i40evf_config_promisc(dev, 1, vf->promisc_multicast_enabled);
1941         if (ret == 0)
1942                 vf->promisc_unicast_enabled = TRUE;
1943 }
1944
1945 static void
1946 i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev)
1947 {
1948         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1949         int ret;
1950
1951         /* If disabled, just return */
1952         if (!vf->promisc_unicast_enabled)
1953                 return;
1954
1955         ret = i40evf_config_promisc(dev, 0, vf->promisc_multicast_enabled);
1956         if (ret == 0)
1957                 vf->promisc_unicast_enabled = FALSE;
1958 }
1959
1960 static void
1961 i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev)
1962 {
1963         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1964         int ret;
1965
1966         /* If enabled, just return */
1967         if (vf->promisc_multicast_enabled)
1968                 return;
1969
1970         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 1);
1971         if (ret == 0)
1972                 vf->promisc_multicast_enabled = TRUE;
1973 }
1974
1975 static void
1976 i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev)
1977 {
1978         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1979         int ret;
1980
1981         /* If enabled, just return */
1982         if (!vf->promisc_multicast_enabled)
1983                 return;
1984
1985         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 0);
1986         if (ret == 0)
1987                 vf->promisc_multicast_enabled = FALSE;
1988 }
1989
1990 static void
1991 i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1992 {
1993         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1994
1995         memset(dev_info, 0, sizeof(*dev_info));
1996         dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
1997         dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
1998         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
1999         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2000         dev_info->hash_key_size = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2001         dev_info->reta_size = ETH_RSS_RETA_SIZE_64;
2002         dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2003         dev_info->max_mac_addrs = I40E_NUM_MACADDR_MAX;
2004         dev_info->rx_offload_capa =
2005                 DEV_RX_OFFLOAD_VLAN_STRIP |
2006                 DEV_RX_OFFLOAD_QINQ_STRIP |
2007                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2008                 DEV_RX_OFFLOAD_UDP_CKSUM |
2009                 DEV_RX_OFFLOAD_TCP_CKSUM;
2010         dev_info->tx_offload_capa =
2011                 DEV_TX_OFFLOAD_VLAN_INSERT |
2012                 DEV_TX_OFFLOAD_QINQ_INSERT |
2013                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2014                 DEV_TX_OFFLOAD_UDP_CKSUM |
2015                 DEV_TX_OFFLOAD_TCP_CKSUM |
2016                 DEV_TX_OFFLOAD_SCTP_CKSUM;
2017
2018         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2019                 .rx_thresh = {
2020                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2021                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2022                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2023                 },
2024                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2025                 .rx_drop_en = 0,
2026         };
2027
2028         dev_info->default_txconf = (struct rte_eth_txconf) {
2029                 .tx_thresh = {
2030                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2031                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2032                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2033                 },
2034                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2035                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2036                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2037                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2038         };
2039
2040         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2041                 .nb_max = I40E_MAX_RING_DESC,
2042                 .nb_min = I40E_MIN_RING_DESC,
2043                 .nb_align = I40E_ALIGN_RING_DESC,
2044         };
2045
2046         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2047                 .nb_max = I40E_MAX_RING_DESC,
2048                 .nb_min = I40E_MIN_RING_DESC,
2049                 .nb_align = I40E_ALIGN_RING_DESC,
2050         };
2051 }
2052
2053 static void
2054 i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2055 {
2056         if (i40evf_get_statics(dev, stats))
2057                 PMD_DRV_LOG(ERR, "Get statics failed");
2058 }
2059
2060 static void
2061 i40evf_dev_close(struct rte_eth_dev *dev)
2062 {
2063         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2064
2065         i40evf_dev_stop(dev);
2066         hw->adapter_stopped = 1;
2067         i40e_dev_free_queues(dev);
2068         i40evf_reset_vf(hw);
2069         i40e_shutdown_adminq(hw);
2070 }
2071
2072 static int
2073 i40evf_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2074 {
2075         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2076         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2077         int ret;
2078
2079         if (!lut)
2080                 return -EINVAL;
2081
2082         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2083                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, FALSE,
2084                                           lut, lut_size);
2085                 if (ret) {
2086                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
2087                         return ret;
2088                 }
2089         } else {
2090                 uint32_t *lut_dw = (uint32_t *)lut;
2091                 uint16_t i, lut_size_dw = lut_size / 4;
2092
2093                 for (i = 0; i < lut_size_dw; i++)
2094                         lut_dw[i] = I40E_READ_REG(hw, I40E_VFQF_HLUT(i));
2095         }
2096
2097         return 0;
2098 }
2099
2100 static int
2101 i40evf_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2102 {
2103         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2104         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2105         int ret;
2106
2107         if (!vsi || !lut)
2108                 return -EINVAL;
2109
2110         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2111                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, FALSE,
2112                                           lut, lut_size);
2113                 if (ret) {
2114                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
2115                         return ret;
2116                 }
2117         } else {
2118                 uint32_t *lut_dw = (uint32_t *)lut;
2119                 uint16_t i, lut_size_dw = lut_size / 4;
2120
2121                 for (i = 0; i < lut_size_dw; i++)
2122                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i), lut_dw[i]);
2123                 I40EVF_WRITE_FLUSH(hw);
2124         }
2125
2126         return 0;
2127 }
2128
2129 static int
2130 i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
2131                            struct rte_eth_rss_reta_entry64 *reta_conf,
2132                            uint16_t reta_size)
2133 {
2134         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2135         uint8_t *lut;
2136         uint16_t i, idx, shift;
2137         int ret;
2138
2139         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2140                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2141                         "(%d) doesn't match the number of hardware can "
2142                         "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2143                 return -EINVAL;
2144         }
2145
2146         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2147         if (!lut) {
2148                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2149                 return -ENOMEM;
2150         }
2151         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2152         if (ret)
2153                 goto out;
2154         for (i = 0; i < reta_size; i++) {
2155                 idx = i / RTE_RETA_GROUP_SIZE;
2156                 shift = i % RTE_RETA_GROUP_SIZE;
2157                 if (reta_conf[idx].mask & (1ULL << shift))
2158                         lut[i] = reta_conf[idx].reta[shift];
2159         }
2160         ret = i40evf_set_rss_lut(&vf->vsi, lut, reta_size);
2161
2162 out:
2163         rte_free(lut);
2164
2165         return ret;
2166 }
2167
2168 static int
2169 i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
2170                           struct rte_eth_rss_reta_entry64 *reta_conf,
2171                           uint16_t reta_size)
2172 {
2173         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2174         uint16_t i, idx, shift;
2175         uint8_t *lut;
2176         int ret;
2177
2178         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2179                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2180                         "(%d) doesn't match the number of hardware can "
2181                         "support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2182                 return -EINVAL;
2183         }
2184
2185         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2186         if (!lut) {
2187                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2188                 return -ENOMEM;
2189         }
2190
2191         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2192         if (ret)
2193                 goto out;
2194         for (i = 0; i < reta_size; i++) {
2195                 idx = i / RTE_RETA_GROUP_SIZE;
2196                 shift = i % RTE_RETA_GROUP_SIZE;
2197                 if (reta_conf[idx].mask & (1ULL << shift))
2198                         reta_conf[idx].reta[shift] = lut[i];
2199         }
2200
2201 out:
2202         rte_free(lut);
2203
2204         return ret;
2205 }
2206
2207 static int
2208 i40evf_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
2209 {
2210         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2211         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2212         int ret = 0;
2213
2214         if (!key || key_len == 0) {
2215                 PMD_DRV_LOG(DEBUG, "No key to be configured");
2216                 return 0;
2217         } else if (key_len != (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2218                 sizeof(uint32_t)) {
2219                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
2220                 return -EINVAL;
2221         }
2222
2223         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2224                 struct i40e_aqc_get_set_rss_key_data *key_dw =
2225                         (struct i40e_aqc_get_set_rss_key_data *)key;
2226
2227                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
2228                 if (ret)
2229                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
2230                                      "via AQ");
2231         } else {
2232                 uint32_t *hash_key = (uint32_t *)key;
2233                 uint16_t i;
2234
2235                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2236                         i40e_write_rx_ctl(hw, I40E_VFQF_HKEY(i), hash_key[i]);
2237                 I40EVF_WRITE_FLUSH(hw);
2238         }
2239
2240         return ret;
2241 }
2242
2243 static int
2244 i40evf_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
2245 {
2246         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2247         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2248         int ret;
2249
2250         if (!key || !key_len)
2251                 return -EINVAL;
2252
2253         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2254                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
2255                         (struct i40e_aqc_get_set_rss_key_data *)key);
2256                 if (ret) {
2257                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
2258                         return ret;
2259                 }
2260         } else {
2261                 uint32_t *key_dw = (uint32_t *)key;
2262                 uint16_t i;
2263
2264                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2265                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_VFQF_HKEY(i));
2266         }
2267         *key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2268
2269         return 0;
2270 }
2271
2272 static int
2273 i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf)
2274 {
2275         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2276         uint64_t rss_hf, hena;
2277         int ret;
2278
2279         ret = i40evf_set_rss_key(&vf->vsi, rss_conf->rss_key,
2280                                  rss_conf->rss_key_len);
2281         if (ret)
2282                 return ret;
2283
2284         rss_hf = rss_conf->rss_hf;
2285         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2286         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2287         hena &= ~I40E_RSS_HENA_ALL;
2288         hena |= i40e_config_hena(rss_hf);
2289         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2290         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2291         I40EVF_WRITE_FLUSH(hw);
2292
2293         return 0;
2294 }
2295
2296 static void
2297 i40evf_disable_rss(struct i40e_vf *vf)
2298 {
2299         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2300         uint64_t hena;
2301
2302         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2303         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2304         hena &= ~I40E_RSS_HENA_ALL;
2305         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2306         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2307         I40EVF_WRITE_FLUSH(hw);
2308 }
2309
2310 static int
2311 i40evf_config_rss(struct i40e_vf *vf)
2312 {
2313         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2314         struct rte_eth_rss_conf rss_conf;
2315         uint32_t i, j, lut = 0, nb_q = (I40E_VFQF_HLUT_MAX_INDEX + 1) * 4;
2316         uint16_t num;
2317
2318         if (vf->dev_data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
2319                 i40evf_disable_rss(vf);
2320                 PMD_DRV_LOG(DEBUG, "RSS not configured\n");
2321                 return 0;
2322         }
2323
2324         num = RTE_MIN(vf->dev_data->nb_rx_queues, I40E_MAX_QP_NUM_PER_VF);
2325         /* Fill out the look up table */
2326         for (i = 0, j = 0; i < nb_q; i++, j++) {
2327                 if (j >= num)
2328                         j = 0;
2329                 lut = (lut << 8) | j;
2330                 if ((i & 3) == 3)
2331                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i >> 2), lut);
2332         }
2333
2334         rss_conf = vf->dev_data->dev_conf.rx_adv_conf.rss_conf;
2335         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
2336                 i40evf_disable_rss(vf);
2337                 PMD_DRV_LOG(DEBUG, "No hash flag is set\n");
2338                 return 0;
2339         }
2340
2341         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
2342                 (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
2343                 /* Calculate the default hash key */
2344                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2345                         rss_key_default[i] = (uint32_t)rte_rand();
2346                 rss_conf.rss_key = (uint8_t *)rss_key_default;
2347                 rss_conf.rss_key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2348                         sizeof(uint32_t);
2349         }
2350
2351         return i40evf_hw_rss_hash_set(vf, &rss_conf);
2352 }
2353
2354 static int
2355 i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
2356                            struct rte_eth_rss_conf *rss_conf)
2357 {
2358         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2359         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2360         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
2361         uint64_t hena;
2362
2363         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2364         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2365         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
2366                 if (rss_hf != 0) /* Enable RSS */
2367                         return -EINVAL;
2368                 return 0;
2369         }
2370
2371         /* RSS enabled */
2372         if (rss_hf == 0) /* Disable RSS */
2373                 return -EINVAL;
2374
2375         return i40evf_hw_rss_hash_set(vf, rss_conf);
2376 }
2377
2378 static int
2379 i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2380                              struct rte_eth_rss_conf *rss_conf)
2381 {
2382         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2383         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2384         uint64_t hena;
2385
2386         i40evf_get_rss_key(&vf->vsi, rss_conf->rss_key,
2387                            &rss_conf->rss_key_len);
2388
2389         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2390         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2391         rss_conf->rss_hf = i40e_parse_hena(hena);
2392
2393         return 0;
2394 }