lib/librte_pmd_i40e/i40e_ethdev_vf.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_tailq.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_dev.h>
  62
  63 #include "i40e_logs.h"
  64 #include "i40e/i40e_prototype.h"
  65 #include "i40e/i40e_adminq_cmd.h"
  66 #include "i40e/i40e_type.h"
  67
  68 #include "i40e_rxtx.h"
  69 #include "i40e_ethdev.h"
  70 #include "i40e_pf.h"
  71 #define I40EVF_VSI_DEFAULT_MSIX_INTR 1
  72
  73 /* busy wait delay in msec */
  74 #define I40EVF_BUSY_WAIT_DELAY 10
  75 #define I40EVF_BUSY_WAIT_COUNT 50
  76 #define MAX_RESET_WAIT_CNT     20
  77
  78 struct i40evf_arq_msg_info {
  79         enum i40e_virtchnl_ops ops;
  80         enum i40e_status_code result;
  81         uint16_t msg_len;
  82         uint8_t *msg;
  83 };
  84
  85 struct vf_cmd_info {
  86         enum i40e_virtchnl_ops ops;
  87         uint8_t *in_args;
  88         uint32_t in_args_size;
  89         uint8_t *out_buffer;
  90         /* Input & output type. pass in buffer size and pass out
  91          * actual return result
  92          */
  93         uint32_t out_size;
  94 };
  95
  96 enum i40evf_aq_result {
  97         I40EVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
  98         I40EVF_MSG_NON,      /* Read nothing from admin queue */
  99         I40EVF_MSG_SYS,      /* Read system msg from admin queue */
 100         I40EVF_MSG_CMD,      /* Read async command result */
 101 };
 102
 103 /* A share buffer to store the command result from PF driver */
 104 static uint8_t cmd_result_buffer[I40E_AQ_BUF_SZ];
 105
 106 static int i40evf_dev_configure(struct rte_eth_dev *dev);
 107 static int i40evf_dev_start(struct rte_eth_dev *dev);
 108 static void i40evf_dev_stop(struct rte_eth_dev *dev);
 109 static void i40evf_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int i40evf_dev_link_update(struct rte_eth_dev *dev,
 112                                   __rte_unused int wait_to_complete);
 113 static void i40evf_dev_stats_get(struct rte_eth_dev *dev,
 114                                 struct rte_eth_stats *stats);
 115 static int i40evf_vlan_filter_set(struct rte_eth_dev *dev,
 116                                   uint16_t vlan_id, int on);
 117 static void i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 118 static int i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid,
 119                                 int on);
 120 static void i40evf_dev_close(struct rte_eth_dev *dev);
 121 static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev);
 122 static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev);
 123 static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev);
 124 static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev);
 125 static int i40evf_get_link_status(struct rte_eth_dev *dev,
 126                                   struct rte_eth_link *link);
 127 static int i40evf_init_vlan(struct rte_eth_dev *dev);
 128 static struct eth_dev_ops i40evf_eth_dev_ops = {
 129         .dev_configure        = i40evf_dev_configure,
 130         .dev_start            = i40evf_dev_start,
 131         .dev_stop             = i40evf_dev_stop,
 132         .promiscuous_enable   = i40evf_dev_promiscuous_enable,
 133         .promiscuous_disable  = i40evf_dev_promiscuous_disable,
 134         .allmulticast_enable  = i40evf_dev_allmulticast_enable,
 135         .allmulticast_disable = i40evf_dev_allmulticast_disable,
 136         .link_update          = i40evf_dev_link_update,
 137         .stats_get            = i40evf_dev_stats_get,
 138         .dev_close            = i40evf_dev_close,
 139         .dev_infos_get        = i40evf_dev_info_get,
 140         .vlan_filter_set      = i40evf_vlan_filter_set,
 141         .vlan_offload_set     = i40evf_vlan_offload_set,
 142         .vlan_pvid_set        = i40evf_vlan_pvid_set,
 143         .rx_queue_setup       = i40e_dev_rx_queue_setup,
 144         .rx_queue_release     = i40e_dev_rx_queue_release,
 145         .tx_queue_setup       = i40e_dev_tx_queue_setup,
 146         .tx_queue_release     = i40e_dev_tx_queue_release,
 147 };
 148
 149 static int
 150 i40evf_set_mac_type(struct i40e_hw *hw)
 151 {
 152         int status = I40E_ERR_DEVICE_NOT_SUPPORTED;
 153
 154         if (hw->vendor_id == I40E_INTEL_VENDOR_ID) {
 155                 switch (hw->device_id) {
 156                 case I40E_DEV_ID_VF:
 157                 case I40E_DEV_ID_VF_HV:
 158                         hw->mac.type = I40E_MAC_VF;
 159                         status = I40E_SUCCESS;
 160                         break;
 161                 default:
 162                         ;
 163                 }
 164         }
 165
 166         return status;
 167 }
 168
 169 /*
 170  * Parse admin queue message.
 171  *
 172  * return value:
 173  *  < 0: meet error
 174  *  0: read sys msg
 175  *  > 0: read cmd result
 176  */
 177 static enum i40evf_aq_result
 178 i40evf_parse_pfmsg(struct i40e_vf *vf,
 179                    struct i40e_arq_event_info *event,
 180                    struct i40evf_arq_msg_info *data)
 181 {
 182         enum i40e_virtchnl_ops opcode = (enum i40e_virtchnl_ops)\
 183                         rte_le_to_cpu_32(event->desc.cookie_high);
 184         enum i40e_status_code retval = (enum i40e_status_code)\
 185                         rte_le_to_cpu_32(event->desc.cookie_low);
 186         enum i40evf_aq_result ret = I40EVF_MSG_CMD;
 187
 188         /* pf sys event */
 189         if (opcode == I40E_VIRTCHNL_OP_EVENT) {
 190                 struct i40e_virtchnl_pf_event *vpe =
 191                         (struct i40e_virtchnl_pf_event *)event->msg_buf;
 192
 193                 /* Initialize ret to sys event */
 194                 ret = I40EVF_MSG_SYS;
 195                 switch (vpe->event) {
 196                 case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
 197                         vf->link_up =
 198                                 vpe->event_data.link_event.link_status;
 199                         vf->pend_msg |= PFMSG_LINK_CHANGE;
 200                         PMD_DRV_LOG(INFO, "Link status update:%s\n",
 201                                         vf->link_up ? "up" : "down");
 202                         break;
 203                 case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
 204                         vf->vf_reset = true;
 205                         vf->pend_msg |= PFMSG_RESET_IMPENDING;
 206                         PMD_DRV_LOG(INFO, "vf is reseting\n");
 207                         break;
 208                 case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
 209                         vf->dev_closed = true;
 210                         vf->pend_msg |= PFMSG_DRIVER_CLOSE;
 211                         PMD_DRV_LOG(INFO, "PF driver closed\n");
 212                         break;
 213                 default:
 214                         PMD_DRV_LOG(ERR,
 215                                 "%s: Unknown event %d from pf\n",
 216                                 __func__, vpe->event);
 217                 }
 218         } else {
 219                 /* async reply msg on command issued by vf previously */
 220                 ret = I40EVF_MSG_CMD;
 221                 /* Actual buffer length read from PF */
 222                 data->msg_len = event->msg_size;
 223         }
 224         /* fill the ops and result to notify VF */
 225         data->result = retval;
 226         data->ops = opcode;
 227
 228         return ret;
 229 }
 230
 231 /*
 232  * Read data in admin queue to get msg from pf driver
 233  */
 234 static enum i40evf_aq_result
 235 i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 236 {
 237         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 238         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 239         struct i40e_arq_event_info event;
 240         int ret;
 241         enum i40evf_aq_result result = I40EVF_MSG_NON;
 242
 243         event.msg_size = data->msg_len;
 244         event.msg_buf = data->msg;
 245         ret = i40e_clean_arq_element(hw, &event, NULL);
 246         /* Can't read any msg from adminQ */
 247         if (ret) {
 248                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
 249                         result = I40EVF_MSG_NON;
 250                 else
 251                         result = I40EVF_MSG_ERR;
 252                 return result;
 253         }
 254
 255         /* Parse the event */
 256         result = i40evf_parse_pfmsg(vf, &event, data);
 257
 258         return result;
 259 }
 260
 261 /*
 262  * Polling read until command result return from pf driver or meet error.
 263  */
 264 static int
 265 i40evf_wait_cmd_done(struct rte_eth_dev *dev,
 266                      struct i40evf_arq_msg_info *data)
 267 {
 268         int i = 0;
 269         enum i40evf_aq_result ret;
 270
 271 #define MAX_TRY_TIMES 10
 272 #define ASQ_DELAY_MS  50
 273         do {
 274                 /* Delay some time first */
 275                 rte_delay_ms(ASQ_DELAY_MS);
 276                 ret = i40evf_read_pfmsg(dev, data);
 277
 278                 if (ret == I40EVF_MSG_CMD)
 279                         return 0;
 280                 else if (ret == I40EVF_MSG_ERR)
 281                         return -1;
 282
 283                 /* If don't read msg or read sys event, continue */
 284         } while(i++ < MAX_TRY_TIMES);
 285
 286         return -1;
 287 }
 288
 289 /**
 290  * clear current command. Only call in case execute
 291  * _atomic_set_cmd successfully.
 292  */
 293 static inline void
 294 _clear_cmd(struct i40e_vf *vf)
 295 {
 296         rte_wmb();
 297         vf->pend_cmd = I40E_VIRTCHNL_OP_UNKNOWN;
 298 }
 299
 300 /*
 301  * Check there is pending cmd in execution. If none, set new command.
 302  */
 303 static inline int
 304 _atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
 305 {
 306         int ret = rte_atomic32_cmpset(&vf->pend_cmd,
 307                         I40E_VIRTCHNL_OP_UNKNOWN, ops);
 308
 309         if (!ret)
 310                 PMD_DRV_LOG(ERR, "There is incomplete cmd %d\n", vf->pend_cmd);
 311
 312         return !ret;
 313 }
 314
 315 static int
 316 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 317 {
 318         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 319         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 320         int err = -1;
 321         struct i40evf_arq_msg_info info;
 322
 323         if (_atomic_set_cmd(vf, args->ops))
 324                 return -1;
 325
 326         info.msg = args->out_buffer;
 327         info.msg_len = args->out_size;
 328         info.ops = I40E_VIRTCHNL_OP_UNKNOWN;
 329         info.result = I40E_SUCCESS;
 330
 331         err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS,
 332                      args->in_args, args->in_args_size, NULL);
 333         if (err) {
 334                 PMD_DRV_LOG(ERR, "fail to send cmd %d\n", args->ops);
 335                 return err;
 336         }
 337
 338         err = i40evf_wait_cmd_done(dev, &info);
 339         /* read message and it's expected one */
 340         if (!err && args->ops == info.ops)
 341                 _clear_cmd(vf);
 342         else if (err)
 343                 PMD_DRV_LOG(ERR, "Failed to read message from AdminQ\n");
 344         else if (args->ops != info.ops)
 345                 PMD_DRV_LOG(ERR, "command mismatch, expect %u, get %u\n",
 346                                 args->ops, info.ops);
 347
 348         return (err | info.result);
 349 }
 350
 351 /*
 352  * Check API version with sync wait until version read or fail from admin queue
 353  */
 354 static int
 355 i40evf_check_api_version(struct rte_eth_dev *dev)
 356 {
 357         struct i40e_virtchnl_version_info version, *pver;
 358         int err;
 359         struct vf_cmd_info args;
 360         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 361
 362         version.major = I40E_VIRTCHNL_VERSION_MAJOR;
 363         version.minor = I40E_VIRTCHNL_VERSION_MINOR;
 364
 365         args.ops = I40E_VIRTCHNL_OP_VERSION;
 366         args.in_args = (uint8_t *)&version;
 367         args.in_args_size = sizeof(version);
 368         args.out_buffer = cmd_result_buffer;
 369         args.out_size = I40E_AQ_BUF_SZ;
 370
 371         err = i40evf_execute_vf_cmd(dev, &args);
 372         if (err) {
 373                 PMD_INIT_LOG(ERR, "fail to execute command OP_VERSION\n");
 374                 return err;
 375         }
 376
 377         pver = (struct i40e_virtchnl_version_info *)args.out_buffer;
 378         /* We are talking with DPDK host */
 379         if (pver->major == I40E_DPDK_VERSION_MAJOR) {
 380                 vf->host_is_dpdk = TRUE;
 381                 PMD_DRV_LOG(INFO, "Detect PF host is DPDK app\n");
 382         }
 383         /* It's linux host driver */
 384         else if ((pver->major != version.major) ||
 385             (pver->minor != version.minor)) {
 386                 PMD_INIT_LOG(ERR, "pf/vf API version mismatch. "
 387                         "(%u.%u)-(%u.%u)\n", pver->major, pver->minor,
 388                                         version.major, version.minor);
 389                 return -1;
 390         }
 391
 392         return 0;
 393 }
 394
 395 static int
 396 i40evf_get_vf_resource(struct rte_eth_dev *dev)
 397 {
 398         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 399         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 400         int err;
 401         struct vf_cmd_info args;
 402         uint32_t len;
 403
 404         args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
 405         args.in_args = NULL;
 406         args.in_args_size = 0;
 407         args.out_buffer = cmd_result_buffer;
 408         args.out_size = I40E_AQ_BUF_SZ;
 409
 410         err = i40evf_execute_vf_cmd(dev, &args);
 411
 412         if (err) {
 413                 PMD_DRV_LOG(ERR, "fail to execute command "
 414                                         "OP_GET_VF_RESOURCE\n");
 415                 return err;
 416         }
 417
 418         len =  sizeof(struct i40e_virtchnl_vf_resource) +
 419                 I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource);
 420
 421         (void)rte_memcpy(vf->vf_res, args.out_buffer,
 422                         RTE_MIN(args.out_size, len));
 423         i40e_vf_parse_hw_config(hw, vf->vf_res);
 424
 425         return 0;
 426 }
 427
 428 static int
 429 i40evf_config_promisc(struct rte_eth_dev *dev,
 430                       bool enable_unicast,
 431                       bool enable_multicast)
 432 {
 433         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 434         int err;
 435         struct vf_cmd_info args;
 436         struct i40e_virtchnl_promisc_info promisc;
 437
 438         promisc.flags = 0;
 439         promisc.vsi_id = vf->vsi_res->vsi_id;
 440
 441         if (enable_unicast)
 442                 promisc.flags |= I40E_FLAG_VF_UNICAST_PROMISC;
 443
 444         if (enable_multicast)
 445                 promisc.flags |= I40E_FLAG_VF_MULTICAST_PROMISC;
 446
 447         args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 448         args.in_args = (uint8_t *)&promisc;
 449         args.in_args_size = sizeof(promisc);
 450         args.out_buffer = cmd_result_buffer;
 451         args.out_size = I40E_AQ_BUF_SZ;
 452
 453         err = i40evf_execute_vf_cmd(dev, &args);
 454
 455         if (err)
 456                 PMD_DRV_LOG(ERR, "fail to execute command "
 457                                 "CONFIG_PROMISCUOUS_MODE\n");
 458         return err;
 459 }
 460
 461 /* Configure vlan and double vlan offload. Use flag to specify which part to configure */
 462 static int
 463 i40evf_config_vlan_offload(struct rte_eth_dev *dev,
 464                                 bool enable_vlan_strip)
 465 {
 466         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 467         int err;
 468         struct vf_cmd_info args;
 469         struct i40e_virtchnl_vlan_offload_info offload;
 470
 471         offload.vsi_id = vf->vsi_res->vsi_id;
 472         offload.enable_vlan_strip = enable_vlan_strip;
 473
 474         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
 475         args.in_args = (uint8_t *)&offload;
 476         args.in_args_size = sizeof(offload);
 477         args.out_buffer = cmd_result_buffer;
 478         args.out_size = I40E_AQ_BUF_SZ;
 479
 480         err = i40evf_execute_vf_cmd(dev, &args);
 481         if (err)
 482                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_OFFLOAD\n");
 483
 484         return err;
 485 }
 486
 487 static int
 488 i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
 489                                 struct i40e_vsi_vlan_pvid_info *info)
 490 {
 491         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 492         int err;
 493         struct vf_cmd_info args;
 494         struct i40e_virtchnl_pvid_info tpid_info;
 495
 496         if (dev == NULL || info == NULL) {
 497                 PMD_DRV_LOG(ERR, "invalid parameters\n");
 498                 return I40E_ERR_PARAM;
 499         }
 500
 501         memset(&tpid_info, 0, sizeof(tpid_info));
 502         tpid_info.vsi_id = vf->vsi_res->vsi_id;
 503         (void)rte_memcpy(&tpid_info.info, info, sizeof(*info));
 504
 505         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
 506         args.in_args = (uint8_t *)&tpid_info;
 507         args.in_args_size = sizeof(tpid_info);
 508         args.out_buffer = cmd_result_buffer;
 509         args.out_size = I40E_AQ_BUF_SZ;
 510
 511         err = i40evf_execute_vf_cmd(dev, &args);
 512         if (err)
 513                 PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_PVID\n");
 514
 515         return err;
 516 }
 517
 518 static int
 519 i40evf_configure_queues(struct rte_eth_dev *dev)
 520 {
 521         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 522         struct i40e_virtchnl_vsi_queue_config_info *queue_info;
 523         struct i40e_virtchnl_queue_pair_info *queue_cfg;
 524         struct i40e_rx_queue **rxq =
 525                 (struct i40e_rx_queue **)dev->data->rx_queues;
 526         struct i40e_tx_queue **txq =
 527                 (struct i40e_tx_queue **)dev->data->tx_queues;
 528         int i, len, nb_qpairs, num_rxq, num_txq;
 529         int err;
 530         struct vf_cmd_info args;
 531         struct rte_pktmbuf_pool_private *mbp_priv;
 532
 533         nb_qpairs = vf->num_queue_pairs;
 534         len = sizeof(*queue_info) + sizeof(*queue_cfg) * nb_qpairs;
 535         queue_info = rte_zmalloc("queue_info", len, 0);
 536         if (queue_info == NULL) {
 537                 PMD_INIT_LOG(ERR, "failed alloc memory for queue_info\n");
 538                 return -1;
 539         }
 540         queue_info->vsi_id = vf->vsi_res->vsi_id;
 541         queue_info->num_queue_pairs = nb_qpairs;
 542         queue_cfg = queue_info->qpair;
 543
 544         num_rxq = dev->data->nb_rx_queues;
 545         num_txq = dev->data->nb_tx_queues;
 546         /*
 547          * PF host driver required to configure queues in pairs, which means
 548          * rxq_num should equals to txq_num. The actual usage won't always
 549          * work that way. The solution is fills 0 with HW ring option in case
 550          * they are not equal.
 551          */
 552         for (i = 0; i < nb_qpairs; i++) {
 553                 /*Fill TX info */
 554                 queue_cfg->txq.vsi_id = queue_info->vsi_id;
 555                 queue_cfg->txq.queue_id = i;
 556                 if (i < num_txq) {
 557                         queue_cfg->txq.ring_len = txq[i]->nb_tx_desc;
 558                         queue_cfg->txq.dma_ring_addr = txq[i]->tx_ring_phys_addr;
 559                 } else {
 560                         queue_cfg->txq.ring_len = 0;
 561                         queue_cfg->txq.dma_ring_addr = 0;
 562                 }
 563
 564                 /* Fill RX info */
 565                 queue_cfg->rxq.vsi_id = queue_info->vsi_id;
 566                 queue_cfg->rxq.queue_id = i;
 567                 queue_cfg->rxq.max_pkt_size = vf->max_pkt_len;
 568                 if (i < num_rxq) {
 569                         mbp_priv = rte_mempool_get_priv(rxq[i]->mp);
 570                         queue_cfg->rxq.databuffer_size = mbp_priv->mbuf_data_room_size -
 571                                                    RTE_PKTMBUF_HEADROOM;;
 572                         queue_cfg->rxq.ring_len = rxq[i]->nb_rx_desc;
 573                         queue_cfg->rxq.dma_ring_addr = rxq[i]->rx_ring_phys_addr;;
 574                 } else {
 575                         queue_cfg->rxq.ring_len = 0;
 576                         queue_cfg->rxq.dma_ring_addr = 0;
 577                         queue_cfg->rxq.databuffer_size = 0;
 578                 }
 579                 queue_cfg++;
 580         }
 581
 582         args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 583         args.in_args = (u8 *)queue_info;
 584         args.in_args_size = len;
 585         args.out_buffer = cmd_result_buffer;
 586         args.out_size = I40E_AQ_BUF_SZ;
 587         err = i40evf_execute_vf_cmd(dev, &args);
 588         if (err)
 589                 PMD_DRV_LOG(ERR, "fail to execute command "
 590                                 "OP_CONFIG_VSI_QUEUES\n");
 591         rte_free(queue_info);
 592
 593         return err;
 594 }
 595
 596 static int
 597 i40evf_config_irq_map(struct rte_eth_dev *dev)
 598 {
 599         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 600         struct vf_cmd_info args;
 601         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_irq_map_info) + \
 602                 sizeof(struct i40e_virtchnl_vector_map)];
 603         struct i40e_virtchnl_irq_map_info *map_info;
 604         int i, err;
 605         map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
 606         map_info->num_vectors = 1;
 607         map_info->vecmap[0].rxitr_idx = RTE_LIBRTE_I40E_ITR_INTERVAL / 2;
 608         map_info->vecmap[0].txitr_idx = RTE_LIBRTE_I40E_ITR_INTERVAL / 2;
 609         map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 610         /* Alway use default dynamic MSIX interrupt */
 611         map_info->vecmap[0].vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR;
 612         /* Don't map any tx queue */
 613         map_info->vecmap[0].txq_map = 0;
 614         map_info->vecmap[0].rxq_map = 0;
 615         for (i = 0; i < dev->data->nb_rx_queues; i++)
 616                 map_info->vecmap[0].rxq_map |= 1 << i;
 617
 618         args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
 619         args.in_args = (u8 *)cmd_buffer;
 620         args.in_args_size = sizeof(cmd_buffer);
 621         args.out_buffer = cmd_result_buffer;
 622         args.out_size = I40E_AQ_BUF_SZ;
 623         err = i40evf_execute_vf_cmd(dev, &args);
 624         if (err)
 625                 PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES\n");
 626
 627         return err;
 628 }
 629
 630 static int
 631 i40evf_enable_queues(struct rte_eth_dev *dev)
 632 {
 633         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 634         struct i40e_virtchnl_queue_select queue_select;
 635         int err, i;
 636         struct vf_cmd_info args;
 637
 638         queue_select.vsi_id = vf->vsi_res->vsi_id;
 639
 640         queue_select.rx_queues = 0;
 641         /* Enable configured RX queues */
 642         for (i = 0; i < dev->data->nb_rx_queues; i++)
 643                 queue_select.rx_queues |= 1 << i;
 644
 645         /* Enable configured TX queues */
 646         queue_select.tx_queues = 0;
 647         for (i = 0; i < dev->data->nb_tx_queues; i++)
 648                 queue_select.tx_queues |= 1 << i;
 649
 650         args.ops = I40E_VIRTCHNL_OP_ENABLE_QUEUES;
 651         args.in_args = (u8 *)&queue_select;
 652         args.in_args_size = sizeof(queue_select);
 653         args.out_buffer = cmd_result_buffer;
 654         args.out_size = I40E_AQ_BUF_SZ;
 655         err = i40evf_execute_vf_cmd(dev, &args);
 656         if (err)
 657                 PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES\n");
 658
 659         return err;
 660 }
 661
 662 static int
 663 i40evf_disable_queues(struct rte_eth_dev *dev)
 664 {
 665         struct i40e_virtchnl_queue_select queue_select;
 666         int err, i;
 667         struct vf_cmd_info args;
 668
 669         /* Enable configured RX queues */
 670         queue_select.rx_queues = 0;
 671         for (i = 0; i < dev->data->nb_rx_queues; i++)
 672                 queue_select.rx_queues |= 1 << i;
 673
 674         /* Enable configured TX queues */
 675         queue_select.tx_queues = 0;
 676         for (i = 0; i < dev->data->nb_tx_queues; i++)
 677                 queue_select.tx_queues |= 1 << i;
 678
 679         args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
 680         args.in_args = (u8 *)&queue_select;
 681         args.in_args_size = sizeof(queue_select);
 682         args.out_buffer = cmd_result_buffer;
 683         args.out_size = I40E_AQ_BUF_SZ;
 684         err = i40evf_execute_vf_cmd(dev, &args);
 685         if (err)
 686                 PMD_DRV_LOG(ERR, "fail to execute command "
 687                                         "OP_DISABLE_QUEUES\n");
 688
 689         return err;
 690 }
 691
 692 static int
 693 i40evf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 694 {
 695         struct i40e_virtchnl_ether_addr_list *list;
 696         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 697         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 698                         sizeof(struct i40e_virtchnl_ether_addr)];
 699         int err;
 700         struct vf_cmd_info args;
 701
 702         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 703                 PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x\n",
 704                         addr->addr_bytes[0], addr->addr_bytes[1],
 705                         addr->addr_bytes[2], addr->addr_bytes[3],
 706                         addr->addr_bytes[4], addr->addr_bytes[5]);
 707                 return -1;
 708         }
 709
 710         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 711         list->vsi_id = vf->vsi_res->vsi_id;
 712         list->num_elements = 1;
 713         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 714                                         sizeof(addr->addr_bytes));
 715
 716         args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
 717         args.in_args = cmd_buffer;
 718         args.in_args_size = sizeof(cmd_buffer);
 719         args.out_buffer = cmd_result_buffer;
 720         args.out_size = I40E_AQ_BUF_SZ;
 721         err = i40evf_execute_vf_cmd(dev, &args);
 722         if (err)
 723                 PMD_DRV_LOG(ERR, "fail to execute command "
 724                                 "OP_ADD_ETHER_ADDRESS\n");
 725
 726         return err;
 727 }
 728
 729 static int
 730 i40evf_del_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
 731 {
 732         struct i40e_virtchnl_ether_addr_list *list;
 733         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 734         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
 735                         sizeof(struct i40e_virtchnl_ether_addr)];
 736         int err;
 737         struct vf_cmd_info args;
 738
 739         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 740                 PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x\n",
 741                         addr->addr_bytes[0], addr->addr_bytes[1],
 742                         addr->addr_bytes[2], addr->addr_bytes[3],
 743                         addr->addr_bytes[4], addr->addr_bytes[5]);
 744                 return -1;
 745         }
 746
 747         list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
 748         list->vsi_id = vf->vsi_res->vsi_id;
 749         list->num_elements = 1;
 750         (void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
 751                         sizeof(addr->addr_bytes));
 752
 753         args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
 754         args.in_args = cmd_buffer;
 755         args.in_args_size = sizeof(cmd_buffer);
 756         args.out_buffer = cmd_result_buffer;
 757         args.out_size = I40E_AQ_BUF_SZ;
 758         err = i40evf_execute_vf_cmd(dev, &args);
 759         if (err)
 760                 PMD_DRV_LOG(ERR, "fail to execute command "
 761                                 "OP_DEL_ETHER_ADDRESS\n");
 762
 763         return err;
 764 }
 765
 766 static int
 767 i40evf_get_statics(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 768 {
 769         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 770         struct i40e_virtchnl_queue_select q_stats;
 771         struct i40e_eth_stats *pstats;
 772         int err;
 773         struct vf_cmd_info args;
 774
 775         memset(&q_stats, 0, sizeof(q_stats));
 776         q_stats.vsi_id = vf->vsi_res->vsi_id;
 777         args.ops = I40E_VIRTCHNL_OP_GET_STATS;
 778         args.in_args = (u8 *)&q_stats;
 779         args.in_args_size = sizeof(q_stats);
 780         args.out_buffer = cmd_result_buffer;
 781         args.out_size = I40E_AQ_BUF_SZ;
 782
 783         err = i40evf_execute_vf_cmd(dev, &args);
 784         if (err) {
 785                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS\n");
 786                 return err;
 787         }
 788         pstats = (struct i40e_eth_stats *)args.out_buffer;
 789         stats->ipackets = pstats->rx_unicast + pstats->rx_multicast +
 790                                                 pstats->rx_broadcast;
 791         stats->opackets = pstats->tx_broadcast + pstats->tx_multicast +
 792                                                 pstats->tx_unicast;
 793         stats->ierrors = pstats->rx_discards;
 794         stats->oerrors = pstats->tx_errors + pstats->tx_discards;
 795         stats->ibytes = pstats->rx_bytes;
 796         stats->obytes = pstats->tx_bytes;
 797
 798         return 0;
 799 }
 800
 801 static int
 802 i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 803 {
 804         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 805         struct i40e_virtchnl_vlan_filter_list *vlan_list;
 806         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
 807                                                         sizeof(uint16_t)];
 808         int err;
 809         struct vf_cmd_info args;
 810
 811         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
 812         vlan_list->vsi_id = vf->vsi_res->vsi_id;
 813         vlan_list->num_elements = 1;
 814         vlan_list->vlan_id[0] = vlanid;
 815
 816         args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
 817         args.in_args = (u8 *)&cmd_buffer;
 818         args.in_args_size = sizeof(cmd_buffer);
 819         args.out_buffer = cmd_result_buffer;
 820         args.out_size = I40E_AQ_BUF_SZ;
 821         err = i40evf_execute_vf_cmd(dev, &args);
 822         if (err)
 823                 PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_VLAN\n");
 824
 825         return err;
 826 }
 827
 828 static int
 829 i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 830 {
 831         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 832         struct i40e_virtchnl_vlan_filter_list *vlan_list;
 833         uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
 834                                                         sizeof(uint16_t)];
 835         int err;
 836         struct vf_cmd_info args;
 837
 838         vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
 839         vlan_list->vsi_id = vf->vsi_res->vsi_id;
 840         vlan_list->num_elements = 1;
 841         vlan_list->vlan_id[0] = vlanid;
 842
 843         args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
 844         args.in_args = (u8 *)&cmd_buffer;
 845         args.in_args_size = sizeof(cmd_buffer);
 846         args.out_buffer = cmd_result_buffer;
 847         args.out_size = I40E_AQ_BUF_SZ;
 848         err = i40evf_execute_vf_cmd(dev, &args);
 849         if (err)
 850                 PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_VLAN\n");
 851
 852         return err;
 853 }
 854
 855 static int
 856 i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link)
 857 {
 858         int err;
 859         struct vf_cmd_info args;
 860         struct rte_eth_link *new_link;
 861
 862         args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT;
 863         args.in_args = NULL;
 864         args.in_args_size = 0;
 865         args.out_buffer = cmd_result_buffer;
 866         args.out_size = I40E_AQ_BUF_SZ;
 867         err = i40evf_execute_vf_cmd(dev, &args);
 868         if (err) {
 869                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_LINK_STAT\n");
 870                 return err;
 871         }
 872
 873         new_link = (struct rte_eth_link *)args.out_buffer;
 874         (void)rte_memcpy(link, new_link, sizeof(*link));
 875
 876         return 0;
 877 }
 878
 879 static struct rte_pci_id pci_id_i40evf_map[] = {
 880 #define RTE_PCI_DEV_ID_DECL_I40EVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 881 #include "rte_pci_dev_ids.h"
 882 { .vendor_id = 0, /* sentinel */ },
 883 };
 884
 885 static inline int
 886 i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 887                                     struct rte_eth_link *link)
 888 {
 889         struct rte_eth_link *dst = &(dev->data->dev_link);
 890         struct rte_eth_link *src = link;
 891
 892         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 893                                         *(uint64_t *)src) == 0)
 894                 return -1;
 895
 896         return 0;
 897 }
 898
 899 static int
 900 i40evf_reset_vf(struct i40e_hw *hw)
 901 {
 902         int i, reset;
 903
 904         if (i40e_vf_reset(hw) != I40E_SUCCESS) {
 905                 PMD_INIT_LOG(ERR, "Reset VF NIC failed\n");
 906                 return -1;
 907         }
 908         /**
 909           * After issuing vf reset command to pf, pf won't necessarily
 910           * reset vf, it depends on what state it exactly is. If it's not
 911           * initialized yet, it won't have vf reset since it's in a certain
 912           * state. If not, it will try to reset. Even vf is reset, pf will
 913           * set I40E_VFGEN_RSTAT to COMPLETE first, then wait 10ms and set
 914           * it to ACTIVE. In this duration, vf may not catch the moment that
 915           * COMPLETE is set. So, for vf, we'll try to wait a long time.
 916           */
 917         rte_delay_ms(200);
 918
 919         for (i = 0; i < MAX_RESET_WAIT_CNT; i++) {
 920                 reset = rd32(hw, I40E_VFGEN_RSTAT) &
 921                         I40E_VFGEN_RSTAT_VFR_STATE_MASK;
 922                 reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT;
 923                 if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset)
 924                         break;
 925                 else
 926                         rte_delay_ms(50);
 927         }
 928
 929         if (i >= MAX_RESET_WAIT_CNT) {
 930                 PMD_INIT_LOG(ERR, "Reset VF NIC failed\n");
 931                 return -1;
 932         }
 933
 934         return 0;
 935 }
 936
 937 static int
 938 i40evf_init_vf(struct rte_eth_dev *dev)
 939 {
 940         int i, err, bufsz;
 941         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 942         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 943
 944         err = i40evf_set_mac_type(hw);
 945         if (err) {
 946                 PMD_INIT_LOG(ERR, "set_mac_type failed: %d\n", err);
 947                 goto err;
 948         }
 949
 950         i40e_init_adminq_parameter(hw);
 951         err = i40e_init_adminq(hw);
 952         if (err) {
 953                 PMD_INIT_LOG(ERR, "init_adminq failed: %d\n", err);
 954                 goto err;
 955         }
 956
 957
 958         /* Reset VF and wait until it's complete */
 959         if (i40evf_reset_vf(hw)) {
 960                 PMD_INIT_LOG(ERR, "reset NIC failed\n");
 961                 goto err_aq;
 962         }
 963
 964         /* VF reset, shutdown admin queue and initialize again */
 965         if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) {
 966                 PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed\n");
 967                 return -1;
 968         }
 969
 970         i40e_init_adminq_parameter(hw);
 971         if (i40e_init_adminq(hw) != I40E_SUCCESS) {
 972                 PMD_INIT_LOG(ERR, "init_adminq failed\n");
 973                 return -1;
 974         }
 975         if (i40evf_check_api_version(dev) != 0) {
 976                 PMD_INIT_LOG(ERR, "check_api version failed\n");
 977                 goto err_aq;
 978         }
 979         bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
 980                 (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource));
 981         vf->vf_res = rte_zmalloc("vf_res", bufsz, 0);
 982         if (!vf->vf_res) {
 983                 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory\n");
 984                         goto err_aq;
 985         }
 986
 987         if (i40evf_get_vf_resource(dev) != 0) {
 988                 PMD_INIT_LOG(ERR, "i40evf_get_vf_config failed\n");
 989                 goto err_alloc;
 990         }
 991
 992         /* got VF config message back from PF, now we can parse it */
 993         for (i = 0; i < vf->vf_res->num_vsis; i++) {
 994                 if (vf->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
 995                         vf->vsi_res = &vf->vf_res->vsi_res[i];
 996         }
 997
 998         if (!vf->vsi_res) {
 999                 PMD_INIT_LOG(ERR, "no LAN VSI found\n");
1000                 goto err_alloc;
1001         }
1002
1003         vf->vsi.vsi_id = vf->vsi_res->vsi_id;
1004         vf->vsi.type = vf->vsi_res->vsi_type;
1005         vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs;
1006
1007         /* check mac addr, if it's not valid, genrate one */
1008         if (I40E_SUCCESS != i40e_validate_mac_addr(\
1009                         vf->vsi_res->default_mac_addr))
1010                 eth_random_addr(vf->vsi_res->default_mac_addr);
1011
1012         ether_addr_copy((struct ether_addr *)vf->vsi_res->default_mac_addr,
1013                                         (struct ether_addr *)hw->mac.addr);
1014
1015         return 0;
1016
1017 err_alloc:
1018         rte_free(vf->vf_res);
1019 err_aq:
1020         i40e_shutdown_adminq(hw); /* ignore error */
1021 err:
1022         return -1;
1023 }
1024
1025 static int
1026 i40evf_dev_init(__rte_unused struct eth_driver *eth_drv,
1027                 struct rte_eth_dev *eth_dev)
1028 {
1029         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
1030                         eth_dev->data->dev_private);
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* assign ops func pointer */
1035         eth_dev->dev_ops = &i40evf_eth_dev_ops;
1036         eth_dev->rx_pkt_burst = &i40e_recv_pkts;
1037         eth_dev->tx_pkt_burst = &i40e_xmit_pkts;
1038
1039         /*
1040          * For secondary processes, we don't initialise any further as primary
1041          * has already done this work.
1042          */
1043         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1044                 if (eth_dev->data->scattered_rx)
1045                         eth_dev->rx_pkt_burst = i40e_recv_scattered_pkts;
1046                 return 0;
1047         }
1048
1049         hw->vendor_id = eth_dev->pci_dev->id.vendor_id;
1050         hw->device_id = eth_dev->pci_dev->id.device_id;
1051         hw->subsystem_vendor_id = eth_dev->pci_dev->id.subsystem_vendor_id;
1052         hw->subsystem_device_id = eth_dev->pci_dev->id.subsystem_device_id;
1053         hw->bus.device = eth_dev->pci_dev->addr.devid;
1054         hw->bus.func = eth_dev->pci_dev->addr.function;
1055         hw->hw_addr = (void *)eth_dev->pci_dev->mem_resource[0].addr;
1056
1057         if(i40evf_init_vf(eth_dev) != 0) {
1058                 PMD_INIT_LOG(ERR, "Init vf failed\n");
1059                 return -1;
1060         }
1061
1062         /* copy mac addr */
1063         eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
1064                                         ETHER_ADDR_LEN, 0);
1065         if (eth_dev->data->mac_addrs == NULL) {
1066                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
1067                                 "store MAC addresses", ETHER_ADDR_LEN);
1068                 return -ENOMEM;
1069         }
1070         ether_addr_copy((struct ether_addr *)hw->mac.addr,
1071                 (struct ether_addr *)eth_dev->data->mac_addrs);
1072
1073         return 0;
1074 }
1075
1076 /*
1077  * virtual function driver struct
1078  */
1079 static struct eth_driver rte_i40evf_pmd = {
1080         {
1081                 .name = "rte_i40evf_pmd",
1082                 .id_table = pci_id_i40evf_map,
1083                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1084         },
1085         .eth_dev_init = i40evf_dev_init,
1086         .dev_private_size = sizeof(struct i40e_vf),
1087 };
1088
1089 /*
1090  * VF Driver initialization routine.
1091  * Invoked one at EAL init time.
1092  * Register itself as the [Virtual Poll Mode] Driver of PCI Fortville devices.
1093  */
1094 static int
1095 rte_i40evf_pmd_init(const char *name __rte_unused,
1096                     const char *params __rte_unused)
1097 {
1098         DEBUGFUNC("rte_i40evf_pmd_init");
1099
1100         rte_eth_driver_register(&rte_i40evf_pmd);
1101
1102         return 0;
1103 }
1104
1105 static struct rte_driver rte_i40evf_driver = {
1106         .type = PMD_PDEV,
1107         .init = rte_i40evf_pmd_init,
1108 };
1109
1110 PMD_REGISTER_DRIVER(rte_i40evf_driver);
1111
1112 static int
1113 i40evf_dev_configure(struct rte_eth_dev *dev)
1114 {
1115         return i40evf_init_vlan(dev);
1116 }
1117
1118 static int
1119 i40evf_init_vlan(struct rte_eth_dev *dev)
1120 {
1121         struct rte_eth_dev_data *data = dev->data;
1122         int ret;
1123
1124         /* Apply vlan offload setting */
1125         i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
1126
1127         /* Apply pvid setting */
1128         ret = i40evf_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
1129                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
1130         return ret;
1131 }
1132
1133 static void
1134 i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1135 {
1136         bool enable_vlan_strip = 0;
1137         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1138         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1139
1140         /* Linux pf host doesn't support vlan offload yet */
1141         if (vf->host_is_dpdk) {
1142                 /* Vlan stripping setting */
1143                 if (mask & ETH_VLAN_STRIP_MASK) {
1144                         /* Enable or disable VLAN stripping */
1145                         if (dev_conf->rxmode.hw_vlan_strip)
1146                                 enable_vlan_strip = 1;
1147                         else
1148                                 enable_vlan_strip = 0;
1149
1150                         i40evf_config_vlan_offload(dev, enable_vlan_strip);
1151                 }
1152         }
1153 }
1154
1155 static int
1156 i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1157 {
1158         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1159         struct i40e_vsi_vlan_pvid_info info;
1160         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1161
1162         memset(&info, 0, sizeof(info));
1163         info.on = on;
1164
1165         /* Linux pf host don't support vlan offload yet */
1166         if (vf->host_is_dpdk) {
1167                 if (info.on)
1168                         info.config.pvid = pvid;
1169                 else {
1170                         info.config.reject.tagged =
1171                                 dev_conf->txmode.hw_vlan_reject_tagged;
1172                         info.config.reject.untagged =
1173                                 dev_conf->txmode.hw_vlan_reject_untagged;
1174                 }
1175                 return i40evf_config_vlan_pvid(dev, &info);
1176         }
1177
1178         return 0;
1179 }
1180
1181 static int
1182 i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1183 {
1184         int ret;
1185
1186         if (on)
1187                 ret = i40evf_add_vlan(dev, vlan_id);
1188         else
1189                 ret = i40evf_del_vlan(dev,vlan_id);
1190
1191         return ret;
1192 }
1193
1194 static int
1195 i40evf_rx_init(struct rte_eth_dev *dev)
1196 {
1197         uint16_t i, j;
1198         struct i40e_rx_queue **rxq =
1199                 (struct i40e_rx_queue **)dev->data->rx_queues;
1200         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1201
1202         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1203                 if (i40e_alloc_rx_queue_mbufs(rxq[i]) != 0) {
1204                         PMD_DRV_LOG(ERR, "alloc rx queues mbufs failed\n");
1205                         goto err;
1206                 }
1207                 rxq[i]->qrx_tail = hw->hw_addr + I40E_QRX_TAIL1(i);
1208                 I40E_PCI_REG_WRITE(rxq[i]->qrx_tail, rxq[i]->nb_rx_desc - 1);
1209         }
1210
1211         /* Flush the operation to write registers */
1212         I40EVF_WRITE_FLUSH(hw);
1213
1214         return 0;
1215
1216 err:
1217         /* Release all mbufs */
1218         for (j = 0; j < i; j++)
1219                 i40e_rx_queue_release_mbufs(rxq[j]);
1220
1221         return -1;
1222 }
1223
1224 static void
1225 i40evf_tx_init(struct rte_eth_dev *dev)
1226 {
1227         uint16_t i;
1228         struct i40e_tx_queue **txq =
1229                 (struct i40e_tx_queue **)dev->data->tx_queues;
1230         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1231
1232         for (i = 0; i < dev->data->nb_tx_queues; i++)
1233                 txq[i]->qtx_tail = hw->hw_addr + I40E_QTX_TAIL1(i);
1234 }
1235
1236 static inline void
1237 i40evf_enable_queues_intr(struct i40e_hw *hw)
1238 {
1239         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
1240                         I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1241                         I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
1242 }
1243
1244 static inline void
1245 i40evf_disable_queues_intr(struct i40e_hw *hw)
1246 {
1247         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
1248                         0);
1249 }
1250
1251 static int
1252 i40evf_dev_start(struct rte_eth_dev *dev)
1253 {
1254         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1255         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1256         struct ether_addr mac_addr;
1257
1258         PMD_DRV_LOG(DEBUG, "i40evf_dev_start");
1259
1260         vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1261         if (dev->data->dev_conf.rxmode.jumbo_frame == 1) {
1262                 if (vf->max_pkt_len <= ETHER_MAX_LEN ||
1263                         vf->max_pkt_len > I40E_FRAME_SIZE_MAX) {
1264                         PMD_DRV_LOG(ERR, "maximum packet length must "
1265                                 "be larger than %u and smaller than %u,"
1266                                         "as jumbo frame is enabled\n",
1267                                                 (uint32_t)ETHER_MAX_LEN,
1268                                         (uint32_t)I40E_FRAME_SIZE_MAX);
1269                         return I40E_ERR_CONFIG;
1270                 }
1271         } else {
1272                 if (vf->max_pkt_len < ETHER_MIN_LEN ||
1273                         vf->max_pkt_len > ETHER_MAX_LEN) {
1274                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1275                                         "larger than %u and smaller than %u, "
1276                                         "as jumbo frame is disabled\n",
1277                                                 (uint32_t)ETHER_MIN_LEN,
1278                                                 (uint32_t)ETHER_MAX_LEN);
1279                         return I40E_ERR_CONFIG;
1280                 }
1281         }
1282
1283         vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
1284                                         dev->data->nb_tx_queues);
1285
1286         if (i40evf_rx_init(dev) != 0){
1287                 PMD_DRV_LOG(ERR, "failed to do RX init\n");
1288                 return -1;
1289         }
1290
1291         i40evf_tx_init(dev);
1292
1293         if (i40evf_configure_queues(dev) != 0) {
1294                 PMD_DRV_LOG(ERR, "configure queues failed\n");
1295                 goto err_queue;
1296         }
1297         if (i40evf_config_irq_map(dev)) {
1298                 PMD_DRV_LOG(ERR, "config_irq_map failed\n");
1299                 goto err_queue;
1300         }
1301
1302         /* Set mac addr */
1303         (void)rte_memcpy(mac_addr.addr_bytes, hw->mac.addr,
1304                                 sizeof(mac_addr.addr_bytes));
1305         if (i40evf_add_mac_addr(dev, &mac_addr)) {
1306                 PMD_DRV_LOG(ERR, "Failed to add mac addr\n");
1307                 goto err_queue;
1308         }
1309
1310         if (i40evf_enable_queues(dev) != 0) {
1311                 PMD_DRV_LOG(ERR, "enable queues failed\n");
1312                 goto err_mac;
1313         }
1314         i40evf_enable_queues_intr(hw);
1315         return 0;
1316
1317 err_mac:
1318         i40evf_del_mac_addr(dev, &mac_addr);
1319 err_queue:
1320         i40e_dev_clear_queues(dev);
1321         return -1;
1322 }
1323
1324 static void
1325 i40evf_dev_stop(struct rte_eth_dev *dev)
1326 {
1327         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1328
1329         PMD_INIT_FUNC_TRACE();
1330
1331         i40evf_disable_queues_intr(hw);
1332         i40evf_disable_queues(dev);
1333         i40e_dev_clear_queues(dev);
1334 }
1335
1336 static int
1337 i40evf_dev_link_update(struct rte_eth_dev *dev,
1338                        __rte_unused int wait_to_complete)
1339 {
1340         struct rte_eth_link new_link;
1341         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1342         /*
1343          * DPDK pf host provide interfacet to acquire link status
1344          * while Linux driver does not
1345          */
1346         if (vf->host_is_dpdk)
1347                 i40evf_get_link_status(dev, &new_link);
1348         else {
1349                 /* Always assume it's up, for Linux driver PF host */
1350                 new_link.link_duplex = ETH_LINK_AUTONEG_DUPLEX;
1351                 new_link.link_speed  = ETH_LINK_SPEED_10000;
1352                 new_link.link_status = 1;
1353         }
1354         i40evf_dev_atomic_write_link_status(dev, &new_link);
1355
1356         return 0;
1357 }
1358
1359 static void
1360 i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev)
1361 {
1362         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1363         int ret;
1364
1365         /* If enabled, just return */
1366         if (vf->promisc_unicast_enabled)
1367                 return;
1368
1369         ret = i40evf_config_promisc(dev, 1, vf->promisc_multicast_enabled);
1370         if (ret == 0)
1371                 vf->promisc_unicast_enabled = TRUE;
1372 }
1373
1374 static void
1375 i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev)
1376 {
1377         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1378         int ret;
1379
1380         /* If disabled, just return */
1381         if (!vf->promisc_unicast_enabled)
1382                 return;
1383
1384         ret = i40evf_config_promisc(dev, 0, vf->promisc_multicast_enabled);
1385         if (ret == 0)
1386                 vf->promisc_unicast_enabled = FALSE;
1387 }
1388
1389 static void
1390 i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev)
1391 {
1392         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1393         int ret;
1394
1395         /* If enabled, just return */
1396         if (vf->promisc_multicast_enabled)
1397                 return;
1398
1399         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 1);
1400         if (ret == 0)
1401                 vf->promisc_multicast_enabled = TRUE;
1402 }
1403
1404 static void
1405 i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev)
1406 {
1407         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1408         int ret;
1409
1410         /* If enabled, just return */
1411         if (!vf->promisc_multicast_enabled)
1412                 return;
1413
1414         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 0);
1415         if (ret == 0)
1416                 vf->promisc_multicast_enabled = FALSE;
1417 }
1418
1419 static void
1420 i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1421 {
1422         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1423
1424         memset(dev_info, 0, sizeof(*dev_info));
1425         dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
1426         dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
1427         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
1428         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
1429 }
1430
1431 static void
1432 i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1433 {
1434         memset(stats, 0, sizeof(*stats));
1435         if (i40evf_get_statics(dev, stats))
1436                 PMD_DRV_LOG(ERR, "Get statics failed\n");
1437 }
1438
1439 static void
1440 i40evf_dev_close(struct rte_eth_dev *dev)
1441 {
1442         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1443
1444         i40evf_dev_stop(dev);
1445         i40evf_reset_vf(hw);
1446         i40e_shutdown_adminq(hw);
1447 }