drivers/net/i40e/i40e_ethdev_vf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12 #include <inttypes.h>
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <rte_cycles.h>
  16
  17 #include <rte_interrupts.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_pci.h>
  21 #include <rte_bus_pci.h>
  22 #include <rte_atomic.h>
  23 #include <rte_branch_prediction.h>
  24 #include <rte_memory.h>
  25 #include <rte_eal.h>
  26 #include <rte_alarm.h>
  27 #include <rte_ether.h>
  28 #include <rte_ethdev_driver.h>
  29 #include <rte_ethdev_pci.h>
  30 #include <rte_malloc.h>
  31 #include <rte_dev.h>
  32
  33 #include "i40e_logs.h"
  34 #include "base/i40e_prototype.h"
  35 #include "base/i40e_adminq_cmd.h"
  36 #include "base/i40e_type.h"
  37
  38 #include "i40e_rxtx.h"
  39 #include "i40e_ethdev.h"
  40 #include "i40e_pf.h"
  41
  42 /* busy wait delay in msec */
  43 #define I40EVF_BUSY_WAIT_DELAY 10
  44 #define I40EVF_BUSY_WAIT_COUNT 50
  45 #define MAX_RESET_WAIT_CNT     20
  46
  47 struct i40evf_arq_msg_info {
  48         enum virtchnl_ops ops;
  49         enum i40e_status_code result;
  50         uint16_t buf_len;
  51         uint16_t msg_len;
  52         uint8_t *msg;
  53 };
  54
  55 struct vf_cmd_info {
  56         enum virtchnl_ops ops;
  57         uint8_t *in_args;
  58         uint32_t in_args_size;
  59         uint8_t *out_buffer;
  60         /* Input & output type. pass in buffer size and pass out
  61          * actual return result
  62          */
  63         uint32_t out_size;
  64 };
  65
  66 enum i40evf_aq_result {
  67         I40EVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
  68         I40EVF_MSG_NON,      /* Read nothing from admin queue */
  69         I40EVF_MSG_SYS,      /* Read system msg from admin queue */
  70         I40EVF_MSG_CMD,      /* Read async command result */
  71 };
  72
  73 static int i40evf_dev_configure(struct rte_eth_dev *dev);
  74 static int i40evf_dev_start(struct rte_eth_dev *dev);
  75 static void i40evf_dev_stop(struct rte_eth_dev *dev);
  76 static void i40evf_dev_info_get(struct rte_eth_dev *dev,
  77                                 struct rte_eth_dev_info *dev_info);
  78 static int i40evf_dev_link_update(struct rte_eth_dev *dev,
  79                                   int wait_to_complete);
  80 static int i40evf_dev_stats_get(struct rte_eth_dev *dev,
  81                                 struct rte_eth_stats *stats);
  82 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
  83                                  struct rte_eth_xstat *xstats, unsigned n);
  84 static int i40evf_dev_xstats_get_names(struct rte_eth_dev *dev,
  85                                        struct rte_eth_xstat_name *xstats_names,
  86                                        unsigned limit);
  87 static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev);
  88 static int i40evf_vlan_filter_set(struct rte_eth_dev *dev,
  89                                   uint16_t vlan_id, int on);
  90 static int i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  91 static void i40evf_dev_close(struct rte_eth_dev *dev);
  92 static int  i40evf_dev_reset(struct rte_eth_dev *dev);
  93 static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev);
  94 static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev);
  95 static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev);
  96 static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev);
  97 static int i40evf_init_vlan(struct rte_eth_dev *dev);
  98 static int i40evf_dev_rx_queue_start(struct rte_eth_dev *dev,
  99                                      uint16_t rx_queue_id);
 100 static int i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev,
 101                                     uint16_t rx_queue_id);
 102 static int i40evf_dev_tx_queue_start(struct rte_eth_dev *dev,
 103                                      uint16_t tx_queue_id);
 104 static int i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev,
 105                                     uint16_t tx_queue_id);
 106 static int i40evf_add_mac_addr(struct rte_eth_dev *dev,
 107                                struct ether_addr *addr,
 108                                uint32_t index,
 109                                uint32_t pool);
 110 static void i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 111 static int i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
 112                         struct rte_eth_rss_reta_entry64 *reta_conf,
 113                         uint16_t reta_size);
 114 static int i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
 115                         struct rte_eth_rss_reta_entry64 *reta_conf,
 116                         uint16_t reta_size);
 117 static int i40evf_config_rss(struct i40e_vf *vf);
 118 static int i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
 119                                       struct rte_eth_rss_conf *rss_conf);
 120 static int i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 121                                         struct rte_eth_rss_conf *rss_conf);
 122 static int i40evf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 123 static void i40evf_set_default_mac_addr(struct rte_eth_dev *dev,
 124                                         struct ether_addr *mac_addr);
 125 static int
 126 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 127 static int
 128 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 129 static void i40evf_handle_pf_event(struct rte_eth_dev *dev,
 130                                    uint8_t *msg,
 131                                    uint16_t msglen);
 132
 133 /* Default hash key buffer for RSS */
 134 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
 135
 136 struct rte_i40evf_xstats_name_off {
 137         char name[RTE_ETH_XSTATS_NAME_SIZE];
 138         unsigned offset;
 139 };
 140
 141 static const struct rte_i40evf_xstats_name_off rte_i40evf_stats_strings[] = {
 142         {"rx_bytes", offsetof(struct i40e_eth_stats, rx_bytes)},
 143         {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
 144         {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
 145         {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
 146         {"rx_dropped_packets", offsetof(struct i40e_eth_stats, rx_discards)},
 147         {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
 148                 rx_unknown_protocol)},
 149         {"tx_bytes", offsetof(struct i40e_eth_stats, tx_bytes)},
 150         {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)},
 151         {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)},
 152         {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)},
 153         {"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_discards)},
 154         {"tx_error_packets", offsetof(struct i40e_eth_stats, tx_errors)},
 155 };
 156
 157 #define I40EVF_NB_XSTATS (sizeof(rte_i40evf_stats_strings) / \
 158                 sizeof(rte_i40evf_stats_strings[0]))
 159
 160 static const struct eth_dev_ops i40evf_eth_dev_ops = {
 161         .dev_configure        = i40evf_dev_configure,
 162         .dev_start            = i40evf_dev_start,
 163         .dev_stop             = i40evf_dev_stop,
 164         .promiscuous_enable   = i40evf_dev_promiscuous_enable,
 165         .promiscuous_disable  = i40evf_dev_promiscuous_disable,
 166         .allmulticast_enable  = i40evf_dev_allmulticast_enable,
 167         .allmulticast_disable = i40evf_dev_allmulticast_disable,
 168         .link_update          = i40evf_dev_link_update,
 169         .stats_get            = i40evf_dev_stats_get,
 170         .stats_reset          = i40evf_dev_xstats_reset,
 171         .xstats_get           = i40evf_dev_xstats_get,
 172         .xstats_get_names     = i40evf_dev_xstats_get_names,
 173         .xstats_reset         = i40evf_dev_xstats_reset,
 174         .dev_close            = i40evf_dev_close,
 175         .dev_reset            = i40evf_dev_reset,
 176         .dev_infos_get        = i40evf_dev_info_get,
 177         .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get,
 178         .vlan_filter_set      = i40evf_vlan_filter_set,
 179         .vlan_offload_set     = i40evf_vlan_offload_set,
 180         .rx_queue_start       = i40evf_dev_rx_queue_start,
 181         .rx_queue_stop        = i40evf_dev_rx_queue_stop,
 182         .tx_queue_start       = i40evf_dev_tx_queue_start,
 183         .tx_queue_stop        = i40evf_dev_tx_queue_stop,
 184         .rx_queue_setup       = i40e_dev_rx_queue_setup,
 185         .rx_queue_release     = i40e_dev_rx_queue_release,
 186         .rx_queue_intr_enable = i40evf_dev_rx_queue_intr_enable,
 187         .rx_queue_intr_disable = i40evf_dev_rx_queue_intr_disable,
 188         .rx_descriptor_done   = i40e_dev_rx_descriptor_done,
 189         .rx_descriptor_status = i40e_dev_rx_descriptor_status,
 190         .tx_descriptor_status = i40e_dev_tx_descriptor_status,
 191         .tx_queue_setup       = i40e_dev_tx_queue_setup,
 192         .tx_queue_release     = i40e_dev_tx_queue_release,
 193         .rx_queue_count       = i40e_dev_rx_queue_count,
 194         .rxq_info_get         = i40e_rxq_info_get,
 195         .txq_info_get         = i40e_txq_info_get,
 196         .mac_addr_add         = i40evf_add_mac_addr,
 197         .mac_addr_remove      = i40evf_del_mac_addr,
 198         .reta_update          = i40evf_dev_rss_reta_update,
 199         .reta_query           = i40evf_dev_rss_reta_query,
 200         .rss_hash_update      = i40evf_dev_rss_hash_update,
 201         .rss_hash_conf_get    = i40evf_dev_rss_hash_conf_get,
 202         .mtu_set              = i40evf_dev_mtu_set,
 203         .mac_addr_set         = i40evf_set_default_mac_addr,
 204 };
 205
 206 /*
 207  * Read data in admin queue to get msg from pf driver
 208  */
 209 static enum i40evf_aq_result
 210 i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
 211 {
 212         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 213         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 214         struct i40e_arq_event_info event;
 215         enum virtchnl_ops opcode;
 216         enum i40e_status_code retval;
 217         int ret;
 218         enum i40evf_aq_result result = I40EVF_MSG_NON;
 219
 220         event.buf_len = data->buf_len;
 221         event.msg_buf = data->msg;
 222         ret = i40e_clean_arq_element(hw, &event, NULL);
 223         /* Can't read any msg from adminQ */
 224         if (ret) {
 225                 if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
 226                         result = I40EVF_MSG_ERR;
 227                 return result;
 228         }
 229
 230         opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
 231         retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
 232         /* pf sys event */
 233         if (opcode == VIRTCHNL_OP_EVENT) {
 234                 struct virtchnl_pf_event *vpe =
 235                         (struct virtchnl_pf_event *)event.msg_buf;
 236
 237                 result = I40EVF_MSG_SYS;
 238                 switch (vpe->event) {
 239                 case VIRTCHNL_EVENT_LINK_CHANGE:
 240                         vf->link_up =
 241                                 vpe->event_data.link_event.link_status;
 242                         vf->link_speed =
 243                                 vpe->event_data.link_event.link_speed;
 244                         vf->pend_msg |= PFMSG_LINK_CHANGE;
 245                         PMD_DRV_LOG(INFO, "Link status update:%s",
 246                                     vf->link_up ? "up" : "down");
 247                         break;
 248                 case VIRTCHNL_EVENT_RESET_IMPENDING:
 249                         vf->vf_reset = true;
 250                         vf->pend_msg |= PFMSG_RESET_IMPENDING;
 251                         PMD_DRV_LOG(INFO, "vf is reseting");
 252                         break;
 253                 case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
 254                         vf->dev_closed = true;
 255                         vf->pend_msg |= PFMSG_DRIVER_CLOSE;
 256                         PMD_DRV_LOG(INFO, "PF driver closed");
 257                         break;
 258                 default:
 259                         PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
 260                                     __func__, vpe->event);
 261                 }
 262         } else {
 263                 /* async reply msg on command issued by vf previously */
 264                 result = I40EVF_MSG_CMD;
 265                 /* Actual data length read from PF */
 266                 data->msg_len = event.msg_len;
 267         }
 268
 269         data->result = retval;
 270         data->ops = opcode;
 271
 272         return result;
 273 }
 274
 275 /**
 276  * clear current command. Only call in case execute
 277  * _atomic_set_cmd successfully.
 278  */
 279 static inline void
 280 _clear_cmd(struct i40e_vf *vf)
 281 {
 282         rte_wmb();
 283         vf->pend_cmd = VIRTCHNL_OP_UNKNOWN;
 284 }
 285
 286 /*
 287  * Check there is pending cmd in execution. If none, set new command.
 288  */
 289 static inline int
 290 _atomic_set_cmd(struct i40e_vf *vf, enum virtchnl_ops ops)
 291 {
 292         int ret = rte_atomic32_cmpset(&vf->pend_cmd,
 293                         VIRTCHNL_OP_UNKNOWN, ops);
 294
 295         if (!ret)
 296                 PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd);
 297
 298         return !ret;
 299 }
 300
 301 #define MAX_TRY_TIMES 200
 302 #define ASQ_DELAY_MS  10
 303
 304 static int
 305 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
 306 {
 307         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 308         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 309         struct i40evf_arq_msg_info info;
 310         enum i40evf_aq_result ret;
 311         int err, i = 0;
 312
 313         if (_atomic_set_cmd(vf, args->ops))
 314                 return -1;
 315
 316         info.msg = args->out_buffer;
 317         info.buf_len = args->out_size;
 318         info.ops = VIRTCHNL_OP_UNKNOWN;
 319         info.result = I40E_SUCCESS;
 320
 321         err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS,
 322                      args->in_args, args->in_args_size, NULL);
 323         if (err) {
 324                 PMD_DRV_LOG(ERR, "fail to send cmd %d", args->ops);
 325                 _clear_cmd(vf);
 326                 return err;
 327         }
 328
 329         switch (args->ops) {
 330         case VIRTCHNL_OP_RESET_VF:
 331                 /*no need to process in this function */
 332                 err = 0;
 333                 break;
 334         case VIRTCHNL_OP_VERSION:
 335         case VIRTCHNL_OP_GET_VF_RESOURCES:
 336                 /* for init adminq commands, need to poll the response */
 337                 err = -1;
 338                 do {
 339                         ret = i40evf_read_pfmsg(dev, &info);
 340                         vf->cmd_retval = info.result;
 341                         if (ret == I40EVF_MSG_CMD) {
 342                                 err = 0;
 343                                 break;
 344                         } else if (ret == I40EVF_MSG_ERR)
 345                                 break;
 346                         rte_delay_ms(ASQ_DELAY_MS);
 347                         /* If don't read msg or read sys event, continue */
 348                 } while (i++ < MAX_TRY_TIMES);
 349                 _clear_cmd(vf);
 350                 break;
 351
 352         default:
 353                 /* for other adminq in running time, waiting the cmd done flag */
 354                 err = -1;
 355                 do {
 356                         if (vf->pend_cmd == VIRTCHNL_OP_UNKNOWN) {
 357                                 err = 0;
 358                                 break;
 359                         }
 360                         rte_delay_ms(ASQ_DELAY_MS);
 361                         /* If don't read msg or read sys event, continue */
 362                 } while (i++ < MAX_TRY_TIMES);
 363                 /* If there's no response is received, clear command */
 364                 if (i >= MAX_TRY_TIMES) {
 365                         PMD_DRV_LOG(WARNING, "No response for %d", args->ops);
 366                         _clear_cmd(vf);
 367                 }
 368                 break;
 369         }
 370
 371         return err | vf->cmd_retval;
 372 }
 373
 374 /*
 375  * Check API version with sync wait until version read or fail from admin queue
 376  */
 377 static int
 378 i40evf_check_api_version(struct rte_eth_dev *dev)
 379 {
 380         struct virtchnl_version_info version, *pver;
 381         int err;
 382         struct vf_cmd_info args;
 383         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 384
 385         version.major = VIRTCHNL_VERSION_MAJOR;
 386         version.minor = VIRTCHNL_VERSION_MINOR;
 387
 388         args.ops = VIRTCHNL_OP_VERSION;
 389         args.in_args = (uint8_t *)&version;
 390         args.in_args_size = sizeof(version);
 391         args.out_buffer = vf->aq_resp;
 392         args.out_size = I40E_AQ_BUF_SZ;
 393
 394         err = i40evf_execute_vf_cmd(dev, &args);
 395         if (err) {
 396                 PMD_INIT_LOG(ERR, "fail to execute command OP_VERSION");
 397                 return err;
 398         }
 399
 400         pver = (struct virtchnl_version_info *)args.out_buffer;
 401         vf->version_major = pver->major;
 402         vf->version_minor = pver->minor;
 403         if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) &&
 404                 (vf->version_minor <= VIRTCHNL_VERSION_MINOR))
 405                 PMD_DRV_LOG(INFO, "Peer is Linux PF host");
 406         else {
 407                 PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)",
 408                                         vf->version_major, vf->version_minor,
 409                                                 VIRTCHNL_VERSION_MAJOR,
 410                                                 VIRTCHNL_VERSION_MINOR);
 411                 return -1;
 412         }
 413
 414         return 0;
 415 }
 416
 417 static int
 418 i40evf_get_vf_resource(struct rte_eth_dev *dev)
 419 {
 420         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 421         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 422         int err;
 423         struct vf_cmd_info args;
 424         uint32_t caps, len;
 425
 426         args.ops = VIRTCHNL_OP_GET_VF_RESOURCES;
 427         args.out_buffer = vf->aq_resp;
 428         args.out_size = I40E_AQ_BUF_SZ;
 429         if (PF_IS_V11(vf)) {
 430                 caps = VIRTCHNL_VF_OFFLOAD_L2 |
 431                        VIRTCHNL_VF_OFFLOAD_RSS_AQ |
 432                        VIRTCHNL_VF_OFFLOAD_RSS_REG |
 433                        VIRTCHNL_VF_OFFLOAD_VLAN |
 434                        VIRTCHNL_VF_OFFLOAD_RX_POLLING;
 435                 args.in_args = (uint8_t *)&caps;
 436                 args.in_args_size = sizeof(caps);
 437         } else {
 438                 args.in_args = NULL;
 439                 args.in_args_size = 0;
 440         }
 441         err = i40evf_execute_vf_cmd(dev, &args);
 442
 443         if (err) {
 444                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_VF_RESOURCE");
 445                 return err;
 446         }
 447
 448         len =  sizeof(struct virtchnl_vf_resource) +
 449                 I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource);
 450
 451         rte_memcpy(vf->vf_res, args.out_buffer,
 452                         RTE_MIN(args.out_size, len));
 453         i40e_vf_parse_hw_config(hw, vf->vf_res);
 454
 455         return 0;
 456 }
 457
 458 static int
 459 i40evf_config_promisc(struct rte_eth_dev *dev,
 460                       bool enable_unicast,
 461                       bool enable_multicast)
 462 {
 463         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 464         int err;
 465         struct vf_cmd_info args;
 466         struct virtchnl_promisc_info promisc;
 467
 468         promisc.flags = 0;
 469         promisc.vsi_id = vf->vsi_res->vsi_id;
 470
 471         if (enable_unicast)
 472                 promisc.flags |= FLAG_VF_UNICAST_PROMISC;
 473
 474         if (enable_multicast)
 475                 promisc.flags |= FLAG_VF_MULTICAST_PROMISC;
 476
 477         args.ops = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
 478         args.in_args = (uint8_t *)&promisc;
 479         args.in_args_size = sizeof(promisc);
 480         args.out_buffer = vf->aq_resp;
 481         args.out_size = I40E_AQ_BUF_SZ;
 482
 483         err = i40evf_execute_vf_cmd(dev, &args);
 484
 485         if (err)
 486                 PMD_DRV_LOG(ERR, "fail to execute command "
 487                             "CONFIG_PROMISCUOUS_MODE");
 488         return err;
 489 }
 490
 491 static int
 492 i40evf_enable_vlan_strip(struct rte_eth_dev *dev)
 493 {
 494         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 495         struct vf_cmd_info args;
 496         int ret;
 497
 498         memset(&args, 0, sizeof(args));
 499         args.ops = VIRTCHNL_OP_ENABLE_VLAN_STRIPPING;
 500         args.in_args = NULL;
 501         args.in_args_size = 0;
 502         args.out_buffer = vf->aq_resp;
 503         args.out_size = I40E_AQ_BUF_SZ;
 504         ret = i40evf_execute_vf_cmd(dev, &args);
 505         if (ret)
 506                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 507                             "VIRTCHNL_OP_ENABLE_VLAN_STRIPPING");
 508
 509         return ret;
 510 }
 511
 512 static int
 513 i40evf_disable_vlan_strip(struct rte_eth_dev *dev)
 514 {
 515         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 516         struct vf_cmd_info args;
 517         int ret;
 518
 519         memset(&args, 0, sizeof(args));
 520         args.ops = VIRTCHNL_OP_DISABLE_VLAN_STRIPPING;
 521         args.in_args = NULL;
 522         args.in_args_size = 0;
 523         args.out_buffer = vf->aq_resp;
 524         args.out_size = I40E_AQ_BUF_SZ;
 525         ret = i40evf_execute_vf_cmd(dev, &args);
 526         if (ret)
 527                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 528                             "VIRTCHNL_OP_DISABLE_VLAN_STRIPPING");
 529
 530         return ret;
 531 }
 532
 533 static void
 534 i40evf_fill_virtchnl_vsi_txq_info(struct virtchnl_txq_info *txq_info,
 535                                   uint16_t vsi_id,
 536                                   uint16_t queue_id,
 537                                   uint16_t nb_txq,
 538                                   struct i40e_tx_queue *txq)
 539 {
 540         txq_info->vsi_id = vsi_id;
 541         txq_info->queue_id = queue_id;
 542         if (queue_id < nb_txq) {
 543                 txq_info->ring_len = txq->nb_tx_desc;
 544                 txq_info->dma_ring_addr = txq->tx_ring_phys_addr;
 545         }
 546 }
 547
 548 static void
 549 i40evf_fill_virtchnl_vsi_rxq_info(struct virtchnl_rxq_info *rxq_info,
 550                                   uint16_t vsi_id,
 551                                   uint16_t queue_id,
 552                                   uint16_t nb_rxq,
 553                                   uint32_t max_pkt_size,
 554                                   struct i40e_rx_queue *rxq)
 555 {
 556         rxq_info->vsi_id = vsi_id;
 557         rxq_info->queue_id = queue_id;
 558         rxq_info->max_pkt_size = max_pkt_size;
 559         if (queue_id < nb_rxq) {
 560                 rxq_info->ring_len = rxq->nb_rx_desc;
 561                 rxq_info->dma_ring_addr = rxq->rx_ring_phys_addr;
 562                 rxq_info->databuffer_size =
 563                         (rte_pktmbuf_data_room_size(rxq->mp) -
 564                                 RTE_PKTMBUF_HEADROOM);
 565         }
 566 }
 567
 568 static int
 569 i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
 570 {
 571         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 572         struct i40e_rx_queue **rxq =
 573                 (struct i40e_rx_queue **)dev->data->rx_queues;
 574         struct i40e_tx_queue **txq =
 575                 (struct i40e_tx_queue **)dev->data->tx_queues;
 576         struct virtchnl_vsi_queue_config_info *vc_vqci;
 577         struct virtchnl_queue_pair_info *vc_qpi;
 578         struct vf_cmd_info args;
 579         uint16_t i, nb_qp = vf->num_queue_pairs;
 580         const uint32_t size =
 581                 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqci, nb_qp);
 582         uint8_t buff[size];
 583         int ret;
 584
 585         memset(buff, 0, sizeof(buff));
 586         vc_vqci = (struct virtchnl_vsi_queue_config_info *)buff;
 587         vc_vqci->vsi_id = vf->vsi_res->vsi_id;
 588         vc_vqci->num_queue_pairs = nb_qp;
 589
 590         for (i = 0, vc_qpi = vc_vqci->qpair; i < nb_qp; i++, vc_qpi++) {
 591                 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpi->txq,
 592                         vc_vqci->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
 593                 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpi->rxq,
 594                         vc_vqci->vsi_id, i, dev->data->nb_rx_queues,
 595                                         vf->max_pkt_len, rxq[i]);
 596         }
 597         memset(&args, 0, sizeof(args));
 598         args.ops = VIRTCHNL_OP_CONFIG_VSI_QUEUES;
 599         args.in_args = (uint8_t *)vc_vqci;
 600         args.in_args_size = size;
 601         args.out_buffer = vf->aq_resp;
 602         args.out_size = I40E_AQ_BUF_SZ;
 603         ret = i40evf_execute_vf_cmd(dev, &args);
 604         if (ret)
 605                 PMD_DRV_LOG(ERR, "Failed to execute command of "
 606                         "VIRTCHNL_OP_CONFIG_VSI_QUEUES");
 607
 608         return ret;
 609 }
 610
 611 static int
 612 i40evf_config_irq_map(struct rte_eth_dev *dev)
 613 {
 614         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 615         struct vf_cmd_info args;
 616         uint8_t cmd_buffer[sizeof(struct virtchnl_irq_map_info) + \
 617                 sizeof(struct virtchnl_vector_map)];
 618         struct virtchnl_irq_map_info *map_info;
 619         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 620         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 621         uint32_t vector_id;
 622         int i, err;
 623
 624         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
 625             rte_intr_allow_others(intr_handle))
 626                 vector_id = I40E_RX_VEC_START;
 627         else
 628                 vector_id = I40E_MISC_VEC_ID;
 629
 630         map_info = (struct virtchnl_irq_map_info *)cmd_buffer;
 631         map_info->num_vectors = 1;
 632         map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
 633         map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
 634         /* Alway use default dynamic MSIX interrupt */
 635         map_info->vecmap[0].vector_id = vector_id;
 636         /* Don't map any tx queue */
 637         map_info->vecmap[0].txq_map = 0;
 638         map_info->vecmap[0].rxq_map = 0;
 639         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 640                 map_info->vecmap[0].rxq_map |= 1 << i;
 641                 if (rte_intr_dp_is_en(intr_handle))
 642                         intr_handle->intr_vec[i] = vector_id;
 643         }
 644
 645         args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
 646         args.in_args = (u8 *)cmd_buffer;
 647         args.in_args_size = sizeof(cmd_buffer);
 648         args.out_buffer = vf->aq_resp;
 649         args.out_size = I40E_AQ_BUF_SZ;
 650         err = i40evf_execute_vf_cmd(dev, &args);
 651         if (err)
 652                 PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES");
 653
 654         return err;
 655 }
 656
 657 static int
 658 i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
 659                                 bool on)
 660 {
 661         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 662         struct virtchnl_queue_select queue_select;
 663         int err;
 664         struct vf_cmd_info args;
 665         memset(&queue_select, 0, sizeof(queue_select));
 666         queue_select.vsi_id = vf->vsi_res->vsi_id;
 667
 668         if (isrx)
 669                 queue_select.rx_queues |= 1 << qid;
 670         else
 671                 queue_select.tx_queues |= 1 << qid;
 672
 673         if (on)
 674                 args.ops = VIRTCHNL_OP_ENABLE_QUEUES;
 675         else
 676                 args.ops = VIRTCHNL_OP_DISABLE_QUEUES;
 677         args.in_args = (u8 *)&queue_select;
 678         args.in_args_size = sizeof(queue_select);
 679         args.out_buffer = vf->aq_resp;
 680         args.out_size = I40E_AQ_BUF_SZ;
 681         err = i40evf_execute_vf_cmd(dev, &args);
 682         if (err)
 683                 PMD_DRV_LOG(ERR, "fail to switch %s %u %s",
 684                             isrx ? "RX" : "TX", qid, on ? "on" : "off");
 685
 686         return err;
 687 }
 688
 689 static int
 690 i40evf_start_queues(struct rte_eth_dev *dev)
 691 {
 692         struct rte_eth_dev_data *dev_data = dev->data;
 693         int i;
 694         struct i40e_rx_queue *rxq;
 695         struct i40e_tx_queue *txq;
 696
 697         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 698                 rxq = dev_data->rx_queues[i];
 699                 if (rxq->rx_deferred_start)
 700                         continue;
 701                 if (i40evf_dev_rx_queue_start(dev, i) != 0) {
 702                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 703                         return -1;
 704                 }
 705         }
 706
 707         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 708                 txq = dev_data->tx_queues[i];
 709                 if (txq->tx_deferred_start)
 710                         continue;
 711                 if (i40evf_dev_tx_queue_start(dev, i) != 0) {
 712                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 713                         return -1;
 714                 }
 715         }
 716
 717         return 0;
 718 }
 719
 720 static int
 721 i40evf_stop_queues(struct rte_eth_dev *dev)
 722 {
 723         int i;
 724
 725         /* Stop TX queues first */
 726         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 727                 if (i40evf_dev_tx_queue_stop(dev, i) != 0) {
 728                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 729                         return -1;
 730                 }
 731         }
 732
 733         /* Then stop RX queues */
 734         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 735                 if (i40evf_dev_rx_queue_stop(dev, i) != 0) {
 736                         PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
 737                         return -1;
 738                 }
 739         }
 740
 741         return 0;
 742 }
 743
 744 static int
 745 i40evf_add_mac_addr(struct rte_eth_dev *dev,
 746                     struct ether_addr *addr,
 747                     __rte_unused uint32_t index,
 748                     __rte_unused uint32_t pool)
 749 {
 750         struct virtchnl_ether_addr_list *list;
 751         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 752         uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \
 753                         sizeof(struct virtchnl_ether_addr)];
 754         int err;
 755         struct vf_cmd_info args;
 756
 757         if (is_zero_ether_addr(addr)) {
 758                 PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x",
 759                             addr->addr_bytes[0], addr->addr_bytes[1],
 760                             addr->addr_bytes[2], addr->addr_bytes[3],
 761                             addr->addr_bytes[4], addr->addr_bytes[5]);
 762                 return I40E_ERR_INVALID_MAC_ADDR;
 763         }
 764
 765         list = (struct virtchnl_ether_addr_list *)cmd_buffer;
 766         list->vsi_id = vf->vsi_res->vsi_id;
 767         list->num_elements = 1;
 768         rte_memcpy(list->list[0].addr, addr->addr_bytes,
 769                                         sizeof(addr->addr_bytes));
 770
 771         args.ops = VIRTCHNL_OP_ADD_ETH_ADDR;
 772         args.in_args = cmd_buffer;
 773         args.in_args_size = sizeof(cmd_buffer);
 774         args.out_buffer = vf->aq_resp;
 775         args.out_size = I40E_AQ_BUF_SZ;
 776         err = i40evf_execute_vf_cmd(dev, &args);
 777         if (err)
 778                 PMD_DRV_LOG(ERR, "fail to execute command "
 779                             "OP_ADD_ETHER_ADDRESS");
 780         else
 781                 vf->vsi.mac_num++;
 782
 783         return err;
 784 }
 785
 786 static void
 787 i40evf_del_mac_addr_by_addr(struct rte_eth_dev *dev,
 788                             struct ether_addr *addr)
 789 {
 790         struct virtchnl_ether_addr_list *list;
 791         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 792         uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \
 793                         sizeof(struct virtchnl_ether_addr)];
 794         int err;
 795         struct vf_cmd_info args;
 796
 797         if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
 798                 PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x",
 799                             addr->addr_bytes[0], addr->addr_bytes[1],
 800                             addr->addr_bytes[2], addr->addr_bytes[3],
 801                             addr->addr_bytes[4], addr->addr_bytes[5]);
 802                 return;
 803         }
 804
 805         list = (struct virtchnl_ether_addr_list *)cmd_buffer;
 806         list->vsi_id = vf->vsi_res->vsi_id;
 807         list->num_elements = 1;
 808         rte_memcpy(list->list[0].addr, addr->addr_bytes,
 809                         sizeof(addr->addr_bytes));
 810
 811         args.ops = VIRTCHNL_OP_DEL_ETH_ADDR;
 812         args.in_args = cmd_buffer;
 813         args.in_args_size = sizeof(cmd_buffer);
 814         args.out_buffer = vf->aq_resp;
 815         args.out_size = I40E_AQ_BUF_SZ;
 816         err = i40evf_execute_vf_cmd(dev, &args);
 817         if (err)
 818                 PMD_DRV_LOG(ERR, "fail to execute command "
 819                             "OP_DEL_ETHER_ADDRESS");
 820         else
 821                 vf->vsi.mac_num--;
 822         return;
 823 }
 824
 825 static void
 826 i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index)
 827 {
 828         struct rte_eth_dev_data *data = dev->data;
 829         struct ether_addr *addr;
 830
 831         addr = &data->mac_addrs[index];
 832
 833         i40evf_del_mac_addr_by_addr(dev, addr);
 834 }
 835
 836 static int
 837 i40evf_query_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
 838 {
 839         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 840         struct virtchnl_queue_select q_stats;
 841         int err;
 842         struct vf_cmd_info args;
 843
 844         memset(&q_stats, 0, sizeof(q_stats));
 845         q_stats.vsi_id = vf->vsi_res->vsi_id;
 846         args.ops = VIRTCHNL_OP_GET_STATS;
 847         args.in_args = (u8 *)&q_stats;
 848         args.in_args_size = sizeof(q_stats);
 849         args.out_buffer = vf->aq_resp;
 850         args.out_size = I40E_AQ_BUF_SZ;
 851
 852         err = i40evf_execute_vf_cmd(dev, &args);
 853         if (err) {
 854                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS");
 855                 *pstats = NULL;
 856                 return err;
 857         }
 858         *pstats = (struct i40e_eth_stats *)args.out_buffer;
 859         return 0;
 860 }
 861
 862 static void
 863 i40evf_stat_update_48(uint64_t *offset,
 864                    uint64_t *stat)
 865 {
 866         if (*stat >= *offset)
 867                 *stat = *stat - *offset;
 868         else
 869                 *stat = (uint64_t)((*stat +
 870                         ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset);
 871
 872         *stat &= I40E_48_BIT_MASK;
 873 }
 874
 875 static void
 876 i40evf_stat_update_32(uint64_t *offset,
 877                    uint64_t *stat)
 878 {
 879         if (*stat >= *offset)
 880                 *stat = (uint64_t)(*stat - *offset);
 881         else
 882                 *stat = (uint64_t)((*stat +
 883                         ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset);
 884 }
 885
 886 static void
 887 i40evf_update_stats(struct i40e_vsi *vsi,
 888                                         struct i40e_eth_stats *nes)
 889 {
 890         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
 891
 892         i40evf_stat_update_48(&oes->rx_bytes,
 893                             &nes->rx_bytes);
 894         i40evf_stat_update_48(&oes->rx_unicast,
 895                             &nes->rx_unicast);
 896         i40evf_stat_update_48(&oes->rx_multicast,
 897                             &nes->rx_multicast);
 898         i40evf_stat_update_48(&oes->rx_broadcast,
 899                             &nes->rx_broadcast);
 900         i40evf_stat_update_32(&oes->rx_discards,
 901                                 &nes->rx_discards);
 902         i40evf_stat_update_32(&oes->rx_unknown_protocol,
 903                             &nes->rx_unknown_protocol);
 904         i40evf_stat_update_48(&oes->tx_bytes,
 905                             &nes->tx_bytes);
 906         i40evf_stat_update_48(&oes->tx_unicast,
 907                             &nes->tx_unicast);
 908         i40evf_stat_update_48(&oes->tx_multicast,
 909                             &nes->tx_multicast);
 910         i40evf_stat_update_48(&oes->tx_broadcast,
 911                             &nes->tx_broadcast);
 912         i40evf_stat_update_32(&oes->tx_errors, &nes->tx_errors);
 913         i40evf_stat_update_32(&oes->tx_discards, &nes->tx_discards);
 914 }
 915
 916 static void
 917 i40evf_dev_xstats_reset(struct rte_eth_dev *dev)
 918 {
 919         int ret;
 920         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 921         struct i40e_eth_stats *pstats = NULL;
 922
 923         /* read stat values to clear hardware registers */
 924         ret = i40evf_query_stats(dev, &pstats);
 925
 926         /* set stats offset base on current values */
 927         if (ret == 0)
 928                 vf->vsi.eth_stats_offset = *pstats;
 929 }
 930
 931 static int i40evf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
 932                                       struct rte_eth_xstat_name *xstats_names,
 933                                       __rte_unused unsigned limit)
 934 {
 935         unsigned i;
 936
 937         if (xstats_names != NULL)
 938                 for (i = 0; i < I40EVF_NB_XSTATS; i++) {
 939                         snprintf(xstats_names[i].name,
 940                                 sizeof(xstats_names[i].name),
 941                                 "%s", rte_i40evf_stats_strings[i].name);
 942                 }
 943         return I40EVF_NB_XSTATS;
 944 }
 945
 946 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
 947                                  struct rte_eth_xstat *xstats, unsigned n)
 948 {
 949         int ret;
 950         unsigned i;
 951         struct i40e_eth_stats *pstats = NULL;
 952         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 953         struct i40e_vsi *vsi = &vf->vsi;
 954
 955         if (n < I40EVF_NB_XSTATS)
 956                 return I40EVF_NB_XSTATS;
 957
 958         ret = i40evf_query_stats(dev, &pstats);
 959         if (ret != 0)
 960                 return 0;
 961
 962         if (!xstats)
 963                 return 0;
 964
 965         i40evf_update_stats(vsi, pstats);
 966
 967         /* loop over xstats array and values from pstats */
 968         for (i = 0; i < I40EVF_NB_XSTATS; i++) {
 969                 xstats[i].id = i;
 970                 xstats[i].value = *(uint64_t *)(((char *)pstats) +
 971                         rte_i40evf_stats_strings[i].offset);
 972         }
 973
 974         return I40EVF_NB_XSTATS;
 975 }
 976
 977 static int
 978 i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
 979 {
 980         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 981         struct virtchnl_vlan_filter_list *vlan_list;
 982         uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) +
 983                                                         sizeof(uint16_t)];
 984         int err;
 985         struct vf_cmd_info args;
 986
 987         vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer;
 988         vlan_list->vsi_id = vf->vsi_res->vsi_id;
 989         vlan_list->num_elements = 1;
 990         vlan_list->vlan_id[0] = vlanid;
 991
 992         args.ops = VIRTCHNL_OP_ADD_VLAN;
 993         args.in_args = (u8 *)&cmd_buffer;
 994         args.in_args_size = sizeof(cmd_buffer);
 995         args.out_buffer = vf->aq_resp;
 996         args.out_size = I40E_AQ_BUF_SZ;
 997         err = i40evf_execute_vf_cmd(dev, &args);
 998         if (err)
 999                 PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_VLAN");
1000
1001         return err;
1002 }
1003
1004 static int
1005 i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1006 {
1007         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1008         struct virtchnl_vlan_filter_list *vlan_list;
1009         uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) +
1010                                                         sizeof(uint16_t)];
1011         int err;
1012         struct vf_cmd_info args;
1013
1014         vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer;
1015         vlan_list->vsi_id = vf->vsi_res->vsi_id;
1016         vlan_list->num_elements = 1;
1017         vlan_list->vlan_id[0] = vlanid;
1018
1019         args.ops = VIRTCHNL_OP_DEL_VLAN;
1020         args.in_args = (u8 *)&cmd_buffer;
1021         args.in_args_size = sizeof(cmd_buffer);
1022         args.out_buffer = vf->aq_resp;
1023         args.out_size = I40E_AQ_BUF_SZ;
1024         err = i40evf_execute_vf_cmd(dev, &args);
1025         if (err)
1026                 PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_VLAN");
1027
1028         return err;
1029 }
1030
1031 static const struct rte_pci_id pci_id_i40evf_map[] = {
1032         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF) },
1033         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV) },
1034         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF) },
1035         { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF) },
1036         { .vendor_id = 0, /* sentinel */ },
1037 };
1038
1039 /* Disable IRQ0 */
1040 static inline void
1041 i40evf_disable_irq0(struct i40e_hw *hw)
1042 {
1043         /* Disable all interrupt types */
1044         I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
1045         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1046                        I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1047         I40EVF_WRITE_FLUSH(hw);
1048 }
1049
1050 /* Enable IRQ0 */
1051 static inline void
1052 i40evf_enable_irq0(struct i40e_hw *hw)
1053 {
1054         /* Enable admin queue interrupt trigger */
1055         uint32_t val;
1056
1057         i40evf_disable_irq0(hw);
1058         val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
1059         val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
1060                 I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
1061         I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
1062
1063         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1064                 I40E_VFINT_DYN_CTL01_INTENA_MASK |
1065                 I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1066                 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1067
1068         I40EVF_WRITE_FLUSH(hw);
1069 }
1070
1071 static int
1072 i40evf_check_vf_reset_done(struct i40e_hw *hw)
1073 {
1074         int i, reset;
1075
1076         for (i = 0; i < MAX_RESET_WAIT_CNT; i++) {
1077                 reset = I40E_READ_REG(hw, I40E_VFGEN_RSTAT) &
1078                         I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1079                 reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT;
1080                 if (reset == VIRTCHNL_VFR_VFACTIVE ||
1081                     reset == VIRTCHNL_VFR_COMPLETED)
1082                         break;
1083                 rte_delay_ms(50);
1084         }
1085
1086         if (i >= MAX_RESET_WAIT_CNT)
1087                 return -1;
1088
1089         return 0;
1090 }
1091 static int
1092 i40evf_reset_vf(struct i40e_hw *hw)
1093 {
1094         int ret;
1095
1096         if (i40e_vf_reset(hw) != I40E_SUCCESS) {
1097                 PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1098                 return -1;
1099         }
1100         /**
1101           * After issuing vf reset command to pf, pf won't necessarily
1102           * reset vf, it depends on what state it exactly is. If it's not
1103           * initialized yet, it won't have vf reset since it's in a certain
1104           * state. If not, it will try to reset. Even vf is reset, pf will
1105           * set I40E_VFGEN_RSTAT to COMPLETE first, then wait 10ms and set
1106           * it to ACTIVE. In this duration, vf may not catch the moment that
1107           * COMPLETE is set. So, for vf, we'll try to wait a long time.
1108           */
1109         rte_delay_ms(200);
1110
1111         ret = i40evf_check_vf_reset_done(hw);
1112         if (ret) {
1113                 PMD_INIT_LOG(ERR, "VF is still resetting");
1114                 return ret;
1115         }
1116
1117         return 0;
1118 }
1119
1120 static int
1121 i40evf_init_vf(struct rte_eth_dev *dev)
1122 {
1123         int i, err, bufsz;
1124         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1125         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1126         uint16_t interval =
1127                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL, 0, 0);
1128
1129         vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1130         vf->dev_data = dev->data;
1131         err = i40e_set_mac_type(hw);
1132         if (err) {
1133                 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err);
1134                 goto err;
1135         }
1136
1137         err = i40evf_check_vf_reset_done(hw);
1138         if (err)
1139                 goto err;
1140
1141         i40e_init_adminq_parameter(hw);
1142         err = i40e_init_adminq(hw);
1143         if (err) {
1144                 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err);
1145                 goto err;
1146         }
1147
1148         /* Reset VF and wait until it's complete */
1149         if (i40evf_reset_vf(hw)) {
1150                 PMD_INIT_LOG(ERR, "reset NIC failed");
1151                 goto err_aq;
1152         }
1153
1154         /* VF reset, shutdown admin queue and initialize again */
1155         if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) {
1156                 PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed");
1157                 goto err;
1158         }
1159
1160         i40e_init_adminq_parameter(hw);
1161         if (i40e_init_adminq(hw) != I40E_SUCCESS) {
1162                 PMD_INIT_LOG(ERR, "init_adminq failed");
1163                 goto err;
1164         }
1165
1166         vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
1167         if (!vf->aq_resp) {
1168                 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
1169                 goto err_aq;
1170         }
1171         if (i40evf_check_api_version(dev) != 0) {
1172                 PMD_INIT_LOG(ERR, "check_api version failed");
1173                 goto err_api;
1174         }
1175         bufsz = sizeof(struct virtchnl_vf_resource) +
1176                 (I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource));
1177         vf->vf_res = rte_zmalloc("vf_res", bufsz, 0);
1178         if (!vf->vf_res) {
1179                 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory");
1180                 goto err_api;
1181         }
1182
1183         if (i40evf_get_vf_resource(dev) != 0) {
1184                 PMD_INIT_LOG(ERR, "i40evf_get_vf_config failed");
1185                 goto err_alloc;
1186         }
1187
1188         /* got VF config message back from PF, now we can parse it */
1189         for (i = 0; i < vf->vf_res->num_vsis; i++) {
1190                 if (vf->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
1191                         vf->vsi_res = &vf->vf_res->vsi_res[i];
1192         }
1193
1194         if (!vf->vsi_res) {
1195                 PMD_INIT_LOG(ERR, "no LAN VSI found");
1196                 goto err_alloc;
1197         }
1198
1199         if (hw->mac.type == I40E_MAC_X722_VF)
1200                 vf->flags = I40E_FLAG_RSS_AQ_CAPABLE;
1201         vf->vsi.vsi_id = vf->vsi_res->vsi_id;
1202
1203         switch (vf->vsi_res->vsi_type) {
1204         case VIRTCHNL_VSI_SRIOV:
1205                 vf->vsi.type = I40E_VSI_SRIOV;
1206                 break;
1207         default:
1208                 vf->vsi.type = I40E_VSI_TYPE_UNKNOWN;
1209                 break;
1210         }
1211         vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs;
1212         vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1213
1214         /* Store the MAC address configured by host, or generate random one */
1215         if (is_valid_assigned_ether_addr((struct ether_addr *)hw->mac.addr))
1216                 vf->flags |= I40E_FLAG_VF_MAC_BY_PF;
1217         else
1218                 eth_random_addr(hw->mac.addr); /* Generate a random one */
1219
1220         I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1221                        (I40E_ITR_INDEX_DEFAULT <<
1222                         I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
1223                        (interval <<
1224                         I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
1225         I40EVF_WRITE_FLUSH(hw);
1226
1227         return 0;
1228
1229 err_alloc:
1230         rte_free(vf->vf_res);
1231         vf->vsi_res = NULL;
1232 err_api:
1233         rte_free(vf->aq_resp);
1234 err_aq:
1235         i40e_shutdown_adminq(hw); /* ignore error */
1236 err:
1237         return -1;
1238 }
1239
1240 static int
1241 i40evf_uninit_vf(struct rte_eth_dev *dev)
1242 {
1243         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1244         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1245
1246         PMD_INIT_FUNC_TRACE();
1247
1248         if (hw->adapter_stopped == 0)
1249                 i40evf_dev_close(dev);
1250         rte_free(vf->vf_res);
1251         vf->vf_res = NULL;
1252         rte_free(vf->aq_resp);
1253         vf->aq_resp = NULL;
1254
1255         return 0;
1256 }
1257
1258 static void
1259 i40evf_handle_pf_event(struct rte_eth_dev *dev, uint8_t *msg,
1260                 __rte_unused uint16_t msglen)
1261 {
1262         struct virtchnl_pf_event *pf_msg =
1263                         (struct virtchnl_pf_event *)msg;
1264         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1265
1266         switch (pf_msg->event) {
1267         case VIRTCHNL_EVENT_RESET_IMPENDING:
1268                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event");
1269                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET,
1270                                               NULL);
1271                 break;
1272         case VIRTCHNL_EVENT_LINK_CHANGE:
1273                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event");
1274                 vf->link_up = pf_msg->event_data.link_event.link_status;
1275                 vf->link_speed = pf_msg->event_data.link_event.link_speed;
1276                 break;
1277         case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
1278                 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event");
1279                 break;
1280         default:
1281                 PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
1282                 break;
1283         }
1284 }
1285
1286 static void
1287 i40evf_handle_aq_msg(struct rte_eth_dev *dev)
1288 {
1289         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1290         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1291         struct i40e_arq_event_info info;
1292         uint16_t pending, aq_opc;
1293         enum virtchnl_ops msg_opc;
1294         enum i40e_status_code msg_ret;
1295         int ret;
1296
1297         info.buf_len = I40E_AQ_BUF_SZ;
1298         if (!vf->aq_resp) {
1299                 PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
1300                 return;
1301         }
1302         info.msg_buf = vf->aq_resp;
1303
1304         pending = 1;
1305         while (pending) {
1306                 ret = i40e_clean_arq_element(hw, &info, &pending);
1307
1308                 if (ret != I40E_SUCCESS) {
1309                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
1310                                     "ret: %d", ret);
1311                         break;
1312                 }
1313                 aq_opc = rte_le_to_cpu_16(info.desc.opcode);
1314                 /* For the message sent from pf to vf, opcode is stored in
1315                  * cookie_high of struct i40e_aq_desc, while return error code
1316                  * are stored in cookie_low, Which is done by
1317                  * i40e_aq_send_msg_to_vf in PF driver.*/
1318                 msg_opc = (enum virtchnl_ops)rte_le_to_cpu_32(
1319                                                   info.desc.cookie_high);
1320                 msg_ret = (enum i40e_status_code)rte_le_to_cpu_32(
1321                                                   info.desc.cookie_low);
1322                 switch (aq_opc) {
1323                 case i40e_aqc_opc_send_msg_to_vf:
1324                         if (msg_opc == VIRTCHNL_OP_EVENT)
1325                                 /* process event*/
1326                                 i40evf_handle_pf_event(dev, info.msg_buf,
1327                                                        info.msg_len);
1328                         else {
1329                                 /* read message and it's expected one */
1330                                 if (msg_opc == vf->pend_cmd) {
1331                                         vf->cmd_retval = msg_ret;
1332                                         /* prevent compiler reordering */
1333                                         rte_compiler_barrier();
1334                                         _clear_cmd(vf);
1335                                 } else
1336                                         PMD_DRV_LOG(ERR, "command mismatch,"
1337                                                 "expect %u, get %u",
1338                                                 vf->pend_cmd, msg_opc);
1339                                 PMD_DRV_LOG(DEBUG, "adminq response is received,"
1340                                              " opcode = %d", msg_opc);
1341                         }
1342                         break;
1343                 default:
1344                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
1345                                     aq_opc);
1346                         break;
1347                 }
1348         }
1349 }
1350
1351 /**
1352  * Interrupt handler triggered by NIC  for handling
1353  * specific interrupt. Only adminq interrupt is processed in VF.
1354  *
1355  * @param handle
1356  *  Pointer to interrupt handle.
1357  * @param param
1358  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1359  *
1360  * @return
1361  *  void
1362  */
1363 static void
1364 i40evf_dev_interrupt_handler(void *param)
1365 {
1366         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1367         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368         uint32_t icr0;
1369
1370         i40evf_disable_irq0(hw);
1371
1372         /* read out interrupt causes */
1373         icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
1374
1375         /* No interrupt event indicated */
1376         if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
1377                 PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do");
1378                 goto done;
1379         }
1380
1381         if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
1382                 PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported");
1383                 i40evf_handle_aq_msg(dev);
1384         }
1385
1386         /* Link Status Change interrupt */
1387         if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
1388                 PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
1389                                    " do nothing");
1390
1391 done:
1392         i40evf_enable_irq0(hw);
1393 }
1394
1395 static int
1396 i40evf_dev_init(struct rte_eth_dev *eth_dev)
1397 {
1398         struct i40e_hw *hw
1399                 = I40E_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1400         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1401
1402         PMD_INIT_FUNC_TRACE();
1403
1404         /* assign ops func pointer */
1405         eth_dev->dev_ops = &i40evf_eth_dev_ops;
1406         eth_dev->rx_pkt_burst = &i40e_recv_pkts;
1407         eth_dev->tx_pkt_burst = &i40e_xmit_pkts;
1408
1409         /*
1410          * For secondary processes, we don't initialise any further as primary
1411          * has already done this work.
1412          */
1413         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1414                 i40e_set_rx_function(eth_dev);
1415                 i40e_set_tx_function(eth_dev);
1416                 return 0;
1417         }
1418         i40e_set_default_ptype_table(eth_dev);
1419         i40e_set_default_pctype_table(eth_dev);
1420         rte_eth_copy_pci_info(eth_dev, pci_dev);
1421
1422         hw->vendor_id = pci_dev->id.vendor_id;
1423         hw->device_id = pci_dev->id.device_id;
1424         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1425         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1426         hw->bus.device = pci_dev->addr.devid;
1427         hw->bus.func = pci_dev->addr.function;
1428         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1429         hw->adapter_stopped = 0;
1430
1431         if(i40evf_init_vf(eth_dev) != 0) {
1432                 PMD_INIT_LOG(ERR, "Init vf failed");
1433                 return -1;
1434         }
1435
1436         /* register callback func to eal lib */
1437         rte_intr_callback_register(&pci_dev->intr_handle,
1438                 i40evf_dev_interrupt_handler, (void *)eth_dev);
1439
1440         /* enable uio intr after callback register */
1441         rte_intr_enable(&pci_dev->intr_handle);
1442
1443         /* configure and enable device interrupt */
1444         i40evf_enable_irq0(hw);
1445
1446         /* copy mac addr */
1447         eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
1448                                         ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX,
1449                                         0);
1450         if (eth_dev->data->mac_addrs == NULL) {
1451                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to"
1452                                 " store MAC addresses",
1453                                 ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX);
1454                 return -ENOMEM;
1455         }
1456         ether_addr_copy((struct ether_addr *)hw->mac.addr,
1457                         &eth_dev->data->mac_addrs[0]);
1458
1459         return 0;
1460 }
1461
1462 static int
1463 i40evf_dev_uninit(struct rte_eth_dev *eth_dev)
1464 {
1465         PMD_INIT_FUNC_TRACE();
1466
1467         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1468                 return -EPERM;
1469
1470         eth_dev->dev_ops = NULL;
1471         eth_dev->rx_pkt_burst = NULL;
1472         eth_dev->tx_pkt_burst = NULL;
1473
1474         if (i40evf_uninit_vf(eth_dev) != 0) {
1475                 PMD_INIT_LOG(ERR, "i40evf_uninit_vf failed");
1476                 return -1;
1477         }
1478
1479         rte_free(eth_dev->data->mac_addrs);
1480         eth_dev->data->mac_addrs = NULL;
1481
1482         return 0;
1483 }
1484
1485 static int eth_i40evf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1486         struct rte_pci_device *pci_dev)
1487 {
1488         return rte_eth_dev_pci_generic_probe(pci_dev,
1489                 sizeof(struct i40e_adapter), i40evf_dev_init);
1490 }
1491
1492 static int eth_i40evf_pci_remove(struct rte_pci_device *pci_dev)
1493 {
1494         return rte_eth_dev_pci_generic_remove(pci_dev, i40evf_dev_uninit);
1495 }
1496
1497 /*
1498  * virtual function driver struct
1499  */
1500 static struct rte_pci_driver rte_i40evf_pmd = {
1501         .id_table = pci_id_i40evf_map,
1502         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA,
1503         .probe = eth_i40evf_pci_probe,
1504         .remove = eth_i40evf_pci_remove,
1505 };
1506
1507 RTE_PMD_REGISTER_PCI(net_i40e_vf, rte_i40evf_pmd);
1508 RTE_PMD_REGISTER_PCI_TABLE(net_i40e_vf, pci_id_i40evf_map);
1509 RTE_PMD_REGISTER_KMOD_DEP(net_i40e_vf, "* igb_uio | vfio-pci");
1510
1511 static int
1512 i40evf_dev_configure(struct rte_eth_dev *dev)
1513 {
1514         struct i40e_adapter *ad =
1515                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1516         struct rte_eth_conf *conf = &dev->data->dev_conf;
1517         struct i40e_vf *vf;
1518
1519         /* Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1520          * allocation or vector Rx preconditions we will reset it.
1521          */
1522         ad->rx_bulk_alloc_allowed = true;
1523         ad->rx_vec_allowed = true;
1524         ad->tx_simple_allowed = true;
1525         ad->tx_vec_allowed = true;
1526
1527         /* For non-DPDK PF drivers, VF has no ability to disable HW
1528          * CRC strip, and is implicitly enabled by the PF.
1529          */
1530         if (!conf->rxmode.hw_strip_crc) {
1531                 vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1532                 if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) &&
1533                     (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) {
1534                         /* Peer is running non-DPDK PF driver. */
1535                         PMD_INIT_LOG(ERR, "VF can't disable HW CRC Strip");
1536                         return -EINVAL;
1537                 }
1538         }
1539
1540         return i40evf_init_vlan(dev);
1541 }
1542
1543 static int
1544 i40evf_init_vlan(struct rte_eth_dev *dev)
1545 {
1546         /* Apply vlan offload setting */
1547         i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
1548
1549         return 0;
1550 }
1551
1552 static int
1553 i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1554 {
1555         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1556         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1557
1558         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
1559                 return -ENOTSUP;
1560
1561         /* Vlan stripping setting */
1562         if (mask & ETH_VLAN_STRIP_MASK) {
1563                 /* Enable or disable VLAN stripping */
1564                 if (dev_conf->rxmode.hw_vlan_strip)
1565                         i40evf_enable_vlan_strip(dev);
1566                 else
1567                         i40evf_disable_vlan_strip(dev);
1568         }
1569
1570         return 0;
1571 }
1572
1573 static int
1574 i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1575 {
1576         struct i40e_rx_queue *rxq;
1577         int err = 0;
1578         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1579
1580         PMD_INIT_FUNC_TRACE();
1581
1582         if (rx_queue_id < dev->data->nb_rx_queues) {
1583                 rxq = dev->data->rx_queues[rx_queue_id];
1584
1585                 err = i40e_alloc_rx_queue_mbufs(rxq);
1586                 if (err) {
1587                         PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1588                         return err;
1589                 }
1590
1591                 rte_wmb();
1592
1593                 /* Init the RX tail register. */
1594                 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1595                 I40EVF_WRITE_FLUSH(hw);
1596
1597                 /* Ready to switch the queue on */
1598                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE);
1599
1600                 if (err)
1601                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
1602                                     rx_queue_id);
1603                 else
1604                         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1605         }
1606
1607         return err;
1608 }
1609
1610 static int
1611 i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1612 {
1613         struct i40e_rx_queue *rxq;
1614         int err;
1615
1616         if (rx_queue_id < dev->data->nb_rx_queues) {
1617                 rxq = dev->data->rx_queues[rx_queue_id];
1618
1619                 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE);
1620
1621                 if (err) {
1622                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
1623                                     rx_queue_id);
1624                         return err;
1625                 }
1626
1627                 i40e_rx_queue_release_mbufs(rxq);
1628                 i40e_reset_rx_queue(rxq);
1629                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1630         }
1631
1632         return 0;
1633 }
1634
1635 static int
1636 i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1637 {
1638         int err = 0;
1639
1640         PMD_INIT_FUNC_TRACE();
1641
1642         if (tx_queue_id < dev->data->nb_tx_queues) {
1643
1644                 /* Ready to switch the queue on */
1645                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE);
1646
1647                 if (err)
1648                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
1649                                     tx_queue_id);
1650                 else
1651                         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1652         }
1653
1654         return err;
1655 }
1656
1657 static int
1658 i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1659 {
1660         struct i40e_tx_queue *txq;
1661         int err;
1662
1663         if (tx_queue_id < dev->data->nb_tx_queues) {
1664                 txq = dev->data->tx_queues[tx_queue_id];
1665
1666                 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE);
1667
1668                 if (err) {
1669                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
1670                                     tx_queue_id);
1671                         return err;
1672                 }
1673
1674                 i40e_tx_queue_release_mbufs(txq);
1675                 i40e_reset_tx_queue(txq);
1676                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1677         }
1678
1679         return 0;
1680 }
1681
1682 static int
1683 i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1684 {
1685         int ret;
1686
1687         if (on)
1688                 ret = i40evf_add_vlan(dev, vlan_id);
1689         else
1690                 ret = i40evf_del_vlan(dev,vlan_id);
1691
1692         return ret;
1693 }
1694
1695 static int
1696 i40evf_rxq_init(struct rte_eth_dev *dev, struct i40e_rx_queue *rxq)
1697 {
1698         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1699         struct rte_eth_dev_data *dev_data = dev->data;
1700         struct rte_pktmbuf_pool_private *mbp_priv;
1701         uint16_t buf_size, len;
1702
1703         rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL1(rxq->queue_id);
1704         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1705         I40EVF_WRITE_FLUSH(hw);
1706
1707         /* Calculate the maximum packet length allowed */
1708         mbp_priv = rte_mempool_get_priv(rxq->mp);
1709         buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
1710                                         RTE_PKTMBUF_HEADROOM);
1711         rxq->hs_mode = i40e_header_split_none;
1712         rxq->rx_hdr_len = 0;
1713         rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
1714         len = rxq->rx_buf_len * I40E_MAX_CHAINED_RX_BUFFERS;
1715         rxq->max_pkt_len = RTE_MIN(len,
1716                 dev_data->dev_conf.rxmode.max_rx_pkt_len);
1717
1718         /**
1719          * Check if the jumbo frame and maximum packet length are set correctly
1720          */
1721         if (dev_data->dev_conf.rxmode.jumbo_frame == 1) {
1722                 if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
1723                     rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
1724                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1725                                 "larger than %u and smaller than %u, as jumbo "
1726                                 "frame is enabled", (uint32_t)ETHER_MAX_LEN,
1727                                         (uint32_t)I40E_FRAME_SIZE_MAX);
1728                         return I40E_ERR_CONFIG;
1729                 }
1730         } else {
1731                 if (rxq->max_pkt_len < ETHER_MIN_LEN ||
1732                     rxq->max_pkt_len > ETHER_MAX_LEN) {
1733                         PMD_DRV_LOG(ERR, "maximum packet length must be "
1734                                 "larger than %u and smaller than %u, as jumbo "
1735                                 "frame is disabled", (uint32_t)ETHER_MIN_LEN,
1736                                                 (uint32_t)ETHER_MAX_LEN);
1737                         return I40E_ERR_CONFIG;
1738                 }
1739         }
1740
1741         if (dev_data->dev_conf.rxmode.enable_scatter ||
1742             (rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
1743                 dev_data->scattered_rx = 1;
1744         }
1745
1746         return 0;
1747 }
1748
1749 static int
1750 i40evf_rx_init(struct rte_eth_dev *dev)
1751 {
1752         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1753         uint16_t i;
1754         int ret = I40E_SUCCESS;
1755         struct i40e_rx_queue **rxq =
1756                 (struct i40e_rx_queue **)dev->data->rx_queues;
1757
1758         i40evf_config_rss(vf);
1759         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1760                 if (!rxq[i] || !rxq[i]->q_set)
1761                         continue;
1762                 ret = i40evf_rxq_init(dev, rxq[i]);
1763                 if (ret != I40E_SUCCESS)
1764                         break;
1765         }
1766         if (ret == I40E_SUCCESS)
1767                 i40e_set_rx_function(dev);
1768
1769         return ret;
1770 }
1771
1772 static void
1773 i40evf_tx_init(struct rte_eth_dev *dev)
1774 {
1775         uint16_t i;
1776         struct i40e_tx_queue **txq =
1777                 (struct i40e_tx_queue **)dev->data->tx_queues;
1778         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1779
1780         for (i = 0; i < dev->data->nb_tx_queues; i++)
1781                 txq[i]->qtx_tail = hw->hw_addr + I40E_QTX_TAIL1(i);
1782
1783         i40e_set_tx_function(dev);
1784 }
1785
1786 static inline void
1787 i40evf_enable_queues_intr(struct rte_eth_dev *dev)
1788 {
1789         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1790         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1791         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1792
1793         if (!rte_intr_allow_others(intr_handle)) {
1794                 I40E_WRITE_REG(hw,
1795                                I40E_VFINT_DYN_CTL01,
1796                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1797                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1798                                I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1799                 I40EVF_WRITE_FLUSH(hw);
1800                 return;
1801         }
1802
1803         I40EVF_WRITE_FLUSH(hw);
1804 }
1805
1806 static inline void
1807 i40evf_disable_queues_intr(struct rte_eth_dev *dev)
1808 {
1809         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1810         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1811         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1812
1813         if (!rte_intr_allow_others(intr_handle)) {
1814                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1815                                I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1816                 I40EVF_WRITE_FLUSH(hw);
1817                 return;
1818         }
1819
1820         I40EVF_WRITE_FLUSH(hw);
1821 }
1822
1823 static int
1824 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1825 {
1826         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1827         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1828         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1829         uint16_t interval =
1830                 i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL, 0, 0);
1831         uint16_t msix_intr;
1832
1833         msix_intr = intr_handle->intr_vec[queue_id];
1834         if (msix_intr == I40E_MISC_VEC_ID)
1835                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1836                                I40E_VFINT_DYN_CTL01_INTENA_MASK |
1837                                I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1838                                (0 << I40E_VFINT_DYN_CTL01_ITR_INDX_SHIFT) |
1839                                (interval <<
1840                                 I40E_VFINT_DYN_CTL01_INTERVAL_SHIFT));
1841         else
1842                 I40E_WRITE_REG(hw,
1843                                I40E_VFINT_DYN_CTLN1(msix_intr -
1844                                                     I40E_RX_VEC_START),
1845                                I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1846                                I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
1847                                (0 << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1848                                (interval <<
1849                                 I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT));
1850
1851         I40EVF_WRITE_FLUSH(hw);
1852
1853         rte_intr_enable(&pci_dev->intr_handle);
1854
1855         return 0;
1856 }
1857
1858 static int
1859 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1860 {
1861         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1862         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1863         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1864         uint16_t msix_intr;
1865
1866         msix_intr = intr_handle->intr_vec[queue_id];
1867         if (msix_intr == I40E_MISC_VEC_ID)
1868                 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
1869         else
1870                 I40E_WRITE_REG(hw,
1871                                I40E_VFINT_DYN_CTLN1(msix_intr -
1872                                                     I40E_RX_VEC_START),
1873                                0);
1874
1875         I40EVF_WRITE_FLUSH(hw);
1876
1877         return 0;
1878 }
1879
1880 static void
1881 i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add)
1882 {
1883         struct virtchnl_ether_addr_list *list;
1884         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1885         int err, i, j;
1886         int next_begin = 0;
1887         int begin = 0;
1888         uint32_t len;
1889         struct ether_addr *addr;
1890         struct vf_cmd_info args;
1891
1892         do {
1893                 j = 0;
1894                 len = sizeof(struct virtchnl_ether_addr_list);
1895                 for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) {
1896                         if (is_zero_ether_addr(&dev->data->mac_addrs[i]))
1897                                 continue;
1898                         len += sizeof(struct virtchnl_ether_addr);
1899                         if (len >= I40E_AQ_BUF_SZ) {
1900                                 next_begin = i + 1;
1901                                 break;
1902                         }
1903                 }
1904
1905                 list = rte_zmalloc("i40evf_del_mac_buffer", len, 0);
1906                 if (!list) {
1907                         PMD_DRV_LOG(ERR, "fail to allocate memory");
1908                         return;
1909                 }
1910
1911                 for (i = begin; i < next_begin; i++) {
1912                         addr = &dev->data->mac_addrs[i];
1913                         if (is_zero_ether_addr(addr))
1914                                 continue;
1915                         rte_memcpy(list->list[j].addr, addr->addr_bytes,
1916                                          sizeof(addr->addr_bytes));
1917                         PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x",
1918                                     addr->addr_bytes[0], addr->addr_bytes[1],
1919                                     addr->addr_bytes[2], addr->addr_bytes[3],
1920                                     addr->addr_bytes[4], addr->addr_bytes[5]);
1921                         j++;
1922                 }
1923                 list->vsi_id = vf->vsi_res->vsi_id;
1924                 list->num_elements = j;
1925                 args.ops = add ? VIRTCHNL_OP_ADD_ETH_ADDR :
1926                            VIRTCHNL_OP_DEL_ETH_ADDR;
1927                 args.in_args = (uint8_t *)list;
1928                 args.in_args_size = len;
1929                 args.out_buffer = vf->aq_resp;
1930                 args.out_size = I40E_AQ_BUF_SZ;
1931                 err = i40evf_execute_vf_cmd(dev, &args);
1932                 if (err) {
1933                         PMD_DRV_LOG(ERR, "fail to execute command %s",
1934                                     add ? "OP_ADD_ETHER_ADDRESS" :
1935                                     "OP_DEL_ETHER_ADDRESS");
1936                 } else {
1937                         if (add)
1938                                 vf->vsi.mac_num++;
1939                         else
1940                                 vf->vsi.mac_num--;
1941                 }
1942                 rte_free(list);
1943                 begin = next_begin;
1944         } while (begin < I40E_NUM_MACADDR_MAX);
1945 }
1946
1947 static int
1948 i40evf_dev_start(struct rte_eth_dev *dev)
1949 {
1950         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1951         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1952         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1953         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1954         uint32_t intr_vector = 0;
1955
1956         PMD_INIT_FUNC_TRACE();
1957
1958         hw->adapter_stopped = 0;
1959
1960         vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1961         vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
1962                                         dev->data->nb_tx_queues);
1963
1964         /* check and configure queue intr-vector mapping */
1965         if (rte_intr_cap_multiple(intr_handle) &&
1966             dev->data->dev_conf.intr_conf.rxq) {
1967                 intr_vector = dev->data->nb_rx_queues;
1968                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1969                         return -1;
1970         }
1971
1972         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1973                 intr_handle->intr_vec =
1974                         rte_zmalloc("intr_vec",
1975                                     dev->data->nb_rx_queues * sizeof(int), 0);
1976                 if (!intr_handle->intr_vec) {
1977                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1978                                      " intr_vec", dev->data->nb_rx_queues);
1979                         return -ENOMEM;
1980                 }
1981         }
1982
1983         if (i40evf_rx_init(dev) != 0){
1984                 PMD_DRV_LOG(ERR, "failed to do RX init");
1985                 return -1;
1986         }
1987
1988         i40evf_tx_init(dev);
1989
1990         if (i40evf_configure_vsi_queues(dev) != 0) {
1991                 PMD_DRV_LOG(ERR, "configure queues failed");
1992                 goto err_queue;
1993         }
1994         if (i40evf_config_irq_map(dev)) {
1995                 PMD_DRV_LOG(ERR, "config_irq_map failed");
1996                 goto err_queue;
1997         }
1998
1999         /* Set all mac addrs */
2000         i40evf_add_del_all_mac_addr(dev, TRUE);
2001
2002         if (i40evf_start_queues(dev) != 0) {
2003                 PMD_DRV_LOG(ERR, "enable queues failed");
2004                 goto err_mac;
2005         }
2006
2007         /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt
2008          * is mapped to VFIO vector 0 in i40evf_dev_init( ).
2009          * If previous VFIO interrupt mapping set in i40evf_dev_init( ) is
2010          * not cleared, it will fail when rte_intr_enable( ) tries to map Rx
2011          * queue interrupt to other VFIO vectors.
2012          * So clear uio/vfio intr/evevnfd first to avoid failure.
2013          */
2014         if (dev->data->dev_conf.intr_conf.rxq != 0) {
2015                 rte_intr_disable(intr_handle);
2016                 rte_intr_enable(intr_handle);
2017         }
2018
2019         i40evf_enable_queues_intr(dev);
2020
2021         return 0;
2022
2023 err_mac:
2024         i40evf_add_del_all_mac_addr(dev, FALSE);
2025 err_queue:
2026         return -1;
2027 }
2028
2029 static void
2030 i40evf_dev_stop(struct rte_eth_dev *dev)
2031 {
2032         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2033         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2034         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2035
2036         PMD_INIT_FUNC_TRACE();
2037
2038         if (hw->adapter_stopped == 1)
2039                 return;
2040         i40evf_stop_queues(dev);
2041         i40evf_disable_queues_intr(dev);
2042         i40e_dev_clear_queues(dev);
2043
2044         /* Clean datapath event and queue/vec mapping */
2045         rte_intr_efd_disable(intr_handle);
2046         if (intr_handle->intr_vec) {
2047                 rte_free(intr_handle->intr_vec);
2048                 intr_handle->intr_vec = NULL;
2049         }
2050         /* remove all mac addrs */
2051         i40evf_add_del_all_mac_addr(dev, FALSE);
2052         hw->adapter_stopped = 1;
2053
2054 }
2055
2056 static int
2057 i40evf_dev_link_update(struct rte_eth_dev *dev,
2058                        __rte_unused int wait_to_complete)
2059 {
2060         struct rte_eth_link new_link;
2061         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2062         /*
2063          * DPDK pf host provide interfacet to acquire link status
2064          * while Linux driver does not
2065          */
2066
2067         memset(&new_link, 0, sizeof(new_link));
2068         /* Linux driver PF host */
2069         switch (vf->link_speed) {
2070         case I40E_LINK_SPEED_100MB:
2071                 new_link.link_speed = ETH_SPEED_NUM_100M;
2072                 break;
2073         case I40E_LINK_SPEED_1GB:
2074                 new_link.link_speed = ETH_SPEED_NUM_1G;
2075                 break;
2076         case I40E_LINK_SPEED_10GB:
2077                 new_link.link_speed = ETH_SPEED_NUM_10G;
2078                 break;
2079         case I40E_LINK_SPEED_20GB:
2080                 new_link.link_speed = ETH_SPEED_NUM_20G;
2081                 break;
2082         case I40E_LINK_SPEED_25GB:
2083                 new_link.link_speed = ETH_SPEED_NUM_25G;
2084                 break;
2085         case I40E_LINK_SPEED_40GB:
2086                 new_link.link_speed = ETH_SPEED_NUM_40G;
2087                 break;
2088         default:
2089                 new_link.link_speed = ETH_SPEED_NUM_100M;
2090                 break;
2091         }
2092         /* full duplex only */
2093         new_link.link_duplex = ETH_LINK_FULL_DUPLEX;
2094         new_link.link_status = vf->link_up ? ETH_LINK_UP :
2095                                              ETH_LINK_DOWN;
2096         new_link.link_autoneg =
2097                 dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED;
2098
2099         return rte_eth_linkstatus_set(dev, &new_link);
2100 }
2101
2102 static void
2103 i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev)
2104 {
2105         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2106         int ret;
2107
2108         /* If enabled, just return */
2109         if (vf->promisc_unicast_enabled)
2110                 return;
2111
2112         ret = i40evf_config_promisc(dev, 1, vf->promisc_multicast_enabled);
2113         if (ret == 0)
2114                 vf->promisc_unicast_enabled = TRUE;
2115 }
2116
2117 static void
2118 i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev)
2119 {
2120         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2121         int ret;
2122
2123         /* If disabled, just return */
2124         if (!vf->promisc_unicast_enabled)
2125                 return;
2126
2127         ret = i40evf_config_promisc(dev, 0, vf->promisc_multicast_enabled);
2128         if (ret == 0)
2129                 vf->promisc_unicast_enabled = FALSE;
2130 }
2131
2132 static void
2133 i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev)
2134 {
2135         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2136         int ret;
2137
2138         /* If enabled, just return */
2139         if (vf->promisc_multicast_enabled)
2140                 return;
2141
2142         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 1);
2143         if (ret == 0)
2144                 vf->promisc_multicast_enabled = TRUE;
2145 }
2146
2147 static void
2148 i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev)
2149 {
2150         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2151         int ret;
2152
2153         /* If enabled, just return */
2154         if (!vf->promisc_multicast_enabled)
2155                 return;
2156
2157         ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 0);
2158         if (ret == 0)
2159                 vf->promisc_multicast_enabled = FALSE;
2160 }
2161
2162 static void
2163 i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2164 {
2165         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2166
2167         memset(dev_info, 0, sizeof(*dev_info));
2168         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2169         dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
2170         dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
2171         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2172         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2173         dev_info->hash_key_size = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2174         dev_info->reta_size = ETH_RSS_RETA_SIZE_64;
2175         dev_info->flow_type_rss_offloads = vf->adapter->flow_types_mask;
2176         dev_info->max_mac_addrs = I40E_NUM_MACADDR_MAX;
2177         dev_info->rx_offload_capa =
2178                 DEV_RX_OFFLOAD_VLAN_STRIP |
2179                 DEV_RX_OFFLOAD_QINQ_STRIP |
2180                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2181                 DEV_RX_OFFLOAD_UDP_CKSUM |
2182                 DEV_RX_OFFLOAD_TCP_CKSUM |
2183                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
2184                 DEV_RX_OFFLOAD_CRC_STRIP;
2185         dev_info->tx_offload_capa =
2186                 DEV_TX_OFFLOAD_VLAN_INSERT |
2187                 DEV_TX_OFFLOAD_QINQ_INSERT |
2188                 DEV_TX_OFFLOAD_IPV4_CKSUM |
2189                 DEV_TX_OFFLOAD_UDP_CKSUM |
2190                 DEV_TX_OFFLOAD_TCP_CKSUM |
2191                 DEV_TX_OFFLOAD_SCTP_CKSUM |
2192                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
2193                 DEV_TX_OFFLOAD_TCP_TSO |
2194                 DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
2195                 DEV_TX_OFFLOAD_GRE_TNL_TSO |
2196                 DEV_TX_OFFLOAD_IPIP_TNL_TSO |
2197                 DEV_TX_OFFLOAD_GENEVE_TNL_TSO;
2198
2199         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2200                 .rx_thresh = {
2201                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
2202                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
2203                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
2204                 },
2205                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2206                 .rx_drop_en = 0,
2207         };
2208
2209         dev_info->default_txconf = (struct rte_eth_txconf) {
2210                 .tx_thresh = {
2211                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
2212                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
2213                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
2214                 },
2215                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2216                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2217                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2218                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2219         };
2220
2221         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2222                 .nb_max = I40E_MAX_RING_DESC,
2223                 .nb_min = I40E_MIN_RING_DESC,
2224                 .nb_align = I40E_ALIGN_RING_DESC,
2225         };
2226
2227         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2228                 .nb_max = I40E_MAX_RING_DESC,
2229                 .nb_min = I40E_MIN_RING_DESC,
2230                 .nb_align = I40E_ALIGN_RING_DESC,
2231         };
2232 }
2233
2234 static int
2235 i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2236 {
2237         int ret;
2238         struct i40e_eth_stats *pstats = NULL;
2239         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2240         struct i40e_vsi *vsi = &vf->vsi;
2241
2242         ret = i40evf_query_stats(dev, &pstats);
2243         if (ret == 0) {
2244                 i40evf_update_stats(vsi, pstats);
2245
2246                 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast +
2247                                                 pstats->rx_broadcast;
2248                 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast +
2249                                                 pstats->tx_unicast;
2250                 stats->imissed = pstats->rx_discards;
2251                 stats->oerrors = pstats->tx_errors + pstats->tx_discards;
2252                 stats->ibytes = pstats->rx_bytes;
2253                 stats->obytes = pstats->tx_bytes;
2254         } else {
2255                 PMD_DRV_LOG(ERR, "Get statistics failed");
2256         }
2257         return ret;
2258 }
2259
2260 static void
2261 i40evf_dev_close(struct rte_eth_dev *dev)
2262 {
2263         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2264         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2265         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2266
2267         i40evf_dev_stop(dev);
2268         i40e_dev_free_queues(dev);
2269         i40evf_reset_vf(hw);
2270         i40e_shutdown_adminq(hw);
2271         /* disable uio intr before callback unregister */
2272         rte_intr_disable(intr_handle);
2273
2274         /* unregister callback func from eal lib */
2275         rte_intr_callback_unregister(intr_handle,
2276                                      i40evf_dev_interrupt_handler, dev);
2277         i40evf_disable_irq0(hw);
2278 }
2279
2280 /*
2281  * Reset VF device only to re-initialize resources in PMD layer
2282  */
2283 static int
2284 i40evf_dev_reset(struct rte_eth_dev *dev)
2285 {
2286         int ret;
2287
2288         ret = i40evf_dev_uninit(dev);
2289         if (ret)
2290                 return ret;
2291
2292         ret = i40evf_dev_init(dev);
2293
2294         return ret;
2295 }
2296
2297 static int
2298 i40evf_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2299 {
2300         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2301         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2302         int ret;
2303
2304         if (!lut)
2305                 return -EINVAL;
2306
2307         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2308                 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, FALSE,
2309                                           lut, lut_size);
2310                 if (ret) {
2311                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
2312                         return ret;
2313                 }
2314         } else {
2315                 uint32_t *lut_dw = (uint32_t *)lut;
2316                 uint16_t i, lut_size_dw = lut_size / 4;
2317
2318                 for (i = 0; i < lut_size_dw; i++)
2319                         lut_dw[i] = I40E_READ_REG(hw, I40E_VFQF_HLUT(i));
2320         }
2321
2322         return 0;
2323 }
2324
2325 static int
2326 i40evf_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2327 {
2328         struct i40e_vf *vf;
2329         struct i40e_hw *hw;
2330         int ret;
2331
2332         if (!vsi || !lut)
2333                 return -EINVAL;
2334
2335         vf = I40E_VSI_TO_VF(vsi);
2336         hw = I40E_VSI_TO_HW(vsi);
2337
2338         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2339                 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, FALSE,
2340                                           lut, lut_size);
2341                 if (ret) {
2342                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
2343                         return ret;
2344                 }
2345         } else {
2346                 uint32_t *lut_dw = (uint32_t *)lut;
2347                 uint16_t i, lut_size_dw = lut_size / 4;
2348
2349                 for (i = 0; i < lut_size_dw; i++)
2350                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i), lut_dw[i]);
2351                 I40EVF_WRITE_FLUSH(hw);
2352         }
2353
2354         return 0;
2355 }
2356
2357 static int
2358 i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
2359                            struct rte_eth_rss_reta_entry64 *reta_conf,
2360                            uint16_t reta_size)
2361 {
2362         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2363         uint8_t *lut;
2364         uint16_t i, idx, shift;
2365         int ret;
2366
2367         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2368                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2369                         "(%d) doesn't match the number of hardware can "
2370                         "support (%d)", reta_size, ETH_RSS_RETA_SIZE_64);
2371                 return -EINVAL;
2372         }
2373
2374         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2375         if (!lut) {
2376                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2377                 return -ENOMEM;
2378         }
2379         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2380         if (ret)
2381                 goto out;
2382         for (i = 0; i < reta_size; i++) {
2383                 idx = i / RTE_RETA_GROUP_SIZE;
2384                 shift = i % RTE_RETA_GROUP_SIZE;
2385                 if (reta_conf[idx].mask & (1ULL << shift))
2386                         lut[i] = reta_conf[idx].reta[shift];
2387         }
2388         ret = i40evf_set_rss_lut(&vf->vsi, lut, reta_size);
2389
2390 out:
2391         rte_free(lut);
2392
2393         return ret;
2394 }
2395
2396 static int
2397 i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
2398                           struct rte_eth_rss_reta_entry64 *reta_conf,
2399                           uint16_t reta_size)
2400 {
2401         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2402         uint16_t i, idx, shift;
2403         uint8_t *lut;
2404         int ret;
2405
2406         if (reta_size != ETH_RSS_RETA_SIZE_64) {
2407                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2408                         "(%d) doesn't match the number of hardware can "
2409                         "support (%d)", reta_size, ETH_RSS_RETA_SIZE_64);
2410                 return -EINVAL;
2411         }
2412
2413         lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2414         if (!lut) {
2415                 PMD_DRV_LOG(ERR, "No memory can be allocated");
2416                 return -ENOMEM;
2417         }
2418
2419         ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2420         if (ret)
2421                 goto out;
2422         for (i = 0; i < reta_size; i++) {
2423                 idx = i / RTE_RETA_GROUP_SIZE;
2424                 shift = i % RTE_RETA_GROUP_SIZE;
2425                 if (reta_conf[idx].mask & (1ULL << shift))
2426                         reta_conf[idx].reta[shift] = lut[i];
2427         }
2428
2429 out:
2430         rte_free(lut);
2431
2432         return ret;
2433 }
2434
2435 static int
2436 i40evf_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
2437 {
2438         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2439         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2440         int ret = 0;
2441
2442         if (!key || key_len == 0) {
2443                 PMD_DRV_LOG(DEBUG, "No key to be configured");
2444                 return 0;
2445         } else if (key_len != (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2446                 sizeof(uint32_t)) {
2447                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
2448                 return -EINVAL;
2449         }
2450
2451         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2452                 struct i40e_aqc_get_set_rss_key_data *key_dw =
2453                         (struct i40e_aqc_get_set_rss_key_data *)key;
2454
2455                 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
2456                 if (ret)
2457                         PMD_INIT_LOG(ERR, "Failed to configure RSS key "
2458                                      "via AQ");
2459         } else {
2460                 uint32_t *hash_key = (uint32_t *)key;
2461                 uint16_t i;
2462
2463                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2464                         i40e_write_rx_ctl(hw, I40E_VFQF_HKEY(i), hash_key[i]);
2465                 I40EVF_WRITE_FLUSH(hw);
2466         }
2467
2468         return ret;
2469 }
2470
2471 static int
2472 i40evf_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
2473 {
2474         struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2475         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2476         int ret;
2477
2478         if (!key || !key_len)
2479                 return -EINVAL;
2480
2481         if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2482                 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
2483                         (struct i40e_aqc_get_set_rss_key_data *)key);
2484                 if (ret) {
2485                         PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
2486                         return ret;
2487                 }
2488         } else {
2489                 uint32_t *key_dw = (uint32_t *)key;
2490                 uint16_t i;
2491
2492                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2493                         key_dw[i] = i40e_read_rx_ctl(hw, I40E_VFQF_HKEY(i));
2494         }
2495         *key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2496
2497         return 0;
2498 }
2499
2500 static int
2501 i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf)
2502 {
2503         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2504         uint64_t hena;
2505         int ret;
2506
2507         ret = i40evf_set_rss_key(&vf->vsi, rss_conf->rss_key,
2508                                  rss_conf->rss_key_len);
2509         if (ret)
2510                 return ret;
2511
2512         hena = i40e_config_hena(vf->adapter, rss_conf->rss_hf);
2513         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2514         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2515         I40EVF_WRITE_FLUSH(hw);
2516
2517         return 0;
2518 }
2519
2520 static void
2521 i40evf_disable_rss(struct i40e_vf *vf)
2522 {
2523         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2524
2525         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), 0);
2526         i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), 0);
2527         I40EVF_WRITE_FLUSH(hw);
2528 }
2529
2530 static int
2531 i40evf_config_rss(struct i40e_vf *vf)
2532 {
2533         struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2534         struct rte_eth_rss_conf rss_conf;
2535         uint32_t i, j, lut = 0, nb_q = (I40E_VFQF_HLUT_MAX_INDEX + 1) * 4;
2536         uint16_t num;
2537
2538         if (vf->dev_data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
2539                 i40evf_disable_rss(vf);
2540                 PMD_DRV_LOG(DEBUG, "RSS not configured");
2541                 return 0;
2542         }
2543
2544         num = RTE_MIN(vf->dev_data->nb_rx_queues, I40E_MAX_QP_NUM_PER_VF);
2545         /* Fill out the look up table */
2546         for (i = 0, j = 0; i < nb_q; i++, j++) {
2547                 if (j >= num)
2548                         j = 0;
2549                 lut = (lut << 8) | j;
2550                 if ((i & 3) == 3)
2551                         I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i >> 2), lut);
2552         }
2553
2554         rss_conf = vf->dev_data->dev_conf.rx_adv_conf.rss_conf;
2555         if ((rss_conf.rss_hf & vf->adapter->flow_types_mask) == 0) {
2556                 i40evf_disable_rss(vf);
2557                 PMD_DRV_LOG(DEBUG, "No hash flag is set");
2558                 return 0;
2559         }
2560
2561         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
2562                 (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
2563                 /* Calculate the default hash key */
2564                 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2565                         rss_key_default[i] = (uint32_t)rte_rand();
2566                 rss_conf.rss_key = (uint8_t *)rss_key_default;
2567                 rss_conf.rss_key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2568                         sizeof(uint32_t);
2569         }
2570
2571         return i40evf_hw_rss_hash_set(vf, &rss_conf);
2572 }
2573
2574 static int
2575 i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
2576                            struct rte_eth_rss_conf *rss_conf)
2577 {
2578         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2579         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2580         uint64_t rss_hf = rss_conf->rss_hf & vf->adapter->flow_types_mask;
2581         uint64_t hena;
2582
2583         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2584         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2585
2586         if (!(hena & vf->adapter->pctypes_mask)) { /* RSS disabled */
2587                 if (rss_hf != 0) /* Enable RSS */
2588                         return -EINVAL;
2589                 return 0;
2590         }
2591
2592         /* RSS enabled */
2593         if (rss_hf == 0) /* Disable RSS */
2594                 return -EINVAL;
2595
2596         return i40evf_hw_rss_hash_set(vf, rss_conf);
2597 }
2598
2599 static int
2600 i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2601                              struct rte_eth_rss_conf *rss_conf)
2602 {
2603         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2604         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2605         uint64_t hena;
2606
2607         i40evf_get_rss_key(&vf->vsi, rss_conf->rss_key,
2608                            &rss_conf->rss_key_len);
2609
2610         hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2611         hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2612         rss_conf->rss_hf = i40e_parse_hena(vf->adapter, hena);
2613
2614         return 0;
2615 }
2616
2617 static int
2618 i40evf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
2619 {
2620         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2621         struct rte_eth_dev_data *dev_data = vf->dev_data;
2622         uint32_t frame_size = mtu + I40E_ETH_OVERHEAD;
2623         int ret = 0;
2624
2625         /* check if mtu is within the allowed range */
2626         if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX))
2627                 return -EINVAL;
2628
2629         /* mtu setting is forbidden if port is start */
2630         if (dev_data->dev_started) {
2631                 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration",
2632                             dev_data->port_id);
2633                 return -EBUSY;
2634         }
2635
2636         if (frame_size > ETHER_MAX_LEN)
2637                 dev_data->dev_conf.rxmode.jumbo_frame = 1;
2638         else
2639                 dev_data->dev_conf.rxmode.jumbo_frame = 0;
2640
2641         dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
2642
2643         return ret;
2644 }
2645
2646 static void
2647 i40evf_set_default_mac_addr(struct rte_eth_dev *dev,
2648                             struct ether_addr *mac_addr)
2649 {
2650         struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2651         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2652
2653         if (!is_valid_assigned_ether_addr(mac_addr)) {
2654                 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
2655                 return;
2656         }
2657
2658         if (vf->flags & I40E_FLAG_VF_MAC_BY_PF)
2659                 return;
2660
2661         i40evf_del_mac_addr_by_addr(dev, (struct ether_addr *)hw->mac.addr);
2662
2663         i40evf_add_mac_addr(dev, mac_addr, 0, 0);
2664
2665         ether_addr_copy(mac_addr, (struct ether_addr *)hw->mac.addr);
2666 }