drivers/net/avf/avf_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12 #include <inttypes.h>
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15
  16 #include <rte_interrupts.h>
  17 #include <rte_debug.h>
  18 #include <rte_pci.h>
  19 #include <rte_atomic.h>
  20 #include <rte_eal.h>
  21 #include <rte_ether.h>
  22 #include <rte_ethdev.h>
  23 #include <rte_ethdev_pci.h>
  24 #include <rte_malloc.h>
  25 #include <rte_memzone.h>
  26 #include <rte_dev.h>
  27
  28 #include "avf_log.h"
  29 #include "base/avf_prototype.h"
  30 #include "base/avf_adminq_cmd.h"
  31 #include "base/avf_type.h"
  32
  33 #include "avf.h"
  34 #include "avf_rxtx.h"
  35
  36 static int avf_dev_configure(struct rte_eth_dev *dev);
  37 static int avf_dev_start(struct rte_eth_dev *dev);
  38 static void avf_dev_stop(struct rte_eth_dev *dev);
  39 static void avf_dev_close(struct rte_eth_dev *dev);
  40 static void avf_dev_info_get(struct rte_eth_dev *dev,
  41                              struct rte_eth_dev_info *dev_info);
  42 static const uint32_t *avf_dev_supported_ptypes_get(struct rte_eth_dev *dev);
  43 static int avf_dev_stats_get(struct rte_eth_dev *dev,
  44                              struct rte_eth_stats *stats);
  45 static void avf_dev_promiscuous_enable(struct rte_eth_dev *dev);
  46 static void avf_dev_promiscuous_disable(struct rte_eth_dev *dev);
  47 static void avf_dev_allmulticast_enable(struct rte_eth_dev *dev);
  48 static void avf_dev_allmulticast_disable(struct rte_eth_dev *dev);
  49 static int avf_dev_add_mac_addr(struct rte_eth_dev *dev,
  50                                 struct ether_addr *addr,
  51                                 uint32_t index,
  52                                 uint32_t pool);
  53 static void avf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index);
  54 static int avf_dev_vlan_filter_set(struct rte_eth_dev *dev,
  55                                    uint16_t vlan_id, int on);
  56 static int avf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  57 static int avf_dev_rss_reta_update(struct rte_eth_dev *dev,
  58                                    struct rte_eth_rss_reta_entry64 *reta_conf,
  59                                    uint16_t reta_size);
  60 static int avf_dev_rss_reta_query(struct rte_eth_dev *dev,
  61                                   struct rte_eth_rss_reta_entry64 *reta_conf,
  62                                   uint16_t reta_size);
  63 static int avf_dev_rss_hash_update(struct rte_eth_dev *dev,
  64                                    struct rte_eth_rss_conf *rss_conf);
  65 static int avf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
  66                                      struct rte_eth_rss_conf *rss_conf);
  67 static void avf_dev_set_default_mac_addr(struct rte_eth_dev *dev,
  68                                          struct ether_addr *mac_addr);
  69
  70 int avf_logtype_init;
  71 int avf_logtype_driver;
  72 static const struct rte_pci_id pci_id_avf_map[] = {
  73         { RTE_PCI_DEVICE(AVF_INTEL_VENDOR_ID, AVF_DEV_ID_ADAPTIVE_VF) },
  74         { .vendor_id = 0, /* sentinel */ },
  75 };
  76
  77 static const struct eth_dev_ops avf_eth_dev_ops = {
  78         .dev_configure              = avf_dev_configure,
  79         .dev_start                  = avf_dev_start,
  80         .dev_stop                   = avf_dev_stop,
  81         .dev_close                  = avf_dev_close,
  82         .dev_infos_get              = avf_dev_info_get,
  83         .dev_supported_ptypes_get   = avf_dev_supported_ptypes_get,
  84         .link_update                = avf_dev_link_update,
  85         .stats_get                  = avf_dev_stats_get,
  86         .promiscuous_enable         = avf_dev_promiscuous_enable,
  87         .promiscuous_disable        = avf_dev_promiscuous_disable,
  88         .allmulticast_enable        = avf_dev_allmulticast_enable,
  89         .allmulticast_disable       = avf_dev_allmulticast_disable,
  90         .mac_addr_add               = avf_dev_add_mac_addr,
  91         .mac_addr_remove            = avf_dev_del_mac_addr,
  92         .vlan_filter_set            = avf_dev_vlan_filter_set,
  93         .vlan_offload_set           = avf_dev_vlan_offload_set,
  94         .rx_queue_start             = avf_dev_rx_queue_start,
  95         .rx_queue_stop              = avf_dev_rx_queue_stop,
  96         .tx_queue_start             = avf_dev_tx_queue_start,
  97         .tx_queue_stop              = avf_dev_tx_queue_stop,
  98         .rx_queue_setup             = avf_dev_rx_queue_setup,
  99         .rx_queue_release           = avf_dev_rx_queue_release,
 100         .tx_queue_setup             = avf_dev_tx_queue_setup,
 101         .tx_queue_release           = avf_dev_tx_queue_release,
 102         .mac_addr_set               = avf_dev_set_default_mac_addr,
 103         .reta_update                = avf_dev_rss_reta_update,
 104         .reta_query                 = avf_dev_rss_reta_query,
 105         .rss_hash_update            = avf_dev_rss_hash_update,
 106         .rss_hash_conf_get          = avf_dev_rss_hash_conf_get,
 107 };
 108
 109 static int
 110 avf_dev_configure(struct rte_eth_dev *dev)
 111 {
 112         struct avf_adapter *ad =
 113                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 114         struct avf_info *vf =  AVF_DEV_PRIVATE_TO_VF(ad);
 115         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 116
 117         /* Vlan stripping setting */
 118         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 119                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
 120                         avf_enable_vlan_strip(ad);
 121                 else
 122                         avf_disable_vlan_strip(ad);
 123         }
 124         return 0;
 125 }
 126
 127 static int
 128 avf_init_rss(struct avf_adapter *adapter)
 129 {
 130         struct avf_info *vf =  AVF_DEV_PRIVATE_TO_VF(adapter);
 131         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(adapter);
 132         struct rte_eth_rss_conf *rss_conf;
 133         uint8_t i, j, nb_q;
 134         int ret;
 135
 136         rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 137         nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
 138                        AVF_MAX_NUM_QUEUES);
 139
 140         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 141                 PMD_DRV_LOG(DEBUG, "RSS is not supported");
 142                 return -ENOTSUP;
 143         }
 144         if (adapter->eth_dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 145                 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default");
 146                 /* set all lut items to default queue */
 147                 for (i = 0; i < vf->vf_res->rss_lut_size; i++)
 148                         vf->rss_lut[i] = 0;
 149                 ret = avf_configure_rss_lut(adapter);
 150                 return ret;
 151         }
 152
 153         /* In AVF, RSS enablement is set by PF driver. It is not supported
 154          * to set based on rss_conf->rss_hf.
 155          */
 156
 157         /* configure RSS key */
 158         if (!rss_conf->rss_key) {
 159                 /* Calculate the default hash key */
 160                 for (i = 0; i <= vf->vf_res->rss_key_size; i++)
 161                         vf->rss_key[i] = (uint8_t)rte_rand();
 162         } else
 163                 rte_memcpy(vf->rss_key, rss_conf->rss_key,
 164                            RTE_MIN(rss_conf->rss_key_len,
 165                                    vf->vf_res->rss_key_size));
 166
 167         /* init RSS LUT table */
 168         for (i = 0; i < vf->vf_res->rss_lut_size; i++, j++) {
 169                 if (j >= nb_q)
 170                         j = 0;
 171                 vf->rss_lut[i] = j;
 172         }
 173         /* send virtchnnl ops to configure rss*/
 174         ret = avf_configure_rss_lut(adapter);
 175         if (ret)
 176                 return ret;
 177         ret = avf_configure_rss_key(adapter);
 178         if (ret)
 179                 return ret;
 180
 181         return 0;
 182 }
 183
 184 static int
 185 avf_init_rxq(struct rte_eth_dev *dev, struct avf_rx_queue *rxq)
 186 {
 187         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 188         struct rte_eth_dev_data *dev_data = dev->data;
 189         uint16_t buf_size, max_pkt_len, len;
 190
 191         buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM;
 192
 193         /* Calculate the maximum packet length allowed */
 194         len = rxq->rx_buf_len * AVF_MAX_CHAINED_RX_BUFFERS;
 195         max_pkt_len = RTE_MIN(len, dev->data->dev_conf.rxmode.max_rx_pkt_len);
 196
 197         /* Check if the jumbo frame and maximum packet length are set
 198          * correctly.
 199          */
 200         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 201                 if (max_pkt_len <= ETHER_MAX_LEN ||
 202                     max_pkt_len > AVF_FRAME_SIZE_MAX) {
 203                         PMD_DRV_LOG(ERR, "maximum packet length must be "
 204                                     "larger than %u and smaller than %u, "
 205                                     "as jumbo frame is enabled",
 206                                     (uint32_t)ETHER_MAX_LEN,
 207                                     (uint32_t)AVF_FRAME_SIZE_MAX);
 208                         return -EINVAL;
 209                 }
 210         } else {
 211                 if (max_pkt_len < ETHER_MIN_LEN ||
 212                     max_pkt_len > ETHER_MAX_LEN) {
 213                         PMD_DRV_LOG(ERR, "maximum packet length must be "
 214                                     "larger than %u and smaller than %u, "
 215                                     "as jumbo frame is disabled",
 216                                     (uint32_t)ETHER_MIN_LEN,
 217                                     (uint32_t)ETHER_MAX_LEN);
 218                         return -EINVAL;
 219                 }
 220         }
 221
 222         rxq->max_pkt_len = max_pkt_len;
 223         if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) ||
 224             (rxq->max_pkt_len + 2 * AVF_VLAN_TAG_SIZE) > buf_size) {
 225                 dev_data->scattered_rx = 1;
 226         }
 227         AVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 228         AVF_WRITE_FLUSH(hw);
 229
 230         return 0;
 231 }
 232
 233 static int
 234 avf_init_queues(struct rte_eth_dev *dev)
 235 {
 236         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 237         struct avf_rx_queue **rxq =
 238                 (struct avf_rx_queue **)dev->data->rx_queues;
 239         struct avf_tx_queue **txq =
 240                 (struct avf_tx_queue **)dev->data->tx_queues;
 241         int i, ret = AVF_SUCCESS;
 242
 243         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 244                 if (!rxq[i] || !rxq[i]->q_set)
 245                         continue;
 246                 ret = avf_init_rxq(dev, rxq[i]);
 247                 if (ret != AVF_SUCCESS)
 248                         break;
 249         }
 250         /* set rx/tx function to vector/scatter/single-segment
 251          * according to parameters
 252          */
 253         avf_set_rx_function(dev);
 254         avf_set_tx_function(dev);
 255
 256         return ret;
 257 }
 258
 259 static int
 260 avf_start_queues(struct rte_eth_dev *dev)
 261 {
 262         struct avf_rx_queue *rxq;
 263         struct avf_tx_queue *txq;
 264         int i;
 265
 266         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 267                 txq = dev->data->tx_queues[i];
 268                 if (txq->tx_deferred_start)
 269                         continue;
 270                 if (avf_dev_tx_queue_start(dev, i) != 0) {
 271                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 272                         return -1;
 273                 }
 274         }
 275
 276         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 277                 rxq = dev->data->rx_queues[i];
 278                 if (rxq->rx_deferred_start)
 279                         continue;
 280                 if (avf_dev_rx_queue_start(dev, i) != 0) {
 281                         PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
 282                         return -1;
 283                 }
 284         }
 285
 286         return 0;
 287 }
 288
 289 static int
 290 avf_dev_start(struct rte_eth_dev *dev)
 291 {
 292         struct avf_adapter *adapter =
 293                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 294         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 295         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 296         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 297         struct rte_intr_handle *intr_handle = dev->intr_handle;
 298         uint16_t interval;
 299         int i;
 300
 301         PMD_INIT_FUNC_TRACE();
 302
 303         hw->adapter_stopped = 0;
 304
 305         vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 306         vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
 307                                       dev->data->nb_tx_queues);
 308
 309         /* TODO: Rx interrupt */
 310
 311         if (avf_init_queues(dev) != 0) {
 312                 PMD_DRV_LOG(ERR, "failed to do Queue init");
 313                 return -1;
 314         }
 315
 316         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 317                 if (avf_init_rss(adapter) != 0) {
 318                         PMD_DRV_LOG(ERR, "configure rss failed");
 319                         goto err_rss;
 320                 }
 321         }
 322
 323         if (avf_configure_queues(adapter) != 0) {
 324                 PMD_DRV_LOG(ERR, "configure queues failed");
 325                 goto err_queue;
 326         }
 327
 328         /* Map interrupt for writeback */
 329         vf->nb_msix = 1;
 330         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) {
 331                 /* If WB_ON_ITR supports, enable it */
 332                 vf->msix_base = AVF_RX_VEC_START;
 333                 AVF_WRITE_REG(hw, AVFINT_DYN_CTLN1(vf->msix_base - 1),
 334                               AVFINT_DYN_CTLN1_ITR_INDX_MASK |
 335                               AVFINT_DYN_CTLN1_WB_ON_ITR_MASK);
 336         } else {
 337                 /* If no WB_ON_ITR offload flags, need to set interrupt for
 338                  * descriptor write back.
 339                  */
 340                 vf->msix_base = AVF_MISC_VEC_ID;
 341
 342                 /* set ITR to max */
 343                 interval = avf_calc_itr_interval(AVF_QUEUE_ITR_INTERVAL_MAX);
 344                 AVF_WRITE_REG(hw, AVFINT_DYN_CTL01,
 345                               AVFINT_DYN_CTL01_INTENA_MASK |
 346                               (AVF_ITR_INDEX_DEFAULT <<
 347                                AVFINT_DYN_CTL01_ITR_INDX_SHIFT) |
 348                               (interval << AVFINT_DYN_CTL01_INTERVAL_SHIFT));
 349         }
 350         AVF_WRITE_FLUSH(hw);
 351         /* map all queues to the same interrupt */
 352         for (i = 0; i < dev->data->nb_rx_queues; i++)
 353                 vf->rxq_map[0] |= 1 << i;
 354         if (avf_config_irq_map(adapter)) {
 355                 PMD_DRV_LOG(ERR, "config interrupt mapping failed");
 356                 goto err_queue;
 357         }
 358
 359         /* Set all mac addrs */
 360         avf_add_del_all_mac_addr(adapter, TRUE);
 361
 362         if (avf_start_queues(dev) != 0) {
 363                 PMD_DRV_LOG(ERR, "enable queues failed");
 364                 goto err_mac;
 365         }
 366
 367         /* TODO: enable interrupt for RX interrupt */
 368         return 0;
 369
 370 err_mac:
 371         avf_add_del_all_mac_addr(adapter, FALSE);
 372 err_queue:
 373 err_rss:
 374         return -1;
 375 }
 376
 377 static void
 378 avf_dev_stop(struct rte_eth_dev *dev)
 379 {
 380         struct avf_adapter *adapter =
 381                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 382         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev);
 383         int ret, i;
 384
 385         PMD_INIT_FUNC_TRACE();
 386
 387         if (hw->adapter_stopped == 1)
 388                 return;
 389
 390         avf_stop_queues(dev);
 391
 392         /*TODO: Disable the interrupt for Rx*/
 393
 394         /* TODO: Rx interrupt vector mapping free */
 395
 396         /* remove all mac addrs */
 397         avf_add_del_all_mac_addr(adapter, FALSE);
 398         hw->adapter_stopped = 1;
 399 }
 400
 401 static void
 402 avf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 403 {
 404         struct avf_adapter *adapter =
 405                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 406         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 407
 408         memset(dev_info, 0, sizeof(*dev_info));
 409         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 410         dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
 411         dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
 412         dev_info->min_rx_bufsize = AVF_BUF_SIZE_MIN;
 413         dev_info->max_rx_pktlen = AVF_FRAME_SIZE_MAX;
 414         dev_info->hash_key_size = vf->vf_res->rss_key_size;
 415         dev_info->reta_size = vf->vf_res->rss_lut_size;
 416         dev_info->flow_type_rss_offloads = AVF_RSS_OFFLOAD_ALL;
 417         dev_info->max_mac_addrs = AVF_NUM_MACADDR_MAX;
 418         dev_info->rx_offload_capa =
 419                 DEV_RX_OFFLOAD_VLAN_STRIP |
 420                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 421                 DEV_RX_OFFLOAD_UDP_CKSUM |
 422                 DEV_RX_OFFLOAD_TCP_CKSUM;
 423         dev_info->tx_offload_capa =
 424                 DEV_TX_OFFLOAD_VLAN_INSERT |
 425                 DEV_TX_OFFLOAD_IPV4_CKSUM |
 426                 DEV_TX_OFFLOAD_UDP_CKSUM |
 427                 DEV_TX_OFFLOAD_TCP_CKSUM |
 428                 DEV_TX_OFFLOAD_SCTP_CKSUM |
 429                 DEV_TX_OFFLOAD_TCP_TSO;
 430
 431         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 432                 .rx_free_thresh = AVF_DEFAULT_RX_FREE_THRESH,
 433                 .rx_drop_en = 0,
 434         };
 435
 436         dev_info->default_txconf = (struct rte_eth_txconf) {
 437                 .tx_free_thresh = AVF_DEFAULT_TX_FREE_THRESH,
 438                 .tx_rs_thresh = AVF_DEFAULT_TX_RS_THRESH,
 439                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 440                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 441         };
 442
 443         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
 444                 .nb_max = AVF_MAX_RING_DESC,
 445                 .nb_min = AVF_MIN_RING_DESC,
 446                 .nb_align = AVF_ALIGN_RING_DESC,
 447         };
 448
 449         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
 450                 .nb_max = AVF_MAX_RING_DESC,
 451                 .nb_min = AVF_MIN_RING_DESC,
 452                 .nb_align = AVF_ALIGN_RING_DESC,
 453         };
 454 }
 455
 456 static const uint32_t *
 457 avf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 458 {
 459         static const uint32_t ptypes[] = {
 460                 RTE_PTYPE_L2_ETHER,
 461                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 462                 RTE_PTYPE_L4_FRAG,
 463                 RTE_PTYPE_L4_ICMP,
 464                 RTE_PTYPE_L4_NONFRAG,
 465                 RTE_PTYPE_L4_SCTP,
 466                 RTE_PTYPE_L4_TCP,
 467                 RTE_PTYPE_L4_UDP,
 468                 RTE_PTYPE_UNKNOWN
 469         };
 470         return ptypes;
 471 }
 472
 473 int
 474 avf_dev_link_update(struct rte_eth_dev *dev,
 475                     __rte_unused int wait_to_complete)
 476 {
 477         struct rte_eth_link new_link;
 478         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 479
 480         /* Only read status info stored in VF, and the info is updated
 481          *  when receive LINK_CHANGE evnet from PF by Virtchnnl.
 482          */
 483         switch (vf->link_speed) {
 484         case VIRTCHNL_LINK_SPEED_100MB:
 485                 new_link.link_speed = ETH_SPEED_NUM_100M;
 486                 break;
 487         case VIRTCHNL_LINK_SPEED_1GB:
 488                 new_link.link_speed = ETH_SPEED_NUM_1G;
 489                 break;
 490         case VIRTCHNL_LINK_SPEED_10GB:
 491                 new_link.link_speed = ETH_SPEED_NUM_10G;
 492                 break;
 493         case VIRTCHNL_LINK_SPEED_20GB:
 494                 new_link.link_speed = ETH_SPEED_NUM_20G;
 495                 break;
 496         case VIRTCHNL_LINK_SPEED_25GB:
 497                 new_link.link_speed = ETH_SPEED_NUM_25G;
 498                 break;
 499         case VIRTCHNL_LINK_SPEED_40GB:
 500                 new_link.link_speed = ETH_SPEED_NUM_40G;
 501                 break;
 502         default:
 503                 new_link.link_speed = ETH_SPEED_NUM_NONE;
 504                 break;
 505         }
 506
 507         new_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 508         new_link.link_status = vf->link_up ? ETH_LINK_UP :
 509                                              ETH_LINK_DOWN;
 510         new_link.link_autoneg = !!(dev->data->dev_conf.link_speeds &
 511                                 ETH_LINK_SPEED_FIXED);
 512
 513         rte_atomic64_cmpset((uint64_t *)&dev->data->dev_link,
 514                             *(uint64_t *)&dev->data->dev_link,
 515                             *(uint64_t *)&new_link);
 516
 517         return 0;
 518 }
 519
 520 static void
 521 avf_dev_promiscuous_enable(struct rte_eth_dev *dev)
 522 {
 523         struct avf_adapter *adapter =
 524                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 525         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 526         int ret;
 527
 528         if (vf->promisc_unicast_enabled)
 529                 return;
 530
 531         ret = avf_config_promisc(adapter, TRUE, vf->promisc_multicast_enabled);
 532         if (!ret)
 533                 vf->promisc_unicast_enabled = TRUE;
 534 }
 535
 536 static void
 537 avf_dev_promiscuous_disable(struct rte_eth_dev *dev)
 538 {
 539         struct avf_adapter *adapter =
 540                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 541         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 542         int ret;
 543
 544         if (!vf->promisc_unicast_enabled)
 545                 return;
 546
 547         ret = avf_config_promisc(adapter, FALSE, vf->promisc_multicast_enabled);
 548         if (!ret)
 549                 vf->promisc_unicast_enabled = FALSE;
 550 }
 551
 552 static void
 553 avf_dev_allmulticast_enable(struct rte_eth_dev *dev)
 554 {
 555         struct avf_adapter *adapter =
 556                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 557         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 558         int ret;
 559
 560         if (vf->promisc_multicast_enabled)
 561                 return;
 562
 563         ret = avf_config_promisc(adapter, vf->promisc_unicast_enabled, TRUE);
 564         if (!ret)
 565                 vf->promisc_multicast_enabled = TRUE;
 566 }
 567
 568 static void
 569 avf_dev_allmulticast_disable(struct rte_eth_dev *dev)
 570 {
 571         struct avf_adapter *adapter =
 572                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 573         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 574         int ret;
 575
 576         if (!vf->promisc_multicast_enabled)
 577                 return;
 578
 579         ret = avf_config_promisc(adapter, vf->promisc_unicast_enabled, FALSE);
 580         if (!ret)
 581                 vf->promisc_multicast_enabled = FALSE;
 582 }
 583
 584 static int
 585 avf_dev_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr,
 586                      __rte_unused uint32_t index,
 587                      __rte_unused uint32_t pool)
 588 {
 589         struct avf_adapter *adapter =
 590                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 591         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 592         int err;
 593
 594         if (is_zero_ether_addr(addr)) {
 595                 PMD_DRV_LOG(ERR, "Invalid Ethernet Address");
 596                 return -EINVAL;
 597         }
 598
 599         err = avf_add_del_eth_addr(adapter, addr, TRUE);
 600         if (err) {
 601                 PMD_DRV_LOG(ERR, "fail to add MAC address");
 602                 return -EIO;
 603         }
 604
 605         vf->mac_num++;
 606
 607         return 0;
 608 }
 609
 610 static void
 611 avf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index)
 612 {
 613         struct avf_adapter *adapter =
 614                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 615         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 616         struct ether_addr *addr;
 617         int err;
 618
 619         addr = &dev->data->mac_addrs[index];
 620
 621         err = avf_add_del_eth_addr(adapter, addr, FALSE);
 622         if (err)
 623                 PMD_DRV_LOG(ERR, "fail to delete MAC address");
 624
 625         vf->mac_num--;
 626 }
 627
 628 static int
 629 avf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 630 {
 631         struct avf_adapter *adapter =
 632                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 633         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 634         int err;
 635
 636         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
 637                 return -ENOTSUP;
 638
 639         err = avf_add_del_vlan(adapter, vlan_id, on);
 640         if (err)
 641                 return -EIO;
 642         return 0;
 643 }
 644
 645 static int
 646 avf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
 647 {
 648         struct avf_adapter *adapter =
 649                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 650         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 651         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 652         int err;
 653
 654         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
 655                 return -ENOTSUP;
 656
 657         /* Vlan stripping setting */
 658         if (mask & ETH_VLAN_STRIP_MASK) {
 659                 /* Enable or disable VLAN stripping */
 660                 if (dev_conf->rxmode.hw_vlan_strip)
 661                         err = avf_enable_vlan_strip(adapter);
 662                 else
 663                         err = avf_disable_vlan_strip(adapter);
 664         }
 665
 666         if (err)
 667                 return -EIO;
 668         return 0;
 669 }
 670
 671 static int
 672 avf_dev_rss_reta_update(struct rte_eth_dev *dev,
 673                         struct rte_eth_rss_reta_entry64 *reta_conf,
 674                         uint16_t reta_size)
 675 {
 676         struct avf_adapter *adapter =
 677                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 678         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 679         uint8_t *lut;
 680         uint16_t i, idx, shift;
 681         int ret;
 682
 683         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF))
 684                 return -ENOTSUP;
 685
 686         if (reta_size != vf->vf_res->rss_lut_size) {
 687                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
 688                         "(%d) doesn't match the number of hardware can "
 689                         "support (%d)", reta_size, vf->vf_res->rss_lut_size);
 690                 return -EINVAL;
 691         }
 692
 693         lut = rte_zmalloc("rss_lut", reta_size, 0);
 694         if (!lut) {
 695                 PMD_DRV_LOG(ERR, "No memory can be allocated");
 696                 return -ENOMEM;
 697         }
 698         /* store the old lut table temporarily */
 699         rte_memcpy(lut, vf->rss_lut, reta_size);
 700
 701         for (i = 0; i < reta_size; i++) {
 702                 idx = i / RTE_RETA_GROUP_SIZE;
 703                 shift = i % RTE_RETA_GROUP_SIZE;
 704                 if (reta_conf[idx].mask & (1ULL << shift))
 705                         lut[i] = reta_conf[idx].reta[shift];
 706         }
 707
 708         rte_memcpy(vf->rss_lut, lut, reta_size);
 709         /* send virtchnnl ops to configure rss*/
 710         ret = avf_configure_rss_lut(adapter);
 711         if (ret) /* revert back */
 712                 rte_memcpy(vf->rss_lut, lut, reta_size);
 713         rte_free(lut);
 714
 715         return ret;
 716 }
 717
 718 static int
 719 avf_dev_rss_reta_query(struct rte_eth_dev *dev,
 720                        struct rte_eth_rss_reta_entry64 *reta_conf,
 721                        uint16_t reta_size)
 722 {
 723         struct avf_adapter *adapter =
 724                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 725         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 726         uint16_t i, idx, shift;
 727
 728         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF))
 729                 return -ENOTSUP;
 730
 731         if (reta_size != vf->vf_res->rss_lut_size) {
 732                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
 733                         "(%d) doesn't match the number of hardware can "
 734                         "support (%d)", reta_size, vf->vf_res->rss_lut_size);
 735                 return -EINVAL;
 736         }
 737
 738         for (i = 0; i < reta_size; i++) {
 739                 idx = i / RTE_RETA_GROUP_SIZE;
 740                 shift = i % RTE_RETA_GROUP_SIZE;
 741                 if (reta_conf[idx].mask & (1ULL << shift))
 742                         reta_conf[idx].reta[shift] = vf->rss_lut[i];
 743         }
 744
 745         return 0;
 746 }
 747
 748 static int
 749 avf_dev_rss_hash_update(struct rte_eth_dev *dev,
 750                         struct rte_eth_rss_conf *rss_conf)
 751 {
 752         struct avf_adapter *adapter =
 753                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 754         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 755
 756         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF))
 757                 return -ENOTSUP;
 758
 759         /* HENA setting, it is enabled by default, no change */
 760         if (!rss_conf->rss_key || rss_conf->rss_key_len == 0) {
 761                 PMD_DRV_LOG(DEBUG, "No key to be configured");
 762                 return 0;
 763         } else if (rss_conf->rss_key_len != vf->vf_res->rss_key_size) {
 764                 PMD_DRV_LOG(ERR, "The size of hash key configured "
 765                         "(%d) doesn't match the size of hardware can "
 766                         "support (%d)", rss_conf->rss_key_len,
 767                         vf->vf_res->rss_key_size);
 768                 return -EINVAL;
 769         }
 770
 771         rte_memcpy(vf->rss_key, rss_conf->rss_key, rss_conf->rss_key_len);
 772
 773         return avf_configure_rss_key(adapter);
 774 }
 775
 776 static int
 777 avf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 778                           struct rte_eth_rss_conf *rss_conf)
 779 {
 780         struct avf_adapter *adapter =
 781                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 782         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
 783
 784         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF))
 785                 return -ENOTSUP;
 786
 787          /* Just set it to default value now. */
 788         rss_conf->rss_hf = AVF_RSS_OFFLOAD_ALL;
 789
 790         if (!rss_conf->rss_key)
 791                 return 0;
 792
 793         rss_conf->rss_key_len = vf->vf_res->rss_key_size;
 794         rte_memcpy(rss_conf->rss_key, vf->rss_key, rss_conf->rss_key_len);
 795
 796         return 0;
 797 }
 798
 799 static void
 800 avf_dev_set_default_mac_addr(struct rte_eth_dev *dev,
 801                              struct ether_addr *mac_addr)
 802 {
 803         struct avf_adapter *adapter =
 804                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 805         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(adapter);
 806         struct ether_addr *perm_addr, *old_addr;
 807         int ret;
 808
 809         old_addr = (struct ether_addr *)hw->mac.addr;
 810         perm_addr = (struct ether_addr *)hw->mac.perm_addr;
 811
 812         if (is_same_ether_addr(mac_addr, old_addr))
 813                 return;
 814
 815         /* If the MAC address is configured by host, skip the setting */
 816         if (is_valid_assigned_ether_addr(perm_addr))
 817                 return;
 818
 819         ret = avf_add_del_eth_addr(adapter, old_addr, FALSE);
 820         if (ret)
 821                 PMD_DRV_LOG(ERR, "Fail to delete old MAC:"
 822                             " %02X:%02X:%02X:%02X:%02X:%02X",
 823                             old_addr->addr_bytes[0],
 824                             old_addr->addr_bytes[1],
 825                             old_addr->addr_bytes[2],
 826                             old_addr->addr_bytes[3],
 827                             old_addr->addr_bytes[4],
 828                             old_addr->addr_bytes[5]);
 829
 830         ret = avf_add_del_eth_addr(adapter, mac_addr, TRUE);
 831         if (ret)
 832                 PMD_DRV_LOG(ERR, "Fail to add new MAC:"
 833                             " %02X:%02X:%02X:%02X:%02X:%02X",
 834                             mac_addr->addr_bytes[0],
 835                             mac_addr->addr_bytes[1],
 836                             mac_addr->addr_bytes[2],
 837                             mac_addr->addr_bytes[3],
 838                             mac_addr->addr_bytes[4],
 839                             mac_addr->addr_bytes[5]);
 840
 841         ether_addr_copy(mac_addr, (struct ether_addr *)hw->mac.addr);
 842 }
 843
 844 static int
 845 avf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 846 {
 847         struct avf_adapter *adapter =
 848                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 849         struct virtchnl_eth_stats *pstats = NULL;
 850         int ret;
 851
 852         ret = avf_query_stats(adapter, &pstats);
 853         if (ret == 0) {
 854                 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast +
 855                                                 pstats->rx_broadcast;
 856                 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast +
 857                                                 pstats->tx_unicast;
 858                 stats->imissed = pstats->rx_discards;
 859                 stats->oerrors = pstats->tx_errors + pstats->tx_discards;
 860                 stats->ibytes = pstats->rx_bytes;
 861                 stats->obytes = pstats->tx_bytes;
 862         } else {
 863                 PMD_DRV_LOG(ERR, "Get statistics failed");
 864         }
 865         return -EIO;
 866 }
 867
 868 static int
 869 avf_check_vf_reset_done(struct avf_hw *hw)
 870 {
 871         int i, reset;
 872
 873         for (i = 0; i < AVF_RESET_WAIT_CNT; i++) {
 874                 reset = AVF_READ_REG(hw, AVFGEN_RSTAT) &
 875                         AVFGEN_RSTAT_VFR_STATE_MASK;
 876                 reset = reset >> AVFGEN_RSTAT_VFR_STATE_SHIFT;
 877                 if (reset == VIRTCHNL_VFR_VFACTIVE ||
 878                     reset == VIRTCHNL_VFR_COMPLETED)
 879                         break;
 880                 rte_delay_ms(20);
 881         }
 882
 883         if (i >= AVF_RESET_WAIT_CNT)
 884                 return -1;
 885
 886         return 0;
 887 }
 888
 889 static int
 890 avf_init_vf(struct rte_eth_dev *dev)
 891 {
 892         int i, err, bufsz;
 893         struct avf_adapter *adapter =
 894                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 895         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 896         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 897
 898         err = avf_set_mac_type(hw);
 899         if (err) {
 900                 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err);
 901                 goto err;
 902         }
 903
 904         err = avf_check_vf_reset_done(hw);
 905         if (err) {
 906                 PMD_INIT_LOG(ERR, "VF is still resetting");
 907                 goto err;
 908         }
 909
 910         avf_init_adminq_parameter(hw);
 911         err = avf_init_adminq(hw);
 912         if (err) {
 913                 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err);
 914                 goto err;
 915         }
 916
 917         vf->aq_resp = rte_zmalloc("vf_aq_resp", AVF_AQ_BUF_SZ, 0);
 918         if (!vf->aq_resp) {
 919                 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
 920                 goto err_aq;
 921         }
 922         if (avf_check_api_version(adapter) != 0) {
 923                 PMD_INIT_LOG(ERR, "check_api version failed");
 924                 goto err_api;
 925         }
 926
 927         bufsz = sizeof(struct virtchnl_vf_resource) +
 928                 (AVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource));
 929         vf->vf_res = rte_zmalloc("vf_res", bufsz, 0);
 930         if (!vf->vf_res) {
 931                 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory");
 932                 goto err_api;
 933         }
 934         if (avf_get_vf_resource(adapter) != 0) {
 935                 PMD_INIT_LOG(ERR, "avf_get_vf_config failed");
 936                 goto err_alloc;
 937         }
 938         /* Allocate memort for RSS info */
 939         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 940                 vf->rss_key = rte_zmalloc("rss_key",
 941                                           vf->vf_res->rss_key_size, 0);
 942                 if (!vf->rss_key) {
 943                         PMD_INIT_LOG(ERR, "unable to allocate rss_key memory");
 944                         goto err_rss;
 945                 }
 946                 vf->rss_lut = rte_zmalloc("rss_lut",
 947                                           vf->vf_res->rss_lut_size, 0);
 948                 if (!vf->rss_lut) {
 949                         PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory");
 950                         goto err_rss;
 951                 }
 952         }
 953         return 0;
 954 err_rss:
 955         rte_free(vf->rss_key);
 956         rte_free(vf->rss_lut);
 957 err_alloc:
 958         rte_free(vf->vf_res);
 959         vf->vsi_res = NULL;
 960 err_api:
 961         rte_free(vf->aq_resp);
 962 err_aq:
 963         avf_shutdown_adminq(hw);
 964 err:
 965         return -1;
 966 }
 967
 968 /* Enable default admin queue interrupt setting */
 969 static inline void
 970 avf_enable_irq0(struct avf_hw *hw)
 971 {
 972         /* Enable admin queue interrupt trigger */
 973         AVF_WRITE_REG(hw, AVFINT_ICR0_ENA1, AVFINT_ICR0_ENA1_ADMINQ_MASK);
 974
 975         AVF_WRITE_REG(hw, AVFINT_DYN_CTL01, AVFINT_DYN_CTL01_INTENA_MASK |
 976                                             AVFINT_DYN_CTL01_ITR_INDX_MASK);
 977
 978         AVF_WRITE_FLUSH(hw);
 979 }
 980
 981 static inline void
 982 avf_disable_irq0(struct avf_hw *hw)
 983 {
 984         /* Disable all interrupt types */
 985         AVF_WRITE_REG(hw, AVFINT_ICR0_ENA1, 0);
 986         AVF_WRITE_REG(hw, AVFINT_DYN_CTL01,
 987                       AVFINT_DYN_CTL01_ITR_INDX_MASK);
 988         AVF_WRITE_FLUSH(hw);
 989 }
 990
 991 static void
 992 avf_dev_interrupt_handler(void *param)
 993 {
 994         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 995         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 996
 997         avf_disable_irq0(hw);
 998
 999         avf_handle_virtchnl_msg(dev);
1000
1001 done:
1002         avf_enable_irq0(hw);
1003 }
1004
1005 static int
1006 avf_dev_init(struct rte_eth_dev *eth_dev)
1007 {
1008         struct avf_adapter *adapter =
1009                 AVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1010         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(adapter);
1011         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1012
1013         PMD_INIT_FUNC_TRACE();
1014
1015         /* assign ops func pointer */
1016         eth_dev->dev_ops = &avf_eth_dev_ops;
1017         eth_dev->rx_pkt_burst = &avf_recv_pkts;
1018         eth_dev->tx_pkt_burst = &avf_xmit_pkts;
1019         eth_dev->tx_pkt_prepare = &avf_prep_pkts;
1020
1021         /* For secondary processes, we don't initialise any further as primary
1022          * has already done this work. Only check if we need a different RX
1023          * and TX function.
1024          */
1025         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1026                 avf_set_rx_function(eth_dev);
1027                 avf_set_tx_function(eth_dev);
1028                 return 0;
1029         }
1030         rte_eth_copy_pci_info(eth_dev, pci_dev);
1031
1032         hw->vendor_id = pci_dev->id.vendor_id;
1033         hw->device_id = pci_dev->id.device_id;
1034         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1035         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1036         hw->bus.bus_id = pci_dev->addr.bus;
1037         hw->bus.device = pci_dev->addr.devid;
1038         hw->bus.func = pci_dev->addr.function;
1039         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1040         hw->back = AVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1041         adapter->eth_dev = eth_dev;
1042
1043         if (avf_init_vf(eth_dev) != 0) {
1044                 PMD_INIT_LOG(ERR, "Init vf failed");
1045                 return -1;
1046         }
1047
1048         /* copy mac addr */
1049         eth_dev->data->mac_addrs = rte_zmalloc(
1050                                         "avf_mac",
1051                                         ETHER_ADDR_LEN * AVF_NUM_MACADDR_MAX,
1052                                         0);
1053         if (!eth_dev->data->mac_addrs) {
1054                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to"
1055                              " store MAC addresses",
1056                              ETHER_ADDR_LEN * AVF_NUM_MACADDR_MAX);
1057                 return -ENOMEM;
1058         }
1059         /* If the MAC address is not configured by host,
1060          * generate a random one.
1061          */
1062         if (!is_valid_assigned_ether_addr((struct ether_addr *)hw->mac.addr))
1063                 eth_random_addr(hw->mac.addr);
1064         ether_addr_copy((struct ether_addr *)hw->mac.addr,
1065                         &eth_dev->data->mac_addrs[0]);
1066
1067         /* register callback func to eal lib */
1068         rte_intr_callback_register(&pci_dev->intr_handle,
1069                                    avf_dev_interrupt_handler,
1070                                    (void *)eth_dev);
1071
1072         /* enable uio intr after callback register */
1073         rte_intr_enable(&pci_dev->intr_handle);
1074
1075         /* configure and enable device interrupt */
1076         avf_enable_irq0(hw);
1077
1078         return 0;
1079 }
1080
1081 static void
1082 avf_dev_close(struct rte_eth_dev *dev)
1083 {
1084         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1085         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1086         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1087
1088         avf_dev_stop(dev);
1089         avf_shutdown_adminq(hw);
1090         /* disable uio intr before callback unregister */
1091         rte_intr_disable(intr_handle);
1092
1093         /* unregister callback func from eal lib */
1094         rte_intr_callback_unregister(intr_handle,
1095                                      avf_dev_interrupt_handler, dev);
1096         avf_disable_irq0(hw);
1097 }
1098
1099 static int
1100 avf_dev_uninit(struct rte_eth_dev *dev)
1101 {
1102         struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1103         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1104
1105         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1106                 return -EPERM;
1107
1108         dev->dev_ops = NULL;
1109         dev->rx_pkt_burst = NULL;
1110         dev->tx_pkt_burst = NULL;
1111         if (hw->adapter_stopped == 0)
1112                 avf_dev_close(dev);
1113
1114         rte_free(vf->vf_res);
1115         vf->vsi_res = NULL;
1116         vf->vf_res = NULL;
1117
1118         rte_free(vf->aq_resp);
1119         vf->aq_resp = NULL;
1120
1121         rte_free(dev->data->mac_addrs);
1122         dev->data->mac_addrs = NULL;
1123
1124         if (vf->rss_lut) {
1125                 rte_free(vf->rss_lut);
1126                 vf->rss_lut = NULL;
1127         }
1128         if (vf->rss_key) {
1129                 rte_free(vf->rss_key);
1130                 vf->rss_key = NULL;
1131         }
1132
1133         return 0;
1134 }
1135
1136 static int eth_avf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1137                              struct rte_pci_device *pci_dev)
1138 {
1139         return rte_eth_dev_pci_generic_probe(pci_dev,
1140                 sizeof(struct avf_adapter), avf_dev_init);
1141 }
1142
1143 static int eth_avf_pci_remove(struct rte_pci_device *pci_dev)
1144 {
1145         return rte_eth_dev_pci_generic_remove(pci_dev, avf_dev_uninit);
1146 }
1147
1148 /* Adaptive virtual function driver struct */
1149 static struct rte_pci_driver rte_avf_pmd = {
1150         .id_table = pci_id_avf_map,
1151         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
1152                      RTE_PCI_DRV_IOVA_AS_VA,
1153         .probe = eth_avf_pci_probe,
1154         .remove = eth_avf_pci_remove,
1155 };
1156
1157 RTE_PMD_REGISTER_PCI(net_avf, rte_avf_pmd);
1158 RTE_PMD_REGISTER_PCI_TABLE(net_avf, pci_id_avf_map);
1159 RTE_PMD_REGISTER_KMOD_DEP(net_avf, "* igb_uio | vfio-pci");
1160 RTE_INIT(avf_init_log);
1161 static void
1162 avf_init_log(void)
1163 {
1164         avf_logtype_init = rte_log_register("pmd.avf.init");
1165         if (avf_logtype_init >= 0)
1166                 rte_log_set_level(avf_logtype_init, RTE_LOG_NOTICE);
1167         avf_logtype_driver = rte_log_register("pmd.avf.driver");
1168         if (avf_logtype_driver >= 0)
1169                 rte_log_set_level(avf_logtype_driver, RTE_LOG_NOTICE);
1170 }
1171
1172 /* memory func for base code */
1173 enum avf_status_code
1174 avf_allocate_dma_mem_d(__rte_unused struct avf_hw *hw,
1175                        struct avf_dma_mem *mem,
1176                        u64 size,
1177                        u32 alignment)
1178 {
1179         const struct rte_memzone *mz = NULL;
1180         char z_name[RTE_MEMZONE_NAMESIZE];
1181
1182         if (!mem)
1183                 return AVF_ERR_PARAM;
1184
1185         snprintf(z_name, sizeof(z_name), "avf_dma_%"PRIu64, rte_rand());
1186         mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
1187                                          alignment, RTE_PGSIZE_2M);
1188         if (!mz)
1189                 return AVF_ERR_NO_MEMORY;
1190
1191         mem->size = size;
1192         mem->va = mz->addr;
1193         mem->pa = mz->phys_addr;
1194         mem->zone = (const void *)mz;
1195         PMD_DRV_LOG(DEBUG,
1196                     "memzone %s allocated with physical address: %"PRIu64,
1197                     mz->name, mem->pa);
1198
1199         return AVF_SUCCESS;
1200 }
1201
1202 enum avf_status_code
1203 avf_free_dma_mem_d(__rte_unused struct avf_hw *hw,
1204                    struct avf_dma_mem *mem)
1205 {
1206         if (!mem)
1207                 return AVF_ERR_PARAM;
1208
1209         PMD_DRV_LOG(DEBUG,
1210                     "memzone %s to be freed with physical address: %"PRIu64,
1211                     ((const struct rte_memzone *)mem->zone)->name, mem->pa);
1212         rte_memzone_free((const struct rte_memzone *)mem->zone);
1213         mem->zone = NULL;
1214         mem->va = NULL;
1215         mem->pa = (u64)0;
1216
1217         return AVF_SUCCESS;
1218 }
1219
1220 enum avf_status_code
1221 avf_allocate_virt_mem_d(__rte_unused struct avf_hw *hw,
1222                         struct avf_virt_mem *mem,
1223                         u32 size)
1224 {
1225         if (!mem)
1226                 return AVF_ERR_PARAM;
1227
1228         mem->size = size;
1229         mem->va = rte_zmalloc("avf", size, 0);
1230
1231         if (mem->va)
1232                 return AVF_SUCCESS;
1233         else
1234                 return AVF_ERR_NO_MEMORY;
1235 }
1236
1237 enum avf_status_code
1238 avf_free_virt_mem_d(__rte_unused struct avf_hw *hw,
1239                     struct avf_virt_mem *mem)
1240 {
1241         if (!mem)
1242                 return AVF_ERR_PARAM;
1243
1244         rte_free(mem->va);
1245         mem->va = NULL;
1246
1247         return AVF_SUCCESS;
1248 }