drivers/net/virtio/virtio_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_memcpy.h>
  14 #include <rte_string_fns.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_branch_prediction.h>
  18 #include <rte_pci.h>
  19 #include <rte_bus_pci.h>
  20 #include <rte_ether.h>
  21 #include <rte_ip.h>
  22 #include <rte_arp.h>
  23 #include <rte_common.h>
  24 #include <rte_errno.h>
  25 #include <rte_cpuflags.h>
  26
  27 #include <rte_memory.h>
  28 #include <rte_eal.h>
  29 #include <rte_dev.h>
  30 #include <rte_cycles.h>
  31 #include <rte_kvargs.h>
  32
  33 #include "virtio_ethdev.h"
  34 #include "virtio_pci.h"
  35 #include "virtio_logs.h"
  36 #include "virtqueue.h"
  37 #include "virtio_rxtx.h"
  38 #include "virtio_user/virtio_user_dev.h"
  39
  40 static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev);
  41 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  42 static int  virtio_dev_start(struct rte_eth_dev *dev);
  43 static void virtio_dev_stop(struct rte_eth_dev *dev);
  44 static int virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
  45 static int virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
  46 static int virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
  47 static int virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
  48 static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
  49 static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
  50         int *vdpa,
  51         uint32_t *speed,
  52         int *vectorized);
  53 static int virtio_dev_info_get(struct rte_eth_dev *dev,
  54                                 struct rte_eth_dev_info *dev_info);
  55 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  56         int wait_to_complete);
  57 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  58
  59 static void virtio_set_hwaddr(struct virtio_hw *hw);
  60 static void virtio_get_hwaddr(struct virtio_hw *hw);
  61
  62 static int virtio_dev_stats_get(struct rte_eth_dev *dev,
  63                                  struct rte_eth_stats *stats);
  64 static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
  65                                  struct rte_eth_xstat *xstats, unsigned n);
  66 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
  67                                        struct rte_eth_xstat_name *xstats_names,
  68                                        unsigned limit);
  69 static int virtio_dev_stats_reset(struct rte_eth_dev *dev);
  70 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  71 static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
  72                                 uint16_t vlan_id, int on);
  73 static int virtio_mac_addr_add(struct rte_eth_dev *dev,
  74                                 struct rte_ether_addr *mac_addr,
  75                                 uint32_t index, uint32_t vmdq);
  76 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  77 static int virtio_mac_addr_set(struct rte_eth_dev *dev,
  78                                 struct rte_ether_addr *mac_addr);
  79
  80 static int virtio_intr_disable(struct rte_eth_dev *dev);
  81
  82 static int virtio_dev_queue_stats_mapping_set(
  83         struct rte_eth_dev *eth_dev,
  84         uint16_t queue_id,
  85         uint8_t stat_idx,
  86         uint8_t is_rx);
  87
  88 int virtio_logtype_init;
  89 int virtio_logtype_driver;
  90
  91 static void virtio_notify_peers(struct rte_eth_dev *dev);
  92 static void virtio_ack_link_announce(struct rte_eth_dev *dev);
  93
  94 /*
  95  * The set of PCI devices this driver supports
  96  */
  97 static const struct rte_pci_id pci_id_virtio_map[] = {
  98         { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_LEGACY_DEVICEID_NET) },
  99         { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_MODERN_DEVICEID_NET) },
 100         { .vendor_id = 0, /* sentinel */ },
 101 };
 102
 103 struct rte_virtio_xstats_name_off {
 104         char name[RTE_ETH_XSTATS_NAME_SIZE];
 105         unsigned offset;
 106 };
 107
 108 /* [rt]x_qX_ is prepended to the name string here */
 109 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
 110         {"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
 111         {"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
 112         {"errors",                 offsetof(struct virtnet_rx, stats.errors)},
 113         {"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
 114         {"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
 115         {"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
 116         {"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
 117         {"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
 118         {"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
 119         {"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
 120         {"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
 121         {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
 122         {"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
 123 };
 124
 125 /* [rt]x_qX_ is prepended to the name string here */
 126 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
 127         {"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
 128         {"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
 129         {"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
 130         {"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
 131         {"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
 132         {"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
 133         {"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
 134         {"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
 135         {"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
 136         {"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
 137         {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
 138         {"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
 139 };
 140
 141 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
 142                             sizeof(rte_virtio_rxq_stat_strings[0]))
 143 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
 144                             sizeof(rte_virtio_txq_stat_strings[0]))
 145
 146 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS];
 147
 148 static struct virtio_pmd_ctrl *
 149 virtio_send_command_packed(struct virtnet_ctl *cvq,
 150                            struct virtio_pmd_ctrl *ctrl,
 151                            int *dlen, int pkt_num)
 152 {
 153         struct virtqueue *vq = cvq->vq;
 154         int head;
 155         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 156         struct virtio_pmd_ctrl *result;
 157         uint16_t flags;
 158         int sum = 0;
 159         int nb_descs = 0;
 160         int k;
 161
 162         /*
 163          * Format is enforced in qemu code:
 164          * One TX packet for header;
 165          * At least one TX packet per argument;
 166          * One RX packet for ACK.
 167          */
 168         head = vq->vq_avail_idx;
 169         flags = vq->vq_packed.cached_flags;
 170         desc[head].addr = cvq->virtio_net_hdr_mem;
 171         desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 172         vq->vq_free_cnt--;
 173         nb_descs++;
 174         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 175                 vq->vq_avail_idx -= vq->vq_nentries;
 176                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 177         }
 178
 179         for (k = 0; k < pkt_num; k++) {
 180                 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 181                         + sizeof(struct virtio_net_ctrl_hdr)
 182                         + sizeof(ctrl->status) + sizeof(uint8_t) * sum;
 183                 desc[vq->vq_avail_idx].len = dlen[k];
 184                 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT |
 185                         vq->vq_packed.cached_flags;
 186                 sum += dlen[k];
 187                 vq->vq_free_cnt--;
 188                 nb_descs++;
 189                 if (++vq->vq_avail_idx >= vq->vq_nentries) {
 190                         vq->vq_avail_idx -= vq->vq_nentries;
 191                         vq->vq_packed.cached_flags ^=
 192                                 VRING_PACKED_DESC_F_AVAIL_USED;
 193                 }
 194         }
 195
 196         desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 197                 + sizeof(struct virtio_net_ctrl_hdr);
 198         desc[vq->vq_avail_idx].len = sizeof(ctrl->status);
 199         desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE |
 200                 vq->vq_packed.cached_flags;
 201         vq->vq_free_cnt--;
 202         nb_descs++;
 203         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 204                 vq->vq_avail_idx -= vq->vq_nentries;
 205                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 206         }
 207
 208         virtio_wmb(vq->hw->weak_barriers);
 209         desc[head].flags = VRING_DESC_F_NEXT | flags;
 210
 211         virtio_wmb(vq->hw->weak_barriers);
 212         virtqueue_notify(vq);
 213
 214         /* wait for used descriptors in virtqueue */
 215         while (!desc_is_used(&desc[head], vq))
 216                 usleep(100);
 217
 218         virtio_rmb(vq->hw->weak_barriers);
 219
 220         /* now get used descriptors */
 221         vq->vq_free_cnt += nb_descs;
 222         vq->vq_used_cons_idx += nb_descs;
 223         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 224                 vq->vq_used_cons_idx -= vq->vq_nentries;
 225                 vq->vq_packed.used_wrap_counter ^= 1;
 226         }
 227
 228         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n"
 229                         "vq->vq_avail_idx=%d\n"
 230                         "vq->vq_used_cons_idx=%d\n"
 231                         "vq->vq_packed.cached_flags=0x%x\n"
 232                         "vq->vq_packed.used_wrap_counter=%d\n",
 233                         vq->vq_free_cnt,
 234                         vq->vq_avail_idx,
 235                         vq->vq_used_cons_idx,
 236                         vq->vq_packed.cached_flags,
 237                         vq->vq_packed.used_wrap_counter);
 238
 239         result = cvq->virtio_net_hdr_mz->addr;
 240         return result;
 241 }
 242
 243 static struct virtio_pmd_ctrl *
 244 virtio_send_command_split(struct virtnet_ctl *cvq,
 245                           struct virtio_pmd_ctrl *ctrl,
 246                           int *dlen, int pkt_num)
 247 {
 248         struct virtio_pmd_ctrl *result;
 249         struct virtqueue *vq = cvq->vq;
 250         uint32_t head, i;
 251         int k, sum = 0;
 252
 253         head = vq->vq_desc_head_idx;
 254
 255         /*
 256          * Format is enforced in qemu code:
 257          * One TX packet for header;
 258          * At least one TX packet per argument;
 259          * One RX packet for ACK.
 260          */
 261         vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT;
 262         vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem;
 263         vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 264         vq->vq_free_cnt--;
 265         i = vq->vq_split.ring.desc[head].next;
 266
 267         for (k = 0; k < pkt_num; k++) {
 268                 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT;
 269                 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 270                         + sizeof(struct virtio_net_ctrl_hdr)
 271                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 272                 vq->vq_split.ring.desc[i].len = dlen[k];
 273                 sum += dlen[k];
 274                 vq->vq_free_cnt--;
 275                 i = vq->vq_split.ring.desc[i].next;
 276         }
 277
 278         vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE;
 279         vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 280                         + sizeof(struct virtio_net_ctrl_hdr);
 281         vq->vq_split.ring.desc[i].len = sizeof(ctrl->status);
 282         vq->vq_free_cnt--;
 283
 284         vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next;
 285
 286         vq_update_avail_ring(vq, head);
 287         vq_update_avail_idx(vq);
 288
 289         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
 290
 291         virtqueue_notify(vq);
 292
 293         rte_rmb();
 294         while (VIRTQUEUE_NUSED(vq) == 0) {
 295                 rte_rmb();
 296                 usleep(100);
 297         }
 298
 299         while (VIRTQUEUE_NUSED(vq)) {
 300                 uint32_t idx, desc_idx, used_idx;
 301                 struct vring_used_elem *uep;
 302
 303                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 304                                 & (vq->vq_nentries - 1));
 305                 uep = &vq->vq_split.ring.used->ring[used_idx];
 306                 idx = (uint32_t) uep->id;
 307                 desc_idx = idx;
 308
 309                 while (vq->vq_split.ring.desc[desc_idx].flags &
 310                                 VRING_DESC_F_NEXT) {
 311                         desc_idx = vq->vq_split.ring.desc[desc_idx].next;
 312                         vq->vq_free_cnt++;
 313                 }
 314
 315                 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 316                 vq->vq_desc_head_idx = idx;
 317
 318                 vq->vq_used_cons_idx++;
 319                 vq->vq_free_cnt++;
 320         }
 321
 322         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
 323                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 324
 325         result = cvq->virtio_net_hdr_mz->addr;
 326         return result;
 327 }
 328
 329 static int
 330 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
 331                     int *dlen, int pkt_num)
 332 {
 333         virtio_net_ctrl_ack status = ~0;
 334         struct virtio_pmd_ctrl *result;
 335         struct virtqueue *vq;
 336
 337         ctrl->status = status;
 338
 339         if (!cvq || !cvq->vq) {
 340                 PMD_INIT_LOG(ERR, "Control queue is not supported.");
 341                 return -1;
 342         }
 343
 344         rte_spinlock_lock(&cvq->lock);
 345         vq = cvq->vq;
 346
 347         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 348                 "vq->hw->cvq = %p vq = %p",
 349                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 350
 351         if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) {
 352                 rte_spinlock_unlock(&cvq->lock);
 353                 return -1;
 354         }
 355
 356         memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
 357                 sizeof(struct virtio_pmd_ctrl));
 358
 359         if (vtpci_packed_queue(vq->hw))
 360                 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num);
 361         else
 362                 result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num);
 363
 364         rte_spinlock_unlock(&cvq->lock);
 365         return result->status;
 366 }
 367
 368 static int
 369 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 370 {
 371         struct virtio_hw *hw = dev->data->dev_private;
 372         struct virtio_pmd_ctrl ctrl;
 373         int dlen[1];
 374         int ret;
 375
 376         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 377         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 378         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 379
 380         dlen[0] = sizeof(uint16_t);
 381
 382         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 383         if (ret) {
 384                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 385                           "failed, this is too late now...");
 386                 return -EINVAL;
 387         }
 388
 389         return 0;
 390 }
 391
 392 static void
 393 virtio_dev_queue_release(void *queue __rte_unused)
 394 {
 395         /* do nothing */
 396 }
 397
 398 static uint16_t
 399 virtio_get_nr_vq(struct virtio_hw *hw)
 400 {
 401         uint16_t nr_vq = hw->max_queue_pairs * 2;
 402
 403         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
 404                 nr_vq += 1;
 405
 406         return nr_vq;
 407 }
 408
 409 static void
 410 virtio_init_vring(struct virtqueue *vq)
 411 {
 412         int size = vq->vq_nentries;
 413         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 414
 415         PMD_INIT_FUNC_TRACE();
 416
 417         memset(ring_mem, 0, vq->vq_ring_size);
 418
 419         vq->vq_used_cons_idx = 0;
 420         vq->vq_desc_head_idx = 0;
 421         vq->vq_avail_idx = 0;
 422         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 423         vq->vq_free_cnt = vq->vq_nentries;
 424         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 425         if (vtpci_packed_queue(vq->hw)) {
 426                 vring_init_packed(&vq->vq_packed.ring, ring_mem,
 427                                   VIRTIO_PCI_VRING_ALIGN, size);
 428                 vring_desc_init_packed(vq, size);
 429         } else {
 430                 struct vring *vr = &vq->vq_split.ring;
 431
 432                 vring_init_split(vr, ring_mem, VIRTIO_PCI_VRING_ALIGN, size);
 433                 vring_desc_init_split(vr->desc, size);
 434         }
 435         /*
 436          * Disable device(host) interrupting guest
 437          */
 438         virtqueue_disable_intr(vq);
 439 }
 440
 441 static int
 442 virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx)
 443 {
 444         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 445         char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
 446         const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
 447         unsigned int vq_size, size;
 448         struct virtio_hw *hw = dev->data->dev_private;
 449         struct virtnet_rx *rxvq = NULL;
 450         struct virtnet_tx *txvq = NULL;
 451         struct virtnet_ctl *cvq = NULL;
 452         struct virtqueue *vq;
 453         size_t sz_hdr_mz = 0;
 454         void *sw_ring = NULL;
 455         int queue_type = virtio_get_queue_type(hw, vtpci_queue_idx);
 456         int ret;
 457         int numa_node = dev->device->numa_node;
 458
 459         PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d",
 460                         vtpci_queue_idx, numa_node);
 461
 462         /*
 463          * Read the virtqueue size from the Queue Size field
 464          * Always power of 2 and if 0 virtqueue does not exist
 465          */
 466         vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
 467         PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
 468         if (vq_size == 0) {
 469                 PMD_INIT_LOG(ERR, "virtqueue does not exist");
 470                 return -EINVAL;
 471         }
 472
 473         if (!vtpci_packed_queue(hw) && !rte_is_power_of_2(vq_size)) {
 474                 PMD_INIT_LOG(ERR, "split virtqueue size is not power of 2");
 475                 return -EINVAL;
 476         }
 477
 478         snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
 479                  dev->data->port_id, vtpci_queue_idx);
 480
 481         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 482                                 vq_size * sizeof(struct vq_desc_extra),
 483                                 RTE_CACHE_LINE_SIZE);
 484         if (queue_type == VTNET_TQ) {
 485                 /*
 486                  * For each xmit packet, allocate a virtio_net_hdr
 487                  * and indirect ring elements
 488                  */
 489                 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
 490         } else if (queue_type == VTNET_CQ) {
 491                 /* Allocate a page for control vq command, data and status */
 492                 sz_hdr_mz = PAGE_SIZE;
 493         }
 494
 495         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
 496                                 numa_node);
 497         if (vq == NULL) {
 498                 PMD_INIT_LOG(ERR, "can not allocate vq");
 499                 return -ENOMEM;
 500         }
 501         hw->vqs[vtpci_queue_idx] = vq;
 502
 503         vq->hw = hw;
 504         vq->vq_queue_index = vtpci_queue_idx;
 505         vq->vq_nentries = vq_size;
 506         if (vtpci_packed_queue(hw)) {
 507                 vq->vq_packed.used_wrap_counter = 1;
 508                 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
 509                 vq->vq_packed.event_flags_shadow = 0;
 510                 if (queue_type == VTNET_RQ)
 511                         vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
 512         }
 513
 514         /*
 515          * Reserve a memzone for vring elements
 516          */
 517         size = vring_size(hw, vq_size, VIRTIO_PCI_VRING_ALIGN);
 518         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
 519         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
 520                      size, vq->vq_ring_size);
 521
 522         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 523                         numa_node, RTE_MEMZONE_IOVA_CONTIG,
 524                         VIRTIO_PCI_VRING_ALIGN);
 525         if (mz == NULL) {
 526                 if (rte_errno == EEXIST)
 527                         mz = rte_memzone_lookup(vq_name);
 528                 if (mz == NULL) {
 529                         ret = -ENOMEM;
 530                         goto fail_q_alloc;
 531                 }
 532         }
 533
 534         memset(mz->addr, 0, mz->len);
 535
 536         vq->vq_ring_mem = mz->iova;
 537         vq->vq_ring_virt_mem = mz->addr;
 538         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem:      0x%" PRIx64,
 539                      (uint64_t)mz->iova);
 540         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%" PRIx64,
 541                      (uint64_t)(uintptr_t)mz->addr);
 542
 543         virtio_init_vring(vq);
 544
 545         if (sz_hdr_mz) {
 546                 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 547                          dev->data->port_id, vtpci_queue_idx);
 548                 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
 549                                 numa_node, RTE_MEMZONE_IOVA_CONTIG,
 550                                 RTE_CACHE_LINE_SIZE);
 551                 if (hdr_mz == NULL) {
 552                         if (rte_errno == EEXIST)
 553                                 hdr_mz = rte_memzone_lookup(vq_hdr_name);
 554                         if (hdr_mz == NULL) {
 555                                 ret = -ENOMEM;
 556                                 goto fail_q_alloc;
 557                         }
 558                 }
 559         }
 560
 561         if (queue_type == VTNET_RQ) {
 562                 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
 563                                sizeof(vq->sw_ring[0]);
 564
 565                 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
 566                                 RTE_CACHE_LINE_SIZE, numa_node);
 567                 if (!sw_ring) {
 568                         PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
 569                         ret = -ENOMEM;
 570                         goto fail_q_alloc;
 571                 }
 572
 573                 vq->sw_ring = sw_ring;
 574                 rxvq = &vq->rxq;
 575                 rxvq->vq = vq;
 576                 rxvq->port_id = dev->data->port_id;
 577                 rxvq->mz = mz;
 578         } else if (queue_type == VTNET_TQ) {
 579                 txvq = &vq->txq;
 580                 txvq->vq = vq;
 581                 txvq->port_id = dev->data->port_id;
 582                 txvq->mz = mz;
 583                 txvq->virtio_net_hdr_mz = hdr_mz;
 584                 txvq->virtio_net_hdr_mem = hdr_mz->iova;
 585         } else if (queue_type == VTNET_CQ) {
 586                 cvq = &vq->cq;
 587                 cvq->vq = vq;
 588                 cvq->mz = mz;
 589                 cvq->virtio_net_hdr_mz = hdr_mz;
 590                 cvq->virtio_net_hdr_mem = hdr_mz->iova;
 591                 memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE);
 592
 593                 hw->cvq = cvq;
 594         }
 595
 596         /* For virtio_user case (that is when hw->virtio_user_dev is not NULL),
 597          * we use virtual address. And we need properly set _offset_, please see
 598          * VIRTIO_MBUF_DATA_DMA_ADDR in virtqueue.h for more information.
 599          */
 600         if (!hw->virtio_user_dev)
 601                 vq->offset = offsetof(struct rte_mbuf, buf_iova);
 602         else {
 603                 vq->vq_ring_mem = (uintptr_t)mz->addr;
 604                 vq->offset = offsetof(struct rte_mbuf, buf_addr);
 605                 if (queue_type == VTNET_TQ)
 606                         txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 607                 else if (queue_type == VTNET_CQ)
 608                         cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 609         }
 610
 611         if (queue_type == VTNET_TQ) {
 612                 struct virtio_tx_region *txr;
 613                 unsigned int i;
 614
 615                 txr = hdr_mz->addr;
 616                 memset(txr, 0, vq_size * sizeof(*txr));
 617                 for (i = 0; i < vq_size; i++) {
 618                         struct vring_desc *start_dp = txr[i].tx_indir;
 619
 620                         /* first indirect descriptor is always the tx header */
 621                         if (!vtpci_packed_queue(hw)) {
 622                                 vring_desc_init_split(start_dp,
 623                                                       RTE_DIM(txr[i].tx_indir));
 624                                 start_dp->addr = txvq->virtio_net_hdr_mem
 625                                         + i * sizeof(*txr)
 626                                         + offsetof(struct virtio_tx_region,
 627                                                    tx_hdr);
 628                                 start_dp->len = hw->vtnet_hdr_size;
 629                                 start_dp->flags = VRING_DESC_F_NEXT;
 630                         }
 631                 }
 632         }
 633
 634         if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) {
 635                 PMD_INIT_LOG(ERR, "setup_queue failed");
 636                 return -EINVAL;
 637         }
 638
 639         return 0;
 640
 641 fail_q_alloc:
 642         rte_free(sw_ring);
 643         rte_memzone_free(hdr_mz);
 644         rte_memzone_free(mz);
 645         rte_free(vq);
 646
 647         return ret;
 648 }
 649
 650 static void
 651 virtio_free_queues(struct virtio_hw *hw)
 652 {
 653         uint16_t nr_vq = virtio_get_nr_vq(hw);
 654         struct virtqueue *vq;
 655         int queue_type;
 656         uint16_t i;
 657
 658         if (hw->vqs == NULL)
 659                 return;
 660
 661         for (i = 0; i < nr_vq; i++) {
 662                 vq = hw->vqs[i];
 663                 if (!vq)
 664                         continue;
 665
 666                 queue_type = virtio_get_queue_type(hw, i);
 667                 if (queue_type == VTNET_RQ) {
 668                         rte_free(vq->sw_ring);
 669                         rte_memzone_free(vq->rxq.mz);
 670                 } else if (queue_type == VTNET_TQ) {
 671                         rte_memzone_free(vq->txq.mz);
 672                         rte_memzone_free(vq->txq.virtio_net_hdr_mz);
 673                 } else {
 674                         rte_memzone_free(vq->cq.mz);
 675                         rte_memzone_free(vq->cq.virtio_net_hdr_mz);
 676                 }
 677
 678                 rte_free(vq);
 679                 hw->vqs[i] = NULL;
 680         }
 681
 682         rte_free(hw->vqs);
 683         hw->vqs = NULL;
 684 }
 685
 686 static int
 687 virtio_alloc_queues(struct rte_eth_dev *dev)
 688 {
 689         struct virtio_hw *hw = dev->data->dev_private;
 690         uint16_t nr_vq = virtio_get_nr_vq(hw);
 691         uint16_t i;
 692         int ret;
 693
 694         hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0);
 695         if (!hw->vqs) {
 696                 PMD_INIT_LOG(ERR, "failed to allocate vqs");
 697                 return -ENOMEM;
 698         }
 699
 700         for (i = 0; i < nr_vq; i++) {
 701                 ret = virtio_init_queue(dev, i);
 702                 if (ret < 0) {
 703                         virtio_free_queues(hw);
 704                         return ret;
 705                 }
 706         }
 707
 708         return 0;
 709 }
 710
 711 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev);
 712
 713 static void
 714 virtio_dev_close(struct rte_eth_dev *dev)
 715 {
 716         struct virtio_hw *hw = dev->data->dev_private;
 717         struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
 718
 719         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 720
 721         if (!hw->opened)
 722                 return;
 723         hw->opened = false;
 724
 725         /* reset the NIC */
 726         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 727                 VTPCI_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR);
 728         if (intr_conf->rxq)
 729                 virtio_queues_unbind_intr(dev);
 730
 731         if (intr_conf->lsc || intr_conf->rxq) {
 732                 virtio_intr_disable(dev);
 733                 rte_intr_efd_disable(dev->intr_handle);
 734                 rte_free(dev->intr_handle->intr_vec);
 735                 dev->intr_handle->intr_vec = NULL;
 736         }
 737
 738         vtpci_reset(hw);
 739         virtio_dev_free_mbufs(dev);
 740         virtio_free_queues(hw);
 741
 742 #ifdef RTE_VIRTIO_USER
 743         if (hw->virtio_user_dev)
 744                 virtio_user_dev_uninit(hw->virtio_user_dev);
 745         else
 746 #endif
 747         if (dev->device) {
 748                 rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(dev));
 749                 if (!hw->modern)
 750                         rte_pci_ioport_unmap(VTPCI_IO(hw));
 751         }
 752 }
 753
 754 static int
 755 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
 756 {
 757         struct virtio_hw *hw = dev->data->dev_private;
 758         struct virtio_pmd_ctrl ctrl;
 759         int dlen[1];
 760         int ret;
 761
 762         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 763                 PMD_INIT_LOG(INFO, "host does not support rx control");
 764                 return -ENOTSUP;
 765         }
 766
 767         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 768         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 769         ctrl.data[0] = 1;
 770         dlen[0] = 1;
 771
 772         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 773         if (ret) {
 774                 PMD_INIT_LOG(ERR, "Failed to enable promisc");
 775                 return -EAGAIN;
 776         }
 777
 778         return 0;
 779 }
 780
 781 static int
 782 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
 783 {
 784         struct virtio_hw *hw = dev->data->dev_private;
 785         struct virtio_pmd_ctrl ctrl;
 786         int dlen[1];
 787         int ret;
 788
 789         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 790                 PMD_INIT_LOG(INFO, "host does not support rx control");
 791                 return -ENOTSUP;
 792         }
 793
 794         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 795         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 796         ctrl.data[0] = 0;
 797         dlen[0] = 1;
 798
 799         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 800         if (ret) {
 801                 PMD_INIT_LOG(ERR, "Failed to disable promisc");
 802                 return -EAGAIN;
 803         }
 804
 805         return 0;
 806 }
 807
 808 static int
 809 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
 810 {
 811         struct virtio_hw *hw = dev->data->dev_private;
 812         struct virtio_pmd_ctrl ctrl;
 813         int dlen[1];
 814         int ret;
 815
 816         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 817                 PMD_INIT_LOG(INFO, "host does not support rx control");
 818                 return -ENOTSUP;
 819         }
 820
 821         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 822         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 823         ctrl.data[0] = 1;
 824         dlen[0] = 1;
 825
 826         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 827         if (ret) {
 828                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
 829                 return -EAGAIN;
 830         }
 831
 832         return 0;
 833 }
 834
 835 static int
 836 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
 837 {
 838         struct virtio_hw *hw = dev->data->dev_private;
 839         struct virtio_pmd_ctrl ctrl;
 840         int dlen[1];
 841         int ret;
 842
 843         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 844                 PMD_INIT_LOG(INFO, "host does not support rx control");
 845                 return -ENOTSUP;
 846         }
 847
 848         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 849         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 850         ctrl.data[0] = 0;
 851         dlen[0] = 1;
 852
 853         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 854         if (ret) {
 855                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
 856                 return -EAGAIN;
 857         }
 858
 859         return 0;
 860 }
 861
 862 #define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
 863 static int
 864 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 865 {
 866         struct virtio_hw *hw = dev->data->dev_private;
 867         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
 868                                  hw->vtnet_hdr_size;
 869         uint32_t frame_size = mtu + ether_hdr_len;
 870         uint32_t max_frame_size = hw->max_mtu + ether_hdr_len;
 871
 872         max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN);
 873
 874         if (mtu < RTE_ETHER_MIN_MTU || frame_size > max_frame_size) {
 875                 PMD_INIT_LOG(ERR, "MTU should be between %d and %d",
 876                         RTE_ETHER_MIN_MTU, max_frame_size - ether_hdr_len);
 877                 return -EINVAL;
 878         }
 879         return 0;
 880 }
 881
 882 static int
 883 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 884 {
 885         struct virtio_hw *hw = dev->data->dev_private;
 886         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 887         struct virtqueue *vq = rxvq->vq;
 888
 889         virtqueue_enable_intr(vq);
 890         virtio_mb(hw->weak_barriers);
 891         return 0;
 892 }
 893
 894 static int
 895 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 896 {
 897         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 898         struct virtqueue *vq = rxvq->vq;
 899
 900         virtqueue_disable_intr(vq);
 901         return 0;
 902 }
 903
 904 /*
 905  * dev_ops for virtio, bare necessities for basic operation
 906  */
 907 static const struct eth_dev_ops virtio_eth_dev_ops = {
 908         .dev_configure           = virtio_dev_configure,
 909         .dev_start               = virtio_dev_start,
 910         .dev_stop                = virtio_dev_stop,
 911         .dev_close               = virtio_dev_close,
 912         .promiscuous_enable      = virtio_dev_promiscuous_enable,
 913         .promiscuous_disable     = virtio_dev_promiscuous_disable,
 914         .allmulticast_enable     = virtio_dev_allmulticast_enable,
 915         .allmulticast_disable    = virtio_dev_allmulticast_disable,
 916         .mtu_set                 = virtio_mtu_set,
 917         .dev_infos_get           = virtio_dev_info_get,
 918         .stats_get               = virtio_dev_stats_get,
 919         .xstats_get              = virtio_dev_xstats_get,
 920         .xstats_get_names        = virtio_dev_xstats_get_names,
 921         .stats_reset             = virtio_dev_stats_reset,
 922         .xstats_reset            = virtio_dev_stats_reset,
 923         .link_update             = virtio_dev_link_update,
 924         .vlan_offload_set        = virtio_dev_vlan_offload_set,
 925         .rx_queue_setup          = virtio_dev_rx_queue_setup,
 926         .rx_queue_intr_enable    = virtio_dev_rx_queue_intr_enable,
 927         .rx_queue_intr_disable   = virtio_dev_rx_queue_intr_disable,
 928         .rx_queue_release        = virtio_dev_queue_release,
 929         .rx_descriptor_done      = virtio_dev_rx_queue_done,
 930         .tx_queue_setup          = virtio_dev_tx_queue_setup,
 931         .tx_queue_release        = virtio_dev_queue_release,
 932         /* collect stats per queue */
 933         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 934         .vlan_filter_set         = virtio_vlan_filter_set,
 935         .mac_addr_add            = virtio_mac_addr_add,
 936         .mac_addr_remove         = virtio_mac_addr_remove,
 937         .mac_addr_set            = virtio_mac_addr_set,
 938 };
 939
 940 /*
 941  * dev_ops for virtio-user in secondary processes, as we just have
 942  * some limited supports currently.
 943  */
 944 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops = {
 945         .dev_infos_get           = virtio_dev_info_get,
 946         .stats_get               = virtio_dev_stats_get,
 947         .xstats_get              = virtio_dev_xstats_get,
 948         .xstats_get_names        = virtio_dev_xstats_get_names,
 949         .stats_reset             = virtio_dev_stats_reset,
 950         .xstats_reset            = virtio_dev_stats_reset,
 951         /* collect stats per queue */
 952         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 953 };
 954
 955 static void
 956 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 957 {
 958         unsigned i;
 959
 960         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 961                 const struct virtnet_tx *txvq = dev->data->tx_queues[i];
 962                 if (txvq == NULL)
 963                         continue;
 964
 965                 stats->opackets += txvq->stats.packets;
 966                 stats->obytes += txvq->stats.bytes;
 967
 968                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 969                         stats->q_opackets[i] = txvq->stats.packets;
 970                         stats->q_obytes[i] = txvq->stats.bytes;
 971                 }
 972         }
 973
 974         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 975                 const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
 976                 if (rxvq == NULL)
 977                         continue;
 978
 979                 stats->ipackets += rxvq->stats.packets;
 980                 stats->ibytes += rxvq->stats.bytes;
 981                 stats->ierrors += rxvq->stats.errors;
 982
 983                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 984                         stats->q_ipackets[i] = rxvq->stats.packets;
 985                         stats->q_ibytes[i] = rxvq->stats.bytes;
 986                 }
 987         }
 988
 989         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
 990 }
 991
 992 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
 993                                        struct rte_eth_xstat_name *xstats_names,
 994                                        __rte_unused unsigned limit)
 995 {
 996         unsigned i;
 997         unsigned count = 0;
 998         unsigned t;
 999
1000         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1001                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1002
1003         if (xstats_names != NULL) {
1004                 /* Note: limit checked in rte_eth_xstats_names() */
1005
1006                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1007                         struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1008                         if (rxvq == NULL)
1009                                 continue;
1010                         for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1011                                 snprintf(xstats_names[count].name,
1012                                         sizeof(xstats_names[count].name),
1013                                         "rx_q%u_%s", i,
1014                                         rte_virtio_rxq_stat_strings[t].name);
1015                                 count++;
1016                         }
1017                 }
1018
1019                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1020                         struct virtnet_tx *txvq = dev->data->tx_queues[i];
1021                         if (txvq == NULL)
1022                                 continue;
1023                         for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1024                                 snprintf(xstats_names[count].name,
1025                                         sizeof(xstats_names[count].name),
1026                                         "tx_q%u_%s", i,
1027                                         rte_virtio_txq_stat_strings[t].name);
1028                                 count++;
1029                         }
1030                 }
1031                 return count;
1032         }
1033         return nstats;
1034 }
1035
1036 static int
1037 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1038                       unsigned n)
1039 {
1040         unsigned i;
1041         unsigned count = 0;
1042
1043         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1044                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1045
1046         if (n < nstats)
1047                 return nstats;
1048
1049         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1050                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1051
1052                 if (rxvq == NULL)
1053                         continue;
1054
1055                 unsigned t;
1056
1057                 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1058                         xstats[count].value = *(uint64_t *)(((char *)rxvq) +
1059                                 rte_virtio_rxq_stat_strings[t].offset);
1060                         xstats[count].id = count;
1061                         count++;
1062                 }
1063         }
1064
1065         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1066                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1067
1068                 if (txvq == NULL)
1069                         continue;
1070
1071                 unsigned t;
1072
1073                 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1074                         xstats[count].value = *(uint64_t *)(((char *)txvq) +
1075                                 rte_virtio_txq_stat_strings[t].offset);
1076                         xstats[count].id = count;
1077                         count++;
1078                 }
1079         }
1080
1081         return count;
1082 }
1083
1084 static int
1085 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1086 {
1087         virtio_update_stats(dev, stats);
1088
1089         return 0;
1090 }
1091
1092 static int
1093 virtio_dev_stats_reset(struct rte_eth_dev *dev)
1094 {
1095         unsigned int i;
1096
1097         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1098                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1099                 if (txvq == NULL)
1100                         continue;
1101
1102                 txvq->stats.packets = 0;
1103                 txvq->stats.bytes = 0;
1104                 txvq->stats.multicast = 0;
1105                 txvq->stats.broadcast = 0;
1106                 memset(txvq->stats.size_bins, 0,
1107                        sizeof(txvq->stats.size_bins[0]) * 8);
1108         }
1109
1110         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1111                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1112                 if (rxvq == NULL)
1113                         continue;
1114
1115                 rxvq->stats.packets = 0;
1116                 rxvq->stats.bytes = 0;
1117                 rxvq->stats.errors = 0;
1118                 rxvq->stats.multicast = 0;
1119                 rxvq->stats.broadcast = 0;
1120                 memset(rxvq->stats.size_bins, 0,
1121                        sizeof(rxvq->stats.size_bins[0]) * 8);
1122         }
1123
1124         return 0;
1125 }
1126
1127 static void
1128 virtio_set_hwaddr(struct virtio_hw *hw)
1129 {
1130         vtpci_write_dev_config(hw,
1131                         offsetof(struct virtio_net_config, mac),
1132                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1133 }
1134
1135 static void
1136 virtio_get_hwaddr(struct virtio_hw *hw)
1137 {
1138         if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) {
1139                 vtpci_read_dev_config(hw,
1140                         offsetof(struct virtio_net_config, mac),
1141                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1142         } else {
1143                 rte_eth_random_addr(&hw->mac_addr[0]);
1144                 virtio_set_hwaddr(hw);
1145         }
1146 }
1147
1148 static int
1149 virtio_mac_table_set(struct virtio_hw *hw,
1150                      const struct virtio_net_ctrl_mac *uc,
1151                      const struct virtio_net_ctrl_mac *mc)
1152 {
1153         struct virtio_pmd_ctrl ctrl;
1154         int err, len[2];
1155
1156         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1157                 PMD_DRV_LOG(INFO, "host does not support mac table");
1158                 return -1;
1159         }
1160
1161         ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1162         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1163
1164         len[0] = uc->entries * RTE_ETHER_ADDR_LEN + sizeof(uc->entries);
1165         memcpy(ctrl.data, uc, len[0]);
1166
1167         len[1] = mc->entries * RTE_ETHER_ADDR_LEN + sizeof(mc->entries);
1168         memcpy(ctrl.data + len[0], mc, len[1]);
1169
1170         err = virtio_send_command(hw->cvq, &ctrl, len, 2);
1171         if (err != 0)
1172                 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
1173         return err;
1174 }
1175
1176 static int
1177 virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1178                     uint32_t index, uint32_t vmdq __rte_unused)
1179 {
1180         struct virtio_hw *hw = dev->data->dev_private;
1181         const struct rte_ether_addr *addrs = dev->data->mac_addrs;
1182         unsigned int i;
1183         struct virtio_net_ctrl_mac *uc, *mc;
1184
1185         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1186                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1187                 return -EINVAL;
1188         }
1189
1190         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1191                 sizeof(uc->entries));
1192         uc->entries = 0;
1193         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1194                 sizeof(mc->entries));
1195         mc->entries = 0;
1196
1197         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1198                 const struct rte_ether_addr *addr
1199                         = (i == index) ? mac_addr : addrs + i;
1200                 struct virtio_net_ctrl_mac *tbl
1201                         = rte_is_multicast_ether_addr(addr) ? mc : uc;
1202
1203                 memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN);
1204         }
1205
1206         return virtio_mac_table_set(hw, uc, mc);
1207 }
1208
1209 static void
1210 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1211 {
1212         struct virtio_hw *hw = dev->data->dev_private;
1213         struct rte_ether_addr *addrs = dev->data->mac_addrs;
1214         struct virtio_net_ctrl_mac *uc, *mc;
1215         unsigned int i;
1216
1217         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1218                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1219                 return;
1220         }
1221
1222         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1223                 sizeof(uc->entries));
1224         uc->entries = 0;
1225         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1226                 sizeof(mc->entries));
1227         mc->entries = 0;
1228
1229         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1230                 struct virtio_net_ctrl_mac *tbl;
1231
1232                 if (i == index || rte_is_zero_ether_addr(addrs + i))
1233                         continue;
1234
1235                 tbl = rte_is_multicast_ether_addr(addrs + i) ? mc : uc;
1236                 memcpy(&tbl->macs[tbl->entries++], addrs + i,
1237                         RTE_ETHER_ADDR_LEN);
1238         }
1239
1240         virtio_mac_table_set(hw, uc, mc);
1241 }
1242
1243 static int
1244 virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1245 {
1246         struct virtio_hw *hw = dev->data->dev_private;
1247
1248         memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1249
1250         /* Use atomic update if available */
1251         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1252                 struct virtio_pmd_ctrl ctrl;
1253                 int len = RTE_ETHER_ADDR_LEN;
1254
1255                 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1256                 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1257
1258                 memcpy(ctrl.data, mac_addr, RTE_ETHER_ADDR_LEN);
1259                 return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1260         }
1261
1262         if (!vtpci_with_feature(hw, VIRTIO_NET_F_MAC))
1263                 return -ENOTSUP;
1264
1265         virtio_set_hwaddr(hw);
1266         return 0;
1267 }
1268
1269 static int
1270 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1271 {
1272         struct virtio_hw *hw = dev->data->dev_private;
1273         struct virtio_pmd_ctrl ctrl;
1274         int len;
1275
1276         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1277                 return -ENOTSUP;
1278
1279         ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1280         ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1281         memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1282         len = sizeof(vlan_id);
1283
1284         return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1285 }
1286
1287 static int
1288 virtio_intr_unmask(struct rte_eth_dev *dev)
1289 {
1290         struct virtio_hw *hw = dev->data->dev_private;
1291
1292         if (rte_intr_ack(dev->intr_handle) < 0)
1293                 return -1;
1294
1295         if (!hw->virtio_user_dev)
1296                 hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev));
1297
1298         return 0;
1299 }
1300
1301 static int
1302 virtio_intr_enable(struct rte_eth_dev *dev)
1303 {
1304         struct virtio_hw *hw = dev->data->dev_private;
1305
1306         if (rte_intr_enable(dev->intr_handle) < 0)
1307                 return -1;
1308
1309         if (!hw->virtio_user_dev)
1310                 hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev));
1311
1312         return 0;
1313 }
1314
1315 static int
1316 virtio_intr_disable(struct rte_eth_dev *dev)
1317 {
1318         struct virtio_hw *hw = dev->data->dev_private;
1319
1320         if (rte_intr_disable(dev->intr_handle) < 0)
1321                 return -1;
1322
1323         if (!hw->virtio_user_dev)
1324                 hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev));
1325
1326         return 0;
1327 }
1328
1329 static int
1330 virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1331 {
1332         uint64_t host_features;
1333
1334         /* Prepare guest_features: feature that driver wants to support */
1335         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1336                 req_features);
1337
1338         /* Read device(host) feature bits */
1339         host_features = VTPCI_OPS(hw)->get_features(hw);
1340         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1341                 host_features);
1342
1343         /* If supported, ensure MTU value is valid before acknowledging it. */
1344         if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) {
1345                 struct virtio_net_config config;
1346
1347                 vtpci_read_dev_config(hw,
1348                         offsetof(struct virtio_net_config, mtu),
1349                         &config.mtu, sizeof(config.mtu));
1350
1351                 if (config.mtu < RTE_ETHER_MIN_MTU)
1352                         req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
1353         }
1354
1355         /*
1356          * Negotiate features: Subset of device feature bits are written back
1357          * guest feature bits.
1358          */
1359         hw->guest_features = req_features;
1360         hw->guest_features = vtpci_negotiate_features(hw, host_features);
1361         PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1362                 hw->guest_features);
1363
1364         if (hw->modern) {
1365                 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
1366                         PMD_INIT_LOG(ERR,
1367                                 "VIRTIO_F_VERSION_1 features is not enabled.");
1368                         return -1;
1369                 }
1370                 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1371                 if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1372                         PMD_INIT_LOG(ERR,
1373                                 "failed to set FEATURES_OK status!");
1374                         return -1;
1375                 }
1376         }
1377
1378         hw->req_guest_features = req_features;
1379
1380         return 0;
1381 }
1382
1383 int
1384 virtio_dev_pause(struct rte_eth_dev *dev)
1385 {
1386         struct virtio_hw *hw = dev->data->dev_private;
1387
1388         rte_spinlock_lock(&hw->state_lock);
1389
1390         if (hw->started == 0) {
1391                 /* Device is just stopped. */
1392                 rte_spinlock_unlock(&hw->state_lock);
1393                 return -1;
1394         }
1395         hw->started = 0;
1396         /*
1397          * Prevent the worker threads from touching queues to avoid contention,
1398          * 1 ms should be enough for the ongoing Tx function to finish.
1399          */
1400         rte_delay_ms(1);
1401         return 0;
1402 }
1403
1404 /*
1405  * Recover hw state to let the worker threads continue.
1406  */
1407 void
1408 virtio_dev_resume(struct rte_eth_dev *dev)
1409 {
1410         struct virtio_hw *hw = dev->data->dev_private;
1411
1412         hw->started = 1;
1413         rte_spinlock_unlock(&hw->state_lock);
1414 }
1415
1416 /*
1417  * Should be called only after device is paused.
1418  */
1419 int
1420 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts,
1421                 int nb_pkts)
1422 {
1423         struct virtio_hw *hw = dev->data->dev_private;
1424         struct virtnet_tx *txvq = dev->data->tx_queues[0];
1425         int ret;
1426
1427         hw->inject_pkts = tx_pkts;
1428         ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts);
1429         hw->inject_pkts = NULL;
1430
1431         return ret;
1432 }
1433
1434 static void
1435 virtio_notify_peers(struct rte_eth_dev *dev)
1436 {
1437         struct virtio_hw *hw = dev->data->dev_private;
1438         struct virtnet_rx *rxvq;
1439         struct rte_mbuf *rarp_mbuf;
1440
1441         if (!dev->data->rx_queues)
1442                 return;
1443
1444         rxvq = dev->data->rx_queues[0];
1445         if (!rxvq)
1446                 return;
1447
1448         rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool,
1449                         (struct rte_ether_addr *)hw->mac_addr);
1450         if (rarp_mbuf == NULL) {
1451                 PMD_DRV_LOG(ERR, "failed to make RARP packet.");
1452                 return;
1453         }
1454
1455         /* If virtio port just stopped, no need to send RARP */
1456         if (virtio_dev_pause(dev) < 0) {
1457                 rte_pktmbuf_free(rarp_mbuf);
1458                 return;
1459         }
1460
1461         virtio_inject_pkts(dev, &rarp_mbuf, 1);
1462         virtio_dev_resume(dev);
1463 }
1464
1465 static void
1466 virtio_ack_link_announce(struct rte_eth_dev *dev)
1467 {
1468         struct virtio_hw *hw = dev->data->dev_private;
1469         struct virtio_pmd_ctrl ctrl;
1470
1471         ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE;
1472         ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK;
1473
1474         virtio_send_command(hw->cvq, &ctrl, NULL, 0);
1475 }
1476
1477 /*
1478  * Process virtio config changed interrupt. Call the callback
1479  * if link state changed, generate gratuitous RARP packet if
1480  * the status indicates an ANNOUNCE.
1481  */
1482 void
1483 virtio_interrupt_handler(void *param)
1484 {
1485         struct rte_eth_dev *dev = param;
1486         struct virtio_hw *hw = dev->data->dev_private;
1487         uint8_t isr;
1488         uint16_t status;
1489
1490         /* Read interrupt status which clears interrupt */
1491         isr = vtpci_isr(hw);
1492         PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1493
1494         if (virtio_intr_unmask(dev) < 0)
1495                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1496
1497         if (isr & VIRTIO_PCI_ISR_CONFIG) {
1498                 if (virtio_dev_link_update(dev, 0) == 0)
1499                         _rte_eth_dev_callback_process(dev,
1500                                                       RTE_ETH_EVENT_INTR_LSC,
1501                                                       NULL);
1502
1503                 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1504                         vtpci_read_dev_config(hw,
1505                                 offsetof(struct virtio_net_config, status),
1506                                 &status, sizeof(status));
1507                         if (status & VIRTIO_NET_S_ANNOUNCE) {
1508                                 virtio_notify_peers(dev);
1509                                 if (hw->cvq)
1510                                         virtio_ack_link_announce(dev);
1511                         }
1512                 }
1513         }
1514 }
1515
1516 /* set rx and tx handlers according to what is supported */
1517 static void
1518 set_rxtx_funcs(struct rte_eth_dev *eth_dev)
1519 {
1520         struct virtio_hw *hw = eth_dev->data->dev_private;
1521
1522         eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare;
1523         if (vtpci_packed_queue(hw)) {
1524                 PMD_INIT_LOG(INFO,
1525                         "virtio: using packed ring %s Tx path on port %u",
1526                         hw->use_inorder_tx ? "inorder" : "standard",
1527                         eth_dev->data->port_id);
1528                 eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
1529         } else {
1530                 if (hw->use_inorder_tx) {
1531                         PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u",
1532                                 eth_dev->data->port_id);
1533                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder;
1534                 } else {
1535                         PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u",
1536                                 eth_dev->data->port_id);
1537                         eth_dev->tx_pkt_burst = virtio_xmit_pkts;
1538                 }
1539         }
1540
1541         if (vtpci_packed_queue(hw)) {
1542                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1543                         PMD_INIT_LOG(INFO,
1544                                 "virtio: using packed ring mergeable buffer Rx path on port %u",
1545                                 eth_dev->data->port_id);
1546                         eth_dev->rx_pkt_burst =
1547                                 &virtio_recv_mergeable_pkts_packed;
1548                 } else {
1549                         PMD_INIT_LOG(INFO,
1550                                 "virtio: using packed ring standard Rx path on port %u",
1551                                 eth_dev->data->port_id);
1552                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
1553                 }
1554         } else {
1555                 if (hw->use_vec_rx) {
1556                         PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on port %u",
1557                                 eth_dev->data->port_id);
1558                         eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
1559                 } else if (hw->use_inorder_rx) {
1560                         PMD_INIT_LOG(INFO,
1561                                 "virtio: using inorder Rx path on port %u",
1562                                 eth_dev->data->port_id);
1563                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder;
1564                 } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1565                         PMD_INIT_LOG(INFO,
1566                                 "virtio: using mergeable buffer Rx path on port %u",
1567                                 eth_dev->data->port_id);
1568                         eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1569                 } else {
1570                         PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
1571                                 eth_dev->data->port_id);
1572                         eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1573                 }
1574         }
1575
1576 }
1577
1578 /* Only support 1:1 queue/interrupt mapping so far.
1579  * TODO: support n:1 queue/interrupt mapping when there are limited number of
1580  * interrupt vectors (<N+1).
1581  */
1582 static int
1583 virtio_queues_bind_intr(struct rte_eth_dev *dev)
1584 {
1585         uint32_t i;
1586         struct virtio_hw *hw = dev->data->dev_private;
1587
1588         PMD_INIT_LOG(INFO, "queue/interrupt binding");
1589         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
1590                 dev->intr_handle->intr_vec[i] = i + 1;
1591                 if (VTPCI_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) ==
1592                                                  VIRTIO_MSI_NO_VECTOR) {
1593                         PMD_DRV_LOG(ERR, "failed to set queue vector");
1594                         return -EBUSY;
1595                 }
1596         }
1597
1598         return 0;
1599 }
1600
1601 static void
1602 virtio_queues_unbind_intr(struct rte_eth_dev *dev)
1603 {
1604         uint32_t i;
1605         struct virtio_hw *hw = dev->data->dev_private;
1606
1607         PMD_INIT_LOG(INFO, "queue/interrupt unbinding");
1608         for (i = 0; i < dev->data->nb_rx_queues; ++i)
1609                 VTPCI_OPS(hw)->set_queue_irq(hw,
1610                                              hw->vqs[i * VTNET_CQ],
1611                                              VIRTIO_MSI_NO_VECTOR);
1612 }
1613
1614 static int
1615 virtio_configure_intr(struct rte_eth_dev *dev)
1616 {
1617         struct virtio_hw *hw = dev->data->dev_private;
1618
1619         if (!rte_intr_cap_multiple(dev->intr_handle)) {
1620                 PMD_INIT_LOG(ERR, "Multiple intr vector not supported");
1621                 return -ENOTSUP;
1622         }
1623
1624         if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) {
1625                 PMD_INIT_LOG(ERR, "Fail to create eventfd");
1626                 return -1;
1627         }
1628
1629         if (!dev->intr_handle->intr_vec) {
1630                 dev->intr_handle->intr_vec =
1631                         rte_zmalloc("intr_vec",
1632                                     hw->max_queue_pairs * sizeof(int), 0);
1633                 if (!dev->intr_handle->intr_vec) {
1634                         PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors",
1635                                      hw->max_queue_pairs);
1636                         return -ENOMEM;
1637                 }
1638         }
1639
1640         /* Re-register callback to update max_intr */
1641         rte_intr_callback_unregister(dev->intr_handle,
1642                                      virtio_interrupt_handler,
1643                                      dev);
1644         rte_intr_callback_register(dev->intr_handle,
1645                                    virtio_interrupt_handler,
1646                                    dev);
1647
1648         /* DO NOT try to remove this! This function will enable msix, or QEMU
1649          * will encounter SIGSEGV when DRIVER_OK is sent.
1650          * And for legacy devices, this should be done before queue/vec binding
1651          * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1652          * (22) will be ignored.
1653          */
1654         if (virtio_intr_enable(dev) < 0) {
1655                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1656                 return -1;
1657         }
1658
1659         if (virtio_queues_bind_intr(dev) < 0) {
1660                 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt");
1661                 return -1;
1662         }
1663
1664         return 0;
1665 }
1666 #define SPEED_UNKNOWN    0xffffffff
1667 #define DUPLEX_UNKNOWN   0xff
1668 /* reset device and renegotiate features if needed */
1669 static int
1670 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
1671 {
1672         struct virtio_hw *hw = eth_dev->data->dev_private;
1673         struct virtio_net_config *config;
1674         struct virtio_net_config local_config;
1675         struct rte_pci_device *pci_dev = NULL;
1676         int ret;
1677
1678         /* Reset the device although not necessary at startup */
1679         vtpci_reset(hw);
1680
1681         if (hw->vqs) {
1682                 virtio_dev_free_mbufs(eth_dev);
1683                 virtio_free_queues(hw);
1684         }
1685
1686         /* Tell the host we've noticed this device. */
1687         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1688
1689         /* Tell the host we've known how to drive the device. */
1690         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1691         if (virtio_negotiate_features(hw, req_features) < 0)
1692                 return -1;
1693
1694         hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
1695
1696         if (!hw->virtio_user_dev)
1697                 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1698
1699         /* If host does not support both status and MSI-X then disable LSC */
1700         if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS) &&
1701             hw->use_msix != VIRTIO_MSIX_NONE)
1702                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
1703         else
1704                 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
1705
1706         /* Setting up rx_header size for the device */
1707         if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
1708             vtpci_with_feature(hw, VIRTIO_F_VERSION_1) ||
1709             vtpci_with_feature(hw, VIRTIO_F_RING_PACKED))
1710                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1711         else
1712                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
1713
1714         /* Copy the permanent MAC address to: virtio_hw */
1715         virtio_get_hwaddr(hw);
1716         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
1717                         &eth_dev->data->mac_addrs[0]);
1718         PMD_INIT_LOG(DEBUG,
1719                      "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1720                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1721                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1722
1723         if (hw->speed == SPEED_UNKNOWN) {
1724                 if (vtpci_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) {
1725                         config = &local_config;
1726                         vtpci_read_dev_config(hw,
1727                                 offsetof(struct virtio_net_config, speed),
1728                                 &config->speed, sizeof(config->speed));
1729                         vtpci_read_dev_config(hw,
1730                                 offsetof(struct virtio_net_config, duplex),
1731                                 &config->duplex, sizeof(config->duplex));
1732                         hw->speed = config->speed;
1733                         hw->duplex = config->duplex;
1734                 }
1735         }
1736         if (hw->speed == SPEED_UNKNOWN)
1737                 hw->speed = ETH_SPEED_NUM_10G;
1738         if (hw->duplex == DUPLEX_UNKNOWN)
1739                 hw->duplex = ETH_LINK_FULL_DUPLEX;
1740         PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d",
1741                 hw->speed, hw->duplex);
1742         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1743                 config = &local_config;
1744
1745                 vtpci_read_dev_config(hw,
1746                         offsetof(struct virtio_net_config, mac),
1747                         &config->mac, sizeof(config->mac));
1748
1749                 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1750                         vtpci_read_dev_config(hw,
1751                                 offsetof(struct virtio_net_config, status),
1752                                 &config->status, sizeof(config->status));
1753                 } else {
1754                         PMD_INIT_LOG(DEBUG,
1755                                      "VIRTIO_NET_F_STATUS is not supported");
1756                         config->status = 0;
1757                 }
1758
1759                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) {
1760                         vtpci_read_dev_config(hw,
1761                                 offsetof(struct virtio_net_config, max_virtqueue_pairs),
1762                                 &config->max_virtqueue_pairs,
1763                                 sizeof(config->max_virtqueue_pairs));
1764                 } else {
1765                         PMD_INIT_LOG(DEBUG,
1766                                      "VIRTIO_NET_F_MQ is not supported");
1767                         config->max_virtqueue_pairs = 1;
1768                 }
1769
1770                 hw->max_queue_pairs = config->max_virtqueue_pairs;
1771
1772                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MTU)) {
1773                         vtpci_read_dev_config(hw,
1774                                 offsetof(struct virtio_net_config, mtu),
1775                                 &config->mtu,
1776                                 sizeof(config->mtu));
1777
1778                         /*
1779                          * MTU value has already been checked at negotiation
1780                          * time, but check again in case it has changed since
1781                          * then, which should not happen.
1782                          */
1783                         if (config->mtu < RTE_ETHER_MIN_MTU) {
1784                                 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)",
1785                                                 config->mtu);
1786                                 return -1;
1787                         }
1788
1789                         hw->max_mtu = config->mtu;
1790                         /* Set initial MTU to maximum one supported by vhost */
1791                         eth_dev->data->mtu = config->mtu;
1792
1793                 } else {
1794                         hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1795                                 VLAN_TAG_LEN - hw->vtnet_hdr_size;
1796                 }
1797
1798                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
1799                                 config->max_virtqueue_pairs);
1800                 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
1801                 PMD_INIT_LOG(DEBUG,
1802                                 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1803                                 config->mac[0], config->mac[1],
1804                                 config->mac[2], config->mac[3],
1805                                 config->mac[4], config->mac[5]);
1806         } else {
1807                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
1808                 hw->max_queue_pairs = 1;
1809                 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1810                         VLAN_TAG_LEN - hw->vtnet_hdr_size;
1811         }
1812
1813         ret = virtio_alloc_queues(eth_dev);
1814         if (ret < 0)
1815                 return ret;
1816
1817         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1818                 if (virtio_configure_intr(eth_dev) < 0) {
1819                         PMD_INIT_LOG(ERR, "failed to configure interrupt");
1820                         virtio_free_queues(hw);
1821                         return -1;
1822                 }
1823         }
1824
1825         vtpci_reinit_complete(hw);
1826
1827         if (pci_dev)
1828                 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1829                         eth_dev->data->port_id, pci_dev->id.vendor_id,
1830                         pci_dev->id.device_id);
1831
1832         return 0;
1833 }
1834
1835 /*
1836  * Remap the PCI device again (IO port map for legacy device and
1837  * memory map for modern device), so that the secondary process
1838  * could have the PCI initiated correctly.
1839  */
1840 static int
1841 virtio_remap_pci(struct rte_pci_device *pci_dev, struct virtio_hw *hw)
1842 {
1843         if (hw->modern) {
1844                 /*
1845                  * We don't have to re-parse the PCI config space, since
1846                  * rte_pci_map_device() makes sure the mapped address
1847                  * in secondary process would equal to the one mapped in
1848                  * the primary process: error will be returned if that
1849                  * requirement is not met.
1850                  *
1851                  * That said, we could simply reuse all cap pointers
1852                  * (such as dev_cfg, common_cfg, etc.) parsed from the
1853                  * primary process, which is stored in shared memory.
1854                  */
1855                 if (rte_pci_map_device(pci_dev)) {
1856                         PMD_INIT_LOG(DEBUG, "failed to map pci device!");
1857                         return -1;
1858                 }
1859         } else {
1860                 if (rte_pci_ioport_map(pci_dev, 0, VTPCI_IO(hw)) < 0)
1861                         return -1;
1862         }
1863
1864         return 0;
1865 }
1866
1867 static void
1868 virtio_set_vtpci_ops(struct virtio_hw *hw)
1869 {
1870 #ifdef RTE_VIRTIO_USER
1871         if (hw->virtio_user_dev)
1872                 VTPCI_OPS(hw) = &virtio_user_ops;
1873         else
1874 #endif
1875         if (hw->modern)
1876                 VTPCI_OPS(hw) = &modern_ops;
1877         else
1878                 VTPCI_OPS(hw) = &legacy_ops;
1879 }
1880
1881 /*
1882  * This function is based on probe() function in virtio_pci.c
1883  * It returns 0 on success.
1884  */
1885 int
1886 eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
1887 {
1888         struct virtio_hw *hw = eth_dev->data->dev_private;
1889         uint32_t speed = SPEED_UNKNOWN;
1890         int vectorized = 0;
1891         int ret;
1892
1893         if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
1894                 PMD_INIT_LOG(ERR,
1895                         "Not sufficient headroom required = %d, avail = %d",
1896                         (int)sizeof(struct virtio_net_hdr_mrg_rxbuf),
1897                         RTE_PKTMBUF_HEADROOM);
1898
1899                 return -1;
1900         }
1901
1902         eth_dev->dev_ops = &virtio_eth_dev_ops;
1903
1904         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1905                 if (!hw->virtio_user_dev) {
1906                         ret = virtio_remap_pci(RTE_ETH_DEV_TO_PCI(eth_dev), hw);
1907                         if (ret)
1908                                 return ret;
1909                 }
1910
1911                 virtio_set_vtpci_ops(hw);
1912                 set_rxtx_funcs(eth_dev);
1913
1914                 return 0;
1915         }
1916         ret = virtio_dev_devargs_parse(eth_dev->device->devargs,
1917                  NULL, &speed, &vectorized);
1918         if (ret < 0)
1919                 return ret;
1920         hw->speed = speed;
1921         /*
1922          * Pass the information to the rte_eth_dev_close() that it should also
1923          * release the private port resources.
1924          */
1925         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
1926
1927         /* Allocate memory for storing MAC addresses */
1928         eth_dev->data->mac_addrs = rte_zmalloc("virtio",
1929                                 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN, 0);
1930         if (eth_dev->data->mac_addrs == NULL) {
1931                 PMD_INIT_LOG(ERR,
1932                         "Failed to allocate %d bytes needed to store MAC addresses",
1933                         VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN);
1934                 return -ENOMEM;
1935         }
1936
1937         hw->port_id = eth_dev->data->port_id;
1938         /* For virtio_user case the hw->virtio_user_dev is populated by
1939          * virtio_user_eth_dev_alloc() before eth_virtio_dev_init() is called.
1940          */
1941         if (!hw->virtio_user_dev) {
1942                 ret = vtpci_init(RTE_ETH_DEV_TO_PCI(eth_dev), hw);
1943                 if (ret)
1944                         goto err_vtpci_init;
1945         }
1946
1947         rte_spinlock_init(&hw->state_lock);
1948
1949         /* reset device and negotiate default features */
1950         ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
1951         if (ret < 0)
1952                 goto err_virtio_init;
1953
1954         if (vectorized) {
1955                 if (!vtpci_packed_queue(hw))
1956                         hw->use_vec_rx = 1;
1957         }
1958
1959         hw->opened = true;
1960
1961         return 0;
1962
1963 err_virtio_init:
1964         if (!hw->virtio_user_dev) {
1965                 rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev));
1966                 if (!hw->modern)
1967                         rte_pci_ioport_unmap(VTPCI_IO(hw));
1968         }
1969 err_vtpci_init:
1970         rte_free(eth_dev->data->mac_addrs);
1971         eth_dev->data->mac_addrs = NULL;
1972         return ret;
1973 }
1974
1975 static int
1976 eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev)
1977 {
1978         PMD_INIT_FUNC_TRACE();
1979
1980         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
1981                 return 0;
1982
1983         virtio_dev_stop(eth_dev);
1984         virtio_dev_close(eth_dev);
1985
1986         eth_dev->dev_ops = NULL;
1987         eth_dev->tx_pkt_burst = NULL;
1988         eth_dev->rx_pkt_burst = NULL;
1989
1990         PMD_INIT_LOG(DEBUG, "dev_uninit completed");
1991
1992         return 0;
1993 }
1994
1995
1996 static int vdpa_check_handler(__rte_unused const char *key,
1997                 const char *value, void *ret_val)
1998 {
1999         if (strcmp(value, "1") == 0)
2000                 *(int *)ret_val = 1;
2001         else
2002                 *(int *)ret_val = 0;
2003
2004         return 0;
2005 }
2006
2007
2008 static uint32_t
2009 virtio_dev_speed_capa_get(uint32_t speed)
2010 {
2011         switch (speed) {
2012         case ETH_SPEED_NUM_10G:
2013                 return ETH_LINK_SPEED_10G;
2014         case ETH_SPEED_NUM_20G:
2015                 return ETH_LINK_SPEED_20G;
2016         case ETH_SPEED_NUM_25G:
2017                 return ETH_LINK_SPEED_25G;
2018         case ETH_SPEED_NUM_40G:
2019                 return ETH_LINK_SPEED_40G;
2020         case ETH_SPEED_NUM_50G:
2021                 return ETH_LINK_SPEED_50G;
2022         case ETH_SPEED_NUM_56G:
2023                 return ETH_LINK_SPEED_56G;
2024         case ETH_SPEED_NUM_100G:
2025                 return ETH_LINK_SPEED_100G;
2026         default:
2027                 return 0;
2028         }
2029 }
2030
2031 static int vectorized_check_handler(__rte_unused const char *key,
2032                 const char *value, void *ret_val)
2033 {
2034         if (strcmp(value, "1") == 0)
2035                 *(int *)ret_val = 1;
2036         else
2037                 *(int *)ret_val = 0;
2038
2039         return 0;
2040 }
2041
2042 #define VIRTIO_ARG_SPEED      "speed"
2043 #define VIRTIO_ARG_VDPA       "vdpa"
2044 #define VIRTIO_ARG_VECTORIZED "vectorized"
2045
2046
2047 static int
2048 link_speed_handler(const char *key __rte_unused,
2049                 const char *value, void *ret_val)
2050 {
2051         uint32_t val;
2052         if (!value || !ret_val)
2053                 return -EINVAL;
2054         val = strtoul(value, NULL, 0);
2055         /* validate input */
2056         if (virtio_dev_speed_capa_get(val) == 0)
2057                 return -EINVAL;
2058         *(uint32_t *)ret_val = val;
2059
2060         return 0;
2061 }
2062
2063
2064 static int
2065 virtio_dev_devargs_parse(struct rte_devargs *devargs, int *vdpa,
2066         uint32_t *speed, int *vectorized)
2067 {
2068         struct rte_kvargs *kvlist;
2069         int ret = 0;
2070
2071         if (devargs == NULL)
2072                 return 0;
2073
2074         kvlist = rte_kvargs_parse(devargs->args, NULL);
2075         if (kvlist == NULL) {
2076                 PMD_INIT_LOG(ERR, "error when parsing param");
2077                 return 0;
2078         }
2079         if (vdpa && rte_kvargs_count(kvlist, VIRTIO_ARG_VDPA) == 1) {
2080                 /* vdpa mode selected when there's a key-value pair:
2081                  * vdpa=1
2082                  */
2083                 ret = rte_kvargs_process(kvlist, VIRTIO_ARG_VDPA,
2084                                 vdpa_check_handler, vdpa);
2085                 if (ret < 0) {
2086                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2087                                 VIRTIO_ARG_VDPA);
2088                         goto exit;
2089                 }
2090         }
2091         if (speed && rte_kvargs_count(kvlist, VIRTIO_ARG_SPEED) == 1) {
2092                 ret = rte_kvargs_process(kvlist,
2093                                         VIRTIO_ARG_SPEED,
2094                                         link_speed_handler, speed);
2095                 if (ret < 0) {
2096                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2097                                         VIRTIO_ARG_SPEED);
2098                         goto exit;
2099                 }
2100         }
2101
2102         if (vectorized &&
2103                 rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
2104                 ret = rte_kvargs_process(kvlist,
2105                                 VIRTIO_ARG_VECTORIZED,
2106                                 vectorized_check_handler, vectorized);
2107                 if (ret < 0) {
2108                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2109                                         VIRTIO_ARG_VECTORIZED);
2110                         goto exit;
2111                 }
2112         }
2113
2114 exit:
2115         rte_kvargs_free(kvlist);
2116         return ret;
2117 }
2118
2119 static int eth_virtio_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2120         struct rte_pci_device *pci_dev)
2121 {
2122         int vdpa = 0;
2123         int ret = 0;
2124
2125         ret = virtio_dev_devargs_parse(pci_dev->device.devargs, &vdpa, NULL,
2126                 NULL);
2127         if (ret < 0) {
2128                 PMD_INIT_LOG(ERR, "devargs parsing is failed");
2129                 return ret;
2130         }
2131         /* virtio pmd skips probe if device needs to work in vdpa mode */
2132         if (vdpa == 1)
2133                 return 1;
2134
2135         return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct virtio_hw),
2136                 eth_virtio_dev_init);
2137 }
2138
2139 static int eth_virtio_pci_remove(struct rte_pci_device *pci_dev)
2140 {
2141         int ret;
2142
2143         ret = rte_eth_dev_pci_generic_remove(pci_dev, eth_virtio_dev_uninit);
2144         /* Port has already been released by close. */
2145         if (ret == -ENODEV)
2146                 ret = 0;
2147         return ret;
2148 }
2149
2150 static struct rte_pci_driver rte_virtio_pmd = {
2151         .driver = {
2152                 .name = "net_virtio",
2153         },
2154         .id_table = pci_id_virtio_map,
2155         .drv_flags = 0,
2156         .probe = eth_virtio_pci_probe,
2157         .remove = eth_virtio_pci_remove,
2158 };
2159
2160 RTE_INIT(rte_virtio_pmd_init)
2161 {
2162         rte_eal_iopl_init();
2163         rte_pci_register(&rte_virtio_pmd);
2164 }
2165
2166 static bool
2167 rx_offload_enabled(struct virtio_hw *hw)
2168 {
2169         return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
2170                 vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2171                 vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
2172 }
2173
2174 static bool
2175 tx_offload_enabled(struct virtio_hw *hw)
2176 {
2177         return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) ||
2178                 vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
2179                 vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
2180 }
2181
2182 /*
2183  * Configure virtio device
2184  * It returns 0 on success.
2185  */
2186 static int
2187 virtio_dev_configure(struct rte_eth_dev *dev)
2188 {
2189         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2190         const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
2191         struct virtio_hw *hw = dev->data->dev_private;
2192         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
2193                 hw->vtnet_hdr_size;
2194         uint64_t rx_offloads = rxmode->offloads;
2195         uint64_t tx_offloads = txmode->offloads;
2196         uint64_t req_features;
2197         int ret;
2198
2199         PMD_INIT_LOG(DEBUG, "configure");
2200         req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
2201
2202         if (rxmode->mq_mode != ETH_MQ_RX_NONE) {
2203                 PMD_DRV_LOG(ERR,
2204                         "Unsupported Rx multi queue mode %d",
2205                         rxmode->mq_mode);
2206                 return -EINVAL;
2207         }
2208
2209         if (txmode->mq_mode != ETH_MQ_TX_NONE) {
2210                 PMD_DRV_LOG(ERR,
2211                         "Unsupported Tx multi queue mode %d",
2212                         txmode->mq_mode);
2213                 return -EINVAL;
2214         }
2215
2216         if (dev->data->dev_conf.intr_conf.rxq) {
2217                 ret = virtio_init_device(dev, hw->req_guest_features);
2218                 if (ret < 0)
2219                         return ret;
2220         }
2221
2222         if (rxmode->max_rx_pkt_len > hw->max_mtu + ether_hdr_len)
2223                 req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
2224
2225         if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2226                            DEV_RX_OFFLOAD_TCP_CKSUM))
2227                 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
2228
2229         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO)
2230                 req_features |=
2231                         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2232                         (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2233
2234         if (tx_offloads & (DEV_TX_OFFLOAD_UDP_CKSUM |
2235                            DEV_TX_OFFLOAD_TCP_CKSUM))
2236                 req_features |= (1ULL << VIRTIO_NET_F_CSUM);
2237
2238         if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO)
2239                 req_features |=
2240                         (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2241                         (1ULL << VIRTIO_NET_F_HOST_TSO6);
2242
2243         /* if request features changed, reinit the device */
2244         if (req_features != hw->req_guest_features) {
2245                 ret = virtio_init_device(dev, req_features);
2246                 if (ret < 0)
2247                         return ret;
2248         }
2249
2250         if ((rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2251                             DEV_RX_OFFLOAD_TCP_CKSUM)) &&
2252                 !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
2253                 PMD_DRV_LOG(ERR,
2254                         "rx checksum not available on this host");
2255                 return -ENOTSUP;
2256         }
2257
2258         if ((rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) &&
2259                 (!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2260                  !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) {
2261                 PMD_DRV_LOG(ERR,
2262                         "Large Receive Offload not available on this host");
2263                 return -ENOTSUP;
2264         }
2265
2266         /* start control queue */
2267         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
2268                 virtio_dev_cq_start(dev);
2269
2270         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2271                 hw->vlan_strip = 1;
2272
2273         if ((rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
2274             && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2275                 PMD_DRV_LOG(ERR,
2276                             "vlan filtering not available on this host");
2277                 return -ENOTSUP;
2278         }
2279
2280         hw->has_tx_offload = tx_offload_enabled(hw);
2281         hw->has_rx_offload = rx_offload_enabled(hw);
2282
2283         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2284                 /* Enable vector (0) for Link State Intrerrupt */
2285                 if (VTPCI_OPS(hw)->set_config_irq(hw, 0) ==
2286                                 VIRTIO_MSI_NO_VECTOR) {
2287                         PMD_DRV_LOG(ERR, "failed to set config vector");
2288                         return -EBUSY;
2289                 }
2290
2291         if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
2292                 hw->use_inorder_tx = 1;
2293                 hw->use_inorder_rx = 1;
2294                 hw->use_vec_rx = 0;
2295         }
2296
2297         if (vtpci_packed_queue(hw)) {
2298                 hw->use_vec_rx = 0;
2299                 hw->use_inorder_rx = 0;
2300         }
2301
2302 #if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM
2303         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
2304                 hw->use_vec_rx = 0;
2305         }
2306 #endif
2307         if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2308                 hw->use_vec_rx = 0;
2309         }
2310
2311         if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2312                            DEV_RX_OFFLOAD_TCP_CKSUM |
2313                            DEV_RX_OFFLOAD_TCP_LRO |
2314                            DEV_RX_OFFLOAD_VLAN_STRIP))
2315                 hw->use_vec_rx = 0;
2316
2317         return 0;
2318 }
2319
2320
2321 static int
2322 virtio_dev_start(struct rte_eth_dev *dev)
2323 {
2324         uint16_t nb_queues, i;
2325         struct virtnet_rx *rxvq;
2326         struct virtnet_tx *txvq __rte_unused;
2327         struct virtio_hw *hw = dev->data->dev_private;
2328         int ret;
2329
2330         /* Finish the initialization of the queues */
2331         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2332                 ret = virtio_dev_rx_queue_setup_finish(dev, i);
2333                 if (ret < 0)
2334                         return ret;
2335         }
2336         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2337                 ret = virtio_dev_tx_queue_setup_finish(dev, i);
2338                 if (ret < 0)
2339                         return ret;
2340         }
2341
2342         /* check if lsc interrupt feature is enabled */
2343         if (dev->data->dev_conf.intr_conf.lsc) {
2344                 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
2345                         PMD_DRV_LOG(ERR, "link status not supported by host");
2346                         return -ENOTSUP;
2347                 }
2348         }
2349
2350         /* Enable uio/vfio intr/eventfd mapping: althrough we already did that
2351          * in device configure, but it could be unmapped  when device is
2352          * stopped.
2353          */
2354         if (dev->data->dev_conf.intr_conf.lsc ||
2355             dev->data->dev_conf.intr_conf.rxq) {
2356                 virtio_intr_disable(dev);
2357
2358                 /* Setup interrupt callback  */
2359                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2360                         rte_intr_callback_register(dev->intr_handle,
2361                                                    virtio_interrupt_handler,
2362                                                    dev);
2363
2364                 if (virtio_intr_enable(dev) < 0) {
2365                         PMD_DRV_LOG(ERR, "interrupt enable failed");
2366                         return -EIO;
2367                 }
2368         }
2369
2370         /*Notify the backend
2371          *Otherwise the tap backend might already stop its queue due to fullness.
2372          *vhost backend will have no chance to be waked up
2373          */
2374         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
2375         if (hw->max_queue_pairs > 1) {
2376                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
2377                         return -EINVAL;
2378         }
2379
2380         PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
2381
2382         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2383                 rxvq = dev->data->rx_queues[i];
2384                 /* Flush the old packets */
2385                 virtqueue_rxvq_flush(rxvq->vq);
2386                 virtqueue_notify(rxvq->vq);
2387         }
2388
2389         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2390                 txvq = dev->data->tx_queues[i];
2391                 virtqueue_notify(txvq->vq);
2392         }
2393
2394         PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
2395
2396         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2397                 rxvq = dev->data->rx_queues[i];
2398                 VIRTQUEUE_DUMP(rxvq->vq);
2399         }
2400
2401         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2402                 txvq = dev->data->tx_queues[i];
2403                 VIRTQUEUE_DUMP(txvq->vq);
2404         }
2405
2406         set_rxtx_funcs(dev);
2407         hw->started = true;
2408
2409         /* Initialize Link state */
2410         virtio_dev_link_update(dev, 0);
2411
2412         return 0;
2413 }
2414
2415 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
2416 {
2417         struct virtio_hw *hw = dev->data->dev_private;
2418         uint16_t nr_vq = virtio_get_nr_vq(hw);
2419         const char *type __rte_unused;
2420         unsigned int i, mbuf_num = 0;
2421         struct virtqueue *vq;
2422         struct rte_mbuf *buf;
2423         int queue_type;
2424
2425         if (hw->vqs == NULL)
2426                 return;
2427
2428         for (i = 0; i < nr_vq; i++) {
2429                 vq = hw->vqs[i];
2430                 if (!vq)
2431                         continue;
2432
2433                 queue_type = virtio_get_queue_type(hw, i);
2434                 if (queue_type == VTNET_RQ)
2435                         type = "rxq";
2436                 else if (queue_type == VTNET_TQ)
2437                         type = "txq";
2438                 else
2439                         continue;
2440
2441                 PMD_INIT_LOG(DEBUG,
2442                         "Before freeing %s[%d] used and unused buf",
2443                         type, i);
2444                 VIRTQUEUE_DUMP(vq);
2445
2446                 while ((buf = virtqueue_detach_unused(vq)) != NULL) {
2447                         rte_pktmbuf_free(buf);
2448                         mbuf_num++;
2449                 }
2450
2451                 PMD_INIT_LOG(DEBUG,
2452                         "After freeing %s[%d] used and unused buf",
2453                         type, i);
2454                 VIRTQUEUE_DUMP(vq);
2455         }
2456
2457         PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num);
2458 }
2459
2460 /*
2461  * Stop device: disable interrupt and mark link down
2462  */
2463 static void
2464 virtio_dev_stop(struct rte_eth_dev *dev)
2465 {
2466         struct virtio_hw *hw = dev->data->dev_private;
2467         struct rte_eth_link link;
2468         struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
2469
2470         PMD_INIT_LOG(DEBUG, "stop");
2471
2472         rte_spinlock_lock(&hw->state_lock);
2473         if (!hw->started)
2474                 goto out_unlock;
2475         hw->started = false;
2476
2477         if (intr_conf->lsc || intr_conf->rxq) {
2478                 virtio_intr_disable(dev);
2479
2480                 /* Reset interrupt callback  */
2481                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
2482                         rte_intr_callback_unregister(dev->intr_handle,
2483                                                      virtio_interrupt_handler,
2484                                                      dev);
2485                 }
2486         }
2487
2488         memset(&link, 0, sizeof(link));
2489         rte_eth_linkstatus_set(dev, &link);
2490 out_unlock:
2491         rte_spinlock_unlock(&hw->state_lock);
2492 }
2493
2494 static int
2495 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
2496 {
2497         struct rte_eth_link link;
2498         uint16_t status;
2499         struct virtio_hw *hw = dev->data->dev_private;
2500
2501         memset(&link, 0, sizeof(link));
2502         link.link_duplex = hw->duplex;
2503         link.link_speed  = hw->speed;
2504         link.link_autoneg = ETH_LINK_AUTONEG;
2505
2506         if (!hw->started) {
2507                 link.link_status = ETH_LINK_DOWN;
2508         } else if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
2509                 PMD_INIT_LOG(DEBUG, "Get link status from hw");
2510                 vtpci_read_dev_config(hw,
2511                                 offsetof(struct virtio_net_config, status),
2512                                 &status, sizeof(status));
2513                 if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
2514                         link.link_status = ETH_LINK_DOWN;
2515                         PMD_INIT_LOG(DEBUG, "Port %d is down",
2516                                      dev->data->port_id);
2517                 } else {
2518                         link.link_status = ETH_LINK_UP;
2519                         PMD_INIT_LOG(DEBUG, "Port %d is up",
2520                                      dev->data->port_id);
2521                 }
2522         } else {
2523                 link.link_status = ETH_LINK_UP;
2524         }
2525
2526         return rte_eth_linkstatus_set(dev, &link);
2527 }
2528
2529 static int
2530 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2531 {
2532         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2533         struct virtio_hw *hw = dev->data->dev_private;
2534         uint64_t offloads = rxmode->offloads;
2535
2536         if (mask & ETH_VLAN_FILTER_MASK) {
2537                 if ((offloads & DEV_RX_OFFLOAD_VLAN_FILTER) &&
2538                                 !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2539
2540                         PMD_DRV_LOG(NOTICE,
2541                                 "vlan filtering not available on this host");
2542
2543                         return -ENOTSUP;
2544                 }
2545         }
2546
2547         if (mask & ETH_VLAN_STRIP_MASK)
2548                 hw->vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
2549
2550         return 0;
2551 }
2552
2553 static int
2554 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2555 {
2556         uint64_t tso_mask, host_features;
2557         struct virtio_hw *hw = dev->data->dev_private;
2558         dev_info->speed_capa = virtio_dev_speed_capa_get(hw->speed);
2559
2560         dev_info->max_rx_queues =
2561                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
2562         dev_info->max_tx_queues =
2563                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
2564         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
2565         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
2566         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
2567
2568         host_features = VTPCI_OPS(hw)->get_features(hw);
2569         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
2570         dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
2571         if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
2572                 dev_info->rx_offload_capa |=
2573                         DEV_RX_OFFLOAD_TCP_CKSUM |
2574                         DEV_RX_OFFLOAD_UDP_CKSUM;
2575         }
2576         if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN))
2577                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_VLAN_FILTER;
2578         tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2579                 (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2580         if ((host_features & tso_mask) == tso_mask)
2581                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2582
2583         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
2584                                     DEV_TX_OFFLOAD_VLAN_INSERT;
2585         if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) {
2586                 dev_info->tx_offload_capa |=
2587                         DEV_TX_OFFLOAD_UDP_CKSUM |
2588                         DEV_TX_OFFLOAD_TCP_CKSUM;
2589         }
2590         tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2591                 (1ULL << VIRTIO_NET_F_HOST_TSO6);
2592         if ((host_features & tso_mask) == tso_mask)
2593                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
2594
2595         return 0;
2596 }
2597
2598 /*
2599  * It enables testpmd to collect per queue stats.
2600  */
2601 static int
2602 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
2603 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
2604 __rte_unused uint8_t is_rx)
2605 {
2606         return 0;
2607 }
2608
2609 RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__);
2610 RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map);
2611 RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio-pci");
2612
2613 RTE_INIT(virtio_init_log)
2614 {
2615         virtio_logtype_init = rte_log_register("pmd.net.virtio.init");
2616         if (virtio_logtype_init >= 0)
2617                 rte_log_set_level(virtio_logtype_init, RTE_LOG_NOTICE);
2618         virtio_logtype_driver = rte_log_register("pmd.net.virtio.driver");
2619         if (virtio_logtype_driver >= 0)
2620                 rte_log_set_level(virtio_logtype_driver, RTE_LOG_NOTICE);
2621 }