drivers/net/virtio/virtio_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_memcpy.h>
  14 #include <rte_string_fns.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_branch_prediction.h>
  18 #include <rte_pci.h>
  19 #include <rte_bus_pci.h>
  20 #include <rte_ether.h>
  21 #include <rte_ip.h>
  22 #include <rte_arp.h>
  23 #include <rte_common.h>
  24 #include <rte_errno.h>
  25 #include <rte_cpuflags.h>
  26
  27 #include <rte_memory.h>
  28 #include <rte_eal.h>
  29 #include <rte_dev.h>
  30 #include <rte_cycles.h>
  31 #include <rte_kvargs.h>
  32
  33 #include "virtio_ethdev.h"
  34 #include "virtio_pci.h"
  35 #include "virtio_logs.h"
  36 #include "virtqueue.h"
  37 #include "virtio_rxtx.h"
  38
  39 static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev);
  40 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  41 static int  virtio_dev_start(struct rte_eth_dev *dev);
  42 static void virtio_dev_stop(struct rte_eth_dev *dev);
  43 static void virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
  44 static void virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
  45 static void virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
  46 static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
  47 static void virtio_dev_info_get(struct rte_eth_dev *dev,
  48                                 struct rte_eth_dev_info *dev_info);
  49 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  50         int wait_to_complete);
  51 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  52
  53 static void virtio_set_hwaddr(struct virtio_hw *hw);
  54 static void virtio_get_hwaddr(struct virtio_hw *hw);
  55
  56 static int virtio_dev_stats_get(struct rte_eth_dev *dev,
  57                                  struct rte_eth_stats *stats);
  58 static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
  59                                  struct rte_eth_xstat *xstats, unsigned n);
  60 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
  61                                        struct rte_eth_xstat_name *xstats_names,
  62                                        unsigned limit);
  63 static void virtio_dev_stats_reset(struct rte_eth_dev *dev);
  64 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  65 static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
  66                                 uint16_t vlan_id, int on);
  67 static int virtio_mac_addr_add(struct rte_eth_dev *dev,
  68                                 struct ether_addr *mac_addr,
  69                                 uint32_t index, uint32_t vmdq);
  70 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  71 static int virtio_mac_addr_set(struct rte_eth_dev *dev,
  72                                 struct ether_addr *mac_addr);
  73
  74 static int virtio_intr_disable(struct rte_eth_dev *dev);
  75
  76 static int virtio_dev_queue_stats_mapping_set(
  77         struct rte_eth_dev *eth_dev,
  78         uint16_t queue_id,
  79         uint8_t stat_idx,
  80         uint8_t is_rx);
  81
  82 int virtio_logtype_init;
  83 int virtio_logtype_driver;
  84
  85 static void virtio_notify_peers(struct rte_eth_dev *dev);
  86 static void virtio_ack_link_announce(struct rte_eth_dev *dev);
  87
  88 /*
  89  * The set of PCI devices this driver supports
  90  */
  91 static const struct rte_pci_id pci_id_virtio_map[] = {
  92         { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_LEGACY_DEVICEID_NET) },
  93         { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_MODERN_DEVICEID_NET) },
  94         { .vendor_id = 0, /* sentinel */ },
  95 };
  96
  97 struct rte_virtio_xstats_name_off {
  98         char name[RTE_ETH_XSTATS_NAME_SIZE];
  99         unsigned offset;
 100 };
 101
 102 /* [rt]x_qX_ is prepended to the name string here */
 103 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
 104         {"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
 105         {"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
 106         {"errors",                 offsetof(struct virtnet_rx, stats.errors)},
 107         {"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
 108         {"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
 109         {"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
 110         {"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
 111         {"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
 112         {"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
 113         {"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
 114         {"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
 115         {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
 116         {"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
 117 };
 118
 119 /* [rt]x_qX_ is prepended to the name string here */
 120 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
 121         {"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
 122         {"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
 123         {"errors",                 offsetof(struct virtnet_tx, stats.errors)},
 124         {"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
 125         {"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
 126         {"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
 127         {"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
 128         {"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
 129         {"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
 130         {"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
 131         {"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
 132         {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
 133         {"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
 134 };
 135
 136 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
 137                             sizeof(rte_virtio_rxq_stat_strings[0]))
 138 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
 139                             sizeof(rte_virtio_txq_stat_strings[0]))
 140
 141 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS];
 142
 143 static struct virtio_pmd_ctrl *
 144 virtio_send_command_packed(struct virtnet_ctl *cvq,
 145                            struct virtio_pmd_ctrl *ctrl,
 146                            int *dlen, int pkt_num)
 147 {
 148         struct virtqueue *vq = cvq->vq;
 149         int head;
 150         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 151         struct virtio_pmd_ctrl *result;
 152         uint16_t flags;
 153         int sum = 0;
 154         int nb_descs = 0;
 155         int k;
 156
 157         /*
 158          * Format is enforced in qemu code:
 159          * One TX packet for header;
 160          * At least one TX packet per argument;
 161          * One RX packet for ACK.
 162          */
 163         head = vq->vq_avail_idx;
 164         flags = vq->vq_packed.cached_flags;
 165         desc[head].addr = cvq->virtio_net_hdr_mem;
 166         desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 167         vq->vq_free_cnt--;
 168         nb_descs++;
 169         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 170                 vq->vq_avail_idx -= vq->vq_nentries;
 171                 vq->vq_packed.cached_flags ^=
 172                         VRING_DESC_F_AVAIL(1) | VRING_DESC_F_USED(1);
 173         }
 174
 175         for (k = 0; k < pkt_num; k++) {
 176                 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 177                         + sizeof(struct virtio_net_ctrl_hdr)
 178                         + sizeof(ctrl->status) + sizeof(uint8_t) * sum;
 179                 desc[vq->vq_avail_idx].len = dlen[k];
 180                 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT |
 181                         vq->vq_packed.cached_flags;
 182                 sum += dlen[k];
 183                 vq->vq_free_cnt--;
 184                 nb_descs++;
 185                 if (++vq->vq_avail_idx >= vq->vq_nentries) {
 186                         vq->vq_avail_idx -= vq->vq_nentries;
 187                         vq->vq_packed.cached_flags ^=
 188                                 VRING_DESC_F_AVAIL(1) | VRING_DESC_F_USED(1);
 189                 }
 190         }
 191
 192         desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 193                 + sizeof(struct virtio_net_ctrl_hdr);
 194         desc[vq->vq_avail_idx].len = sizeof(ctrl->status);
 195         desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE |
 196                 vq->vq_packed.cached_flags;
 197         vq->vq_free_cnt--;
 198         nb_descs++;
 199         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 200                 vq->vq_avail_idx -= vq->vq_nentries;
 201                 vq->vq_packed.cached_flags ^=
 202                         VRING_DESC_F_AVAIL(1) | VRING_DESC_F_USED(1);
 203         }
 204
 205         virtio_wmb(vq->hw->weak_barriers);
 206         desc[head].flags = VRING_DESC_F_NEXT | flags;
 207
 208         virtio_wmb(vq->hw->weak_barriers);
 209         virtqueue_notify(vq);
 210
 211         /* wait for used descriptors in virtqueue */
 212         while (!desc_is_used(&desc[head], vq))
 213                 usleep(100);
 214
 215         virtio_rmb(vq->hw->weak_barriers);
 216
 217         /* now get used descriptors */
 218         vq->vq_free_cnt += nb_descs;
 219         vq->vq_used_cons_idx += nb_descs;
 220         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 221                 vq->vq_used_cons_idx -= vq->vq_nentries;
 222                 vq->vq_packed.used_wrap_counter ^= 1;
 223         }
 224
 225         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n"
 226                         "vq->vq_avail_idx=%d\n"
 227                         "vq->vq_used_cons_idx=%d\n"
 228                         "vq->vq_packed.cached_flags=0x%x\n"
 229                         "vq->vq_packed.used_wrap_counter=%d\n",
 230                         vq->vq_free_cnt,
 231                         vq->vq_avail_idx,
 232                         vq->vq_used_cons_idx,
 233                         vq->vq_packed.cached_flags,
 234                         vq->vq_packed.used_wrap_counter);
 235
 236         result = cvq->virtio_net_hdr_mz->addr;
 237         return result;
 238 }
 239
 240 static int
 241 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
 242                 int *dlen, int pkt_num)
 243 {
 244         uint32_t head, i;
 245         int k, sum = 0;
 246         virtio_net_ctrl_ack status = ~0;
 247         struct virtio_pmd_ctrl *result;
 248         struct virtqueue *vq;
 249
 250         ctrl->status = status;
 251
 252         if (!cvq || !cvq->vq) {
 253                 PMD_INIT_LOG(ERR, "Control queue is not supported.");
 254                 return -1;
 255         }
 256
 257         rte_spinlock_lock(&cvq->lock);
 258         vq = cvq->vq;
 259         head = vq->vq_desc_head_idx;
 260
 261         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 262                 "vq->hw->cvq = %p vq = %p",
 263                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 264
 265         if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) {
 266                 rte_spinlock_unlock(&cvq->lock);
 267                 return -1;
 268         }
 269
 270         memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
 271                 sizeof(struct virtio_pmd_ctrl));
 272
 273         if (vtpci_packed_queue(vq->hw)) {
 274                 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num);
 275                 goto out_unlock;
 276         }
 277
 278         /*
 279          * Format is enforced in qemu code:
 280          * One TX packet for header;
 281          * At least one TX packet per argument;
 282          * One RX packet for ACK.
 283          */
 284         vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT;
 285         vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem;
 286         vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 287         vq->vq_free_cnt--;
 288         i = vq->vq_split.ring.desc[head].next;
 289
 290         for (k = 0; k < pkt_num; k++) {
 291                 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT;
 292                 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 293                         + sizeof(struct virtio_net_ctrl_hdr)
 294                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 295                 vq->vq_split.ring.desc[i].len = dlen[k];
 296                 sum += dlen[k];
 297                 vq->vq_free_cnt--;
 298                 i = vq->vq_split.ring.desc[i].next;
 299         }
 300
 301         vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE;
 302         vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 303                         + sizeof(struct virtio_net_ctrl_hdr);
 304         vq->vq_split.ring.desc[i].len = sizeof(ctrl->status);
 305         vq->vq_free_cnt--;
 306
 307         vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next;
 308
 309         vq_update_avail_ring(vq, head);
 310         vq_update_avail_idx(vq);
 311
 312         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
 313
 314         virtqueue_notify(vq);
 315
 316         rte_rmb();
 317         while (VIRTQUEUE_NUSED(vq) == 0) {
 318                 rte_rmb();
 319                 usleep(100);
 320         }
 321
 322         while (VIRTQUEUE_NUSED(vq)) {
 323                 uint32_t idx, desc_idx, used_idx;
 324                 struct vring_used_elem *uep;
 325
 326                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 327                                 & (vq->vq_nentries - 1));
 328                 uep = &vq->vq_split.ring.used->ring[used_idx];
 329                 idx = (uint32_t) uep->id;
 330                 desc_idx = idx;
 331
 332                 while (vq->vq_split.ring.desc[desc_idx].flags &
 333                                 VRING_DESC_F_NEXT) {
 334                         desc_idx = vq->vq_split.ring.desc[desc_idx].next;
 335                         vq->vq_free_cnt++;
 336                 }
 337
 338                 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 339                 vq->vq_desc_head_idx = idx;
 340
 341                 vq->vq_used_cons_idx++;
 342                 vq->vq_free_cnt++;
 343         }
 344
 345         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
 346                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 347
 348         result = cvq->virtio_net_hdr_mz->addr;
 349
 350 out_unlock:
 351         rte_spinlock_unlock(&cvq->lock);
 352         return result->status;
 353 }
 354
 355 static int
 356 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 357 {
 358         struct virtio_hw *hw = dev->data->dev_private;
 359         struct virtio_pmd_ctrl ctrl;
 360         int dlen[1];
 361         int ret;
 362
 363         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 364         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 365         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 366
 367         dlen[0] = sizeof(uint16_t);
 368
 369         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 370         if (ret) {
 371                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 372                           "failed, this is too late now...");
 373                 return -EINVAL;
 374         }
 375
 376         return 0;
 377 }
 378
 379 static void
 380 virtio_dev_queue_release(void *queue __rte_unused)
 381 {
 382         /* do nothing */
 383 }
 384
 385 static uint16_t
 386 virtio_get_nr_vq(struct virtio_hw *hw)
 387 {
 388         uint16_t nr_vq = hw->max_queue_pairs * 2;
 389
 390         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
 391                 nr_vq += 1;
 392
 393         return nr_vq;
 394 }
 395
 396 static void
 397 virtio_init_vring(struct virtqueue *vq)
 398 {
 399         int size = vq->vq_nentries;
 400         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 401
 402         PMD_INIT_FUNC_TRACE();
 403
 404         memset(ring_mem, 0, vq->vq_ring_size);
 405
 406         vq->vq_used_cons_idx = 0;
 407         vq->vq_desc_head_idx = 0;
 408         vq->vq_avail_idx = 0;
 409         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 410         vq->vq_free_cnt = vq->vq_nentries;
 411         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 412         if (vtpci_packed_queue(vq->hw)) {
 413                 vring_init_packed(&vq->vq_packed.ring, ring_mem,
 414                                   VIRTIO_PCI_VRING_ALIGN, size);
 415                 vring_desc_init_packed(vq, size);
 416         } else {
 417                 struct vring *vr = &vq->vq_split.ring;
 418
 419                 vring_init_split(vr, ring_mem, VIRTIO_PCI_VRING_ALIGN, size);
 420                 vring_desc_init_split(vr->desc, size);
 421         }
 422         /*
 423          * Disable device(host) interrupting guest
 424          */
 425         virtqueue_disable_intr(vq);
 426 }
 427
 428 static int
 429 virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx)
 430 {
 431         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 432         char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
 433         const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
 434         unsigned int vq_size, size;
 435         struct virtio_hw *hw = dev->data->dev_private;
 436         struct virtnet_rx *rxvq = NULL;
 437         struct virtnet_tx *txvq = NULL;
 438         struct virtnet_ctl *cvq = NULL;
 439         struct virtqueue *vq;
 440         size_t sz_hdr_mz = 0;
 441         void *sw_ring = NULL;
 442         int queue_type = virtio_get_queue_type(hw, vtpci_queue_idx);
 443         int ret;
 444         int numa_node = dev->device->numa_node;
 445
 446         PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d",
 447                         vtpci_queue_idx, numa_node);
 448
 449         /*
 450          * Read the virtqueue size from the Queue Size field
 451          * Always power of 2 and if 0 virtqueue does not exist
 452          */
 453         vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
 454         PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
 455         if (vq_size == 0) {
 456                 PMD_INIT_LOG(ERR, "virtqueue does not exist");
 457                 return -EINVAL;
 458         }
 459
 460         if (!rte_is_power_of_2(vq_size)) {
 461                 PMD_INIT_LOG(ERR, "virtqueue size is not powerof 2");
 462                 return -EINVAL;
 463         }
 464
 465         snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
 466                  dev->data->port_id, vtpci_queue_idx);
 467
 468         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 469                                 vq_size * sizeof(struct vq_desc_extra),
 470                                 RTE_CACHE_LINE_SIZE);
 471         if (queue_type == VTNET_TQ) {
 472                 /*
 473                  * For each xmit packet, allocate a virtio_net_hdr
 474                  * and indirect ring elements
 475                  */
 476                 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
 477         } else if (queue_type == VTNET_CQ) {
 478                 /* Allocate a page for control vq command, data and status */
 479                 sz_hdr_mz = PAGE_SIZE;
 480         }
 481
 482         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
 483                                 numa_node);
 484         if (vq == NULL) {
 485                 PMD_INIT_LOG(ERR, "can not allocate vq");
 486                 return -ENOMEM;
 487         }
 488         hw->vqs[vtpci_queue_idx] = vq;
 489
 490         vq->hw = hw;
 491         vq->vq_queue_index = vtpci_queue_idx;
 492         vq->vq_nentries = vq_size;
 493         if (vtpci_packed_queue(hw)) {
 494                 vq->vq_packed.used_wrap_counter = 1;
 495                 vq->vq_packed.cached_flags = VRING_DESC_F_AVAIL(1);
 496                 vq->vq_packed.event_flags_shadow = 0;
 497                 if (queue_type == VTNET_RQ)
 498                         vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
 499         }
 500
 501         /*
 502          * Reserve a memzone for vring elements
 503          */
 504         size = vring_size(hw, vq_size, VIRTIO_PCI_VRING_ALIGN);
 505         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
 506         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
 507                      size, vq->vq_ring_size);
 508
 509         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 510                         numa_node, RTE_MEMZONE_IOVA_CONTIG,
 511                         VIRTIO_PCI_VRING_ALIGN);
 512         if (mz == NULL) {
 513                 if (rte_errno == EEXIST)
 514                         mz = rte_memzone_lookup(vq_name);
 515                 if (mz == NULL) {
 516                         ret = -ENOMEM;
 517                         goto fail_q_alloc;
 518                 }
 519         }
 520
 521         memset(mz->addr, 0, mz->len);
 522
 523         vq->vq_ring_mem = mz->iova;
 524         vq->vq_ring_virt_mem = mz->addr;
 525         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem:      0x%" PRIx64,
 526                      (uint64_t)mz->iova);
 527         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%" PRIx64,
 528                      (uint64_t)(uintptr_t)mz->addr);
 529
 530         virtio_init_vring(vq);
 531
 532         if (sz_hdr_mz) {
 533                 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 534                          dev->data->port_id, vtpci_queue_idx);
 535                 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
 536                                 numa_node, RTE_MEMZONE_IOVA_CONTIG,
 537                                 RTE_CACHE_LINE_SIZE);
 538                 if (hdr_mz == NULL) {
 539                         if (rte_errno == EEXIST)
 540                                 hdr_mz = rte_memzone_lookup(vq_hdr_name);
 541                         if (hdr_mz == NULL) {
 542                                 ret = -ENOMEM;
 543                                 goto fail_q_alloc;
 544                         }
 545                 }
 546         }
 547
 548         if (queue_type == VTNET_RQ) {
 549                 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
 550                                sizeof(vq->sw_ring[0]);
 551
 552                 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
 553                                 RTE_CACHE_LINE_SIZE, numa_node);
 554                 if (!sw_ring) {
 555                         PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
 556                         ret = -ENOMEM;
 557                         goto fail_q_alloc;
 558                 }
 559
 560                 vq->sw_ring = sw_ring;
 561                 rxvq = &vq->rxq;
 562                 rxvq->vq = vq;
 563                 rxvq->port_id = dev->data->port_id;
 564                 rxvq->mz = mz;
 565         } else if (queue_type == VTNET_TQ) {
 566                 txvq = &vq->txq;
 567                 txvq->vq = vq;
 568                 txvq->port_id = dev->data->port_id;
 569                 txvq->mz = mz;
 570                 txvq->virtio_net_hdr_mz = hdr_mz;
 571                 txvq->virtio_net_hdr_mem = hdr_mz->iova;
 572         } else if (queue_type == VTNET_CQ) {
 573                 cvq = &vq->cq;
 574                 cvq->vq = vq;
 575                 cvq->mz = mz;
 576                 cvq->virtio_net_hdr_mz = hdr_mz;
 577                 cvq->virtio_net_hdr_mem = hdr_mz->iova;
 578                 memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE);
 579
 580                 hw->cvq = cvq;
 581         }
 582
 583         /* For virtio_user case (that is when hw->dev is NULL), we use
 584          * virtual address. And we need properly set _offset_, please see
 585          * VIRTIO_MBUF_DATA_DMA_ADDR in virtqueue.h for more information.
 586          */
 587         if (!hw->virtio_user_dev)
 588                 vq->offset = offsetof(struct rte_mbuf, buf_iova);
 589         else {
 590                 vq->vq_ring_mem = (uintptr_t)mz->addr;
 591                 vq->offset = offsetof(struct rte_mbuf, buf_addr);
 592                 if (queue_type == VTNET_TQ)
 593                         txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 594                 else if (queue_type == VTNET_CQ)
 595                         cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 596         }
 597
 598         if (queue_type == VTNET_TQ) {
 599                 struct virtio_tx_region *txr;
 600                 unsigned int i;
 601
 602                 txr = hdr_mz->addr;
 603                 memset(txr, 0, vq_size * sizeof(*txr));
 604                 for (i = 0; i < vq_size; i++) {
 605                         struct vring_desc *start_dp = txr[i].tx_indir;
 606
 607                         /* first indirect descriptor is always the tx header */
 608                         if (!vtpci_packed_queue(hw)) {
 609                                 vring_desc_init_split(start_dp,
 610                                                       RTE_DIM(txr[i].tx_indir));
 611                                 start_dp->addr = txvq->virtio_net_hdr_mem
 612                                         + i * sizeof(*txr)
 613                                         + offsetof(struct virtio_tx_region,
 614                                                    tx_hdr);
 615                                 start_dp->len = hw->vtnet_hdr_size;
 616                                 start_dp->flags = VRING_DESC_F_NEXT;
 617                         }
 618                 }
 619         }
 620
 621         if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) {
 622                 PMD_INIT_LOG(ERR, "setup_queue failed");
 623                 return -EINVAL;
 624         }
 625
 626         return 0;
 627
 628 fail_q_alloc:
 629         rte_free(sw_ring);
 630         rte_memzone_free(hdr_mz);
 631         rte_memzone_free(mz);
 632         rte_free(vq);
 633
 634         return ret;
 635 }
 636
 637 static void
 638 virtio_free_queues(struct virtio_hw *hw)
 639 {
 640         uint16_t nr_vq = virtio_get_nr_vq(hw);
 641         struct virtqueue *vq;
 642         int queue_type;
 643         uint16_t i;
 644
 645         if (hw->vqs == NULL)
 646                 return;
 647
 648         for (i = 0; i < nr_vq; i++) {
 649                 vq = hw->vqs[i];
 650                 if (!vq)
 651                         continue;
 652
 653                 queue_type = virtio_get_queue_type(hw, i);
 654                 if (queue_type == VTNET_RQ) {
 655                         rte_free(vq->sw_ring);
 656                         rte_memzone_free(vq->rxq.mz);
 657                 } else if (queue_type == VTNET_TQ) {
 658                         rte_memzone_free(vq->txq.mz);
 659                         rte_memzone_free(vq->txq.virtio_net_hdr_mz);
 660                 } else {
 661                         rte_memzone_free(vq->cq.mz);
 662                         rte_memzone_free(vq->cq.virtio_net_hdr_mz);
 663                 }
 664
 665                 rte_free(vq);
 666                 hw->vqs[i] = NULL;
 667         }
 668
 669         rte_free(hw->vqs);
 670         hw->vqs = NULL;
 671 }
 672
 673 static int
 674 virtio_alloc_queues(struct rte_eth_dev *dev)
 675 {
 676         struct virtio_hw *hw = dev->data->dev_private;
 677         uint16_t nr_vq = virtio_get_nr_vq(hw);
 678         uint16_t i;
 679         int ret;
 680
 681         hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0);
 682         if (!hw->vqs) {
 683                 PMD_INIT_LOG(ERR, "failed to allocate vqs");
 684                 return -ENOMEM;
 685         }
 686
 687         for (i = 0; i < nr_vq; i++) {
 688                 ret = virtio_init_queue(dev, i);
 689                 if (ret < 0) {
 690                         virtio_free_queues(hw);
 691                         return ret;
 692                 }
 693         }
 694
 695         return 0;
 696 }
 697
 698 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev);
 699
 700 static void
 701 virtio_dev_close(struct rte_eth_dev *dev)
 702 {
 703         struct virtio_hw *hw = dev->data->dev_private;
 704         struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
 705
 706         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 707
 708         if (!hw->opened)
 709                 return;
 710         hw->opened = false;
 711
 712         /* reset the NIC */
 713         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 714                 VTPCI_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR);
 715         if (intr_conf->rxq)
 716                 virtio_queues_unbind_intr(dev);
 717
 718         if (intr_conf->lsc || intr_conf->rxq) {
 719                 virtio_intr_disable(dev);
 720                 rte_intr_efd_disable(dev->intr_handle);
 721                 rte_free(dev->intr_handle->intr_vec);
 722                 dev->intr_handle->intr_vec = NULL;
 723         }
 724
 725         vtpci_reset(hw);
 726         virtio_dev_free_mbufs(dev);
 727         virtio_free_queues(hw);
 728 }
 729
 730 static void
 731 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
 732 {
 733         struct virtio_hw *hw = dev->data->dev_private;
 734         struct virtio_pmd_ctrl ctrl;
 735         int dlen[1];
 736         int ret;
 737
 738         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 739                 PMD_INIT_LOG(INFO, "host does not support rx control");
 740                 return;
 741         }
 742
 743         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 744         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 745         ctrl.data[0] = 1;
 746         dlen[0] = 1;
 747
 748         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 749         if (ret)
 750                 PMD_INIT_LOG(ERR, "Failed to enable promisc");
 751 }
 752
 753 static void
 754 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
 755 {
 756         struct virtio_hw *hw = dev->data->dev_private;
 757         struct virtio_pmd_ctrl ctrl;
 758         int dlen[1];
 759         int ret;
 760
 761         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 762                 PMD_INIT_LOG(INFO, "host does not support rx control");
 763                 return;
 764         }
 765
 766         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 767         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 768         ctrl.data[0] = 0;
 769         dlen[0] = 1;
 770
 771         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 772         if (ret)
 773                 PMD_INIT_LOG(ERR, "Failed to disable promisc");
 774 }
 775
 776 static void
 777 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
 778 {
 779         struct virtio_hw *hw = dev->data->dev_private;
 780         struct virtio_pmd_ctrl ctrl;
 781         int dlen[1];
 782         int ret;
 783
 784         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 785                 PMD_INIT_LOG(INFO, "host does not support rx control");
 786                 return;
 787         }
 788
 789         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 790         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 791         ctrl.data[0] = 1;
 792         dlen[0] = 1;
 793
 794         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 795         if (ret)
 796                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
 797 }
 798
 799 static void
 800 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
 801 {
 802         struct virtio_hw *hw = dev->data->dev_private;
 803         struct virtio_pmd_ctrl ctrl;
 804         int dlen[1];
 805         int ret;
 806
 807         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 808                 PMD_INIT_LOG(INFO, "host does not support rx control");
 809                 return;
 810         }
 811
 812         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 813         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 814         ctrl.data[0] = 0;
 815         dlen[0] = 1;
 816
 817         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 818         if (ret)
 819                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
 820 }
 821
 822 #define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
 823 static int
 824 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 825 {
 826         struct virtio_hw *hw = dev->data->dev_private;
 827         uint32_t ether_hdr_len = ETHER_HDR_LEN + VLAN_TAG_LEN +
 828                                  hw->vtnet_hdr_size;
 829         uint32_t frame_size = mtu + ether_hdr_len;
 830         uint32_t max_frame_size = hw->max_mtu + ether_hdr_len;
 831
 832         max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN);
 833
 834         if (mtu < ETHER_MIN_MTU || frame_size > max_frame_size) {
 835                 PMD_INIT_LOG(ERR, "MTU should be between %d and %d",
 836                         ETHER_MIN_MTU, max_frame_size - ether_hdr_len);
 837                 return -EINVAL;
 838         }
 839         return 0;
 840 }
 841
 842 static int
 843 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 844 {
 845         struct virtio_hw *hw = dev->data->dev_private;
 846         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 847         struct virtqueue *vq = rxvq->vq;
 848
 849         virtqueue_enable_intr(vq);
 850         virtio_mb(hw->weak_barriers);
 851         return 0;
 852 }
 853
 854 static int
 855 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 856 {
 857         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 858         struct virtqueue *vq = rxvq->vq;
 859
 860         virtqueue_disable_intr(vq);
 861         return 0;
 862 }
 863
 864 /*
 865  * dev_ops for virtio, bare necessities for basic operation
 866  */
 867 static const struct eth_dev_ops virtio_eth_dev_ops = {
 868         .dev_configure           = virtio_dev_configure,
 869         .dev_start               = virtio_dev_start,
 870         .dev_stop                = virtio_dev_stop,
 871         .dev_close               = virtio_dev_close,
 872         .promiscuous_enable      = virtio_dev_promiscuous_enable,
 873         .promiscuous_disable     = virtio_dev_promiscuous_disable,
 874         .allmulticast_enable     = virtio_dev_allmulticast_enable,
 875         .allmulticast_disable    = virtio_dev_allmulticast_disable,
 876         .mtu_set                 = virtio_mtu_set,
 877         .dev_infos_get           = virtio_dev_info_get,
 878         .stats_get               = virtio_dev_stats_get,
 879         .xstats_get              = virtio_dev_xstats_get,
 880         .xstats_get_names        = virtio_dev_xstats_get_names,
 881         .stats_reset             = virtio_dev_stats_reset,
 882         .xstats_reset            = virtio_dev_stats_reset,
 883         .link_update             = virtio_dev_link_update,
 884         .vlan_offload_set        = virtio_dev_vlan_offload_set,
 885         .rx_queue_setup          = virtio_dev_rx_queue_setup,
 886         .rx_queue_intr_enable    = virtio_dev_rx_queue_intr_enable,
 887         .rx_queue_intr_disable   = virtio_dev_rx_queue_intr_disable,
 888         .rx_queue_release        = virtio_dev_queue_release,
 889         .rx_descriptor_done      = virtio_dev_rx_queue_done,
 890         .tx_queue_setup          = virtio_dev_tx_queue_setup,
 891         .tx_queue_release        = virtio_dev_queue_release,
 892         /* collect stats per queue */
 893         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 894         .vlan_filter_set         = virtio_vlan_filter_set,
 895         .mac_addr_add            = virtio_mac_addr_add,
 896         .mac_addr_remove         = virtio_mac_addr_remove,
 897         .mac_addr_set            = virtio_mac_addr_set,
 898 };
 899
 900 static void
 901 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 902 {
 903         unsigned i;
 904
 905         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 906                 const struct virtnet_tx *txvq = dev->data->tx_queues[i];
 907                 if (txvq == NULL)
 908                         continue;
 909
 910                 stats->opackets += txvq->stats.packets;
 911                 stats->obytes += txvq->stats.bytes;
 912                 stats->oerrors += txvq->stats.errors;
 913
 914                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 915                         stats->q_opackets[i] = txvq->stats.packets;
 916                         stats->q_obytes[i] = txvq->stats.bytes;
 917                 }
 918         }
 919
 920         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 921                 const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
 922                 if (rxvq == NULL)
 923                         continue;
 924
 925                 stats->ipackets += rxvq->stats.packets;
 926                 stats->ibytes += rxvq->stats.bytes;
 927                 stats->ierrors += rxvq->stats.errors;
 928
 929                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 930                         stats->q_ipackets[i] = rxvq->stats.packets;
 931                         stats->q_ibytes[i] = rxvq->stats.bytes;
 932                 }
 933         }
 934
 935         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
 936 }
 937
 938 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
 939                                        struct rte_eth_xstat_name *xstats_names,
 940                                        __rte_unused unsigned limit)
 941 {
 942         unsigned i;
 943         unsigned count = 0;
 944         unsigned t;
 945
 946         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
 947                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
 948
 949         if (xstats_names != NULL) {
 950                 /* Note: limit checked in rte_eth_xstats_names() */
 951
 952                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 953                         struct virtnet_rx *rxvq = dev->data->rx_queues[i];
 954                         if (rxvq == NULL)
 955                                 continue;
 956                         for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
 957                                 snprintf(xstats_names[count].name,
 958                                         sizeof(xstats_names[count].name),
 959                                         "rx_q%u_%s", i,
 960                                         rte_virtio_rxq_stat_strings[t].name);
 961                                 count++;
 962                         }
 963                 }
 964
 965                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
 966                         struct virtnet_tx *txvq = dev->data->tx_queues[i];
 967                         if (txvq == NULL)
 968                                 continue;
 969                         for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
 970                                 snprintf(xstats_names[count].name,
 971                                         sizeof(xstats_names[count].name),
 972                                         "tx_q%u_%s", i,
 973                                         rte_virtio_txq_stat_strings[t].name);
 974                                 count++;
 975                         }
 976                 }
 977                 return count;
 978         }
 979         return nstats;
 980 }
 981
 982 static int
 983 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 984                       unsigned n)
 985 {
 986         unsigned i;
 987         unsigned count = 0;
 988
 989         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
 990                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
 991
 992         if (n < nstats)
 993                 return nstats;
 994
 995         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 996                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
 997
 998                 if (rxvq == NULL)
 999                         continue;
1000
1001                 unsigned t;
1002
1003                 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1004                         xstats[count].value = *(uint64_t *)(((char *)rxvq) +
1005                                 rte_virtio_rxq_stat_strings[t].offset);
1006                         xstats[count].id = count;
1007                         count++;
1008                 }
1009         }
1010
1011         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1012                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1013
1014                 if (txvq == NULL)
1015                         continue;
1016
1017                 unsigned t;
1018
1019                 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1020                         xstats[count].value = *(uint64_t *)(((char *)txvq) +
1021                                 rte_virtio_txq_stat_strings[t].offset);
1022                         xstats[count].id = count;
1023                         count++;
1024                 }
1025         }
1026
1027         return count;
1028 }
1029
1030 static int
1031 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1032 {
1033         virtio_update_stats(dev, stats);
1034
1035         return 0;
1036 }
1037
1038 static void
1039 virtio_dev_stats_reset(struct rte_eth_dev *dev)
1040 {
1041         unsigned int i;
1042
1043         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1044                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1045                 if (txvq == NULL)
1046                         continue;
1047
1048                 txvq->stats.packets = 0;
1049                 txvq->stats.bytes = 0;
1050                 txvq->stats.errors = 0;
1051                 txvq->stats.multicast = 0;
1052                 txvq->stats.broadcast = 0;
1053                 memset(txvq->stats.size_bins, 0,
1054                        sizeof(txvq->stats.size_bins[0]) * 8);
1055         }
1056
1057         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1058                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1059                 if (rxvq == NULL)
1060                         continue;
1061
1062                 rxvq->stats.packets = 0;
1063                 rxvq->stats.bytes = 0;
1064                 rxvq->stats.errors = 0;
1065                 rxvq->stats.multicast = 0;
1066                 rxvq->stats.broadcast = 0;
1067                 memset(rxvq->stats.size_bins, 0,
1068                        sizeof(rxvq->stats.size_bins[0]) * 8);
1069         }
1070 }
1071
1072 static void
1073 virtio_set_hwaddr(struct virtio_hw *hw)
1074 {
1075         vtpci_write_dev_config(hw,
1076                         offsetof(struct virtio_net_config, mac),
1077                         &hw->mac_addr, ETHER_ADDR_LEN);
1078 }
1079
1080 static void
1081 virtio_get_hwaddr(struct virtio_hw *hw)
1082 {
1083         if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) {
1084                 vtpci_read_dev_config(hw,
1085                         offsetof(struct virtio_net_config, mac),
1086                         &hw->mac_addr, ETHER_ADDR_LEN);
1087         } else {
1088                 eth_random_addr(&hw->mac_addr[0]);
1089                 virtio_set_hwaddr(hw);
1090         }
1091 }
1092
1093 static int
1094 virtio_mac_table_set(struct virtio_hw *hw,
1095                      const struct virtio_net_ctrl_mac *uc,
1096                      const struct virtio_net_ctrl_mac *mc)
1097 {
1098         struct virtio_pmd_ctrl ctrl;
1099         int err, len[2];
1100
1101         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1102                 PMD_DRV_LOG(INFO, "host does not support mac table");
1103                 return -1;
1104         }
1105
1106         ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1107         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1108
1109         len[0] = uc->entries * ETHER_ADDR_LEN + sizeof(uc->entries);
1110         memcpy(ctrl.data, uc, len[0]);
1111
1112         len[1] = mc->entries * ETHER_ADDR_LEN + sizeof(mc->entries);
1113         memcpy(ctrl.data + len[0], mc, len[1]);
1114
1115         err = virtio_send_command(hw->cvq, &ctrl, len, 2);
1116         if (err != 0)
1117                 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
1118         return err;
1119 }
1120
1121 static int
1122 virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1123                     uint32_t index, uint32_t vmdq __rte_unused)
1124 {
1125         struct virtio_hw *hw = dev->data->dev_private;
1126         const struct ether_addr *addrs = dev->data->mac_addrs;
1127         unsigned int i;
1128         struct virtio_net_ctrl_mac *uc, *mc;
1129
1130         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1131                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1132                 return -EINVAL;
1133         }
1134
1135         uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries));
1136         uc->entries = 0;
1137         mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries));
1138         mc->entries = 0;
1139
1140         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1141                 const struct ether_addr *addr
1142                         = (i == index) ? mac_addr : addrs + i;
1143                 struct virtio_net_ctrl_mac *tbl
1144                         = is_multicast_ether_addr(addr) ? mc : uc;
1145
1146                 memcpy(&tbl->macs[tbl->entries++], addr, ETHER_ADDR_LEN);
1147         }
1148
1149         return virtio_mac_table_set(hw, uc, mc);
1150 }
1151
1152 static void
1153 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1154 {
1155         struct virtio_hw *hw = dev->data->dev_private;
1156         struct ether_addr *addrs = dev->data->mac_addrs;
1157         struct virtio_net_ctrl_mac *uc, *mc;
1158         unsigned int i;
1159
1160         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1161                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1162                 return;
1163         }
1164
1165         uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries));
1166         uc->entries = 0;
1167         mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries));
1168         mc->entries = 0;
1169
1170         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1171                 struct virtio_net_ctrl_mac *tbl;
1172
1173                 if (i == index || is_zero_ether_addr(addrs + i))
1174                         continue;
1175
1176                 tbl = is_multicast_ether_addr(addrs + i) ? mc : uc;
1177                 memcpy(&tbl->macs[tbl->entries++], addrs + i, ETHER_ADDR_LEN);
1178         }
1179
1180         virtio_mac_table_set(hw, uc, mc);
1181 }
1182
1183 static int
1184 virtio_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1185 {
1186         struct virtio_hw *hw = dev->data->dev_private;
1187
1188         memcpy(hw->mac_addr, mac_addr, ETHER_ADDR_LEN);
1189
1190         /* Use atomic update if available */
1191         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1192                 struct virtio_pmd_ctrl ctrl;
1193                 int len = ETHER_ADDR_LEN;
1194
1195                 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1196                 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1197
1198                 memcpy(ctrl.data, mac_addr, ETHER_ADDR_LEN);
1199                 return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1200         }
1201
1202         if (!vtpci_with_feature(hw, VIRTIO_NET_F_MAC))
1203                 return -ENOTSUP;
1204
1205         virtio_set_hwaddr(hw);
1206         return 0;
1207 }
1208
1209 static int
1210 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1211 {
1212         struct virtio_hw *hw = dev->data->dev_private;
1213         struct virtio_pmd_ctrl ctrl;
1214         int len;
1215
1216         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1217                 return -ENOTSUP;
1218
1219         ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1220         ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1221         memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1222         len = sizeof(vlan_id);
1223
1224         return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1225 }
1226
1227 static int
1228 virtio_intr_enable(struct rte_eth_dev *dev)
1229 {
1230         struct virtio_hw *hw = dev->data->dev_private;
1231
1232         if (rte_intr_enable(dev->intr_handle) < 0)
1233                 return -1;
1234
1235         if (!hw->virtio_user_dev)
1236                 hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev));
1237
1238         return 0;
1239 }
1240
1241 static int
1242 virtio_intr_disable(struct rte_eth_dev *dev)
1243 {
1244         struct virtio_hw *hw = dev->data->dev_private;
1245
1246         if (rte_intr_disable(dev->intr_handle) < 0)
1247                 return -1;
1248
1249         if (!hw->virtio_user_dev)
1250                 hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev));
1251
1252         return 0;
1253 }
1254
1255 static int
1256 virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1257 {
1258         uint64_t host_features;
1259
1260         /* Prepare guest_features: feature that driver wants to support */
1261         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1262                 req_features);
1263
1264         /* Read device(host) feature bits */
1265         host_features = VTPCI_OPS(hw)->get_features(hw);
1266         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1267                 host_features);
1268
1269         /* If supported, ensure MTU value is valid before acknowledging it. */
1270         if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) {
1271                 struct virtio_net_config config;
1272
1273                 vtpci_read_dev_config(hw,
1274                         offsetof(struct virtio_net_config, mtu),
1275                         &config.mtu, sizeof(config.mtu));
1276
1277                 if (config.mtu < ETHER_MIN_MTU)
1278                         req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
1279         }
1280
1281         /*
1282          * Negotiate features: Subset of device feature bits are written back
1283          * guest feature bits.
1284          */
1285         hw->guest_features = req_features;
1286         hw->guest_features = vtpci_negotiate_features(hw, host_features);
1287         PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1288                 hw->guest_features);
1289
1290         if (hw->modern) {
1291                 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
1292                         PMD_INIT_LOG(ERR,
1293                                 "VIRTIO_F_VERSION_1 features is not enabled.");
1294                         return -1;
1295                 }
1296                 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1297                 if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1298                         PMD_INIT_LOG(ERR,
1299                                 "failed to set FEATURES_OK status!");
1300                         return -1;
1301                 }
1302         }
1303
1304         hw->req_guest_features = req_features;
1305
1306         return 0;
1307 }
1308
1309 int
1310 virtio_dev_pause(struct rte_eth_dev *dev)
1311 {
1312         struct virtio_hw *hw = dev->data->dev_private;
1313
1314         rte_spinlock_lock(&hw->state_lock);
1315
1316         if (hw->started == 0) {
1317                 /* Device is just stopped. */
1318                 rte_spinlock_unlock(&hw->state_lock);
1319                 return -1;
1320         }
1321         hw->started = 0;
1322         /*
1323          * Prevent the worker threads from touching queues to avoid contention,
1324          * 1 ms should be enough for the ongoing Tx function to finish.
1325          */
1326         rte_delay_ms(1);
1327         return 0;
1328 }
1329
1330 /*
1331  * Recover hw state to let the worker threads continue.
1332  */
1333 void
1334 virtio_dev_resume(struct rte_eth_dev *dev)
1335 {
1336         struct virtio_hw *hw = dev->data->dev_private;
1337
1338         hw->started = 1;
1339         rte_spinlock_unlock(&hw->state_lock);
1340 }
1341
1342 /*
1343  * Should be called only after device is paused.
1344  */
1345 int
1346 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts,
1347                 int nb_pkts)
1348 {
1349         struct virtio_hw *hw = dev->data->dev_private;
1350         struct virtnet_tx *txvq = dev->data->tx_queues[0];
1351         int ret;
1352
1353         hw->inject_pkts = tx_pkts;
1354         ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts);
1355         hw->inject_pkts = NULL;
1356
1357         return ret;
1358 }
1359
1360 static void
1361 virtio_notify_peers(struct rte_eth_dev *dev)
1362 {
1363         struct virtio_hw *hw = dev->data->dev_private;
1364         struct virtnet_rx *rxvq;
1365         struct rte_mbuf *rarp_mbuf;
1366
1367         if (!dev->data->rx_queues)
1368                 return;
1369
1370         rxvq = dev->data->rx_queues[0];
1371         if (!rxvq)
1372                 return;
1373
1374         rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool,
1375                         (struct ether_addr *)hw->mac_addr);
1376         if (rarp_mbuf == NULL) {
1377                 PMD_DRV_LOG(ERR, "failed to make RARP packet.");
1378                 return;
1379         }
1380
1381         /* If virtio port just stopped, no need to send RARP */
1382         if (virtio_dev_pause(dev) < 0) {
1383                 rte_pktmbuf_free(rarp_mbuf);
1384                 return;
1385         }
1386
1387         virtio_inject_pkts(dev, &rarp_mbuf, 1);
1388         virtio_dev_resume(dev);
1389 }
1390
1391 static void
1392 virtio_ack_link_announce(struct rte_eth_dev *dev)
1393 {
1394         struct virtio_hw *hw = dev->data->dev_private;
1395         struct virtio_pmd_ctrl ctrl;
1396
1397         ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE;
1398         ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK;
1399
1400         virtio_send_command(hw->cvq, &ctrl, NULL, 0);
1401 }
1402
1403 /*
1404  * Process virtio config changed interrupt. Call the callback
1405  * if link state changed, generate gratuitous RARP packet if
1406  * the status indicates an ANNOUNCE.
1407  */
1408 void
1409 virtio_interrupt_handler(void *param)
1410 {
1411         struct rte_eth_dev *dev = param;
1412         struct virtio_hw *hw = dev->data->dev_private;
1413         uint8_t isr;
1414         uint16_t status;
1415
1416         /* Read interrupt status which clears interrupt */
1417         isr = vtpci_isr(hw);
1418         PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1419
1420         if (virtio_intr_enable(dev) < 0)
1421                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1422
1423         if (isr & VIRTIO_PCI_ISR_CONFIG) {
1424                 if (virtio_dev_link_update(dev, 0) == 0)
1425                         _rte_eth_dev_callback_process(dev,
1426                                                       RTE_ETH_EVENT_INTR_LSC,
1427                                                       NULL);
1428
1429                 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1430                         vtpci_read_dev_config(hw,
1431                                 offsetof(struct virtio_net_config, status),
1432                                 &status, sizeof(status));
1433                         if (status & VIRTIO_NET_S_ANNOUNCE) {
1434                                 virtio_notify_peers(dev);
1435                                 if (hw->cvq)
1436                                         virtio_ack_link_announce(dev);
1437                         }
1438                 }
1439         }
1440 }
1441
1442 /* set rx and tx handlers according to what is supported */
1443 static void
1444 set_rxtx_funcs(struct rte_eth_dev *eth_dev)
1445 {
1446         struct virtio_hw *hw = eth_dev->data->dev_private;
1447
1448         if (vtpci_packed_queue(hw)) {
1449                 PMD_INIT_LOG(INFO,
1450                         "virtio: using packed ring %s Tx path on port %u",
1451                         hw->use_inorder_tx ? "inorder" : "standard",
1452                         eth_dev->data->port_id);
1453                 eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
1454         } else {
1455                 if (hw->use_inorder_tx) {
1456                         PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u",
1457                                 eth_dev->data->port_id);
1458                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder;
1459                 } else {
1460                         PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u",
1461                                 eth_dev->data->port_id);
1462                         eth_dev->tx_pkt_burst = virtio_xmit_pkts;
1463                 }
1464         }
1465
1466         if (vtpci_packed_queue(hw)) {
1467                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1468                         PMD_INIT_LOG(INFO,
1469                                 "virtio: using packed ring mergeable buffer Rx path on port %u",
1470                                 eth_dev->data->port_id);
1471                         eth_dev->rx_pkt_burst =
1472                                 &virtio_recv_mergeable_pkts_packed;
1473                 } else {
1474                         PMD_INIT_LOG(INFO,
1475                                 "virtio: using packed ring standard Rx path on port %u",
1476                                 eth_dev->data->port_id);
1477                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
1478                 }
1479         } else {
1480                 if (hw->use_simple_rx) {
1481                         PMD_INIT_LOG(INFO, "virtio: using simple Rx path on port %u",
1482                                 eth_dev->data->port_id);
1483                         eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
1484                 } else if (hw->use_inorder_rx) {
1485                         PMD_INIT_LOG(INFO,
1486                                 "virtio: using inorder Rx path on port %u",
1487                                 eth_dev->data->port_id);
1488                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder;
1489                 } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1490                         PMD_INIT_LOG(INFO,
1491                                 "virtio: using mergeable buffer Rx path on port %u",
1492                                 eth_dev->data->port_id);
1493                         eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1494                 } else {
1495                         PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
1496                                 eth_dev->data->port_id);
1497                         eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1498                 }
1499         }
1500
1501 }
1502
1503 /* Only support 1:1 queue/interrupt mapping so far.
1504  * TODO: support n:1 queue/interrupt mapping when there are limited number of
1505  * interrupt vectors (<N+1).
1506  */
1507 static int
1508 virtio_queues_bind_intr(struct rte_eth_dev *dev)
1509 {
1510         uint32_t i;
1511         struct virtio_hw *hw = dev->data->dev_private;
1512
1513         PMD_INIT_LOG(INFO, "queue/interrupt binding");
1514         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
1515                 dev->intr_handle->intr_vec[i] = i + 1;
1516                 if (VTPCI_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) ==
1517                                                  VIRTIO_MSI_NO_VECTOR) {
1518                         PMD_DRV_LOG(ERR, "failed to set queue vector");
1519                         return -EBUSY;
1520                 }
1521         }
1522
1523         return 0;
1524 }
1525
1526 static void
1527 virtio_queues_unbind_intr(struct rte_eth_dev *dev)
1528 {
1529         uint32_t i;
1530         struct virtio_hw *hw = dev->data->dev_private;
1531
1532         PMD_INIT_LOG(INFO, "queue/interrupt unbinding");
1533         for (i = 0; i < dev->data->nb_rx_queues; ++i)
1534                 VTPCI_OPS(hw)->set_queue_irq(hw,
1535                                              hw->vqs[i * VTNET_CQ],
1536                                              VIRTIO_MSI_NO_VECTOR);
1537 }
1538
1539 static int
1540 virtio_configure_intr(struct rte_eth_dev *dev)
1541 {
1542         struct virtio_hw *hw = dev->data->dev_private;
1543
1544         if (!rte_intr_cap_multiple(dev->intr_handle)) {
1545                 PMD_INIT_LOG(ERR, "Multiple intr vector not supported");
1546                 return -ENOTSUP;
1547         }
1548
1549         if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) {
1550                 PMD_INIT_LOG(ERR, "Fail to create eventfd");
1551                 return -1;
1552         }
1553
1554         if (!dev->intr_handle->intr_vec) {
1555                 dev->intr_handle->intr_vec =
1556                         rte_zmalloc("intr_vec",
1557                                     hw->max_queue_pairs * sizeof(int), 0);
1558                 if (!dev->intr_handle->intr_vec) {
1559                         PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors",
1560                                      hw->max_queue_pairs);
1561                         return -ENOMEM;
1562                 }
1563         }
1564
1565         /* Re-register callback to update max_intr */
1566         rte_intr_callback_unregister(dev->intr_handle,
1567                                      virtio_interrupt_handler,
1568                                      dev);
1569         rte_intr_callback_register(dev->intr_handle,
1570                                    virtio_interrupt_handler,
1571                                    dev);
1572
1573         /* DO NOT try to remove this! This function will enable msix, or QEMU
1574          * will encounter SIGSEGV when DRIVER_OK is sent.
1575          * And for legacy devices, this should be done before queue/vec binding
1576          * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1577          * (22) will be ignored.
1578          */
1579         if (virtio_intr_enable(dev) < 0) {
1580                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1581                 return -1;
1582         }
1583
1584         if (virtio_queues_bind_intr(dev) < 0) {
1585                 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt");
1586                 return -1;
1587         }
1588
1589         return 0;
1590 }
1591
1592 /* reset device and renegotiate features if needed */
1593 static int
1594 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
1595 {
1596         struct virtio_hw *hw = eth_dev->data->dev_private;
1597         struct virtio_net_config *config;
1598         struct virtio_net_config local_config;
1599         struct rte_pci_device *pci_dev = NULL;
1600         int ret;
1601
1602         /* Reset the device although not necessary at startup */
1603         vtpci_reset(hw);
1604
1605         if (hw->vqs) {
1606                 virtio_dev_free_mbufs(eth_dev);
1607                 virtio_free_queues(hw);
1608         }
1609
1610         /* Tell the host we've noticed this device. */
1611         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1612
1613         /* Tell the host we've known how to drive the device. */
1614         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1615         if (virtio_negotiate_features(hw, req_features) < 0)
1616                 return -1;
1617
1618         hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
1619
1620         if (!hw->virtio_user_dev) {
1621                 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1622                 rte_eth_copy_pci_info(eth_dev, pci_dev);
1623         }
1624
1625         /* If host does not support both status and MSI-X then disable LSC */
1626         if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS) &&
1627             hw->use_msix != VIRTIO_MSIX_NONE)
1628                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
1629         else
1630                 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
1631
1632         /* Setting up rx_header size for the device */
1633         if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
1634             vtpci_with_feature(hw, VIRTIO_F_VERSION_1) ||
1635             vtpci_with_feature(hw, VIRTIO_F_RING_PACKED))
1636                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1637         else
1638                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
1639
1640         /* Copy the permanent MAC address to: virtio_hw */
1641         virtio_get_hwaddr(hw);
1642         ether_addr_copy((struct ether_addr *) hw->mac_addr,
1643                         &eth_dev->data->mac_addrs[0]);
1644         PMD_INIT_LOG(DEBUG,
1645                      "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1646                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1647                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1648
1649         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1650                 config = &local_config;
1651
1652                 vtpci_read_dev_config(hw,
1653                         offsetof(struct virtio_net_config, mac),
1654                         &config->mac, sizeof(config->mac));
1655
1656                 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1657                         vtpci_read_dev_config(hw,
1658                                 offsetof(struct virtio_net_config, status),
1659                                 &config->status, sizeof(config->status));
1660                 } else {
1661                         PMD_INIT_LOG(DEBUG,
1662                                      "VIRTIO_NET_F_STATUS is not supported");
1663                         config->status = 0;
1664                 }
1665
1666                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) {
1667                         vtpci_read_dev_config(hw,
1668                                 offsetof(struct virtio_net_config, max_virtqueue_pairs),
1669                                 &config->max_virtqueue_pairs,
1670                                 sizeof(config->max_virtqueue_pairs));
1671                 } else {
1672                         PMD_INIT_LOG(DEBUG,
1673                                      "VIRTIO_NET_F_MQ is not supported");
1674                         config->max_virtqueue_pairs = 1;
1675                 }
1676
1677                 hw->max_queue_pairs = config->max_virtqueue_pairs;
1678
1679                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MTU)) {
1680                         vtpci_read_dev_config(hw,
1681                                 offsetof(struct virtio_net_config, mtu),
1682                                 &config->mtu,
1683                                 sizeof(config->mtu));
1684
1685                         /*
1686                          * MTU value has already been checked at negotiation
1687                          * time, but check again in case it has changed since
1688                          * then, which should not happen.
1689                          */
1690                         if (config->mtu < ETHER_MIN_MTU) {
1691                                 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)",
1692                                                 config->mtu);
1693                                 return -1;
1694                         }
1695
1696                         hw->max_mtu = config->mtu;
1697                         /* Set initial MTU to maximum one supported by vhost */
1698                         eth_dev->data->mtu = config->mtu;
1699
1700                 } else {
1701                         hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - ETHER_HDR_LEN -
1702                                 VLAN_TAG_LEN - hw->vtnet_hdr_size;
1703                 }
1704
1705                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
1706                                 config->max_virtqueue_pairs);
1707                 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
1708                 PMD_INIT_LOG(DEBUG,
1709                                 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1710                                 config->mac[0], config->mac[1],
1711                                 config->mac[2], config->mac[3],
1712                                 config->mac[4], config->mac[5]);
1713         } else {
1714                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
1715                 hw->max_queue_pairs = 1;
1716                 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - ETHER_HDR_LEN -
1717                         VLAN_TAG_LEN - hw->vtnet_hdr_size;
1718         }
1719
1720         ret = virtio_alloc_queues(eth_dev);
1721         if (ret < 0)
1722                 return ret;
1723
1724         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1725                 if (virtio_configure_intr(eth_dev) < 0) {
1726                         PMD_INIT_LOG(ERR, "failed to configure interrupt");
1727                         return -1;
1728                 }
1729         }
1730
1731         vtpci_reinit_complete(hw);
1732
1733         if (pci_dev)
1734                 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1735                         eth_dev->data->port_id, pci_dev->id.vendor_id,
1736                         pci_dev->id.device_id);
1737
1738         return 0;
1739 }
1740
1741 /*
1742  * Remap the PCI device again (IO port map for legacy device and
1743  * memory map for modern device), so that the secondary process
1744  * could have the PCI initiated correctly.
1745  */
1746 static int
1747 virtio_remap_pci(struct rte_pci_device *pci_dev, struct virtio_hw *hw)
1748 {
1749         if (hw->modern) {
1750                 /*
1751                  * We don't have to re-parse the PCI config space, since
1752                  * rte_pci_map_device() makes sure the mapped address
1753                  * in secondary process would equal to the one mapped in
1754                  * the primary process: error will be returned if that
1755                  * requirement is not met.
1756                  *
1757                  * That said, we could simply reuse all cap pointers
1758                  * (such as dev_cfg, common_cfg, etc.) parsed from the
1759                  * primary process, which is stored in shared memory.
1760                  */
1761                 if (rte_pci_map_device(pci_dev)) {
1762                         PMD_INIT_LOG(DEBUG, "failed to map pci device!");
1763                         return -1;
1764                 }
1765         } else {
1766                 if (rte_pci_ioport_map(pci_dev, 0, VTPCI_IO(hw)) < 0)
1767                         return -1;
1768         }
1769
1770         return 0;
1771 }
1772
1773 static void
1774 virtio_set_vtpci_ops(struct virtio_hw *hw)
1775 {
1776 #ifdef RTE_VIRTIO_USER
1777         if (hw->virtio_user_dev)
1778                 VTPCI_OPS(hw) = &virtio_user_ops;
1779         else
1780 #endif
1781         if (hw->modern)
1782                 VTPCI_OPS(hw) = &modern_ops;
1783         else
1784                 VTPCI_OPS(hw) = &legacy_ops;
1785 }
1786
1787 /*
1788  * This function is based on probe() function in virtio_pci.c
1789  * It returns 0 on success.
1790  */
1791 int
1792 eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
1793 {
1794         struct virtio_hw *hw = eth_dev->data->dev_private;
1795         int ret;
1796
1797         RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr_mrg_rxbuf));
1798
1799         eth_dev->dev_ops = &virtio_eth_dev_ops;
1800
1801         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1802                 if (!hw->virtio_user_dev) {
1803                         ret = virtio_remap_pci(RTE_ETH_DEV_TO_PCI(eth_dev), hw);
1804                         if (ret)
1805                                 return ret;
1806                 }
1807
1808                 virtio_set_vtpci_ops(hw);
1809                 set_rxtx_funcs(eth_dev);
1810
1811                 return 0;
1812         }
1813
1814         /* Allocate memory for storing MAC addresses */
1815         eth_dev->data->mac_addrs = rte_zmalloc("virtio", VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN, 0);
1816         if (eth_dev->data->mac_addrs == NULL) {
1817                 PMD_INIT_LOG(ERR,
1818                         "Failed to allocate %d bytes needed to store MAC addresses",
1819                         VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN);
1820                 return -ENOMEM;
1821         }
1822
1823         hw->port_id = eth_dev->data->port_id;
1824         /* For virtio_user case the hw->virtio_user_dev is populated by
1825          * virtio_user_eth_dev_alloc() before eth_virtio_dev_init() is called.
1826          */
1827         if (!hw->virtio_user_dev) {
1828                 ret = vtpci_init(RTE_ETH_DEV_TO_PCI(eth_dev), hw);
1829                 if (ret)
1830                         goto out;
1831         }
1832
1833         /* reset device and negotiate default features */
1834         ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
1835         if (ret < 0)
1836                 goto out;
1837
1838         return 0;
1839
1840 out:
1841         rte_free(eth_dev->data->mac_addrs);
1842         return ret;
1843 }
1844
1845 static int
1846 eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev)
1847 {
1848         PMD_INIT_FUNC_TRACE();
1849
1850         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
1851                 return 0;
1852
1853         virtio_dev_stop(eth_dev);
1854         virtio_dev_close(eth_dev);
1855
1856         eth_dev->dev_ops = NULL;
1857         eth_dev->tx_pkt_burst = NULL;
1858         eth_dev->rx_pkt_burst = NULL;
1859
1860         if (eth_dev->device)
1861                 rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev));
1862
1863         PMD_INIT_LOG(DEBUG, "dev_uninit completed");
1864
1865         return 0;
1866 }
1867
1868 static int vdpa_check_handler(__rte_unused const char *key,
1869                 const char *value, __rte_unused void *opaque)
1870 {
1871         if (strcmp(value, "1"))
1872                 return -1;
1873
1874         return 0;
1875 }
1876
1877 static int
1878 vdpa_mode_selected(struct rte_devargs *devargs)
1879 {
1880         struct rte_kvargs *kvlist;
1881         const char *key = "vdpa";
1882         int ret = 0;
1883
1884         if (devargs == NULL)
1885                 return 0;
1886
1887         kvlist = rte_kvargs_parse(devargs->args, NULL);
1888         if (kvlist == NULL)
1889                 return 0;
1890
1891         if (!rte_kvargs_count(kvlist, key))
1892                 goto exit;
1893
1894         /* vdpa mode selected when there's a key-value pair: vdpa=1 */
1895         if (rte_kvargs_process(kvlist, key,
1896                                 vdpa_check_handler, NULL) < 0) {
1897                 goto exit;
1898         }
1899         ret = 1;
1900
1901 exit:
1902         rte_kvargs_free(kvlist);
1903         return ret;
1904 }
1905
1906 static int eth_virtio_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1907         struct rte_pci_device *pci_dev)
1908 {
1909         if (rte_eal_iopl_init() != 0) {
1910                 PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD");
1911                 return 1;
1912         }
1913
1914         /* virtio pmd skips probe if device needs to work in vdpa mode */
1915         if (vdpa_mode_selected(pci_dev->device.devargs))
1916                 return 1;
1917
1918         return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct virtio_hw),
1919                 eth_virtio_dev_init);
1920 }
1921
1922 static int eth_virtio_pci_remove(struct rte_pci_device *pci_dev)
1923 {
1924         return rte_eth_dev_pci_generic_remove(pci_dev, eth_virtio_dev_uninit);
1925 }
1926
1927 static struct rte_pci_driver rte_virtio_pmd = {
1928         .driver = {
1929                 .name = "net_virtio",
1930         },
1931         .id_table = pci_id_virtio_map,
1932         .drv_flags = 0,
1933         .probe = eth_virtio_pci_probe,
1934         .remove = eth_virtio_pci_remove,
1935 };
1936
1937 RTE_INIT(rte_virtio_pmd_init)
1938 {
1939         rte_eal_iopl_init();
1940         rte_pci_register(&rte_virtio_pmd);
1941 }
1942
1943 static bool
1944 rx_offload_enabled(struct virtio_hw *hw)
1945 {
1946         return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
1947                 vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
1948                 vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
1949 }
1950
1951 static bool
1952 tx_offload_enabled(struct virtio_hw *hw)
1953 {
1954         return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) ||
1955                 vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
1956                 vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
1957 }
1958
1959 /*
1960  * Configure virtio device
1961  * It returns 0 on success.
1962  */
1963 static int
1964 virtio_dev_configure(struct rte_eth_dev *dev)
1965 {
1966         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
1967         const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
1968         struct virtio_hw *hw = dev->data->dev_private;
1969         uint32_t ether_hdr_len = ETHER_HDR_LEN + VLAN_TAG_LEN +
1970                 hw->vtnet_hdr_size;
1971         uint64_t rx_offloads = rxmode->offloads;
1972         uint64_t tx_offloads = txmode->offloads;
1973         uint64_t req_features;
1974         int ret;
1975
1976         PMD_INIT_LOG(DEBUG, "configure");
1977         req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
1978
1979         if (dev->data->dev_conf.intr_conf.rxq) {
1980                 ret = virtio_init_device(dev, hw->req_guest_features);
1981                 if (ret < 0)
1982                         return ret;
1983         }
1984
1985         if (rxmode->max_rx_pkt_len > hw->max_mtu + ether_hdr_len)
1986                 req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
1987
1988         if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
1989                            DEV_RX_OFFLOAD_TCP_CKSUM))
1990                 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
1991
1992         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO)
1993                 req_features |=
1994                         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
1995                         (1ULL << VIRTIO_NET_F_GUEST_TSO6);
1996
1997         if (tx_offloads & (DEV_TX_OFFLOAD_UDP_CKSUM |
1998                            DEV_TX_OFFLOAD_TCP_CKSUM))
1999                 req_features |= (1ULL << VIRTIO_NET_F_CSUM);
2000
2001         if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO)
2002                 req_features |=
2003                         (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2004                         (1ULL << VIRTIO_NET_F_HOST_TSO6);
2005
2006         /* if request features changed, reinit the device */
2007         if (req_features != hw->req_guest_features) {
2008                 ret = virtio_init_device(dev, req_features);
2009                 if (ret < 0)
2010                         return ret;
2011         }
2012
2013         if ((rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2014                             DEV_RX_OFFLOAD_TCP_CKSUM)) &&
2015                 !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
2016                 PMD_DRV_LOG(ERR,
2017                         "rx checksum not available on this host");
2018                 return -ENOTSUP;
2019         }
2020
2021         if ((rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) &&
2022                 (!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2023                  !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) {
2024                 PMD_DRV_LOG(ERR,
2025                         "Large Receive Offload not available on this host");
2026                 return -ENOTSUP;
2027         }
2028
2029         /* start control queue */
2030         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
2031                 virtio_dev_cq_start(dev);
2032
2033         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2034                 hw->vlan_strip = 1;
2035
2036         if ((rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
2037             && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2038                 PMD_DRV_LOG(ERR,
2039                             "vlan filtering not available on this host");
2040                 return -ENOTSUP;
2041         }
2042
2043         hw->has_tx_offload = tx_offload_enabled(hw);
2044         hw->has_rx_offload = rx_offload_enabled(hw);
2045
2046         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2047                 /* Enable vector (0) for Link State Intrerrupt */
2048                 if (VTPCI_OPS(hw)->set_config_irq(hw, 0) ==
2049                                 VIRTIO_MSI_NO_VECTOR) {
2050                         PMD_DRV_LOG(ERR, "failed to set config vector");
2051                         return -EBUSY;
2052                 }
2053
2054         rte_spinlock_init(&hw->state_lock);
2055
2056         hw->use_simple_rx = 1;
2057
2058         if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
2059                 hw->use_inorder_tx = 1;
2060                 hw->use_inorder_rx = 1;
2061                 hw->use_simple_rx = 0;
2062         }
2063
2064         if (vtpci_packed_queue(hw)) {
2065                 hw->use_simple_rx = 0;
2066                 hw->use_inorder_rx = 0;
2067         }
2068
2069 #if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM
2070         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
2071                 hw->use_simple_rx = 0;
2072         }
2073 #endif
2074         if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2075                  hw->use_simple_rx = 0;
2076         }
2077
2078         if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2079                            DEV_RX_OFFLOAD_TCP_CKSUM |
2080                            DEV_RX_OFFLOAD_TCP_LRO |
2081                            DEV_RX_OFFLOAD_VLAN_STRIP))
2082                 hw->use_simple_rx = 0;
2083
2084         hw->opened = true;
2085
2086         return 0;
2087 }
2088
2089
2090 static int
2091 virtio_dev_start(struct rte_eth_dev *dev)
2092 {
2093         uint16_t nb_queues, i;
2094         struct virtnet_rx *rxvq;
2095         struct virtnet_tx *txvq __rte_unused;
2096         struct virtio_hw *hw = dev->data->dev_private;
2097         int ret;
2098
2099         /* Finish the initialization of the queues */
2100         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2101                 ret = virtio_dev_rx_queue_setup_finish(dev, i);
2102                 if (ret < 0)
2103                         return ret;
2104         }
2105         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2106                 ret = virtio_dev_tx_queue_setup_finish(dev, i);
2107                 if (ret < 0)
2108                         return ret;
2109         }
2110
2111         /* check if lsc interrupt feature is enabled */
2112         if (dev->data->dev_conf.intr_conf.lsc) {
2113                 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
2114                         PMD_DRV_LOG(ERR, "link status not supported by host");
2115                         return -ENOTSUP;
2116                 }
2117         }
2118
2119         /* Enable uio/vfio intr/eventfd mapping: althrough we already did that
2120          * in device configure, but it could be unmapped  when device is
2121          * stopped.
2122          */
2123         if (dev->data->dev_conf.intr_conf.lsc ||
2124             dev->data->dev_conf.intr_conf.rxq) {
2125                 virtio_intr_disable(dev);
2126
2127                 /* Setup interrupt callback  */
2128                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2129                         rte_intr_callback_register(dev->intr_handle,
2130                                                    virtio_interrupt_handler,
2131                                                    dev);
2132
2133                 if (virtio_intr_enable(dev) < 0) {
2134                         PMD_DRV_LOG(ERR, "interrupt enable failed");
2135                         return -EIO;
2136                 }
2137         }
2138
2139         /*Notify the backend
2140          *Otherwise the tap backend might already stop its queue due to fullness.
2141          *vhost backend will have no chance to be waked up
2142          */
2143         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
2144         if (hw->max_queue_pairs > 1) {
2145                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
2146                         return -EINVAL;
2147         }
2148
2149         PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
2150
2151         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2152                 rxvq = dev->data->rx_queues[i];
2153                 /* Flush the old packets */
2154                 virtqueue_rxvq_flush(rxvq->vq);
2155                 virtqueue_notify(rxvq->vq);
2156         }
2157
2158         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2159                 txvq = dev->data->tx_queues[i];
2160                 virtqueue_notify(txvq->vq);
2161         }
2162
2163         PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
2164
2165         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2166                 rxvq = dev->data->rx_queues[i];
2167                 VIRTQUEUE_DUMP(rxvq->vq);
2168         }
2169
2170         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2171                 txvq = dev->data->tx_queues[i];
2172                 VIRTQUEUE_DUMP(txvq->vq);
2173         }
2174
2175         set_rxtx_funcs(dev);
2176         hw->started = true;
2177
2178         /* Initialize Link state */
2179         virtio_dev_link_update(dev, 0);
2180
2181         return 0;
2182 }
2183
2184 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
2185 {
2186         struct virtio_hw *hw = dev->data->dev_private;
2187         uint16_t nr_vq = virtio_get_nr_vq(hw);
2188         const char *type __rte_unused;
2189         unsigned int i, mbuf_num = 0;
2190         struct virtqueue *vq;
2191         struct rte_mbuf *buf;
2192         int queue_type;
2193
2194         if (hw->vqs == NULL)
2195                 return;
2196
2197         for (i = 0; i < nr_vq; i++) {
2198                 vq = hw->vqs[i];
2199                 if (!vq)
2200                         continue;
2201
2202                 queue_type = virtio_get_queue_type(hw, i);
2203                 if (queue_type == VTNET_RQ)
2204                         type = "rxq";
2205                 else if (queue_type == VTNET_TQ)
2206                         type = "txq";
2207                 else
2208                         continue;
2209
2210                 PMD_INIT_LOG(DEBUG,
2211                         "Before freeing %s[%d] used and unused buf",
2212                         type, i);
2213                 VIRTQUEUE_DUMP(vq);
2214
2215                 while ((buf = virtqueue_detach_unused(vq)) != NULL) {
2216                         rte_pktmbuf_free(buf);
2217                         mbuf_num++;
2218                 }
2219
2220                 PMD_INIT_LOG(DEBUG,
2221                         "After freeing %s[%d] used and unused buf",
2222                         type, i);
2223                 VIRTQUEUE_DUMP(vq);
2224         }
2225
2226         PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num);
2227 }
2228
2229 /*
2230  * Stop device: disable interrupt and mark link down
2231  */
2232 static void
2233 virtio_dev_stop(struct rte_eth_dev *dev)
2234 {
2235         struct virtio_hw *hw = dev->data->dev_private;
2236         struct rte_eth_link link;
2237         struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
2238
2239         PMD_INIT_LOG(DEBUG, "stop");
2240
2241         rte_spinlock_lock(&hw->state_lock);
2242         if (!hw->started)
2243                 goto out_unlock;
2244         hw->started = false;
2245
2246         if (intr_conf->lsc || intr_conf->rxq) {
2247                 virtio_intr_disable(dev);
2248
2249                 /* Reset interrupt callback  */
2250                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
2251                         rte_intr_callback_unregister(dev->intr_handle,
2252                                                      virtio_interrupt_handler,
2253                                                      dev);
2254                 }
2255         }
2256
2257         memset(&link, 0, sizeof(link));
2258         rte_eth_linkstatus_set(dev, &link);
2259 out_unlock:
2260         rte_spinlock_unlock(&hw->state_lock);
2261 }
2262
2263 static int
2264 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
2265 {
2266         struct rte_eth_link link;
2267         uint16_t status;
2268         struct virtio_hw *hw = dev->data->dev_private;
2269
2270         memset(&link, 0, sizeof(link));
2271         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2272         link.link_speed  = ETH_SPEED_NUM_10G;
2273         link.link_autoneg = ETH_LINK_FIXED;
2274
2275         if (!hw->started) {
2276                 link.link_status = ETH_LINK_DOWN;
2277         } else if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
2278                 PMD_INIT_LOG(DEBUG, "Get link status from hw");
2279                 vtpci_read_dev_config(hw,
2280                                 offsetof(struct virtio_net_config, status),
2281                                 &status, sizeof(status));
2282                 if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
2283                         link.link_status = ETH_LINK_DOWN;
2284                         PMD_INIT_LOG(DEBUG, "Port %d is down",
2285                                      dev->data->port_id);
2286                 } else {
2287                         link.link_status = ETH_LINK_UP;
2288                         PMD_INIT_LOG(DEBUG, "Port %d is up",
2289                                      dev->data->port_id);
2290                 }
2291         } else {
2292                 link.link_status = ETH_LINK_UP;
2293         }
2294
2295         return rte_eth_linkstatus_set(dev, &link);
2296 }
2297
2298 static int
2299 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2300 {
2301         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2302         struct virtio_hw *hw = dev->data->dev_private;
2303         uint64_t offloads = rxmode->offloads;
2304
2305         if (mask & ETH_VLAN_FILTER_MASK) {
2306                 if ((offloads & DEV_RX_OFFLOAD_VLAN_FILTER) &&
2307                                 !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2308
2309                         PMD_DRV_LOG(NOTICE,
2310                                 "vlan filtering not available on this host");
2311
2312                         return -ENOTSUP;
2313                 }
2314         }
2315
2316         if (mask & ETH_VLAN_STRIP_MASK)
2317                 hw->vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
2318
2319         return 0;
2320 }
2321
2322 static void
2323 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2324 {
2325         uint64_t tso_mask, host_features;
2326         struct virtio_hw *hw = dev->data->dev_private;
2327
2328         dev_info->speed_capa = ETH_LINK_SPEED_10G; /* fake value */
2329
2330         dev_info->max_rx_queues =
2331                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
2332         dev_info->max_tx_queues =
2333                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
2334         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
2335         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
2336         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
2337
2338         host_features = VTPCI_OPS(hw)->get_features(hw);
2339         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
2340         dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
2341         if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
2342                 dev_info->rx_offload_capa |=
2343                         DEV_RX_OFFLOAD_TCP_CKSUM |
2344                         DEV_RX_OFFLOAD_UDP_CKSUM;
2345         }
2346         if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN))
2347                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_VLAN_FILTER;
2348         tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2349                 (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2350         if ((host_features & tso_mask) == tso_mask)
2351                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2352
2353         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
2354                                     DEV_TX_OFFLOAD_VLAN_INSERT;
2355         if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) {
2356                 dev_info->tx_offload_capa |=
2357                         DEV_TX_OFFLOAD_UDP_CKSUM |
2358                         DEV_TX_OFFLOAD_TCP_CKSUM;
2359         }
2360         tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2361                 (1ULL << VIRTIO_NET_F_HOST_TSO6);
2362         if ((host_features & tso_mask) == tso_mask)
2363                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
2364 }
2365
2366 /*
2367  * It enables testpmd to collect per queue stats.
2368  */
2369 static int
2370 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
2371 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
2372 __rte_unused uint8_t is_rx)
2373 {
2374         return 0;
2375 }
2376
2377 RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__);
2378 RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map);
2379 RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio-pci");
2380
2381 RTE_INIT(virtio_init_log)
2382 {
2383         virtio_logtype_init = rte_log_register("pmd.net.virtio.init");
2384         if (virtio_logtype_init >= 0)
2385                 rte_log_set_level(virtio_logtype_init, RTE_LOG_NOTICE);
2386         virtio_logtype_driver = rte_log_register("pmd.net.virtio.driver");
2387         if (virtio_logtype_driver >= 0)
2388                 rte_log_set_level(virtio_logtype_driver, RTE_LOG_NOTICE);
2389 }