drivers/net/virtio/virtio_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <string.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <unistd.h>
  39
  40 #include <rte_ethdev.h>
  41 #include <rte_memcpy.h>
  42 #include <rte_string_fns.h>
  43 #include <rte_memzone.h>
  44 #include <rte_malloc.h>
  45 #include <rte_atomic.h>
  46 #include <rte_branch_prediction.h>
  47 #include <rte_pci.h>
  48 #include <rte_ether.h>
  49 #include <rte_common.h>
  50 #include <rte_errno.h>
  51
  52 #include <rte_memory.h>
  53 #include <rte_eal.h>
  54 #include <rte_dev.h>
  55
  56 #include "virtio_ethdev.h"
  57 #include "virtio_pci.h"
  58 #include "virtio_logs.h"
  59 #include "virtqueue.h"
  60 #include "virtio_rxtx.h"
  61
  62 static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev);
  63 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  64 static int  virtio_dev_start(struct rte_eth_dev *dev);
  65 static void virtio_dev_stop(struct rte_eth_dev *dev);
  66 static void virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
  67 static void virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
  68 static void virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
  69 static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
  70 static void virtio_dev_info_get(struct rte_eth_dev *dev,
  71                                 struct rte_eth_dev_info *dev_info);
  72 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  73         __rte_unused int wait_to_complete);
  74
  75 static void virtio_set_hwaddr(struct virtio_hw *hw);
  76 static void virtio_get_hwaddr(struct virtio_hw *hw);
  77
  78 static void virtio_dev_stats_get(struct rte_eth_dev *dev,
  79                                  struct rte_eth_stats *stats);
  80 static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
  81                                  struct rte_eth_xstat *xstats, unsigned n);
  82 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
  83                                        struct rte_eth_xstat_name *xstats_names,
  84                                        unsigned limit);
  85 static void virtio_dev_stats_reset(struct rte_eth_dev *dev);
  86 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  87 static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
  88                                 uint16_t vlan_id, int on);
  89 static void virtio_mac_addr_add(struct rte_eth_dev *dev,
  90                                 struct ether_addr *mac_addr,
  91                                 uint32_t index, uint32_t vmdq __rte_unused);
  92 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  93 static void virtio_mac_addr_set(struct rte_eth_dev *dev,
  94                                 struct ether_addr *mac_addr);
  95
  96 static int virtio_dev_queue_stats_mapping_set(
  97         __rte_unused struct rte_eth_dev *eth_dev,
  98         __rte_unused uint16_t queue_id,
  99         __rte_unused uint8_t stat_idx,
 100         __rte_unused uint8_t is_rx);
 101
 102 /*
 103  * The set of PCI devices this driver supports
 104  */
 105 static const struct rte_pci_id pci_id_virtio_map[] = {
 106         { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_LEGACY_DEVICEID_NET) },
 107         { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_MODERN_DEVICEID_NET) },
 108         { .vendor_id = 0, /* sentinel */ },
 109 };
 110
 111 struct rte_virtio_xstats_name_off {
 112         char name[RTE_ETH_XSTATS_NAME_SIZE];
 113         unsigned offset;
 114 };
 115
 116 /* [rt]x_qX_ is prepended to the name string here */
 117 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
 118         {"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
 119         {"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
 120         {"errors",                 offsetof(struct virtnet_rx, stats.errors)},
 121         {"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
 122         {"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
 123         {"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
 124         {"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
 125         {"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
 126         {"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
 127         {"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
 128         {"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
 129         {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
 130         {"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
 131 };
 132
 133 /* [rt]x_qX_ is prepended to the name string here */
 134 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
 135         {"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
 136         {"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
 137         {"errors",                 offsetof(struct virtnet_tx, stats.errors)},
 138         {"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
 139         {"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
 140         {"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
 141         {"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
 142         {"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
 143         {"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
 144         {"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
 145         {"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
 146         {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
 147         {"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
 148 };
 149
 150 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
 151                             sizeof(rte_virtio_rxq_stat_strings[0]))
 152 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
 153                             sizeof(rte_virtio_txq_stat_strings[0]))
 154
 155 static int
 156 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
 157                 int *dlen, int pkt_num)
 158 {
 159         uint32_t head, i;
 160         int k, sum = 0;
 161         virtio_net_ctrl_ack status = ~0;
 162         struct virtio_pmd_ctrl result;
 163         struct virtqueue *vq;
 164
 165         ctrl->status = status;
 166
 167         if (!cvq || !cvq->vq) {
 168                 PMD_INIT_LOG(ERR, "Control queue is not supported.");
 169                 return -1;
 170         }
 171         vq = cvq->vq;
 172         head = vq->vq_desc_head_idx;
 173
 174         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 175                 "vq->hw->cvq = %p vq = %p",
 176                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 177
 178         if ((vq->vq_free_cnt < ((uint32_t)pkt_num + 2)) || (pkt_num < 1))
 179                 return -1;
 180
 181         memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
 182                 sizeof(struct virtio_pmd_ctrl));
 183
 184         /*
 185          * Format is enforced in qemu code:
 186          * One TX packet for header;
 187          * At least one TX packet per argument;
 188          * One RX packet for ACK.
 189          */
 190         vq->vq_ring.desc[head].flags = VRING_DESC_F_NEXT;
 191         vq->vq_ring.desc[head].addr = cvq->virtio_net_hdr_mem;
 192         vq->vq_ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 193         vq->vq_free_cnt--;
 194         i = vq->vq_ring.desc[head].next;
 195
 196         for (k = 0; k < pkt_num; k++) {
 197                 vq->vq_ring.desc[i].flags = VRING_DESC_F_NEXT;
 198                 vq->vq_ring.desc[i].addr = cvq->virtio_net_hdr_mem
 199                         + sizeof(struct virtio_net_ctrl_hdr)
 200                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 201                 vq->vq_ring.desc[i].len = dlen[k];
 202                 sum += dlen[k];
 203                 vq->vq_free_cnt--;
 204                 i = vq->vq_ring.desc[i].next;
 205         }
 206
 207         vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
 208         vq->vq_ring.desc[i].addr = cvq->virtio_net_hdr_mem
 209                         + sizeof(struct virtio_net_ctrl_hdr);
 210         vq->vq_ring.desc[i].len = sizeof(ctrl->status);
 211         vq->vq_free_cnt--;
 212
 213         vq->vq_desc_head_idx = vq->vq_ring.desc[i].next;
 214
 215         vq_update_avail_ring(vq, head);
 216         vq_update_avail_idx(vq);
 217
 218         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
 219
 220         virtqueue_notify(vq);
 221
 222         rte_rmb();
 223         while (VIRTQUEUE_NUSED(vq) == 0) {
 224                 rte_rmb();
 225                 usleep(100);
 226         }
 227
 228         while (VIRTQUEUE_NUSED(vq)) {
 229                 uint32_t idx, desc_idx, used_idx;
 230                 struct vring_used_elem *uep;
 231
 232                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 233                                 & (vq->vq_nentries - 1));
 234                 uep = &vq->vq_ring.used->ring[used_idx];
 235                 idx = (uint32_t) uep->id;
 236                 desc_idx = idx;
 237
 238                 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
 239                         desc_idx = vq->vq_ring.desc[desc_idx].next;
 240                         vq->vq_free_cnt++;
 241                 }
 242
 243                 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 244                 vq->vq_desc_head_idx = idx;
 245
 246                 vq->vq_used_cons_idx++;
 247                 vq->vq_free_cnt++;
 248         }
 249
 250         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
 251                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 252
 253         memcpy(&result, cvq->virtio_net_hdr_mz->addr,
 254                         sizeof(struct virtio_pmd_ctrl));
 255
 256         return result.status;
 257 }
 258
 259 static int
 260 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 261 {
 262         struct virtio_hw *hw = dev->data->dev_private;
 263         struct virtio_pmd_ctrl ctrl;
 264         int dlen[1];
 265         int ret;
 266
 267         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 268         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 269         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 270
 271         dlen[0] = sizeof(uint16_t);
 272
 273         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 274         if (ret) {
 275                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 276                           "failed, this is too late now...");
 277                 return -EINVAL;
 278         }
 279
 280         return 0;
 281 }
 282
 283 void
 284 virtio_dev_queue_release(struct virtqueue *vq)
 285 {
 286         struct virtio_hw *hw;
 287
 288         if (vq) {
 289                 hw = vq->hw;
 290                 if (vq->configured)
 291                         hw->vtpci_ops->del_queue(hw, vq);
 292
 293                 rte_free(vq->sw_ring);
 294                 rte_free(vq);
 295         }
 296 }
 297
 298 int virtio_dev_queue_setup(struct rte_eth_dev *dev,
 299                         int queue_type,
 300                         uint16_t queue_idx,
 301                         uint16_t vtpci_queue_idx,
 302                         uint16_t nb_desc,
 303                         unsigned int socket_id,
 304                         void **pvq)
 305 {
 306         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 307         char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
 308         const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
 309         unsigned int vq_size, size;
 310         struct virtio_hw *hw = dev->data->dev_private;
 311         struct virtnet_rx *rxvq = NULL;
 312         struct virtnet_tx *txvq = NULL;
 313         struct virtnet_ctl *cvq = NULL;
 314         struct virtqueue *vq;
 315         size_t sz_hdr_mz = 0;
 316         void *sw_ring = NULL;
 317         int ret;
 318
 319         PMD_INIT_LOG(DEBUG, "setting up queue: %u", vtpci_queue_idx);
 320
 321         /*
 322          * Read the virtqueue size from the Queue Size field
 323          * Always power of 2 and if 0 virtqueue does not exist
 324          */
 325         vq_size = hw->vtpci_ops->get_queue_num(hw, vtpci_queue_idx);
 326         PMD_INIT_LOG(DEBUG, "vq_size: %u nb_desc:%u", vq_size, nb_desc);
 327         if (vq_size == 0) {
 328                 PMD_INIT_LOG(ERR, "virtqueue does not exist");
 329                 return -EINVAL;
 330         }
 331
 332         if (!rte_is_power_of_2(vq_size)) {
 333                 PMD_INIT_LOG(ERR, "virtqueue size is not powerof 2");
 334                 return -EINVAL;
 335         }
 336
 337         snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
 338                  dev->data->port_id, vtpci_queue_idx);
 339
 340         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 341                                 vq_size * sizeof(struct vq_desc_extra),
 342                                 RTE_CACHE_LINE_SIZE);
 343         if (queue_type == VTNET_TQ) {
 344                 /*
 345                  * For each xmit packet, allocate a virtio_net_hdr
 346                  * and indirect ring elements
 347                  */
 348                 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
 349         } else if (queue_type == VTNET_CQ) {
 350                 /* Allocate a page for control vq command, data and status */
 351                 sz_hdr_mz = PAGE_SIZE;
 352         }
 353
 354         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, socket_id);
 355         if (vq == NULL) {
 356                 PMD_INIT_LOG(ERR, "can not allocate vq");
 357                 return -ENOMEM;
 358         }
 359         vq->hw = hw;
 360         vq->vq_queue_index = vtpci_queue_idx;
 361         vq->vq_nentries = vq_size;
 362
 363         if (nb_desc == 0 || nb_desc > vq_size)
 364                 nb_desc = vq_size;
 365         vq->vq_free_cnt = nb_desc;
 366
 367         /*
 368          * Reserve a memzone for vring elements
 369          */
 370         size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
 371         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
 372         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
 373                      size, vq->vq_ring_size);
 374
 375         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, socket_id,
 376                                          0, VIRTIO_PCI_VRING_ALIGN);
 377         if (mz == NULL) {
 378                 if (rte_errno == EEXIST)
 379                         mz = rte_memzone_lookup(vq_name);
 380                 if (mz == NULL) {
 381                         ret = -ENOMEM;
 382                         goto fail_q_alloc;
 383                 }
 384         }
 385
 386         memset(mz->addr, 0, sizeof(mz->len));
 387
 388         vq->vq_ring_mem = mz->phys_addr;
 389         vq->vq_ring_virt_mem = mz->addr;
 390         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem:      0x%" PRIx64,
 391                      (uint64_t)mz->phys_addr);
 392         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%" PRIx64,
 393                      (uint64_t)(uintptr_t)mz->addr);
 394
 395         if (sz_hdr_mz) {
 396                 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 397                          dev->data->port_id, vtpci_queue_idx);
 398                 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
 399                                                      socket_id, 0,
 400                                                      RTE_CACHE_LINE_SIZE);
 401                 if (hdr_mz == NULL) {
 402                         if (rte_errno == EEXIST)
 403                                 hdr_mz = rte_memzone_lookup(vq_hdr_name);
 404                         if (hdr_mz == NULL) {
 405                                 ret = -ENOMEM;
 406                                 goto fail_q_alloc;
 407                         }
 408                 }
 409         }
 410
 411         if (queue_type == VTNET_RQ) {
 412                 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
 413                                sizeof(vq->sw_ring[0]);
 414
 415                 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
 416                                              RTE_CACHE_LINE_SIZE, socket_id);
 417                 if (!sw_ring) {
 418                         PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
 419                         ret = -ENOMEM;
 420                         goto fail_q_alloc;
 421                 }
 422
 423                 vq->sw_ring = sw_ring;
 424                 rxvq = &vq->rxq;
 425                 rxvq->vq = vq;
 426                 rxvq->port_id = dev->data->port_id;
 427                 rxvq->queue_id = queue_idx;
 428                 rxvq->mz = mz;
 429                 *pvq = rxvq;
 430         } else if (queue_type == VTNET_TQ) {
 431                 txvq = &vq->txq;
 432                 txvq->vq = vq;
 433                 txvq->port_id = dev->data->port_id;
 434                 txvq->queue_id = queue_idx;
 435                 txvq->mz = mz;
 436                 txvq->virtio_net_hdr_mz = hdr_mz;
 437                 txvq->virtio_net_hdr_mem = hdr_mz->phys_addr;
 438
 439                 *pvq = txvq;
 440         } else if (queue_type == VTNET_CQ) {
 441                 cvq = &vq->cq;
 442                 cvq->vq = vq;
 443                 cvq->mz = mz;
 444                 cvq->virtio_net_hdr_mz = hdr_mz;
 445                 cvq->virtio_net_hdr_mem = hdr_mz->phys_addr;
 446                 memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE);
 447                 *pvq = cvq;
 448         }
 449
 450         /* For virtio_user case (that is when dev->pci_dev is NULL), we use
 451          * virtual address. And we need properly set _offset_, please see
 452          * VIRTIO_MBUF_DATA_DMA_ADDR in virtqueue.h for more information.
 453          */
 454         if (dev->pci_dev)
 455                 vq->offset = offsetof(struct rte_mbuf, buf_physaddr);
 456         else {
 457                 vq->vq_ring_mem = (uintptr_t)mz->addr;
 458                 vq->offset = offsetof(struct rte_mbuf, buf_addr);
 459                 if (queue_type == VTNET_TQ)
 460                         txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 461                 else if (queue_type == VTNET_CQ)
 462                         cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 463         }
 464
 465         if (queue_type == VTNET_TQ) {
 466                 struct virtio_tx_region *txr;
 467                 unsigned int i;
 468
 469                 txr = hdr_mz->addr;
 470                 memset(txr, 0, vq_size * sizeof(*txr));
 471                 for (i = 0; i < vq_size; i++) {
 472                         struct vring_desc *start_dp = txr[i].tx_indir;
 473
 474                         vring_desc_init(start_dp, RTE_DIM(txr[i].tx_indir));
 475
 476                         /* first indirect descriptor is always the tx header */
 477                         start_dp->addr = txvq->virtio_net_hdr_mem
 478                                 + i * sizeof(*txr)
 479                                 + offsetof(struct virtio_tx_region, tx_hdr);
 480
 481                         start_dp->len = hw->vtnet_hdr_size;
 482                         start_dp->flags = VRING_DESC_F_NEXT;
 483                 }
 484         }
 485
 486         if (hw->vtpci_ops->setup_queue(hw, vq) < 0) {
 487                 PMD_INIT_LOG(ERR, "setup_queue failed");
 488                 virtio_dev_queue_release(vq);
 489                 return -EINVAL;
 490         }
 491
 492         vq->configured = 1;
 493         return 0;
 494
 495 fail_q_alloc:
 496         rte_free(sw_ring);
 497         rte_memzone_free(hdr_mz);
 498         rte_memzone_free(mz);
 499         rte_free(vq);
 500
 501         return ret;
 502 }
 503
 504 static int
 505 virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx,
 506                 uint32_t socket_id)
 507 {
 508         struct virtnet_ctl *cvq;
 509         int ret;
 510         struct virtio_hw *hw = dev->data->dev_private;
 511
 512         PMD_INIT_FUNC_TRACE();
 513         ret = virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX,
 514                         vtpci_queue_idx, 0, socket_id, (void **)&cvq);
 515         if (ret < 0) {
 516                 PMD_INIT_LOG(ERR, "control vq initialization failed");
 517                 return ret;
 518         }
 519
 520         hw->cvq = cvq;
 521         return 0;
 522 }
 523
 524 static void
 525 virtio_free_queues(struct rte_eth_dev *dev)
 526 {
 527         unsigned int i;
 528
 529         for (i = 0; i < dev->data->nb_rx_queues; i++)
 530                 virtio_dev_rx_queue_release(dev->data->rx_queues[i]);
 531
 532         dev->data->nb_rx_queues = 0;
 533
 534         for (i = 0; i < dev->data->nb_tx_queues; i++)
 535                 virtio_dev_tx_queue_release(dev->data->tx_queues[i]);
 536
 537         dev->data->nb_tx_queues = 0;
 538 }
 539
 540 static void
 541 virtio_dev_close(struct rte_eth_dev *dev)
 542 {
 543         struct virtio_hw *hw = dev->data->dev_private;
 544
 545         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 546
 547         if (hw->cvq)
 548                 virtio_dev_queue_release(hw->cvq->vq);
 549
 550         /* reset the NIC */
 551         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 552                 vtpci_irq_config(hw, VIRTIO_MSI_NO_VECTOR);
 553         vtpci_reset(hw);
 554         hw->started = 0;
 555         virtio_dev_free_mbufs(dev);
 556         virtio_free_queues(dev);
 557 }
 558
 559 static void
 560 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
 561 {
 562         struct virtio_hw *hw = dev->data->dev_private;
 563         struct virtio_pmd_ctrl ctrl;
 564         int dlen[1];
 565         int ret;
 566
 567         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 568                 PMD_INIT_LOG(INFO, "host does not support rx control\n");
 569                 return;
 570         }
 571
 572         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 573         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 574         ctrl.data[0] = 1;
 575         dlen[0] = 1;
 576
 577         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 578         if (ret)
 579                 PMD_INIT_LOG(ERR, "Failed to enable promisc");
 580 }
 581
 582 static void
 583 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
 584 {
 585         struct virtio_hw *hw = dev->data->dev_private;
 586         struct virtio_pmd_ctrl ctrl;
 587         int dlen[1];
 588         int ret;
 589
 590         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 591                 PMD_INIT_LOG(INFO, "host does not support rx control\n");
 592                 return;
 593         }
 594
 595         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 596         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 597         ctrl.data[0] = 0;
 598         dlen[0] = 1;
 599
 600         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 601         if (ret)
 602                 PMD_INIT_LOG(ERR, "Failed to disable promisc");
 603 }
 604
 605 static void
 606 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
 607 {
 608         struct virtio_hw *hw = dev->data->dev_private;
 609         struct virtio_pmd_ctrl ctrl;
 610         int dlen[1];
 611         int ret;
 612
 613         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 614                 PMD_INIT_LOG(INFO, "host does not support rx control\n");
 615                 return;
 616         }
 617
 618         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 619         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 620         ctrl.data[0] = 1;
 621         dlen[0] = 1;
 622
 623         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 624         if (ret)
 625                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
 626 }
 627
 628 static void
 629 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
 630 {
 631         struct virtio_hw *hw = dev->data->dev_private;
 632         struct virtio_pmd_ctrl ctrl;
 633         int dlen[1];
 634         int ret;
 635
 636         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 637                 PMD_INIT_LOG(INFO, "host does not support rx control\n");
 638                 return;
 639         }
 640
 641         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 642         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 643         ctrl.data[0] = 0;
 644         dlen[0] = 1;
 645
 646         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 647         if (ret)
 648                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
 649 }
 650
 651 #define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
 652 static int
 653 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 654 {
 655         struct virtio_hw *hw = dev->data->dev_private;
 656         uint32_t ether_hdr_len = ETHER_HDR_LEN + VLAN_TAG_LEN +
 657                                  hw->vtnet_hdr_size;
 658         uint32_t frame_size = mtu + ether_hdr_len;
 659
 660         if (mtu < ETHER_MIN_MTU || frame_size > VIRTIO_MAX_RX_PKTLEN) {
 661                 PMD_INIT_LOG(ERR, "MTU should be between %d and %d\n",
 662                         ETHER_MIN_MTU, VIRTIO_MAX_RX_PKTLEN - ether_hdr_len);
 663                 return -EINVAL;
 664         }
 665         return 0;
 666 }
 667
 668 /*
 669  * dev_ops for virtio, bare necessities for basic operation
 670  */
 671 static const struct eth_dev_ops virtio_eth_dev_ops = {
 672         .dev_configure           = virtio_dev_configure,
 673         .dev_start               = virtio_dev_start,
 674         .dev_stop                = virtio_dev_stop,
 675         .dev_close               = virtio_dev_close,
 676         .promiscuous_enable      = virtio_dev_promiscuous_enable,
 677         .promiscuous_disable     = virtio_dev_promiscuous_disable,
 678         .allmulticast_enable     = virtio_dev_allmulticast_enable,
 679         .allmulticast_disable    = virtio_dev_allmulticast_disable,
 680         .mtu_set                 = virtio_mtu_set,
 681         .dev_infos_get           = virtio_dev_info_get,
 682         .stats_get               = virtio_dev_stats_get,
 683         .xstats_get              = virtio_dev_xstats_get,
 684         .xstats_get_names        = virtio_dev_xstats_get_names,
 685         .stats_reset             = virtio_dev_stats_reset,
 686         .xstats_reset            = virtio_dev_stats_reset,
 687         .link_update             = virtio_dev_link_update,
 688         .rx_queue_setup          = virtio_dev_rx_queue_setup,
 689         .rx_queue_release        = virtio_dev_rx_queue_release,
 690         .tx_queue_setup          = virtio_dev_tx_queue_setup,
 691         .tx_queue_release        = virtio_dev_tx_queue_release,
 692         /* collect stats per queue */
 693         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 694         .vlan_filter_set         = virtio_vlan_filter_set,
 695         .mac_addr_add            = virtio_mac_addr_add,
 696         .mac_addr_remove         = virtio_mac_addr_remove,
 697         .mac_addr_set            = virtio_mac_addr_set,
 698 };
 699
 700 static inline int
 701 virtio_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 702                                 struct rte_eth_link *link)
 703 {
 704         struct rte_eth_link *dst = link;
 705         struct rte_eth_link *src = &(dev->data->dev_link);
 706
 707         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 708                         *(uint64_t *)src) == 0)
 709                 return -1;
 710
 711         return 0;
 712 }
 713
 714 /**
 715  * Atomically writes the link status information into global
 716  * structure rte_eth_dev.
 717  *
 718  * @param dev
 719  *   - Pointer to the structure rte_eth_dev to read from.
 720  *   - Pointer to the buffer to be saved with the link status.
 721  *
 722  * @return
 723  *   - On success, zero.
 724  *   - On failure, negative value.
 725  */
 726 static inline int
 727 virtio_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 728                 struct rte_eth_link *link)
 729 {
 730         struct rte_eth_link *dst = &(dev->data->dev_link);
 731         struct rte_eth_link *src = link;
 732
 733         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 734                                         *(uint64_t *)src) == 0)
 735                 return -1;
 736
 737         return 0;
 738 }
 739
 740 static void
 741 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 742 {
 743         unsigned i;
 744
 745         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 746                 const struct virtnet_tx *txvq = dev->data->tx_queues[i];
 747                 if (txvq == NULL)
 748                         continue;
 749
 750                 stats->opackets += txvq->stats.packets;
 751                 stats->obytes += txvq->stats.bytes;
 752                 stats->oerrors += txvq->stats.errors;
 753
 754                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 755                         stats->q_opackets[i] = txvq->stats.packets;
 756                         stats->q_obytes[i] = txvq->stats.bytes;
 757                 }
 758         }
 759
 760         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 761                 const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
 762                 if (rxvq == NULL)
 763                         continue;
 764
 765                 stats->ipackets += rxvq->stats.packets;
 766                 stats->ibytes += rxvq->stats.bytes;
 767                 stats->ierrors += rxvq->stats.errors;
 768
 769                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 770                         stats->q_ipackets[i] = rxvq->stats.packets;
 771                         stats->q_ibytes[i] = rxvq->stats.bytes;
 772                 }
 773         }
 774
 775         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
 776 }
 777
 778 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
 779                                        struct rte_eth_xstat_name *xstats_names,
 780                                        __rte_unused unsigned limit)
 781 {
 782         unsigned i;
 783         unsigned count = 0;
 784         unsigned t;
 785
 786         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
 787                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
 788
 789         if (xstats_names != NULL) {
 790                 /* Note: limit checked in rte_eth_xstats_names() */
 791
 792                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 793                         struct virtqueue *rxvq = dev->data->rx_queues[i];
 794                         if (rxvq == NULL)
 795                                 continue;
 796                         for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
 797                                 snprintf(xstats_names[count].name,
 798                                         sizeof(xstats_names[count].name),
 799                                         "rx_q%u_%s", i,
 800                                         rte_virtio_rxq_stat_strings[t].name);
 801                                 count++;
 802                         }
 803                 }
 804
 805                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
 806                         struct virtqueue *txvq = dev->data->tx_queues[i];
 807                         if (txvq == NULL)
 808                                 continue;
 809                         for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
 810                                 snprintf(xstats_names[count].name,
 811                                         sizeof(xstats_names[count].name),
 812                                         "tx_q%u_%s", i,
 813                                         rte_virtio_txq_stat_strings[t].name);
 814                                 count++;
 815                         }
 816                 }
 817                 return count;
 818         }
 819         return nstats;
 820 }
 821
 822 static int
 823 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 824                       unsigned n)
 825 {
 826         unsigned i;
 827         unsigned count = 0;
 828
 829         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
 830                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
 831
 832         if (n < nstats)
 833                 return nstats;
 834
 835         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 836                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
 837
 838                 if (rxvq == NULL)
 839                         continue;
 840
 841                 unsigned t;
 842
 843                 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
 844                         xstats[count].value = *(uint64_t *)(((char *)rxvq) +
 845                                 rte_virtio_rxq_stat_strings[t].offset);
 846                         count++;
 847                 }
 848         }
 849
 850         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 851                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
 852
 853                 if (txvq == NULL)
 854                         continue;
 855
 856                 unsigned t;
 857
 858                 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
 859                         xstats[count].value = *(uint64_t *)(((char *)txvq) +
 860                                 rte_virtio_txq_stat_strings[t].offset);
 861                         count++;
 862                 }
 863         }
 864
 865         return count;
 866 }
 867
 868 static void
 869 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 870 {
 871         virtio_update_stats(dev, stats);
 872 }
 873
 874 static void
 875 virtio_dev_stats_reset(struct rte_eth_dev *dev)
 876 {
 877         unsigned int i;
 878
 879         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 880                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
 881                 if (txvq == NULL)
 882                         continue;
 883
 884                 txvq->stats.packets = 0;
 885                 txvq->stats.bytes = 0;
 886                 txvq->stats.errors = 0;
 887                 txvq->stats.multicast = 0;
 888                 txvq->stats.broadcast = 0;
 889                 memset(txvq->stats.size_bins, 0,
 890                        sizeof(txvq->stats.size_bins[0]) * 8);
 891         }
 892
 893         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 894                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
 895                 if (rxvq == NULL)
 896                         continue;
 897
 898                 rxvq->stats.packets = 0;
 899                 rxvq->stats.bytes = 0;
 900                 rxvq->stats.errors = 0;
 901                 rxvq->stats.multicast = 0;
 902                 rxvq->stats.broadcast = 0;
 903                 memset(rxvq->stats.size_bins, 0,
 904                        sizeof(rxvq->stats.size_bins[0]) * 8);
 905         }
 906 }
 907
 908 static void
 909 virtio_set_hwaddr(struct virtio_hw *hw)
 910 {
 911         vtpci_write_dev_config(hw,
 912                         offsetof(struct virtio_net_config, mac),
 913                         &hw->mac_addr, ETHER_ADDR_LEN);
 914 }
 915
 916 static void
 917 virtio_get_hwaddr(struct virtio_hw *hw)
 918 {
 919         if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) {
 920                 vtpci_read_dev_config(hw,
 921                         offsetof(struct virtio_net_config, mac),
 922                         &hw->mac_addr, ETHER_ADDR_LEN);
 923         } else {
 924                 eth_random_addr(&hw->mac_addr[0]);
 925                 virtio_set_hwaddr(hw);
 926         }
 927 }
 928
 929 static void
 930 virtio_mac_table_set(struct virtio_hw *hw,
 931                      const struct virtio_net_ctrl_mac *uc,
 932                      const struct virtio_net_ctrl_mac *mc)
 933 {
 934         struct virtio_pmd_ctrl ctrl;
 935         int err, len[2];
 936
 937         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
 938                 PMD_DRV_LOG(INFO, "host does not support mac table");
 939                 return;
 940         }
 941
 942         ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
 943         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
 944
 945         len[0] = uc->entries * ETHER_ADDR_LEN + sizeof(uc->entries);
 946         memcpy(ctrl.data, uc, len[0]);
 947
 948         len[1] = mc->entries * ETHER_ADDR_LEN + sizeof(mc->entries);
 949         memcpy(ctrl.data + len[0], mc, len[1]);
 950
 951         err = virtio_send_command(hw->cvq, &ctrl, len, 2);
 952         if (err != 0)
 953                 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
 954 }
 955
 956 static void
 957 virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 958                     uint32_t index, uint32_t vmdq __rte_unused)
 959 {
 960         struct virtio_hw *hw = dev->data->dev_private;
 961         const struct ether_addr *addrs = dev->data->mac_addrs;
 962         unsigned int i;
 963         struct virtio_net_ctrl_mac *uc, *mc;
 964
 965         if (index >= VIRTIO_MAX_MAC_ADDRS) {
 966                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
 967                 return;
 968         }
 969
 970         uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries));
 971         uc->entries = 0;
 972         mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries));
 973         mc->entries = 0;
 974
 975         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
 976                 const struct ether_addr *addr
 977                         = (i == index) ? mac_addr : addrs + i;
 978                 struct virtio_net_ctrl_mac *tbl
 979                         = is_multicast_ether_addr(addr) ? mc : uc;
 980
 981                 memcpy(&tbl->macs[tbl->entries++], addr, ETHER_ADDR_LEN);
 982         }
 983
 984         virtio_mac_table_set(hw, uc, mc);
 985 }
 986
 987 static void
 988 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 989 {
 990         struct virtio_hw *hw = dev->data->dev_private;
 991         struct ether_addr *addrs = dev->data->mac_addrs;
 992         struct virtio_net_ctrl_mac *uc, *mc;
 993         unsigned int i;
 994
 995         if (index >= VIRTIO_MAX_MAC_ADDRS) {
 996                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
 997                 return;
 998         }
 999
1000         uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries));
1001         uc->entries = 0;
1002         mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries));
1003         mc->entries = 0;
1004
1005         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1006                 struct virtio_net_ctrl_mac *tbl;
1007
1008                 if (i == index || is_zero_ether_addr(addrs + i))
1009                         continue;
1010
1011                 tbl = is_multicast_ether_addr(addrs + i) ? mc : uc;
1012                 memcpy(&tbl->macs[tbl->entries++], addrs + i, ETHER_ADDR_LEN);
1013         }
1014
1015         virtio_mac_table_set(hw, uc, mc);
1016 }
1017
1018 static void
1019 virtio_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1020 {
1021         struct virtio_hw *hw = dev->data->dev_private;
1022
1023         memcpy(hw->mac_addr, mac_addr, ETHER_ADDR_LEN);
1024
1025         /* Use atomic update if available */
1026         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1027                 struct virtio_pmd_ctrl ctrl;
1028                 int len = ETHER_ADDR_LEN;
1029
1030                 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1031                 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1032
1033                 memcpy(ctrl.data, mac_addr, ETHER_ADDR_LEN);
1034                 virtio_send_command(hw->cvq, &ctrl, &len, 1);
1035         } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC))
1036                 virtio_set_hwaddr(hw);
1037 }
1038
1039 static int
1040 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1041 {
1042         struct virtio_hw *hw = dev->data->dev_private;
1043         struct virtio_pmd_ctrl ctrl;
1044         int len;
1045
1046         if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1047                 return -ENOTSUP;
1048
1049         ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1050         ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1051         memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1052         len = sizeof(vlan_id);
1053
1054         return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1055 }
1056
1057 static int
1058 virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1059 {
1060         uint64_t host_features;
1061
1062         /* Prepare guest_features: feature that driver wants to support */
1063         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1064                 req_features);
1065
1066         /* Read device(host) feature bits */
1067         host_features = hw->vtpci_ops->get_features(hw);
1068         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1069                 host_features);
1070
1071         /*
1072          * Negotiate features: Subset of device feature bits are written back
1073          * guest feature bits.
1074          */
1075         hw->guest_features = req_features;
1076         hw->guest_features = vtpci_negotiate_features(hw, host_features);
1077         PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1078                 hw->guest_features);
1079
1080         if (hw->modern) {
1081                 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
1082                         PMD_INIT_LOG(ERR,
1083                                 "VIRTIO_F_VERSION_1 features is not enabled.");
1084                         return -1;
1085                 }
1086                 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1087                 if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1088                         PMD_INIT_LOG(ERR,
1089                                 "failed to set FEATURES_OK status!");
1090                         return -1;
1091                 }
1092         }
1093
1094         hw->req_guest_features = req_features;
1095
1096         return 0;
1097 }
1098
1099 /*
1100  * Process Virtio Config changed interrupt and call the callback
1101  * if link state changed.
1102  */
1103 static void
1104 virtio_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1105                          void *param)
1106 {
1107         struct rte_eth_dev *dev = param;
1108         struct virtio_hw *hw = dev->data->dev_private;
1109         uint8_t isr;
1110
1111         /* Read interrupt status which clears interrupt */
1112         isr = vtpci_isr(hw);
1113         PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1114
1115         if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0)
1116                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1117
1118         if (isr & VIRTIO_PCI_ISR_CONFIG) {
1119                 if (virtio_dev_link_update(dev, 0) == 0)
1120                         _rte_eth_dev_callback_process(dev,
1121                                                       RTE_ETH_EVENT_INTR_LSC, NULL);
1122         }
1123
1124 }
1125
1126 static void
1127 rx_func_get(struct rte_eth_dev *eth_dev)
1128 {
1129         struct virtio_hw *hw = eth_dev->data->dev_private;
1130         if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF))
1131                 eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1132         else
1133                 eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1134 }
1135
1136 /* reset device and renegotiate features if needed */
1137 static int
1138 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
1139 {
1140         struct virtio_hw *hw = eth_dev->data->dev_private;
1141         struct virtio_net_config *config;
1142         struct virtio_net_config local_config;
1143         struct rte_pci_device *pci_dev = eth_dev->pci_dev;
1144
1145         /* Reset the device although not necessary at startup */
1146         vtpci_reset(hw);
1147
1148         /* Tell the host we've noticed this device. */
1149         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1150
1151         /* Tell the host we've known how to drive the device. */
1152         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1153         if (virtio_negotiate_features(hw, req_features) < 0)
1154                 return -1;
1155
1156         /* If host does not support status then disable LSC */
1157         if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS))
1158                 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
1159         else
1160                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
1161
1162         rte_eth_copy_pci_info(eth_dev, pci_dev);
1163
1164         rx_func_get(eth_dev);
1165
1166         /* Setting up rx_header size for the device */
1167         if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
1168             vtpci_with_feature(hw, VIRTIO_F_VERSION_1))
1169                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1170         else
1171                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
1172
1173         /* Copy the permanent MAC address to: virtio_hw */
1174         virtio_get_hwaddr(hw);
1175         ether_addr_copy((struct ether_addr *) hw->mac_addr,
1176                         &eth_dev->data->mac_addrs[0]);
1177         PMD_INIT_LOG(DEBUG,
1178                      "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1179                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1180                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1181
1182         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1183                 config = &local_config;
1184
1185                 vtpci_read_dev_config(hw,
1186                         offsetof(struct virtio_net_config, mac),
1187                         &config->mac, sizeof(config->mac));
1188
1189                 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1190                         vtpci_read_dev_config(hw,
1191                                 offsetof(struct virtio_net_config, status),
1192                                 &config->status, sizeof(config->status));
1193                 } else {
1194                         PMD_INIT_LOG(DEBUG,
1195                                      "VIRTIO_NET_F_STATUS is not supported");
1196                         config->status = 0;
1197                 }
1198
1199                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) {
1200                         vtpci_read_dev_config(hw,
1201                                 offsetof(struct virtio_net_config, max_virtqueue_pairs),
1202                                 &config->max_virtqueue_pairs,
1203                                 sizeof(config->max_virtqueue_pairs));
1204                 } else {
1205                         PMD_INIT_LOG(DEBUG,
1206                                      "VIRTIO_NET_F_MQ is not supported");
1207                         config->max_virtqueue_pairs = 1;
1208                 }
1209
1210                 hw->max_queue_pairs = config->max_virtqueue_pairs;
1211
1212                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
1213                                 config->max_virtqueue_pairs);
1214                 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
1215                 PMD_INIT_LOG(DEBUG,
1216                                 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1217                                 config->mac[0], config->mac[1],
1218                                 config->mac[2], config->mac[3],
1219                                 config->mac[4], config->mac[5]);
1220         } else {
1221                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
1222                 hw->max_queue_pairs = 1;
1223         }
1224
1225         if (pci_dev)
1226                 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1227                         eth_dev->data->port_id, pci_dev->id.vendor_id,
1228                         pci_dev->id.device_id);
1229
1230         return 0;
1231 }
1232
1233 /*
1234  * This function is based on probe() function in virtio_pci.c
1235  * It returns 0 on success.
1236  */
1237 int
1238 eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
1239 {
1240         struct virtio_hw *hw = eth_dev->data->dev_private;
1241         struct rte_pci_device *pci_dev;
1242         uint32_t dev_flags = RTE_ETH_DEV_DETACHABLE;
1243         int ret;
1244
1245         RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr_mrg_rxbuf));
1246
1247         eth_dev->dev_ops = &virtio_eth_dev_ops;
1248         eth_dev->tx_pkt_burst = &virtio_xmit_pkts;
1249
1250         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1251                 rx_func_get(eth_dev);
1252                 return 0;
1253         }
1254
1255         /* Allocate memory for storing MAC addresses */
1256         eth_dev->data->mac_addrs = rte_zmalloc("virtio", VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN, 0);
1257         if (eth_dev->data->mac_addrs == NULL) {
1258                 PMD_INIT_LOG(ERR,
1259                         "Failed to allocate %d bytes needed to store MAC addresses",
1260                         VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN);
1261                 return -ENOMEM;
1262         }
1263
1264         pci_dev = eth_dev->pci_dev;
1265
1266         if (pci_dev) {
1267                 ret = vtpci_init(pci_dev, hw, &dev_flags);
1268                 if (ret)
1269                         return ret;
1270         }
1271
1272         eth_dev->data->dev_flags = dev_flags;
1273
1274         /* reset device and negotiate default features */
1275         ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
1276         if (ret < 0)
1277                 return ret;
1278
1279         /* Setup interrupt callback  */
1280         if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1281                 rte_intr_callback_register(&pci_dev->intr_handle,
1282                         virtio_interrupt_handler, eth_dev);
1283
1284         return 0;
1285 }
1286
1287 static int
1288 eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev)
1289 {
1290         struct rte_pci_device *pci_dev;
1291         struct virtio_hw *hw = eth_dev->data->dev_private;
1292
1293         PMD_INIT_FUNC_TRACE();
1294
1295         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
1296                 return -EPERM;
1297
1298         if (hw->started == 1) {
1299                 virtio_dev_stop(eth_dev);
1300                 virtio_dev_close(eth_dev);
1301         }
1302         pci_dev = eth_dev->pci_dev;
1303
1304         eth_dev->dev_ops = NULL;
1305         eth_dev->tx_pkt_burst = NULL;
1306         eth_dev->rx_pkt_burst = NULL;
1307
1308         rte_free(eth_dev->data->mac_addrs);
1309         eth_dev->data->mac_addrs = NULL;
1310
1311         /* reset interrupt callback  */
1312         if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1313                 rte_intr_callback_unregister(&pci_dev->intr_handle,
1314                                                 virtio_interrupt_handler,
1315                                                 eth_dev);
1316         rte_eal_pci_unmap_device(pci_dev);
1317
1318         PMD_INIT_LOG(DEBUG, "dev_uninit completed");
1319
1320         return 0;
1321 }
1322
1323 static struct eth_driver rte_virtio_pmd = {
1324         .pci_drv = {
1325                 .driver = {
1326                         .name = "net_virtio",
1327                 },
1328                 .id_table = pci_id_virtio_map,
1329                 .drv_flags = RTE_PCI_DRV_DETACHABLE,
1330                 .probe = rte_eth_dev_pci_probe,
1331                 .remove = rte_eth_dev_pci_remove,
1332         },
1333         .eth_dev_init = eth_virtio_dev_init,
1334         .eth_dev_uninit = eth_virtio_dev_uninit,
1335         .dev_private_size = sizeof(struct virtio_hw),
1336 };
1337
1338 RTE_INIT(rte_virtio_pmd_init);
1339 static void
1340 rte_virtio_pmd_init(void)
1341 {
1342         if (rte_eal_iopl_init() != 0) {
1343                 PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD");
1344                 return;
1345         }
1346
1347         rte_eal_pci_register(&rte_virtio_pmd.pci_drv);
1348 }
1349
1350 /*
1351  * Configure virtio device
1352  * It returns 0 on success.
1353  */
1354 static int
1355 virtio_dev_configure(struct rte_eth_dev *dev)
1356 {
1357         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
1358         struct virtio_hw *hw = dev->data->dev_private;
1359         uint64_t req_features;
1360         int ret;
1361
1362         PMD_INIT_LOG(DEBUG, "configure");
1363         req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
1364         if (rxmode->hw_ip_checksum)
1365                 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
1366         if (rxmode->enable_lro)
1367                 req_features |=
1368                         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
1369                         (1ULL << VIRTIO_NET_F_GUEST_TSO6);
1370
1371         /* if request features changed, reinit the device */
1372         if (req_features != hw->req_guest_features) {
1373                 ret = virtio_init_device(dev, req_features);
1374                 if (ret < 0)
1375                         return ret;
1376         }
1377
1378         if (rxmode->hw_ip_checksum &&
1379                 !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
1380                 PMD_DRV_LOG(NOTICE,
1381                         "rx ip checksum not available on this host");
1382                 return -ENOTSUP;
1383         }
1384
1385         if (rxmode->enable_lro &&
1386                 (!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
1387                         !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4))) {
1388                 PMD_DRV_LOG(NOTICE,
1389                         "lro not available on this host");
1390                 return -ENOTSUP;
1391         }
1392
1393         /* Setup and start control queue */
1394         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1395                 ret = virtio_dev_cq_queue_setup(dev,
1396                         hw->max_queue_pairs * 2,
1397                         SOCKET_ID_ANY);
1398                 if (ret < 0)
1399                         return ret;
1400                 virtio_dev_cq_start(dev);
1401         }
1402
1403         hw->vlan_strip = rxmode->hw_vlan_strip;
1404
1405         if (rxmode->hw_vlan_filter
1406             && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
1407                 PMD_DRV_LOG(NOTICE,
1408                             "vlan filtering not available on this host");
1409                 return -ENOTSUP;
1410         }
1411
1412         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1413                 if (vtpci_irq_config(hw, 0) == VIRTIO_MSI_NO_VECTOR) {
1414                         PMD_DRV_LOG(ERR, "failed to set config vector");
1415                         return -EBUSY;
1416                 }
1417
1418         return 0;
1419 }
1420
1421
1422 static int
1423 virtio_dev_start(struct rte_eth_dev *dev)
1424 {
1425         uint16_t nb_queues, i;
1426         struct virtio_hw *hw = dev->data->dev_private;
1427         struct virtnet_rx *rxvq;
1428         struct virtnet_tx *txvq __rte_unused;
1429
1430         /* check if lsc interrupt feature is enabled */
1431         if (dev->data->dev_conf.intr_conf.lsc) {
1432                 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
1433                         PMD_DRV_LOG(ERR, "link status not supported by host");
1434                         return -ENOTSUP;
1435                 }
1436
1437                 if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0) {
1438                         PMD_DRV_LOG(ERR, "interrupt enable failed");
1439                         return -EIO;
1440                 }
1441         }
1442
1443         /* Initialize Link state */
1444         virtio_dev_link_update(dev, 0);
1445
1446         /* On restart after stop do not touch queues */
1447         if (hw->started)
1448                 return 0;
1449
1450         /* Do final configuration before rx/tx engine starts */
1451         virtio_dev_rxtx_start(dev);
1452         vtpci_reinit_complete(hw);
1453
1454         hw->started = 1;
1455
1456         /*Notify the backend
1457          *Otherwise the tap backend might already stop its queue due to fullness.
1458          *vhost backend will have no chance to be waked up
1459          */
1460         nb_queues = dev->data->nb_rx_queues;
1461         if (nb_queues > 1) {
1462                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
1463                         return -EINVAL;
1464         }
1465
1466         PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
1467
1468         for (i = 0; i < nb_queues; i++) {
1469                 rxvq = dev->data->rx_queues[i];
1470                 virtqueue_notify(rxvq->vq);
1471         }
1472
1473         PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
1474
1475         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1476                 rxvq = dev->data->rx_queues[i];
1477                 VIRTQUEUE_DUMP(rxvq->vq);
1478         }
1479
1480         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1481                 txvq = dev->data->tx_queues[i];
1482                 VIRTQUEUE_DUMP(txvq->vq);
1483         }
1484
1485         return 0;
1486 }
1487
1488 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
1489 {
1490         struct rte_mbuf *buf;
1491         int i, mbuf_num = 0;
1492
1493         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1494                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1495
1496                 PMD_INIT_LOG(DEBUG,
1497                              "Before freeing rxq[%d] used and unused buf", i);
1498                 VIRTQUEUE_DUMP(rxvq->vq);
1499
1500                 PMD_INIT_LOG(DEBUG, "rx_queues[%d]=%p", i, rxvq);
1501                 while ((buf = virtqueue_detatch_unused(rxvq->vq)) != NULL) {
1502                         rte_pktmbuf_free(buf);
1503                         mbuf_num++;
1504                 }
1505
1506                 PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num);
1507                 PMD_INIT_LOG(DEBUG,
1508                              "After freeing rxq[%d] used and unused buf", i);
1509                 VIRTQUEUE_DUMP(rxvq->vq);
1510         }
1511
1512         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1513                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1514
1515                 PMD_INIT_LOG(DEBUG,
1516                              "Before freeing txq[%d] used and unused bufs",
1517                              i);
1518                 VIRTQUEUE_DUMP(txvq->vq);
1519
1520                 mbuf_num = 0;
1521                 while ((buf = virtqueue_detatch_unused(txvq->vq)) != NULL) {
1522                         rte_pktmbuf_free(buf);
1523                         mbuf_num++;
1524                 }
1525
1526                 PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num);
1527                 PMD_INIT_LOG(DEBUG,
1528                              "After freeing txq[%d] used and unused buf", i);
1529                 VIRTQUEUE_DUMP(txvq->vq);
1530         }
1531 }
1532
1533 /*
1534  * Stop device: disable interrupt and mark link down
1535  */
1536 static void
1537 virtio_dev_stop(struct rte_eth_dev *dev)
1538 {
1539         struct rte_eth_link link;
1540
1541         PMD_INIT_LOG(DEBUG, "stop");
1542
1543         if (dev->data->dev_conf.intr_conf.lsc)
1544                 rte_intr_disable(&dev->pci_dev->intr_handle);
1545
1546         memset(&link, 0, sizeof(link));
1547         virtio_dev_atomic_write_link_status(dev, &link);
1548 }
1549
1550 static int
1551 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
1552 {
1553         struct rte_eth_link link, old;
1554         uint16_t status;
1555         struct virtio_hw *hw = dev->data->dev_private;
1556         memset(&link, 0, sizeof(link));
1557         virtio_dev_atomic_read_link_status(dev, &link);
1558         old = link;
1559         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1560         link.link_speed  = SPEED_10G;
1561
1562         if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1563                 PMD_INIT_LOG(DEBUG, "Get link status from hw");
1564                 vtpci_read_dev_config(hw,
1565                                 offsetof(struct virtio_net_config, status),
1566                                 &status, sizeof(status));
1567                 if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
1568                         link.link_status = ETH_LINK_DOWN;
1569                         PMD_INIT_LOG(DEBUG, "Port %d is down",
1570                                      dev->data->port_id);
1571                 } else {
1572                         link.link_status = ETH_LINK_UP;
1573                         PMD_INIT_LOG(DEBUG, "Port %d is up",
1574                                      dev->data->port_id);
1575                 }
1576         } else {
1577                 link.link_status = ETH_LINK_UP;
1578         }
1579         virtio_dev_atomic_write_link_status(dev, &link);
1580
1581         return (old.link_status == link.link_status) ? -1 : 0;
1582 }
1583
1584 static void
1585 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1586 {
1587         uint64_t tso_mask;
1588         struct virtio_hw *hw = dev->data->dev_private;
1589
1590         if (dev->pci_dev)
1591                 dev_info->driver_name = dev->driver->pci_drv.driver.name;
1592         else
1593                 dev_info->driver_name = "virtio_user PMD";
1594         dev_info->max_rx_queues =
1595                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
1596         dev_info->max_tx_queues =
1597                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
1598         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
1599         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
1600         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
1601         dev_info->default_txconf = (struct rte_eth_txconf) {
1602                 .txq_flags = ETH_TXQ_FLAGS_NOOFFLOADS
1603         };
1604         dev_info->rx_offload_capa =
1605                 DEV_RX_OFFLOAD_TCP_CKSUM |
1606                 DEV_RX_OFFLOAD_UDP_CKSUM |
1607                 DEV_RX_OFFLOAD_TCP_LRO;
1608         dev_info->tx_offload_capa = 0;
1609
1610         if (hw->guest_features & (1ULL << VIRTIO_NET_F_CSUM)) {
1611                 dev_info->tx_offload_capa |=
1612                         DEV_TX_OFFLOAD_UDP_CKSUM |
1613                         DEV_TX_OFFLOAD_TCP_CKSUM;
1614         }
1615
1616         tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
1617                 (1ULL << VIRTIO_NET_F_HOST_TSO6);
1618         if ((hw->guest_features & tso_mask) == tso_mask)
1619                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
1620 }
1621
1622 /*
1623  * It enables testpmd to collect per queue stats.
1624  */
1625 static int
1626 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
1627 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
1628 __rte_unused uint8_t is_rx)
1629 {
1630         return 0;
1631 }
1632
1633 RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__);
1634 RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map);