lib/librte_pmd_virtio/virtio_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <string.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <unistd.h>
  39 #ifdef RTE_EXEC_ENV_LINUXAPP
  40 #include <dirent.h>
  41 #endif
  42
  43 #include <rte_ethdev.h>
  44 #include <rte_memcpy.h>
  45 #include <rte_string_fns.h>
  46 #include <rte_memzone.h>
  47 #include <rte_malloc.h>
  48 #include <rte_atomic.h>
  49 #include <rte_branch_prediction.h>
  50 #include <rte_pci.h>
  51 #include <rte_ether.h>
  52 #include <rte_common.h>
  53
  54 #include <rte_memory.h>
  55 #include <rte_eal.h>
  56 #include <rte_dev.h>
  57
  58 #include "virtio_ethdev.h"
  59 #include "virtio_pci.h"
  60 #include "virtio_logs.h"
  61 #include "virtqueue.h"
  62
  63
  64 static int eth_virtio_dev_init(struct eth_driver *eth_drv,
  65                 struct rte_eth_dev *eth_dev);
  66 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  67 static int  virtio_dev_start(struct rte_eth_dev *dev);
  68 static void virtio_dev_stop(struct rte_eth_dev *dev);
  69 static void virtio_dev_info_get(struct rte_eth_dev *dev,
  70                                 struct rte_eth_dev_info *dev_info);
  71 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  72         __rte_unused int wait_to_complete);
  73
  74 static void virtio_set_hwaddr(struct virtio_hw *hw);
  75 static void virtio_get_hwaddr(struct virtio_hw *hw);
  76
  77 static void virtio_dev_rx_queue_release(__rte_unused void *rxq);
  78 static void virtio_dev_tx_queue_release(__rte_unused void *txq);
  79
  80 static void virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
  81 static void virtio_dev_stats_reset(struct rte_eth_dev *dev);
  82 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  83
  84 static int virtio_dev_queue_stats_mapping_set(
  85         __rte_unused struct rte_eth_dev *eth_dev,
  86         __rte_unused uint16_t queue_id,
  87         __rte_unused uint8_t stat_idx,
  88         __rte_unused uint8_t is_rx);
  89
  90 /*
  91  * The set of PCI devices this driver supports
  92  */
  93 static struct rte_pci_id pci_id_virtio_map[] = {
  94
  95 #define RTE_PCI_DEV_ID_DECL_VIRTIO(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
  96 #include "rte_pci_dev_ids.h"
  97
  98 { .vendor_id = 0, /* sentinel */ },
  99 };
 100
 101 static int
 102 virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl,
 103                 int *dlen, int pkt_num)
 104 {
 105         uint32_t head = vq->vq_desc_head_idx, i;
 106         int k, sum = 0;
 107         virtio_net_ctrl_ack status = ~0;
 108         struct virtio_pmd_ctrl result;
 109
 110         ctrl->status = status;
 111
 112         if (!vq->hw->cvq) {
 113                 PMD_INIT_LOG(ERR, "%s(): Control queue is "
 114                         "not supported by this device.\n", __func__);
 115                 return -1;
 116         }
 117
 118         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 119                 "vq->hw->cvq = %p vq = %p\n",
 120                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 121
 122         if ((vq->vq_free_cnt < ((uint32_t)pkt_num + 2)) || (pkt_num < 1))
 123                 return -1;
 124
 125         memcpy(vq->virtio_net_hdr_mz->addr, ctrl,
 126                 sizeof(struct virtio_pmd_ctrl));
 127
 128         /*
 129          * Format is enforced in qemu code:
 130          * One TX packet for header;
 131          * At least one TX packet per argument;
 132          * One RX packet for ACK.
 133          */
 134         vq->vq_ring.desc[head].flags = VRING_DESC_F_NEXT;
 135         vq->vq_ring.desc[head].addr = vq->virtio_net_hdr_mz->phys_addr;
 136         vq->vq_ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 137         vq->vq_free_cnt--;
 138         i = vq->vq_ring.desc[head].next;
 139
 140         for (k = 0; k < pkt_num; k++) {
 141                 vq->vq_ring.desc[i].flags = VRING_DESC_F_NEXT;
 142                 vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr
 143                         + sizeof(struct virtio_net_ctrl_hdr)
 144                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 145                 vq->vq_ring.desc[i].len = dlen[k];
 146                 sum += dlen[k];
 147                 vq->vq_free_cnt--;
 148                 i = vq->vq_ring.desc[i].next;
 149         }
 150
 151         vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
 152         vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr
 153                         + sizeof(struct virtio_net_ctrl_hdr);
 154         vq->vq_ring.desc[i].len = sizeof(ctrl->status);
 155         vq->vq_free_cnt--;
 156
 157         vq->vq_desc_head_idx = vq->vq_ring.desc[i].next;
 158
 159         vq_update_avail_ring(vq, head);
 160         vq_update_avail_idx(vq);
 161
 162         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d\n", vq->vq_queue_index);
 163
 164         virtqueue_notify(vq);
 165
 166         while (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
 167                 usleep(100);
 168
 169         while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
 170                 uint32_t idx, desc_idx, used_idx;
 171                 struct vring_used_elem *uep;
 172
 173                 rmb();
 174
 175                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 176                                 & (vq->vq_nentries - 1));
 177                 uep = &vq->vq_ring.used->ring[used_idx];
 178                 idx = (uint32_t) uep->id;
 179                 desc_idx = idx;
 180
 181                 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
 182                         desc_idx = vq->vq_ring.desc[desc_idx].next;
 183                         vq->vq_free_cnt++;
 184                 }
 185
 186                 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 187                 vq->vq_desc_head_idx = idx;
 188
 189                 vq->vq_used_cons_idx++;
 190                 vq->vq_free_cnt++;
 191         }
 192
 193         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d\n",
 194                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 195
 196         memcpy(&result, vq->virtio_net_hdr_mz->addr,
 197                         sizeof(struct virtio_pmd_ctrl));
 198
 199         return result.status;
 200 }
 201
 202 static int
 203 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 204 {
 205         struct virtio_hw *hw
 206                 = VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 207         struct virtio_pmd_ctrl ctrl;
 208         int dlen[1];
 209         int ret;
 210
 211         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 212         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 213         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 214
 215         PMD_INIT_LOG(DEBUG, "ctrl.data=%d\n", *(int *)ctrl.data);
 216
 217         dlen[0] = sizeof(uint16_t);
 218
 219         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 220
 221         if (ret) {
 222                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 223                           "failed, this is too late now...\n");
 224                 return -EINVAL;
 225         }
 226
 227         return 0;
 228 }
 229
 230 int virtio_dev_queue_setup(struct rte_eth_dev *dev,
 231                         int queue_type,
 232                         uint16_t queue_idx,
 233                         uint8_t  vtpci_queue_idx,
 234                         uint16_t nb_desc,
 235                         unsigned int socket_id,
 236                         struct virtqueue **pvq)
 237 {
 238         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 239         const struct rte_memzone *mz;
 240         uint16_t vq_size;
 241         int size;
 242         struct virtio_hw *hw =
 243                 VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 244         struct virtqueue  *vq = NULL;
 245
 246         /* Write the virtqueue index to the Queue Select Field */
 247         VIRTIO_WRITE_REG_2(hw, VIRTIO_PCI_QUEUE_SEL, vtpci_queue_idx);
 248         PMD_INIT_LOG(DEBUG, "selecting queue: %d\n", vtpci_queue_idx);
 249
 250         /*
 251          * Read the virtqueue size from the Queue Size field
 252          * Always power of 2 and if 0 virtqueue does not exist
 253          */
 254         vq_size = VIRTIO_READ_REG_2(hw, VIRTIO_PCI_QUEUE_NUM);
 255         PMD_INIT_LOG(DEBUG, "vq_size: %d nb_desc:%d\n", vq_size, nb_desc);
 256         if (nb_desc == 0)
 257                 nb_desc = vq_size;
 258         if (vq_size == 0) {
 259                 PMD_INIT_LOG(ERR, "%s: virtqueue does not exist\n", __func__);
 260                 return (-EINVAL);
 261         } else if (!rte_is_power_of_2(vq_size)) {
 262                 PMD_INIT_LOG(ERR, "%s: virtqueue size is not powerof 2\n", __func__);
 263                 return (-EINVAL);
 264         } else if (nb_desc != vq_size) {
 265                 PMD_INIT_LOG(ERR, "Warning: nb_desc(%d) is not equal to vq size (%d), fall to vq size\n",
 266                         nb_desc, vq_size);
 267                 nb_desc = vq_size;
 268         }
 269
 270         if (queue_type == VTNET_RQ) {
 271                 rte_snprintf(vq_name, sizeof(vq_name), "port%d_rvq%d",
 272                         dev->data->port_id, queue_idx);
 273                 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) +
 274                         vq_size * sizeof(struct vq_desc_extra), CACHE_LINE_SIZE);
 275                 memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name));
 276         } else if(queue_type == VTNET_TQ) {
 277                 rte_snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d",
 278                         dev->data->port_id, queue_idx);
 279                 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) +
 280                         vq_size * sizeof(struct vq_desc_extra), CACHE_LINE_SIZE);
 281                 memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name));
 282         } else if(queue_type == VTNET_CQ) {
 283                 rte_snprintf(vq_name, sizeof(vq_name), "port%d_cvq",
 284                         dev->data->port_id);
 285                 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) +
 286                         vq_size * sizeof(struct vq_desc_extra),
 287                         CACHE_LINE_SIZE);
 288                 memcpy(vq->vq_name, vq_name, sizeof(vq->vq_name));
 289         }
 290         if (vq == NULL) {
 291                 PMD_INIT_LOG(ERR, "%s: Can not allocate virtqueue\n", __func__);
 292                 return (-ENOMEM);
 293         }
 294
 295         vq->hw = hw;
 296         vq->port_id = dev->data->port_id;
 297         vq->queue_id = queue_idx;
 298         vq->vq_queue_index = vtpci_queue_idx;
 299         vq->vq_alignment = VIRTIO_PCI_VRING_ALIGN;
 300         vq->vq_nentries = vq_size;
 301         vq->vq_free_cnt = vq_size;
 302
 303         /*
 304          * Reserve a memzone for vring elements
 305          */
 306         size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
 307         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
 308         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d\n", size, vq->vq_ring_size);
 309
 310         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 311                 socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
 312         if (mz == NULL) {
 313                 rte_free(vq);
 314                 return (-ENOMEM);
 315         }
 316
 317         /*
 318         * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
 319         * and only accepts 32 bit page frame number.
 320         * Check if the allocated physical memory exceeds 16TB.
 321         */
 322         if ( (mz->phys_addr + vq->vq_ring_size - 1) >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32) ) {
 323                 PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!\n");
 324                 rte_free(vq);
 325                 return (-ENOMEM);
 326         }
 327
 328         memset(mz->addr, 0, sizeof(mz->len));
 329         vq->mz = mz;
 330         vq->vq_ring_mem = mz->phys_addr;
 331         vq->vq_ring_virt_mem = mz->addr;
 332         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem:      0x%"PRIx64"\n", (uint64_t)mz->phys_addr);
 333         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%"PRIx64"\n", (uint64_t)mz->addr);
 334         vq->virtio_net_hdr_mz  = NULL;
 335         vq->virtio_net_hdr_mem = (void *)NULL;
 336
 337         if (queue_type == VTNET_TQ) {
 338                 /*
 339                  * For each xmit packet, allocate a virtio_net_hdr
 340                  */
 341                 rte_snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d_hdrzone",
 342                         dev->data->port_id, queue_idx);
 343                 vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name,
 344                         vq_size * sizeof(struct virtio_net_hdr),
 345                         socket_id, 0, CACHE_LINE_SIZE);
 346                 if (vq->virtio_net_hdr_mz == NULL) {
 347                         rte_free(vq);
 348                         return -ENOMEM;
 349                 }
 350                 vq->virtio_net_hdr_mem =
 351                         (void *)(uintptr_t)vq->virtio_net_hdr_mz->phys_addr;
 352                 memset(vq->virtio_net_hdr_mz->addr, 0,
 353                         vq_size * sizeof(struct virtio_net_hdr));
 354         } else if (queue_type == VTNET_CQ) {
 355                 /* Allocate a page for control vq command, data and status */
 356                 rte_snprintf(vq_name, sizeof(vq_name), "port%d_cvq_hdrzone",
 357                         dev->data->port_id);
 358                 vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name,
 359                         PAGE_SIZE, socket_id, 0, CACHE_LINE_SIZE);
 360                 if (vq->virtio_net_hdr_mz == NULL) {
 361                         rte_free(vq);
 362                         return -ENOMEM;
 363                 }
 364                 vq->virtio_net_hdr_mem =
 365                         (void *)(uintptr_t)vq->virtio_net_hdr_mz->phys_addr;
 366                 memset(vq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE);
 367         }
 368
 369         /*
 370          * Set guest physical address of the virtqueue
 371          * in VIRTIO_PCI_QUEUE_PFN config register of device
 372          */
 373         VIRTIO_WRITE_REG_4(hw, VIRTIO_PCI_QUEUE_PFN,
 374                         mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 375         *pvq = vq;
 376         return 0;
 377 }
 378
 379 static int
 380 virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx,
 381                 uint32_t socket_id)
 382 {
 383         struct virtqueue *vq;
 384         uint16_t nb_desc = 0;
 385         int ret;
 386         struct virtio_hw *hw =
 387                 VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 388
 389         PMD_INIT_FUNC_TRACE();
 390         ret = virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX,
 391                         vtpci_queue_idx, nb_desc, socket_id, &vq);
 392
 393         if (ret < 0) {
 394                 PMD_INIT_LOG(ERR, "control vq initialization failed\n");
 395                 return ret;
 396         }
 397
 398         hw->cvq = vq;
 399         return 0;
 400 }
 401
 402 static void
 403 virtio_dev_close(struct rte_eth_dev *dev)
 404 {
 405         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 406
 407         virtio_dev_stop(dev);
 408 }
 409
 410 /*
 411  * dev_ops for virtio, bare necessities for basic operation
 412  */
 413 static struct eth_dev_ops virtio_eth_dev_ops = {
 414         .dev_configure           = virtio_dev_configure,
 415         .dev_start               = virtio_dev_start,
 416         .dev_stop                = virtio_dev_stop,
 417         .dev_close               = virtio_dev_close,
 418
 419         .dev_infos_get           = virtio_dev_info_get,
 420         .stats_get               = virtio_dev_stats_get,
 421         .stats_reset             = virtio_dev_stats_reset,
 422         .link_update             = virtio_dev_link_update,
 423         .mac_addr_add            = NULL,
 424         .mac_addr_remove         = NULL,
 425         .rx_queue_setup          = virtio_dev_rx_queue_setup,
 426         /* meaningfull only to multiple queue */
 427         .rx_queue_release        = virtio_dev_rx_queue_release,
 428         .tx_queue_setup          = virtio_dev_tx_queue_setup,
 429         /* meaningfull only to multiple queue */
 430         .tx_queue_release        = virtio_dev_tx_queue_release,
 431         /* collect stats per queue */
 432         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set
 433 };
 434
 435 static inline int
 436 virtio_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 437                                 struct rte_eth_link *link)
 438 {
 439         struct rte_eth_link *dst = link;
 440         struct rte_eth_link *src = &(dev->data->dev_link);
 441
 442         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 443                         *(uint64_t *)src) == 0)
 444                 return (-1);
 445
 446         return (0);
 447 }
 448
 449 /**
 450  * Atomically writes the link status information into global
 451  * structure rte_eth_dev.
 452  *
 453  * @param dev
 454  *   - Pointer to the structure rte_eth_dev to read from.
 455  *   - Pointer to the buffer to be saved with the link status.
 456  *
 457  * @return
 458  *   - On success, zero.
 459  *   - On failure, negative value.
 460  */
 461 static inline int
 462 virtio_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 463                 struct rte_eth_link *link)
 464 {
 465         struct rte_eth_link *dst = &(dev->data->dev_link);
 466         struct rte_eth_link *src = link;
 467
 468         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 469                                         *(uint64_t *)src) == 0)
 470                 return (-1);
 471
 472         return (0);
 473 }
 474
 475 static void
 476 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 477 {
 478         struct virtio_hw *hw =
 479                 VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 480         if(stats)
 481                 memcpy(stats, &hw->eth_stats, sizeof(*stats));
 482 }
 483
 484 static void
 485 virtio_dev_stats_reset(struct rte_eth_dev *dev)
 486 {
 487         struct virtio_hw *hw =
 488                 VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489         /* Reset software totals */
 490         memset(&hw->eth_stats, 0, sizeof(hw->eth_stats));
 491 }
 492
 493 static void
 494 virtio_set_hwaddr(struct virtio_hw *hw)
 495 {
 496         vtpci_write_dev_config(hw,
 497                         offsetof(struct virtio_net_config, mac),
 498                         &hw->mac_addr, ETHER_ADDR_LEN);
 499 }
 500
 501 static void
 502 virtio_get_hwaddr(struct virtio_hw *hw)
 503 {
 504         if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) {
 505                 vtpci_read_dev_config(hw,
 506                         offsetof(struct virtio_net_config, mac),
 507                         &hw->mac_addr, ETHER_ADDR_LEN);
 508         } else {
 509                 eth_random_addr(&hw->mac_addr[0]);
 510                 virtio_set_hwaddr(hw);
 511         }
 512 }
 513
 514
 515 static void
 516 virtio_negotiate_features(struct virtio_hw *hw)
 517 {
 518         uint32_t guest_features, mask;
 519         mask = VIRTIO_NET_F_CTRL_RX | VIRTIO_NET_F_CTRL_VLAN;
 520         mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM ;
 521
 522         /* TSO and LRO are only available when their corresponding
 523          * checksum offload feature is also negotiated.
 524          */
 525         mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 | VIRTIO_NET_F_HOST_ECN;
 526         mask |= VIRTIO_NET_F_GUEST_TSO4 | VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN;
 527         mask |= VTNET_LRO_FEATURES;
 528
 529         /* rx_mbuf should not be in multiple merged segments */
 530         mask |= VIRTIO_NET_F_MRG_RXBUF;
 531
 532         /* not negotiating INDIRECT descriptor table support */
 533         mask |= VIRTIO_RING_F_INDIRECT_DESC;
 534
 535         /* Prepare guest_features: feature that driver wants to support */
 536         guest_features = VTNET_FEATURES & ~mask;
 537         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %x\n",
 538                 guest_features);
 539
 540         /* Read device(host) feature bits */
 541         hw->host_features = VIRTIO_READ_REG_4(hw, VIRTIO_PCI_HOST_FEATURES);
 542         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %x\n",
 543                 hw->host_features);
 544
 545         /*
 546          * Negotiate features: Subset of device feature bits are written back
 547          * guest feature bits.
 548          */
 549         hw->guest_features = vtpci_negotiate_features(hw, guest_features);
 550         PMD_INIT_LOG(DEBUG, "features after negotiate = %x\n",
 551                 hw->guest_features);
 552 }
 553
 554 #ifdef RTE_EXEC_ENV_LINUXAPP
 555 static int
 556 parse_sysfs_value(const char *filename, unsigned long *val)
 557 {
 558         FILE *f;
 559         char buf[BUFSIZ];
 560         char *end = NULL;
 561
 562         if ((f = fopen(filename, "r")) == NULL) {
 563                 PMD_INIT_LOG(ERR, "%s(): cannot open sysfs value %s\n",
 564                              __func__, filename);
 565                 return -1;
 566         }
 567
 568         if (fgets(buf, sizeof(buf), f) == NULL) {
 569                 PMD_INIT_LOG(ERR, "%s(): cannot read sysfs value %s\n",
 570                              __func__, filename);
 571                 fclose(f);
 572                 return -1;
 573         }
 574         *val = strtoul(buf, &end, 0);
 575         if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) {
 576                 PMD_INIT_LOG(ERR, "%s(): cannot parse sysfs value %s\n",
 577                              __func__, filename);
 578                 fclose(f);
 579                 return -1;
 580         }
 581         fclose(f);
 582         return 0;
 583 }
 584
 585 static int get_uio_dev(struct rte_pci_addr *loc, char *buf, unsigned int buflen)
 586 {
 587         unsigned int uio_num;
 588         struct dirent *e;
 589         DIR *dir;
 590         char dirname[PATH_MAX];
 591
 592         /* depending on kernel version, uio can be located in uio/uioX
 593          * or uio:uioX */
 594         rte_snprintf(dirname, sizeof(dirname),
 595                  SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/uio",
 596                  loc->domain, loc->bus, loc->devid, loc->function);
 597         dir = opendir(dirname);
 598         if (dir == NULL) {
 599                 /* retry with the parent directory */
 600                 rte_snprintf(dirname, sizeof(dirname),
 601                          SYSFS_PCI_DEVICES "/" PCI_PRI_FMT,
 602                          loc->domain, loc->bus, loc->devid, loc->function);
 603                 dir = opendir(dirname);
 604
 605                 if (dir == NULL) {
 606                         PMD_INIT_LOG(ERR, "Cannot opendir %s\n", dirname);
 607                         return -1;
 608                 }
 609         }
 610
 611         /* take the first file starting with "uio" */
 612         while ((e = readdir(dir)) != NULL) {
 613                 /* format could be uio%d ...*/
 614                 int shortprefix_len = sizeof("uio") - 1;
 615                 /* ... or uio:uio%d */
 616                 int longprefix_len = sizeof("uio:uio") - 1;
 617                 char *endptr;
 618
 619                 if (strncmp(e->d_name, "uio", 3) != 0)
 620                         continue;
 621
 622                 /* first try uio%d */
 623                 errno = 0;
 624                 uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10);
 625                 if (errno == 0 && endptr != (e->d_name + shortprefix_len)) {
 626                         rte_snprintf(buf, buflen, "%s/uio%u", dirname, uio_num);
 627                         break;
 628                 }
 629
 630                 /* then try uio:uio%d */
 631                 errno = 0;
 632                 uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10);
 633                 if (errno == 0 && endptr != (e->d_name + longprefix_len)) {
 634                         rte_snprintf(buf, buflen, "%s/uio:uio%u", dirname,
 635                                      uio_num);
 636                         break;
 637                 }
 638         }
 639         closedir(dir);
 640
 641         /* No uio resource found */
 642         if (e == NULL) {
 643                 PMD_INIT_LOG(ERR, "Could not find uio resource\n");
 644                 return -1;
 645         }
 646
 647         return 0;
 648 }
 649 #endif
 650
 651 /*
 652  * This function is based on probe() function in virtio_pci.c
 653  * It returns 0 on success.
 654  */
 655 static int
 656 eth_virtio_dev_init(__rte_unused struct eth_driver *eth_drv,
 657                 struct rte_eth_dev *eth_dev)
 658 {
 659         struct virtio_net_config *config;
 660         struct virtio_net_config local_config;
 661         uint32_t offset_conf = sizeof(config->mac);
 662         struct rte_pci_device *pci_dev;
 663         struct virtio_hw *hw =
 664                 VIRTIO_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 665
 666         if (RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr) ) {
 667                 PMD_INIT_LOG(ERR,
 668                         "MBUF HEADROOM should be enough to hold virtio net hdr\n");
 669                 return (-1);
 670         }
 671
 672         if (! (rte_eal_get_configuration()->flags & EAL_FLG_HIGH_IOPL)) {
 673                 PMD_INIT_LOG(ERR,
 674                         "IOPL call failed in EAL init - cannot use virtio PMD driver\n");
 675                 return (-1);
 676         }
 677
 678         eth_dev->dev_ops = &virtio_eth_dev_ops;
 679         eth_dev->rx_pkt_burst = &virtio_recv_pkts;
 680         eth_dev->tx_pkt_burst = &virtio_xmit_pkts;
 681
 682         if(rte_eal_process_type() == RTE_PROC_SECONDARY)
 683                 return 0;
 684
 685         pci_dev = eth_dev->pci_dev;
 686
 687         hw->device_id = pci_dev->id.device_id;
 688         hw->vendor_id = pci_dev->id.vendor_id;
 689 #ifdef RTE_EXEC_ENV_LINUXAPP
 690         {
 691                 char dirname[PATH_MAX];
 692                 char filename[PATH_MAX];
 693                 unsigned long start,size;
 694
 695                 if (get_uio_dev(&pci_dev->addr, dirname, sizeof(dirname)) < 0)
 696                         return -1;
 697
 698                 /* get portio size */
 699                 rte_snprintf(filename, sizeof(filename),
 700                              "%s/portio/port0/size", dirname);
 701                 if (parse_sysfs_value(filename, &size) < 0) {
 702                         PMD_INIT_LOG(ERR, "%s(): cannot parse size\n",
 703                                      __func__);
 704                         return -1;
 705                 }
 706
 707                 /* get portio start */
 708                 rte_snprintf(filename, sizeof(filename),
 709                              "%s/portio/port0/start", dirname);
 710                 if (parse_sysfs_value(filename, &start) < 0) {
 711                         PMD_INIT_LOG(ERR, "%s(): cannot parse portio start\n",
 712                                      __func__);
 713                         return -1;
 714                 }
 715                 pci_dev->mem_resource[0].addr = (void *)(uintptr_t)start;
 716                 pci_dev->mem_resource[0].len =  (uint64_t)size;
 717                 PMD_INIT_LOG(DEBUG, "PCI Port IO found start=0x%lx with "
 718                              "size=0x%lx\n", start, size);
 719         }
 720 #endif
 721         hw->io_base = (uint32_t)(uintptr_t)pci_dev->mem_resource[0].addr;
 722
 723         /* Reset the device although not necessary at startup */
 724         vtpci_reset(hw);
 725
 726         /* Tell the host we've noticed this device. */
 727         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
 728
 729         /* Tell the host we've known how to drive the device. */
 730         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
 731         virtio_negotiate_features(hw);
 732
 733         /* Setting up rx_header size for the device */
 734         if(vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF))
 735                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 736         else
 737                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
 738
 739         /* Allocate memory for storing MAC addresses */
 740         eth_dev->data->mac_addrs = rte_zmalloc("virtio", ETHER_ADDR_LEN, 0);
 741         if (eth_dev->data->mac_addrs == NULL) {
 742                 PMD_INIT_LOG(ERR,
 743                         "Failed to allocate %d bytes needed to store MAC addresses",
 744                         ETHER_ADDR_LEN);
 745                 return (-ENOMEM);
 746         }
 747
 748         /* Copy the permanent MAC address to: virtio_hw */
 749         virtio_get_hwaddr(hw);
 750         ether_addr_copy((struct ether_addr *) hw->mac_addr,
 751                         &eth_dev->data->mac_addrs[0]);
 752         PMD_INIT_LOG(DEBUG, "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X\n", hw->mac_addr[0],
 753                         hw->mac_addr[1],hw->mac_addr[2], hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
 754
 755         if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
 756                 config = &local_config;
 757
 758                 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
 759                         offset_conf += sizeof(config->status);
 760                 } else {
 761                         PMD_INIT_LOG(DEBUG, "VIRTIO_NET_F_STATUS is "
 762                                 "not supported\n");
 763                         config->status = 0;
 764                 }
 765
 766                 if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) {
 767                         offset_conf += sizeof(config->max_virtqueue_pairs);
 768                 } else {
 769                         PMD_INIT_LOG(DEBUG, "VIRTIO_NET_F_MQ is "
 770                                 "not supported\n");
 771                         config->max_virtqueue_pairs = 1;
 772                 }
 773
 774                 vtpci_read_dev_config(hw, 0, (uint8_t *)config, offset_conf);
 775
 776                 hw->max_rx_queues =
 777                         (VIRTIO_MAX_RX_QUEUES < config->max_virtqueue_pairs) ?
 778                         VIRTIO_MAX_RX_QUEUES : config->max_virtqueue_pairs;
 779                 hw->max_tx_queues =
 780                         (VIRTIO_MAX_TX_QUEUES < config->max_virtqueue_pairs) ?
 781                         VIRTIO_MAX_TX_QUEUES : config->max_virtqueue_pairs;
 782
 783                 virtio_dev_cq_queue_setup(eth_dev,
 784                                         config->max_virtqueue_pairs * 2,
 785                                         SOCKET_ID_ANY);
 786
 787                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d\n",
 788                                 config->max_virtqueue_pairs);
 789                 PMD_INIT_LOG(DEBUG, "config->status=%d\n", config->status);
 790                 PMD_INIT_LOG(DEBUG,
 791                                 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X\n",
 792                                 config->mac[0], config->mac[1],
 793                                 config->mac[2], config->mac[3],
 794                                 config->mac[4], config->mac[5]);
 795         } else {
 796                 hw->max_rx_queues = 1;
 797                 hw->max_tx_queues = 1;
 798         }
 799
 800         eth_dev->data->nb_rx_queues = hw->max_rx_queues;
 801         eth_dev->data->nb_tx_queues = hw->max_tx_queues;
 802
 803         PMD_INIT_LOG(DEBUG, "hw->max_rx_queues=%d   hw->max_tx_queues=%d\n",
 804                         hw->max_rx_queues, hw->max_tx_queues);
 805         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 806                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 807                         pci_dev->id.device_id);
 808         return 0;
 809 }
 810
 811 static struct eth_driver rte_virtio_pmd = {
 812         {
 813                 .name = "rte_virtio_pmd",
 814                 .id_table = pci_id_virtio_map,
 815                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 816         },
 817         .eth_dev_init = eth_virtio_dev_init,
 818         .dev_private_size = sizeof(struct virtio_adapter),
 819 };
 820
 821 /*
 822  * Driver initialization routine.
 823  * Invoked once at EAL init time.
 824  * Register itself as the [Poll Mode] Driver of PCI virtio devices.
 825  * Returns 0 on success.
 826  */
 827 static int
 828 rte_virtio_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
 829 {
 830         rte_eth_driver_register(&rte_virtio_pmd);
 831         return (0);
 832 }
 833
 834 /*
 835  * Only 1 queue is supported, no queue release related operation
 836  */
 837 static void
 838 virtio_dev_rx_queue_release(__rte_unused void *rxq)
 839 {
 840 }
 841
 842 static void
 843 virtio_dev_tx_queue_release(__rte_unused void *txq)
 844 {
 845 }
 846
 847 /*
 848  * Configure virtio device
 849  * It returns 0 on success.
 850  */
 851 static int
 852 virtio_dev_configure(__rte_unused struct rte_eth_dev *dev)
 853 {
 854         return (0);
 855 }
 856
 857
 858 static int
 859 virtio_dev_start(struct rte_eth_dev *dev)
 860 {
 861         uint16_t nb_queues, i;
 862         uint16_t status;
 863         struct virtio_hw *hw =
 864                 VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 865
 866         /* Tell the host we've noticed this device. */
 867         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
 868
 869         /* Tell the host we've known how to drive the device. */
 870         vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
 871
 872         hw->adapter_stopped = 0;
 873
 874         virtio_dev_cq_start(dev);
 875
 876         /* Do final configuration before rx/tx engine starts */
 877         virtio_dev_rxtx_start(dev);
 878
 879         /* Check VIRTIO_NET_F_STATUS for link status*/
 880         if(vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
 881                 vtpci_read_dev_config(hw,
 882                                 offsetof(struct virtio_net_config, status),
 883                                 &status, sizeof(status));
 884                 if((status & VIRTIO_NET_S_LINK_UP) == 0) {
 885                         PMD_INIT_LOG(ERR,     "Port: %d Link is DOWN\n", dev->data->port_id);
 886                         return (-EIO);
 887                 } else {
 888                         PMD_INIT_LOG(DEBUG, "Port: %d Link is UP\n",  dev->data->port_id);
 889                 }
 890         }
 891         vtpci_reinit_complete(hw);
 892
 893         /*Notify the backend
 894          *Otherwise the tap backend might already stop its queue due to fullness.
 895          *vhost backend will have no chance to be waked up
 896          */
 897         nb_queues = dev->data->nb_rx_queues;
 898         if (nb_queues > 1) {
 899                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
 900                         return -EINVAL;
 901         }
 902
 903         PMD_INIT_LOG(DEBUG, "nb_queues=%d\n", nb_queues);
 904
 905         for (i = 0; i < nb_queues; i++)
 906                 virtqueue_notify(dev->data->rx_queues[i]);
 907
 908         PMD_INIT_LOG(DEBUG, "Notified backend at initialization\n");
 909
 910         for (i = 0; i < dev->data->nb_rx_queues; i++)
 911                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 912
 913         for (i = 0; i < dev->data->nb_tx_queues; i++)
 914                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 915
 916         return 0;
 917 }
 918
 919 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
 920 {
 921         struct rte_mbuf *buf;
 922         int i, mbuf_num = 0;
 923         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 924                 PMD_INIT_LOG(DEBUG, "Before freeing rxq[%d] used and "
 925                         "unused buf\n", i);
 926                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 927
 928                 while ((buf = (struct rte_mbuf *)virtqueue_detatch_unused(
 929                                         dev->data->rx_queues[i])) != NULL) {
 930                         rte_pktmbuf_free_seg(buf);
 931                         mbuf_num++;
 932                 }
 933
 934                 PMD_INIT_LOG(DEBUG, "free %d mbufs\n", mbuf_num);
 935                 PMD_INIT_LOG(DEBUG, "After freeing rxq[%d] used and "
 936                         "unused buf\n", i);
 937                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]);
 938         }
 939
 940         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 941                 PMD_INIT_LOG(DEBUG, "Before freeing txq[%d] used and "
 942                         "unused bufs\n", i);
 943                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 944
 945                 mbuf_num = 0;
 946                 while ((buf = (struct rte_mbuf *)virtqueue_detatch_unused(
 947                                         dev->data->tx_queues[i])) != NULL) {
 948                         rte_pktmbuf_free_seg(buf);
 949                         mbuf_num++;
 950                 }
 951
 952                 PMD_INIT_LOG(DEBUG, "free %d mbufs\n", mbuf_num);
 953                 PMD_INIT_LOG(DEBUG, "After freeing txq[%d] used and "
 954                         "unused buf\n", i);
 955                 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]);
 956         }
 957 }
 958
 959 /*
 960  * Stop device: disable rx and tx functions to allow for reconfiguring.
 961  */
 962 static void
 963 virtio_dev_stop(struct rte_eth_dev *dev)
 964 {
 965         struct virtio_hw *hw =
 966                 VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 967
 968         /* reset the NIC */
 969         vtpci_reset(hw);
 970         virtio_dev_free_mbufs(dev);
 971 }
 972
 973 static int
 974 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
 975 {
 976         struct rte_eth_link link, old;
 977         uint16_t status;
 978         struct virtio_hw *hw =
 979                 VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 980         memset(&link, 0, sizeof(link));
 981         virtio_dev_atomic_read_link_status(dev, &link);
 982         old = link;
 983         link.link_duplex = FULL_DUPLEX ;
 984         link.link_speed  = SPEED_10G ;
 985         if(vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
 986                 PMD_INIT_LOG(DEBUG, "Get link status from hw\n");
 987                 vtpci_read_dev_config(hw,
 988                                 offsetof(struct virtio_net_config, status),
 989                                 &status, sizeof(status));
 990                 if((status & VIRTIO_NET_S_LINK_UP) == 0) {
 991                         link.link_status = 0;
 992                         PMD_INIT_LOG(DEBUG, "Port %d is down\n",dev->data->port_id);
 993                 } else {
 994                         link.link_status = 1;
 995                         PMD_INIT_LOG(DEBUG, "Port %d is up\n",dev->data->port_id);
 996                 }
 997         } else {
 998                 link.link_status = 1;   //Link up
 999         }
1000         virtio_dev_atomic_write_link_status(dev, &link);
1001         if(old.link_status == link.link_status)
1002                 return (-1);
1003         /*changed*/
1004         return (0);
1005 }
1006
1007 static void
1008 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1009 {
1010         struct virtio_hw *hw = VIRTIO_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1011         dev_info->driver_name = dev->driver->pci_drv.name;
1012         dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues;
1013         dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues;
1014         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
1015         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
1016         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
1017 }
1018
1019 /*
1020  * It enables testpmd to collect per queue stats.
1021  */
1022 static int
1023 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
1024 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
1025 __rte_unused uint8_t is_rx)
1026 {
1027         return 0;
1028 }
1029
1030 static struct rte_driver rte_virtio_driver = {
1031         .type = PMD_PDEV,
1032         .init = rte_virtio_pmd_init,
1033 };
1034
1035 PMD_REGISTER_DRIVER(rte_virtio_driver);