drivers/net/virtio/virtio_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10
  11 #include <ethdev_driver.h>
  12 #include <rte_memcpy.h>
  13 #include <rte_string_fns.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_branch_prediction.h>
  17 #include <rte_ether.h>
  18 #include <rte_ip.h>
  19 #include <rte_arp.h>
  20 #include <rte_common.h>
  21 #include <rte_errno.h>
  22 #include <rte_cpuflags.h>
  23 #include <rte_vect.h>
  24 #include <rte_memory.h>
  25 #include <rte_eal_paging.h>
  26 #include <rte_eal.h>
  27 #include <rte_dev.h>
  28 #include <rte_cycles.h>
  29 #include <rte_kvargs.h>
  30
  31 #include "virtio_ethdev.h"
  32 #include "virtio.h"
  33 #include "virtio_logs.h"
  34 #include "virtqueue.h"
  35 #include "virtio_rxtx.h"
  36 #include "virtio_rxtx_simple.h"
  37 #include "virtio_user/virtio_user_dev.h"
  38
  39 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  40 static int  virtio_dev_start(struct rte_eth_dev *dev);
  41 static int virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
  42 static int virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
  43 static int virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
  44 static int virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
  45 static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
  46 static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
  47         uint32_t *speed,
  48         int *vectorized);
  49 static int virtio_dev_info_get(struct rte_eth_dev *dev,
  50                                 struct rte_eth_dev_info *dev_info);
  51 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  52         int wait_to_complete);
  53 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  54
  55 static void virtio_set_hwaddr(struct virtio_hw *hw);
  56 static void virtio_get_hwaddr(struct virtio_hw *hw);
  57
  58 static int virtio_dev_stats_get(struct rte_eth_dev *dev,
  59                                  struct rte_eth_stats *stats);
  60 static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
  61                                  struct rte_eth_xstat *xstats, unsigned n);
  62 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
  63                                        struct rte_eth_xstat_name *xstats_names,
  64                                        unsigned limit);
  65 static int virtio_dev_stats_reset(struct rte_eth_dev *dev);
  66 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  67 static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
  68                                 uint16_t vlan_id, int on);
  69 static int virtio_mac_addr_add(struct rte_eth_dev *dev,
  70                                 struct rte_ether_addr *mac_addr,
  71                                 uint32_t index, uint32_t vmdq);
  72 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  73 static int virtio_mac_addr_set(struct rte_eth_dev *dev,
  74                                 struct rte_ether_addr *mac_addr);
  75
  76 static int virtio_intr_disable(struct rte_eth_dev *dev);
  77
  78 static int virtio_dev_queue_stats_mapping_set(
  79         struct rte_eth_dev *eth_dev,
  80         uint16_t queue_id,
  81         uint8_t stat_idx,
  82         uint8_t is_rx);
  83
  84 static void virtio_notify_peers(struct rte_eth_dev *dev);
  85 static void virtio_ack_link_announce(struct rte_eth_dev *dev);
  86
  87 struct rte_virtio_xstats_name_off {
  88         char name[RTE_ETH_XSTATS_NAME_SIZE];
  89         unsigned offset;
  90 };
  91
  92 /* [rt]x_qX_ is prepended to the name string here */
  93 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
  94         {"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
  95         {"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
  96         {"errors",                 offsetof(struct virtnet_rx, stats.errors)},
  97         {"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
  98         {"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
  99         {"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
 100         {"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
 101         {"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
 102         {"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
 103         {"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
 104         {"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
 105         {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
 106         {"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
 107 };
 108
 109 /* [rt]x_qX_ is prepended to the name string here */
 110 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
 111         {"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
 112         {"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
 113         {"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
 114         {"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
 115         {"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
 116         {"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
 117         {"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
 118         {"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
 119         {"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
 120         {"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
 121         {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
 122         {"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
 123 };
 124
 125 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
 126                             sizeof(rte_virtio_rxq_stat_strings[0]))
 127 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
 128                             sizeof(rte_virtio_txq_stat_strings[0]))
 129
 130 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS];
 131
 132 static struct virtio_pmd_ctrl *
 133 virtio_send_command_packed(struct virtnet_ctl *cvq,
 134                            struct virtio_pmd_ctrl *ctrl,
 135                            int *dlen, int pkt_num)
 136 {
 137         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 138         int head;
 139         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 140         struct virtio_pmd_ctrl *result;
 141         uint16_t flags;
 142         int sum = 0;
 143         int nb_descs = 0;
 144         int k;
 145
 146         /*
 147          * Format is enforced in qemu code:
 148          * One TX packet for header;
 149          * At least one TX packet per argument;
 150          * One RX packet for ACK.
 151          */
 152         head = vq->vq_avail_idx;
 153         flags = vq->vq_packed.cached_flags;
 154         desc[head].addr = cvq->virtio_net_hdr_mem;
 155         desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 156         vq->vq_free_cnt--;
 157         nb_descs++;
 158         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 159                 vq->vq_avail_idx -= vq->vq_nentries;
 160                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 161         }
 162
 163         for (k = 0; k < pkt_num; k++) {
 164                 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 165                         + sizeof(struct virtio_net_ctrl_hdr)
 166                         + sizeof(ctrl->status) + sizeof(uint8_t) * sum;
 167                 desc[vq->vq_avail_idx].len = dlen[k];
 168                 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT |
 169                         vq->vq_packed.cached_flags;
 170                 sum += dlen[k];
 171                 vq->vq_free_cnt--;
 172                 nb_descs++;
 173                 if (++vq->vq_avail_idx >= vq->vq_nentries) {
 174                         vq->vq_avail_idx -= vq->vq_nentries;
 175                         vq->vq_packed.cached_flags ^=
 176                                 VRING_PACKED_DESC_F_AVAIL_USED;
 177                 }
 178         }
 179
 180         desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 181                 + sizeof(struct virtio_net_ctrl_hdr);
 182         desc[vq->vq_avail_idx].len = sizeof(ctrl->status);
 183         desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE |
 184                 vq->vq_packed.cached_flags;
 185         vq->vq_free_cnt--;
 186         nb_descs++;
 187         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 188                 vq->vq_avail_idx -= vq->vq_nentries;
 189                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 190         }
 191
 192         virtqueue_store_flags_packed(&desc[head], VRING_DESC_F_NEXT | flags,
 193                         vq->hw->weak_barriers);
 194
 195         virtio_wmb(vq->hw->weak_barriers);
 196         virtqueue_notify(vq);
 197
 198         /* wait for used desc in virtqueue
 199          * desc_is_used has a load-acquire or rte_io_rmb inside
 200          */
 201         while (!desc_is_used(&desc[head], vq))
 202                 usleep(100);
 203
 204         /* now get used descriptors */
 205         vq->vq_free_cnt += nb_descs;
 206         vq->vq_used_cons_idx += nb_descs;
 207         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 208                 vq->vq_used_cons_idx -= vq->vq_nentries;
 209                 vq->vq_packed.used_wrap_counter ^= 1;
 210         }
 211
 212         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n"
 213                         "vq->vq_avail_idx=%d\n"
 214                         "vq->vq_used_cons_idx=%d\n"
 215                         "vq->vq_packed.cached_flags=0x%x\n"
 216                         "vq->vq_packed.used_wrap_counter=%d",
 217                         vq->vq_free_cnt,
 218                         vq->vq_avail_idx,
 219                         vq->vq_used_cons_idx,
 220                         vq->vq_packed.cached_flags,
 221                         vq->vq_packed.used_wrap_counter);
 222
 223         result = cvq->virtio_net_hdr_mz->addr;
 224         return result;
 225 }
 226
 227 static struct virtio_pmd_ctrl *
 228 virtio_send_command_split(struct virtnet_ctl *cvq,
 229                           struct virtio_pmd_ctrl *ctrl,
 230                           int *dlen, int pkt_num)
 231 {
 232         struct virtio_pmd_ctrl *result;
 233         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 234         uint32_t head, i;
 235         int k, sum = 0;
 236
 237         head = vq->vq_desc_head_idx;
 238
 239         /*
 240          * Format is enforced in qemu code:
 241          * One TX packet for header;
 242          * At least one TX packet per argument;
 243          * One RX packet for ACK.
 244          */
 245         vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT;
 246         vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem;
 247         vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 248         vq->vq_free_cnt--;
 249         i = vq->vq_split.ring.desc[head].next;
 250
 251         for (k = 0; k < pkt_num; k++) {
 252                 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT;
 253                 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 254                         + sizeof(struct virtio_net_ctrl_hdr)
 255                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 256                 vq->vq_split.ring.desc[i].len = dlen[k];
 257                 sum += dlen[k];
 258                 vq->vq_free_cnt--;
 259                 i = vq->vq_split.ring.desc[i].next;
 260         }
 261
 262         vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE;
 263         vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 264                         + sizeof(struct virtio_net_ctrl_hdr);
 265         vq->vq_split.ring.desc[i].len = sizeof(ctrl->status);
 266         vq->vq_free_cnt--;
 267
 268         vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next;
 269
 270         vq_update_avail_ring(vq, head);
 271         vq_update_avail_idx(vq);
 272
 273         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
 274
 275         virtqueue_notify(vq);
 276
 277         while (virtqueue_nused(vq) == 0)
 278                 usleep(100);
 279
 280         while (virtqueue_nused(vq)) {
 281                 uint32_t idx, desc_idx, used_idx;
 282                 struct vring_used_elem *uep;
 283
 284                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 285                                 & (vq->vq_nentries - 1));
 286                 uep = &vq->vq_split.ring.used->ring[used_idx];
 287                 idx = (uint32_t) uep->id;
 288                 desc_idx = idx;
 289
 290                 while (vq->vq_split.ring.desc[desc_idx].flags &
 291                                 VRING_DESC_F_NEXT) {
 292                         desc_idx = vq->vq_split.ring.desc[desc_idx].next;
 293                         vq->vq_free_cnt++;
 294                 }
 295
 296                 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 297                 vq->vq_desc_head_idx = idx;
 298
 299                 vq->vq_used_cons_idx++;
 300                 vq->vq_free_cnt++;
 301         }
 302
 303         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
 304                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 305
 306         result = cvq->virtio_net_hdr_mz->addr;
 307         return result;
 308 }
 309
 310 static int
 311 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
 312                     int *dlen, int pkt_num)
 313 {
 314         virtio_net_ctrl_ack status = ~0;
 315         struct virtio_pmd_ctrl *result;
 316         struct virtqueue *vq;
 317
 318         ctrl->status = status;
 319
 320         if (!cvq) {
 321                 PMD_INIT_LOG(ERR, "Control queue is not supported.");
 322                 return -1;
 323         }
 324
 325         rte_spinlock_lock(&cvq->lock);
 326         vq = virtnet_cq_to_vq(cvq);
 327
 328         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 329                 "vq->hw->cvq = %p vq = %p",
 330                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 331
 332         if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) {
 333                 rte_spinlock_unlock(&cvq->lock);
 334                 return -1;
 335         }
 336
 337         memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
 338                 sizeof(struct virtio_pmd_ctrl));
 339
 340         if (virtio_with_packed_queue(vq->hw))
 341                 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num);
 342         else
 343                 result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num);
 344
 345         rte_spinlock_unlock(&cvq->lock);
 346         return result->status;
 347 }
 348
 349 static int
 350 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 351 {
 352         struct virtio_hw *hw = dev->data->dev_private;
 353         struct virtio_pmd_ctrl ctrl;
 354         int dlen[1];
 355         int ret;
 356
 357         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 358         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 359         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 360
 361         dlen[0] = sizeof(uint16_t);
 362
 363         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 364         if (ret) {
 365                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 366                           "failed, this is too late now...");
 367                 return -EINVAL;
 368         }
 369
 370         return 0;
 371 }
 372
 373 static uint16_t
 374 virtio_get_nr_vq(struct virtio_hw *hw)
 375 {
 376         uint16_t nr_vq = hw->max_queue_pairs * 2;
 377
 378         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
 379                 nr_vq += 1;
 380
 381         return nr_vq;
 382 }
 383
 384 static void
 385 virtio_init_vring(struct virtqueue *vq)
 386 {
 387         int size = vq->vq_nentries;
 388         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 389
 390         PMD_INIT_FUNC_TRACE();
 391
 392         memset(ring_mem, 0, vq->vq_ring_size);
 393
 394         vq->vq_used_cons_idx = 0;
 395         vq->vq_desc_head_idx = 0;
 396         vq->vq_avail_idx = 0;
 397         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 398         vq->vq_free_cnt = vq->vq_nentries;
 399         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 400         if (virtio_with_packed_queue(vq->hw)) {
 401                 vring_init_packed(&vq->vq_packed.ring, ring_mem,
 402                                   VIRTIO_VRING_ALIGN, size);
 403                 vring_desc_init_packed(vq, size);
 404         } else {
 405                 struct vring *vr = &vq->vq_split.ring;
 406
 407                 vring_init_split(vr, ring_mem, VIRTIO_VRING_ALIGN, size);
 408                 vring_desc_init_split(vr->desc, size);
 409         }
 410         /*
 411          * Disable device(host) interrupting guest
 412          */
 413         virtqueue_disable_intr(vq);
 414 }
 415
 416 static int
 417 virtio_init_queue(struct rte_eth_dev *dev, uint16_t queue_idx)
 418 {
 419         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 420         char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
 421         const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
 422         unsigned int vq_size, size;
 423         struct virtio_hw *hw = dev->data->dev_private;
 424         struct virtnet_rx *rxvq = NULL;
 425         struct virtnet_tx *txvq = NULL;
 426         struct virtnet_ctl *cvq = NULL;
 427         struct virtqueue *vq;
 428         size_t sz_hdr_mz = 0;
 429         void *sw_ring = NULL;
 430         int queue_type = virtio_get_queue_type(hw, queue_idx);
 431         int ret;
 432         int numa_node = dev->device->numa_node;
 433         struct rte_mbuf *fake_mbuf = NULL;
 434
 435         PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d",
 436                         queue_idx, numa_node);
 437
 438         /*
 439          * Read the virtqueue size from the Queue Size field
 440          * Always power of 2 and if 0 virtqueue does not exist
 441          */
 442         vq_size = VIRTIO_OPS(hw)->get_queue_num(hw, queue_idx);
 443         PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
 444         if (vq_size == 0) {
 445                 PMD_INIT_LOG(ERR, "virtqueue does not exist");
 446                 return -EINVAL;
 447         }
 448
 449         if (!virtio_with_packed_queue(hw) && !rte_is_power_of_2(vq_size)) {
 450                 PMD_INIT_LOG(ERR, "split virtqueue size is not power of 2");
 451                 return -EINVAL;
 452         }
 453
 454         snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
 455                  dev->data->port_id, queue_idx);
 456
 457         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 458                                 vq_size * sizeof(struct vq_desc_extra),
 459                                 RTE_CACHE_LINE_SIZE);
 460         if (queue_type == VTNET_TQ) {
 461                 /*
 462                  * For each xmit packet, allocate a virtio_net_hdr
 463                  * and indirect ring elements
 464                  */
 465                 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
 466         } else if (queue_type == VTNET_CQ) {
 467                 /* Allocate a page for control vq command, data and status */
 468                 sz_hdr_mz = rte_mem_page_size();
 469         }
 470
 471         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
 472                                 numa_node);
 473         if (vq == NULL) {
 474                 PMD_INIT_LOG(ERR, "can not allocate vq");
 475                 return -ENOMEM;
 476         }
 477         hw->vqs[queue_idx] = vq;
 478
 479         vq->hw = hw;
 480         vq->vq_queue_index = queue_idx;
 481         vq->vq_nentries = vq_size;
 482         if (virtio_with_packed_queue(hw)) {
 483                 vq->vq_packed.used_wrap_counter = 1;
 484                 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
 485                 vq->vq_packed.event_flags_shadow = 0;
 486                 if (queue_type == VTNET_RQ)
 487                         vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
 488         }
 489
 490         /*
 491          * Reserve a memzone for vring elements
 492          */
 493         size = vring_size(hw, vq_size, VIRTIO_VRING_ALIGN);
 494         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN);
 495         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
 496                      size, vq->vq_ring_size);
 497
 498         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 499                         numa_node, RTE_MEMZONE_IOVA_CONTIG,
 500                         VIRTIO_VRING_ALIGN);
 501         if (mz == NULL) {
 502                 if (rte_errno == EEXIST)
 503                         mz = rte_memzone_lookup(vq_name);
 504                 if (mz == NULL) {
 505                         ret = -ENOMEM;
 506                         goto free_vq;
 507                 }
 508         }
 509
 510         memset(mz->addr, 0, mz->len);
 511
 512         if (hw->use_va)
 513                 vq->vq_ring_mem = (uintptr_t)mz->addr;
 514         else
 515                 vq->vq_ring_mem = mz->iova;
 516
 517         vq->vq_ring_virt_mem = mz->addr;
 518         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem);
 519         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem);
 520
 521         virtio_init_vring(vq);
 522
 523         if (sz_hdr_mz) {
 524                 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 525                          dev->data->port_id, queue_idx);
 526                 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
 527                                 numa_node, RTE_MEMZONE_IOVA_CONTIG,
 528                                 RTE_CACHE_LINE_SIZE);
 529                 if (hdr_mz == NULL) {
 530                         if (rte_errno == EEXIST)
 531                                 hdr_mz = rte_memzone_lookup(vq_hdr_name);
 532                         if (hdr_mz == NULL) {
 533                                 ret = -ENOMEM;
 534                                 goto free_mz;
 535                         }
 536                 }
 537         }
 538
 539         if (queue_type == VTNET_RQ) {
 540                 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
 541                                sizeof(vq->sw_ring[0]);
 542
 543                 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
 544                                 RTE_CACHE_LINE_SIZE, numa_node);
 545                 if (!sw_ring) {
 546                         PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
 547                         ret = -ENOMEM;
 548                         goto free_hdr_mz;
 549                 }
 550
 551                 fake_mbuf = rte_zmalloc_socket("sw_ring", sizeof(*fake_mbuf),
 552                                 RTE_CACHE_LINE_SIZE, numa_node);
 553                 if (!fake_mbuf) {
 554                         PMD_INIT_LOG(ERR, "can not allocate fake mbuf");
 555                         ret = -ENOMEM;
 556                         goto free_sw_ring;
 557                 }
 558
 559                 vq->sw_ring = sw_ring;
 560                 rxvq = &vq->rxq;
 561                 rxvq->port_id = dev->data->port_id;
 562                 rxvq->mz = mz;
 563                 rxvq->fake_mbuf = fake_mbuf;
 564         } else if (queue_type == VTNET_TQ) {
 565                 txvq = &vq->txq;
 566                 txvq->port_id = dev->data->port_id;
 567                 txvq->mz = mz;
 568                 txvq->virtio_net_hdr_mz = hdr_mz;
 569                 if (hw->use_va)
 570                         txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 571                 else
 572                         txvq->virtio_net_hdr_mem = hdr_mz->iova;
 573         } else if (queue_type == VTNET_CQ) {
 574                 cvq = &vq->cq;
 575                 cvq->mz = mz;
 576                 cvq->virtio_net_hdr_mz = hdr_mz;
 577                 if (hw->use_va)
 578                         cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 579                 else
 580                         cvq->virtio_net_hdr_mem = hdr_mz->iova;
 581                 memset(cvq->virtio_net_hdr_mz->addr, 0, rte_mem_page_size());
 582
 583                 hw->cvq = cvq;
 584         }
 585
 586         if (hw->use_va)
 587                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr);
 588         else
 589                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova);
 590
 591         if (queue_type == VTNET_TQ) {
 592                 struct virtio_tx_region *txr;
 593                 unsigned int i;
 594
 595                 txr = hdr_mz->addr;
 596                 memset(txr, 0, vq_size * sizeof(*txr));
 597                 for (i = 0; i < vq_size; i++) {
 598                         /* first indirect descriptor is always the tx header */
 599                         if (!virtio_with_packed_queue(hw)) {
 600                                 struct vring_desc *start_dp = txr[i].tx_indir;
 601                                 vring_desc_init_split(start_dp,
 602                                                       RTE_DIM(txr[i].tx_indir));
 603                                 start_dp->addr = txvq->virtio_net_hdr_mem
 604                                         + i * sizeof(*txr)
 605                                         + offsetof(struct virtio_tx_region,
 606                                                    tx_hdr);
 607                                 start_dp->len = hw->vtnet_hdr_size;
 608                                 start_dp->flags = VRING_DESC_F_NEXT;
 609                         } else {
 610                                 struct vring_packed_desc *start_dp =
 611                                         txr[i].tx_packed_indir;
 612                                 vring_desc_init_indirect_packed(start_dp,
 613                                       RTE_DIM(txr[i].tx_packed_indir));
 614                                 start_dp->addr = txvq->virtio_net_hdr_mem
 615                                         + i * sizeof(*txr)
 616                                         + offsetof(struct virtio_tx_region,
 617                                                    tx_hdr);
 618                                 start_dp->len = hw->vtnet_hdr_size;
 619                         }
 620                 }
 621         }
 622
 623         if (VIRTIO_OPS(hw)->setup_queue(hw, vq) < 0) {
 624                 PMD_INIT_LOG(ERR, "setup_queue failed");
 625                 ret = -EINVAL;
 626                 goto clean_vq;
 627         }
 628
 629         return 0;
 630
 631 clean_vq:
 632         hw->cvq = NULL;
 633         rte_free(fake_mbuf);
 634 free_sw_ring:
 635         rte_free(sw_ring);
 636 free_hdr_mz:
 637         rte_memzone_free(hdr_mz);
 638 free_mz:
 639         rte_memzone_free(mz);
 640 free_vq:
 641         rte_free(vq);
 642         hw->vqs[queue_idx] = NULL;
 643
 644         return ret;
 645 }
 646
 647 static void
 648 virtio_free_queues(struct virtio_hw *hw)
 649 {
 650         uint16_t nr_vq = virtio_get_nr_vq(hw);
 651         struct virtqueue *vq;
 652         int queue_type;
 653         uint16_t i;
 654
 655         if (hw->vqs == NULL)
 656                 return;
 657
 658         for (i = 0; i < nr_vq; i++) {
 659                 vq = hw->vqs[i];
 660                 if (!vq)
 661                         continue;
 662
 663                 queue_type = virtio_get_queue_type(hw, i);
 664                 if (queue_type == VTNET_RQ) {
 665                         rte_free(vq->rxq.fake_mbuf);
 666                         rte_free(vq->sw_ring);
 667                         rte_memzone_free(vq->rxq.mz);
 668                 } else if (queue_type == VTNET_TQ) {
 669                         rte_memzone_free(vq->txq.mz);
 670                         rte_memzone_free(vq->txq.virtio_net_hdr_mz);
 671                 } else {
 672                         rte_memzone_free(vq->cq.mz);
 673                         rte_memzone_free(vq->cq.virtio_net_hdr_mz);
 674                 }
 675
 676                 rte_free(vq);
 677                 hw->vqs[i] = NULL;
 678         }
 679
 680         rte_free(hw->vqs);
 681         hw->vqs = NULL;
 682 }
 683
 684 static int
 685 virtio_alloc_queues(struct rte_eth_dev *dev)
 686 {
 687         struct virtio_hw *hw = dev->data->dev_private;
 688         uint16_t nr_vq = virtio_get_nr_vq(hw);
 689         uint16_t i;
 690         int ret;
 691
 692         hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0);
 693         if (!hw->vqs) {
 694                 PMD_INIT_LOG(ERR, "failed to allocate vqs");
 695                 return -ENOMEM;
 696         }
 697
 698         for (i = 0; i < nr_vq; i++) {
 699                 ret = virtio_init_queue(dev, i);
 700                 if (ret < 0) {
 701                         virtio_free_queues(hw);
 702                         return ret;
 703                 }
 704         }
 705
 706         return 0;
 707 }
 708
 709 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev);
 710
 711 int
 712 virtio_dev_close(struct rte_eth_dev *dev)
 713 {
 714         struct virtio_hw *hw = dev->data->dev_private;
 715         struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
 716
 717         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 718         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 719                 return 0;
 720
 721         if (!hw->opened)
 722                 return 0;
 723         hw->opened = 0;
 724
 725         /* reset the NIC */
 726         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 727                 VIRTIO_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR);
 728         if (intr_conf->rxq)
 729                 virtio_queues_unbind_intr(dev);
 730
 731         if (intr_conf->lsc || intr_conf->rxq) {
 732                 virtio_intr_disable(dev);
 733                 rte_intr_efd_disable(dev->intr_handle);
 734                 rte_free(dev->intr_handle->intr_vec);
 735                 dev->intr_handle->intr_vec = NULL;
 736         }
 737
 738         virtio_reset(hw);
 739         virtio_dev_free_mbufs(dev);
 740         virtio_free_queues(hw);
 741
 742         return VIRTIO_OPS(hw)->dev_close(hw);
 743 }
 744
 745 static int
 746 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
 747 {
 748         struct virtio_hw *hw = dev->data->dev_private;
 749         struct virtio_pmd_ctrl ctrl;
 750         int dlen[1];
 751         int ret;
 752
 753         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 754                 PMD_INIT_LOG(INFO, "host does not support rx control");
 755                 return -ENOTSUP;
 756         }
 757
 758         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 759         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 760         ctrl.data[0] = 1;
 761         dlen[0] = 1;
 762
 763         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 764         if (ret) {
 765                 PMD_INIT_LOG(ERR, "Failed to enable promisc");
 766                 return -EAGAIN;
 767         }
 768
 769         return 0;
 770 }
 771
 772 static int
 773 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
 774 {
 775         struct virtio_hw *hw = dev->data->dev_private;
 776         struct virtio_pmd_ctrl ctrl;
 777         int dlen[1];
 778         int ret;
 779
 780         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 781                 PMD_INIT_LOG(INFO, "host does not support rx control");
 782                 return -ENOTSUP;
 783         }
 784
 785         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 786         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 787         ctrl.data[0] = 0;
 788         dlen[0] = 1;
 789
 790         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 791         if (ret) {
 792                 PMD_INIT_LOG(ERR, "Failed to disable promisc");
 793                 return -EAGAIN;
 794         }
 795
 796         return 0;
 797 }
 798
 799 static int
 800 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
 801 {
 802         struct virtio_hw *hw = dev->data->dev_private;
 803         struct virtio_pmd_ctrl ctrl;
 804         int dlen[1];
 805         int ret;
 806
 807         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 808                 PMD_INIT_LOG(INFO, "host does not support rx control");
 809                 return -ENOTSUP;
 810         }
 811
 812         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 813         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 814         ctrl.data[0] = 1;
 815         dlen[0] = 1;
 816
 817         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 818         if (ret) {
 819                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
 820                 return -EAGAIN;
 821         }
 822
 823         return 0;
 824 }
 825
 826 static int
 827 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
 828 {
 829         struct virtio_hw *hw = dev->data->dev_private;
 830         struct virtio_pmd_ctrl ctrl;
 831         int dlen[1];
 832         int ret;
 833
 834         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 835                 PMD_INIT_LOG(INFO, "host does not support rx control");
 836                 return -ENOTSUP;
 837         }
 838
 839         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 840         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 841         ctrl.data[0] = 0;
 842         dlen[0] = 1;
 843
 844         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 845         if (ret) {
 846                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
 847                 return -EAGAIN;
 848         }
 849
 850         return 0;
 851 }
 852
 853 uint16_t
 854 virtio_rx_mem_pool_buf_size(struct rte_mempool *mp)
 855 {
 856         return rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
 857 }
 858
 859 bool
 860 virtio_rx_check_scatter(uint16_t max_rx_pkt_len, uint16_t rx_buf_size,
 861                         bool rx_scatter_enabled, const char **error)
 862 {
 863         if (!rx_scatter_enabled && max_rx_pkt_len > rx_buf_size) {
 864                 *error = "Rx scatter is disabled and RxQ mbuf pool object size is too small";
 865                 return false;
 866         }
 867
 868         return true;
 869 }
 870
 871 static bool
 872 virtio_check_scatter_on_all_rx_queues(struct rte_eth_dev *dev,
 873                                       uint16_t frame_size)
 874 {
 875         struct virtio_hw *hw = dev->data->dev_private;
 876         struct virtnet_rx *rxvq;
 877         struct virtqueue *vq;
 878         unsigned int qidx;
 879         uint16_t buf_size;
 880         const char *error;
 881
 882         if (hw->vqs == NULL)
 883                 return true;
 884
 885         for (qidx = 0; (vq = hw->vqs[2 * qidx + VTNET_SQ_RQ_QUEUE_IDX]) != NULL;
 886              qidx++) {
 887                 rxvq = &vq->rxq;
 888                 if (rxvq->mpool == NULL)
 889                         continue;
 890                 buf_size = virtio_rx_mem_pool_buf_size(rxvq->mpool);
 891
 892                 if (!virtio_rx_check_scatter(frame_size, buf_size,
 893                                              hw->rx_ol_scatter, &error)) {
 894                         PMD_INIT_LOG(ERR, "MTU check for RxQ %u failed: %s",
 895                                      qidx, error);
 896                         return false;
 897                 }
 898         }
 899
 900         return true;
 901 }
 902
 903 #define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
 904 static int
 905 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 906 {
 907         struct virtio_hw *hw = dev->data->dev_private;
 908         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
 909                                  hw->vtnet_hdr_size;
 910         uint32_t frame_size = mtu + ether_hdr_len;
 911         uint32_t max_frame_size = hw->max_mtu + ether_hdr_len;
 912
 913         max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN);
 914
 915         if (mtu < RTE_ETHER_MIN_MTU || frame_size > max_frame_size) {
 916                 PMD_INIT_LOG(ERR, "MTU should be between %d and %d",
 917                         RTE_ETHER_MIN_MTU, max_frame_size - ether_hdr_len);
 918                 return -EINVAL;
 919         }
 920
 921         if (!virtio_check_scatter_on_all_rx_queues(dev, frame_size)) {
 922                 PMD_INIT_LOG(ERR, "MTU vs Rx scatter and Rx buffers check failed");
 923                 return -EINVAL;
 924         }
 925
 926         hw->max_rx_pkt_len = frame_size;
 927
 928         return 0;
 929 }
 930
 931 static int
 932 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 933 {
 934         struct virtio_hw *hw = dev->data->dev_private;
 935         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 936         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
 937
 938         virtqueue_enable_intr(vq);
 939         virtio_mb(hw->weak_barriers);
 940         return 0;
 941 }
 942
 943 static int
 944 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 945 {
 946         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 947         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
 948
 949         virtqueue_disable_intr(vq);
 950         return 0;
 951 }
 952
 953 /*
 954  * dev_ops for virtio, bare necessities for basic operation
 955  */
 956 static const struct eth_dev_ops virtio_eth_dev_ops = {
 957         .dev_configure           = virtio_dev_configure,
 958         .dev_start               = virtio_dev_start,
 959         .dev_stop                = virtio_dev_stop,
 960         .dev_close               = virtio_dev_close,
 961         .promiscuous_enable      = virtio_dev_promiscuous_enable,
 962         .promiscuous_disable     = virtio_dev_promiscuous_disable,
 963         .allmulticast_enable     = virtio_dev_allmulticast_enable,
 964         .allmulticast_disable    = virtio_dev_allmulticast_disable,
 965         .mtu_set                 = virtio_mtu_set,
 966         .dev_infos_get           = virtio_dev_info_get,
 967         .stats_get               = virtio_dev_stats_get,
 968         .xstats_get              = virtio_dev_xstats_get,
 969         .xstats_get_names        = virtio_dev_xstats_get_names,
 970         .stats_reset             = virtio_dev_stats_reset,
 971         .xstats_reset            = virtio_dev_stats_reset,
 972         .link_update             = virtio_dev_link_update,
 973         .vlan_offload_set        = virtio_dev_vlan_offload_set,
 974         .rx_queue_setup          = virtio_dev_rx_queue_setup,
 975         .rx_queue_intr_enable    = virtio_dev_rx_queue_intr_enable,
 976         .rx_queue_intr_disable   = virtio_dev_rx_queue_intr_disable,
 977         .tx_queue_setup          = virtio_dev_tx_queue_setup,
 978         /* collect stats per queue */
 979         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 980         .vlan_filter_set         = virtio_vlan_filter_set,
 981         .mac_addr_add            = virtio_mac_addr_add,
 982         .mac_addr_remove         = virtio_mac_addr_remove,
 983         .mac_addr_set            = virtio_mac_addr_set,
 984 };
 985
 986 /*
 987  * dev_ops for virtio-user in secondary processes, as we just have
 988  * some limited supports currently.
 989  */
 990 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops = {
 991         .dev_infos_get           = virtio_dev_info_get,
 992         .stats_get               = virtio_dev_stats_get,
 993         .xstats_get              = virtio_dev_xstats_get,
 994         .xstats_get_names        = virtio_dev_xstats_get_names,
 995         .stats_reset             = virtio_dev_stats_reset,
 996         .xstats_reset            = virtio_dev_stats_reset,
 997         /* collect stats per queue */
 998         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 999 };
1000
1001 static void
1002 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1003 {
1004         unsigned i;
1005
1006         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1007                 const struct virtnet_tx *txvq = dev->data->tx_queues[i];
1008                 if (txvq == NULL)
1009                         continue;
1010
1011                 stats->opackets += txvq->stats.packets;
1012                 stats->obytes += txvq->stats.bytes;
1013
1014                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1015                         stats->q_opackets[i] = txvq->stats.packets;
1016                         stats->q_obytes[i] = txvq->stats.bytes;
1017                 }
1018         }
1019
1020         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1021                 const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1022                 if (rxvq == NULL)
1023                         continue;
1024
1025                 stats->ipackets += rxvq->stats.packets;
1026                 stats->ibytes += rxvq->stats.bytes;
1027                 stats->ierrors += rxvq->stats.errors;
1028
1029                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1030                         stats->q_ipackets[i] = rxvq->stats.packets;
1031                         stats->q_ibytes[i] = rxvq->stats.bytes;
1032                 }
1033         }
1034
1035         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1036 }
1037
1038 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
1039                                        struct rte_eth_xstat_name *xstats_names,
1040                                        __rte_unused unsigned limit)
1041 {
1042         unsigned i;
1043         unsigned count = 0;
1044         unsigned t;
1045
1046         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1047                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1048
1049         if (xstats_names != NULL) {
1050                 /* Note: limit checked in rte_eth_xstats_names() */
1051
1052                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1053                         struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1054                         if (rxvq == NULL)
1055                                 continue;
1056                         for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1057                                 snprintf(xstats_names[count].name,
1058                                         sizeof(xstats_names[count].name),
1059                                         "rx_q%u_%s", i,
1060                                         rte_virtio_rxq_stat_strings[t].name);
1061                                 count++;
1062                         }
1063                 }
1064
1065                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1066                         struct virtnet_tx *txvq = dev->data->tx_queues[i];
1067                         if (txvq == NULL)
1068                                 continue;
1069                         for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1070                                 snprintf(xstats_names[count].name,
1071                                         sizeof(xstats_names[count].name),
1072                                         "tx_q%u_%s", i,
1073                                         rte_virtio_txq_stat_strings[t].name);
1074                                 count++;
1075                         }
1076                 }
1077                 return count;
1078         }
1079         return nstats;
1080 }
1081
1082 static int
1083 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1084                       unsigned n)
1085 {
1086         unsigned i;
1087         unsigned count = 0;
1088
1089         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1090                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1091
1092         if (n < nstats)
1093                 return nstats;
1094
1095         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1096                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1097
1098                 if (rxvq == NULL)
1099                         continue;
1100
1101                 unsigned t;
1102
1103                 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1104                         xstats[count].value = *(uint64_t *)(((char *)rxvq) +
1105                                 rte_virtio_rxq_stat_strings[t].offset);
1106                         xstats[count].id = count;
1107                         count++;
1108                 }
1109         }
1110
1111         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1112                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1113
1114                 if (txvq == NULL)
1115                         continue;
1116
1117                 unsigned t;
1118
1119                 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1120                         xstats[count].value = *(uint64_t *)(((char *)txvq) +
1121                                 rte_virtio_txq_stat_strings[t].offset);
1122                         xstats[count].id = count;
1123                         count++;
1124                 }
1125         }
1126
1127         return count;
1128 }
1129
1130 static int
1131 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1132 {
1133         virtio_update_stats(dev, stats);
1134
1135         return 0;
1136 }
1137
1138 static int
1139 virtio_dev_stats_reset(struct rte_eth_dev *dev)
1140 {
1141         unsigned int i;
1142
1143         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1144                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1145                 if (txvq == NULL)
1146                         continue;
1147
1148                 txvq->stats.packets = 0;
1149                 txvq->stats.bytes = 0;
1150                 txvq->stats.multicast = 0;
1151                 txvq->stats.broadcast = 0;
1152                 memset(txvq->stats.size_bins, 0,
1153                        sizeof(txvq->stats.size_bins[0]) * 8);
1154         }
1155
1156         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1157                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1158                 if (rxvq == NULL)
1159                         continue;
1160
1161                 rxvq->stats.packets = 0;
1162                 rxvq->stats.bytes = 0;
1163                 rxvq->stats.errors = 0;
1164                 rxvq->stats.multicast = 0;
1165                 rxvq->stats.broadcast = 0;
1166                 memset(rxvq->stats.size_bins, 0,
1167                        sizeof(rxvq->stats.size_bins[0]) * 8);
1168         }
1169
1170         return 0;
1171 }
1172
1173 static void
1174 virtio_set_hwaddr(struct virtio_hw *hw)
1175 {
1176         virtio_write_dev_config(hw,
1177                         offsetof(struct virtio_net_config, mac),
1178                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1179 }
1180
1181 static void
1182 virtio_get_hwaddr(struct virtio_hw *hw)
1183 {
1184         if (virtio_with_feature(hw, VIRTIO_NET_F_MAC)) {
1185                 virtio_read_dev_config(hw,
1186                         offsetof(struct virtio_net_config, mac),
1187                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1188         } else {
1189                 rte_eth_random_addr(&hw->mac_addr[0]);
1190                 virtio_set_hwaddr(hw);
1191         }
1192 }
1193
1194 static int
1195 virtio_mac_table_set(struct virtio_hw *hw,
1196                      const struct virtio_net_ctrl_mac *uc,
1197                      const struct virtio_net_ctrl_mac *mc)
1198 {
1199         struct virtio_pmd_ctrl ctrl;
1200         int err, len[2];
1201
1202         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1203                 PMD_DRV_LOG(INFO, "host does not support mac table");
1204                 return -1;
1205         }
1206
1207         ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1208         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1209
1210         len[0] = uc->entries * RTE_ETHER_ADDR_LEN + sizeof(uc->entries);
1211         memcpy(ctrl.data, uc, len[0]);
1212
1213         len[1] = mc->entries * RTE_ETHER_ADDR_LEN + sizeof(mc->entries);
1214         memcpy(ctrl.data + len[0], mc, len[1]);
1215
1216         err = virtio_send_command(hw->cvq, &ctrl, len, 2);
1217         if (err != 0)
1218                 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
1219         return err;
1220 }
1221
1222 static int
1223 virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1224                     uint32_t index, uint32_t vmdq __rte_unused)
1225 {
1226         struct virtio_hw *hw = dev->data->dev_private;
1227         const struct rte_ether_addr *addrs = dev->data->mac_addrs;
1228         unsigned int i;
1229         struct virtio_net_ctrl_mac *uc, *mc;
1230
1231         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1232                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1233                 return -EINVAL;
1234         }
1235
1236         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1237                 sizeof(uc->entries));
1238         uc->entries = 0;
1239         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1240                 sizeof(mc->entries));
1241         mc->entries = 0;
1242
1243         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1244                 const struct rte_ether_addr *addr
1245                         = (i == index) ? mac_addr : addrs + i;
1246                 struct virtio_net_ctrl_mac *tbl
1247                         = rte_is_multicast_ether_addr(addr) ? mc : uc;
1248
1249                 memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN);
1250         }
1251
1252         return virtio_mac_table_set(hw, uc, mc);
1253 }
1254
1255 static void
1256 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1257 {
1258         struct virtio_hw *hw = dev->data->dev_private;
1259         struct rte_ether_addr *addrs = dev->data->mac_addrs;
1260         struct virtio_net_ctrl_mac *uc, *mc;
1261         unsigned int i;
1262
1263         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1264                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1265                 return;
1266         }
1267
1268         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1269                 sizeof(uc->entries));
1270         uc->entries = 0;
1271         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1272                 sizeof(mc->entries));
1273         mc->entries = 0;
1274
1275         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1276                 struct virtio_net_ctrl_mac *tbl;
1277
1278                 if (i == index || rte_is_zero_ether_addr(addrs + i))
1279                         continue;
1280
1281                 tbl = rte_is_multicast_ether_addr(addrs + i) ? mc : uc;
1282                 memcpy(&tbl->macs[tbl->entries++], addrs + i,
1283                         RTE_ETHER_ADDR_LEN);
1284         }
1285
1286         virtio_mac_table_set(hw, uc, mc);
1287 }
1288
1289 static int
1290 virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1291 {
1292         struct virtio_hw *hw = dev->data->dev_private;
1293
1294         memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1295
1296         /* Use atomic update if available */
1297         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1298                 struct virtio_pmd_ctrl ctrl;
1299                 int len = RTE_ETHER_ADDR_LEN;
1300
1301                 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1302                 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1303
1304                 memcpy(ctrl.data, mac_addr, RTE_ETHER_ADDR_LEN);
1305                 return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1306         }
1307
1308         if (!virtio_with_feature(hw, VIRTIO_NET_F_MAC))
1309                 return -ENOTSUP;
1310
1311         virtio_set_hwaddr(hw);
1312         return 0;
1313 }
1314
1315 static int
1316 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1317 {
1318         struct virtio_hw *hw = dev->data->dev_private;
1319         struct virtio_pmd_ctrl ctrl;
1320         int len;
1321
1322         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1323                 return -ENOTSUP;
1324
1325         ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1326         ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1327         memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1328         len = sizeof(vlan_id);
1329
1330         return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1331 }
1332
1333 static int
1334 virtio_intr_unmask(struct rte_eth_dev *dev)
1335 {
1336         struct virtio_hw *hw = dev->data->dev_private;
1337
1338         if (rte_intr_ack(dev->intr_handle) < 0)
1339                 return -1;
1340
1341         if (VIRTIO_OPS(hw)->intr_detect)
1342                 VIRTIO_OPS(hw)->intr_detect(hw);
1343
1344         return 0;
1345 }
1346
1347 static int
1348 virtio_intr_enable(struct rte_eth_dev *dev)
1349 {
1350         struct virtio_hw *hw = dev->data->dev_private;
1351
1352         if (rte_intr_enable(dev->intr_handle) < 0)
1353                 return -1;
1354
1355         if (VIRTIO_OPS(hw)->intr_detect)
1356                 VIRTIO_OPS(hw)->intr_detect(hw);
1357
1358         return 0;
1359 }
1360
1361 static int
1362 virtio_intr_disable(struct rte_eth_dev *dev)
1363 {
1364         struct virtio_hw *hw = dev->data->dev_private;
1365
1366         if (rte_intr_disable(dev->intr_handle) < 0)
1367                 return -1;
1368
1369         if (VIRTIO_OPS(hw)->intr_detect)
1370                 VIRTIO_OPS(hw)->intr_detect(hw);
1371
1372         return 0;
1373 }
1374
1375 static int
1376 virtio_ethdev_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1377 {
1378         uint64_t host_features;
1379
1380         /* Prepare guest_features: feature that driver wants to support */
1381         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1382                 req_features);
1383
1384         /* Read device(host) feature bits */
1385         host_features = VIRTIO_OPS(hw)->get_features(hw);
1386         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1387                 host_features);
1388
1389         /* If supported, ensure MTU value is valid before acknowledging it. */
1390         if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) {
1391                 struct virtio_net_config config;
1392
1393                 virtio_read_dev_config(hw,
1394                         offsetof(struct virtio_net_config, mtu),
1395                         &config.mtu, sizeof(config.mtu));
1396
1397                 if (config.mtu < RTE_ETHER_MIN_MTU)
1398                         req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
1399         }
1400
1401         /*
1402          * Negotiate features: Subset of device feature bits are written back
1403          * guest feature bits.
1404          */
1405         hw->guest_features = req_features;
1406         hw->guest_features = virtio_negotiate_features(hw, host_features);
1407         PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1408                 hw->guest_features);
1409
1410         if (VIRTIO_OPS(hw)->features_ok(hw) < 0)
1411                 return -1;
1412
1413         if (virtio_with_feature(hw, VIRTIO_F_VERSION_1)) {
1414                 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1415
1416                 if (!(virtio_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1417                         PMD_INIT_LOG(ERR, "Failed to set FEATURES_OK status!");
1418                         return -1;
1419                 }
1420         }
1421
1422         hw->req_guest_features = req_features;
1423
1424         return 0;
1425 }
1426
1427 int
1428 virtio_dev_pause(struct rte_eth_dev *dev)
1429 {
1430         struct virtio_hw *hw = dev->data->dev_private;
1431
1432         rte_spinlock_lock(&hw->state_lock);
1433
1434         if (hw->started == 0) {
1435                 /* Device is just stopped. */
1436                 rte_spinlock_unlock(&hw->state_lock);
1437                 return -1;
1438         }
1439         hw->started = 0;
1440         /*
1441          * Prevent the worker threads from touching queues to avoid contention,
1442          * 1 ms should be enough for the ongoing Tx function to finish.
1443          */
1444         rte_delay_ms(1);
1445         return 0;
1446 }
1447
1448 /*
1449  * Recover hw state to let the worker threads continue.
1450  */
1451 void
1452 virtio_dev_resume(struct rte_eth_dev *dev)
1453 {
1454         struct virtio_hw *hw = dev->data->dev_private;
1455
1456         hw->started = 1;
1457         rte_spinlock_unlock(&hw->state_lock);
1458 }
1459
1460 /*
1461  * Should be called only after device is paused.
1462  */
1463 int
1464 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts,
1465                 int nb_pkts)
1466 {
1467         struct virtio_hw *hw = dev->data->dev_private;
1468         struct virtnet_tx *txvq = dev->data->tx_queues[0];
1469         int ret;
1470
1471         hw->inject_pkts = tx_pkts;
1472         ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts);
1473         hw->inject_pkts = NULL;
1474
1475         return ret;
1476 }
1477
1478 static void
1479 virtio_notify_peers(struct rte_eth_dev *dev)
1480 {
1481         struct virtio_hw *hw = dev->data->dev_private;
1482         struct virtnet_rx *rxvq;
1483         struct rte_mbuf *rarp_mbuf;
1484
1485         if (!dev->data->rx_queues)
1486                 return;
1487
1488         rxvq = dev->data->rx_queues[0];
1489         if (!rxvq)
1490                 return;
1491
1492         rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool,
1493                         (struct rte_ether_addr *)hw->mac_addr);
1494         if (rarp_mbuf == NULL) {
1495                 PMD_DRV_LOG(ERR, "failed to make RARP packet.");
1496                 return;
1497         }
1498
1499         /* If virtio port just stopped, no need to send RARP */
1500         if (virtio_dev_pause(dev) < 0) {
1501                 rte_pktmbuf_free(rarp_mbuf);
1502                 return;
1503         }
1504
1505         virtio_inject_pkts(dev, &rarp_mbuf, 1);
1506         virtio_dev_resume(dev);
1507 }
1508
1509 static void
1510 virtio_ack_link_announce(struct rte_eth_dev *dev)
1511 {
1512         struct virtio_hw *hw = dev->data->dev_private;
1513         struct virtio_pmd_ctrl ctrl;
1514
1515         ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE;
1516         ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK;
1517
1518         virtio_send_command(hw->cvq, &ctrl, NULL, 0);
1519 }
1520
1521 /*
1522  * Process virtio config changed interrupt. Call the callback
1523  * if link state changed, generate gratuitous RARP packet if
1524  * the status indicates an ANNOUNCE.
1525  */
1526 void
1527 virtio_interrupt_handler(void *param)
1528 {
1529         struct rte_eth_dev *dev = param;
1530         struct virtio_hw *hw = dev->data->dev_private;
1531         uint8_t isr;
1532         uint16_t status;
1533
1534         /* Read interrupt status which clears interrupt */
1535         isr = virtio_get_isr(hw);
1536         PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1537
1538         if (virtio_intr_unmask(dev) < 0)
1539                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1540
1541         if (isr & VIRTIO_ISR_CONFIG) {
1542                 if (virtio_dev_link_update(dev, 0) == 0)
1543                         rte_eth_dev_callback_process(dev,
1544                                                      RTE_ETH_EVENT_INTR_LSC,
1545                                                      NULL);
1546
1547                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1548                         virtio_read_dev_config(hw,
1549                                 offsetof(struct virtio_net_config, status),
1550                                 &status, sizeof(status));
1551                         if (status & VIRTIO_NET_S_ANNOUNCE) {
1552                                 virtio_notify_peers(dev);
1553                                 if (hw->cvq)
1554                                         virtio_ack_link_announce(dev);
1555                         }
1556                 }
1557         }
1558 }
1559
1560 /* set rx and tx handlers according to what is supported */
1561 static void
1562 set_rxtx_funcs(struct rte_eth_dev *eth_dev)
1563 {
1564         struct virtio_hw *hw = eth_dev->data->dev_private;
1565
1566         eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare;
1567         if (virtio_with_packed_queue(hw)) {
1568                 PMD_INIT_LOG(INFO,
1569                         "virtio: using packed ring %s Tx path on port %u",
1570                         hw->use_vec_tx ? "vectorized" : "standard",
1571                         eth_dev->data->port_id);
1572                 if (hw->use_vec_tx)
1573                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
1574                 else
1575                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
1576         } else {
1577                 if (hw->use_inorder_tx) {
1578                         PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u",
1579                                 eth_dev->data->port_id);
1580                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder;
1581                 } else {
1582                         PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u",
1583                                 eth_dev->data->port_id);
1584                         eth_dev->tx_pkt_burst = virtio_xmit_pkts;
1585                 }
1586         }
1587
1588         if (virtio_with_packed_queue(hw)) {
1589                 if (hw->use_vec_rx) {
1590                         PMD_INIT_LOG(INFO,
1591                                 "virtio: using packed ring vectorized Rx path on port %u",
1592                                 eth_dev->data->port_id);
1593                         eth_dev->rx_pkt_burst =
1594                                 &virtio_recv_pkts_packed_vec;
1595                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1596                         PMD_INIT_LOG(INFO,
1597                                 "virtio: using packed ring mergeable buffer Rx path on port %u",
1598                                 eth_dev->data->port_id);
1599                         eth_dev->rx_pkt_burst =
1600                                 &virtio_recv_mergeable_pkts_packed;
1601                 } else {
1602                         PMD_INIT_LOG(INFO,
1603                                 "virtio: using packed ring standard Rx path on port %u",
1604                                 eth_dev->data->port_id);
1605                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
1606                 }
1607         } else {
1608                 if (hw->use_vec_rx) {
1609                         PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on port %u",
1610                                 eth_dev->data->port_id);
1611                         eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
1612                 } else if (hw->use_inorder_rx) {
1613                         PMD_INIT_LOG(INFO,
1614                                 "virtio: using inorder Rx path on port %u",
1615                                 eth_dev->data->port_id);
1616                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder;
1617                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1618                         PMD_INIT_LOG(INFO,
1619                                 "virtio: using mergeable buffer Rx path on port %u",
1620                                 eth_dev->data->port_id);
1621                         eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1622                 } else {
1623                         PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
1624                                 eth_dev->data->port_id);
1625                         eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1626                 }
1627         }
1628
1629 }
1630
1631 /* Only support 1:1 queue/interrupt mapping so far.
1632  * TODO: support n:1 queue/interrupt mapping when there are limited number of
1633  * interrupt vectors (<N+1).
1634  */
1635 static int
1636 virtio_queues_bind_intr(struct rte_eth_dev *dev)
1637 {
1638         uint32_t i;
1639         struct virtio_hw *hw = dev->data->dev_private;
1640
1641         PMD_INIT_LOG(INFO, "queue/interrupt binding");
1642         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
1643                 dev->intr_handle->intr_vec[i] = i + 1;
1644                 if (VIRTIO_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) ==
1645                                                  VIRTIO_MSI_NO_VECTOR) {
1646                         PMD_DRV_LOG(ERR, "failed to set queue vector");
1647                         return -EBUSY;
1648                 }
1649         }
1650
1651         return 0;
1652 }
1653
1654 static void
1655 virtio_queues_unbind_intr(struct rte_eth_dev *dev)
1656 {
1657         uint32_t i;
1658         struct virtio_hw *hw = dev->data->dev_private;
1659
1660         PMD_INIT_LOG(INFO, "queue/interrupt unbinding");
1661         for (i = 0; i < dev->data->nb_rx_queues; ++i)
1662                 VIRTIO_OPS(hw)->set_queue_irq(hw,
1663                                              hw->vqs[i * VTNET_CQ],
1664                                              VIRTIO_MSI_NO_VECTOR);
1665 }
1666
1667 static int
1668 virtio_configure_intr(struct rte_eth_dev *dev)
1669 {
1670         struct virtio_hw *hw = dev->data->dev_private;
1671
1672         if (!rte_intr_cap_multiple(dev->intr_handle)) {
1673                 PMD_INIT_LOG(ERR, "Multiple intr vector not supported");
1674                 return -ENOTSUP;
1675         }
1676
1677         if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) {
1678                 PMD_INIT_LOG(ERR, "Fail to create eventfd");
1679                 return -1;
1680         }
1681
1682         if (!dev->intr_handle->intr_vec) {
1683                 dev->intr_handle->intr_vec =
1684                         rte_zmalloc("intr_vec",
1685                                     hw->max_queue_pairs * sizeof(int), 0);
1686                 if (!dev->intr_handle->intr_vec) {
1687                         PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors",
1688                                      hw->max_queue_pairs);
1689                         return -ENOMEM;
1690                 }
1691         }
1692
1693         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1694                 /* Re-register callback to update max_intr */
1695                 rte_intr_callback_unregister(dev->intr_handle,
1696                                              virtio_interrupt_handler,
1697                                              dev);
1698                 rte_intr_callback_register(dev->intr_handle,
1699                                            virtio_interrupt_handler,
1700                                            dev);
1701         }
1702
1703         /* DO NOT try to remove this! This function will enable msix, or QEMU
1704          * will encounter SIGSEGV when DRIVER_OK is sent.
1705          * And for legacy devices, this should be done before queue/vec binding
1706          * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1707          * (22) will be ignored.
1708          */
1709         if (virtio_intr_enable(dev) < 0) {
1710                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1711                 return -1;
1712         }
1713
1714         if (virtio_queues_bind_intr(dev) < 0) {
1715                 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt");
1716                 return -1;
1717         }
1718
1719         return 0;
1720 }
1721 #define DUPLEX_UNKNOWN   0xff
1722 /* reset device and renegotiate features if needed */
1723 static int
1724 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
1725 {
1726         struct virtio_hw *hw = eth_dev->data->dev_private;
1727         struct virtio_net_config *config;
1728         struct virtio_net_config local_config;
1729         int ret;
1730
1731         /* Reset the device although not necessary at startup */
1732         virtio_reset(hw);
1733
1734         if (hw->vqs) {
1735                 virtio_dev_free_mbufs(eth_dev);
1736                 virtio_free_queues(hw);
1737         }
1738
1739         /* Tell the host we've noticed this device. */
1740         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1741
1742         /* Tell the host we've known how to drive the device. */
1743         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1744         if (virtio_ethdev_negotiate_features(hw, req_features) < 0)
1745                 return -1;
1746
1747         hw->weak_barriers = !virtio_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
1748
1749         /* If host does not support both status and MSI-X then disable LSC */
1750         if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS) && hw->intr_lsc)
1751                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
1752         else
1753                 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
1754
1755         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
1756
1757         /* Setting up rx_header size for the device */
1758         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
1759             virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
1760             virtio_with_packed_queue(hw))
1761                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1762         else
1763                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
1764
1765         /* Copy the permanent MAC address to: virtio_hw */
1766         virtio_get_hwaddr(hw);
1767         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
1768                         &eth_dev->data->mac_addrs[0]);
1769         PMD_INIT_LOG(DEBUG,
1770                      "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
1771                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1772                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1773
1774         if (hw->speed == ETH_SPEED_NUM_UNKNOWN) {
1775                 if (virtio_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) {
1776                         config = &local_config;
1777                         virtio_read_dev_config(hw,
1778                                 offsetof(struct virtio_net_config, speed),
1779                                 &config->speed, sizeof(config->speed));
1780                         virtio_read_dev_config(hw,
1781                                 offsetof(struct virtio_net_config, duplex),
1782                                 &config->duplex, sizeof(config->duplex));
1783                         hw->speed = config->speed;
1784                         hw->duplex = config->duplex;
1785                 }
1786         }
1787         if (hw->duplex == DUPLEX_UNKNOWN)
1788                 hw->duplex = ETH_LINK_FULL_DUPLEX;
1789         PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d",
1790                 hw->speed, hw->duplex);
1791         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1792                 config = &local_config;
1793
1794                 virtio_read_dev_config(hw,
1795                         offsetof(struct virtio_net_config, mac),
1796                         &config->mac, sizeof(config->mac));
1797
1798                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1799                         virtio_read_dev_config(hw,
1800                                 offsetof(struct virtio_net_config, status),
1801                                 &config->status, sizeof(config->status));
1802                 } else {
1803                         PMD_INIT_LOG(DEBUG,
1804                                      "VIRTIO_NET_F_STATUS is not supported");
1805                         config->status = 0;
1806                 }
1807
1808                 if (virtio_with_feature(hw, VIRTIO_NET_F_MQ)) {
1809                         virtio_read_dev_config(hw,
1810                                 offsetof(struct virtio_net_config, max_virtqueue_pairs),
1811                                 &config->max_virtqueue_pairs,
1812                                 sizeof(config->max_virtqueue_pairs));
1813                 } else {
1814                         PMD_INIT_LOG(DEBUG,
1815                                      "VIRTIO_NET_F_MQ is not supported");
1816                         config->max_virtqueue_pairs = 1;
1817                 }
1818
1819                 hw->max_queue_pairs = config->max_virtqueue_pairs;
1820
1821                 if (virtio_with_feature(hw, VIRTIO_NET_F_MTU)) {
1822                         virtio_read_dev_config(hw,
1823                                 offsetof(struct virtio_net_config, mtu),
1824                                 &config->mtu,
1825                                 sizeof(config->mtu));
1826
1827                         /*
1828                          * MTU value has already been checked at negotiation
1829                          * time, but check again in case it has changed since
1830                          * then, which should not happen.
1831                          */
1832                         if (config->mtu < RTE_ETHER_MIN_MTU) {
1833                                 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)",
1834                                                 config->mtu);
1835                                 return -1;
1836                         }
1837
1838                         hw->max_mtu = config->mtu;
1839                         /* Set initial MTU to maximum one supported by vhost */
1840                         eth_dev->data->mtu = config->mtu;
1841
1842                 } else {
1843                         hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1844                                 VLAN_TAG_LEN - hw->vtnet_hdr_size;
1845                 }
1846
1847                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
1848                                 config->max_virtqueue_pairs);
1849                 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
1850                 PMD_INIT_LOG(DEBUG,
1851                                 "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
1852                                 config->mac[0], config->mac[1],
1853                                 config->mac[2], config->mac[3],
1854                                 config->mac[4], config->mac[5]);
1855         } else {
1856                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
1857                 hw->max_queue_pairs = 1;
1858                 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1859                         VLAN_TAG_LEN - hw->vtnet_hdr_size;
1860         }
1861
1862         ret = virtio_alloc_queues(eth_dev);
1863         if (ret < 0)
1864                 return ret;
1865
1866         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1867                 if (virtio_configure_intr(eth_dev) < 0) {
1868                         PMD_INIT_LOG(ERR, "failed to configure interrupt");
1869                         virtio_free_queues(hw);
1870                         return -1;
1871                 }
1872         }
1873
1874         virtio_reinit_complete(hw);
1875
1876         return 0;
1877 }
1878
1879 /*
1880  * This function is based on probe() function in virtio_pci.c
1881  * It returns 0 on success.
1882  */
1883 int
1884 eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
1885 {
1886         struct virtio_hw *hw = eth_dev->data->dev_private;
1887         uint32_t speed = ETH_SPEED_NUM_UNKNOWN;
1888         int vectorized = 0;
1889         int ret;
1890
1891         if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
1892                 PMD_INIT_LOG(ERR,
1893                         "Not sufficient headroom required = %d, avail = %d",
1894                         (int)sizeof(struct virtio_net_hdr_mrg_rxbuf),
1895                         RTE_PKTMBUF_HEADROOM);
1896
1897                 return -1;
1898         }
1899
1900         eth_dev->dev_ops = &virtio_eth_dev_ops;
1901
1902         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1903                 set_rxtx_funcs(eth_dev);
1904                 return 0;
1905         }
1906
1907         ret = virtio_dev_devargs_parse(eth_dev->device->devargs, &speed, &vectorized);
1908         if (ret < 0)
1909                 return ret;
1910         hw->speed = speed;
1911         hw->duplex = DUPLEX_UNKNOWN;
1912
1913         /* Allocate memory for storing MAC addresses */
1914         eth_dev->data->mac_addrs = rte_zmalloc("virtio",
1915                                 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN, 0);
1916         if (eth_dev->data->mac_addrs == NULL) {
1917                 PMD_INIT_LOG(ERR,
1918                         "Failed to allocate %d bytes needed to store MAC addresses",
1919                         VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN);
1920                 return -ENOMEM;
1921         }
1922
1923         rte_spinlock_init(&hw->state_lock);
1924
1925         /* reset device and negotiate default features */
1926         ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
1927         if (ret < 0)
1928                 goto err_virtio_init;
1929
1930         if (vectorized) {
1931                 if (!virtio_with_packed_queue(hw)) {
1932                         hw->use_vec_rx = 1;
1933                 } else {
1934 #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
1935                         hw->use_vec_rx = 1;
1936                         hw->use_vec_tx = 1;
1937 #else
1938                         PMD_DRV_LOG(INFO,
1939                                 "building environment do not support packed ring vectorized");
1940 #endif
1941                 }
1942         }
1943
1944         hw->opened = 1;
1945
1946         return 0;
1947
1948 err_virtio_init:
1949         rte_free(eth_dev->data->mac_addrs);
1950         eth_dev->data->mac_addrs = NULL;
1951         return ret;
1952 }
1953
1954 static uint32_t
1955 virtio_dev_speed_capa_get(uint32_t speed)
1956 {
1957         switch (speed) {
1958         case ETH_SPEED_NUM_10G:
1959                 return ETH_LINK_SPEED_10G;
1960         case ETH_SPEED_NUM_20G:
1961                 return ETH_LINK_SPEED_20G;
1962         case ETH_SPEED_NUM_25G:
1963                 return ETH_LINK_SPEED_25G;
1964         case ETH_SPEED_NUM_40G:
1965                 return ETH_LINK_SPEED_40G;
1966         case ETH_SPEED_NUM_50G:
1967                 return ETH_LINK_SPEED_50G;
1968         case ETH_SPEED_NUM_56G:
1969                 return ETH_LINK_SPEED_56G;
1970         case ETH_SPEED_NUM_100G:
1971                 return ETH_LINK_SPEED_100G;
1972         case ETH_SPEED_NUM_200G:
1973                 return ETH_LINK_SPEED_200G;
1974         default:
1975                 return 0;
1976         }
1977 }
1978
1979 static int vectorized_check_handler(__rte_unused const char *key,
1980                 const char *value, void *ret_val)
1981 {
1982         if (strcmp(value, "1") == 0)
1983                 *(int *)ret_val = 1;
1984         else
1985                 *(int *)ret_val = 0;
1986
1987         return 0;
1988 }
1989
1990 #define VIRTIO_ARG_SPEED      "speed"
1991 #define VIRTIO_ARG_VECTORIZED "vectorized"
1992
1993 static int
1994 link_speed_handler(const char *key __rte_unused,
1995                 const char *value, void *ret_val)
1996 {
1997         uint32_t val;
1998         if (!value || !ret_val)
1999                 return -EINVAL;
2000         val = strtoul(value, NULL, 0);
2001         /* validate input */
2002         if (virtio_dev_speed_capa_get(val) == 0)
2003                 return -EINVAL;
2004         *(uint32_t *)ret_val = val;
2005
2006         return 0;
2007 }
2008
2009
2010 static int
2011 virtio_dev_devargs_parse(struct rte_devargs *devargs, uint32_t *speed, int *vectorized)
2012 {
2013         struct rte_kvargs *kvlist;
2014         int ret = 0;
2015
2016         if (devargs == NULL)
2017                 return 0;
2018
2019         kvlist = rte_kvargs_parse(devargs->args, NULL);
2020         if (kvlist == NULL) {
2021                 PMD_INIT_LOG(ERR, "error when parsing param");
2022                 return 0;
2023         }
2024
2025         if (speed && rte_kvargs_count(kvlist, VIRTIO_ARG_SPEED) == 1) {
2026                 ret = rte_kvargs_process(kvlist,
2027                                         VIRTIO_ARG_SPEED,
2028                                         link_speed_handler, speed);
2029                 if (ret < 0) {
2030                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2031                                         VIRTIO_ARG_SPEED);
2032                         goto exit;
2033                 }
2034         }
2035
2036         if (vectorized &&
2037                 rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
2038                 ret = rte_kvargs_process(kvlist,
2039                                 VIRTIO_ARG_VECTORIZED,
2040                                 vectorized_check_handler, vectorized);
2041                 if (ret < 0) {
2042                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2043                                         VIRTIO_ARG_VECTORIZED);
2044                         goto exit;
2045                 }
2046         }
2047
2048 exit:
2049         rte_kvargs_free(kvlist);
2050         return ret;
2051 }
2052
2053 static uint8_t
2054 rx_offload_enabled(struct virtio_hw *hw)
2055 {
2056         return virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
2057                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2058                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
2059 }
2060
2061 static uint8_t
2062 tx_offload_enabled(struct virtio_hw *hw)
2063 {
2064         return virtio_with_feature(hw, VIRTIO_NET_F_CSUM) ||
2065                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
2066                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
2067 }
2068
2069 /*
2070  * Configure virtio device
2071  * It returns 0 on success.
2072  */
2073 static int
2074 virtio_dev_configure(struct rte_eth_dev *dev)
2075 {
2076         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2077         const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
2078         struct virtio_hw *hw = dev->data->dev_private;
2079         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
2080                 hw->vtnet_hdr_size;
2081         uint64_t rx_offloads = rxmode->offloads;
2082         uint64_t tx_offloads = txmode->offloads;
2083         uint64_t req_features;
2084         int ret;
2085
2086         PMD_INIT_LOG(DEBUG, "configure");
2087         req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
2088
2089         if (rxmode->mq_mode != ETH_MQ_RX_NONE) {
2090                 PMD_DRV_LOG(ERR,
2091                         "Unsupported Rx multi queue mode %d",
2092                         rxmode->mq_mode);
2093                 return -EINVAL;
2094         }
2095
2096         if (txmode->mq_mode != ETH_MQ_TX_NONE) {
2097                 PMD_DRV_LOG(ERR,
2098                         "Unsupported Tx multi queue mode %d",
2099                         txmode->mq_mode);
2100                 return -EINVAL;
2101         }
2102
2103         if (dev->data->dev_conf.intr_conf.rxq) {
2104                 ret = virtio_init_device(dev, hw->req_guest_features);
2105                 if (ret < 0)
2106                         return ret;
2107         }
2108
2109         if (rxmode->mtu > hw->max_mtu)
2110                 req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
2111
2112         hw->max_rx_pkt_len = ether_hdr_len + rxmode->mtu;
2113
2114         if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2115                            DEV_RX_OFFLOAD_TCP_CKSUM))
2116                 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
2117
2118         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO)
2119                 req_features |=
2120                         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2121                         (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2122
2123         if (tx_offloads & (DEV_TX_OFFLOAD_UDP_CKSUM |
2124                            DEV_TX_OFFLOAD_TCP_CKSUM))
2125                 req_features |= (1ULL << VIRTIO_NET_F_CSUM);
2126
2127         if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO)
2128                 req_features |=
2129                         (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2130                         (1ULL << VIRTIO_NET_F_HOST_TSO6);
2131
2132         /* if request features changed, reinit the device */
2133         if (req_features != hw->req_guest_features) {
2134                 ret = virtio_init_device(dev, req_features);
2135                 if (ret < 0)
2136                         return ret;
2137         }
2138
2139         if ((rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2140                             DEV_RX_OFFLOAD_TCP_CKSUM)) &&
2141                 !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
2142                 PMD_DRV_LOG(ERR,
2143                         "rx checksum not available on this host");
2144                 return -ENOTSUP;
2145         }
2146
2147         if ((rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) &&
2148                 (!virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2149                  !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) {
2150                 PMD_DRV_LOG(ERR,
2151                         "Large Receive Offload not available on this host");
2152                 return -ENOTSUP;
2153         }
2154
2155         /* start control queue */
2156         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
2157                 virtio_dev_cq_start(dev);
2158
2159         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2160                 hw->vlan_strip = 1;
2161
2162         hw->rx_ol_scatter = (rx_offloads & DEV_RX_OFFLOAD_SCATTER);
2163
2164         if ((rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER) &&
2165                         !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2166                 PMD_DRV_LOG(ERR,
2167                             "vlan filtering not available on this host");
2168                 return -ENOTSUP;
2169         }
2170
2171         hw->has_tx_offload = tx_offload_enabled(hw);
2172         hw->has_rx_offload = rx_offload_enabled(hw);
2173
2174         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2175                 /* Enable vector (0) for Link State Intrerrupt */
2176                 if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) ==
2177                                 VIRTIO_MSI_NO_VECTOR) {
2178                         PMD_DRV_LOG(ERR, "failed to set config vector");
2179                         return -EBUSY;
2180                 }
2181
2182         if (virtio_with_packed_queue(hw)) {
2183 #if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT)
2184                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2185                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
2186                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2187                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2188                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_512)) {
2189                         PMD_DRV_LOG(INFO,
2190                                 "disabled packed ring vectorized path for requirements not met");
2191                         hw->use_vec_rx = 0;
2192                         hw->use_vec_tx = 0;
2193                 }
2194 #elif defined(RTE_ARCH_ARM)
2195                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2196                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) ||
2197                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2198                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2199                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)) {
2200                         PMD_DRV_LOG(INFO,
2201                                 "disabled packed ring vectorized path for requirements not met");
2202                         hw->use_vec_rx = 0;
2203                         hw->use_vec_tx = 0;
2204                 }
2205 #else
2206                 hw->use_vec_rx = 0;
2207                 hw->use_vec_tx = 0;
2208 #endif
2209
2210                 if (hw->use_vec_rx) {
2211                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2212                                 PMD_DRV_LOG(INFO,
2213                                         "disabled packed ring vectorized rx for mrg_rxbuf enabled");
2214                                 hw->use_vec_rx = 0;
2215                         }
2216
2217                         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
2218                                 PMD_DRV_LOG(INFO,
2219                                         "disabled packed ring vectorized rx for TCP_LRO enabled");
2220                                 hw->use_vec_rx = 0;
2221                         }
2222                 }
2223         } else {
2224                 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) {
2225                         hw->use_inorder_tx = 1;
2226                         hw->use_inorder_rx = 1;
2227                         hw->use_vec_rx = 0;
2228                 }
2229
2230                 if (hw->use_vec_rx) {
2231 #if defined RTE_ARCH_ARM
2232                         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
2233                                 PMD_DRV_LOG(INFO,
2234                                         "disabled split ring vectorized path for requirement not met");
2235                                 hw->use_vec_rx = 0;
2236                         }
2237 #endif
2238                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2239                                 PMD_DRV_LOG(INFO,
2240                                         "disabled split ring vectorized rx for mrg_rxbuf enabled");
2241                                 hw->use_vec_rx = 0;
2242                         }
2243
2244                         if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM |
2245                                            DEV_RX_OFFLOAD_TCP_CKSUM |
2246                                            DEV_RX_OFFLOAD_TCP_LRO |
2247                                            DEV_RX_OFFLOAD_VLAN_STRIP)) {
2248                                 PMD_DRV_LOG(INFO,
2249                                         "disabled split ring vectorized rx for offloading enabled");
2250                                 hw->use_vec_rx = 0;
2251                         }
2252
2253                         if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
2254                                 PMD_DRV_LOG(INFO,
2255                                         "disabled split ring vectorized rx, max SIMD bitwidth too low");
2256                                 hw->use_vec_rx = 0;
2257                         }
2258                 }
2259         }
2260
2261         return 0;
2262 }
2263
2264
2265 static int
2266 virtio_dev_start(struct rte_eth_dev *dev)
2267 {
2268         uint16_t nb_queues, i;
2269         struct virtqueue *vq;
2270         struct virtio_hw *hw = dev->data->dev_private;
2271         int ret;
2272
2273         /* Finish the initialization of the queues */
2274         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2275                 ret = virtio_dev_rx_queue_setup_finish(dev, i);
2276                 if (ret < 0)
2277                         return ret;
2278         }
2279         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2280                 ret = virtio_dev_tx_queue_setup_finish(dev, i);
2281                 if (ret < 0)
2282                         return ret;
2283         }
2284
2285         /* check if lsc interrupt feature is enabled */
2286         if (dev->data->dev_conf.intr_conf.lsc) {
2287                 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
2288                         PMD_DRV_LOG(ERR, "link status not supported by host");
2289                         return -ENOTSUP;
2290                 }
2291         }
2292
2293         /* Enable uio/vfio intr/eventfd mapping: althrough we already did that
2294          * in device configure, but it could be unmapped  when device is
2295          * stopped.
2296          */
2297         if (dev->data->dev_conf.intr_conf.lsc ||
2298             dev->data->dev_conf.intr_conf.rxq) {
2299                 virtio_intr_disable(dev);
2300
2301                 /* Setup interrupt callback  */
2302                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2303                         rte_intr_callback_register(dev->intr_handle,
2304                                                    virtio_interrupt_handler,
2305                                                    dev);
2306
2307                 if (virtio_intr_enable(dev) < 0) {
2308                         PMD_DRV_LOG(ERR, "interrupt enable failed");
2309                         return -EIO;
2310                 }
2311         }
2312
2313         /*Notify the backend
2314          *Otherwise the tap backend might already stop its queue due to fullness.
2315          *vhost backend will have no chance to be waked up
2316          */
2317         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
2318         if (hw->max_queue_pairs > 1) {
2319                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
2320                         return -EINVAL;
2321         }
2322
2323         PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
2324
2325         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2326                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2327                 /* Flush the old packets */
2328                 virtqueue_rxvq_flush(vq);
2329                 virtqueue_notify(vq);
2330         }
2331
2332         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2333                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2334                 virtqueue_notify(vq);
2335         }
2336
2337         PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
2338
2339         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2340                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2341                 VIRTQUEUE_DUMP(vq);
2342         }
2343
2344         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2345                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2346                 VIRTQUEUE_DUMP(vq);
2347         }
2348
2349         set_rxtx_funcs(dev);
2350         hw->started = 1;
2351
2352         /* Initialize Link state */
2353         virtio_dev_link_update(dev, 0);
2354
2355         return 0;
2356 }
2357
2358 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
2359 {
2360         struct virtio_hw *hw = dev->data->dev_private;
2361         uint16_t nr_vq = virtio_get_nr_vq(hw);
2362         const char *type __rte_unused;
2363         unsigned int i, mbuf_num = 0;
2364         struct virtqueue *vq;
2365         struct rte_mbuf *buf;
2366         int queue_type;
2367
2368         if (hw->vqs == NULL)
2369                 return;
2370
2371         for (i = 0; i < nr_vq; i++) {
2372                 vq = hw->vqs[i];
2373                 if (!vq)
2374                         continue;
2375
2376                 queue_type = virtio_get_queue_type(hw, i);
2377                 if (queue_type == VTNET_RQ)
2378                         type = "rxq";
2379                 else if (queue_type == VTNET_TQ)
2380                         type = "txq";
2381                 else
2382                         continue;
2383
2384                 PMD_INIT_LOG(DEBUG,
2385                         "Before freeing %s[%d] used and unused buf",
2386                         type, i);
2387                 VIRTQUEUE_DUMP(vq);
2388
2389                 while ((buf = virtqueue_detach_unused(vq)) != NULL) {
2390                         rte_pktmbuf_free(buf);
2391                         mbuf_num++;
2392                 }
2393
2394                 PMD_INIT_LOG(DEBUG,
2395                         "After freeing %s[%d] used and unused buf",
2396                         type, i);
2397                 VIRTQUEUE_DUMP(vq);
2398         }
2399
2400         PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num);
2401 }
2402
2403 static void
2404 virtio_tx_completed_cleanup(struct rte_eth_dev *dev)
2405 {
2406         struct virtio_hw *hw = dev->data->dev_private;
2407         struct virtqueue *vq;
2408         int qidx;
2409         void (*xmit_cleanup)(struct virtqueue *vq, uint16_t nb_used);
2410
2411         if (virtio_with_packed_queue(hw)) {
2412                 if (hw->use_vec_tx)
2413                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2414                 else if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER))
2415                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2416                 else
2417                         xmit_cleanup = &virtio_xmit_cleanup_normal_packed;
2418         } else {
2419                 if (hw->use_inorder_tx)
2420                         xmit_cleanup = &virtio_xmit_cleanup_inorder;
2421                 else
2422                         xmit_cleanup = &virtio_xmit_cleanup;
2423         }
2424
2425         for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) {
2426                 vq = hw->vqs[2 * qidx + VTNET_SQ_TQ_QUEUE_IDX];
2427                 if (vq != NULL)
2428                         xmit_cleanup(vq, virtqueue_nused(vq));
2429         }
2430 }
2431
2432 /*
2433  * Stop device: disable interrupt and mark link down
2434  */
2435 int
2436 virtio_dev_stop(struct rte_eth_dev *dev)
2437 {
2438         struct virtio_hw *hw = dev->data->dev_private;
2439         struct rte_eth_link link;
2440         struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
2441
2442         PMD_INIT_LOG(DEBUG, "stop");
2443         dev->data->dev_started = 0;
2444
2445         rte_spinlock_lock(&hw->state_lock);
2446         if (!hw->started)
2447                 goto out_unlock;
2448         hw->started = 0;
2449
2450         virtio_tx_completed_cleanup(dev);
2451
2452         if (intr_conf->lsc || intr_conf->rxq) {
2453                 virtio_intr_disable(dev);
2454
2455                 /* Reset interrupt callback  */
2456                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
2457                         rte_intr_callback_unregister(dev->intr_handle,
2458                                                      virtio_interrupt_handler,
2459                                                      dev);
2460                 }
2461         }
2462
2463         memset(&link, 0, sizeof(link));
2464         rte_eth_linkstatus_set(dev, &link);
2465 out_unlock:
2466         rte_spinlock_unlock(&hw->state_lock);
2467
2468         return 0;
2469 }
2470
2471 static int
2472 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
2473 {
2474         struct rte_eth_link link;
2475         uint16_t status;
2476         struct virtio_hw *hw = dev->data->dev_private;
2477
2478         memset(&link, 0, sizeof(link));
2479         link.link_duplex = hw->duplex;
2480         link.link_speed  = hw->speed;
2481         link.link_autoneg = ETH_LINK_AUTONEG;
2482
2483         if (!hw->started) {
2484                 link.link_status = ETH_LINK_DOWN;
2485                 link.link_speed = ETH_SPEED_NUM_NONE;
2486         } else if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
2487                 PMD_INIT_LOG(DEBUG, "Get link status from hw");
2488                 virtio_read_dev_config(hw,
2489                                 offsetof(struct virtio_net_config, status),
2490                                 &status, sizeof(status));
2491                 if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
2492                         link.link_status = ETH_LINK_DOWN;
2493                         link.link_speed = ETH_SPEED_NUM_NONE;
2494                         PMD_INIT_LOG(DEBUG, "Port %d is down",
2495                                      dev->data->port_id);
2496                 } else {
2497                         link.link_status = ETH_LINK_UP;
2498                         PMD_INIT_LOG(DEBUG, "Port %d is up",
2499                                      dev->data->port_id);
2500                 }
2501         } else {
2502                 link.link_status = ETH_LINK_UP;
2503         }
2504
2505         return rte_eth_linkstatus_set(dev, &link);
2506 }
2507
2508 static int
2509 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2510 {
2511         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2512         struct virtio_hw *hw = dev->data->dev_private;
2513         uint64_t offloads = rxmode->offloads;
2514
2515         if (mask & ETH_VLAN_FILTER_MASK) {
2516                 if ((offloads & DEV_RX_OFFLOAD_VLAN_FILTER) &&
2517                                 !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2518
2519                         PMD_DRV_LOG(NOTICE,
2520                                 "vlan filtering not available on this host");
2521
2522                         return -ENOTSUP;
2523                 }
2524         }
2525
2526         if (mask & ETH_VLAN_STRIP_MASK)
2527                 hw->vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
2528
2529         return 0;
2530 }
2531
2532 static int
2533 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2534 {
2535         uint64_t tso_mask, host_features;
2536         struct virtio_hw *hw = dev->data->dev_private;
2537         dev_info->speed_capa = virtio_dev_speed_capa_get(hw->speed);
2538
2539         dev_info->max_rx_queues =
2540                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
2541         dev_info->max_tx_queues =
2542                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
2543         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
2544         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
2545         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
2546         dev_info->max_mtu = hw->max_mtu;
2547
2548         host_features = VIRTIO_OPS(hw)->get_features(hw);
2549         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
2550         dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
2551         if (host_features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))
2552                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_SCATTER;
2553         if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
2554                 dev_info->rx_offload_capa |=
2555                         DEV_RX_OFFLOAD_TCP_CKSUM |
2556                         DEV_RX_OFFLOAD_UDP_CKSUM;
2557         }
2558         if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN))
2559                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_VLAN_FILTER;
2560         tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2561                 (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2562         if ((host_features & tso_mask) == tso_mask)
2563                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2564
2565         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
2566                                     DEV_TX_OFFLOAD_VLAN_INSERT;
2567         if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) {
2568                 dev_info->tx_offload_capa |=
2569                         DEV_TX_OFFLOAD_UDP_CKSUM |
2570                         DEV_TX_OFFLOAD_TCP_CKSUM;
2571         }
2572         tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2573                 (1ULL << VIRTIO_NET_F_HOST_TSO6);
2574         if ((host_features & tso_mask) == tso_mask)
2575                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
2576
2577         if (host_features & (1ULL << VIRTIO_F_RING_PACKED)) {
2578                 /*
2579                  * According to 2.7 Packed Virtqueues,
2580                  * 2.7.10.1 Structure Size and Alignment:
2581                  * The Queue Size value does not have to be a power of 2.
2582                  */
2583                 dev_info->rx_desc_lim.nb_max = UINT16_MAX;
2584                 dev_info->tx_desc_lim.nb_max = UINT16_MAX;
2585         } else {
2586                 /*
2587                  * According to 2.6 Split Virtqueues:
2588                  * Queue Size value is always a power of 2. The maximum Queue
2589                  * Size value is 32768.
2590                  */
2591                 dev_info->rx_desc_lim.nb_max = 32768;
2592                 dev_info->tx_desc_lim.nb_max = 32768;
2593         }
2594         /*
2595          * Actual minimum is not the same for virtqueues of different kinds,
2596          * but to avoid tangling the code with separate branches, rely on
2597          * default thresholds since desc number must be at least of their size.
2598          */
2599         dev_info->rx_desc_lim.nb_min = RTE_MAX(DEFAULT_RX_FREE_THRESH,
2600                                                RTE_VIRTIO_VPMD_RX_REARM_THRESH);
2601         dev_info->tx_desc_lim.nb_min = DEFAULT_TX_FREE_THRESH;
2602         dev_info->rx_desc_lim.nb_align = 1;
2603         dev_info->tx_desc_lim.nb_align = 1;
2604
2605         return 0;
2606 }
2607
2608 /*
2609  * It enables testpmd to collect per queue stats.
2610  */
2611 static int
2612 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
2613 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
2614 __rte_unused uint8_t is_rx)
2615 {
2616         return 0;
2617 }
2618
2619 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_init, init, NOTICE);
2620 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_driver, driver, NOTICE);