drivers/net/virtio/virtio_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10
  11 #include <ethdev_driver.h>
  12 #include <rte_memcpy.h>
  13 #include <rte_string_fns.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_branch_prediction.h>
  17 #include <rte_ether.h>
  18 #include <rte_ip.h>
  19 #include <rte_arp.h>
  20 #include <rte_common.h>
  21 #include <rte_errno.h>
  22 #include <rte_cpuflags.h>
  23 #include <rte_vect.h>
  24 #include <rte_memory.h>
  25 #include <rte_eal_paging.h>
  26 #include <rte_eal.h>
  27 #include <rte_dev.h>
  28 #include <rte_cycles.h>
  29 #include <rte_kvargs.h>
  30
  31 #include "virtio_ethdev.h"
  32 #include "virtio.h"
  33 #include "virtio_logs.h"
  34 #include "virtqueue.h"
  35 #include "virtio_rxtx.h"
  36 #include "virtio_rxtx_simple.h"
  37 #include "virtio_user/virtio_user_dev.h"
  38
  39 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  40 static int  virtio_dev_start(struct rte_eth_dev *dev);
  41 static int virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
  42 static int virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
  43 static int virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
  44 static int virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
  45 static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
  46 static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
  47         uint32_t *speed,
  48         int *vectorized);
  49 static int virtio_dev_info_get(struct rte_eth_dev *dev,
  50                                 struct rte_eth_dev_info *dev_info);
  51 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  52         int wait_to_complete);
  53 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  54
  55 static void virtio_set_hwaddr(struct virtio_hw *hw);
  56 static void virtio_get_hwaddr(struct virtio_hw *hw);
  57
  58 static int virtio_dev_stats_get(struct rte_eth_dev *dev,
  59                                  struct rte_eth_stats *stats);
  60 static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
  61                                  struct rte_eth_xstat *xstats, unsigned n);
  62 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
  63                                        struct rte_eth_xstat_name *xstats_names,
  64                                        unsigned limit);
  65 static int virtio_dev_stats_reset(struct rte_eth_dev *dev);
  66 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  67 static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
  68                                 uint16_t vlan_id, int on);
  69 static int virtio_mac_addr_add(struct rte_eth_dev *dev,
  70                                 struct rte_ether_addr *mac_addr,
  71                                 uint32_t index, uint32_t vmdq);
  72 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  73 static int virtio_mac_addr_set(struct rte_eth_dev *dev,
  74                                 struct rte_ether_addr *mac_addr);
  75
  76 static int virtio_intr_disable(struct rte_eth_dev *dev);
  77
  78 static int virtio_dev_queue_stats_mapping_set(
  79         struct rte_eth_dev *eth_dev,
  80         uint16_t queue_id,
  81         uint8_t stat_idx,
  82         uint8_t is_rx);
  83
  84 static void virtio_notify_peers(struct rte_eth_dev *dev);
  85 static void virtio_ack_link_announce(struct rte_eth_dev *dev);
  86
  87 struct rte_virtio_xstats_name_off {
  88         char name[RTE_ETH_XSTATS_NAME_SIZE];
  89         unsigned offset;
  90 };
  91
  92 /* [rt]x_qX_ is prepended to the name string here */
  93 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
  94         {"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
  95         {"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
  96         {"errors",                 offsetof(struct virtnet_rx, stats.errors)},
  97         {"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
  98         {"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
  99         {"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
 100         {"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
 101         {"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
 102         {"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
 103         {"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
 104         {"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
 105         {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
 106         {"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
 107 };
 108
 109 /* [rt]x_qX_ is prepended to the name string here */
 110 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
 111         {"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
 112         {"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
 113         {"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
 114         {"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
 115         {"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
 116         {"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
 117         {"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
 118         {"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
 119         {"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
 120         {"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
 121         {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
 122         {"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
 123 };
 124
 125 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
 126                             sizeof(rte_virtio_rxq_stat_strings[0]))
 127 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
 128                             sizeof(rte_virtio_txq_stat_strings[0]))
 129
 130 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS];
 131
 132 static struct virtio_pmd_ctrl *
 133 virtio_send_command_packed(struct virtnet_ctl *cvq,
 134                            struct virtio_pmd_ctrl *ctrl,
 135                            int *dlen, int pkt_num)
 136 {
 137         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 138         int head;
 139         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 140         struct virtio_pmd_ctrl *result;
 141         uint16_t flags;
 142         int sum = 0;
 143         int nb_descs = 0;
 144         int k;
 145
 146         /*
 147          * Format is enforced in qemu code:
 148          * One TX packet for header;
 149          * At least one TX packet per argument;
 150          * One RX packet for ACK.
 151          */
 152         head = vq->vq_avail_idx;
 153         flags = vq->vq_packed.cached_flags;
 154         desc[head].addr = cvq->virtio_net_hdr_mem;
 155         desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 156         vq->vq_free_cnt--;
 157         nb_descs++;
 158         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 159                 vq->vq_avail_idx -= vq->vq_nentries;
 160                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 161         }
 162
 163         for (k = 0; k < pkt_num; k++) {
 164                 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 165                         + sizeof(struct virtio_net_ctrl_hdr)
 166                         + sizeof(ctrl->status) + sizeof(uint8_t) * sum;
 167                 desc[vq->vq_avail_idx].len = dlen[k];
 168                 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT |
 169                         vq->vq_packed.cached_flags;
 170                 sum += dlen[k];
 171                 vq->vq_free_cnt--;
 172                 nb_descs++;
 173                 if (++vq->vq_avail_idx >= vq->vq_nentries) {
 174                         vq->vq_avail_idx -= vq->vq_nentries;
 175                         vq->vq_packed.cached_flags ^=
 176                                 VRING_PACKED_DESC_F_AVAIL_USED;
 177                 }
 178         }
 179
 180         desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 181                 + sizeof(struct virtio_net_ctrl_hdr);
 182         desc[vq->vq_avail_idx].len = sizeof(ctrl->status);
 183         desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE |
 184                 vq->vq_packed.cached_flags;
 185         vq->vq_free_cnt--;
 186         nb_descs++;
 187         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 188                 vq->vq_avail_idx -= vq->vq_nentries;
 189                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 190         }
 191
 192         virtqueue_store_flags_packed(&desc[head], VRING_DESC_F_NEXT | flags,
 193                         vq->hw->weak_barriers);
 194
 195         virtio_wmb(vq->hw->weak_barriers);
 196         virtqueue_notify(vq);
 197
 198         /* wait for used desc in virtqueue
 199          * desc_is_used has a load-acquire or rte_io_rmb inside
 200          */
 201         while (!desc_is_used(&desc[head], vq))
 202                 usleep(100);
 203
 204         /* now get used descriptors */
 205         vq->vq_free_cnt += nb_descs;
 206         vq->vq_used_cons_idx += nb_descs;
 207         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 208                 vq->vq_used_cons_idx -= vq->vq_nentries;
 209                 vq->vq_packed.used_wrap_counter ^= 1;
 210         }
 211
 212         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n"
 213                         "vq->vq_avail_idx=%d\n"
 214                         "vq->vq_used_cons_idx=%d\n"
 215                         "vq->vq_packed.cached_flags=0x%x\n"
 216                         "vq->vq_packed.used_wrap_counter=%d",
 217                         vq->vq_free_cnt,
 218                         vq->vq_avail_idx,
 219                         vq->vq_used_cons_idx,
 220                         vq->vq_packed.cached_flags,
 221                         vq->vq_packed.used_wrap_counter);
 222
 223         result = cvq->virtio_net_hdr_mz->addr;
 224         return result;
 225 }
 226
 227 static struct virtio_pmd_ctrl *
 228 virtio_send_command_split(struct virtnet_ctl *cvq,
 229                           struct virtio_pmd_ctrl *ctrl,
 230                           int *dlen, int pkt_num)
 231 {
 232         struct virtio_pmd_ctrl *result;
 233         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 234         uint32_t head, i;
 235         int k, sum = 0;
 236
 237         head = vq->vq_desc_head_idx;
 238
 239         /*
 240          * Format is enforced in qemu code:
 241          * One TX packet for header;
 242          * At least one TX packet per argument;
 243          * One RX packet for ACK.
 244          */
 245         vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT;
 246         vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem;
 247         vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 248         vq->vq_free_cnt--;
 249         i = vq->vq_split.ring.desc[head].next;
 250
 251         for (k = 0; k < pkt_num; k++) {
 252                 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT;
 253                 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 254                         + sizeof(struct virtio_net_ctrl_hdr)
 255                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 256                 vq->vq_split.ring.desc[i].len = dlen[k];
 257                 sum += dlen[k];
 258                 vq->vq_free_cnt--;
 259                 i = vq->vq_split.ring.desc[i].next;
 260         }
 261
 262         vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE;
 263         vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 264                         + sizeof(struct virtio_net_ctrl_hdr);
 265         vq->vq_split.ring.desc[i].len = sizeof(ctrl->status);
 266         vq->vq_free_cnt--;
 267
 268         vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next;
 269
 270         vq_update_avail_ring(vq, head);
 271         vq_update_avail_idx(vq);
 272
 273         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
 274
 275         virtqueue_notify(vq);
 276
 277         while (virtqueue_nused(vq) == 0)
 278                 usleep(100);
 279
 280         while (virtqueue_nused(vq)) {
 281                 uint32_t idx, desc_idx, used_idx;
 282                 struct vring_used_elem *uep;
 283
 284                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 285                                 & (vq->vq_nentries - 1));
 286                 uep = &vq->vq_split.ring.used->ring[used_idx];
 287                 idx = (uint32_t) uep->id;
 288                 desc_idx = idx;
 289
 290                 while (vq->vq_split.ring.desc[desc_idx].flags &
 291                                 VRING_DESC_F_NEXT) {
 292                         desc_idx = vq->vq_split.ring.desc[desc_idx].next;
 293                         vq->vq_free_cnt++;
 294                 }
 295
 296                 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 297                 vq->vq_desc_head_idx = idx;
 298
 299                 vq->vq_used_cons_idx++;
 300                 vq->vq_free_cnt++;
 301         }
 302
 303         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
 304                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 305
 306         result = cvq->virtio_net_hdr_mz->addr;
 307         return result;
 308 }
 309
 310 static int
 311 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
 312                     int *dlen, int pkt_num)
 313 {
 314         virtio_net_ctrl_ack status = ~0;
 315         struct virtio_pmd_ctrl *result;
 316         struct virtqueue *vq;
 317
 318         ctrl->status = status;
 319
 320         if (!cvq) {
 321                 PMD_INIT_LOG(ERR, "Control queue is not supported.");
 322                 return -1;
 323         }
 324
 325         rte_spinlock_lock(&cvq->lock);
 326         vq = virtnet_cq_to_vq(cvq);
 327
 328         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 329                 "vq->hw->cvq = %p vq = %p",
 330                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 331
 332         if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) {
 333                 rte_spinlock_unlock(&cvq->lock);
 334                 return -1;
 335         }
 336
 337         memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
 338                 sizeof(struct virtio_pmd_ctrl));
 339
 340         if (virtio_with_packed_queue(vq->hw))
 341                 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num);
 342         else
 343                 result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num);
 344
 345         rte_spinlock_unlock(&cvq->lock);
 346         return result->status;
 347 }
 348
 349 static int
 350 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 351 {
 352         struct virtio_hw *hw = dev->data->dev_private;
 353         struct virtio_pmd_ctrl ctrl;
 354         int dlen[1];
 355         int ret;
 356
 357         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 358         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 359         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 360
 361         dlen[0] = sizeof(uint16_t);
 362
 363         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 364         if (ret) {
 365                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 366                           "failed, this is too late now...");
 367                 return -EINVAL;
 368         }
 369
 370         return 0;
 371 }
 372
 373 static uint16_t
 374 virtio_get_nr_vq(struct virtio_hw *hw)
 375 {
 376         uint16_t nr_vq = hw->max_queue_pairs * 2;
 377
 378         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
 379                 nr_vq += 1;
 380
 381         return nr_vq;
 382 }
 383
 384 static void
 385 virtio_init_vring(struct virtqueue *vq)
 386 {
 387         int size = vq->vq_nentries;
 388         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 389
 390         PMD_INIT_FUNC_TRACE();
 391
 392         memset(ring_mem, 0, vq->vq_ring_size);
 393
 394         vq->vq_used_cons_idx = 0;
 395         vq->vq_desc_head_idx = 0;
 396         vq->vq_avail_idx = 0;
 397         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 398         vq->vq_free_cnt = vq->vq_nentries;
 399         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 400         if (virtio_with_packed_queue(vq->hw)) {
 401                 vring_init_packed(&vq->vq_packed.ring, ring_mem,
 402                                   VIRTIO_VRING_ALIGN, size);
 403                 vring_desc_init_packed(vq, size);
 404         } else {
 405                 struct vring *vr = &vq->vq_split.ring;
 406
 407                 vring_init_split(vr, ring_mem, VIRTIO_VRING_ALIGN, size);
 408                 vring_desc_init_split(vr->desc, size);
 409         }
 410         /*
 411          * Disable device(host) interrupting guest
 412          */
 413         virtqueue_disable_intr(vq);
 414 }
 415
 416 static int
 417 virtio_init_queue(struct rte_eth_dev *dev, uint16_t queue_idx)
 418 {
 419         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 420         char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
 421         const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
 422         unsigned int vq_size, size;
 423         struct virtio_hw *hw = dev->data->dev_private;
 424         struct virtnet_rx *rxvq = NULL;
 425         struct virtnet_tx *txvq = NULL;
 426         struct virtnet_ctl *cvq = NULL;
 427         struct virtqueue *vq;
 428         size_t sz_hdr_mz = 0;
 429         void *sw_ring = NULL;
 430         int queue_type = virtio_get_queue_type(hw, queue_idx);
 431         int ret;
 432         int numa_node = dev->device->numa_node;
 433         struct rte_mbuf *fake_mbuf = NULL;
 434
 435         PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d",
 436                         queue_idx, numa_node);
 437
 438         /*
 439          * Read the virtqueue size from the Queue Size field
 440          * Always power of 2 and if 0 virtqueue does not exist
 441          */
 442         vq_size = VIRTIO_OPS(hw)->get_queue_num(hw, queue_idx);
 443         PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
 444         if (vq_size == 0) {
 445                 PMD_INIT_LOG(ERR, "virtqueue does not exist");
 446                 return -EINVAL;
 447         }
 448
 449         if (!virtio_with_packed_queue(hw) && !rte_is_power_of_2(vq_size)) {
 450                 PMD_INIT_LOG(ERR, "split virtqueue size is not power of 2");
 451                 return -EINVAL;
 452         }
 453
 454         snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
 455                  dev->data->port_id, queue_idx);
 456
 457         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 458                                 vq_size * sizeof(struct vq_desc_extra),
 459                                 RTE_CACHE_LINE_SIZE);
 460         if (queue_type == VTNET_TQ) {
 461                 /*
 462                  * For each xmit packet, allocate a virtio_net_hdr
 463                  * and indirect ring elements
 464                  */
 465                 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
 466         } else if (queue_type == VTNET_CQ) {
 467                 /* Allocate a page for control vq command, data and status */
 468                 sz_hdr_mz = rte_mem_page_size();
 469         }
 470
 471         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
 472                                 numa_node);
 473         if (vq == NULL) {
 474                 PMD_INIT_LOG(ERR, "can not allocate vq");
 475                 return -ENOMEM;
 476         }
 477         hw->vqs[queue_idx] = vq;
 478
 479         vq->hw = hw;
 480         vq->vq_queue_index = queue_idx;
 481         vq->vq_nentries = vq_size;
 482         if (virtio_with_packed_queue(hw)) {
 483                 vq->vq_packed.used_wrap_counter = 1;
 484                 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
 485                 vq->vq_packed.event_flags_shadow = 0;
 486                 if (queue_type == VTNET_RQ)
 487                         vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
 488         }
 489
 490         /*
 491          * Reserve a memzone for vring elements
 492          */
 493         size = vring_size(hw, vq_size, VIRTIO_VRING_ALIGN);
 494         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN);
 495         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
 496                      size, vq->vq_ring_size);
 497
 498         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 499                         numa_node, RTE_MEMZONE_IOVA_CONTIG,
 500                         VIRTIO_VRING_ALIGN);
 501         if (mz == NULL) {
 502                 if (rte_errno == EEXIST)
 503                         mz = rte_memzone_lookup(vq_name);
 504                 if (mz == NULL) {
 505                         ret = -ENOMEM;
 506                         goto free_vq;
 507                 }
 508         }
 509
 510         memset(mz->addr, 0, mz->len);
 511
 512         if (hw->use_va)
 513                 vq->vq_ring_mem = (uintptr_t)mz->addr;
 514         else
 515                 vq->vq_ring_mem = mz->iova;
 516
 517         vq->vq_ring_virt_mem = mz->addr;
 518         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem);
 519         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem);
 520
 521         virtio_init_vring(vq);
 522
 523         if (sz_hdr_mz) {
 524                 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 525                          dev->data->port_id, queue_idx);
 526                 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
 527                                 numa_node, RTE_MEMZONE_IOVA_CONTIG,
 528                                 RTE_CACHE_LINE_SIZE);
 529                 if (hdr_mz == NULL) {
 530                         if (rte_errno == EEXIST)
 531                                 hdr_mz = rte_memzone_lookup(vq_hdr_name);
 532                         if (hdr_mz == NULL) {
 533                                 ret = -ENOMEM;
 534                                 goto free_mz;
 535                         }
 536                 }
 537         }
 538
 539         if (queue_type == VTNET_RQ) {
 540                 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
 541                                sizeof(vq->sw_ring[0]);
 542
 543                 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
 544                                 RTE_CACHE_LINE_SIZE, numa_node);
 545                 if (!sw_ring) {
 546                         PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
 547                         ret = -ENOMEM;
 548                         goto free_hdr_mz;
 549                 }
 550
 551                 fake_mbuf = rte_zmalloc_socket("sw_ring", sizeof(*fake_mbuf),
 552                                 RTE_CACHE_LINE_SIZE, numa_node);
 553                 if (!fake_mbuf) {
 554                         PMD_INIT_LOG(ERR, "can not allocate fake mbuf");
 555                         ret = -ENOMEM;
 556                         goto free_sw_ring;
 557                 }
 558
 559                 vq->sw_ring = sw_ring;
 560                 rxvq = &vq->rxq;
 561                 rxvq->port_id = dev->data->port_id;
 562                 rxvq->mz = mz;
 563                 rxvq->fake_mbuf = fake_mbuf;
 564         } else if (queue_type == VTNET_TQ) {
 565                 txvq = &vq->txq;
 566                 txvq->port_id = dev->data->port_id;
 567                 txvq->mz = mz;
 568                 txvq->virtio_net_hdr_mz = hdr_mz;
 569                 if (hw->use_va)
 570                         txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 571                 else
 572                         txvq->virtio_net_hdr_mem = hdr_mz->iova;
 573         } else if (queue_type == VTNET_CQ) {
 574                 cvq = &vq->cq;
 575                 cvq->mz = mz;
 576                 cvq->virtio_net_hdr_mz = hdr_mz;
 577                 if (hw->use_va)
 578                         cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 579                 else
 580                         cvq->virtio_net_hdr_mem = hdr_mz->iova;
 581                 memset(cvq->virtio_net_hdr_mz->addr, 0, rte_mem_page_size());
 582
 583                 hw->cvq = cvq;
 584         }
 585
 586         if (hw->use_va)
 587                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr);
 588         else
 589                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova);
 590
 591         if (queue_type == VTNET_TQ) {
 592                 struct virtio_tx_region *txr;
 593                 unsigned int i;
 594
 595                 txr = hdr_mz->addr;
 596                 memset(txr, 0, vq_size * sizeof(*txr));
 597                 for (i = 0; i < vq_size; i++) {
 598                         /* first indirect descriptor is always the tx header */
 599                         if (!virtio_with_packed_queue(hw)) {
 600                                 struct vring_desc *start_dp = txr[i].tx_indir;
 601                                 vring_desc_init_split(start_dp,
 602                                                       RTE_DIM(txr[i].tx_indir));
 603                                 start_dp->addr = txvq->virtio_net_hdr_mem
 604                                         + i * sizeof(*txr)
 605                                         + offsetof(struct virtio_tx_region,
 606                                                    tx_hdr);
 607                                 start_dp->len = hw->vtnet_hdr_size;
 608                                 start_dp->flags = VRING_DESC_F_NEXT;
 609                         } else {
 610                                 struct vring_packed_desc *start_dp =
 611                                         txr[i].tx_packed_indir;
 612                                 vring_desc_init_indirect_packed(start_dp,
 613                                       RTE_DIM(txr[i].tx_packed_indir));
 614                                 start_dp->addr = txvq->virtio_net_hdr_mem
 615                                         + i * sizeof(*txr)
 616                                         + offsetof(struct virtio_tx_region,
 617                                                    tx_hdr);
 618                                 start_dp->len = hw->vtnet_hdr_size;
 619                         }
 620                 }
 621         }
 622
 623         if (VIRTIO_OPS(hw)->setup_queue(hw, vq) < 0) {
 624                 PMD_INIT_LOG(ERR, "setup_queue failed");
 625                 ret = -EINVAL;
 626                 goto clean_vq;
 627         }
 628
 629         return 0;
 630
 631 clean_vq:
 632         hw->cvq = NULL;
 633         rte_free(fake_mbuf);
 634 free_sw_ring:
 635         rte_free(sw_ring);
 636 free_hdr_mz:
 637         rte_memzone_free(hdr_mz);
 638 free_mz:
 639         rte_memzone_free(mz);
 640 free_vq:
 641         rte_free(vq);
 642         hw->vqs[queue_idx] = NULL;
 643
 644         return ret;
 645 }
 646
 647 static void
 648 virtio_free_queues(struct virtio_hw *hw)
 649 {
 650         uint16_t nr_vq = virtio_get_nr_vq(hw);
 651         struct virtqueue *vq;
 652         int queue_type;
 653         uint16_t i;
 654
 655         if (hw->vqs == NULL)
 656                 return;
 657
 658         for (i = 0; i < nr_vq; i++) {
 659                 vq = hw->vqs[i];
 660                 if (!vq)
 661                         continue;
 662
 663                 queue_type = virtio_get_queue_type(hw, i);
 664                 if (queue_type == VTNET_RQ) {
 665                         rte_free(vq->rxq.fake_mbuf);
 666                         rte_free(vq->sw_ring);
 667                         rte_memzone_free(vq->rxq.mz);
 668                 } else if (queue_type == VTNET_TQ) {
 669                         rte_memzone_free(vq->txq.mz);
 670                         rte_memzone_free(vq->txq.virtio_net_hdr_mz);
 671                 } else {
 672                         rte_memzone_free(vq->cq.mz);
 673                         rte_memzone_free(vq->cq.virtio_net_hdr_mz);
 674                 }
 675
 676                 rte_free(vq);
 677                 hw->vqs[i] = NULL;
 678         }
 679
 680         rte_free(hw->vqs);
 681         hw->vqs = NULL;
 682 }
 683
 684 static int
 685 virtio_alloc_queues(struct rte_eth_dev *dev)
 686 {
 687         struct virtio_hw *hw = dev->data->dev_private;
 688         uint16_t nr_vq = virtio_get_nr_vq(hw);
 689         uint16_t i;
 690         int ret;
 691
 692         hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0);
 693         if (!hw->vqs) {
 694                 PMD_INIT_LOG(ERR, "failed to allocate vqs");
 695                 return -ENOMEM;
 696         }
 697
 698         for (i = 0; i < nr_vq; i++) {
 699                 ret = virtio_init_queue(dev, i);
 700                 if (ret < 0) {
 701                         virtio_free_queues(hw);
 702                         return ret;
 703                 }
 704         }
 705
 706         return 0;
 707 }
 708
 709 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev);
 710
 711 int
 712 virtio_dev_close(struct rte_eth_dev *dev)
 713 {
 714         struct virtio_hw *hw = dev->data->dev_private;
 715         struct rte_eth_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
 716
 717         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 718         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 719                 return 0;
 720
 721         if (!hw->opened)
 722                 return 0;
 723         hw->opened = 0;
 724
 725         /* reset the NIC */
 726         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 727                 VIRTIO_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR);
 728         if (intr_conf->rxq)
 729                 virtio_queues_unbind_intr(dev);
 730
 731         if (intr_conf->lsc || intr_conf->rxq) {
 732                 virtio_intr_disable(dev);
 733                 rte_intr_efd_disable(dev->intr_handle);
 734                 rte_intr_vec_list_free(dev->intr_handle);
 735         }
 736
 737         virtio_reset(hw);
 738         virtio_dev_free_mbufs(dev);
 739         virtio_free_queues(hw);
 740
 741         return VIRTIO_OPS(hw)->dev_close(hw);
 742 }
 743
 744 static int
 745 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
 746 {
 747         struct virtio_hw *hw = dev->data->dev_private;
 748         struct virtio_pmd_ctrl ctrl;
 749         int dlen[1];
 750         int ret;
 751
 752         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 753                 PMD_INIT_LOG(INFO, "host does not support rx control");
 754                 return -ENOTSUP;
 755         }
 756
 757         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 758         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 759         ctrl.data[0] = 1;
 760         dlen[0] = 1;
 761
 762         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 763         if (ret) {
 764                 PMD_INIT_LOG(ERR, "Failed to enable promisc");
 765                 return -EAGAIN;
 766         }
 767
 768         return 0;
 769 }
 770
 771 static int
 772 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
 773 {
 774         struct virtio_hw *hw = dev->data->dev_private;
 775         struct virtio_pmd_ctrl ctrl;
 776         int dlen[1];
 777         int ret;
 778
 779         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 780                 PMD_INIT_LOG(INFO, "host does not support rx control");
 781                 return -ENOTSUP;
 782         }
 783
 784         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 785         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 786         ctrl.data[0] = 0;
 787         dlen[0] = 1;
 788
 789         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 790         if (ret) {
 791                 PMD_INIT_LOG(ERR, "Failed to disable promisc");
 792                 return -EAGAIN;
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
 800 {
 801         struct virtio_hw *hw = dev->data->dev_private;
 802         struct virtio_pmd_ctrl ctrl;
 803         int dlen[1];
 804         int ret;
 805
 806         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 807                 PMD_INIT_LOG(INFO, "host does not support rx control");
 808                 return -ENOTSUP;
 809         }
 810
 811         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 812         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 813         ctrl.data[0] = 1;
 814         dlen[0] = 1;
 815
 816         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 817         if (ret) {
 818                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
 819                 return -EAGAIN;
 820         }
 821
 822         return 0;
 823 }
 824
 825 static int
 826 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
 827 {
 828         struct virtio_hw *hw = dev->data->dev_private;
 829         struct virtio_pmd_ctrl ctrl;
 830         int dlen[1];
 831         int ret;
 832
 833         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 834                 PMD_INIT_LOG(INFO, "host does not support rx control");
 835                 return -ENOTSUP;
 836         }
 837
 838         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 839         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 840         ctrl.data[0] = 0;
 841         dlen[0] = 1;
 842
 843         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 844         if (ret) {
 845                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
 846                 return -EAGAIN;
 847         }
 848
 849         return 0;
 850 }
 851
 852 uint16_t
 853 virtio_rx_mem_pool_buf_size(struct rte_mempool *mp)
 854 {
 855         return rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
 856 }
 857
 858 bool
 859 virtio_rx_check_scatter(uint16_t max_rx_pkt_len, uint16_t rx_buf_size,
 860                         bool rx_scatter_enabled, const char **error)
 861 {
 862         if (!rx_scatter_enabled && max_rx_pkt_len > rx_buf_size) {
 863                 *error = "Rx scatter is disabled and RxQ mbuf pool object size is too small";
 864                 return false;
 865         }
 866
 867         return true;
 868 }
 869
 870 static bool
 871 virtio_check_scatter_on_all_rx_queues(struct rte_eth_dev *dev,
 872                                       uint16_t frame_size)
 873 {
 874         struct virtio_hw *hw = dev->data->dev_private;
 875         struct virtnet_rx *rxvq;
 876         struct virtqueue *vq;
 877         unsigned int qidx;
 878         uint16_t buf_size;
 879         const char *error;
 880
 881         if (hw->vqs == NULL)
 882                 return true;
 883
 884         for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) {
 885                 vq = hw->vqs[2 * qidx + VTNET_SQ_RQ_QUEUE_IDX];
 886                 if (vq == NULL)
 887                         continue;
 888
 889                 rxvq = &vq->rxq;
 890                 if (rxvq->mpool == NULL)
 891                         continue;
 892                 buf_size = virtio_rx_mem_pool_buf_size(rxvq->mpool);
 893
 894                 if (!virtio_rx_check_scatter(frame_size, buf_size,
 895                                              hw->rx_ol_scatter, &error)) {
 896                         PMD_INIT_LOG(ERR, "MTU check for RxQ %u failed: %s",
 897                                      qidx, error);
 898                         return false;
 899                 }
 900         }
 901
 902         return true;
 903 }
 904
 905 #define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
 906 static int
 907 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 908 {
 909         struct virtio_hw *hw = dev->data->dev_private;
 910         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
 911                                  hw->vtnet_hdr_size;
 912         uint32_t frame_size = mtu + ether_hdr_len;
 913         uint32_t max_frame_size = hw->max_mtu + ether_hdr_len;
 914
 915         max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN);
 916
 917         if (mtu < RTE_ETHER_MIN_MTU || frame_size > max_frame_size) {
 918                 PMD_INIT_LOG(ERR, "MTU should be between %d and %d",
 919                         RTE_ETHER_MIN_MTU, max_frame_size - ether_hdr_len);
 920                 return -EINVAL;
 921         }
 922
 923         if (!virtio_check_scatter_on_all_rx_queues(dev, frame_size)) {
 924                 PMD_INIT_LOG(ERR, "MTU vs Rx scatter and Rx buffers check failed");
 925                 return -EINVAL;
 926         }
 927
 928         hw->max_rx_pkt_len = frame_size;
 929
 930         return 0;
 931 }
 932
 933 static int
 934 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 935 {
 936         struct virtio_hw *hw = dev->data->dev_private;
 937         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 938         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
 939
 940         virtqueue_enable_intr(vq);
 941         virtio_mb(hw->weak_barriers);
 942         return 0;
 943 }
 944
 945 static int
 946 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 947 {
 948         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 949         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
 950
 951         virtqueue_disable_intr(vq);
 952         return 0;
 953 }
 954
 955 /*
 956  * dev_ops for virtio, bare necessities for basic operation
 957  */
 958 static const struct eth_dev_ops virtio_eth_dev_ops = {
 959         .dev_configure           = virtio_dev_configure,
 960         .dev_start               = virtio_dev_start,
 961         .dev_stop                = virtio_dev_stop,
 962         .dev_close               = virtio_dev_close,
 963         .promiscuous_enable      = virtio_dev_promiscuous_enable,
 964         .promiscuous_disable     = virtio_dev_promiscuous_disable,
 965         .allmulticast_enable     = virtio_dev_allmulticast_enable,
 966         .allmulticast_disable    = virtio_dev_allmulticast_disable,
 967         .mtu_set                 = virtio_mtu_set,
 968         .dev_infos_get           = virtio_dev_info_get,
 969         .stats_get               = virtio_dev_stats_get,
 970         .xstats_get              = virtio_dev_xstats_get,
 971         .xstats_get_names        = virtio_dev_xstats_get_names,
 972         .stats_reset             = virtio_dev_stats_reset,
 973         .xstats_reset            = virtio_dev_stats_reset,
 974         .link_update             = virtio_dev_link_update,
 975         .vlan_offload_set        = virtio_dev_vlan_offload_set,
 976         .rx_queue_setup          = virtio_dev_rx_queue_setup,
 977         .rx_queue_intr_enable    = virtio_dev_rx_queue_intr_enable,
 978         .rx_queue_intr_disable   = virtio_dev_rx_queue_intr_disable,
 979         .tx_queue_setup          = virtio_dev_tx_queue_setup,
 980         /* collect stats per queue */
 981         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 982         .vlan_filter_set         = virtio_vlan_filter_set,
 983         .mac_addr_add            = virtio_mac_addr_add,
 984         .mac_addr_remove         = virtio_mac_addr_remove,
 985         .mac_addr_set            = virtio_mac_addr_set,
 986 };
 987
 988 /*
 989  * dev_ops for virtio-user in secondary processes, as we just have
 990  * some limited supports currently.
 991  */
 992 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops = {
 993         .dev_infos_get           = virtio_dev_info_get,
 994         .stats_get               = virtio_dev_stats_get,
 995         .xstats_get              = virtio_dev_xstats_get,
 996         .xstats_get_names        = virtio_dev_xstats_get_names,
 997         .stats_reset             = virtio_dev_stats_reset,
 998         .xstats_reset            = virtio_dev_stats_reset,
 999         /* collect stats per queue */
1000         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
1001 };
1002
1003 static void
1004 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1005 {
1006         unsigned i;
1007
1008         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1009                 const struct virtnet_tx *txvq = dev->data->tx_queues[i];
1010                 if (txvq == NULL)
1011                         continue;
1012
1013                 stats->opackets += txvq->stats.packets;
1014                 stats->obytes += txvq->stats.bytes;
1015
1016                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1017                         stats->q_opackets[i] = txvq->stats.packets;
1018                         stats->q_obytes[i] = txvq->stats.bytes;
1019                 }
1020         }
1021
1022         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1023                 const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1024                 if (rxvq == NULL)
1025                         continue;
1026
1027                 stats->ipackets += rxvq->stats.packets;
1028                 stats->ibytes += rxvq->stats.bytes;
1029                 stats->ierrors += rxvq->stats.errors;
1030
1031                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1032                         stats->q_ipackets[i] = rxvq->stats.packets;
1033                         stats->q_ibytes[i] = rxvq->stats.bytes;
1034                 }
1035         }
1036
1037         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1038 }
1039
1040 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
1041                                        struct rte_eth_xstat_name *xstats_names,
1042                                        __rte_unused unsigned limit)
1043 {
1044         unsigned i;
1045         unsigned count = 0;
1046         unsigned t;
1047
1048         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1049                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1050
1051         if (xstats_names != NULL) {
1052                 /* Note: limit checked in rte_eth_xstats_names() */
1053
1054                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1055                         struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1056                         if (rxvq == NULL)
1057                                 continue;
1058                         for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1059                                 snprintf(xstats_names[count].name,
1060                                         sizeof(xstats_names[count].name),
1061                                         "rx_q%u_%s", i,
1062                                         rte_virtio_rxq_stat_strings[t].name);
1063                                 count++;
1064                         }
1065                 }
1066
1067                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1068                         struct virtnet_tx *txvq = dev->data->tx_queues[i];
1069                         if (txvq == NULL)
1070                                 continue;
1071                         for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1072                                 snprintf(xstats_names[count].name,
1073                                         sizeof(xstats_names[count].name),
1074                                         "tx_q%u_%s", i,
1075                                         rte_virtio_txq_stat_strings[t].name);
1076                                 count++;
1077                         }
1078                 }
1079                 return count;
1080         }
1081         return nstats;
1082 }
1083
1084 static int
1085 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1086                       unsigned n)
1087 {
1088         unsigned i;
1089         unsigned count = 0;
1090
1091         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1092                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1093
1094         if (n < nstats)
1095                 return nstats;
1096
1097         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1098                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1099
1100                 if (rxvq == NULL)
1101                         continue;
1102
1103                 unsigned t;
1104
1105                 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1106                         xstats[count].value = *(uint64_t *)(((char *)rxvq) +
1107                                 rte_virtio_rxq_stat_strings[t].offset);
1108                         xstats[count].id = count;
1109                         count++;
1110                 }
1111         }
1112
1113         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1114                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1115
1116                 if (txvq == NULL)
1117                         continue;
1118
1119                 unsigned t;
1120
1121                 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1122                         xstats[count].value = *(uint64_t *)(((char *)txvq) +
1123                                 rte_virtio_txq_stat_strings[t].offset);
1124                         xstats[count].id = count;
1125                         count++;
1126                 }
1127         }
1128
1129         return count;
1130 }
1131
1132 static int
1133 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1134 {
1135         virtio_update_stats(dev, stats);
1136
1137         return 0;
1138 }
1139
1140 static int
1141 virtio_dev_stats_reset(struct rte_eth_dev *dev)
1142 {
1143         unsigned int i;
1144
1145         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1146                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1147                 if (txvq == NULL)
1148                         continue;
1149
1150                 txvq->stats.packets = 0;
1151                 txvq->stats.bytes = 0;
1152                 txvq->stats.multicast = 0;
1153                 txvq->stats.broadcast = 0;
1154                 memset(txvq->stats.size_bins, 0,
1155                        sizeof(txvq->stats.size_bins[0]) * 8);
1156         }
1157
1158         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1159                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1160                 if (rxvq == NULL)
1161                         continue;
1162
1163                 rxvq->stats.packets = 0;
1164                 rxvq->stats.bytes = 0;
1165                 rxvq->stats.errors = 0;
1166                 rxvq->stats.multicast = 0;
1167                 rxvq->stats.broadcast = 0;
1168                 memset(rxvq->stats.size_bins, 0,
1169                        sizeof(rxvq->stats.size_bins[0]) * 8);
1170         }
1171
1172         return 0;
1173 }
1174
1175 static void
1176 virtio_set_hwaddr(struct virtio_hw *hw)
1177 {
1178         virtio_write_dev_config(hw,
1179                         offsetof(struct virtio_net_config, mac),
1180                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1181 }
1182
1183 static void
1184 virtio_get_hwaddr(struct virtio_hw *hw)
1185 {
1186         if (virtio_with_feature(hw, VIRTIO_NET_F_MAC)) {
1187                 virtio_read_dev_config(hw,
1188                         offsetof(struct virtio_net_config, mac),
1189                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1190         } else {
1191                 rte_eth_random_addr(&hw->mac_addr[0]);
1192                 virtio_set_hwaddr(hw);
1193         }
1194 }
1195
1196 static int
1197 virtio_mac_table_set(struct virtio_hw *hw,
1198                      const struct virtio_net_ctrl_mac *uc,
1199                      const struct virtio_net_ctrl_mac *mc)
1200 {
1201         struct virtio_pmd_ctrl ctrl;
1202         int err, len[2];
1203
1204         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1205                 PMD_DRV_LOG(INFO, "host does not support mac table");
1206                 return -1;
1207         }
1208
1209         ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1210         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1211
1212         len[0] = uc->entries * RTE_ETHER_ADDR_LEN + sizeof(uc->entries);
1213         memcpy(ctrl.data, uc, len[0]);
1214
1215         len[1] = mc->entries * RTE_ETHER_ADDR_LEN + sizeof(mc->entries);
1216         memcpy(ctrl.data + len[0], mc, len[1]);
1217
1218         err = virtio_send_command(hw->cvq, &ctrl, len, 2);
1219         if (err != 0)
1220                 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
1221         return err;
1222 }
1223
1224 static int
1225 virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1226                     uint32_t index, uint32_t vmdq __rte_unused)
1227 {
1228         struct virtio_hw *hw = dev->data->dev_private;
1229         const struct rte_ether_addr *addrs = dev->data->mac_addrs;
1230         unsigned int i;
1231         struct virtio_net_ctrl_mac *uc, *mc;
1232
1233         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1234                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1235                 return -EINVAL;
1236         }
1237
1238         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1239                 sizeof(uc->entries));
1240         uc->entries = 0;
1241         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1242                 sizeof(mc->entries));
1243         mc->entries = 0;
1244
1245         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1246                 const struct rte_ether_addr *addr
1247                         = (i == index) ? mac_addr : addrs + i;
1248                 struct virtio_net_ctrl_mac *tbl
1249                         = rte_is_multicast_ether_addr(addr) ? mc : uc;
1250
1251                 memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN);
1252         }
1253
1254         return virtio_mac_table_set(hw, uc, mc);
1255 }
1256
1257 static void
1258 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1259 {
1260         struct virtio_hw *hw = dev->data->dev_private;
1261         struct rte_ether_addr *addrs = dev->data->mac_addrs;
1262         struct virtio_net_ctrl_mac *uc, *mc;
1263         unsigned int i;
1264
1265         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1266                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1267                 return;
1268         }
1269
1270         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1271                 sizeof(uc->entries));
1272         uc->entries = 0;
1273         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1274                 sizeof(mc->entries));
1275         mc->entries = 0;
1276
1277         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1278                 struct virtio_net_ctrl_mac *tbl;
1279
1280                 if (i == index || rte_is_zero_ether_addr(addrs + i))
1281                         continue;
1282
1283                 tbl = rte_is_multicast_ether_addr(addrs + i) ? mc : uc;
1284                 memcpy(&tbl->macs[tbl->entries++], addrs + i,
1285                         RTE_ETHER_ADDR_LEN);
1286         }
1287
1288         virtio_mac_table_set(hw, uc, mc);
1289 }
1290
1291 static int
1292 virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1293 {
1294         struct virtio_hw *hw = dev->data->dev_private;
1295
1296         memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1297
1298         /* Use atomic update if available */
1299         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1300                 struct virtio_pmd_ctrl ctrl;
1301                 int len = RTE_ETHER_ADDR_LEN;
1302
1303                 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1304                 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1305
1306                 memcpy(ctrl.data, mac_addr, RTE_ETHER_ADDR_LEN);
1307                 return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1308         }
1309
1310         if (!virtio_with_feature(hw, VIRTIO_NET_F_MAC))
1311                 return -ENOTSUP;
1312
1313         virtio_set_hwaddr(hw);
1314         return 0;
1315 }
1316
1317 static int
1318 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1319 {
1320         struct virtio_hw *hw = dev->data->dev_private;
1321         struct virtio_pmd_ctrl ctrl;
1322         int len;
1323
1324         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1325                 return -ENOTSUP;
1326
1327         ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1328         ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1329         memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1330         len = sizeof(vlan_id);
1331
1332         return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1333 }
1334
1335 static int
1336 virtio_intr_unmask(struct rte_eth_dev *dev)
1337 {
1338         struct virtio_hw *hw = dev->data->dev_private;
1339
1340         if (rte_intr_ack(dev->intr_handle) < 0)
1341                 return -1;
1342
1343         if (VIRTIO_OPS(hw)->intr_detect)
1344                 VIRTIO_OPS(hw)->intr_detect(hw);
1345
1346         return 0;
1347 }
1348
1349 static int
1350 virtio_intr_enable(struct rte_eth_dev *dev)
1351 {
1352         struct virtio_hw *hw = dev->data->dev_private;
1353
1354         if (rte_intr_enable(dev->intr_handle) < 0)
1355                 return -1;
1356
1357         if (VIRTIO_OPS(hw)->intr_detect)
1358                 VIRTIO_OPS(hw)->intr_detect(hw);
1359
1360         return 0;
1361 }
1362
1363 static int
1364 virtio_intr_disable(struct rte_eth_dev *dev)
1365 {
1366         struct virtio_hw *hw = dev->data->dev_private;
1367
1368         if (rte_intr_disable(dev->intr_handle) < 0)
1369                 return -1;
1370
1371         if (VIRTIO_OPS(hw)->intr_detect)
1372                 VIRTIO_OPS(hw)->intr_detect(hw);
1373
1374         return 0;
1375 }
1376
1377 static int
1378 virtio_ethdev_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1379 {
1380         uint64_t host_features;
1381
1382         /* Prepare guest_features: feature that driver wants to support */
1383         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1384                 req_features);
1385
1386         /* Read device(host) feature bits */
1387         host_features = VIRTIO_OPS(hw)->get_features(hw);
1388         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1389                 host_features);
1390
1391         /* If supported, ensure MTU value is valid before acknowledging it. */
1392         if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) {
1393                 struct virtio_net_config config;
1394
1395                 virtio_read_dev_config(hw,
1396                         offsetof(struct virtio_net_config, mtu),
1397                         &config.mtu, sizeof(config.mtu));
1398
1399                 if (config.mtu < RTE_ETHER_MIN_MTU)
1400                         req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
1401         }
1402
1403         /*
1404          * Negotiate features: Subset of device feature bits are written back
1405          * guest feature bits.
1406          */
1407         hw->guest_features = req_features;
1408         hw->guest_features = virtio_negotiate_features(hw, host_features);
1409         PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1410                 hw->guest_features);
1411
1412         if (VIRTIO_OPS(hw)->features_ok(hw) < 0)
1413                 return -1;
1414
1415         if (virtio_with_feature(hw, VIRTIO_F_VERSION_1)) {
1416                 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1417
1418                 if (!(virtio_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1419                         PMD_INIT_LOG(ERR, "Failed to set FEATURES_OK status!");
1420                         return -1;
1421                 }
1422         }
1423
1424         hw->req_guest_features = req_features;
1425
1426         return 0;
1427 }
1428
1429 int
1430 virtio_dev_pause(struct rte_eth_dev *dev)
1431 {
1432         struct virtio_hw *hw = dev->data->dev_private;
1433
1434         rte_spinlock_lock(&hw->state_lock);
1435
1436         if (hw->started == 0) {
1437                 /* Device is just stopped. */
1438                 rte_spinlock_unlock(&hw->state_lock);
1439                 return -1;
1440         }
1441         hw->started = 0;
1442         /*
1443          * Prevent the worker threads from touching queues to avoid contention,
1444          * 1 ms should be enough for the ongoing Tx function to finish.
1445          */
1446         rte_delay_ms(1);
1447         return 0;
1448 }
1449
1450 /*
1451  * Recover hw state to let the worker threads continue.
1452  */
1453 void
1454 virtio_dev_resume(struct rte_eth_dev *dev)
1455 {
1456         struct virtio_hw *hw = dev->data->dev_private;
1457
1458         hw->started = 1;
1459         rte_spinlock_unlock(&hw->state_lock);
1460 }
1461
1462 /*
1463  * Should be called only after device is paused.
1464  */
1465 int
1466 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts,
1467                 int nb_pkts)
1468 {
1469         struct virtio_hw *hw = dev->data->dev_private;
1470         struct virtnet_tx *txvq = dev->data->tx_queues[0];
1471         int ret;
1472
1473         hw->inject_pkts = tx_pkts;
1474         ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts);
1475         hw->inject_pkts = NULL;
1476
1477         return ret;
1478 }
1479
1480 static void
1481 virtio_notify_peers(struct rte_eth_dev *dev)
1482 {
1483         struct virtio_hw *hw = dev->data->dev_private;
1484         struct virtnet_rx *rxvq;
1485         struct rte_mbuf *rarp_mbuf;
1486
1487         if (!dev->data->rx_queues)
1488                 return;
1489
1490         rxvq = dev->data->rx_queues[0];
1491         if (!rxvq)
1492                 return;
1493
1494         rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool,
1495                         (struct rte_ether_addr *)hw->mac_addr);
1496         if (rarp_mbuf == NULL) {
1497                 PMD_DRV_LOG(ERR, "failed to make RARP packet.");
1498                 return;
1499         }
1500
1501         /* If virtio port just stopped, no need to send RARP */
1502         if (virtio_dev_pause(dev) < 0) {
1503                 rte_pktmbuf_free(rarp_mbuf);
1504                 return;
1505         }
1506
1507         virtio_inject_pkts(dev, &rarp_mbuf, 1);
1508         virtio_dev_resume(dev);
1509 }
1510
1511 static void
1512 virtio_ack_link_announce(struct rte_eth_dev *dev)
1513 {
1514         struct virtio_hw *hw = dev->data->dev_private;
1515         struct virtio_pmd_ctrl ctrl;
1516
1517         ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE;
1518         ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK;
1519
1520         virtio_send_command(hw->cvq, &ctrl, NULL, 0);
1521 }
1522
1523 /*
1524  * Process virtio config changed interrupt. Call the callback
1525  * if link state changed, generate gratuitous RARP packet if
1526  * the status indicates an ANNOUNCE.
1527  */
1528 void
1529 virtio_interrupt_handler(void *param)
1530 {
1531         struct rte_eth_dev *dev = param;
1532         struct virtio_hw *hw = dev->data->dev_private;
1533         uint8_t isr;
1534         uint16_t status;
1535
1536         /* Read interrupt status which clears interrupt */
1537         isr = virtio_get_isr(hw);
1538         PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1539
1540         if (virtio_intr_unmask(dev) < 0)
1541                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1542
1543         if (isr & VIRTIO_ISR_CONFIG) {
1544                 if (virtio_dev_link_update(dev, 0) == 0)
1545                         rte_eth_dev_callback_process(dev,
1546                                                      RTE_ETH_EVENT_INTR_LSC,
1547                                                      NULL);
1548
1549                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1550                         virtio_read_dev_config(hw,
1551                                 offsetof(struct virtio_net_config, status),
1552                                 &status, sizeof(status));
1553                         if (status & VIRTIO_NET_S_ANNOUNCE) {
1554                                 virtio_notify_peers(dev);
1555                                 if (hw->cvq)
1556                                         virtio_ack_link_announce(dev);
1557                         }
1558                 }
1559         }
1560 }
1561
1562 /* set rx and tx handlers according to what is supported */
1563 static void
1564 set_rxtx_funcs(struct rte_eth_dev *eth_dev)
1565 {
1566         struct virtio_hw *hw = eth_dev->data->dev_private;
1567
1568         eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare;
1569         if (virtio_with_packed_queue(hw)) {
1570                 PMD_INIT_LOG(INFO,
1571                         "virtio: using packed ring %s Tx path on port %u",
1572                         hw->use_vec_tx ? "vectorized" : "standard",
1573                         eth_dev->data->port_id);
1574                 if (hw->use_vec_tx)
1575                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
1576                 else
1577                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
1578         } else {
1579                 if (hw->use_inorder_tx) {
1580                         PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u",
1581                                 eth_dev->data->port_id);
1582                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder;
1583                 } else {
1584                         PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u",
1585                                 eth_dev->data->port_id);
1586                         eth_dev->tx_pkt_burst = virtio_xmit_pkts;
1587                 }
1588         }
1589
1590         if (virtio_with_packed_queue(hw)) {
1591                 if (hw->use_vec_rx) {
1592                         PMD_INIT_LOG(INFO,
1593                                 "virtio: using packed ring vectorized Rx path on port %u",
1594                                 eth_dev->data->port_id);
1595                         eth_dev->rx_pkt_burst =
1596                                 &virtio_recv_pkts_packed_vec;
1597                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1598                         PMD_INIT_LOG(INFO,
1599                                 "virtio: using packed ring mergeable buffer Rx path on port %u",
1600                                 eth_dev->data->port_id);
1601                         eth_dev->rx_pkt_burst =
1602                                 &virtio_recv_mergeable_pkts_packed;
1603                 } else {
1604                         PMD_INIT_LOG(INFO,
1605                                 "virtio: using packed ring standard Rx path on port %u",
1606                                 eth_dev->data->port_id);
1607                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
1608                 }
1609         } else {
1610                 if (hw->use_vec_rx) {
1611                         PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on port %u",
1612                                 eth_dev->data->port_id);
1613                         eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
1614                 } else if (hw->use_inorder_rx) {
1615                         PMD_INIT_LOG(INFO,
1616                                 "virtio: using inorder Rx path on port %u",
1617                                 eth_dev->data->port_id);
1618                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder;
1619                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1620                         PMD_INIT_LOG(INFO,
1621                                 "virtio: using mergeable buffer Rx path on port %u",
1622                                 eth_dev->data->port_id);
1623                         eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1624                 } else {
1625                         PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
1626                                 eth_dev->data->port_id);
1627                         eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1628                 }
1629         }
1630
1631 }
1632
1633 /* Only support 1:1 queue/interrupt mapping so far.
1634  * TODO: support n:1 queue/interrupt mapping when there are limited number of
1635  * interrupt vectors (<N+1).
1636  */
1637 static int
1638 virtio_queues_bind_intr(struct rte_eth_dev *dev)
1639 {
1640         uint32_t i;
1641         struct virtio_hw *hw = dev->data->dev_private;
1642
1643         PMD_INIT_LOG(INFO, "queue/interrupt binding");
1644         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
1645                 if (rte_intr_vec_list_index_set(dev->intr_handle, i,
1646                                                        i + 1))
1647                         return -rte_errno;
1648                 if (VIRTIO_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) ==
1649                                                  VIRTIO_MSI_NO_VECTOR) {
1650                         PMD_DRV_LOG(ERR, "failed to set queue vector");
1651                         return -EBUSY;
1652                 }
1653         }
1654
1655         return 0;
1656 }
1657
1658 static void
1659 virtio_queues_unbind_intr(struct rte_eth_dev *dev)
1660 {
1661         uint32_t i;
1662         struct virtio_hw *hw = dev->data->dev_private;
1663
1664         PMD_INIT_LOG(INFO, "queue/interrupt unbinding");
1665         for (i = 0; i < dev->data->nb_rx_queues; ++i)
1666                 VIRTIO_OPS(hw)->set_queue_irq(hw,
1667                                              hw->vqs[i * VTNET_CQ],
1668                                              VIRTIO_MSI_NO_VECTOR);
1669 }
1670
1671 static int
1672 virtio_configure_intr(struct rte_eth_dev *dev)
1673 {
1674         struct virtio_hw *hw = dev->data->dev_private;
1675
1676         if (!rte_intr_cap_multiple(dev->intr_handle)) {
1677                 PMD_INIT_LOG(ERR, "Multiple intr vector not supported");
1678                 return -ENOTSUP;
1679         }
1680
1681         if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) {
1682                 PMD_INIT_LOG(ERR, "Fail to create eventfd");
1683                 return -1;
1684         }
1685
1686         if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec",
1687                                     hw->max_queue_pairs)) {
1688                 PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors",
1689                              hw->max_queue_pairs);
1690                 return -ENOMEM;
1691         }
1692
1693         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1694                 /* Re-register callback to update max_intr */
1695                 rte_intr_callback_unregister(dev->intr_handle,
1696                                              virtio_interrupt_handler,
1697                                              dev);
1698                 rte_intr_callback_register(dev->intr_handle,
1699                                            virtio_interrupt_handler,
1700                                            dev);
1701         }
1702
1703         /* DO NOT try to remove this! This function will enable msix, or QEMU
1704          * will encounter SIGSEGV when DRIVER_OK is sent.
1705          * And for legacy devices, this should be done before queue/vec binding
1706          * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1707          * (22) will be ignored.
1708          */
1709         if (virtio_intr_enable(dev) < 0) {
1710                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1711                 return -1;
1712         }
1713
1714         if (virtio_queues_bind_intr(dev) < 0) {
1715                 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt");
1716                 return -1;
1717         }
1718
1719         return 0;
1720 }
1721 #define DUPLEX_UNKNOWN   0xff
1722 /* reset device and renegotiate features if needed */
1723 static int
1724 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
1725 {
1726         struct virtio_hw *hw = eth_dev->data->dev_private;
1727         struct virtio_net_config *config;
1728         struct virtio_net_config local_config;
1729         int ret;
1730
1731         /* Reset the device although not necessary at startup */
1732         virtio_reset(hw);
1733
1734         if (hw->vqs) {
1735                 virtio_dev_free_mbufs(eth_dev);
1736                 virtio_free_queues(hw);
1737         }
1738
1739         /* Tell the host we've noticed this device. */
1740         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1741
1742         /* Tell the host we've known how to drive the device. */
1743         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1744         if (virtio_ethdev_negotiate_features(hw, req_features) < 0)
1745                 return -1;
1746
1747         hw->weak_barriers = !virtio_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
1748
1749         /* If host does not support both status and MSI-X then disable LSC */
1750         if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS) && hw->intr_lsc)
1751                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
1752         else
1753                 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
1754
1755         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
1756
1757         /* Setting up rx_header size for the device */
1758         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
1759             virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
1760             virtio_with_packed_queue(hw))
1761                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1762         else
1763                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
1764
1765         /* Copy the permanent MAC address to: virtio_hw */
1766         virtio_get_hwaddr(hw);
1767         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
1768                         &eth_dev->data->mac_addrs[0]);
1769         PMD_INIT_LOG(DEBUG,
1770                      "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
1771                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1772                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1773
1774         if (hw->speed == RTE_ETH_SPEED_NUM_UNKNOWN) {
1775                 if (virtio_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) {
1776                         config = &local_config;
1777                         virtio_read_dev_config(hw,
1778                                 offsetof(struct virtio_net_config, speed),
1779                                 &config->speed, sizeof(config->speed));
1780                         virtio_read_dev_config(hw,
1781                                 offsetof(struct virtio_net_config, duplex),
1782                                 &config->duplex, sizeof(config->duplex));
1783                         hw->speed = config->speed;
1784                         hw->duplex = config->duplex;
1785                 }
1786         }
1787         if (hw->duplex == DUPLEX_UNKNOWN)
1788                 hw->duplex = RTE_ETH_LINK_FULL_DUPLEX;
1789         PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d",
1790                 hw->speed, hw->duplex);
1791         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1792                 config = &local_config;
1793
1794                 virtio_read_dev_config(hw,
1795                         offsetof(struct virtio_net_config, mac),
1796                         &config->mac, sizeof(config->mac));
1797
1798                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1799                         virtio_read_dev_config(hw,
1800                                 offsetof(struct virtio_net_config, status),
1801                                 &config->status, sizeof(config->status));
1802                 } else {
1803                         PMD_INIT_LOG(DEBUG,
1804                                      "VIRTIO_NET_F_STATUS is not supported");
1805                         config->status = 0;
1806                 }
1807
1808                 if (virtio_with_feature(hw, VIRTIO_NET_F_MQ)) {
1809                         virtio_read_dev_config(hw,
1810                                 offsetof(struct virtio_net_config, max_virtqueue_pairs),
1811                                 &config->max_virtqueue_pairs,
1812                                 sizeof(config->max_virtqueue_pairs));
1813                 } else {
1814                         PMD_INIT_LOG(DEBUG,
1815                                      "VIRTIO_NET_F_MQ is not supported");
1816                         config->max_virtqueue_pairs = 1;
1817                 }
1818
1819                 hw->max_queue_pairs = config->max_virtqueue_pairs;
1820
1821                 if (virtio_with_feature(hw, VIRTIO_NET_F_MTU)) {
1822                         virtio_read_dev_config(hw,
1823                                 offsetof(struct virtio_net_config, mtu),
1824                                 &config->mtu,
1825                                 sizeof(config->mtu));
1826
1827                         /*
1828                          * MTU value has already been checked at negotiation
1829                          * time, but check again in case it has changed since
1830                          * then, which should not happen.
1831                          */
1832                         if (config->mtu < RTE_ETHER_MIN_MTU) {
1833                                 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)",
1834                                                 config->mtu);
1835                                 return -1;
1836                         }
1837
1838                         hw->max_mtu = config->mtu;
1839                         /* Set initial MTU to maximum one supported by vhost */
1840                         eth_dev->data->mtu = config->mtu;
1841
1842                 } else {
1843                         hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1844                                 VLAN_TAG_LEN - hw->vtnet_hdr_size;
1845                 }
1846
1847                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
1848                                 config->max_virtqueue_pairs);
1849                 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
1850                 PMD_INIT_LOG(DEBUG,
1851                                 "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
1852                                 config->mac[0], config->mac[1],
1853                                 config->mac[2], config->mac[3],
1854                                 config->mac[4], config->mac[5]);
1855         } else {
1856                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
1857                 hw->max_queue_pairs = 1;
1858                 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1859                         VLAN_TAG_LEN - hw->vtnet_hdr_size;
1860         }
1861
1862         ret = virtio_alloc_queues(eth_dev);
1863         if (ret < 0)
1864                 return ret;
1865
1866         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1867                 if (virtio_configure_intr(eth_dev) < 0) {
1868                         PMD_INIT_LOG(ERR, "failed to configure interrupt");
1869                         virtio_free_queues(hw);
1870                         return -1;
1871                 }
1872         }
1873
1874         virtio_reinit_complete(hw);
1875
1876         return 0;
1877 }
1878
1879 /*
1880  * This function is based on probe() function in virtio_pci.c
1881  * It returns 0 on success.
1882  */
1883 int
1884 eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
1885 {
1886         struct virtio_hw *hw = eth_dev->data->dev_private;
1887         uint32_t speed = RTE_ETH_SPEED_NUM_UNKNOWN;
1888         int vectorized = 0;
1889         int ret;
1890
1891         if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
1892                 PMD_INIT_LOG(ERR,
1893                         "Not sufficient headroom required = %d, avail = %d",
1894                         (int)sizeof(struct virtio_net_hdr_mrg_rxbuf),
1895                         RTE_PKTMBUF_HEADROOM);
1896
1897                 return -1;
1898         }
1899
1900         eth_dev->dev_ops = &virtio_eth_dev_ops;
1901
1902         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1903                 set_rxtx_funcs(eth_dev);
1904                 return 0;
1905         }
1906
1907         ret = virtio_dev_devargs_parse(eth_dev->device->devargs, &speed, &vectorized);
1908         if (ret < 0)
1909                 return ret;
1910         hw->speed = speed;
1911         hw->duplex = DUPLEX_UNKNOWN;
1912
1913         /* Allocate memory for storing MAC addresses */
1914         eth_dev->data->mac_addrs = rte_zmalloc("virtio",
1915                                 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN, 0);
1916         if (eth_dev->data->mac_addrs == NULL) {
1917                 PMD_INIT_LOG(ERR,
1918                         "Failed to allocate %d bytes needed to store MAC addresses",
1919                         VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN);
1920                 return -ENOMEM;
1921         }
1922
1923         rte_spinlock_init(&hw->state_lock);
1924
1925         /* reset device and negotiate default features */
1926         ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
1927         if (ret < 0)
1928                 goto err_virtio_init;
1929
1930         if (vectorized) {
1931                 if (!virtio_with_packed_queue(hw)) {
1932                         hw->use_vec_rx = 1;
1933                 } else {
1934 #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
1935                         hw->use_vec_rx = 1;
1936                         hw->use_vec_tx = 1;
1937 #else
1938                         PMD_DRV_LOG(INFO,
1939                                 "building environment do not support packed ring vectorized");
1940 #endif
1941                 }
1942         }
1943
1944         hw->opened = 1;
1945
1946         return 0;
1947
1948 err_virtio_init:
1949         rte_free(eth_dev->data->mac_addrs);
1950         eth_dev->data->mac_addrs = NULL;
1951         return ret;
1952 }
1953
1954 static uint32_t
1955 virtio_dev_speed_capa_get(uint32_t speed)
1956 {
1957         switch (speed) {
1958         case RTE_ETH_SPEED_NUM_10G:
1959                 return RTE_ETH_LINK_SPEED_10G;
1960         case RTE_ETH_SPEED_NUM_20G:
1961                 return RTE_ETH_LINK_SPEED_20G;
1962         case RTE_ETH_SPEED_NUM_25G:
1963                 return RTE_ETH_LINK_SPEED_25G;
1964         case RTE_ETH_SPEED_NUM_40G:
1965                 return RTE_ETH_LINK_SPEED_40G;
1966         case RTE_ETH_SPEED_NUM_50G:
1967                 return RTE_ETH_LINK_SPEED_50G;
1968         case RTE_ETH_SPEED_NUM_56G:
1969                 return RTE_ETH_LINK_SPEED_56G;
1970         case RTE_ETH_SPEED_NUM_100G:
1971                 return RTE_ETH_LINK_SPEED_100G;
1972         case RTE_ETH_SPEED_NUM_200G:
1973                 return RTE_ETH_LINK_SPEED_200G;
1974         default:
1975                 return 0;
1976         }
1977 }
1978
1979 static int vectorized_check_handler(__rte_unused const char *key,
1980                 const char *value, void *ret_val)
1981 {
1982         if (strcmp(value, "1") == 0)
1983                 *(int *)ret_val = 1;
1984         else
1985                 *(int *)ret_val = 0;
1986
1987         return 0;
1988 }
1989
1990 #define VIRTIO_ARG_SPEED      "speed"
1991 #define VIRTIO_ARG_VECTORIZED "vectorized"
1992
1993 static int
1994 link_speed_handler(const char *key __rte_unused,
1995                 const char *value, void *ret_val)
1996 {
1997         uint32_t val;
1998         if (!value || !ret_val)
1999                 return -EINVAL;
2000         val = strtoul(value, NULL, 0);
2001         /* validate input */
2002         if (virtio_dev_speed_capa_get(val) == 0)
2003                 return -EINVAL;
2004         *(uint32_t *)ret_val = val;
2005
2006         return 0;
2007 }
2008
2009
2010 static int
2011 virtio_dev_devargs_parse(struct rte_devargs *devargs, uint32_t *speed, int *vectorized)
2012 {
2013         struct rte_kvargs *kvlist;
2014         int ret = 0;
2015
2016         if (devargs == NULL)
2017                 return 0;
2018
2019         kvlist = rte_kvargs_parse(devargs->args, NULL);
2020         if (kvlist == NULL) {
2021                 PMD_INIT_LOG(ERR, "error when parsing param");
2022                 return 0;
2023         }
2024
2025         if (speed && rte_kvargs_count(kvlist, VIRTIO_ARG_SPEED) == 1) {
2026                 ret = rte_kvargs_process(kvlist,
2027                                         VIRTIO_ARG_SPEED,
2028                                         link_speed_handler, speed);
2029                 if (ret < 0) {
2030                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2031                                         VIRTIO_ARG_SPEED);
2032                         goto exit;
2033                 }
2034         }
2035
2036         if (vectorized &&
2037                 rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
2038                 ret = rte_kvargs_process(kvlist,
2039                                 VIRTIO_ARG_VECTORIZED,
2040                                 vectorized_check_handler, vectorized);
2041                 if (ret < 0) {
2042                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2043                                         VIRTIO_ARG_VECTORIZED);
2044                         goto exit;
2045                 }
2046         }
2047
2048 exit:
2049         rte_kvargs_free(kvlist);
2050         return ret;
2051 }
2052
2053 static uint8_t
2054 rx_offload_enabled(struct virtio_hw *hw)
2055 {
2056         return virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
2057                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2058                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
2059 }
2060
2061 static uint8_t
2062 tx_offload_enabled(struct virtio_hw *hw)
2063 {
2064         return virtio_with_feature(hw, VIRTIO_NET_F_CSUM) ||
2065                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
2066                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
2067 }
2068
2069 /*
2070  * Configure virtio device
2071  * It returns 0 on success.
2072  */
2073 static int
2074 virtio_dev_configure(struct rte_eth_dev *dev)
2075 {
2076         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2077         const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
2078         struct virtio_hw *hw = dev->data->dev_private;
2079         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
2080                 hw->vtnet_hdr_size;
2081         uint64_t rx_offloads = rxmode->offloads;
2082         uint64_t tx_offloads = txmode->offloads;
2083         uint64_t req_features;
2084         int ret;
2085
2086         PMD_INIT_LOG(DEBUG, "configure");
2087         req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
2088
2089         if (rxmode->mq_mode != RTE_ETH_MQ_RX_NONE) {
2090                 PMD_DRV_LOG(ERR,
2091                         "Unsupported Rx multi queue mode %d",
2092                         rxmode->mq_mode);
2093                 return -EINVAL;
2094         }
2095
2096         if (txmode->mq_mode != RTE_ETH_MQ_TX_NONE) {
2097                 PMD_DRV_LOG(ERR,
2098                         "Unsupported Tx multi queue mode %d",
2099                         txmode->mq_mode);
2100                 return -EINVAL;
2101         }
2102
2103         if (dev->data->dev_conf.intr_conf.rxq) {
2104                 ret = virtio_init_device(dev, hw->req_guest_features);
2105                 if (ret < 0)
2106                         return ret;
2107         }
2108
2109         if (rxmode->mtu > hw->max_mtu)
2110                 req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
2111
2112         hw->max_rx_pkt_len = ether_hdr_len + rxmode->mtu;
2113
2114         if (rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2115                            RTE_ETH_RX_OFFLOAD_TCP_CKSUM))
2116                 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
2117
2118         if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)
2119                 req_features |=
2120                         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2121                         (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2122
2123         if (tx_offloads & (RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2124                            RTE_ETH_TX_OFFLOAD_TCP_CKSUM))
2125                 req_features |= (1ULL << VIRTIO_NET_F_CSUM);
2126
2127         if (tx_offloads & RTE_ETH_TX_OFFLOAD_TCP_TSO)
2128                 req_features |=
2129                         (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2130                         (1ULL << VIRTIO_NET_F_HOST_TSO6);
2131
2132         /* if request features changed, reinit the device */
2133         if (req_features != hw->req_guest_features) {
2134                 ret = virtio_init_device(dev, req_features);
2135                 if (ret < 0)
2136                         return ret;
2137         }
2138
2139         if ((rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2140                             RTE_ETH_RX_OFFLOAD_TCP_CKSUM)) &&
2141                 !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
2142                 PMD_DRV_LOG(ERR,
2143                         "rx checksum not available on this host");
2144                 return -ENOTSUP;
2145         }
2146
2147         if ((rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) &&
2148                 (!virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2149                  !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) {
2150                 PMD_DRV_LOG(ERR,
2151                         "Large Receive Offload not available on this host");
2152                 return -ENOTSUP;
2153         }
2154
2155         /* start control queue */
2156         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
2157                 virtio_dev_cq_start(dev);
2158
2159         if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
2160                 hw->vlan_strip = 1;
2161
2162         hw->rx_ol_scatter = (rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER);
2163
2164         if ((rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) &&
2165                         !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2166                 PMD_DRV_LOG(ERR,
2167                             "vlan filtering not available on this host");
2168                 return -ENOTSUP;
2169         }
2170
2171         hw->has_tx_offload = tx_offload_enabled(hw);
2172         hw->has_rx_offload = rx_offload_enabled(hw);
2173
2174         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2175                 /* Enable vector (0) for Link State Intrerrupt */
2176                 if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) ==
2177                                 VIRTIO_MSI_NO_VECTOR) {
2178                         PMD_DRV_LOG(ERR, "failed to set config vector");
2179                         return -EBUSY;
2180                 }
2181
2182         if (virtio_with_packed_queue(hw)) {
2183 #if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT)
2184                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2185                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
2186                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2187                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2188                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_512)) {
2189                         PMD_DRV_LOG(INFO,
2190                                 "disabled packed ring vectorized path for requirements not met");
2191                         hw->use_vec_rx = 0;
2192                         hw->use_vec_tx = 0;
2193                 }
2194 #elif defined(RTE_ARCH_ARM)
2195                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2196                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) ||
2197                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2198                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2199                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)) {
2200                         PMD_DRV_LOG(INFO,
2201                                 "disabled packed ring vectorized path for requirements not met");
2202                         hw->use_vec_rx = 0;
2203                         hw->use_vec_tx = 0;
2204                 }
2205 #else
2206                 hw->use_vec_rx = 0;
2207                 hw->use_vec_tx = 0;
2208 #endif
2209
2210                 if (hw->use_vec_rx) {
2211                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2212                                 PMD_DRV_LOG(INFO,
2213                                         "disabled packed ring vectorized rx for mrg_rxbuf enabled");
2214                                 hw->use_vec_rx = 0;
2215                         }
2216
2217                         if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) {
2218                                 PMD_DRV_LOG(INFO,
2219                                         "disabled packed ring vectorized rx for TCP_LRO enabled");
2220                                 hw->use_vec_rx = 0;
2221                         }
2222                 }
2223         } else {
2224                 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) {
2225                         hw->use_inorder_tx = 1;
2226                         hw->use_inorder_rx = 1;
2227                         hw->use_vec_rx = 0;
2228                 }
2229
2230                 if (hw->use_vec_rx) {
2231 #if defined RTE_ARCH_ARM
2232                         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
2233                                 PMD_DRV_LOG(INFO,
2234                                         "disabled split ring vectorized path for requirement not met");
2235                                 hw->use_vec_rx = 0;
2236                         }
2237 #endif
2238                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2239                                 PMD_DRV_LOG(INFO,
2240                                         "disabled split ring vectorized rx for mrg_rxbuf enabled");
2241                                 hw->use_vec_rx = 0;
2242                         }
2243
2244                         if (rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2245                                            RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
2246                                            RTE_ETH_RX_OFFLOAD_TCP_LRO |
2247                                            RTE_ETH_RX_OFFLOAD_VLAN_STRIP)) {
2248                                 PMD_DRV_LOG(INFO,
2249                                         "disabled split ring vectorized rx for offloading enabled");
2250                                 hw->use_vec_rx = 0;
2251                         }
2252
2253                         if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
2254                                 PMD_DRV_LOG(INFO,
2255                                         "disabled split ring vectorized rx, max SIMD bitwidth too low");
2256                                 hw->use_vec_rx = 0;
2257                         }
2258                 }
2259         }
2260
2261         return 0;
2262 }
2263
2264
2265 static int
2266 virtio_dev_start(struct rte_eth_dev *dev)
2267 {
2268         uint16_t nb_queues, i;
2269         struct virtqueue *vq;
2270         struct virtio_hw *hw = dev->data->dev_private;
2271         int ret;
2272
2273         /* Finish the initialization of the queues */
2274         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2275                 ret = virtio_dev_rx_queue_setup_finish(dev, i);
2276                 if (ret < 0)
2277                         return ret;
2278         }
2279         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2280                 ret = virtio_dev_tx_queue_setup_finish(dev, i);
2281                 if (ret < 0)
2282                         return ret;
2283         }
2284
2285         /* check if lsc interrupt feature is enabled */
2286         if (dev->data->dev_conf.intr_conf.lsc) {
2287                 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
2288                         PMD_DRV_LOG(ERR, "link status not supported by host");
2289                         return -ENOTSUP;
2290                 }
2291         }
2292
2293         /* Enable uio/vfio intr/eventfd mapping: althrough we already did that
2294          * in device configure, but it could be unmapped  when device is
2295          * stopped.
2296          */
2297         if (dev->data->dev_conf.intr_conf.lsc ||
2298             dev->data->dev_conf.intr_conf.rxq) {
2299                 virtio_intr_disable(dev);
2300
2301                 /* Setup interrupt callback  */
2302                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2303                         rte_intr_callback_register(dev->intr_handle,
2304                                                    virtio_interrupt_handler,
2305                                                    dev);
2306
2307                 if (virtio_intr_enable(dev) < 0) {
2308                         PMD_DRV_LOG(ERR, "interrupt enable failed");
2309                         return -EIO;
2310                 }
2311         }
2312
2313         /*Notify the backend
2314          *Otherwise the tap backend might already stop its queue due to fullness.
2315          *vhost backend will have no chance to be waked up
2316          */
2317         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
2318         if (hw->max_queue_pairs > 1) {
2319                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
2320                         return -EINVAL;
2321         }
2322
2323         PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
2324
2325         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2326                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2327                 /* Flush the old packets */
2328                 virtqueue_rxvq_flush(vq);
2329                 virtqueue_notify(vq);
2330         }
2331
2332         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2333                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2334                 virtqueue_notify(vq);
2335         }
2336
2337         PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
2338
2339         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2340                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2341                 VIRTQUEUE_DUMP(vq);
2342         }
2343
2344         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2345                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2346                 VIRTQUEUE_DUMP(vq);
2347         }
2348
2349         set_rxtx_funcs(dev);
2350         hw->started = 1;
2351
2352         /* Initialize Link state */
2353         virtio_dev_link_update(dev, 0);
2354
2355         return 0;
2356 }
2357
2358 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
2359 {
2360         struct virtio_hw *hw = dev->data->dev_private;
2361         uint16_t nr_vq = virtio_get_nr_vq(hw);
2362         const char *type __rte_unused;
2363         unsigned int i, mbuf_num = 0;
2364         struct virtqueue *vq;
2365         struct rte_mbuf *buf;
2366         int queue_type;
2367
2368         if (hw->vqs == NULL)
2369                 return;
2370
2371         for (i = 0; i < nr_vq; i++) {
2372                 vq = hw->vqs[i];
2373                 if (!vq)
2374                         continue;
2375
2376                 queue_type = virtio_get_queue_type(hw, i);
2377                 if (queue_type == VTNET_RQ)
2378                         type = "rxq";
2379                 else if (queue_type == VTNET_TQ)
2380                         type = "txq";
2381                 else
2382                         continue;
2383
2384                 PMD_INIT_LOG(DEBUG,
2385                         "Before freeing %s[%d] used and unused buf",
2386                         type, i);
2387                 VIRTQUEUE_DUMP(vq);
2388
2389                 while ((buf = virtqueue_detach_unused(vq)) != NULL) {
2390                         rte_pktmbuf_free(buf);
2391                         mbuf_num++;
2392                 }
2393
2394                 PMD_INIT_LOG(DEBUG,
2395                         "After freeing %s[%d] used and unused buf",
2396                         type, i);
2397                 VIRTQUEUE_DUMP(vq);
2398         }
2399
2400         PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num);
2401 }
2402
2403 static void
2404 virtio_tx_completed_cleanup(struct rte_eth_dev *dev)
2405 {
2406         struct virtio_hw *hw = dev->data->dev_private;
2407         struct virtqueue *vq;
2408         int qidx;
2409         void (*xmit_cleanup)(struct virtqueue *vq, uint16_t nb_used);
2410
2411         if (virtio_with_packed_queue(hw)) {
2412                 if (hw->use_vec_tx)
2413                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2414                 else if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER))
2415                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2416                 else
2417                         xmit_cleanup = &virtio_xmit_cleanup_normal_packed;
2418         } else {
2419                 if (hw->use_inorder_tx)
2420                         xmit_cleanup = &virtio_xmit_cleanup_inorder;
2421                 else
2422                         xmit_cleanup = &virtio_xmit_cleanup;
2423         }
2424
2425         for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) {
2426                 vq = hw->vqs[2 * qidx + VTNET_SQ_TQ_QUEUE_IDX];
2427                 if (vq != NULL)
2428                         xmit_cleanup(vq, virtqueue_nused(vq));
2429         }
2430 }
2431
2432 /*
2433  * Stop device: disable interrupt and mark link down
2434  */
2435 int
2436 virtio_dev_stop(struct rte_eth_dev *dev)
2437 {
2438         struct virtio_hw *hw = dev->data->dev_private;
2439         struct rte_eth_link link;
2440         struct rte_eth_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
2441
2442         PMD_INIT_LOG(DEBUG, "stop");
2443         dev->data->dev_started = 0;
2444
2445         rte_spinlock_lock(&hw->state_lock);
2446         if (!hw->started)
2447                 goto out_unlock;
2448         hw->started = 0;
2449
2450         virtio_tx_completed_cleanup(dev);
2451
2452         if (intr_conf->lsc || intr_conf->rxq) {
2453                 virtio_intr_disable(dev);
2454
2455                 /* Reset interrupt callback  */
2456                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
2457                         rte_intr_callback_unregister(dev->intr_handle,
2458                                                      virtio_interrupt_handler,
2459                                                      dev);
2460                 }
2461         }
2462
2463         memset(&link, 0, sizeof(link));
2464         rte_eth_linkstatus_set(dev, &link);
2465 out_unlock:
2466         rte_spinlock_unlock(&hw->state_lock);
2467
2468         return 0;
2469 }
2470
2471 static int
2472 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
2473 {
2474         struct rte_eth_link link;
2475         uint16_t status;
2476         struct virtio_hw *hw = dev->data->dev_private;
2477
2478         memset(&link, 0, sizeof(link));
2479         link.link_duplex = hw->duplex;
2480         link.link_speed  = hw->speed;
2481         link.link_autoneg = RTE_ETH_LINK_AUTONEG;
2482
2483         if (!hw->started) {
2484                 link.link_status = RTE_ETH_LINK_DOWN;
2485                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2486         } else if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
2487                 PMD_INIT_LOG(DEBUG, "Get link status from hw");
2488                 virtio_read_dev_config(hw,
2489                                 offsetof(struct virtio_net_config, status),
2490                                 &status, sizeof(status));
2491                 if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
2492                         link.link_status = RTE_ETH_LINK_DOWN;
2493                         link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2494                         PMD_INIT_LOG(DEBUG, "Port %d is down",
2495                                      dev->data->port_id);
2496                 } else {
2497                         link.link_status = RTE_ETH_LINK_UP;
2498                         PMD_INIT_LOG(DEBUG, "Port %d is up",
2499                                      dev->data->port_id);
2500                 }
2501         } else {
2502                 link.link_status = RTE_ETH_LINK_UP;
2503         }
2504
2505         return rte_eth_linkstatus_set(dev, &link);
2506 }
2507
2508 static int
2509 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2510 {
2511         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2512         struct virtio_hw *hw = dev->data->dev_private;
2513         uint64_t offloads = rxmode->offloads;
2514
2515         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
2516                 if ((offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) &&
2517                                 !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2518
2519                         PMD_DRV_LOG(NOTICE,
2520                                 "vlan filtering not available on this host");
2521
2522                         return -ENOTSUP;
2523                 }
2524         }
2525
2526         if (mask & RTE_ETH_VLAN_STRIP_MASK)
2527                 hw->vlan_strip = !!(offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP);
2528
2529         return 0;
2530 }
2531
2532 static int
2533 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2534 {
2535         uint64_t tso_mask, host_features;
2536         struct virtio_hw *hw = dev->data->dev_private;
2537         dev_info->speed_capa = virtio_dev_speed_capa_get(hw->speed);
2538
2539         dev_info->max_rx_queues =
2540                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
2541         dev_info->max_tx_queues =
2542                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
2543         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
2544         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
2545         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
2546         dev_info->max_mtu = hw->max_mtu;
2547
2548         host_features = VIRTIO_OPS(hw)->get_features(hw);
2549         dev_info->rx_offload_capa = RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2550         if (host_features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))
2551                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_SCATTER;
2552         if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
2553                 dev_info->rx_offload_capa |=
2554                         RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
2555                         RTE_ETH_RX_OFFLOAD_UDP_CKSUM;
2556         }
2557         if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN))
2558                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
2559         tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2560                 (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2561         if ((host_features & tso_mask) == tso_mask)
2562                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO;
2563
2564         dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
2565                                     RTE_ETH_TX_OFFLOAD_VLAN_INSERT;
2566         if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) {
2567                 dev_info->tx_offload_capa |=
2568                         RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2569                         RTE_ETH_TX_OFFLOAD_TCP_CKSUM;
2570         }
2571         tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2572                 (1ULL << VIRTIO_NET_F_HOST_TSO6);
2573         if ((host_features & tso_mask) == tso_mask)
2574                 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_TCP_TSO;
2575
2576         if (host_features & (1ULL << VIRTIO_F_RING_PACKED)) {
2577                 /*
2578                  * According to 2.7 Packed Virtqueues,
2579                  * 2.7.10.1 Structure Size and Alignment:
2580                  * The Queue Size value does not have to be a power of 2.
2581                  */
2582                 dev_info->rx_desc_lim.nb_max = UINT16_MAX;
2583                 dev_info->tx_desc_lim.nb_max = UINT16_MAX;
2584         } else {
2585                 /*
2586                  * According to 2.6 Split Virtqueues:
2587                  * Queue Size value is always a power of 2. The maximum Queue
2588                  * Size value is 32768.
2589                  */
2590                 dev_info->rx_desc_lim.nb_max = 32768;
2591                 dev_info->tx_desc_lim.nb_max = 32768;
2592         }
2593         /*
2594          * Actual minimum is not the same for virtqueues of different kinds,
2595          * but to avoid tangling the code with separate branches, rely on
2596          * default thresholds since desc number must be at least of their size.
2597          */
2598         dev_info->rx_desc_lim.nb_min = RTE_MAX(DEFAULT_RX_FREE_THRESH,
2599                                                RTE_VIRTIO_VPMD_RX_REARM_THRESH);
2600         dev_info->tx_desc_lim.nb_min = DEFAULT_TX_FREE_THRESH;
2601         dev_info->rx_desc_lim.nb_align = 1;
2602         dev_info->tx_desc_lim.nb_align = 1;
2603
2604         return 0;
2605 }
2606
2607 /*
2608  * It enables testpmd to collect per queue stats.
2609  */
2610 static int
2611 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
2612 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
2613 __rte_unused uint8_t is_rx)
2614 {
2615         return 0;
2616 }
2617
2618 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_init, init, NOTICE);
2619 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_driver, driver, NOTICE);