drivers/net/virtio/virtio_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10
  11 #include <ethdev_driver.h>
  12 #include <rte_memcpy.h>
  13 #include <rte_string_fns.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_branch_prediction.h>
  17 #include <rte_ether.h>
  18 #include <rte_ip.h>
  19 #include <rte_arp.h>
  20 #include <rte_common.h>
  21 #include <rte_errno.h>
  22 #include <rte_cpuflags.h>
  23 #include <rte_vect.h>
  24 #include <rte_memory.h>
  25 #include <rte_eal_paging.h>
  26 #include <rte_eal.h>
  27 #include <rte_dev.h>
  28 #include <rte_cycles.h>
  29 #include <rte_kvargs.h>
  30
  31 #include "virtio_ethdev.h"
  32 #include "virtio.h"
  33 #include "virtio_logs.h"
  34 #include "virtqueue.h"
  35 #include "virtio_rxtx.h"
  36 #include "virtio_rxtx_simple.h"
  37 #include "virtio_user/virtio_user_dev.h"
  38
  39 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  40 static int  virtio_dev_start(struct rte_eth_dev *dev);
  41 static int virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
  42 static int virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
  43 static int virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
  44 static int virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
  45 static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
  46 static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
  47         uint32_t *speed,
  48         int *vectorized);
  49 static int virtio_dev_info_get(struct rte_eth_dev *dev,
  50                                 struct rte_eth_dev_info *dev_info);
  51 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  52         int wait_to_complete);
  53 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  54
  55 static void virtio_set_hwaddr(struct virtio_hw *hw);
  56 static void virtio_get_hwaddr(struct virtio_hw *hw);
  57
  58 static int virtio_dev_stats_get(struct rte_eth_dev *dev,
  59                                  struct rte_eth_stats *stats);
  60 static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
  61                                  struct rte_eth_xstat *xstats, unsigned n);
  62 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
  63                                        struct rte_eth_xstat_name *xstats_names,
  64                                        unsigned limit);
  65 static int virtio_dev_stats_reset(struct rte_eth_dev *dev);
  66 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  67 static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
  68                                 uint16_t vlan_id, int on);
  69 static int virtio_mac_addr_add(struct rte_eth_dev *dev,
  70                                 struct rte_ether_addr *mac_addr,
  71                                 uint32_t index, uint32_t vmdq);
  72 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  73 static int virtio_mac_addr_set(struct rte_eth_dev *dev,
  74                                 struct rte_ether_addr *mac_addr);
  75
  76 static int virtio_intr_disable(struct rte_eth_dev *dev);
  77
  78 static int virtio_dev_queue_stats_mapping_set(
  79         struct rte_eth_dev *eth_dev,
  80         uint16_t queue_id,
  81         uint8_t stat_idx,
  82         uint8_t is_rx);
  83
  84 static void virtio_notify_peers(struct rte_eth_dev *dev);
  85 static void virtio_ack_link_announce(struct rte_eth_dev *dev);
  86
  87 struct rte_virtio_xstats_name_off {
  88         char name[RTE_ETH_XSTATS_NAME_SIZE];
  89         unsigned offset;
  90 };
  91
  92 /* [rt]x_qX_ is prepended to the name string here */
  93 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
  94         {"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
  95         {"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
  96         {"errors",                 offsetof(struct virtnet_rx, stats.errors)},
  97         {"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
  98         {"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
  99         {"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
 100         {"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
 101         {"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
 102         {"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
 103         {"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
 104         {"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
 105         {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
 106         {"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
 107 };
 108
 109 /* [rt]x_qX_ is prepended to the name string here */
 110 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
 111         {"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
 112         {"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
 113         {"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
 114         {"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
 115         {"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
 116         {"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
 117         {"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
 118         {"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
 119         {"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
 120         {"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
 121         {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
 122         {"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
 123 };
 124
 125 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
 126                             sizeof(rte_virtio_rxq_stat_strings[0]))
 127 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
 128                             sizeof(rte_virtio_txq_stat_strings[0]))
 129
 130 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS];
 131
 132 static struct virtio_pmd_ctrl *
 133 virtio_send_command_packed(struct virtnet_ctl *cvq,
 134                            struct virtio_pmd_ctrl *ctrl,
 135                            int *dlen, int pkt_num)
 136 {
 137         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 138         int head;
 139         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 140         struct virtio_pmd_ctrl *result;
 141         uint16_t flags;
 142         int sum = 0;
 143         int nb_descs = 0;
 144         int k;
 145
 146         /*
 147          * Format is enforced in qemu code:
 148          * One TX packet for header;
 149          * At least one TX packet per argument;
 150          * One RX packet for ACK.
 151          */
 152         head = vq->vq_avail_idx;
 153         flags = vq->vq_packed.cached_flags;
 154         desc[head].addr = cvq->virtio_net_hdr_mem;
 155         desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 156         vq->vq_free_cnt--;
 157         nb_descs++;
 158         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 159                 vq->vq_avail_idx -= vq->vq_nentries;
 160                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 161         }
 162
 163         for (k = 0; k < pkt_num; k++) {
 164                 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 165                         + sizeof(struct virtio_net_ctrl_hdr)
 166                         + sizeof(ctrl->status) + sizeof(uint8_t) * sum;
 167                 desc[vq->vq_avail_idx].len = dlen[k];
 168                 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT |
 169                         vq->vq_packed.cached_flags;
 170                 sum += dlen[k];
 171                 vq->vq_free_cnt--;
 172                 nb_descs++;
 173                 if (++vq->vq_avail_idx >= vq->vq_nentries) {
 174                         vq->vq_avail_idx -= vq->vq_nentries;
 175                         vq->vq_packed.cached_flags ^=
 176                                 VRING_PACKED_DESC_F_AVAIL_USED;
 177                 }
 178         }
 179
 180         desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 181                 + sizeof(struct virtio_net_ctrl_hdr);
 182         desc[vq->vq_avail_idx].len = sizeof(ctrl->status);
 183         desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE |
 184                 vq->vq_packed.cached_flags;
 185         vq->vq_free_cnt--;
 186         nb_descs++;
 187         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 188                 vq->vq_avail_idx -= vq->vq_nentries;
 189                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 190         }
 191
 192         virtqueue_store_flags_packed(&desc[head], VRING_DESC_F_NEXT | flags,
 193                         vq->hw->weak_barriers);
 194
 195         virtio_wmb(vq->hw->weak_barriers);
 196         virtqueue_notify(vq);
 197
 198         /* wait for used desc in virtqueue
 199          * desc_is_used has a load-acquire or rte_io_rmb inside
 200          */
 201         while (!desc_is_used(&desc[head], vq))
 202                 usleep(100);
 203
 204         /* now get used descriptors */
 205         vq->vq_free_cnt += nb_descs;
 206         vq->vq_used_cons_idx += nb_descs;
 207         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 208                 vq->vq_used_cons_idx -= vq->vq_nentries;
 209                 vq->vq_packed.used_wrap_counter ^= 1;
 210         }
 211
 212         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n"
 213                         "vq->vq_avail_idx=%d\n"
 214                         "vq->vq_used_cons_idx=%d\n"
 215                         "vq->vq_packed.cached_flags=0x%x\n"
 216                         "vq->vq_packed.used_wrap_counter=%d",
 217                         vq->vq_free_cnt,
 218                         vq->vq_avail_idx,
 219                         vq->vq_used_cons_idx,
 220                         vq->vq_packed.cached_flags,
 221                         vq->vq_packed.used_wrap_counter);
 222
 223         result = cvq->virtio_net_hdr_mz->addr;
 224         return result;
 225 }
 226
 227 static struct virtio_pmd_ctrl *
 228 virtio_send_command_split(struct virtnet_ctl *cvq,
 229                           struct virtio_pmd_ctrl *ctrl,
 230                           int *dlen, int pkt_num)
 231 {
 232         struct virtio_pmd_ctrl *result;
 233         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 234         uint32_t head, i;
 235         int k, sum = 0;
 236
 237         head = vq->vq_desc_head_idx;
 238
 239         /*
 240          * Format is enforced in qemu code:
 241          * One TX packet for header;
 242          * At least one TX packet per argument;
 243          * One RX packet for ACK.
 244          */
 245         vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT;
 246         vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem;
 247         vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 248         vq->vq_free_cnt--;
 249         i = vq->vq_split.ring.desc[head].next;
 250
 251         for (k = 0; k < pkt_num; k++) {
 252                 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT;
 253                 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 254                         + sizeof(struct virtio_net_ctrl_hdr)
 255                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 256                 vq->vq_split.ring.desc[i].len = dlen[k];
 257                 sum += dlen[k];
 258                 vq->vq_free_cnt--;
 259                 i = vq->vq_split.ring.desc[i].next;
 260         }
 261
 262         vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE;
 263         vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 264                         + sizeof(struct virtio_net_ctrl_hdr);
 265         vq->vq_split.ring.desc[i].len = sizeof(ctrl->status);
 266         vq->vq_free_cnt--;
 267
 268         vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next;
 269
 270         vq_update_avail_ring(vq, head);
 271         vq_update_avail_idx(vq);
 272
 273         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
 274
 275         virtqueue_notify(vq);
 276
 277         while (virtqueue_nused(vq) == 0)
 278                 usleep(100);
 279
 280         while (virtqueue_nused(vq)) {
 281                 uint32_t idx, desc_idx, used_idx;
 282                 struct vring_used_elem *uep;
 283
 284                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 285                                 & (vq->vq_nentries - 1));
 286                 uep = &vq->vq_split.ring.used->ring[used_idx];
 287                 idx = (uint32_t) uep->id;
 288                 desc_idx = idx;
 289
 290                 while (vq->vq_split.ring.desc[desc_idx].flags &
 291                                 VRING_DESC_F_NEXT) {
 292                         desc_idx = vq->vq_split.ring.desc[desc_idx].next;
 293                         vq->vq_free_cnt++;
 294                 }
 295
 296                 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 297                 vq->vq_desc_head_idx = idx;
 298
 299                 vq->vq_used_cons_idx++;
 300                 vq->vq_free_cnt++;
 301         }
 302
 303         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
 304                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 305
 306         result = cvq->virtio_net_hdr_mz->addr;
 307         return result;
 308 }
 309
 310 static int
 311 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
 312                     int *dlen, int pkt_num)
 313 {
 314         virtio_net_ctrl_ack status = ~0;
 315         struct virtio_pmd_ctrl *result;
 316         struct virtqueue *vq;
 317
 318         ctrl->status = status;
 319
 320         if (!cvq) {
 321                 PMD_INIT_LOG(ERR, "Control queue is not supported.");
 322                 return -1;
 323         }
 324
 325         rte_spinlock_lock(&cvq->lock);
 326         vq = virtnet_cq_to_vq(cvq);
 327
 328         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 329                 "vq->hw->cvq = %p vq = %p",
 330                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 331
 332         if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) {
 333                 rte_spinlock_unlock(&cvq->lock);
 334                 return -1;
 335         }
 336
 337         memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
 338                 sizeof(struct virtio_pmd_ctrl));
 339
 340         if (virtio_with_packed_queue(vq->hw))
 341                 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num);
 342         else
 343                 result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num);
 344
 345         rte_spinlock_unlock(&cvq->lock);
 346         return result->status;
 347 }
 348
 349 static int
 350 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 351 {
 352         struct virtio_hw *hw = dev->data->dev_private;
 353         struct virtio_pmd_ctrl ctrl;
 354         int dlen[1];
 355         int ret;
 356
 357         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 358         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 359         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 360
 361         dlen[0] = sizeof(uint16_t);
 362
 363         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 364         if (ret) {
 365                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 366                           "failed, this is too late now...");
 367                 return -EINVAL;
 368         }
 369
 370         return 0;
 371 }
 372
 373 static uint16_t
 374 virtio_get_nr_vq(struct virtio_hw *hw)
 375 {
 376         uint16_t nr_vq = hw->max_queue_pairs * 2;
 377
 378         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
 379                 nr_vq += 1;
 380
 381         return nr_vq;
 382 }
 383
 384 static void
 385 virtio_init_vring(struct virtqueue *vq)
 386 {
 387         int size = vq->vq_nentries;
 388         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 389
 390         PMD_INIT_FUNC_TRACE();
 391
 392         memset(ring_mem, 0, vq->vq_ring_size);
 393
 394         vq->vq_used_cons_idx = 0;
 395         vq->vq_desc_head_idx = 0;
 396         vq->vq_avail_idx = 0;
 397         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 398         vq->vq_free_cnt = vq->vq_nentries;
 399         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 400         if (virtio_with_packed_queue(vq->hw)) {
 401                 vring_init_packed(&vq->vq_packed.ring, ring_mem,
 402                                   VIRTIO_VRING_ALIGN, size);
 403                 vring_desc_init_packed(vq, size);
 404         } else {
 405                 struct vring *vr = &vq->vq_split.ring;
 406
 407                 vring_init_split(vr, ring_mem, VIRTIO_VRING_ALIGN, size);
 408                 vring_desc_init_split(vr->desc, size);
 409         }
 410         /*
 411          * Disable device(host) interrupting guest
 412          */
 413         virtqueue_disable_intr(vq);
 414 }
 415
 416 static int
 417 virtio_init_queue(struct rte_eth_dev *dev, uint16_t queue_idx)
 418 {
 419         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 420         char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
 421         const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
 422         unsigned int vq_size, size;
 423         struct virtio_hw *hw = dev->data->dev_private;
 424         struct virtnet_rx *rxvq = NULL;
 425         struct virtnet_tx *txvq = NULL;
 426         struct virtnet_ctl *cvq = NULL;
 427         struct virtqueue *vq;
 428         size_t sz_hdr_mz = 0;
 429         void *sw_ring = NULL;
 430         int queue_type = virtio_get_queue_type(hw, queue_idx);
 431         int ret;
 432         int numa_node = dev->device->numa_node;
 433         struct rte_mbuf *fake_mbuf = NULL;
 434
 435         PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d",
 436                         queue_idx, numa_node);
 437
 438         /*
 439          * Read the virtqueue size from the Queue Size field
 440          * Always power of 2 and if 0 virtqueue does not exist
 441          */
 442         vq_size = VIRTIO_OPS(hw)->get_queue_num(hw, queue_idx);
 443         PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
 444         if (vq_size == 0) {
 445                 PMD_INIT_LOG(ERR, "virtqueue does not exist");
 446                 return -EINVAL;
 447         }
 448
 449         if (!virtio_with_packed_queue(hw) && !rte_is_power_of_2(vq_size)) {
 450                 PMD_INIT_LOG(ERR, "split virtqueue size is not power of 2");
 451                 return -EINVAL;
 452         }
 453
 454         snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
 455                  dev->data->port_id, queue_idx);
 456
 457         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 458                                 vq_size * sizeof(struct vq_desc_extra),
 459                                 RTE_CACHE_LINE_SIZE);
 460         if (queue_type == VTNET_TQ) {
 461                 /*
 462                  * For each xmit packet, allocate a virtio_net_hdr
 463                  * and indirect ring elements
 464                  */
 465                 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
 466         } else if (queue_type == VTNET_CQ) {
 467                 /* Allocate a page for control vq command, data and status */
 468                 sz_hdr_mz = rte_mem_page_size();
 469         }
 470
 471         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
 472                                 numa_node);
 473         if (vq == NULL) {
 474                 PMD_INIT_LOG(ERR, "can not allocate vq");
 475                 return -ENOMEM;
 476         }
 477         hw->vqs[queue_idx] = vq;
 478
 479         vq->hw = hw;
 480         vq->vq_queue_index = queue_idx;
 481         vq->vq_nentries = vq_size;
 482         if (virtio_with_packed_queue(hw)) {
 483                 vq->vq_packed.used_wrap_counter = 1;
 484                 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
 485                 vq->vq_packed.event_flags_shadow = 0;
 486                 if (queue_type == VTNET_RQ)
 487                         vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
 488         }
 489
 490         /*
 491          * Reserve a memzone for vring elements
 492          */
 493         size = vring_size(hw, vq_size, VIRTIO_VRING_ALIGN);
 494         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN);
 495         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
 496                      size, vq->vq_ring_size);
 497
 498         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 499                         numa_node, RTE_MEMZONE_IOVA_CONTIG,
 500                         VIRTIO_VRING_ALIGN);
 501         if (mz == NULL) {
 502                 if (rte_errno == EEXIST)
 503                         mz = rte_memzone_lookup(vq_name);
 504                 if (mz == NULL) {
 505                         ret = -ENOMEM;
 506                         goto free_vq;
 507                 }
 508         }
 509
 510         memset(mz->addr, 0, mz->len);
 511
 512         if (hw->use_va)
 513                 vq->vq_ring_mem = (uintptr_t)mz->addr;
 514         else
 515                 vq->vq_ring_mem = mz->iova;
 516
 517         vq->vq_ring_virt_mem = mz->addr;
 518         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem);
 519         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem);
 520
 521         virtio_init_vring(vq);
 522
 523         if (sz_hdr_mz) {
 524                 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 525                          dev->data->port_id, queue_idx);
 526                 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
 527                                 numa_node, RTE_MEMZONE_IOVA_CONTIG,
 528                                 RTE_CACHE_LINE_SIZE);
 529                 if (hdr_mz == NULL) {
 530                         if (rte_errno == EEXIST)
 531                                 hdr_mz = rte_memzone_lookup(vq_hdr_name);
 532                         if (hdr_mz == NULL) {
 533                                 ret = -ENOMEM;
 534                                 goto free_mz;
 535                         }
 536                 }
 537         }
 538
 539         if (queue_type == VTNET_RQ) {
 540                 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
 541                                sizeof(vq->sw_ring[0]);
 542
 543                 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
 544                                 RTE_CACHE_LINE_SIZE, numa_node);
 545                 if (!sw_ring) {
 546                         PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
 547                         ret = -ENOMEM;
 548                         goto free_hdr_mz;
 549                 }
 550
 551                 fake_mbuf = rte_zmalloc_socket("sw_ring", sizeof(*fake_mbuf),
 552                                 RTE_CACHE_LINE_SIZE, numa_node);
 553                 if (!fake_mbuf) {
 554                         PMD_INIT_LOG(ERR, "can not allocate fake mbuf");
 555                         ret = -ENOMEM;
 556                         goto free_sw_ring;
 557                 }
 558
 559                 vq->sw_ring = sw_ring;
 560                 rxvq = &vq->rxq;
 561                 rxvq->port_id = dev->data->port_id;
 562                 rxvq->mz = mz;
 563                 rxvq->fake_mbuf = fake_mbuf;
 564         } else if (queue_type == VTNET_TQ) {
 565                 txvq = &vq->txq;
 566                 txvq->port_id = dev->data->port_id;
 567                 txvq->mz = mz;
 568                 txvq->virtio_net_hdr_mz = hdr_mz;
 569                 if (hw->use_va)
 570                         txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 571                 else
 572                         txvq->virtio_net_hdr_mem = hdr_mz->iova;
 573         } else if (queue_type == VTNET_CQ) {
 574                 cvq = &vq->cq;
 575                 cvq->mz = mz;
 576                 cvq->virtio_net_hdr_mz = hdr_mz;
 577                 if (hw->use_va)
 578                         cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 579                 else
 580                         cvq->virtio_net_hdr_mem = hdr_mz->iova;
 581                 memset(cvq->virtio_net_hdr_mz->addr, 0, rte_mem_page_size());
 582
 583                 hw->cvq = cvq;
 584         }
 585
 586         if (hw->use_va)
 587                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr);
 588         else
 589                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova);
 590
 591         if (queue_type == VTNET_TQ) {
 592                 struct virtio_tx_region *txr;
 593                 unsigned int i;
 594
 595                 txr = hdr_mz->addr;
 596                 memset(txr, 0, vq_size * sizeof(*txr));
 597                 for (i = 0; i < vq_size; i++) {
 598                         /* first indirect descriptor is always the tx header */
 599                         if (!virtio_with_packed_queue(hw)) {
 600                                 struct vring_desc *start_dp = txr[i].tx_indir;
 601                                 vring_desc_init_split(start_dp,
 602                                                       RTE_DIM(txr[i].tx_indir));
 603                                 start_dp->addr = txvq->virtio_net_hdr_mem
 604                                         + i * sizeof(*txr)
 605                                         + offsetof(struct virtio_tx_region,
 606                                                    tx_hdr);
 607                                 start_dp->len = hw->vtnet_hdr_size;
 608                                 start_dp->flags = VRING_DESC_F_NEXT;
 609                         } else {
 610                                 struct vring_packed_desc *start_dp =
 611                                         txr[i].tx_packed_indir;
 612                                 vring_desc_init_indirect_packed(start_dp,
 613                                       RTE_DIM(txr[i].tx_packed_indir));
 614                                 start_dp->addr = txvq->virtio_net_hdr_mem
 615                                         + i * sizeof(*txr)
 616                                         + offsetof(struct virtio_tx_region,
 617                                                    tx_hdr);
 618                                 start_dp->len = hw->vtnet_hdr_size;
 619                         }
 620                 }
 621         }
 622
 623         if (VIRTIO_OPS(hw)->setup_queue(hw, vq) < 0) {
 624                 PMD_INIT_LOG(ERR, "setup_queue failed");
 625                 ret = -EINVAL;
 626                 goto clean_vq;
 627         }
 628
 629         return 0;
 630
 631 clean_vq:
 632         hw->cvq = NULL;
 633         rte_free(fake_mbuf);
 634 free_sw_ring:
 635         rte_free(sw_ring);
 636 free_hdr_mz:
 637         rte_memzone_free(hdr_mz);
 638 free_mz:
 639         rte_memzone_free(mz);
 640 free_vq:
 641         rte_free(vq);
 642         hw->vqs[queue_idx] = NULL;
 643
 644         return ret;
 645 }
 646
 647 static void
 648 virtio_free_queues(struct virtio_hw *hw)
 649 {
 650         uint16_t nr_vq = virtio_get_nr_vq(hw);
 651         struct virtqueue *vq;
 652         int queue_type;
 653         uint16_t i;
 654
 655         if (hw->vqs == NULL)
 656                 return;
 657
 658         for (i = 0; i < nr_vq; i++) {
 659                 vq = hw->vqs[i];
 660                 if (!vq)
 661                         continue;
 662
 663                 queue_type = virtio_get_queue_type(hw, i);
 664                 if (queue_type == VTNET_RQ) {
 665                         rte_free(vq->rxq.fake_mbuf);
 666                         rte_free(vq->sw_ring);
 667                         rte_memzone_free(vq->rxq.mz);
 668                 } else if (queue_type == VTNET_TQ) {
 669                         rte_memzone_free(vq->txq.mz);
 670                         rte_memzone_free(vq->txq.virtio_net_hdr_mz);
 671                 } else {
 672                         rte_memzone_free(vq->cq.mz);
 673                         rte_memzone_free(vq->cq.virtio_net_hdr_mz);
 674                 }
 675
 676                 rte_free(vq);
 677                 hw->vqs[i] = NULL;
 678         }
 679
 680         rte_free(hw->vqs);
 681         hw->vqs = NULL;
 682 }
 683
 684 static int
 685 virtio_alloc_queues(struct rte_eth_dev *dev)
 686 {
 687         struct virtio_hw *hw = dev->data->dev_private;
 688         uint16_t nr_vq = virtio_get_nr_vq(hw);
 689         uint16_t i;
 690         int ret;
 691
 692         hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0);
 693         if (!hw->vqs) {
 694                 PMD_INIT_LOG(ERR, "failed to allocate vqs");
 695                 return -ENOMEM;
 696         }
 697
 698         for (i = 0; i < nr_vq; i++) {
 699                 ret = virtio_init_queue(dev, i);
 700                 if (ret < 0) {
 701                         virtio_free_queues(hw);
 702                         return ret;
 703                 }
 704         }
 705
 706         return 0;
 707 }
 708
 709 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev);
 710
 711 int
 712 virtio_dev_close(struct rte_eth_dev *dev)
 713 {
 714         struct virtio_hw *hw = dev->data->dev_private;
 715         struct rte_eth_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
 716
 717         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 718         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 719                 return 0;
 720
 721         if (!hw->opened)
 722                 return 0;
 723         hw->opened = 0;
 724
 725         /* reset the NIC */
 726         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 727                 VIRTIO_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR);
 728         if (intr_conf->rxq)
 729                 virtio_queues_unbind_intr(dev);
 730
 731         if (intr_conf->lsc || intr_conf->rxq) {
 732                 virtio_intr_disable(dev);
 733                 rte_intr_efd_disable(dev->intr_handle);
 734                 rte_free(dev->intr_handle->intr_vec);
 735                 dev->intr_handle->intr_vec = NULL;
 736         }
 737
 738         virtio_reset(hw);
 739         virtio_dev_free_mbufs(dev);
 740         virtio_free_queues(hw);
 741
 742         return VIRTIO_OPS(hw)->dev_close(hw);
 743 }
 744
 745 static int
 746 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
 747 {
 748         struct virtio_hw *hw = dev->data->dev_private;
 749         struct virtio_pmd_ctrl ctrl;
 750         int dlen[1];
 751         int ret;
 752
 753         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 754                 PMD_INIT_LOG(INFO, "host does not support rx control");
 755                 return -ENOTSUP;
 756         }
 757
 758         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 759         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 760         ctrl.data[0] = 1;
 761         dlen[0] = 1;
 762
 763         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 764         if (ret) {
 765                 PMD_INIT_LOG(ERR, "Failed to enable promisc");
 766                 return -EAGAIN;
 767         }
 768
 769         return 0;
 770 }
 771
 772 static int
 773 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
 774 {
 775         struct virtio_hw *hw = dev->data->dev_private;
 776         struct virtio_pmd_ctrl ctrl;
 777         int dlen[1];
 778         int ret;
 779
 780         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 781                 PMD_INIT_LOG(INFO, "host does not support rx control");
 782                 return -ENOTSUP;
 783         }
 784
 785         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 786         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 787         ctrl.data[0] = 0;
 788         dlen[0] = 1;
 789
 790         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 791         if (ret) {
 792                 PMD_INIT_LOG(ERR, "Failed to disable promisc");
 793                 return -EAGAIN;
 794         }
 795
 796         return 0;
 797 }
 798
 799 static int
 800 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
 801 {
 802         struct virtio_hw *hw = dev->data->dev_private;
 803         struct virtio_pmd_ctrl ctrl;
 804         int dlen[1];
 805         int ret;
 806
 807         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 808                 PMD_INIT_LOG(INFO, "host does not support rx control");
 809                 return -ENOTSUP;
 810         }
 811
 812         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 813         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 814         ctrl.data[0] = 1;
 815         dlen[0] = 1;
 816
 817         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 818         if (ret) {
 819                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
 820                 return -EAGAIN;
 821         }
 822
 823         return 0;
 824 }
 825
 826 static int
 827 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
 828 {
 829         struct virtio_hw *hw = dev->data->dev_private;
 830         struct virtio_pmd_ctrl ctrl;
 831         int dlen[1];
 832         int ret;
 833
 834         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 835                 PMD_INIT_LOG(INFO, "host does not support rx control");
 836                 return -ENOTSUP;
 837         }
 838
 839         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 840         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 841         ctrl.data[0] = 0;
 842         dlen[0] = 1;
 843
 844         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 845         if (ret) {
 846                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
 847                 return -EAGAIN;
 848         }
 849
 850         return 0;
 851 }
 852
 853 uint16_t
 854 virtio_rx_mem_pool_buf_size(struct rte_mempool *mp)
 855 {
 856         return rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
 857 }
 858
 859 bool
 860 virtio_rx_check_scatter(uint16_t max_rx_pkt_len, uint16_t rx_buf_size,
 861                         bool rx_scatter_enabled, const char **error)
 862 {
 863         if (!rx_scatter_enabled && max_rx_pkt_len > rx_buf_size) {
 864                 *error = "Rx scatter is disabled and RxQ mbuf pool object size is too small";
 865                 return false;
 866         }
 867
 868         return true;
 869 }
 870
 871 static bool
 872 virtio_check_scatter_on_all_rx_queues(struct rte_eth_dev *dev,
 873                                       uint16_t frame_size)
 874 {
 875         struct virtio_hw *hw = dev->data->dev_private;
 876         struct virtnet_rx *rxvq;
 877         struct virtqueue *vq;
 878         unsigned int qidx;
 879         uint16_t buf_size;
 880         const char *error;
 881
 882         if (hw->vqs == NULL)
 883                 return true;
 884
 885         for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) {
 886                 vq = hw->vqs[2 * qidx + VTNET_SQ_RQ_QUEUE_IDX];
 887                 if (vq == NULL)
 888                         continue;
 889
 890                 rxvq = &vq->rxq;
 891                 if (rxvq->mpool == NULL)
 892                         continue;
 893                 buf_size = virtio_rx_mem_pool_buf_size(rxvq->mpool);
 894
 895                 if (!virtio_rx_check_scatter(frame_size, buf_size,
 896                                              hw->rx_ol_scatter, &error)) {
 897                         PMD_INIT_LOG(ERR, "MTU check for RxQ %u failed: %s",
 898                                      qidx, error);
 899                         return false;
 900                 }
 901         }
 902
 903         return true;
 904 }
 905
 906 #define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
 907 static int
 908 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 909 {
 910         struct virtio_hw *hw = dev->data->dev_private;
 911         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
 912                                  hw->vtnet_hdr_size;
 913         uint32_t frame_size = mtu + ether_hdr_len;
 914         uint32_t max_frame_size = hw->max_mtu + ether_hdr_len;
 915
 916         max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN);
 917
 918         if (mtu < RTE_ETHER_MIN_MTU || frame_size > max_frame_size) {
 919                 PMD_INIT_LOG(ERR, "MTU should be between %d and %d",
 920                         RTE_ETHER_MIN_MTU, max_frame_size - ether_hdr_len);
 921                 return -EINVAL;
 922         }
 923
 924         if (!virtio_check_scatter_on_all_rx_queues(dev, frame_size)) {
 925                 PMD_INIT_LOG(ERR, "MTU vs Rx scatter and Rx buffers check failed");
 926                 return -EINVAL;
 927         }
 928
 929         hw->max_rx_pkt_len = frame_size;
 930
 931         return 0;
 932 }
 933
 934 static int
 935 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
 936 {
 937         struct virtio_hw *hw = dev->data->dev_private;
 938         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 939         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
 940
 941         virtqueue_enable_intr(vq);
 942         virtio_mb(hw->weak_barriers);
 943         return 0;
 944 }
 945
 946 static int
 947 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 948 {
 949         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
 950         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
 951
 952         virtqueue_disable_intr(vq);
 953         return 0;
 954 }
 955
 956 /*
 957  * dev_ops for virtio, bare necessities for basic operation
 958  */
 959 static const struct eth_dev_ops virtio_eth_dev_ops = {
 960         .dev_configure           = virtio_dev_configure,
 961         .dev_start               = virtio_dev_start,
 962         .dev_stop                = virtio_dev_stop,
 963         .dev_close               = virtio_dev_close,
 964         .promiscuous_enable      = virtio_dev_promiscuous_enable,
 965         .promiscuous_disable     = virtio_dev_promiscuous_disable,
 966         .allmulticast_enable     = virtio_dev_allmulticast_enable,
 967         .allmulticast_disable    = virtio_dev_allmulticast_disable,
 968         .mtu_set                 = virtio_mtu_set,
 969         .dev_infos_get           = virtio_dev_info_get,
 970         .stats_get               = virtio_dev_stats_get,
 971         .xstats_get              = virtio_dev_xstats_get,
 972         .xstats_get_names        = virtio_dev_xstats_get_names,
 973         .stats_reset             = virtio_dev_stats_reset,
 974         .xstats_reset            = virtio_dev_stats_reset,
 975         .link_update             = virtio_dev_link_update,
 976         .vlan_offload_set        = virtio_dev_vlan_offload_set,
 977         .rx_queue_setup          = virtio_dev_rx_queue_setup,
 978         .rx_queue_intr_enable    = virtio_dev_rx_queue_intr_enable,
 979         .rx_queue_intr_disable   = virtio_dev_rx_queue_intr_disable,
 980         .tx_queue_setup          = virtio_dev_tx_queue_setup,
 981         /* collect stats per queue */
 982         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
 983         .vlan_filter_set         = virtio_vlan_filter_set,
 984         .mac_addr_add            = virtio_mac_addr_add,
 985         .mac_addr_remove         = virtio_mac_addr_remove,
 986         .mac_addr_set            = virtio_mac_addr_set,
 987 };
 988
 989 /*
 990  * dev_ops for virtio-user in secondary processes, as we just have
 991  * some limited supports currently.
 992  */
 993 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops = {
 994         .dev_infos_get           = virtio_dev_info_get,
 995         .stats_get               = virtio_dev_stats_get,
 996         .xstats_get              = virtio_dev_xstats_get,
 997         .xstats_get_names        = virtio_dev_xstats_get_names,
 998         .stats_reset             = virtio_dev_stats_reset,
 999         .xstats_reset            = virtio_dev_stats_reset,
1000         /* collect stats per queue */
1001         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
1002 };
1003
1004 static void
1005 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1006 {
1007         unsigned i;
1008
1009         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1010                 const struct virtnet_tx *txvq = dev->data->tx_queues[i];
1011                 if (txvq == NULL)
1012                         continue;
1013
1014                 stats->opackets += txvq->stats.packets;
1015                 stats->obytes += txvq->stats.bytes;
1016
1017                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1018                         stats->q_opackets[i] = txvq->stats.packets;
1019                         stats->q_obytes[i] = txvq->stats.bytes;
1020                 }
1021         }
1022
1023         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1024                 const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1025                 if (rxvq == NULL)
1026                         continue;
1027
1028                 stats->ipackets += rxvq->stats.packets;
1029                 stats->ibytes += rxvq->stats.bytes;
1030                 stats->ierrors += rxvq->stats.errors;
1031
1032                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1033                         stats->q_ipackets[i] = rxvq->stats.packets;
1034                         stats->q_ibytes[i] = rxvq->stats.bytes;
1035                 }
1036         }
1037
1038         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1039 }
1040
1041 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
1042                                        struct rte_eth_xstat_name *xstats_names,
1043                                        __rte_unused unsigned limit)
1044 {
1045         unsigned i;
1046         unsigned count = 0;
1047         unsigned t;
1048
1049         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1050                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1051
1052         if (xstats_names != NULL) {
1053                 /* Note: limit checked in rte_eth_xstats_names() */
1054
1055                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1056                         struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1057                         if (rxvq == NULL)
1058                                 continue;
1059                         for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1060                                 snprintf(xstats_names[count].name,
1061                                         sizeof(xstats_names[count].name),
1062                                         "rx_q%u_%s", i,
1063                                         rte_virtio_rxq_stat_strings[t].name);
1064                                 count++;
1065                         }
1066                 }
1067
1068                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1069                         struct virtnet_tx *txvq = dev->data->tx_queues[i];
1070                         if (txvq == NULL)
1071                                 continue;
1072                         for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1073                                 snprintf(xstats_names[count].name,
1074                                         sizeof(xstats_names[count].name),
1075                                         "tx_q%u_%s", i,
1076                                         rte_virtio_txq_stat_strings[t].name);
1077                                 count++;
1078                         }
1079                 }
1080                 return count;
1081         }
1082         return nstats;
1083 }
1084
1085 static int
1086 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1087                       unsigned n)
1088 {
1089         unsigned i;
1090         unsigned count = 0;
1091
1092         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1093                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1094
1095         if (n < nstats)
1096                 return nstats;
1097
1098         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1099                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1100
1101                 if (rxvq == NULL)
1102                         continue;
1103
1104                 unsigned t;
1105
1106                 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1107                         xstats[count].value = *(uint64_t *)(((char *)rxvq) +
1108                                 rte_virtio_rxq_stat_strings[t].offset);
1109                         xstats[count].id = count;
1110                         count++;
1111                 }
1112         }
1113
1114         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1115                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1116
1117                 if (txvq == NULL)
1118                         continue;
1119
1120                 unsigned t;
1121
1122                 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1123                         xstats[count].value = *(uint64_t *)(((char *)txvq) +
1124                                 rte_virtio_txq_stat_strings[t].offset);
1125                         xstats[count].id = count;
1126                         count++;
1127                 }
1128         }
1129
1130         return count;
1131 }
1132
1133 static int
1134 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1135 {
1136         virtio_update_stats(dev, stats);
1137
1138         return 0;
1139 }
1140
1141 static int
1142 virtio_dev_stats_reset(struct rte_eth_dev *dev)
1143 {
1144         unsigned int i;
1145
1146         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1147                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1148                 if (txvq == NULL)
1149                         continue;
1150
1151                 txvq->stats.packets = 0;
1152                 txvq->stats.bytes = 0;
1153                 txvq->stats.multicast = 0;
1154                 txvq->stats.broadcast = 0;
1155                 memset(txvq->stats.size_bins, 0,
1156                        sizeof(txvq->stats.size_bins[0]) * 8);
1157         }
1158
1159         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1160                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1161                 if (rxvq == NULL)
1162                         continue;
1163
1164                 rxvq->stats.packets = 0;
1165                 rxvq->stats.bytes = 0;
1166                 rxvq->stats.errors = 0;
1167                 rxvq->stats.multicast = 0;
1168                 rxvq->stats.broadcast = 0;
1169                 memset(rxvq->stats.size_bins, 0,
1170                        sizeof(rxvq->stats.size_bins[0]) * 8);
1171         }
1172
1173         return 0;
1174 }
1175
1176 static void
1177 virtio_set_hwaddr(struct virtio_hw *hw)
1178 {
1179         virtio_write_dev_config(hw,
1180                         offsetof(struct virtio_net_config, mac),
1181                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1182 }
1183
1184 static void
1185 virtio_get_hwaddr(struct virtio_hw *hw)
1186 {
1187         if (virtio_with_feature(hw, VIRTIO_NET_F_MAC)) {
1188                 virtio_read_dev_config(hw,
1189                         offsetof(struct virtio_net_config, mac),
1190                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1191         } else {
1192                 rte_eth_random_addr(&hw->mac_addr[0]);
1193                 virtio_set_hwaddr(hw);
1194         }
1195 }
1196
1197 static int
1198 virtio_mac_table_set(struct virtio_hw *hw,
1199                      const struct virtio_net_ctrl_mac *uc,
1200                      const struct virtio_net_ctrl_mac *mc)
1201 {
1202         struct virtio_pmd_ctrl ctrl;
1203         int err, len[2];
1204
1205         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1206                 PMD_DRV_LOG(INFO, "host does not support mac table");
1207                 return -1;
1208         }
1209
1210         ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1211         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1212
1213         len[0] = uc->entries * RTE_ETHER_ADDR_LEN + sizeof(uc->entries);
1214         memcpy(ctrl.data, uc, len[0]);
1215
1216         len[1] = mc->entries * RTE_ETHER_ADDR_LEN + sizeof(mc->entries);
1217         memcpy(ctrl.data + len[0], mc, len[1]);
1218
1219         err = virtio_send_command(hw->cvq, &ctrl, len, 2);
1220         if (err != 0)
1221                 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
1222         return err;
1223 }
1224
1225 static int
1226 virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1227                     uint32_t index, uint32_t vmdq __rte_unused)
1228 {
1229         struct virtio_hw *hw = dev->data->dev_private;
1230         const struct rte_ether_addr *addrs = dev->data->mac_addrs;
1231         unsigned int i;
1232         struct virtio_net_ctrl_mac *uc, *mc;
1233
1234         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1235                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1236                 return -EINVAL;
1237         }
1238
1239         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1240                 sizeof(uc->entries));
1241         uc->entries = 0;
1242         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1243                 sizeof(mc->entries));
1244         mc->entries = 0;
1245
1246         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1247                 const struct rte_ether_addr *addr
1248                         = (i == index) ? mac_addr : addrs + i;
1249                 struct virtio_net_ctrl_mac *tbl
1250                         = rte_is_multicast_ether_addr(addr) ? mc : uc;
1251
1252                 memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN);
1253         }
1254
1255         return virtio_mac_table_set(hw, uc, mc);
1256 }
1257
1258 static void
1259 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1260 {
1261         struct virtio_hw *hw = dev->data->dev_private;
1262         struct rte_ether_addr *addrs = dev->data->mac_addrs;
1263         struct virtio_net_ctrl_mac *uc, *mc;
1264         unsigned int i;
1265
1266         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1267                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1268                 return;
1269         }
1270
1271         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1272                 sizeof(uc->entries));
1273         uc->entries = 0;
1274         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1275                 sizeof(mc->entries));
1276         mc->entries = 0;
1277
1278         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1279                 struct virtio_net_ctrl_mac *tbl;
1280
1281                 if (i == index || rte_is_zero_ether_addr(addrs + i))
1282                         continue;
1283
1284                 tbl = rte_is_multicast_ether_addr(addrs + i) ? mc : uc;
1285                 memcpy(&tbl->macs[tbl->entries++], addrs + i,
1286                         RTE_ETHER_ADDR_LEN);
1287         }
1288
1289         virtio_mac_table_set(hw, uc, mc);
1290 }
1291
1292 static int
1293 virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1294 {
1295         struct virtio_hw *hw = dev->data->dev_private;
1296
1297         memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1298
1299         /* Use atomic update if available */
1300         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1301                 struct virtio_pmd_ctrl ctrl;
1302                 int len = RTE_ETHER_ADDR_LEN;
1303
1304                 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1305                 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1306
1307                 memcpy(ctrl.data, mac_addr, RTE_ETHER_ADDR_LEN);
1308                 return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1309         }
1310
1311         if (!virtio_with_feature(hw, VIRTIO_NET_F_MAC))
1312                 return -ENOTSUP;
1313
1314         virtio_set_hwaddr(hw);
1315         return 0;
1316 }
1317
1318 static int
1319 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1320 {
1321         struct virtio_hw *hw = dev->data->dev_private;
1322         struct virtio_pmd_ctrl ctrl;
1323         int len;
1324
1325         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1326                 return -ENOTSUP;
1327
1328         ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1329         ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1330         memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1331         len = sizeof(vlan_id);
1332
1333         return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1334 }
1335
1336 static int
1337 virtio_intr_unmask(struct rte_eth_dev *dev)
1338 {
1339         struct virtio_hw *hw = dev->data->dev_private;
1340
1341         if (rte_intr_ack(dev->intr_handle) < 0)
1342                 return -1;
1343
1344         if (VIRTIO_OPS(hw)->intr_detect)
1345                 VIRTIO_OPS(hw)->intr_detect(hw);
1346
1347         return 0;
1348 }
1349
1350 static int
1351 virtio_intr_enable(struct rte_eth_dev *dev)
1352 {
1353         struct virtio_hw *hw = dev->data->dev_private;
1354
1355         if (rte_intr_enable(dev->intr_handle) < 0)
1356                 return -1;
1357
1358         if (VIRTIO_OPS(hw)->intr_detect)
1359                 VIRTIO_OPS(hw)->intr_detect(hw);
1360
1361         return 0;
1362 }
1363
1364 static int
1365 virtio_intr_disable(struct rte_eth_dev *dev)
1366 {
1367         struct virtio_hw *hw = dev->data->dev_private;
1368
1369         if (rte_intr_disable(dev->intr_handle) < 0)
1370                 return -1;
1371
1372         if (VIRTIO_OPS(hw)->intr_detect)
1373                 VIRTIO_OPS(hw)->intr_detect(hw);
1374
1375         return 0;
1376 }
1377
1378 static int
1379 virtio_ethdev_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1380 {
1381         uint64_t host_features;
1382
1383         /* Prepare guest_features: feature that driver wants to support */
1384         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1385                 req_features);
1386
1387         /* Read device(host) feature bits */
1388         host_features = VIRTIO_OPS(hw)->get_features(hw);
1389         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1390                 host_features);
1391
1392         /* If supported, ensure MTU value is valid before acknowledging it. */
1393         if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) {
1394                 struct virtio_net_config config;
1395
1396                 virtio_read_dev_config(hw,
1397                         offsetof(struct virtio_net_config, mtu),
1398                         &config.mtu, sizeof(config.mtu));
1399
1400                 if (config.mtu < RTE_ETHER_MIN_MTU)
1401                         req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
1402         }
1403
1404         /*
1405          * Negotiate features: Subset of device feature bits are written back
1406          * guest feature bits.
1407          */
1408         hw->guest_features = req_features;
1409         hw->guest_features = virtio_negotiate_features(hw, host_features);
1410         PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1411                 hw->guest_features);
1412
1413         if (VIRTIO_OPS(hw)->features_ok(hw) < 0)
1414                 return -1;
1415
1416         if (virtio_with_feature(hw, VIRTIO_F_VERSION_1)) {
1417                 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1418
1419                 if (!(virtio_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1420                         PMD_INIT_LOG(ERR, "Failed to set FEATURES_OK status!");
1421                         return -1;
1422                 }
1423         }
1424
1425         hw->req_guest_features = req_features;
1426
1427         return 0;
1428 }
1429
1430 int
1431 virtio_dev_pause(struct rte_eth_dev *dev)
1432 {
1433         struct virtio_hw *hw = dev->data->dev_private;
1434
1435         rte_spinlock_lock(&hw->state_lock);
1436
1437         if (hw->started == 0) {
1438                 /* Device is just stopped. */
1439                 rte_spinlock_unlock(&hw->state_lock);
1440                 return -1;
1441         }
1442         hw->started = 0;
1443         /*
1444          * Prevent the worker threads from touching queues to avoid contention,
1445          * 1 ms should be enough for the ongoing Tx function to finish.
1446          */
1447         rte_delay_ms(1);
1448         return 0;
1449 }
1450
1451 /*
1452  * Recover hw state to let the worker threads continue.
1453  */
1454 void
1455 virtio_dev_resume(struct rte_eth_dev *dev)
1456 {
1457         struct virtio_hw *hw = dev->data->dev_private;
1458
1459         hw->started = 1;
1460         rte_spinlock_unlock(&hw->state_lock);
1461 }
1462
1463 /*
1464  * Should be called only after device is paused.
1465  */
1466 int
1467 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts,
1468                 int nb_pkts)
1469 {
1470         struct virtio_hw *hw = dev->data->dev_private;
1471         struct virtnet_tx *txvq = dev->data->tx_queues[0];
1472         int ret;
1473
1474         hw->inject_pkts = tx_pkts;
1475         ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts);
1476         hw->inject_pkts = NULL;
1477
1478         return ret;
1479 }
1480
1481 static void
1482 virtio_notify_peers(struct rte_eth_dev *dev)
1483 {
1484         struct virtio_hw *hw = dev->data->dev_private;
1485         struct virtnet_rx *rxvq;
1486         struct rte_mbuf *rarp_mbuf;
1487
1488         if (!dev->data->rx_queues)
1489                 return;
1490
1491         rxvq = dev->data->rx_queues[0];
1492         if (!rxvq)
1493                 return;
1494
1495         rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool,
1496                         (struct rte_ether_addr *)hw->mac_addr);
1497         if (rarp_mbuf == NULL) {
1498                 PMD_DRV_LOG(ERR, "failed to make RARP packet.");
1499                 return;
1500         }
1501
1502         /* If virtio port just stopped, no need to send RARP */
1503         if (virtio_dev_pause(dev) < 0) {
1504                 rte_pktmbuf_free(rarp_mbuf);
1505                 return;
1506         }
1507
1508         virtio_inject_pkts(dev, &rarp_mbuf, 1);
1509         virtio_dev_resume(dev);
1510 }
1511
1512 static void
1513 virtio_ack_link_announce(struct rte_eth_dev *dev)
1514 {
1515         struct virtio_hw *hw = dev->data->dev_private;
1516         struct virtio_pmd_ctrl ctrl;
1517
1518         ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE;
1519         ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK;
1520
1521         virtio_send_command(hw->cvq, &ctrl, NULL, 0);
1522 }
1523
1524 /*
1525  * Process virtio config changed interrupt. Call the callback
1526  * if link state changed, generate gratuitous RARP packet if
1527  * the status indicates an ANNOUNCE.
1528  */
1529 void
1530 virtio_interrupt_handler(void *param)
1531 {
1532         struct rte_eth_dev *dev = param;
1533         struct virtio_hw *hw = dev->data->dev_private;
1534         uint8_t isr;
1535         uint16_t status;
1536
1537         /* Read interrupt status which clears interrupt */
1538         isr = virtio_get_isr(hw);
1539         PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1540
1541         if (virtio_intr_unmask(dev) < 0)
1542                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1543
1544         if (isr & VIRTIO_ISR_CONFIG) {
1545                 if (virtio_dev_link_update(dev, 0) == 0)
1546                         rte_eth_dev_callback_process(dev,
1547                                                      RTE_ETH_EVENT_INTR_LSC,
1548                                                      NULL);
1549
1550                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1551                         virtio_read_dev_config(hw,
1552                                 offsetof(struct virtio_net_config, status),
1553                                 &status, sizeof(status));
1554                         if (status & VIRTIO_NET_S_ANNOUNCE) {
1555                                 virtio_notify_peers(dev);
1556                                 if (hw->cvq)
1557                                         virtio_ack_link_announce(dev);
1558                         }
1559                 }
1560         }
1561 }
1562
1563 /* set rx and tx handlers according to what is supported */
1564 static void
1565 set_rxtx_funcs(struct rte_eth_dev *eth_dev)
1566 {
1567         struct virtio_hw *hw = eth_dev->data->dev_private;
1568
1569         eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare;
1570         if (virtio_with_packed_queue(hw)) {
1571                 PMD_INIT_LOG(INFO,
1572                         "virtio: using packed ring %s Tx path on port %u",
1573                         hw->use_vec_tx ? "vectorized" : "standard",
1574                         eth_dev->data->port_id);
1575                 if (hw->use_vec_tx)
1576                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
1577                 else
1578                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
1579         } else {
1580                 if (hw->use_inorder_tx) {
1581                         PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u",
1582                                 eth_dev->data->port_id);
1583                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder;
1584                 } else {
1585                         PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u",
1586                                 eth_dev->data->port_id);
1587                         eth_dev->tx_pkt_burst = virtio_xmit_pkts;
1588                 }
1589         }
1590
1591         if (virtio_with_packed_queue(hw)) {
1592                 if (hw->use_vec_rx) {
1593                         PMD_INIT_LOG(INFO,
1594                                 "virtio: using packed ring vectorized Rx path on port %u",
1595                                 eth_dev->data->port_id);
1596                         eth_dev->rx_pkt_burst =
1597                                 &virtio_recv_pkts_packed_vec;
1598                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1599                         PMD_INIT_LOG(INFO,
1600                                 "virtio: using packed ring mergeable buffer Rx path on port %u",
1601                                 eth_dev->data->port_id);
1602                         eth_dev->rx_pkt_burst =
1603                                 &virtio_recv_mergeable_pkts_packed;
1604                 } else {
1605                         PMD_INIT_LOG(INFO,
1606                                 "virtio: using packed ring standard Rx path on port %u",
1607                                 eth_dev->data->port_id);
1608                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
1609                 }
1610         } else {
1611                 if (hw->use_vec_rx) {
1612                         PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on port %u",
1613                                 eth_dev->data->port_id);
1614                         eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
1615                 } else if (hw->use_inorder_rx) {
1616                         PMD_INIT_LOG(INFO,
1617                                 "virtio: using inorder Rx path on port %u",
1618                                 eth_dev->data->port_id);
1619                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder;
1620                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1621                         PMD_INIT_LOG(INFO,
1622                                 "virtio: using mergeable buffer Rx path on port %u",
1623                                 eth_dev->data->port_id);
1624                         eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1625                 } else {
1626                         PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
1627                                 eth_dev->data->port_id);
1628                         eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1629                 }
1630         }
1631
1632 }
1633
1634 /* Only support 1:1 queue/interrupt mapping so far.
1635  * TODO: support n:1 queue/interrupt mapping when there are limited number of
1636  * interrupt vectors (<N+1).
1637  */
1638 static int
1639 virtio_queues_bind_intr(struct rte_eth_dev *dev)
1640 {
1641         uint32_t i;
1642         struct virtio_hw *hw = dev->data->dev_private;
1643
1644         PMD_INIT_LOG(INFO, "queue/interrupt binding");
1645         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
1646                 dev->intr_handle->intr_vec[i] = i + 1;
1647                 if (VIRTIO_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) ==
1648                                                  VIRTIO_MSI_NO_VECTOR) {
1649                         PMD_DRV_LOG(ERR, "failed to set queue vector");
1650                         return -EBUSY;
1651                 }
1652         }
1653
1654         return 0;
1655 }
1656
1657 static void
1658 virtio_queues_unbind_intr(struct rte_eth_dev *dev)
1659 {
1660         uint32_t i;
1661         struct virtio_hw *hw = dev->data->dev_private;
1662
1663         PMD_INIT_LOG(INFO, "queue/interrupt unbinding");
1664         for (i = 0; i < dev->data->nb_rx_queues; ++i)
1665                 VIRTIO_OPS(hw)->set_queue_irq(hw,
1666                                              hw->vqs[i * VTNET_CQ],
1667                                              VIRTIO_MSI_NO_VECTOR);
1668 }
1669
1670 static int
1671 virtio_configure_intr(struct rte_eth_dev *dev)
1672 {
1673         struct virtio_hw *hw = dev->data->dev_private;
1674
1675         if (!rte_intr_cap_multiple(dev->intr_handle)) {
1676                 PMD_INIT_LOG(ERR, "Multiple intr vector not supported");
1677                 return -ENOTSUP;
1678         }
1679
1680         if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) {
1681                 PMD_INIT_LOG(ERR, "Fail to create eventfd");
1682                 return -1;
1683         }
1684
1685         if (!dev->intr_handle->intr_vec) {
1686                 dev->intr_handle->intr_vec =
1687                         rte_zmalloc("intr_vec",
1688                                     hw->max_queue_pairs * sizeof(int), 0);
1689                 if (!dev->intr_handle->intr_vec) {
1690                         PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors",
1691                                      hw->max_queue_pairs);
1692                         return -ENOMEM;
1693                 }
1694         }
1695
1696         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1697                 /* Re-register callback to update max_intr */
1698                 rte_intr_callback_unregister(dev->intr_handle,
1699                                              virtio_interrupt_handler,
1700                                              dev);
1701                 rte_intr_callback_register(dev->intr_handle,
1702                                            virtio_interrupt_handler,
1703                                            dev);
1704         }
1705
1706         /* DO NOT try to remove this! This function will enable msix, or QEMU
1707          * will encounter SIGSEGV when DRIVER_OK is sent.
1708          * And for legacy devices, this should be done before queue/vec binding
1709          * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1710          * (22) will be ignored.
1711          */
1712         if (virtio_intr_enable(dev) < 0) {
1713                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1714                 return -1;
1715         }
1716
1717         if (virtio_queues_bind_intr(dev) < 0) {
1718                 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt");
1719                 return -1;
1720         }
1721
1722         return 0;
1723 }
1724 #define DUPLEX_UNKNOWN   0xff
1725 /* reset device and renegotiate features if needed */
1726 static int
1727 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
1728 {
1729         struct virtio_hw *hw = eth_dev->data->dev_private;
1730         struct virtio_net_config *config;
1731         struct virtio_net_config local_config;
1732         int ret;
1733
1734         /* Reset the device although not necessary at startup */
1735         virtio_reset(hw);
1736
1737         if (hw->vqs) {
1738                 virtio_dev_free_mbufs(eth_dev);
1739                 virtio_free_queues(hw);
1740         }
1741
1742         /* Tell the host we've noticed this device. */
1743         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1744
1745         /* Tell the host we've known how to drive the device. */
1746         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1747         if (virtio_ethdev_negotiate_features(hw, req_features) < 0)
1748                 return -1;
1749
1750         hw->weak_barriers = !virtio_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
1751
1752         /* If host does not support both status and MSI-X then disable LSC */
1753         if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS) && hw->intr_lsc)
1754                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
1755         else
1756                 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
1757
1758         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
1759
1760         /* Setting up rx_header size for the device */
1761         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
1762             virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
1763             virtio_with_packed_queue(hw))
1764                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1765         else
1766                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
1767
1768         /* Copy the permanent MAC address to: virtio_hw */
1769         virtio_get_hwaddr(hw);
1770         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
1771                         &eth_dev->data->mac_addrs[0]);
1772         PMD_INIT_LOG(DEBUG,
1773                      "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
1774                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1775                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1776
1777         if (hw->speed == RTE_ETH_SPEED_NUM_UNKNOWN) {
1778                 if (virtio_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) {
1779                         config = &local_config;
1780                         virtio_read_dev_config(hw,
1781                                 offsetof(struct virtio_net_config, speed),
1782                                 &config->speed, sizeof(config->speed));
1783                         virtio_read_dev_config(hw,
1784                                 offsetof(struct virtio_net_config, duplex),
1785                                 &config->duplex, sizeof(config->duplex));
1786                         hw->speed = config->speed;
1787                         hw->duplex = config->duplex;
1788                 }
1789         }
1790         if (hw->duplex == DUPLEX_UNKNOWN)
1791                 hw->duplex = RTE_ETH_LINK_FULL_DUPLEX;
1792         PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d",
1793                 hw->speed, hw->duplex);
1794         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1795                 config = &local_config;
1796
1797                 virtio_read_dev_config(hw,
1798                         offsetof(struct virtio_net_config, mac),
1799                         &config->mac, sizeof(config->mac));
1800
1801                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1802                         virtio_read_dev_config(hw,
1803                                 offsetof(struct virtio_net_config, status),
1804                                 &config->status, sizeof(config->status));
1805                 } else {
1806                         PMD_INIT_LOG(DEBUG,
1807                                      "VIRTIO_NET_F_STATUS is not supported");
1808                         config->status = 0;
1809                 }
1810
1811                 if (virtio_with_feature(hw, VIRTIO_NET_F_MQ)) {
1812                         virtio_read_dev_config(hw,
1813                                 offsetof(struct virtio_net_config, max_virtqueue_pairs),
1814                                 &config->max_virtqueue_pairs,
1815                                 sizeof(config->max_virtqueue_pairs));
1816                 } else {
1817                         PMD_INIT_LOG(DEBUG,
1818                                      "VIRTIO_NET_F_MQ is not supported");
1819                         config->max_virtqueue_pairs = 1;
1820                 }
1821
1822                 hw->max_queue_pairs = config->max_virtqueue_pairs;
1823
1824                 if (virtio_with_feature(hw, VIRTIO_NET_F_MTU)) {
1825                         virtio_read_dev_config(hw,
1826                                 offsetof(struct virtio_net_config, mtu),
1827                                 &config->mtu,
1828                                 sizeof(config->mtu));
1829
1830                         /*
1831                          * MTU value has already been checked at negotiation
1832                          * time, but check again in case it has changed since
1833                          * then, which should not happen.
1834                          */
1835                         if (config->mtu < RTE_ETHER_MIN_MTU) {
1836                                 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)",
1837                                                 config->mtu);
1838                                 return -1;
1839                         }
1840
1841                         hw->max_mtu = config->mtu;
1842                         /* Set initial MTU to maximum one supported by vhost */
1843                         eth_dev->data->mtu = config->mtu;
1844
1845                 } else {
1846                         hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1847                                 VLAN_TAG_LEN - hw->vtnet_hdr_size;
1848                 }
1849
1850                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
1851                                 config->max_virtqueue_pairs);
1852                 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
1853                 PMD_INIT_LOG(DEBUG,
1854                                 "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
1855                                 config->mac[0], config->mac[1],
1856                                 config->mac[2], config->mac[3],
1857                                 config->mac[4], config->mac[5]);
1858         } else {
1859                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
1860                 hw->max_queue_pairs = 1;
1861                 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
1862                         VLAN_TAG_LEN - hw->vtnet_hdr_size;
1863         }
1864
1865         ret = virtio_alloc_queues(eth_dev);
1866         if (ret < 0)
1867                 return ret;
1868
1869         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1870                 if (virtio_configure_intr(eth_dev) < 0) {
1871                         PMD_INIT_LOG(ERR, "failed to configure interrupt");
1872                         virtio_free_queues(hw);
1873                         return -1;
1874                 }
1875         }
1876
1877         virtio_reinit_complete(hw);
1878
1879         return 0;
1880 }
1881
1882 /*
1883  * This function is based on probe() function in virtio_pci.c
1884  * It returns 0 on success.
1885  */
1886 int
1887 eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
1888 {
1889         struct virtio_hw *hw = eth_dev->data->dev_private;
1890         uint32_t speed = RTE_ETH_SPEED_NUM_UNKNOWN;
1891         int vectorized = 0;
1892         int ret;
1893
1894         if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
1895                 PMD_INIT_LOG(ERR,
1896                         "Not sufficient headroom required = %d, avail = %d",
1897                         (int)sizeof(struct virtio_net_hdr_mrg_rxbuf),
1898                         RTE_PKTMBUF_HEADROOM);
1899
1900                 return -1;
1901         }
1902
1903         eth_dev->dev_ops = &virtio_eth_dev_ops;
1904
1905         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1906                 set_rxtx_funcs(eth_dev);
1907                 return 0;
1908         }
1909
1910         ret = virtio_dev_devargs_parse(eth_dev->device->devargs, &speed, &vectorized);
1911         if (ret < 0)
1912                 return ret;
1913         hw->speed = speed;
1914         hw->duplex = DUPLEX_UNKNOWN;
1915
1916         /* Allocate memory for storing MAC addresses */
1917         eth_dev->data->mac_addrs = rte_zmalloc("virtio",
1918                                 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN, 0);
1919         if (eth_dev->data->mac_addrs == NULL) {
1920                 PMD_INIT_LOG(ERR,
1921                         "Failed to allocate %d bytes needed to store MAC addresses",
1922                         VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN);
1923                 return -ENOMEM;
1924         }
1925
1926         rte_spinlock_init(&hw->state_lock);
1927
1928         /* reset device and negotiate default features */
1929         ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
1930         if (ret < 0)
1931                 goto err_virtio_init;
1932
1933         if (vectorized) {
1934                 if (!virtio_with_packed_queue(hw)) {
1935                         hw->use_vec_rx = 1;
1936                 } else {
1937 #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
1938                         hw->use_vec_rx = 1;
1939                         hw->use_vec_tx = 1;
1940 #else
1941                         PMD_DRV_LOG(INFO,
1942                                 "building environment do not support packed ring vectorized");
1943 #endif
1944                 }
1945         }
1946
1947         hw->opened = 1;
1948
1949         return 0;
1950
1951 err_virtio_init:
1952         rte_free(eth_dev->data->mac_addrs);
1953         eth_dev->data->mac_addrs = NULL;
1954         return ret;
1955 }
1956
1957 static uint32_t
1958 virtio_dev_speed_capa_get(uint32_t speed)
1959 {
1960         switch (speed) {
1961         case RTE_ETH_SPEED_NUM_10G:
1962                 return RTE_ETH_LINK_SPEED_10G;
1963         case RTE_ETH_SPEED_NUM_20G:
1964                 return RTE_ETH_LINK_SPEED_20G;
1965         case RTE_ETH_SPEED_NUM_25G:
1966                 return RTE_ETH_LINK_SPEED_25G;
1967         case RTE_ETH_SPEED_NUM_40G:
1968                 return RTE_ETH_LINK_SPEED_40G;
1969         case RTE_ETH_SPEED_NUM_50G:
1970                 return RTE_ETH_LINK_SPEED_50G;
1971         case RTE_ETH_SPEED_NUM_56G:
1972                 return RTE_ETH_LINK_SPEED_56G;
1973         case RTE_ETH_SPEED_NUM_100G:
1974                 return RTE_ETH_LINK_SPEED_100G;
1975         case RTE_ETH_SPEED_NUM_200G:
1976                 return RTE_ETH_LINK_SPEED_200G;
1977         default:
1978                 return 0;
1979         }
1980 }
1981
1982 static int vectorized_check_handler(__rte_unused const char *key,
1983                 const char *value, void *ret_val)
1984 {
1985         if (strcmp(value, "1") == 0)
1986                 *(int *)ret_val = 1;
1987         else
1988                 *(int *)ret_val = 0;
1989
1990         return 0;
1991 }
1992
1993 #define VIRTIO_ARG_SPEED      "speed"
1994 #define VIRTIO_ARG_VECTORIZED "vectorized"
1995
1996 static int
1997 link_speed_handler(const char *key __rte_unused,
1998                 const char *value, void *ret_val)
1999 {
2000         uint32_t val;
2001         if (!value || !ret_val)
2002                 return -EINVAL;
2003         val = strtoul(value, NULL, 0);
2004         /* validate input */
2005         if (virtio_dev_speed_capa_get(val) == 0)
2006                 return -EINVAL;
2007         *(uint32_t *)ret_val = val;
2008
2009         return 0;
2010 }
2011
2012
2013 static int
2014 virtio_dev_devargs_parse(struct rte_devargs *devargs, uint32_t *speed, int *vectorized)
2015 {
2016         struct rte_kvargs *kvlist;
2017         int ret = 0;
2018
2019         if (devargs == NULL)
2020                 return 0;
2021
2022         kvlist = rte_kvargs_parse(devargs->args, NULL);
2023         if (kvlist == NULL) {
2024                 PMD_INIT_LOG(ERR, "error when parsing param");
2025                 return 0;
2026         }
2027
2028         if (speed && rte_kvargs_count(kvlist, VIRTIO_ARG_SPEED) == 1) {
2029                 ret = rte_kvargs_process(kvlist,
2030                                         VIRTIO_ARG_SPEED,
2031                                         link_speed_handler, speed);
2032                 if (ret < 0) {
2033                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2034                                         VIRTIO_ARG_SPEED);
2035                         goto exit;
2036                 }
2037         }
2038
2039         if (vectorized &&
2040                 rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
2041                 ret = rte_kvargs_process(kvlist,
2042                                 VIRTIO_ARG_VECTORIZED,
2043                                 vectorized_check_handler, vectorized);
2044                 if (ret < 0) {
2045                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2046                                         VIRTIO_ARG_VECTORIZED);
2047                         goto exit;
2048                 }
2049         }
2050
2051 exit:
2052         rte_kvargs_free(kvlist);
2053         return ret;
2054 }
2055
2056 static uint8_t
2057 rx_offload_enabled(struct virtio_hw *hw)
2058 {
2059         return virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
2060                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2061                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
2062 }
2063
2064 static uint8_t
2065 tx_offload_enabled(struct virtio_hw *hw)
2066 {
2067         return virtio_with_feature(hw, VIRTIO_NET_F_CSUM) ||
2068                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
2069                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
2070 }
2071
2072 /*
2073  * Configure virtio device
2074  * It returns 0 on success.
2075  */
2076 static int
2077 virtio_dev_configure(struct rte_eth_dev *dev)
2078 {
2079         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2080         const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
2081         struct virtio_hw *hw = dev->data->dev_private;
2082         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
2083                 hw->vtnet_hdr_size;
2084         uint64_t rx_offloads = rxmode->offloads;
2085         uint64_t tx_offloads = txmode->offloads;
2086         uint64_t req_features;
2087         int ret;
2088
2089         PMD_INIT_LOG(DEBUG, "configure");
2090         req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
2091
2092         if (rxmode->mq_mode != RTE_ETH_MQ_RX_NONE) {
2093                 PMD_DRV_LOG(ERR,
2094                         "Unsupported Rx multi queue mode %d",
2095                         rxmode->mq_mode);
2096                 return -EINVAL;
2097         }
2098
2099         if (txmode->mq_mode != RTE_ETH_MQ_TX_NONE) {
2100                 PMD_DRV_LOG(ERR,
2101                         "Unsupported Tx multi queue mode %d",
2102                         txmode->mq_mode);
2103                 return -EINVAL;
2104         }
2105
2106         if (dev->data->dev_conf.intr_conf.rxq) {
2107                 ret = virtio_init_device(dev, hw->req_guest_features);
2108                 if (ret < 0)
2109                         return ret;
2110         }
2111
2112         if (rxmode->mtu > hw->max_mtu)
2113                 req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
2114
2115         hw->max_rx_pkt_len = ether_hdr_len + rxmode->mtu;
2116
2117         if (rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2118                            RTE_ETH_RX_OFFLOAD_TCP_CKSUM))
2119                 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
2120
2121         if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)
2122                 req_features |=
2123                         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2124                         (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2125
2126         if (tx_offloads & (RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2127                            RTE_ETH_TX_OFFLOAD_TCP_CKSUM))
2128                 req_features |= (1ULL << VIRTIO_NET_F_CSUM);
2129
2130         if (tx_offloads & RTE_ETH_TX_OFFLOAD_TCP_TSO)
2131                 req_features |=
2132                         (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2133                         (1ULL << VIRTIO_NET_F_HOST_TSO6);
2134
2135         /* if request features changed, reinit the device */
2136         if (req_features != hw->req_guest_features) {
2137                 ret = virtio_init_device(dev, req_features);
2138                 if (ret < 0)
2139                         return ret;
2140         }
2141
2142         if ((rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2143                             RTE_ETH_RX_OFFLOAD_TCP_CKSUM)) &&
2144                 !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
2145                 PMD_DRV_LOG(ERR,
2146                         "rx checksum not available on this host");
2147                 return -ENOTSUP;
2148         }
2149
2150         if ((rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) &&
2151                 (!virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2152                  !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) {
2153                 PMD_DRV_LOG(ERR,
2154                         "Large Receive Offload not available on this host");
2155                 return -ENOTSUP;
2156         }
2157
2158         /* start control queue */
2159         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
2160                 virtio_dev_cq_start(dev);
2161
2162         if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
2163                 hw->vlan_strip = 1;
2164
2165         hw->rx_ol_scatter = (rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER);
2166
2167         if ((rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) &&
2168                         !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2169                 PMD_DRV_LOG(ERR,
2170                             "vlan filtering not available on this host");
2171                 return -ENOTSUP;
2172         }
2173
2174         hw->has_tx_offload = tx_offload_enabled(hw);
2175         hw->has_rx_offload = rx_offload_enabled(hw);
2176
2177         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2178                 /* Enable vector (0) for Link State Intrerrupt */
2179                 if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) ==
2180                                 VIRTIO_MSI_NO_VECTOR) {
2181                         PMD_DRV_LOG(ERR, "failed to set config vector");
2182                         return -EBUSY;
2183                 }
2184
2185         if (virtio_with_packed_queue(hw)) {
2186 #if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT)
2187                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2188                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
2189                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2190                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2191                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_512)) {
2192                         PMD_DRV_LOG(INFO,
2193                                 "disabled packed ring vectorized path for requirements not met");
2194                         hw->use_vec_rx = 0;
2195                         hw->use_vec_tx = 0;
2196                 }
2197 #elif defined(RTE_ARCH_ARM)
2198                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2199                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) ||
2200                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2201                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2202                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)) {
2203                         PMD_DRV_LOG(INFO,
2204                                 "disabled packed ring vectorized path for requirements not met");
2205                         hw->use_vec_rx = 0;
2206                         hw->use_vec_tx = 0;
2207                 }
2208 #else
2209                 hw->use_vec_rx = 0;
2210                 hw->use_vec_tx = 0;
2211 #endif
2212
2213                 if (hw->use_vec_rx) {
2214                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2215                                 PMD_DRV_LOG(INFO,
2216                                         "disabled packed ring vectorized rx for mrg_rxbuf enabled");
2217                                 hw->use_vec_rx = 0;
2218                         }
2219
2220                         if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) {
2221                                 PMD_DRV_LOG(INFO,
2222                                         "disabled packed ring vectorized rx for TCP_LRO enabled");
2223                                 hw->use_vec_rx = 0;
2224                         }
2225                 }
2226         } else {
2227                 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) {
2228                         hw->use_inorder_tx = 1;
2229                         hw->use_inorder_rx = 1;
2230                         hw->use_vec_rx = 0;
2231                 }
2232
2233                 if (hw->use_vec_rx) {
2234 #if defined RTE_ARCH_ARM
2235                         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
2236                                 PMD_DRV_LOG(INFO,
2237                                         "disabled split ring vectorized path for requirement not met");
2238                                 hw->use_vec_rx = 0;
2239                         }
2240 #endif
2241                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2242                                 PMD_DRV_LOG(INFO,
2243                                         "disabled split ring vectorized rx for mrg_rxbuf enabled");
2244                                 hw->use_vec_rx = 0;
2245                         }
2246
2247                         if (rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2248                                            RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
2249                                            RTE_ETH_RX_OFFLOAD_TCP_LRO |
2250                                            RTE_ETH_RX_OFFLOAD_VLAN_STRIP)) {
2251                                 PMD_DRV_LOG(INFO,
2252                                         "disabled split ring vectorized rx for offloading enabled");
2253                                 hw->use_vec_rx = 0;
2254                         }
2255
2256                         if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
2257                                 PMD_DRV_LOG(INFO,
2258                                         "disabled split ring vectorized rx, max SIMD bitwidth too low");
2259                                 hw->use_vec_rx = 0;
2260                         }
2261                 }
2262         }
2263
2264         return 0;
2265 }
2266
2267
2268 static int
2269 virtio_dev_start(struct rte_eth_dev *dev)
2270 {
2271         uint16_t nb_queues, i;
2272         struct virtqueue *vq;
2273         struct virtio_hw *hw = dev->data->dev_private;
2274         int ret;
2275
2276         /* Finish the initialization of the queues */
2277         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2278                 ret = virtio_dev_rx_queue_setup_finish(dev, i);
2279                 if (ret < 0)
2280                         return ret;
2281         }
2282         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2283                 ret = virtio_dev_tx_queue_setup_finish(dev, i);
2284                 if (ret < 0)
2285                         return ret;
2286         }
2287
2288         /* check if lsc interrupt feature is enabled */
2289         if (dev->data->dev_conf.intr_conf.lsc) {
2290                 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
2291                         PMD_DRV_LOG(ERR, "link status not supported by host");
2292                         return -ENOTSUP;
2293                 }
2294         }
2295
2296         /* Enable uio/vfio intr/eventfd mapping: althrough we already did that
2297          * in device configure, but it could be unmapped  when device is
2298          * stopped.
2299          */
2300         if (dev->data->dev_conf.intr_conf.lsc ||
2301             dev->data->dev_conf.intr_conf.rxq) {
2302                 virtio_intr_disable(dev);
2303
2304                 /* Setup interrupt callback  */
2305                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2306                         rte_intr_callback_register(dev->intr_handle,
2307                                                    virtio_interrupt_handler,
2308                                                    dev);
2309
2310                 if (virtio_intr_enable(dev) < 0) {
2311                         PMD_DRV_LOG(ERR, "interrupt enable failed");
2312                         return -EIO;
2313                 }
2314         }
2315
2316         /*Notify the backend
2317          *Otherwise the tap backend might already stop its queue due to fullness.
2318          *vhost backend will have no chance to be waked up
2319          */
2320         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
2321         if (hw->max_queue_pairs > 1) {
2322                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
2323                         return -EINVAL;
2324         }
2325
2326         PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
2327
2328         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2329                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2330                 /* Flush the old packets */
2331                 virtqueue_rxvq_flush(vq);
2332                 virtqueue_notify(vq);
2333         }
2334
2335         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2336                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2337                 virtqueue_notify(vq);
2338         }
2339
2340         PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
2341
2342         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2343                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2344                 VIRTQUEUE_DUMP(vq);
2345         }
2346
2347         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2348                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2349                 VIRTQUEUE_DUMP(vq);
2350         }
2351
2352         set_rxtx_funcs(dev);
2353         hw->started = 1;
2354
2355         /* Initialize Link state */
2356         virtio_dev_link_update(dev, 0);
2357
2358         return 0;
2359 }
2360
2361 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
2362 {
2363         struct virtio_hw *hw = dev->data->dev_private;
2364         uint16_t nr_vq = virtio_get_nr_vq(hw);
2365         const char *type __rte_unused;
2366         unsigned int i, mbuf_num = 0;
2367         struct virtqueue *vq;
2368         struct rte_mbuf *buf;
2369         int queue_type;
2370
2371         if (hw->vqs == NULL)
2372                 return;
2373
2374         for (i = 0; i < nr_vq; i++) {
2375                 vq = hw->vqs[i];
2376                 if (!vq)
2377                         continue;
2378
2379                 queue_type = virtio_get_queue_type(hw, i);
2380                 if (queue_type == VTNET_RQ)
2381                         type = "rxq";
2382                 else if (queue_type == VTNET_TQ)
2383                         type = "txq";
2384                 else
2385                         continue;
2386
2387                 PMD_INIT_LOG(DEBUG,
2388                         "Before freeing %s[%d] used and unused buf",
2389                         type, i);
2390                 VIRTQUEUE_DUMP(vq);
2391
2392                 while ((buf = virtqueue_detach_unused(vq)) != NULL) {
2393                         rte_pktmbuf_free(buf);
2394                         mbuf_num++;
2395                 }
2396
2397                 PMD_INIT_LOG(DEBUG,
2398                         "After freeing %s[%d] used and unused buf",
2399                         type, i);
2400                 VIRTQUEUE_DUMP(vq);
2401         }
2402
2403         PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num);
2404 }
2405
2406 static void
2407 virtio_tx_completed_cleanup(struct rte_eth_dev *dev)
2408 {
2409         struct virtio_hw *hw = dev->data->dev_private;
2410         struct virtqueue *vq;
2411         int qidx;
2412         void (*xmit_cleanup)(struct virtqueue *vq, uint16_t nb_used);
2413
2414         if (virtio_with_packed_queue(hw)) {
2415                 if (hw->use_vec_tx)
2416                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2417                 else if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER))
2418                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2419                 else
2420                         xmit_cleanup = &virtio_xmit_cleanup_normal_packed;
2421         } else {
2422                 if (hw->use_inorder_tx)
2423                         xmit_cleanup = &virtio_xmit_cleanup_inorder;
2424                 else
2425                         xmit_cleanup = &virtio_xmit_cleanup;
2426         }
2427
2428         for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) {
2429                 vq = hw->vqs[2 * qidx + VTNET_SQ_TQ_QUEUE_IDX];
2430                 if (vq != NULL)
2431                         xmit_cleanup(vq, virtqueue_nused(vq));
2432         }
2433 }
2434
2435 /*
2436  * Stop device: disable interrupt and mark link down
2437  */
2438 int
2439 virtio_dev_stop(struct rte_eth_dev *dev)
2440 {
2441         struct virtio_hw *hw = dev->data->dev_private;
2442         struct rte_eth_link link;
2443         struct rte_eth_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
2444
2445         PMD_INIT_LOG(DEBUG, "stop");
2446         dev->data->dev_started = 0;
2447
2448         rte_spinlock_lock(&hw->state_lock);
2449         if (!hw->started)
2450                 goto out_unlock;
2451         hw->started = 0;
2452
2453         virtio_tx_completed_cleanup(dev);
2454
2455         if (intr_conf->lsc || intr_conf->rxq) {
2456                 virtio_intr_disable(dev);
2457
2458                 /* Reset interrupt callback  */
2459                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
2460                         rte_intr_callback_unregister(dev->intr_handle,
2461                                                      virtio_interrupt_handler,
2462                                                      dev);
2463                 }
2464         }
2465
2466         memset(&link, 0, sizeof(link));
2467         rte_eth_linkstatus_set(dev, &link);
2468 out_unlock:
2469         rte_spinlock_unlock(&hw->state_lock);
2470
2471         return 0;
2472 }
2473
2474 static int
2475 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
2476 {
2477         struct rte_eth_link link;
2478         uint16_t status;
2479         struct virtio_hw *hw = dev->data->dev_private;
2480
2481         memset(&link, 0, sizeof(link));
2482         link.link_duplex = hw->duplex;
2483         link.link_speed  = hw->speed;
2484         link.link_autoneg = RTE_ETH_LINK_AUTONEG;
2485
2486         if (!hw->started) {
2487                 link.link_status = RTE_ETH_LINK_DOWN;
2488                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2489         } else if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
2490                 PMD_INIT_LOG(DEBUG, "Get link status from hw");
2491                 virtio_read_dev_config(hw,
2492                                 offsetof(struct virtio_net_config, status),
2493                                 &status, sizeof(status));
2494                 if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
2495                         link.link_status = RTE_ETH_LINK_DOWN;
2496                         link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2497                         PMD_INIT_LOG(DEBUG, "Port %d is down",
2498                                      dev->data->port_id);
2499                 } else {
2500                         link.link_status = RTE_ETH_LINK_UP;
2501                         PMD_INIT_LOG(DEBUG, "Port %d is up",
2502                                      dev->data->port_id);
2503                 }
2504         } else {
2505                 link.link_status = RTE_ETH_LINK_UP;
2506         }
2507
2508         return rte_eth_linkstatus_set(dev, &link);
2509 }
2510
2511 static int
2512 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2513 {
2514         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2515         struct virtio_hw *hw = dev->data->dev_private;
2516         uint64_t offloads = rxmode->offloads;
2517
2518         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
2519                 if ((offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) &&
2520                                 !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2521
2522                         PMD_DRV_LOG(NOTICE,
2523                                 "vlan filtering not available on this host");
2524
2525                         return -ENOTSUP;
2526                 }
2527         }
2528
2529         if (mask & RTE_ETH_VLAN_STRIP_MASK)
2530                 hw->vlan_strip = !!(offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP);
2531
2532         return 0;
2533 }
2534
2535 static int
2536 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2537 {
2538         uint64_t tso_mask, host_features;
2539         struct virtio_hw *hw = dev->data->dev_private;
2540         dev_info->speed_capa = virtio_dev_speed_capa_get(hw->speed);
2541
2542         dev_info->max_rx_queues =
2543                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
2544         dev_info->max_tx_queues =
2545                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
2546         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
2547         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
2548         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
2549         dev_info->max_mtu = hw->max_mtu;
2550
2551         host_features = VIRTIO_OPS(hw)->get_features(hw);
2552         dev_info->rx_offload_capa = RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2553         if (host_features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))
2554                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_SCATTER;
2555         if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
2556                 dev_info->rx_offload_capa |=
2557                         RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
2558                         RTE_ETH_RX_OFFLOAD_UDP_CKSUM;
2559         }
2560         if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN))
2561                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
2562         tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2563                 (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2564         if ((host_features & tso_mask) == tso_mask)
2565                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO;
2566
2567         dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
2568                                     RTE_ETH_TX_OFFLOAD_VLAN_INSERT;
2569         if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) {
2570                 dev_info->tx_offload_capa |=
2571                         RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2572                         RTE_ETH_TX_OFFLOAD_TCP_CKSUM;
2573         }
2574         tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2575                 (1ULL << VIRTIO_NET_F_HOST_TSO6);
2576         if ((host_features & tso_mask) == tso_mask)
2577                 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_TCP_TSO;
2578
2579         if (host_features & (1ULL << VIRTIO_F_RING_PACKED)) {
2580                 /*
2581                  * According to 2.7 Packed Virtqueues,
2582                  * 2.7.10.1 Structure Size and Alignment:
2583                  * The Queue Size value does not have to be a power of 2.
2584                  */
2585                 dev_info->rx_desc_lim.nb_max = UINT16_MAX;
2586                 dev_info->tx_desc_lim.nb_max = UINT16_MAX;
2587         } else {
2588                 /*
2589                  * According to 2.6 Split Virtqueues:
2590                  * Queue Size value is always a power of 2. The maximum Queue
2591                  * Size value is 32768.
2592                  */
2593                 dev_info->rx_desc_lim.nb_max = 32768;
2594                 dev_info->tx_desc_lim.nb_max = 32768;
2595         }
2596         /*
2597          * Actual minimum is not the same for virtqueues of different kinds,
2598          * but to avoid tangling the code with separate branches, rely on
2599          * default thresholds since desc number must be at least of their size.
2600          */
2601         dev_info->rx_desc_lim.nb_min = RTE_MAX(DEFAULT_RX_FREE_THRESH,
2602                                                RTE_VIRTIO_VPMD_RX_REARM_THRESH);
2603         dev_info->tx_desc_lim.nb_min = DEFAULT_TX_FREE_THRESH;
2604         dev_info->rx_desc_lim.nb_align = 1;
2605         dev_info->tx_desc_lim.nb_align = 1;
2606
2607         return 0;
2608 }
2609
2610 /*
2611  * It enables testpmd to collect per queue stats.
2612  */
2613 static int
2614 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
2615 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
2616 __rte_unused uint8_t is_rx)
2617 {
2618         return 0;
2619 }
2620
2621 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_init, init, NOTICE);
2622 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_driver, driver, NOTICE);