drivers/net/virtio/virtio_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <errno.h>
   9 #include <unistd.h>
  10
  11 #include <ethdev_driver.h>
  12 #include <rte_memcpy.h>
  13 #include <rte_string_fns.h>
  14 #include <rte_memzone.h>
  15 #include <rte_malloc.h>
  16 #include <rte_branch_prediction.h>
  17 #include <rte_ether.h>
  18 #include <rte_ip.h>
  19 #include <rte_arp.h>
  20 #include <rte_common.h>
  21 #include <rte_errno.h>
  22 #include <rte_cpuflags.h>
  23 #include <rte_vect.h>
  24 #include <rte_memory.h>
  25 #include <rte_eal_paging.h>
  26 #include <rte_eal.h>
  27 #include <rte_dev.h>
  28 #include <rte_cycles.h>
  29 #include <rte_kvargs.h>
  30
  31 #include "virtio_ethdev.h"
  32 #include "virtio.h"
  33 #include "virtio_logs.h"
  34 #include "virtqueue.h"
  35 #include "virtio_rxtx.h"
  36 #include "virtio_rxtx_simple.h"
  37 #include "virtio_user/virtio_user_dev.h"
  38
  39 static int  virtio_dev_configure(struct rte_eth_dev *dev);
  40 static int  virtio_dev_start(struct rte_eth_dev *dev);
  41 static int virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
  42 static int virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
  43 static int virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
  44 static int virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
  45 static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
  46 static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
  47         uint32_t *speed,
  48         int *vectorized);
  49 static int virtio_dev_info_get(struct rte_eth_dev *dev,
  50                                 struct rte_eth_dev_info *dev_info);
  51 static int virtio_dev_link_update(struct rte_eth_dev *dev,
  52         int wait_to_complete);
  53 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  54 static int virtio_dev_rss_hash_update(struct rte_eth_dev *dev,
  55                 struct rte_eth_rss_conf *rss_conf);
  56 static int virtio_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
  57                 struct rte_eth_rss_conf *rss_conf);
  58 static int virtio_dev_rss_reta_update(struct rte_eth_dev *dev,
  59                          struct rte_eth_rss_reta_entry64 *reta_conf,
  60                          uint16_t reta_size);
  61 static int virtio_dev_rss_reta_query(struct rte_eth_dev *dev,
  62                          struct rte_eth_rss_reta_entry64 *reta_conf,
  63                          uint16_t reta_size);
  64
  65 static void virtio_set_hwaddr(struct virtio_hw *hw);
  66 static void virtio_get_hwaddr(struct virtio_hw *hw);
  67
  68 static int virtio_dev_stats_get(struct rte_eth_dev *dev,
  69                                  struct rte_eth_stats *stats);
  70 static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
  71                                  struct rte_eth_xstat *xstats, unsigned n);
  72 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
  73                                        struct rte_eth_xstat_name *xstats_names,
  74                                        unsigned limit);
  75 static int virtio_dev_stats_reset(struct rte_eth_dev *dev);
  76 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
  77 static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
  78                                 uint16_t vlan_id, int on);
  79 static int virtio_mac_addr_add(struct rte_eth_dev *dev,
  80                                 struct rte_ether_addr *mac_addr,
  81                                 uint32_t index, uint32_t vmdq);
  82 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  83 static int virtio_mac_addr_set(struct rte_eth_dev *dev,
  84                                 struct rte_ether_addr *mac_addr);
  85
  86 static int virtio_intr_disable(struct rte_eth_dev *dev);
  87 static int virtio_get_monitor_addr(void *rx_queue,
  88                                 struct rte_power_monitor_cond *pmc);
  89
  90 static int virtio_dev_queue_stats_mapping_set(
  91         struct rte_eth_dev *eth_dev,
  92         uint16_t queue_id,
  93         uint8_t stat_idx,
  94         uint8_t is_rx);
  95
  96 static void virtio_notify_peers(struct rte_eth_dev *dev);
  97 static void virtio_ack_link_announce(struct rte_eth_dev *dev);
  98
  99 struct rte_virtio_xstats_name_off {
 100         char name[RTE_ETH_XSTATS_NAME_SIZE];
 101         unsigned offset;
 102 };
 103
 104 /* [rt]x_qX_ is prepended to the name string here */
 105 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
 106         {"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
 107         {"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
 108         {"errors",                 offsetof(struct virtnet_rx, stats.errors)},
 109         {"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
 110         {"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
 111         {"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
 112         {"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
 113         {"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
 114         {"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
 115         {"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
 116         {"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
 117         {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
 118         {"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
 119 };
 120
 121 /* [rt]x_qX_ is prepended to the name string here */
 122 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
 123         {"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
 124         {"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
 125         {"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
 126         {"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
 127         {"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
 128         {"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
 129         {"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
 130         {"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
 131         {"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
 132         {"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
 133         {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
 134         {"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
 135 };
 136
 137 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
 138                             sizeof(rte_virtio_rxq_stat_strings[0]))
 139 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
 140                             sizeof(rte_virtio_txq_stat_strings[0]))
 141
 142 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS];
 143
 144 static struct virtio_pmd_ctrl *
 145 virtio_send_command_packed(struct virtnet_ctl *cvq,
 146                            struct virtio_pmd_ctrl *ctrl,
 147                            int *dlen, int pkt_num)
 148 {
 149         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 150         int head;
 151         struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
 152         struct virtio_pmd_ctrl *result;
 153         uint16_t flags;
 154         int sum = 0;
 155         int nb_descs = 0;
 156         int k;
 157
 158         /*
 159          * Format is enforced in qemu code:
 160          * One TX packet for header;
 161          * At least one TX packet per argument;
 162          * One RX packet for ACK.
 163          */
 164         head = vq->vq_avail_idx;
 165         flags = vq->vq_packed.cached_flags;
 166         desc[head].addr = cvq->virtio_net_hdr_mem;
 167         desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 168         vq->vq_free_cnt--;
 169         nb_descs++;
 170         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 171                 vq->vq_avail_idx -= vq->vq_nentries;
 172                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 173         }
 174
 175         for (k = 0; k < pkt_num; k++) {
 176                 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 177                         + sizeof(struct virtio_net_ctrl_hdr)
 178                         + sizeof(ctrl->status) + sizeof(uint8_t) * sum;
 179                 desc[vq->vq_avail_idx].len = dlen[k];
 180                 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT |
 181                         vq->vq_packed.cached_flags;
 182                 sum += dlen[k];
 183                 vq->vq_free_cnt--;
 184                 nb_descs++;
 185                 if (++vq->vq_avail_idx >= vq->vq_nentries) {
 186                         vq->vq_avail_idx -= vq->vq_nentries;
 187                         vq->vq_packed.cached_flags ^=
 188                                 VRING_PACKED_DESC_F_AVAIL_USED;
 189                 }
 190         }
 191
 192         desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem
 193                 + sizeof(struct virtio_net_ctrl_hdr);
 194         desc[vq->vq_avail_idx].len = sizeof(ctrl->status);
 195         desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE |
 196                 vq->vq_packed.cached_flags;
 197         vq->vq_free_cnt--;
 198         nb_descs++;
 199         if (++vq->vq_avail_idx >= vq->vq_nentries) {
 200                 vq->vq_avail_idx -= vq->vq_nentries;
 201                 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
 202         }
 203
 204         virtqueue_store_flags_packed(&desc[head], VRING_DESC_F_NEXT | flags,
 205                         vq->hw->weak_barriers);
 206
 207         virtio_wmb(vq->hw->weak_barriers);
 208         virtqueue_notify(vq);
 209
 210         /* wait for used desc in virtqueue
 211          * desc_is_used has a load-acquire or rte_io_rmb inside
 212          */
 213         while (!desc_is_used(&desc[head], vq))
 214                 usleep(100);
 215
 216         /* now get used descriptors */
 217         vq->vq_free_cnt += nb_descs;
 218         vq->vq_used_cons_idx += nb_descs;
 219         if (vq->vq_used_cons_idx >= vq->vq_nentries) {
 220                 vq->vq_used_cons_idx -= vq->vq_nentries;
 221                 vq->vq_packed.used_wrap_counter ^= 1;
 222         }
 223
 224         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n"
 225                         "vq->vq_avail_idx=%d\n"
 226                         "vq->vq_used_cons_idx=%d\n"
 227                         "vq->vq_packed.cached_flags=0x%x\n"
 228                         "vq->vq_packed.used_wrap_counter=%d",
 229                         vq->vq_free_cnt,
 230                         vq->vq_avail_idx,
 231                         vq->vq_used_cons_idx,
 232                         vq->vq_packed.cached_flags,
 233                         vq->vq_packed.used_wrap_counter);
 234
 235         result = cvq->virtio_net_hdr_mz->addr;
 236         return result;
 237 }
 238
 239 static struct virtio_pmd_ctrl *
 240 virtio_send_command_split(struct virtnet_ctl *cvq,
 241                           struct virtio_pmd_ctrl *ctrl,
 242                           int *dlen, int pkt_num)
 243 {
 244         struct virtio_pmd_ctrl *result;
 245         struct virtqueue *vq = virtnet_cq_to_vq(cvq);
 246         uint32_t head, i;
 247         int k, sum = 0;
 248
 249         head = vq->vq_desc_head_idx;
 250
 251         /*
 252          * Format is enforced in qemu code:
 253          * One TX packet for header;
 254          * At least one TX packet per argument;
 255          * One RX packet for ACK.
 256          */
 257         vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT;
 258         vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem;
 259         vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
 260         vq->vq_free_cnt--;
 261         i = vq->vq_split.ring.desc[head].next;
 262
 263         for (k = 0; k < pkt_num; k++) {
 264                 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT;
 265                 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 266                         + sizeof(struct virtio_net_ctrl_hdr)
 267                         + sizeof(ctrl->status) + sizeof(uint8_t)*sum;
 268                 vq->vq_split.ring.desc[i].len = dlen[k];
 269                 sum += dlen[k];
 270                 vq->vq_free_cnt--;
 271                 i = vq->vq_split.ring.desc[i].next;
 272         }
 273
 274         vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE;
 275         vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem
 276                         + sizeof(struct virtio_net_ctrl_hdr);
 277         vq->vq_split.ring.desc[i].len = sizeof(ctrl->status);
 278         vq->vq_free_cnt--;
 279
 280         vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next;
 281
 282         vq_update_avail_ring(vq, head);
 283         vq_update_avail_idx(vq);
 284
 285         PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
 286
 287         virtqueue_notify(vq);
 288
 289         while (virtqueue_nused(vq) == 0)
 290                 usleep(100);
 291
 292         while (virtqueue_nused(vq)) {
 293                 uint32_t idx, desc_idx, used_idx;
 294                 struct vring_used_elem *uep;
 295
 296                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 297                                 & (vq->vq_nentries - 1));
 298                 uep = &vq->vq_split.ring.used->ring[used_idx];
 299                 idx = (uint32_t) uep->id;
 300                 desc_idx = idx;
 301
 302                 while (vq->vq_split.ring.desc[desc_idx].flags &
 303                                 VRING_DESC_F_NEXT) {
 304                         desc_idx = vq->vq_split.ring.desc[desc_idx].next;
 305                         vq->vq_free_cnt++;
 306                 }
 307
 308                 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 309                 vq->vq_desc_head_idx = idx;
 310
 311                 vq->vq_used_cons_idx++;
 312                 vq->vq_free_cnt++;
 313         }
 314
 315         PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
 316                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 317
 318         result = cvq->virtio_net_hdr_mz->addr;
 319         return result;
 320 }
 321
 322 static int
 323 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
 324                     int *dlen, int pkt_num)
 325 {
 326         virtio_net_ctrl_ack status = ~0;
 327         struct virtio_pmd_ctrl *result;
 328         struct virtqueue *vq;
 329
 330         ctrl->status = status;
 331
 332         if (!cvq) {
 333                 PMD_INIT_LOG(ERR, "Control queue is not supported.");
 334                 return -1;
 335         }
 336
 337         rte_spinlock_lock(&cvq->lock);
 338         vq = virtnet_cq_to_vq(cvq);
 339
 340         PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
 341                 "vq->hw->cvq = %p vq = %p",
 342                 vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
 343
 344         if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) {
 345                 rte_spinlock_unlock(&cvq->lock);
 346                 return -1;
 347         }
 348
 349         memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
 350                 sizeof(struct virtio_pmd_ctrl));
 351
 352         if (virtio_with_packed_queue(vq->hw))
 353                 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num);
 354         else
 355                 result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num);
 356
 357         rte_spinlock_unlock(&cvq->lock);
 358         return result->status;
 359 }
 360
 361 static int
 362 virtio_set_multiple_queues_rss(struct rte_eth_dev *dev, uint16_t nb_queues)
 363 {
 364         struct virtio_hw *hw = dev->data->dev_private;
 365         struct virtio_pmd_ctrl ctrl;
 366         struct virtio_net_ctrl_rss rss;
 367         int dlen, ret;
 368
 369         rss.hash_types = hw->rss_hash_types & VIRTIO_NET_HASH_TYPE_MASK;
 370         RTE_BUILD_BUG_ON(!RTE_IS_POWER_OF_2(VIRTIO_NET_RSS_RETA_SIZE));
 371         rss.indirection_table_mask = VIRTIO_NET_RSS_RETA_SIZE - 1;
 372         rss.unclassified_queue = 0;
 373         memcpy(rss.indirection_table, hw->rss_reta, VIRTIO_NET_RSS_RETA_SIZE * sizeof(uint16_t));
 374         rss.max_tx_vq = nb_queues;
 375         rss.hash_key_length = VIRTIO_NET_RSS_KEY_SIZE;
 376         memcpy(rss.hash_key_data, hw->rss_key, VIRTIO_NET_RSS_KEY_SIZE);
 377
 378         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 379         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_RSS_CONFIG;
 380         memcpy(ctrl.data, &rss, sizeof(rss));
 381
 382         dlen = sizeof(rss);
 383
 384         ret = virtio_send_command(hw->cvq, &ctrl, &dlen, 1);
 385         if (ret) {
 386                 PMD_INIT_LOG(ERR, "RSS multiqueue configured but send command failed");
 387                 return -EINVAL;
 388         }
 389
 390         return 0;
 391 }
 392
 393 static int
 394 virtio_set_multiple_queues_auto(struct rte_eth_dev *dev, uint16_t nb_queues)
 395 {
 396         struct virtio_hw *hw = dev->data->dev_private;
 397         struct virtio_pmd_ctrl ctrl;
 398         int dlen;
 399         int ret;
 400
 401         ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
 402         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
 403         memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
 404
 405         dlen = sizeof(uint16_t);
 406
 407         ret = virtio_send_command(hw->cvq, &ctrl, &dlen, 1);
 408         if (ret) {
 409                 PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
 410                           "failed, this is too late now...");
 411                 return -EINVAL;
 412         }
 413
 414         return 0;
 415 }
 416
 417 static int
 418 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
 419 {
 420         struct virtio_hw *hw = dev->data->dev_private;
 421
 422         if (virtio_with_feature(hw, VIRTIO_NET_F_RSS))
 423                 return virtio_set_multiple_queues_rss(dev, nb_queues);
 424         else
 425                 return virtio_set_multiple_queues_auto(dev, nb_queues);
 426 }
 427
 428 static uint16_t
 429 virtio_get_nr_vq(struct virtio_hw *hw)
 430 {
 431         uint16_t nr_vq = hw->max_queue_pairs * 2;
 432
 433         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
 434                 nr_vq += 1;
 435
 436         return nr_vq;
 437 }
 438
 439 static void
 440 virtio_init_vring(struct virtqueue *vq)
 441 {
 442         int size = vq->vq_nentries;
 443         uint8_t *ring_mem = vq->vq_ring_virt_mem;
 444
 445         PMD_INIT_FUNC_TRACE();
 446
 447         memset(ring_mem, 0, vq->vq_ring_size);
 448
 449         vq->vq_used_cons_idx = 0;
 450         vq->vq_desc_head_idx = 0;
 451         vq->vq_avail_idx = 0;
 452         vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
 453         vq->vq_free_cnt = vq->vq_nentries;
 454         memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
 455         if (virtio_with_packed_queue(vq->hw)) {
 456                 vring_init_packed(&vq->vq_packed.ring, ring_mem,
 457                                   VIRTIO_VRING_ALIGN, size);
 458                 vring_desc_init_packed(vq, size);
 459         } else {
 460                 struct vring *vr = &vq->vq_split.ring;
 461
 462                 vring_init_split(vr, ring_mem, VIRTIO_VRING_ALIGN, size);
 463                 vring_desc_init_split(vr->desc, size);
 464         }
 465         /*
 466          * Disable device(host) interrupting guest
 467          */
 468         virtqueue_disable_intr(vq);
 469 }
 470
 471 static int
 472 virtio_init_queue(struct rte_eth_dev *dev, uint16_t queue_idx)
 473 {
 474         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 475         char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
 476         const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
 477         unsigned int vq_size, size;
 478         struct virtio_hw *hw = dev->data->dev_private;
 479         struct virtnet_rx *rxvq = NULL;
 480         struct virtnet_tx *txvq = NULL;
 481         struct virtnet_ctl *cvq = NULL;
 482         struct virtqueue *vq;
 483         size_t sz_hdr_mz = 0;
 484         void *sw_ring = NULL;
 485         int queue_type = virtio_get_queue_type(hw, queue_idx);
 486         int ret;
 487         int numa_node = dev->device->numa_node;
 488         struct rte_mbuf *fake_mbuf = NULL;
 489
 490         PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d",
 491                         queue_idx, numa_node);
 492
 493         /*
 494          * Read the virtqueue size from the Queue Size field
 495          * Always power of 2 and if 0 virtqueue does not exist
 496          */
 497         vq_size = VIRTIO_OPS(hw)->get_queue_num(hw, queue_idx);
 498         PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
 499         if (vq_size == 0) {
 500                 PMD_INIT_LOG(ERR, "virtqueue does not exist");
 501                 return -EINVAL;
 502         }
 503
 504         if (!virtio_with_packed_queue(hw) && !rte_is_power_of_2(vq_size)) {
 505                 PMD_INIT_LOG(ERR, "split virtqueue size is not power of 2");
 506                 return -EINVAL;
 507         }
 508
 509         snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
 510                  dev->data->port_id, queue_idx);
 511
 512         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 513                                 vq_size * sizeof(struct vq_desc_extra),
 514                                 RTE_CACHE_LINE_SIZE);
 515         if (queue_type == VTNET_TQ) {
 516                 /*
 517                  * For each xmit packet, allocate a virtio_net_hdr
 518                  * and indirect ring elements
 519                  */
 520                 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
 521         } else if (queue_type == VTNET_CQ) {
 522                 /* Allocate a page for control vq command, data and status */
 523                 sz_hdr_mz = rte_mem_page_size();
 524         }
 525
 526         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
 527                                 numa_node);
 528         if (vq == NULL) {
 529                 PMD_INIT_LOG(ERR, "can not allocate vq");
 530                 return -ENOMEM;
 531         }
 532         hw->vqs[queue_idx] = vq;
 533
 534         vq->hw = hw;
 535         vq->vq_queue_index = queue_idx;
 536         vq->vq_nentries = vq_size;
 537         if (virtio_with_packed_queue(hw)) {
 538                 vq->vq_packed.used_wrap_counter = 1;
 539                 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
 540                 vq->vq_packed.event_flags_shadow = 0;
 541                 if (queue_type == VTNET_RQ)
 542                         vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
 543         }
 544
 545         /*
 546          * Reserve a memzone for vring elements
 547          */
 548         size = vring_size(hw, vq_size, VIRTIO_VRING_ALIGN);
 549         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN);
 550         PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
 551                      size, vq->vq_ring_size);
 552
 553         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 554                         numa_node, RTE_MEMZONE_IOVA_CONTIG,
 555                         VIRTIO_VRING_ALIGN);
 556         if (mz == NULL) {
 557                 if (rte_errno == EEXIST)
 558                         mz = rte_memzone_lookup(vq_name);
 559                 if (mz == NULL) {
 560                         ret = -ENOMEM;
 561                         goto free_vq;
 562                 }
 563         }
 564
 565         memset(mz->addr, 0, mz->len);
 566
 567         if (hw->use_va)
 568                 vq->vq_ring_mem = (uintptr_t)mz->addr;
 569         else
 570                 vq->vq_ring_mem = mz->iova;
 571
 572         vq->vq_ring_virt_mem = mz->addr;
 573         PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem);
 574         PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem);
 575
 576         virtio_init_vring(vq);
 577
 578         if (sz_hdr_mz) {
 579                 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 580                          dev->data->port_id, queue_idx);
 581                 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
 582                                 numa_node, RTE_MEMZONE_IOVA_CONTIG,
 583                                 RTE_CACHE_LINE_SIZE);
 584                 if (hdr_mz == NULL) {
 585                         if (rte_errno == EEXIST)
 586                                 hdr_mz = rte_memzone_lookup(vq_hdr_name);
 587                         if (hdr_mz == NULL) {
 588                                 ret = -ENOMEM;
 589                                 goto free_mz;
 590                         }
 591                 }
 592         }
 593
 594         if (queue_type == VTNET_RQ) {
 595                 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
 596                                sizeof(vq->sw_ring[0]);
 597
 598                 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
 599                                 RTE_CACHE_LINE_SIZE, numa_node);
 600                 if (!sw_ring) {
 601                         PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
 602                         ret = -ENOMEM;
 603                         goto free_hdr_mz;
 604                 }
 605
 606                 fake_mbuf = rte_zmalloc_socket("sw_ring", sizeof(*fake_mbuf),
 607                                 RTE_CACHE_LINE_SIZE, numa_node);
 608                 if (!fake_mbuf) {
 609                         PMD_INIT_LOG(ERR, "can not allocate fake mbuf");
 610                         ret = -ENOMEM;
 611                         goto free_sw_ring;
 612                 }
 613
 614                 vq->sw_ring = sw_ring;
 615                 rxvq = &vq->rxq;
 616                 rxvq->port_id = dev->data->port_id;
 617                 rxvq->mz = mz;
 618                 rxvq->fake_mbuf = fake_mbuf;
 619         } else if (queue_type == VTNET_TQ) {
 620                 txvq = &vq->txq;
 621                 txvq->port_id = dev->data->port_id;
 622                 txvq->mz = mz;
 623                 txvq->virtio_net_hdr_mz = hdr_mz;
 624                 if (hw->use_va)
 625                         txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 626                 else
 627                         txvq->virtio_net_hdr_mem = hdr_mz->iova;
 628         } else if (queue_type == VTNET_CQ) {
 629                 cvq = &vq->cq;
 630                 cvq->mz = mz;
 631                 cvq->virtio_net_hdr_mz = hdr_mz;
 632                 if (hw->use_va)
 633                         cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
 634                 else
 635                         cvq->virtio_net_hdr_mem = hdr_mz->iova;
 636                 memset(cvq->virtio_net_hdr_mz->addr, 0, rte_mem_page_size());
 637
 638                 hw->cvq = cvq;
 639         }
 640
 641         if (hw->use_va)
 642                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr);
 643         else
 644                 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova);
 645
 646         if (queue_type == VTNET_TQ) {
 647                 struct virtio_tx_region *txr;
 648                 unsigned int i;
 649
 650                 txr = hdr_mz->addr;
 651                 memset(txr, 0, vq_size * sizeof(*txr));
 652                 for (i = 0; i < vq_size; i++) {
 653                         /* first indirect descriptor is always the tx header */
 654                         if (!virtio_with_packed_queue(hw)) {
 655                                 struct vring_desc *start_dp = txr[i].tx_indir;
 656                                 vring_desc_init_split(start_dp,
 657                                                       RTE_DIM(txr[i].tx_indir));
 658                                 start_dp->addr = txvq->virtio_net_hdr_mem
 659                                         + i * sizeof(*txr)
 660                                         + offsetof(struct virtio_tx_region,
 661                                                    tx_hdr);
 662                                 start_dp->len = hw->vtnet_hdr_size;
 663                                 start_dp->flags = VRING_DESC_F_NEXT;
 664                         } else {
 665                                 struct vring_packed_desc *start_dp =
 666                                         txr[i].tx_packed_indir;
 667                                 vring_desc_init_indirect_packed(start_dp,
 668                                       RTE_DIM(txr[i].tx_packed_indir));
 669                                 start_dp->addr = txvq->virtio_net_hdr_mem
 670                                         + i * sizeof(*txr)
 671                                         + offsetof(struct virtio_tx_region,
 672                                                    tx_hdr);
 673                                 start_dp->len = hw->vtnet_hdr_size;
 674                         }
 675                 }
 676         }
 677
 678         if (VIRTIO_OPS(hw)->setup_queue(hw, vq) < 0) {
 679                 PMD_INIT_LOG(ERR, "setup_queue failed");
 680                 ret = -EINVAL;
 681                 goto clean_vq;
 682         }
 683
 684         return 0;
 685
 686 clean_vq:
 687         hw->cvq = NULL;
 688         rte_free(fake_mbuf);
 689 free_sw_ring:
 690         rte_free(sw_ring);
 691 free_hdr_mz:
 692         rte_memzone_free(hdr_mz);
 693 free_mz:
 694         rte_memzone_free(mz);
 695 free_vq:
 696         rte_free(vq);
 697         hw->vqs[queue_idx] = NULL;
 698
 699         return ret;
 700 }
 701
 702 static void
 703 virtio_free_queues(struct virtio_hw *hw)
 704 {
 705         uint16_t nr_vq = virtio_get_nr_vq(hw);
 706         struct virtqueue *vq;
 707         int queue_type;
 708         uint16_t i;
 709
 710         if (hw->vqs == NULL)
 711                 return;
 712
 713         for (i = 0; i < nr_vq; i++) {
 714                 vq = hw->vqs[i];
 715                 if (!vq)
 716                         continue;
 717
 718                 queue_type = virtio_get_queue_type(hw, i);
 719                 if (queue_type == VTNET_RQ) {
 720                         rte_free(vq->rxq.fake_mbuf);
 721                         rte_free(vq->sw_ring);
 722                         rte_memzone_free(vq->rxq.mz);
 723                 } else if (queue_type == VTNET_TQ) {
 724                         rte_memzone_free(vq->txq.mz);
 725                         rte_memzone_free(vq->txq.virtio_net_hdr_mz);
 726                 } else {
 727                         rte_memzone_free(vq->cq.mz);
 728                         rte_memzone_free(vq->cq.virtio_net_hdr_mz);
 729                 }
 730
 731                 rte_free(vq);
 732                 hw->vqs[i] = NULL;
 733         }
 734
 735         rte_free(hw->vqs);
 736         hw->vqs = NULL;
 737 }
 738
 739 static int
 740 virtio_alloc_queues(struct rte_eth_dev *dev)
 741 {
 742         struct virtio_hw *hw = dev->data->dev_private;
 743         uint16_t nr_vq = virtio_get_nr_vq(hw);
 744         uint16_t i;
 745         int ret;
 746
 747         hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0);
 748         if (!hw->vqs) {
 749                 PMD_INIT_LOG(ERR, "failed to allocate vqs");
 750                 return -ENOMEM;
 751         }
 752
 753         for (i = 0; i < nr_vq; i++) {
 754                 ret = virtio_init_queue(dev, i);
 755                 if (ret < 0) {
 756                         virtio_free_queues(hw);
 757                         return ret;
 758                 }
 759         }
 760
 761         return 0;
 762 }
 763
 764 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev);
 765
 766 static void
 767 virtio_free_rss(struct virtio_hw *hw)
 768 {
 769         rte_free(hw->rss_key);
 770         hw->rss_key = NULL;
 771
 772         rte_free(hw->rss_reta);
 773         hw->rss_reta = NULL;
 774 }
 775
 776 int
 777 virtio_dev_close(struct rte_eth_dev *dev)
 778 {
 779         struct virtio_hw *hw = dev->data->dev_private;
 780         struct rte_eth_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
 781
 782         PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 783         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 784                 return 0;
 785
 786         if (!hw->opened)
 787                 return 0;
 788         hw->opened = 0;
 789
 790         /* reset the NIC */
 791         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
 792                 VIRTIO_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR);
 793         if (intr_conf->rxq)
 794                 virtio_queues_unbind_intr(dev);
 795
 796         if (intr_conf->lsc || intr_conf->rxq) {
 797                 virtio_intr_disable(dev);
 798                 rte_intr_efd_disable(dev->intr_handle);
 799                 rte_intr_vec_list_free(dev->intr_handle);
 800         }
 801
 802         virtio_reset(hw);
 803         virtio_dev_free_mbufs(dev);
 804         virtio_free_queues(hw);
 805         virtio_free_rss(hw);
 806
 807         return VIRTIO_OPS(hw)->dev_close(hw);
 808 }
 809
 810 static int
 811 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
 812 {
 813         struct virtio_hw *hw = dev->data->dev_private;
 814         struct virtio_pmd_ctrl ctrl;
 815         int dlen[1];
 816         int ret;
 817
 818         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 819                 PMD_INIT_LOG(INFO, "host does not support rx control");
 820                 return -ENOTSUP;
 821         }
 822
 823         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 824         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 825         ctrl.data[0] = 1;
 826         dlen[0] = 1;
 827
 828         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 829         if (ret) {
 830                 PMD_INIT_LOG(ERR, "Failed to enable promisc");
 831                 return -EAGAIN;
 832         }
 833
 834         return 0;
 835 }
 836
 837 static int
 838 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
 839 {
 840         struct virtio_hw *hw = dev->data->dev_private;
 841         struct virtio_pmd_ctrl ctrl;
 842         int dlen[1];
 843         int ret;
 844
 845         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 846                 PMD_INIT_LOG(INFO, "host does not support rx control");
 847                 return -ENOTSUP;
 848         }
 849
 850         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 851         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
 852         ctrl.data[0] = 0;
 853         dlen[0] = 1;
 854
 855         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 856         if (ret) {
 857                 PMD_INIT_LOG(ERR, "Failed to disable promisc");
 858                 return -EAGAIN;
 859         }
 860
 861         return 0;
 862 }
 863
 864 static int
 865 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
 866 {
 867         struct virtio_hw *hw = dev->data->dev_private;
 868         struct virtio_pmd_ctrl ctrl;
 869         int dlen[1];
 870         int ret;
 871
 872         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 873                 PMD_INIT_LOG(INFO, "host does not support rx control");
 874                 return -ENOTSUP;
 875         }
 876
 877         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 878         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 879         ctrl.data[0] = 1;
 880         dlen[0] = 1;
 881
 882         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 883         if (ret) {
 884                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
 885                 return -EAGAIN;
 886         }
 887
 888         return 0;
 889 }
 890
 891 static int
 892 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
 893 {
 894         struct virtio_hw *hw = dev->data->dev_private;
 895         struct virtio_pmd_ctrl ctrl;
 896         int dlen[1];
 897         int ret;
 898
 899         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
 900                 PMD_INIT_LOG(INFO, "host does not support rx control");
 901                 return -ENOTSUP;
 902         }
 903
 904         ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
 905         ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
 906         ctrl.data[0] = 0;
 907         dlen[0] = 1;
 908
 909         ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
 910         if (ret) {
 911                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
 912                 return -EAGAIN;
 913         }
 914
 915         return 0;
 916 }
 917
 918 uint16_t
 919 virtio_rx_mem_pool_buf_size(struct rte_mempool *mp)
 920 {
 921         return rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
 922 }
 923
 924 bool
 925 virtio_rx_check_scatter(uint16_t max_rx_pkt_len, uint16_t rx_buf_size,
 926                         bool rx_scatter_enabled, const char **error)
 927 {
 928         if (!rx_scatter_enabled && max_rx_pkt_len > rx_buf_size) {
 929                 *error = "Rx scatter is disabled and RxQ mbuf pool object size is too small";
 930                 return false;
 931         }
 932
 933         return true;
 934 }
 935
 936 static bool
 937 virtio_check_scatter_on_all_rx_queues(struct rte_eth_dev *dev,
 938                                       uint16_t frame_size)
 939 {
 940         struct virtio_hw *hw = dev->data->dev_private;
 941         struct virtnet_rx *rxvq;
 942         struct virtqueue *vq;
 943         unsigned int qidx;
 944         uint16_t buf_size;
 945         const char *error;
 946
 947         if (hw->vqs == NULL)
 948                 return true;
 949
 950         for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) {
 951                 vq = hw->vqs[2 * qidx + VTNET_SQ_RQ_QUEUE_IDX];
 952                 if (vq == NULL)
 953                         continue;
 954
 955                 rxvq = &vq->rxq;
 956                 if (rxvq->mpool == NULL)
 957                         continue;
 958                 buf_size = virtio_rx_mem_pool_buf_size(rxvq->mpool);
 959
 960                 if (!virtio_rx_check_scatter(frame_size, buf_size,
 961                                              hw->rx_ol_scatter, &error)) {
 962                         PMD_INIT_LOG(ERR, "MTU check for RxQ %u failed: %s",
 963                                      qidx, error);
 964                         return false;
 965                 }
 966         }
 967
 968         return true;
 969 }
 970
 971 #define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
 972 static int
 973 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 974 {
 975         struct virtio_hw *hw = dev->data->dev_private;
 976         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
 977                                  hw->vtnet_hdr_size;
 978         uint32_t frame_size = mtu + ether_hdr_len;
 979         uint32_t max_frame_size = hw->max_mtu + ether_hdr_len;
 980
 981         max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN);
 982
 983         if (mtu < RTE_ETHER_MIN_MTU || frame_size > max_frame_size) {
 984                 PMD_INIT_LOG(ERR, "MTU should be between %d and %d",
 985                         RTE_ETHER_MIN_MTU, max_frame_size - ether_hdr_len);
 986                 return -EINVAL;
 987         }
 988
 989         if (!virtio_check_scatter_on_all_rx_queues(dev, frame_size)) {
 990                 PMD_INIT_LOG(ERR, "MTU vs Rx scatter and Rx buffers check failed");
 991                 return -EINVAL;
 992         }
 993
 994         hw->max_rx_pkt_len = frame_size;
 995
 996         return 0;
 997 }
 998
 999 static int
1000 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1001 {
1002         struct virtio_hw *hw = dev->data->dev_private;
1003         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
1004         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1005
1006         virtqueue_enable_intr(vq);
1007         virtio_mb(hw->weak_barriers);
1008         return 0;
1009 }
1010
1011 static int
1012 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1013 {
1014         struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
1015         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1016
1017         virtqueue_disable_intr(vq);
1018         return 0;
1019 }
1020
1021 /*
1022  * dev_ops for virtio, bare necessities for basic operation
1023  */
1024 static const struct eth_dev_ops virtio_eth_dev_ops = {
1025         .dev_configure           = virtio_dev_configure,
1026         .dev_start               = virtio_dev_start,
1027         .dev_stop                = virtio_dev_stop,
1028         .dev_close               = virtio_dev_close,
1029         .promiscuous_enable      = virtio_dev_promiscuous_enable,
1030         .promiscuous_disable     = virtio_dev_promiscuous_disable,
1031         .allmulticast_enable     = virtio_dev_allmulticast_enable,
1032         .allmulticast_disable    = virtio_dev_allmulticast_disable,
1033         .mtu_set                 = virtio_mtu_set,
1034         .dev_infos_get           = virtio_dev_info_get,
1035         .stats_get               = virtio_dev_stats_get,
1036         .xstats_get              = virtio_dev_xstats_get,
1037         .xstats_get_names        = virtio_dev_xstats_get_names,
1038         .stats_reset             = virtio_dev_stats_reset,
1039         .xstats_reset            = virtio_dev_stats_reset,
1040         .link_update             = virtio_dev_link_update,
1041         .vlan_offload_set        = virtio_dev_vlan_offload_set,
1042         .rx_queue_setup          = virtio_dev_rx_queue_setup,
1043         .rx_queue_intr_enable    = virtio_dev_rx_queue_intr_enable,
1044         .rx_queue_intr_disable   = virtio_dev_rx_queue_intr_disable,
1045         .tx_queue_setup          = virtio_dev_tx_queue_setup,
1046         .rss_hash_update         = virtio_dev_rss_hash_update,
1047         .rss_hash_conf_get       = virtio_dev_rss_hash_conf_get,
1048         .reta_update             = virtio_dev_rss_reta_update,
1049         .reta_query              = virtio_dev_rss_reta_query,
1050         /* collect stats per queue */
1051         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
1052         .vlan_filter_set         = virtio_vlan_filter_set,
1053         .mac_addr_add            = virtio_mac_addr_add,
1054         .mac_addr_remove         = virtio_mac_addr_remove,
1055         .mac_addr_set            = virtio_mac_addr_set,
1056         .get_monitor_addr        = virtio_get_monitor_addr,
1057 };
1058
1059 /*
1060  * dev_ops for virtio-user in secondary processes, as we just have
1061  * some limited supports currently.
1062  */
1063 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops = {
1064         .dev_infos_get           = virtio_dev_info_get,
1065         .stats_get               = virtio_dev_stats_get,
1066         .xstats_get              = virtio_dev_xstats_get,
1067         .xstats_get_names        = virtio_dev_xstats_get_names,
1068         .stats_reset             = virtio_dev_stats_reset,
1069         .xstats_reset            = virtio_dev_stats_reset,
1070         /* collect stats per queue */
1071         .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
1072 };
1073
1074 static void
1075 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1076 {
1077         unsigned i;
1078
1079         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1080                 const struct virtnet_tx *txvq = dev->data->tx_queues[i];
1081                 if (txvq == NULL)
1082                         continue;
1083
1084                 stats->opackets += txvq->stats.packets;
1085                 stats->obytes += txvq->stats.bytes;
1086
1087                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1088                         stats->q_opackets[i] = txvq->stats.packets;
1089                         stats->q_obytes[i] = txvq->stats.bytes;
1090                 }
1091         }
1092
1093         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1094                 const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1095                 if (rxvq == NULL)
1096                         continue;
1097
1098                 stats->ipackets += rxvq->stats.packets;
1099                 stats->ibytes += rxvq->stats.bytes;
1100                 stats->ierrors += rxvq->stats.errors;
1101
1102                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1103                         stats->q_ipackets[i] = rxvq->stats.packets;
1104                         stats->q_ibytes[i] = rxvq->stats.bytes;
1105                 }
1106         }
1107
1108         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1109 }
1110
1111 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
1112                                        struct rte_eth_xstat_name *xstats_names,
1113                                        __rte_unused unsigned limit)
1114 {
1115         unsigned i;
1116         unsigned count = 0;
1117         unsigned t;
1118
1119         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1120                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1121
1122         if (xstats_names != NULL) {
1123                 /* Note: limit checked in rte_eth_xstats_names() */
1124
1125                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1126                         struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1127                         if (rxvq == NULL)
1128                                 continue;
1129                         for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1130                                 snprintf(xstats_names[count].name,
1131                                         sizeof(xstats_names[count].name),
1132                                         "rx_q%u_%s", i,
1133                                         rte_virtio_rxq_stat_strings[t].name);
1134                                 count++;
1135                         }
1136                 }
1137
1138                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1139                         struct virtnet_tx *txvq = dev->data->tx_queues[i];
1140                         if (txvq == NULL)
1141                                 continue;
1142                         for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1143                                 snprintf(xstats_names[count].name,
1144                                         sizeof(xstats_names[count].name),
1145                                         "tx_q%u_%s", i,
1146                                         rte_virtio_txq_stat_strings[t].name);
1147                                 count++;
1148                         }
1149                 }
1150                 return count;
1151         }
1152         return nstats;
1153 }
1154
1155 static int
1156 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1157                       unsigned n)
1158 {
1159         unsigned i;
1160         unsigned count = 0;
1161
1162         unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
1163                 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
1164
1165         if (n < nstats)
1166                 return nstats;
1167
1168         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1169                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1170
1171                 if (rxvq == NULL)
1172                         continue;
1173
1174                 unsigned t;
1175
1176                 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
1177                         xstats[count].value = *(uint64_t *)(((char *)rxvq) +
1178                                 rte_virtio_rxq_stat_strings[t].offset);
1179                         xstats[count].id = count;
1180                         count++;
1181                 }
1182         }
1183
1184         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1185                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1186
1187                 if (txvq == NULL)
1188                         continue;
1189
1190                 unsigned t;
1191
1192                 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
1193                         xstats[count].value = *(uint64_t *)(((char *)txvq) +
1194                                 rte_virtio_txq_stat_strings[t].offset);
1195                         xstats[count].id = count;
1196                         count++;
1197                 }
1198         }
1199
1200         return count;
1201 }
1202
1203 static int
1204 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1205 {
1206         virtio_update_stats(dev, stats);
1207
1208         return 0;
1209 }
1210
1211 static int
1212 virtio_dev_stats_reset(struct rte_eth_dev *dev)
1213 {
1214         unsigned int i;
1215
1216         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1217                 struct virtnet_tx *txvq = dev->data->tx_queues[i];
1218                 if (txvq == NULL)
1219                         continue;
1220
1221                 txvq->stats.packets = 0;
1222                 txvq->stats.bytes = 0;
1223                 txvq->stats.multicast = 0;
1224                 txvq->stats.broadcast = 0;
1225                 memset(txvq->stats.size_bins, 0,
1226                        sizeof(txvq->stats.size_bins[0]) * 8);
1227         }
1228
1229         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1230                 struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1231                 if (rxvq == NULL)
1232                         continue;
1233
1234                 rxvq->stats.packets = 0;
1235                 rxvq->stats.bytes = 0;
1236                 rxvq->stats.errors = 0;
1237                 rxvq->stats.multicast = 0;
1238                 rxvq->stats.broadcast = 0;
1239                 memset(rxvq->stats.size_bins, 0,
1240                        sizeof(rxvq->stats.size_bins[0]) * 8);
1241         }
1242
1243         return 0;
1244 }
1245
1246 static void
1247 virtio_set_hwaddr(struct virtio_hw *hw)
1248 {
1249         virtio_write_dev_config(hw,
1250                         offsetof(struct virtio_net_config, mac),
1251                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1252 }
1253
1254 static void
1255 virtio_get_hwaddr(struct virtio_hw *hw)
1256 {
1257         if (virtio_with_feature(hw, VIRTIO_NET_F_MAC)) {
1258                 virtio_read_dev_config(hw,
1259                         offsetof(struct virtio_net_config, mac),
1260                         &hw->mac_addr, RTE_ETHER_ADDR_LEN);
1261         } else {
1262                 rte_eth_random_addr(&hw->mac_addr[0]);
1263                 virtio_set_hwaddr(hw);
1264         }
1265 }
1266
1267 static int
1268 virtio_mac_table_set(struct virtio_hw *hw,
1269                      const struct virtio_net_ctrl_mac *uc,
1270                      const struct virtio_net_ctrl_mac *mc)
1271 {
1272         struct virtio_pmd_ctrl ctrl;
1273         int err, len[2];
1274
1275         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1276                 PMD_DRV_LOG(INFO, "host does not support mac table");
1277                 return -1;
1278         }
1279
1280         ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1281         ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1282
1283         len[0] = uc->entries * RTE_ETHER_ADDR_LEN + sizeof(uc->entries);
1284         memcpy(ctrl.data, uc, len[0]);
1285
1286         len[1] = mc->entries * RTE_ETHER_ADDR_LEN + sizeof(mc->entries);
1287         memcpy(ctrl.data + len[0], mc, len[1]);
1288
1289         err = virtio_send_command(hw->cvq, &ctrl, len, 2);
1290         if (err != 0)
1291                 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
1292         return err;
1293 }
1294
1295 static int
1296 virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1297                     uint32_t index, uint32_t vmdq __rte_unused)
1298 {
1299         struct virtio_hw *hw = dev->data->dev_private;
1300         const struct rte_ether_addr *addrs = dev->data->mac_addrs;
1301         unsigned int i;
1302         struct virtio_net_ctrl_mac *uc, *mc;
1303
1304         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1305                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1306                 return -EINVAL;
1307         }
1308
1309         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1310                 sizeof(uc->entries));
1311         uc->entries = 0;
1312         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1313                 sizeof(mc->entries));
1314         mc->entries = 0;
1315
1316         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1317                 const struct rte_ether_addr *addr
1318                         = (i == index) ? mac_addr : addrs + i;
1319                 struct virtio_net_ctrl_mac *tbl
1320                         = rte_is_multicast_ether_addr(addr) ? mc : uc;
1321
1322                 memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN);
1323         }
1324
1325         return virtio_mac_table_set(hw, uc, mc);
1326 }
1327
1328 static void
1329 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1330 {
1331         struct virtio_hw *hw = dev->data->dev_private;
1332         struct rte_ether_addr *addrs = dev->data->mac_addrs;
1333         struct virtio_net_ctrl_mac *uc, *mc;
1334         unsigned int i;
1335
1336         if (index >= VIRTIO_MAX_MAC_ADDRS) {
1337                 PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1338                 return;
1339         }
1340
1341         uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1342                 sizeof(uc->entries));
1343         uc->entries = 0;
1344         mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN +
1345                 sizeof(mc->entries));
1346         mc->entries = 0;
1347
1348         for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1349                 struct virtio_net_ctrl_mac *tbl;
1350
1351                 if (i == index || rte_is_zero_ether_addr(addrs + i))
1352                         continue;
1353
1354                 tbl = rte_is_multicast_ether_addr(addrs + i) ? mc : uc;
1355                 memcpy(&tbl->macs[tbl->entries++], addrs + i,
1356                         RTE_ETHER_ADDR_LEN);
1357         }
1358
1359         virtio_mac_table_set(hw, uc, mc);
1360 }
1361
1362 static int
1363 virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1364 {
1365         struct virtio_hw *hw = dev->data->dev_private;
1366
1367         memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1368
1369         /* Use atomic update if available */
1370         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1371                 struct virtio_pmd_ctrl ctrl;
1372                 int len = RTE_ETHER_ADDR_LEN;
1373
1374                 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1375                 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1376
1377                 memcpy(ctrl.data, mac_addr, RTE_ETHER_ADDR_LEN);
1378                 return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1379         }
1380
1381         if (!virtio_with_feature(hw, VIRTIO_NET_F_MAC))
1382                 return -ENOTSUP;
1383
1384         virtio_set_hwaddr(hw);
1385         return 0;
1386 }
1387
1388 #define CLB_VAL_IDX 0
1389 #define CLB_MSK_IDX 1
1390 #define CLB_MATCH_IDX 2
1391 static int
1392 virtio_monitor_callback(const uint64_t value,
1393                 const uint64_t opaque[RTE_POWER_MONITOR_OPAQUE_SZ])
1394 {
1395         const uint64_t m = opaque[CLB_MSK_IDX];
1396         const uint64_t v = opaque[CLB_VAL_IDX];
1397         const uint64_t c = opaque[CLB_MATCH_IDX];
1398
1399         if (c)
1400                 return (value & m) == v ? -1 : 0;
1401         else
1402                 return (value & m) == v ? 0 : -1;
1403 }
1404
1405 static int
1406 virtio_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
1407 {
1408         struct virtnet_rx *rxvq = rx_queue;
1409         struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1410         struct virtio_hw *hw;
1411
1412         if (vq == NULL)
1413                 return -EINVAL;
1414
1415         hw = vq->hw;
1416         if (virtio_with_packed_queue(hw)) {
1417                 struct vring_packed_desc *desc;
1418                 desc = vq->vq_packed.ring.desc;
1419                 pmc->addr = &desc[vq->vq_used_cons_idx].flags;
1420                 if (vq->vq_packed.used_wrap_counter)
1421                         pmc->opaque[CLB_VAL_IDX] =
1422                                                 VRING_PACKED_DESC_F_AVAIL_USED;
1423                 else
1424                         pmc->opaque[CLB_VAL_IDX] = 0;
1425                 pmc->opaque[CLB_MSK_IDX] = VRING_PACKED_DESC_F_AVAIL_USED;
1426                 pmc->opaque[CLB_MATCH_IDX] = 1;
1427                 pmc->size = sizeof(desc[vq->vq_used_cons_idx].flags);
1428         } else {
1429                 pmc->addr = &vq->vq_split.ring.used->idx;
1430                 pmc->opaque[CLB_VAL_IDX] = vq->vq_used_cons_idx
1431                                         & (vq->vq_nentries - 1);
1432                 pmc->opaque[CLB_MSK_IDX] = vq->vq_nentries - 1;
1433                 pmc->opaque[CLB_MATCH_IDX] = 0;
1434                 pmc->size = sizeof(vq->vq_split.ring.used->idx);
1435         }
1436         pmc->fn = virtio_monitor_callback;
1437
1438         return 0;
1439 }
1440
1441 static int
1442 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1443 {
1444         struct virtio_hw *hw = dev->data->dev_private;
1445         struct virtio_pmd_ctrl ctrl;
1446         int len;
1447
1448         if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1449                 return -ENOTSUP;
1450
1451         ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1452         ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1453         memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1454         len = sizeof(vlan_id);
1455
1456         return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1457 }
1458
1459 static int
1460 virtio_intr_unmask(struct rte_eth_dev *dev)
1461 {
1462         struct virtio_hw *hw = dev->data->dev_private;
1463
1464         if (rte_intr_ack(dev->intr_handle) < 0)
1465                 return -1;
1466
1467         if (VIRTIO_OPS(hw)->intr_detect)
1468                 VIRTIO_OPS(hw)->intr_detect(hw);
1469
1470         return 0;
1471 }
1472
1473 static int
1474 virtio_intr_enable(struct rte_eth_dev *dev)
1475 {
1476         struct virtio_hw *hw = dev->data->dev_private;
1477
1478         if (rte_intr_enable(dev->intr_handle) < 0)
1479                 return -1;
1480
1481         if (VIRTIO_OPS(hw)->intr_detect)
1482                 VIRTIO_OPS(hw)->intr_detect(hw);
1483
1484         return 0;
1485 }
1486
1487 static int
1488 virtio_intr_disable(struct rte_eth_dev *dev)
1489 {
1490         struct virtio_hw *hw = dev->data->dev_private;
1491
1492         if (rte_intr_disable(dev->intr_handle) < 0)
1493                 return -1;
1494
1495         if (VIRTIO_OPS(hw)->intr_detect)
1496                 VIRTIO_OPS(hw)->intr_detect(hw);
1497
1498         return 0;
1499 }
1500
1501 static int
1502 virtio_ethdev_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1503 {
1504         uint64_t host_features;
1505
1506         /* Prepare guest_features: feature that driver wants to support */
1507         PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1508                 req_features);
1509
1510         /* Read device(host) feature bits */
1511         host_features = VIRTIO_OPS(hw)->get_features(hw);
1512         PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1513                 host_features);
1514
1515         /* If supported, ensure MTU value is valid before acknowledging it. */
1516         if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) {
1517                 struct virtio_net_config config;
1518
1519                 virtio_read_dev_config(hw,
1520                         offsetof(struct virtio_net_config, mtu),
1521                         &config.mtu, sizeof(config.mtu));
1522
1523                 if (config.mtu < RTE_ETHER_MIN_MTU)
1524                         req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
1525         }
1526
1527         /*
1528          * Negotiate features: Subset of device feature bits are written back
1529          * guest feature bits.
1530          */
1531         hw->guest_features = req_features;
1532         hw->guest_features = virtio_negotiate_features(hw, host_features);
1533         PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1534                 hw->guest_features);
1535
1536         if (VIRTIO_OPS(hw)->features_ok(hw) < 0)
1537                 return -1;
1538
1539         if (virtio_with_feature(hw, VIRTIO_F_VERSION_1)) {
1540                 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1541
1542                 if (!(virtio_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1543                         PMD_INIT_LOG(ERR, "Failed to set FEATURES_OK status!");
1544                         return -1;
1545                 }
1546         }
1547
1548         hw->req_guest_features = req_features;
1549
1550         return 0;
1551 }
1552
1553 int
1554 virtio_dev_pause(struct rte_eth_dev *dev)
1555 {
1556         struct virtio_hw *hw = dev->data->dev_private;
1557
1558         rte_spinlock_lock(&hw->state_lock);
1559
1560         if (hw->started == 0) {
1561                 /* Device is just stopped. */
1562                 rte_spinlock_unlock(&hw->state_lock);
1563                 return -1;
1564         }
1565         hw->started = 0;
1566         /*
1567          * Prevent the worker threads from touching queues to avoid contention,
1568          * 1 ms should be enough for the ongoing Tx function to finish.
1569          */
1570         rte_delay_ms(1);
1571         return 0;
1572 }
1573
1574 /*
1575  * Recover hw state to let the worker threads continue.
1576  */
1577 void
1578 virtio_dev_resume(struct rte_eth_dev *dev)
1579 {
1580         struct virtio_hw *hw = dev->data->dev_private;
1581
1582         hw->started = 1;
1583         rte_spinlock_unlock(&hw->state_lock);
1584 }
1585
1586 /*
1587  * Should be called only after device is paused.
1588  */
1589 int
1590 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts,
1591                 int nb_pkts)
1592 {
1593         struct virtio_hw *hw = dev->data->dev_private;
1594         struct virtnet_tx *txvq = dev->data->tx_queues[0];
1595         int ret;
1596
1597         hw->inject_pkts = tx_pkts;
1598         ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts);
1599         hw->inject_pkts = NULL;
1600
1601         return ret;
1602 }
1603
1604 static void
1605 virtio_notify_peers(struct rte_eth_dev *dev)
1606 {
1607         struct virtio_hw *hw = dev->data->dev_private;
1608         struct virtnet_rx *rxvq;
1609         struct rte_mbuf *rarp_mbuf;
1610
1611         if (!dev->data->rx_queues)
1612                 return;
1613
1614         rxvq = dev->data->rx_queues[0];
1615         if (!rxvq)
1616                 return;
1617
1618         rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool,
1619                         (struct rte_ether_addr *)hw->mac_addr);
1620         if (rarp_mbuf == NULL) {
1621                 PMD_DRV_LOG(ERR, "failed to make RARP packet.");
1622                 return;
1623         }
1624
1625         /* If virtio port just stopped, no need to send RARP */
1626         if (virtio_dev_pause(dev) < 0) {
1627                 rte_pktmbuf_free(rarp_mbuf);
1628                 return;
1629         }
1630
1631         virtio_inject_pkts(dev, &rarp_mbuf, 1);
1632         virtio_dev_resume(dev);
1633 }
1634
1635 static void
1636 virtio_ack_link_announce(struct rte_eth_dev *dev)
1637 {
1638         struct virtio_hw *hw = dev->data->dev_private;
1639         struct virtio_pmd_ctrl ctrl;
1640
1641         ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE;
1642         ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK;
1643
1644         virtio_send_command(hw->cvq, &ctrl, NULL, 0);
1645 }
1646
1647 /*
1648  * Process virtio config changed interrupt. Call the callback
1649  * if link state changed, generate gratuitous RARP packet if
1650  * the status indicates an ANNOUNCE.
1651  */
1652 void
1653 virtio_interrupt_handler(void *param)
1654 {
1655         struct rte_eth_dev *dev = param;
1656         struct virtio_hw *hw = dev->data->dev_private;
1657         uint8_t isr;
1658         uint16_t status;
1659
1660         /* Read interrupt status which clears interrupt */
1661         isr = virtio_get_isr(hw);
1662         PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1663
1664         if (virtio_intr_unmask(dev) < 0)
1665                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1666
1667         if (isr & VIRTIO_ISR_CONFIG) {
1668                 if (virtio_dev_link_update(dev, 0) == 0)
1669                         rte_eth_dev_callback_process(dev,
1670                                                      RTE_ETH_EVENT_INTR_LSC,
1671                                                      NULL);
1672
1673                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1674                         virtio_read_dev_config(hw,
1675                                 offsetof(struct virtio_net_config, status),
1676                                 &status, sizeof(status));
1677                         if (status & VIRTIO_NET_S_ANNOUNCE) {
1678                                 virtio_notify_peers(dev);
1679                                 if (hw->cvq)
1680                                         virtio_ack_link_announce(dev);
1681                         }
1682                 }
1683         }
1684 }
1685
1686 /* set rx and tx handlers according to what is supported */
1687 static void
1688 set_rxtx_funcs(struct rte_eth_dev *eth_dev)
1689 {
1690         struct virtio_hw *hw = eth_dev->data->dev_private;
1691
1692         eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare;
1693         if (virtio_with_packed_queue(hw)) {
1694                 PMD_INIT_LOG(INFO,
1695                         "virtio: using packed ring %s Tx path on port %u",
1696                         hw->use_vec_tx ? "vectorized" : "standard",
1697                         eth_dev->data->port_id);
1698                 if (hw->use_vec_tx)
1699                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
1700                 else
1701                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
1702         } else {
1703                 if (hw->use_inorder_tx) {
1704                         PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u",
1705                                 eth_dev->data->port_id);
1706                         eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder;
1707                 } else {
1708                         PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u",
1709                                 eth_dev->data->port_id);
1710                         eth_dev->tx_pkt_burst = virtio_xmit_pkts;
1711                 }
1712         }
1713
1714         if (virtio_with_packed_queue(hw)) {
1715                 if (hw->use_vec_rx) {
1716                         PMD_INIT_LOG(INFO,
1717                                 "virtio: using packed ring vectorized Rx path on port %u",
1718                                 eth_dev->data->port_id);
1719                         eth_dev->rx_pkt_burst =
1720                                 &virtio_recv_pkts_packed_vec;
1721                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1722                         PMD_INIT_LOG(INFO,
1723                                 "virtio: using packed ring mergeable buffer Rx path on port %u",
1724                                 eth_dev->data->port_id);
1725                         eth_dev->rx_pkt_burst =
1726                                 &virtio_recv_mergeable_pkts_packed;
1727                 } else {
1728                         PMD_INIT_LOG(INFO,
1729                                 "virtio: using packed ring standard Rx path on port %u",
1730                                 eth_dev->data->port_id);
1731                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
1732                 }
1733         } else {
1734                 if (hw->use_vec_rx) {
1735                         PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on port %u",
1736                                 eth_dev->data->port_id);
1737                         eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
1738                 } else if (hw->use_inorder_rx) {
1739                         PMD_INIT_LOG(INFO,
1740                                 "virtio: using inorder Rx path on port %u",
1741                                 eth_dev->data->port_id);
1742                         eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder;
1743                 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1744                         PMD_INIT_LOG(INFO,
1745                                 "virtio: using mergeable buffer Rx path on port %u",
1746                                 eth_dev->data->port_id);
1747                         eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1748                 } else {
1749                         PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
1750                                 eth_dev->data->port_id);
1751                         eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1752                 }
1753         }
1754
1755 }
1756
1757 /* Only support 1:1 queue/interrupt mapping so far.
1758  * TODO: support n:1 queue/interrupt mapping when there are limited number of
1759  * interrupt vectors (<N+1).
1760  */
1761 static int
1762 virtio_queues_bind_intr(struct rte_eth_dev *dev)
1763 {
1764         uint32_t i;
1765         struct virtio_hw *hw = dev->data->dev_private;
1766
1767         PMD_INIT_LOG(INFO, "queue/interrupt binding");
1768         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
1769                 if (rte_intr_vec_list_index_set(dev->intr_handle, i,
1770                                                        i + 1))
1771                         return -rte_errno;
1772                 if (VIRTIO_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) ==
1773                                                  VIRTIO_MSI_NO_VECTOR) {
1774                         PMD_DRV_LOG(ERR, "failed to set queue vector");
1775                         return -EBUSY;
1776                 }
1777         }
1778
1779         return 0;
1780 }
1781
1782 static void
1783 virtio_queues_unbind_intr(struct rte_eth_dev *dev)
1784 {
1785         uint32_t i;
1786         struct virtio_hw *hw = dev->data->dev_private;
1787
1788         PMD_INIT_LOG(INFO, "queue/interrupt unbinding");
1789         for (i = 0; i < dev->data->nb_rx_queues; ++i)
1790                 VIRTIO_OPS(hw)->set_queue_irq(hw,
1791                                              hw->vqs[i * VTNET_CQ],
1792                                              VIRTIO_MSI_NO_VECTOR);
1793 }
1794
1795 static int
1796 virtio_configure_intr(struct rte_eth_dev *dev)
1797 {
1798         struct virtio_hw *hw = dev->data->dev_private;
1799
1800         if (!rte_intr_cap_multiple(dev->intr_handle)) {
1801                 PMD_INIT_LOG(ERR, "Multiple intr vector not supported");
1802                 return -ENOTSUP;
1803         }
1804
1805         if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) {
1806                 PMD_INIT_LOG(ERR, "Fail to create eventfd");
1807                 return -1;
1808         }
1809
1810         if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec",
1811                                     hw->max_queue_pairs)) {
1812                 PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors",
1813                              hw->max_queue_pairs);
1814                 return -ENOMEM;
1815         }
1816
1817         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1818                 /* Re-register callback to update max_intr */
1819                 rte_intr_callback_unregister(dev->intr_handle,
1820                                              virtio_interrupt_handler,
1821                                              dev);
1822                 rte_intr_callback_register(dev->intr_handle,
1823                                            virtio_interrupt_handler,
1824                                            dev);
1825         }
1826
1827         /* DO NOT try to remove this! This function will enable msix, or QEMU
1828          * will encounter SIGSEGV when DRIVER_OK is sent.
1829          * And for legacy devices, this should be done before queue/vec binding
1830          * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1831          * (22) will be ignored.
1832          */
1833         if (virtio_intr_enable(dev) < 0) {
1834                 PMD_DRV_LOG(ERR, "interrupt enable failed");
1835                 return -1;
1836         }
1837
1838         if (virtio_queues_bind_intr(dev) < 0) {
1839                 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt");
1840                 return -1;
1841         }
1842
1843         return 0;
1844 }
1845
1846 static uint64_t
1847 ethdev_to_virtio_rss_offloads(uint64_t ethdev_hash_types)
1848 {
1849         uint64_t virtio_hash_types = 0;
1850
1851         if (ethdev_hash_types & (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
1852                                 RTE_ETH_RSS_NONFRAG_IPV4_OTHER))
1853                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_IPV4;
1854
1855         if (ethdev_hash_types & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
1856                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_TCPV4;
1857
1858         if (ethdev_hash_types & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
1859                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_UDPV4;
1860
1861         if (ethdev_hash_types & (RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
1862                                 RTE_ETH_RSS_NONFRAG_IPV6_OTHER))
1863                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_IPV6;
1864
1865         if (ethdev_hash_types & RTE_ETH_RSS_NONFRAG_IPV6_TCP)
1866                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_TCPV6;
1867
1868         if (ethdev_hash_types & RTE_ETH_RSS_NONFRAG_IPV6_UDP)
1869                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_UDPV6;
1870
1871         if (ethdev_hash_types & RTE_ETH_RSS_IPV6_EX)
1872                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_IP_EX;
1873
1874         if (ethdev_hash_types & RTE_ETH_RSS_IPV6_TCP_EX)
1875                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_TCP_EX;
1876
1877         if (ethdev_hash_types & RTE_ETH_RSS_IPV6_UDP_EX)
1878                 virtio_hash_types |= VIRTIO_NET_HASH_TYPE_UDP_EX;
1879
1880         return virtio_hash_types;
1881 }
1882
1883 static uint64_t
1884 virtio_to_ethdev_rss_offloads(uint64_t virtio_hash_types)
1885 {
1886         uint64_t rss_offloads = 0;
1887
1888         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_IPV4)
1889                 rss_offloads |= RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
1890                         RTE_ETH_RSS_NONFRAG_IPV4_OTHER;
1891
1892         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_TCPV4)
1893                 rss_offloads |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
1894
1895         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_UDPV4)
1896                 rss_offloads |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
1897
1898         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_IPV6)
1899                 rss_offloads |= RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 |
1900                         RTE_ETH_RSS_NONFRAG_IPV6_OTHER;
1901
1902         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_TCPV6)
1903                 rss_offloads |= RTE_ETH_RSS_NONFRAG_IPV6_TCP;
1904
1905         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_UDPV6)
1906                 rss_offloads |= RTE_ETH_RSS_NONFRAG_IPV6_UDP;
1907
1908         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_IP_EX)
1909                 rss_offloads |= RTE_ETH_RSS_IPV6_EX;
1910
1911         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_TCP_EX)
1912                 rss_offloads |= RTE_ETH_RSS_IPV6_TCP_EX;
1913
1914         if (virtio_hash_types & VIRTIO_NET_HASH_TYPE_UDP_EX)
1915                 rss_offloads |= RTE_ETH_RSS_IPV6_UDP_EX;
1916
1917         return rss_offloads;
1918 }
1919
1920 static int
1921 virtio_dev_get_rss_config(struct virtio_hw *hw, uint32_t *rss_hash_types)
1922 {
1923         struct virtio_net_config local_config;
1924         struct virtio_net_config *config = &local_config;
1925
1926         virtio_read_dev_config(hw,
1927                         offsetof(struct virtio_net_config, rss_max_key_size),
1928                         &config->rss_max_key_size,
1929                         sizeof(config->rss_max_key_size));
1930         if (config->rss_max_key_size < VIRTIO_NET_RSS_KEY_SIZE) {
1931                 PMD_INIT_LOG(ERR, "Invalid device RSS max key size (%u)",
1932                                 config->rss_max_key_size);
1933                 return -EINVAL;
1934         }
1935
1936         virtio_read_dev_config(hw,
1937                         offsetof(struct virtio_net_config,
1938                                 rss_max_indirection_table_length),
1939                         &config->rss_max_indirection_table_length,
1940                         sizeof(config->rss_max_indirection_table_length));
1941         if (config->rss_max_indirection_table_length < VIRTIO_NET_RSS_RETA_SIZE) {
1942                 PMD_INIT_LOG(ERR, "Invalid device RSS max reta size (%u)",
1943                                 config->rss_max_indirection_table_length);
1944                 return -EINVAL;
1945         }
1946
1947         virtio_read_dev_config(hw,
1948                         offsetof(struct virtio_net_config, supported_hash_types),
1949                         &config->supported_hash_types,
1950                         sizeof(config->supported_hash_types));
1951         if ((config->supported_hash_types & VIRTIO_NET_HASH_TYPE_MASK) == 0) {
1952                 PMD_INIT_LOG(ERR, "Invalid device RSS hash types (0x%x)",
1953                                 config->supported_hash_types);
1954                 return -EINVAL;
1955         }
1956
1957         *rss_hash_types = config->supported_hash_types & VIRTIO_NET_HASH_TYPE_MASK;
1958
1959         PMD_INIT_LOG(DEBUG, "Device RSS config:");
1960         PMD_INIT_LOG(DEBUG, "\t-Max key size: %u", config->rss_max_key_size);
1961         PMD_INIT_LOG(DEBUG, "\t-Max reta size: %u", config->rss_max_indirection_table_length);
1962         PMD_INIT_LOG(DEBUG, "\t-Supported hash types: 0x%x", *rss_hash_types);
1963
1964         return 0;
1965 }
1966
1967 static int
1968 virtio_dev_rss_hash_update(struct rte_eth_dev *dev,
1969                 struct rte_eth_rss_conf *rss_conf)
1970 {
1971         struct virtio_hw *hw = dev->data->dev_private;
1972         char old_rss_key[VIRTIO_NET_RSS_KEY_SIZE];
1973         uint32_t old_hash_types;
1974         uint16_t nb_queues;
1975         int ret;
1976
1977         if (!virtio_with_feature(hw, VIRTIO_NET_F_RSS))
1978                 return -ENOTSUP;
1979
1980         if (rss_conf->rss_hf & ~virtio_to_ethdev_rss_offloads(VIRTIO_NET_HASH_TYPE_MASK))
1981                 return -EINVAL;
1982
1983         old_hash_types = hw->rss_hash_types;
1984         hw->rss_hash_types = ethdev_to_virtio_rss_offloads(rss_conf->rss_hf);
1985
1986         if (rss_conf->rss_key && rss_conf->rss_key_len) {
1987                 if (rss_conf->rss_key_len != VIRTIO_NET_RSS_KEY_SIZE) {
1988                         PMD_INIT_LOG(ERR, "Driver only supports %u RSS key length",
1989                                         VIRTIO_NET_RSS_KEY_SIZE);
1990                         ret = -EINVAL;
1991                         goto restore_types;
1992                 }
1993                 memcpy(old_rss_key, hw->rss_key, VIRTIO_NET_RSS_KEY_SIZE);
1994                 memcpy(hw->rss_key, rss_conf->rss_key, VIRTIO_NET_RSS_KEY_SIZE);
1995         }
1996
1997         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
1998         ret = virtio_set_multiple_queues_rss(dev, nb_queues);
1999         if (ret < 0) {
2000                 PMD_INIT_LOG(ERR, "Failed to apply new RSS config to the device");
2001                 goto restore_key;
2002         }
2003
2004         return 0;
2005 restore_key:
2006         memcpy(hw->rss_key, old_rss_key, VIRTIO_NET_RSS_KEY_SIZE);
2007 restore_types:
2008         hw->rss_hash_types = old_hash_types;
2009
2010         return ret;
2011 }
2012
2013 static int
2014 virtio_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2015                 struct rte_eth_rss_conf *rss_conf)
2016 {
2017         struct virtio_hw *hw = dev->data->dev_private;
2018
2019         if (!virtio_with_feature(hw, VIRTIO_NET_F_RSS))
2020                 return -ENOTSUP;
2021
2022         if (rss_conf->rss_key && rss_conf->rss_key_len >= VIRTIO_NET_RSS_KEY_SIZE)
2023                 memcpy(rss_conf->rss_key, hw->rss_key, VIRTIO_NET_RSS_KEY_SIZE);
2024         rss_conf->rss_key_len = VIRTIO_NET_RSS_KEY_SIZE;
2025         rss_conf->rss_hf = virtio_to_ethdev_rss_offloads(hw->rss_hash_types);
2026
2027         return 0;
2028 }
2029
2030 static int virtio_dev_rss_reta_update(struct rte_eth_dev *dev,
2031                          struct rte_eth_rss_reta_entry64 *reta_conf,
2032                          uint16_t reta_size)
2033 {
2034         struct virtio_hw *hw = dev->data->dev_private;
2035         uint16_t nb_queues;
2036         uint16_t old_reta[VIRTIO_NET_RSS_RETA_SIZE];
2037         int idx, pos, i, ret;
2038
2039         if (!virtio_with_feature(hw, VIRTIO_NET_F_RSS))
2040                 return -ENOTSUP;
2041
2042         if (reta_size != VIRTIO_NET_RSS_RETA_SIZE)
2043                 return -EINVAL;
2044
2045         memcpy(old_reta, hw->rss_reta, sizeof(old_reta));
2046
2047         for (i = 0; i < reta_size; i++) {
2048                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2049                 pos = i % RTE_ETH_RETA_GROUP_SIZE;
2050
2051                 if (((reta_conf[idx].mask >> pos) & 0x1) == 0)
2052                         continue;
2053
2054                 hw->rss_reta[i] = reta_conf[idx].reta[pos];
2055         }
2056
2057         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
2058         ret = virtio_set_multiple_queues_rss(dev, nb_queues);
2059         if (ret < 0) {
2060                 PMD_INIT_LOG(ERR, "Failed to apply new RETA to the device");
2061                 memcpy(hw->rss_reta, old_reta, sizeof(old_reta));
2062         }
2063
2064         hw->rss_rx_queues = dev->data->nb_rx_queues;
2065
2066         return ret;
2067 }
2068
2069 static int virtio_dev_rss_reta_query(struct rte_eth_dev *dev,
2070                          struct rte_eth_rss_reta_entry64 *reta_conf,
2071                          uint16_t reta_size)
2072 {
2073         struct virtio_hw *hw = dev->data->dev_private;
2074         int idx, i;
2075
2076         if (!virtio_with_feature(hw, VIRTIO_NET_F_RSS))
2077                 return -ENOTSUP;
2078
2079         if (reta_size != VIRTIO_NET_RSS_RETA_SIZE)
2080                 return -EINVAL;
2081
2082         for (i = 0; i < reta_size; i++) {
2083                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2084                 reta_conf[idx].reta[i % RTE_ETH_RETA_GROUP_SIZE] = hw->rss_reta[i];
2085         }
2086
2087         return 0;
2088 }
2089
2090 /*
2091  * As default RSS hash key, it uses the default key of the
2092  * Intel IXGBE devices. It can be updated by the application
2093  * with any 40B key value.
2094  */
2095 static uint8_t rss_intel_key[VIRTIO_NET_RSS_KEY_SIZE] = {
2096         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2097         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2098         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2099         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2100         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2101 };
2102
2103 static int
2104 virtio_dev_rss_init(struct rte_eth_dev *eth_dev)
2105 {
2106         struct virtio_hw *hw = eth_dev->data->dev_private;
2107         uint16_t nb_rx_queues = eth_dev->data->nb_rx_queues;
2108         struct rte_eth_rss_conf *rss_conf;
2109         int ret, i;
2110
2111         if (!nb_rx_queues) {
2112                 PMD_INIT_LOG(ERR, "Cannot init RSS if no Rx queues");
2113                 return -EINVAL;
2114         }
2115
2116         rss_conf = &eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
2117
2118         ret = virtio_dev_get_rss_config(hw, &hw->rss_hash_types);
2119         if (ret)
2120                 return ret;
2121
2122         if (rss_conf->rss_hf) {
2123                 /*  Ensure requested hash types are supported by the device */
2124                 if (rss_conf->rss_hf & ~virtio_to_ethdev_rss_offloads(hw->rss_hash_types))
2125                         return -EINVAL;
2126
2127                 hw->rss_hash_types = ethdev_to_virtio_rss_offloads(rss_conf->rss_hf);
2128         }
2129
2130         if (!hw->rss_key) {
2131                 /* Setup default RSS key if not already setup by the user */
2132                 hw->rss_key = rte_malloc_socket("rss_key",
2133                                 VIRTIO_NET_RSS_KEY_SIZE, 0,
2134                                 eth_dev->device->numa_node);
2135                 if (!hw->rss_key) {
2136                         PMD_INIT_LOG(ERR, "Failed to allocate RSS key");
2137                         return -1;
2138                 }
2139         }
2140
2141         if (rss_conf->rss_key && rss_conf->rss_key_len) {
2142                 if (rss_conf->rss_key_len != VIRTIO_NET_RSS_KEY_SIZE) {
2143                         PMD_INIT_LOG(ERR, "Driver only supports %u RSS key length",
2144                                         VIRTIO_NET_RSS_KEY_SIZE);
2145                         return -EINVAL;
2146                 }
2147                 memcpy(hw->rss_key, rss_conf->rss_key, VIRTIO_NET_RSS_KEY_SIZE);
2148         } else {
2149                 memcpy(hw->rss_key, rss_intel_key, VIRTIO_NET_RSS_KEY_SIZE);
2150         }
2151
2152         if (!hw->rss_reta) {
2153                 /* Setup default RSS reta if not already setup by the user */
2154                 hw->rss_reta = rte_zmalloc_socket("rss_reta",
2155                                 VIRTIO_NET_RSS_RETA_SIZE * sizeof(uint16_t), 0,
2156                                 eth_dev->device->numa_node);
2157                 if (!hw->rss_reta) {
2158                         PMD_INIT_LOG(ERR, "Failed to allocate RSS reta");
2159                         return -1;
2160                 }
2161
2162                 hw->rss_rx_queues = 0;
2163         }
2164
2165         /* Re-initialize the RSS reta if the number of RX queues has changed */
2166         if (hw->rss_rx_queues != nb_rx_queues) {
2167                 for (i = 0; i < VIRTIO_NET_RSS_RETA_SIZE; i++)
2168                         hw->rss_reta[i] = i % nb_rx_queues;
2169                 hw->rss_rx_queues = nb_rx_queues;
2170         }
2171
2172         return 0;
2173 }
2174
2175 #define DUPLEX_UNKNOWN   0xff
2176 /* reset device and renegotiate features if needed */
2177 static int
2178 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
2179 {
2180         struct virtio_hw *hw = eth_dev->data->dev_private;
2181         struct virtio_net_config *config;
2182         struct virtio_net_config local_config;
2183         int ret;
2184
2185         /* Reset the device although not necessary at startup */
2186         virtio_reset(hw);
2187
2188         if (hw->vqs) {
2189                 virtio_dev_free_mbufs(eth_dev);
2190                 virtio_free_queues(hw);
2191         }
2192
2193         /* Tell the host we've noticed this device. */
2194         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
2195
2196         /* Tell the host we've known how to drive the device. */
2197         virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
2198         if (virtio_ethdev_negotiate_features(hw, req_features) < 0)
2199                 return -1;
2200
2201         hw->weak_barriers = !virtio_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
2202
2203         /* If host does not support both status and MSI-X then disable LSC */
2204         if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS) && hw->intr_lsc)
2205                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
2206         else
2207                 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
2208
2209         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2210
2211         /* Setting up rx_header size for the device */
2212         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
2213             virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2214             virtio_with_packed_queue(hw))
2215                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2216         else
2217                 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
2218
2219         /* Copy the permanent MAC address to: virtio_hw */
2220         virtio_get_hwaddr(hw);
2221         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
2222                         &eth_dev->data->mac_addrs[0]);
2223         PMD_INIT_LOG(DEBUG,
2224                      "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
2225                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
2226                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
2227
2228         if (hw->speed == RTE_ETH_SPEED_NUM_UNKNOWN) {
2229                 if (virtio_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) {
2230                         config = &local_config;
2231                         virtio_read_dev_config(hw,
2232                                 offsetof(struct virtio_net_config, speed),
2233                                 &config->speed, sizeof(config->speed));
2234                         virtio_read_dev_config(hw,
2235                                 offsetof(struct virtio_net_config, duplex),
2236                                 &config->duplex, sizeof(config->duplex));
2237                         hw->speed = config->speed;
2238                         hw->duplex = config->duplex;
2239                 }
2240         }
2241         if (hw->duplex == DUPLEX_UNKNOWN)
2242                 hw->duplex = RTE_ETH_LINK_FULL_DUPLEX;
2243         PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d",
2244                 hw->speed, hw->duplex);
2245         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
2246                 config = &local_config;
2247
2248                 virtio_read_dev_config(hw,
2249                         offsetof(struct virtio_net_config, mac),
2250                         &config->mac, sizeof(config->mac));
2251
2252                 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
2253                         virtio_read_dev_config(hw,
2254                                 offsetof(struct virtio_net_config, status),
2255                                 &config->status, sizeof(config->status));
2256                 } else {
2257                         PMD_INIT_LOG(DEBUG,
2258                                      "VIRTIO_NET_F_STATUS is not supported");
2259                         config->status = 0;
2260                 }
2261
2262                 if (virtio_with_feature(hw, VIRTIO_NET_F_MQ) ||
2263                                 virtio_with_feature(hw, VIRTIO_NET_F_RSS)) {
2264                         virtio_read_dev_config(hw,
2265                                 offsetof(struct virtio_net_config, max_virtqueue_pairs),
2266                                 &config->max_virtqueue_pairs,
2267                                 sizeof(config->max_virtqueue_pairs));
2268                 } else {
2269                         PMD_INIT_LOG(DEBUG,
2270                                      "Neither VIRTIO_NET_F_MQ nor VIRTIO_NET_F_RSS are supported");
2271                         config->max_virtqueue_pairs = 1;
2272                 }
2273
2274                 hw->max_queue_pairs = config->max_virtqueue_pairs;
2275
2276                 if (virtio_with_feature(hw, VIRTIO_NET_F_MTU)) {
2277                         virtio_read_dev_config(hw,
2278                                 offsetof(struct virtio_net_config, mtu),
2279                                 &config->mtu,
2280                                 sizeof(config->mtu));
2281
2282                         /*
2283                          * MTU value has already been checked at negotiation
2284                          * time, but check again in case it has changed since
2285                          * then, which should not happen.
2286                          */
2287                         if (config->mtu < RTE_ETHER_MIN_MTU) {
2288                                 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)",
2289                                                 config->mtu);
2290                                 return -1;
2291                         }
2292
2293                         hw->max_mtu = config->mtu;
2294                         /* Set initial MTU to maximum one supported by vhost */
2295                         eth_dev->data->mtu = config->mtu;
2296
2297                 } else {
2298                         hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
2299                                 VLAN_TAG_LEN - hw->vtnet_hdr_size;
2300                 }
2301
2302                 hw->rss_hash_types = 0;
2303                 if (virtio_with_feature(hw, VIRTIO_NET_F_RSS))
2304                         if (virtio_dev_rss_init(eth_dev))
2305                                 return -1;
2306
2307                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
2308                                 config->max_virtqueue_pairs);
2309                 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
2310                 PMD_INIT_LOG(DEBUG,
2311                                 "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT,
2312                                 config->mac[0], config->mac[1],
2313                                 config->mac[2], config->mac[3],
2314                                 config->mac[4], config->mac[5]);
2315         } else {
2316                 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
2317                 hw->max_queue_pairs = 1;
2318                 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN -
2319                         VLAN_TAG_LEN - hw->vtnet_hdr_size;
2320         }
2321
2322         ret = virtio_alloc_queues(eth_dev);
2323         if (ret < 0)
2324                 return ret;
2325
2326         if (eth_dev->data->dev_conf.intr_conf.rxq) {
2327                 if (virtio_configure_intr(eth_dev) < 0) {
2328                         PMD_INIT_LOG(ERR, "failed to configure interrupt");
2329                         virtio_free_queues(hw);
2330                         return -1;
2331                 }
2332         }
2333
2334         virtio_reinit_complete(hw);
2335
2336         return 0;
2337 }
2338
2339 /*
2340  * This function is based on probe() function in virtio_pci.c
2341  * It returns 0 on success.
2342  */
2343 int
2344 eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
2345 {
2346         struct virtio_hw *hw = eth_dev->data->dev_private;
2347         uint32_t speed = RTE_ETH_SPEED_NUM_UNKNOWN;
2348         int vectorized = 0;
2349         int ret;
2350
2351         if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
2352                 PMD_INIT_LOG(ERR,
2353                         "Not sufficient headroom required = %d, avail = %d",
2354                         (int)sizeof(struct virtio_net_hdr_mrg_rxbuf),
2355                         RTE_PKTMBUF_HEADROOM);
2356
2357                 return -1;
2358         }
2359
2360         eth_dev->dev_ops = &virtio_eth_dev_ops;
2361
2362         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
2363                 set_rxtx_funcs(eth_dev);
2364                 return 0;
2365         }
2366
2367         ret = virtio_dev_devargs_parse(eth_dev->device->devargs, &speed, &vectorized);
2368         if (ret < 0)
2369                 return ret;
2370         hw->speed = speed;
2371         hw->duplex = DUPLEX_UNKNOWN;
2372
2373         /* Allocate memory for storing MAC addresses */
2374         eth_dev->data->mac_addrs = rte_zmalloc("virtio",
2375                                 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN, 0);
2376         if (eth_dev->data->mac_addrs == NULL) {
2377                 PMD_INIT_LOG(ERR,
2378                         "Failed to allocate %d bytes needed to store MAC addresses",
2379                         VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN);
2380                 return -ENOMEM;
2381         }
2382
2383         rte_spinlock_init(&hw->state_lock);
2384
2385         /* reset device and negotiate default features */
2386         ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
2387         if (ret < 0)
2388                 goto err_virtio_init;
2389
2390         if (vectorized) {
2391                 if (!virtio_with_packed_queue(hw)) {
2392                         hw->use_vec_rx = 1;
2393                 } else {
2394 #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
2395                         hw->use_vec_rx = 1;
2396                         hw->use_vec_tx = 1;
2397 #else
2398                         PMD_DRV_LOG(INFO,
2399                                 "building environment do not support packed ring vectorized");
2400 #endif
2401                 }
2402         }
2403
2404         hw->opened = 1;
2405
2406         return 0;
2407
2408 err_virtio_init:
2409         rte_free(eth_dev->data->mac_addrs);
2410         eth_dev->data->mac_addrs = NULL;
2411         return ret;
2412 }
2413
2414 static uint32_t
2415 virtio_dev_speed_capa_get(uint32_t speed)
2416 {
2417         switch (speed) {
2418         case RTE_ETH_SPEED_NUM_10G:
2419                 return RTE_ETH_LINK_SPEED_10G;
2420         case RTE_ETH_SPEED_NUM_20G:
2421                 return RTE_ETH_LINK_SPEED_20G;
2422         case RTE_ETH_SPEED_NUM_25G:
2423                 return RTE_ETH_LINK_SPEED_25G;
2424         case RTE_ETH_SPEED_NUM_40G:
2425                 return RTE_ETH_LINK_SPEED_40G;
2426         case RTE_ETH_SPEED_NUM_50G:
2427                 return RTE_ETH_LINK_SPEED_50G;
2428         case RTE_ETH_SPEED_NUM_56G:
2429                 return RTE_ETH_LINK_SPEED_56G;
2430         case RTE_ETH_SPEED_NUM_100G:
2431                 return RTE_ETH_LINK_SPEED_100G;
2432         case RTE_ETH_SPEED_NUM_200G:
2433                 return RTE_ETH_LINK_SPEED_200G;
2434         default:
2435                 return 0;
2436         }
2437 }
2438
2439 static int vectorized_check_handler(__rte_unused const char *key,
2440                 const char *value, void *ret_val)
2441 {
2442         if (strcmp(value, "1") == 0)
2443                 *(int *)ret_val = 1;
2444         else
2445                 *(int *)ret_val = 0;
2446
2447         return 0;
2448 }
2449
2450 #define VIRTIO_ARG_SPEED      "speed"
2451 #define VIRTIO_ARG_VECTORIZED "vectorized"
2452
2453 static int
2454 link_speed_handler(const char *key __rte_unused,
2455                 const char *value, void *ret_val)
2456 {
2457         uint32_t val;
2458         if (!value || !ret_val)
2459                 return -EINVAL;
2460         val = strtoul(value, NULL, 0);
2461         /* validate input */
2462         if (virtio_dev_speed_capa_get(val) == 0)
2463                 return -EINVAL;
2464         *(uint32_t *)ret_val = val;
2465
2466         return 0;
2467 }
2468
2469
2470 static int
2471 virtio_dev_devargs_parse(struct rte_devargs *devargs, uint32_t *speed, int *vectorized)
2472 {
2473         struct rte_kvargs *kvlist;
2474         int ret = 0;
2475
2476         if (devargs == NULL)
2477                 return 0;
2478
2479         kvlist = rte_kvargs_parse(devargs->args, NULL);
2480         if (kvlist == NULL) {
2481                 PMD_INIT_LOG(ERR, "error when parsing param");
2482                 return 0;
2483         }
2484
2485         if (speed && rte_kvargs_count(kvlist, VIRTIO_ARG_SPEED) == 1) {
2486                 ret = rte_kvargs_process(kvlist,
2487                                         VIRTIO_ARG_SPEED,
2488                                         link_speed_handler, speed);
2489                 if (ret < 0) {
2490                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2491                                         VIRTIO_ARG_SPEED);
2492                         goto exit;
2493                 }
2494         }
2495
2496         if (vectorized &&
2497                 rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
2498                 ret = rte_kvargs_process(kvlist,
2499                                 VIRTIO_ARG_VECTORIZED,
2500                                 vectorized_check_handler, vectorized);
2501                 if (ret < 0) {
2502                         PMD_INIT_LOG(ERR, "Failed to parse %s",
2503                                         VIRTIO_ARG_VECTORIZED);
2504                         goto exit;
2505                 }
2506         }
2507
2508 exit:
2509         rte_kvargs_free(kvlist);
2510         return ret;
2511 }
2512
2513 static uint8_t
2514 rx_offload_enabled(struct virtio_hw *hw)
2515 {
2516         return virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) ||
2517                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2518                 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
2519 }
2520
2521 static uint8_t
2522 tx_offload_enabled(struct virtio_hw *hw)
2523 {
2524         return virtio_with_feature(hw, VIRTIO_NET_F_CSUM) ||
2525                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) ||
2526                 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO6);
2527 }
2528
2529 /*
2530  * Configure virtio device
2531  * It returns 0 on success.
2532  */
2533 static int
2534 virtio_dev_configure(struct rte_eth_dev *dev)
2535 {
2536         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2537         const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode;
2538         struct virtio_hw *hw = dev->data->dev_private;
2539         uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN +
2540                 hw->vtnet_hdr_size;
2541         uint64_t rx_offloads = rxmode->offloads;
2542         uint64_t tx_offloads = txmode->offloads;
2543         uint64_t req_features;
2544         int ret;
2545
2546         PMD_INIT_LOG(DEBUG, "configure");
2547         req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
2548
2549         if (rxmode->mq_mode != RTE_ETH_MQ_RX_NONE && rxmode->mq_mode != RTE_ETH_MQ_RX_RSS) {
2550                 PMD_DRV_LOG(ERR,
2551                         "Unsupported Rx multi queue mode %d",
2552                         rxmode->mq_mode);
2553                 return -EINVAL;
2554         }
2555
2556         if (txmode->mq_mode != RTE_ETH_MQ_TX_NONE) {
2557                 PMD_DRV_LOG(ERR,
2558                         "Unsupported Tx multi queue mode %d",
2559                         txmode->mq_mode);
2560                 return -EINVAL;
2561         }
2562
2563         if (dev->data->dev_conf.intr_conf.rxq) {
2564                 ret = virtio_init_device(dev, hw->req_guest_features);
2565                 if (ret < 0)
2566                         return ret;
2567         }
2568
2569         if (rxmode->mq_mode == RTE_ETH_MQ_RX_RSS)
2570                 req_features |= (1ULL << VIRTIO_NET_F_RSS);
2571
2572         if (rxmode->mtu > hw->max_mtu)
2573                 req_features &= ~(1ULL << VIRTIO_NET_F_MTU);
2574
2575         hw->max_rx_pkt_len = ether_hdr_len + rxmode->mtu;
2576
2577         if (rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2578                            RTE_ETH_RX_OFFLOAD_TCP_CKSUM))
2579                 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
2580
2581         if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)
2582                 req_features |=
2583                         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
2584                         (1ULL << VIRTIO_NET_F_GUEST_TSO6);
2585
2586         if (tx_offloads & (RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2587                            RTE_ETH_TX_OFFLOAD_TCP_CKSUM))
2588                 req_features |= (1ULL << VIRTIO_NET_F_CSUM);
2589
2590         if (tx_offloads & RTE_ETH_TX_OFFLOAD_TCP_TSO)
2591                 req_features |=
2592                         (1ULL << VIRTIO_NET_F_HOST_TSO4) |
2593                         (1ULL << VIRTIO_NET_F_HOST_TSO6);
2594
2595         /* if request features changed, reinit the device */
2596         if (req_features != hw->req_guest_features) {
2597                 ret = virtio_init_device(dev, req_features);
2598                 if (ret < 0)
2599                         return ret;
2600         }
2601
2602         if ((rxmode->mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) &&
2603                         !virtio_with_feature(hw, VIRTIO_NET_F_RSS)) {
2604                 PMD_DRV_LOG(ERR, "RSS support requested but not supported by the device");
2605                 return -ENOTSUP;
2606         }
2607
2608         if ((rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2609                             RTE_ETH_RX_OFFLOAD_TCP_CKSUM)) &&
2610                 !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
2611                 PMD_DRV_LOG(ERR,
2612                         "rx checksum not available on this host");
2613                 return -ENOTSUP;
2614         }
2615
2616         if ((rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) &&
2617                 (!virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
2618                  !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) {
2619                 PMD_DRV_LOG(ERR,
2620                         "Large Receive Offload not available on this host");
2621                 return -ENOTSUP;
2622         }
2623
2624         /* start control queue */
2625         if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
2626                 virtio_dev_cq_start(dev);
2627
2628         if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
2629                 hw->vlan_strip = 1;
2630
2631         hw->rx_ol_scatter = (rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER);
2632
2633         if ((rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) &&
2634                         !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2635                 PMD_DRV_LOG(ERR,
2636                             "vlan filtering not available on this host");
2637                 return -ENOTSUP;
2638         }
2639
2640         hw->has_tx_offload = tx_offload_enabled(hw);
2641         hw->has_rx_offload = rx_offload_enabled(hw);
2642
2643         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2644                 /* Enable vector (0) for Link State Intrerrupt */
2645                 if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) ==
2646                                 VIRTIO_MSI_NO_VECTOR) {
2647                         PMD_DRV_LOG(ERR, "failed to set config vector");
2648                         return -EBUSY;
2649                 }
2650
2651         if (virtio_with_packed_queue(hw)) {
2652 #if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT)
2653                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2654                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
2655                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2656                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2657                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_512)) {
2658                         PMD_DRV_LOG(INFO,
2659                                 "disabled packed ring vectorized path for requirements not met");
2660                         hw->use_vec_rx = 0;
2661                         hw->use_vec_tx = 0;
2662                 }
2663 #elif defined(RTE_ARCH_ARM)
2664                 if ((hw->use_vec_rx || hw->use_vec_tx) &&
2665                     (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) ||
2666                      !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) ||
2667                      !virtio_with_feature(hw, VIRTIO_F_VERSION_1) ||
2668                      rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)) {
2669                         PMD_DRV_LOG(INFO,
2670                                 "disabled packed ring vectorized path for requirements not met");
2671                         hw->use_vec_rx = 0;
2672                         hw->use_vec_tx = 0;
2673                 }
2674 #else
2675                 hw->use_vec_rx = 0;
2676                 hw->use_vec_tx = 0;
2677 #endif
2678
2679                 if (hw->use_vec_rx) {
2680                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2681                                 PMD_DRV_LOG(INFO,
2682                                         "disabled packed ring vectorized rx for mrg_rxbuf enabled");
2683                                 hw->use_vec_rx = 0;
2684                         }
2685
2686                         if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) {
2687                                 PMD_DRV_LOG(INFO,
2688                                         "disabled packed ring vectorized rx for TCP_LRO enabled");
2689                                 hw->use_vec_rx = 0;
2690                         }
2691                 }
2692         } else {
2693                 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) {
2694                         hw->use_inorder_tx = 1;
2695                         hw->use_inorder_rx = 1;
2696                         hw->use_vec_rx = 0;
2697                 }
2698
2699                 if (hw->use_vec_rx) {
2700 #if defined RTE_ARCH_ARM
2701                         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
2702                                 PMD_DRV_LOG(INFO,
2703                                         "disabled split ring vectorized path for requirement not met");
2704                                 hw->use_vec_rx = 0;
2705                         }
2706 #endif
2707                         if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
2708                                 PMD_DRV_LOG(INFO,
2709                                         "disabled split ring vectorized rx for mrg_rxbuf enabled");
2710                                 hw->use_vec_rx = 0;
2711                         }
2712
2713                         if (rx_offloads & (RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2714                                            RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
2715                                            RTE_ETH_RX_OFFLOAD_TCP_LRO |
2716                                            RTE_ETH_RX_OFFLOAD_VLAN_STRIP)) {
2717                                 PMD_DRV_LOG(INFO,
2718                                         "disabled split ring vectorized rx for offloading enabled");
2719                                 hw->use_vec_rx = 0;
2720                         }
2721
2722                         if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
2723                                 PMD_DRV_LOG(INFO,
2724                                         "disabled split ring vectorized rx, max SIMD bitwidth too low");
2725                                 hw->use_vec_rx = 0;
2726                         }
2727                 }
2728         }
2729
2730         return 0;
2731 }
2732
2733
2734 static int
2735 virtio_dev_start(struct rte_eth_dev *dev)
2736 {
2737         uint16_t nb_queues, i;
2738         struct virtqueue *vq;
2739         struct virtio_hw *hw = dev->data->dev_private;
2740         int ret;
2741
2742         /* Finish the initialization of the queues */
2743         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2744                 ret = virtio_dev_rx_queue_setup_finish(dev, i);
2745                 if (ret < 0)
2746                         return ret;
2747         }
2748         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2749                 ret = virtio_dev_tx_queue_setup_finish(dev, i);
2750                 if (ret < 0)
2751                         return ret;
2752         }
2753
2754         /* check if lsc interrupt feature is enabled */
2755         if (dev->data->dev_conf.intr_conf.lsc) {
2756                 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
2757                         PMD_DRV_LOG(ERR, "link status not supported by host");
2758                         return -ENOTSUP;
2759                 }
2760         }
2761
2762         /* Enable uio/vfio intr/eventfd mapping: althrough we already did that
2763          * in device configure, but it could be unmapped  when device is
2764          * stopped.
2765          */
2766         if (dev->data->dev_conf.intr_conf.lsc ||
2767             dev->data->dev_conf.intr_conf.rxq) {
2768                 virtio_intr_disable(dev);
2769
2770                 /* Setup interrupt callback  */
2771                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
2772                         rte_intr_callback_register(dev->intr_handle,
2773                                                    virtio_interrupt_handler,
2774                                                    dev);
2775
2776                 if (virtio_intr_enable(dev) < 0) {
2777                         PMD_DRV_LOG(ERR, "interrupt enable failed");
2778                         return -EIO;
2779                 }
2780         }
2781
2782         /*Notify the backend
2783          *Otherwise the tap backend might already stop its queue due to fullness.
2784          *vhost backend will have no chance to be waked up
2785          */
2786         nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
2787         if (hw->max_queue_pairs > 1) {
2788                 if (virtio_set_multiple_queues(dev, nb_queues) != 0)
2789                         return -EINVAL;
2790         }
2791
2792         PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
2793
2794         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2795                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2796                 /* Flush the old packets */
2797                 virtqueue_rxvq_flush(vq);
2798                 virtqueue_notify(vq);
2799         }
2800
2801         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2802                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2803                 virtqueue_notify(vq);
2804         }
2805
2806         PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
2807
2808         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2809                 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]);
2810                 VIRTQUEUE_DUMP(vq);
2811         }
2812
2813         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2814                 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]);
2815                 VIRTQUEUE_DUMP(vq);
2816         }
2817
2818         set_rxtx_funcs(dev);
2819         hw->started = 1;
2820
2821         /* Initialize Link state */
2822         virtio_dev_link_update(dev, 0);
2823
2824         return 0;
2825 }
2826
2827 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
2828 {
2829         struct virtio_hw *hw = dev->data->dev_private;
2830         uint16_t nr_vq = virtio_get_nr_vq(hw);
2831         const char *type __rte_unused;
2832         unsigned int i, mbuf_num = 0;
2833         struct virtqueue *vq;
2834         struct rte_mbuf *buf;
2835         int queue_type;
2836
2837         if (hw->vqs == NULL)
2838                 return;
2839
2840         for (i = 0; i < nr_vq; i++) {
2841                 vq = hw->vqs[i];
2842                 if (!vq)
2843                         continue;
2844
2845                 queue_type = virtio_get_queue_type(hw, i);
2846                 if (queue_type == VTNET_RQ)
2847                         type = "rxq";
2848                 else if (queue_type == VTNET_TQ)
2849                         type = "txq";
2850                 else
2851                         continue;
2852
2853                 PMD_INIT_LOG(DEBUG,
2854                         "Before freeing %s[%d] used and unused buf",
2855                         type, i);
2856                 VIRTQUEUE_DUMP(vq);
2857
2858                 while ((buf = virtqueue_detach_unused(vq)) != NULL) {
2859                         rte_pktmbuf_free(buf);
2860                         mbuf_num++;
2861                 }
2862
2863                 PMD_INIT_LOG(DEBUG,
2864                         "After freeing %s[%d] used and unused buf",
2865                         type, i);
2866                 VIRTQUEUE_DUMP(vq);
2867         }
2868
2869         PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num);
2870 }
2871
2872 static void
2873 virtio_tx_completed_cleanup(struct rte_eth_dev *dev)
2874 {
2875         struct virtio_hw *hw = dev->data->dev_private;
2876         struct virtqueue *vq;
2877         int qidx;
2878         void (*xmit_cleanup)(struct virtqueue *vq, uint16_t nb_used);
2879
2880         if (virtio_with_packed_queue(hw)) {
2881                 if (hw->use_vec_tx)
2882                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2883                 else if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER))
2884                         xmit_cleanup = &virtio_xmit_cleanup_inorder_packed;
2885                 else
2886                         xmit_cleanup = &virtio_xmit_cleanup_normal_packed;
2887         } else {
2888                 if (hw->use_inorder_tx)
2889                         xmit_cleanup = &virtio_xmit_cleanup_inorder;
2890                 else
2891                         xmit_cleanup = &virtio_xmit_cleanup;
2892         }
2893
2894         for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) {
2895                 vq = hw->vqs[2 * qidx + VTNET_SQ_TQ_QUEUE_IDX];
2896                 if (vq != NULL)
2897                         xmit_cleanup(vq, virtqueue_nused(vq));
2898         }
2899 }
2900
2901 /*
2902  * Stop device: disable interrupt and mark link down
2903  */
2904 int
2905 virtio_dev_stop(struct rte_eth_dev *dev)
2906 {
2907         struct virtio_hw *hw = dev->data->dev_private;
2908         struct rte_eth_link link;
2909         struct rte_eth_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
2910
2911         PMD_INIT_LOG(DEBUG, "stop");
2912         dev->data->dev_started = 0;
2913
2914         rte_spinlock_lock(&hw->state_lock);
2915         if (!hw->started)
2916                 goto out_unlock;
2917         hw->started = 0;
2918
2919         virtio_tx_completed_cleanup(dev);
2920
2921         if (intr_conf->lsc || intr_conf->rxq) {
2922                 virtio_intr_disable(dev);
2923
2924                 /* Reset interrupt callback  */
2925                 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
2926                         rte_intr_callback_unregister(dev->intr_handle,
2927                                                      virtio_interrupt_handler,
2928                                                      dev);
2929                 }
2930         }
2931
2932         memset(&link, 0, sizeof(link));
2933         rte_eth_linkstatus_set(dev, &link);
2934 out_unlock:
2935         rte_spinlock_unlock(&hw->state_lock);
2936
2937         return 0;
2938 }
2939
2940 static int
2941 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
2942 {
2943         struct rte_eth_link link;
2944         uint16_t status;
2945         struct virtio_hw *hw = dev->data->dev_private;
2946
2947         memset(&link, 0, sizeof(link));
2948         link.link_duplex = hw->duplex;
2949         link.link_speed  = hw->speed;
2950         link.link_autoneg = RTE_ETH_LINK_AUTONEG;
2951
2952         if (!hw->started) {
2953                 link.link_status = RTE_ETH_LINK_DOWN;
2954                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2955         } else if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) {
2956                 PMD_INIT_LOG(DEBUG, "Get link status from hw");
2957                 virtio_read_dev_config(hw,
2958                                 offsetof(struct virtio_net_config, status),
2959                                 &status, sizeof(status));
2960                 if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
2961                         link.link_status = RTE_ETH_LINK_DOWN;
2962                         link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2963                         PMD_INIT_LOG(DEBUG, "Port %d is down",
2964                                      dev->data->port_id);
2965                 } else {
2966                         link.link_status = RTE_ETH_LINK_UP;
2967                         PMD_INIT_LOG(DEBUG, "Port %d is up",
2968                                      dev->data->port_id);
2969                 }
2970         } else {
2971                 link.link_status = RTE_ETH_LINK_UP;
2972         }
2973
2974         return rte_eth_linkstatus_set(dev, &link);
2975 }
2976
2977 static int
2978 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2979 {
2980         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
2981         struct virtio_hw *hw = dev->data->dev_private;
2982         uint64_t offloads = rxmode->offloads;
2983
2984         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
2985                 if ((offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) &&
2986                                 !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
2987
2988                         PMD_DRV_LOG(NOTICE,
2989                                 "vlan filtering not available on this host");
2990
2991                         return -ENOTSUP;
2992                 }
2993         }
2994
2995         if (mask & RTE_ETH_VLAN_STRIP_MASK)
2996                 hw->vlan_strip = !!(offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP);
2997
2998         return 0;
2999 }
3000
3001 static int
3002 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
3003 {
3004         uint64_t tso_mask, host_features;
3005         uint32_t rss_hash_types = 0;
3006         struct virtio_hw *hw = dev->data->dev_private;
3007         dev_info->speed_capa = virtio_dev_speed_capa_get(hw->speed);
3008
3009         dev_info->max_rx_queues =
3010                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
3011         dev_info->max_tx_queues =
3012                 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
3013         dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
3014         dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
3015         dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
3016         dev_info->max_mtu = hw->max_mtu;
3017
3018         host_features = VIRTIO_OPS(hw)->get_features(hw);
3019         dev_info->rx_offload_capa = RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
3020         if (host_features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))
3021                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_SCATTER;
3022         if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
3023                 dev_info->rx_offload_capa |=
3024                         RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
3025                         RTE_ETH_RX_OFFLOAD_UDP_CKSUM;
3026         }
3027         if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN))
3028                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
3029         tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
3030                 (1ULL << VIRTIO_NET_F_GUEST_TSO6);
3031         if ((host_features & tso_mask) == tso_mask)
3032                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO;
3033
3034         dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
3035                                     RTE_ETH_TX_OFFLOAD_VLAN_INSERT;
3036         if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) {
3037                 dev_info->tx_offload_capa |=
3038                         RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
3039                         RTE_ETH_TX_OFFLOAD_TCP_CKSUM;
3040         }
3041         tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
3042                 (1ULL << VIRTIO_NET_F_HOST_TSO6);
3043         if ((host_features & tso_mask) == tso_mask)
3044                 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_TCP_TSO;
3045
3046         if (host_features & (1ULL << VIRTIO_NET_F_RSS)) {
3047                 virtio_dev_get_rss_config(hw, &rss_hash_types);
3048                 dev_info->hash_key_size = VIRTIO_NET_RSS_KEY_SIZE;
3049                 dev_info->reta_size = VIRTIO_NET_RSS_RETA_SIZE;
3050                 dev_info->flow_type_rss_offloads =
3051                         virtio_to_ethdev_rss_offloads(rss_hash_types);
3052         } else {
3053                 dev_info->hash_key_size = 0;
3054                 dev_info->reta_size = 0;
3055                 dev_info->flow_type_rss_offloads = 0;
3056         }
3057
3058         if (host_features & (1ULL << VIRTIO_F_RING_PACKED)) {
3059                 /*
3060                  * According to 2.7 Packed Virtqueues,
3061                  * 2.7.10.1 Structure Size and Alignment:
3062                  * The Queue Size value does not have to be a power of 2.
3063                  */
3064                 dev_info->rx_desc_lim.nb_max = UINT16_MAX;
3065                 dev_info->tx_desc_lim.nb_max = UINT16_MAX;
3066         } else {
3067                 /*
3068                  * According to 2.6 Split Virtqueues:
3069                  * Queue Size value is always a power of 2. The maximum Queue
3070                  * Size value is 32768.
3071                  */
3072                 dev_info->rx_desc_lim.nb_max = 32768;
3073                 dev_info->tx_desc_lim.nb_max = 32768;
3074         }
3075         /*
3076          * Actual minimum is not the same for virtqueues of different kinds,
3077          * but to avoid tangling the code with separate branches, rely on
3078          * default thresholds since desc number must be at least of their size.
3079          */
3080         dev_info->rx_desc_lim.nb_min = RTE_MAX(DEFAULT_RX_FREE_THRESH,
3081                                                RTE_VIRTIO_VPMD_RX_REARM_THRESH);
3082         dev_info->tx_desc_lim.nb_min = DEFAULT_TX_FREE_THRESH;
3083         dev_info->rx_desc_lim.nb_align = 1;
3084         dev_info->tx_desc_lim.nb_align = 1;
3085
3086         return 0;
3087 }
3088
3089 /*
3090  * It enables testpmd to collect per queue stats.
3091  */
3092 static int
3093 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
3094 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
3095 __rte_unused uint8_t is_rx)
3096 {
3097         return 0;
3098 }
3099
3100 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_init, init, NOTICE);
3101 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_driver, driver, NOTICE);