drivers/net/vhost/rte_eth_vhost.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016 IGEL Co., Ltd.
   3  * Copyright(c) 2016-2018 Intel Corporation
   4  */
   5 #include <unistd.h>
   6 #include <pthread.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_mbuf.h>
  10 #include <rte_ethdev_driver.h>
  11 #include <rte_ethdev_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_memcpy.h>
  14 #include <rte_bus_vdev.h>
  15 #include <rte_kvargs.h>
  16 #include <rte_vhost.h>
  17 #include <rte_spinlock.h>
  18
  19 #include "rte_eth_vhost.h"
  20
  21 static int vhost_logtype;
  22
  23 #define VHOST_LOG(level, ...) \
  24         rte_log(RTE_LOG_ ## level, vhost_logtype, __VA_ARGS__)
  25
  26 enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM};
  27
  28 #define ETH_VHOST_IFACE_ARG             "iface"
  29 #define ETH_VHOST_QUEUES_ARG            "queues"
  30 #define ETH_VHOST_CLIENT_ARG            "client"
  31 #define ETH_VHOST_DEQUEUE_ZERO_COPY     "dequeue-zero-copy"
  32 #define ETH_VHOST_IOMMU_SUPPORT         "iommu-support"
  33 #define ETH_VHOST_POSTCOPY_SUPPORT      "postcopy-support"
  34 #define ETH_VHOST_VIRTIO_NET_F_HOST_TSO "tso"
  35 #define ETH_VHOST_LINEAR_BUF  "linear-buffer"
  36 #define ETH_VHOST_EXT_BUF  "ext-buffer"
  37 #define VHOST_MAX_PKT_BURST 32
  38
  39 static const char *valid_arguments[] = {
  40         ETH_VHOST_IFACE_ARG,
  41         ETH_VHOST_QUEUES_ARG,
  42         ETH_VHOST_CLIENT_ARG,
  43         ETH_VHOST_DEQUEUE_ZERO_COPY,
  44         ETH_VHOST_IOMMU_SUPPORT,
  45         ETH_VHOST_POSTCOPY_SUPPORT,
  46         ETH_VHOST_VIRTIO_NET_F_HOST_TSO,
  47         ETH_VHOST_LINEAR_BUF,
  48         ETH_VHOST_EXT_BUF,
  49         NULL
  50 };
  51
  52 static struct rte_ether_addr base_eth_addr = {
  53         .addr_bytes = {
  54                 0x56 /* V */,
  55                 0x48 /* H */,
  56                 0x4F /* O */,
  57                 0x53 /* S */,
  58                 0x54 /* T */,
  59                 0x00
  60         }
  61 };
  62
  63 enum vhost_xstats_pkts {
  64         VHOST_UNDERSIZE_PKT = 0,
  65         VHOST_64_PKT,
  66         VHOST_65_TO_127_PKT,
  67         VHOST_128_TO_255_PKT,
  68         VHOST_256_TO_511_PKT,
  69         VHOST_512_TO_1023_PKT,
  70         VHOST_1024_TO_1522_PKT,
  71         VHOST_1523_TO_MAX_PKT,
  72         VHOST_BROADCAST_PKT,
  73         VHOST_MULTICAST_PKT,
  74         VHOST_UNICAST_PKT,
  75         VHOST_ERRORS_PKT,
  76         VHOST_ERRORS_FRAGMENTED,
  77         VHOST_ERRORS_JABBER,
  78         VHOST_UNKNOWN_PROTOCOL,
  79         VHOST_XSTATS_MAX,
  80 };
  81
  82 struct vhost_stats {
  83         uint64_t pkts;
  84         uint64_t bytes;
  85         uint64_t missed_pkts;
  86         uint64_t xstats[VHOST_XSTATS_MAX];
  87 };
  88
  89 struct vhost_queue {
  90         int vid;
  91         rte_atomic32_t allow_queuing;
  92         rte_atomic32_t while_queuing;
  93         struct pmd_internal *internal;
  94         struct rte_mempool *mb_pool;
  95         uint16_t port;
  96         uint16_t virtqueue_id;
  97         struct vhost_stats stats;
  98 };
  99
 100 struct pmd_internal {
 101         rte_atomic32_t dev_attached;
 102         char *iface_name;
 103         uint64_t flags;
 104         uint64_t disable_flags;
 105         uint16_t max_queues;
 106         int vid;
 107         rte_atomic32_t started;
 108         uint8_t vlan_strip;
 109 };
 110
 111 struct internal_list {
 112         TAILQ_ENTRY(internal_list) next;
 113         struct rte_eth_dev *eth_dev;
 114 };
 115
 116 TAILQ_HEAD(internal_list_head, internal_list);
 117 static struct internal_list_head internal_list =
 118         TAILQ_HEAD_INITIALIZER(internal_list);
 119
 120 static pthread_mutex_t internal_list_lock = PTHREAD_MUTEX_INITIALIZER;
 121
 122 static struct rte_eth_link pmd_link = {
 123                 .link_speed = 10000,
 124                 .link_duplex = ETH_LINK_FULL_DUPLEX,
 125                 .link_status = ETH_LINK_DOWN
 126 };
 127
 128 struct rte_vhost_vring_state {
 129         rte_spinlock_t lock;
 130
 131         bool cur[RTE_MAX_QUEUES_PER_PORT * 2];
 132         bool seen[RTE_MAX_QUEUES_PER_PORT * 2];
 133         unsigned int index;
 134         unsigned int max_vring;
 135 };
 136
 137 static struct rte_vhost_vring_state *vring_states[RTE_MAX_ETHPORTS];
 138
 139 #define VHOST_XSTATS_NAME_SIZE 64
 140
 141 struct vhost_xstats_name_off {
 142         char name[VHOST_XSTATS_NAME_SIZE];
 143         uint64_t offset;
 144 };
 145
 146 /* [rx]_is prepended to the name string here */
 147 static const struct vhost_xstats_name_off vhost_rxport_stat_strings[] = {
 148         {"good_packets",
 149          offsetof(struct vhost_queue, stats.pkts)},
 150         {"total_bytes",
 151          offsetof(struct vhost_queue, stats.bytes)},
 152         {"missed_pkts",
 153          offsetof(struct vhost_queue, stats.missed_pkts)},
 154         {"broadcast_packets",
 155          offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])},
 156         {"multicast_packets",
 157          offsetof(struct vhost_queue, stats.xstats[VHOST_MULTICAST_PKT])},
 158         {"unicast_packets",
 159          offsetof(struct vhost_queue, stats.xstats[VHOST_UNICAST_PKT])},
 160          {"undersize_packets",
 161          offsetof(struct vhost_queue, stats.xstats[VHOST_UNDERSIZE_PKT])},
 162         {"size_64_packets",
 163          offsetof(struct vhost_queue, stats.xstats[VHOST_64_PKT])},
 164         {"size_65_to_127_packets",
 165          offsetof(struct vhost_queue, stats.xstats[VHOST_65_TO_127_PKT])},
 166         {"size_128_to_255_packets",
 167          offsetof(struct vhost_queue, stats.xstats[VHOST_128_TO_255_PKT])},
 168         {"size_256_to_511_packets",
 169          offsetof(struct vhost_queue, stats.xstats[VHOST_256_TO_511_PKT])},
 170         {"size_512_to_1023_packets",
 171          offsetof(struct vhost_queue, stats.xstats[VHOST_512_TO_1023_PKT])},
 172         {"size_1024_to_1522_packets",
 173          offsetof(struct vhost_queue, stats.xstats[VHOST_1024_TO_1522_PKT])},
 174         {"size_1523_to_max_packets",
 175          offsetof(struct vhost_queue, stats.xstats[VHOST_1523_TO_MAX_PKT])},
 176         {"errors_with_bad_CRC",
 177          offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_PKT])},
 178         {"fragmented_errors",
 179          offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_FRAGMENTED])},
 180         {"jabber_errors",
 181          offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_JABBER])},
 182         {"unknown_protos_packets",
 183          offsetof(struct vhost_queue, stats.xstats[VHOST_UNKNOWN_PROTOCOL])},
 184 };
 185
 186 /* [tx]_ is prepended to the name string here */
 187 static const struct vhost_xstats_name_off vhost_txport_stat_strings[] = {
 188         {"good_packets",
 189          offsetof(struct vhost_queue, stats.pkts)},
 190         {"total_bytes",
 191          offsetof(struct vhost_queue, stats.bytes)},
 192         {"missed_pkts",
 193          offsetof(struct vhost_queue, stats.missed_pkts)},
 194         {"broadcast_packets",
 195          offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])},
 196         {"multicast_packets",
 197          offsetof(struct vhost_queue, stats.xstats[VHOST_MULTICAST_PKT])},
 198         {"unicast_packets",
 199          offsetof(struct vhost_queue, stats.xstats[VHOST_UNICAST_PKT])},
 200         {"undersize_packets",
 201          offsetof(struct vhost_queue, stats.xstats[VHOST_UNDERSIZE_PKT])},
 202         {"size_64_packets",
 203          offsetof(struct vhost_queue, stats.xstats[VHOST_64_PKT])},
 204         {"size_65_to_127_packets",
 205          offsetof(struct vhost_queue, stats.xstats[VHOST_65_TO_127_PKT])},
 206         {"size_128_to_255_packets",
 207          offsetof(struct vhost_queue, stats.xstats[VHOST_128_TO_255_PKT])},
 208         {"size_256_to_511_packets",
 209          offsetof(struct vhost_queue, stats.xstats[VHOST_256_TO_511_PKT])},
 210         {"size_512_to_1023_packets",
 211          offsetof(struct vhost_queue, stats.xstats[VHOST_512_TO_1023_PKT])},
 212         {"size_1024_to_1522_packets",
 213          offsetof(struct vhost_queue, stats.xstats[VHOST_1024_TO_1522_PKT])},
 214         {"size_1523_to_max_packets",
 215          offsetof(struct vhost_queue, stats.xstats[VHOST_1523_TO_MAX_PKT])},
 216         {"errors_with_bad_CRC",
 217          offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_PKT])},
 218 };
 219
 220 #define VHOST_NB_XSTATS_RXPORT (sizeof(vhost_rxport_stat_strings) / \
 221                                 sizeof(vhost_rxport_stat_strings[0]))
 222
 223 #define VHOST_NB_XSTATS_TXPORT (sizeof(vhost_txport_stat_strings) / \
 224                                 sizeof(vhost_txport_stat_strings[0]))
 225
 226 static int
 227 vhost_dev_xstats_reset(struct rte_eth_dev *dev)
 228 {
 229         struct vhost_queue *vq = NULL;
 230         unsigned int i = 0;
 231
 232         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 233                 vq = dev->data->rx_queues[i];
 234                 if (!vq)
 235                         continue;
 236                 memset(&vq->stats, 0, sizeof(vq->stats));
 237         }
 238         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 239                 vq = dev->data->tx_queues[i];
 240                 if (!vq)
 241                         continue;
 242                 memset(&vq->stats, 0, sizeof(vq->stats));
 243         }
 244
 245         return 0;
 246 }
 247
 248 static int
 249 vhost_dev_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
 250                            struct rte_eth_xstat_name *xstats_names,
 251                            unsigned int limit __rte_unused)
 252 {
 253         unsigned int t = 0;
 254         int count = 0;
 255         int nstats = VHOST_NB_XSTATS_RXPORT + VHOST_NB_XSTATS_TXPORT;
 256
 257         if (!xstats_names)
 258                 return nstats;
 259         for (t = 0; t < VHOST_NB_XSTATS_RXPORT; t++) {
 260                 snprintf(xstats_names[count].name,
 261                          sizeof(xstats_names[count].name),
 262                          "rx_%s", vhost_rxport_stat_strings[t].name);
 263                 count++;
 264         }
 265         for (t = 0; t < VHOST_NB_XSTATS_TXPORT; t++) {
 266                 snprintf(xstats_names[count].name,
 267                          sizeof(xstats_names[count].name),
 268                          "tx_%s", vhost_txport_stat_strings[t].name);
 269                 count++;
 270         }
 271         return count;
 272 }
 273
 274 static int
 275 vhost_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 276                      unsigned int n)
 277 {
 278         unsigned int i;
 279         unsigned int t;
 280         unsigned int count = 0;
 281         struct vhost_queue *vq = NULL;
 282         unsigned int nxstats = VHOST_NB_XSTATS_RXPORT + VHOST_NB_XSTATS_TXPORT;
 283
 284         if (n < nxstats)
 285                 return nxstats;
 286
 287         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 288                 vq = dev->data->rx_queues[i];
 289                 if (!vq)
 290                         continue;
 291                 vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts
 292                                 - (vq->stats.xstats[VHOST_BROADCAST_PKT]
 293                                 + vq->stats.xstats[VHOST_MULTICAST_PKT]);
 294         }
 295         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 296                 vq = dev->data->tx_queues[i];
 297                 if (!vq)
 298                         continue;
 299                 vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts
 300                                 + vq->stats.missed_pkts
 301                                 - (vq->stats.xstats[VHOST_BROADCAST_PKT]
 302                                 + vq->stats.xstats[VHOST_MULTICAST_PKT]);
 303         }
 304         for (t = 0; t < VHOST_NB_XSTATS_RXPORT; t++) {
 305                 xstats[count].value = 0;
 306                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 307                         vq = dev->data->rx_queues[i];
 308                         if (!vq)
 309                                 continue;
 310                         xstats[count].value +=
 311                                 *(uint64_t *)(((char *)vq)
 312                                 + vhost_rxport_stat_strings[t].offset);
 313                 }
 314                 xstats[count].id = count;
 315                 count++;
 316         }
 317         for (t = 0; t < VHOST_NB_XSTATS_TXPORT; t++) {
 318                 xstats[count].value = 0;
 319                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
 320                         vq = dev->data->tx_queues[i];
 321                         if (!vq)
 322                                 continue;
 323                         xstats[count].value +=
 324                                 *(uint64_t *)(((char *)vq)
 325                                 + vhost_txport_stat_strings[t].offset);
 326                 }
 327                 xstats[count].id = count;
 328                 count++;
 329         }
 330         return count;
 331 }
 332
 333 static inline void
 334 vhost_count_multicast_broadcast(struct vhost_queue *vq,
 335                                 struct rte_mbuf *mbuf)
 336 {
 337         struct rte_ether_addr *ea = NULL;
 338         struct vhost_stats *pstats = &vq->stats;
 339
 340         ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *);
 341         if (rte_is_multicast_ether_addr(ea)) {
 342                 if (rte_is_broadcast_ether_addr(ea))
 343                         pstats->xstats[VHOST_BROADCAST_PKT]++;
 344                 else
 345                         pstats->xstats[VHOST_MULTICAST_PKT]++;
 346         }
 347 }
 348
 349 static void
 350 vhost_update_packet_xstats(struct vhost_queue *vq,
 351                            struct rte_mbuf **bufs,
 352                            uint16_t count)
 353 {
 354         uint32_t pkt_len = 0;
 355         uint64_t i = 0;
 356         uint64_t index;
 357         struct vhost_stats *pstats = &vq->stats;
 358
 359         for (i = 0; i < count ; i++) {
 360                 pkt_len = bufs[i]->pkt_len;
 361                 if (pkt_len == 64) {
 362                         pstats->xstats[VHOST_64_PKT]++;
 363                 } else if (pkt_len > 64 && pkt_len < 1024) {
 364                         index = (sizeof(pkt_len) * 8)
 365                                 - __builtin_clz(pkt_len) - 5;
 366                         pstats->xstats[index]++;
 367                 } else {
 368                         if (pkt_len < 64)
 369                                 pstats->xstats[VHOST_UNDERSIZE_PKT]++;
 370                         else if (pkt_len <= 1522)
 371                                 pstats->xstats[VHOST_1024_TO_1522_PKT]++;
 372                         else if (pkt_len > 1522)
 373                                 pstats->xstats[VHOST_1523_TO_MAX_PKT]++;
 374                 }
 375                 vhost_count_multicast_broadcast(vq, bufs[i]);
 376         }
 377 }
 378
 379 static uint16_t
 380 eth_vhost_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
 381 {
 382         struct vhost_queue *r = q;
 383         uint16_t i, nb_rx = 0;
 384         uint16_t nb_receive = nb_bufs;
 385
 386         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
 387                 return 0;
 388
 389         rte_atomic32_set(&r->while_queuing, 1);
 390
 391         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
 392                 goto out;
 393
 394         /* Dequeue packets from guest TX queue */
 395         while (nb_receive) {
 396                 uint16_t nb_pkts;
 397                 uint16_t num = (uint16_t)RTE_MIN(nb_receive,
 398                                                  VHOST_MAX_PKT_BURST);
 399
 400                 nb_pkts = rte_vhost_dequeue_burst(r->vid, r->virtqueue_id,
 401                                                   r->mb_pool, &bufs[nb_rx],
 402                                                   num);
 403
 404                 nb_rx += nb_pkts;
 405                 nb_receive -= nb_pkts;
 406                 if (nb_pkts < num)
 407                         break;
 408         }
 409
 410         r->stats.pkts += nb_rx;
 411
 412         for (i = 0; likely(i < nb_rx); i++) {
 413                 bufs[i]->port = r->port;
 414                 bufs[i]->vlan_tci = 0;
 415
 416                 if (r->internal->vlan_strip)
 417                         rte_vlan_strip(bufs[i]);
 418
 419                 r->stats.bytes += bufs[i]->pkt_len;
 420         }
 421
 422         vhost_update_packet_xstats(r, bufs, nb_rx);
 423
 424 out:
 425         rte_atomic32_set(&r->while_queuing, 0);
 426
 427         return nb_rx;
 428 }
 429
 430 static uint16_t
 431 eth_vhost_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
 432 {
 433         struct vhost_queue *r = q;
 434         uint16_t i, nb_tx = 0;
 435         uint16_t nb_send = 0;
 436
 437         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
 438                 return 0;
 439
 440         rte_atomic32_set(&r->while_queuing, 1);
 441
 442         if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
 443                 goto out;
 444
 445         for (i = 0; i < nb_bufs; i++) {
 446                 struct rte_mbuf *m = bufs[i];
 447
 448                 /* Do VLAN tag insertion */
 449                 if (m->ol_flags & PKT_TX_VLAN_PKT) {
 450                         int error = rte_vlan_insert(&m);
 451                         if (unlikely(error)) {
 452                                 rte_pktmbuf_free(m);
 453                                 continue;
 454                         }
 455                 }
 456
 457                 bufs[nb_send] = m;
 458                 ++nb_send;
 459         }
 460
 461         /* Enqueue packets to guest RX queue */
 462         while (nb_send) {
 463                 uint16_t nb_pkts;
 464                 uint16_t num = (uint16_t)RTE_MIN(nb_send,
 465                                                  VHOST_MAX_PKT_BURST);
 466
 467                 nb_pkts = rte_vhost_enqueue_burst(r->vid, r->virtqueue_id,
 468                                                   &bufs[nb_tx], num);
 469
 470                 nb_tx += nb_pkts;
 471                 nb_send -= nb_pkts;
 472                 if (nb_pkts < num)
 473                         break;
 474         }
 475
 476         r->stats.pkts += nb_tx;
 477         r->stats.missed_pkts += nb_bufs - nb_tx;
 478
 479         for (i = 0; likely(i < nb_tx); i++)
 480                 r->stats.bytes += bufs[i]->pkt_len;
 481
 482         vhost_update_packet_xstats(r, bufs, nb_tx);
 483
 484         /* According to RFC2863 page42 section ifHCOutMulticastPkts and
 485          * ifHCOutBroadcastPkts, the counters "multicast" and "broadcast"
 486          * are increased when packets are not transmitted successfully.
 487          */
 488         for (i = nb_tx; i < nb_bufs; i++)
 489                 vhost_count_multicast_broadcast(r, bufs[i]);
 490
 491         for (i = 0; likely(i < nb_tx); i++)
 492                 rte_pktmbuf_free(bufs[i]);
 493 out:
 494         rte_atomic32_set(&r->while_queuing, 0);
 495
 496         return nb_tx;
 497 }
 498
 499 static inline struct internal_list *
 500 find_internal_resource(char *ifname)
 501 {
 502         int found = 0;
 503         struct internal_list *list;
 504         struct pmd_internal *internal;
 505
 506         if (ifname == NULL)
 507                 return NULL;
 508
 509         pthread_mutex_lock(&internal_list_lock);
 510
 511         TAILQ_FOREACH(list, &internal_list, next) {
 512                 internal = list->eth_dev->data->dev_private;
 513                 if (!strcmp(internal->iface_name, ifname)) {
 514                         found = 1;
 515                         break;
 516                 }
 517         }
 518
 519         pthread_mutex_unlock(&internal_list_lock);
 520
 521         if (!found)
 522                 return NULL;
 523
 524         return list;
 525 }
 526
 527 static int
 528 eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
 529 {
 530         struct rte_vhost_vring vring;
 531         struct vhost_queue *vq;
 532         int ret = 0;
 533
 534         vq = dev->data->rx_queues[qid];
 535         if (!vq) {
 536                 VHOST_LOG(ERR, "rxq%d is not setup yet\n", qid);
 537                 return -1;
 538         }
 539
 540         ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
 541         if (ret < 0) {
 542                 VHOST_LOG(ERR, "Failed to get rxq%d's vring\n", qid);
 543                 return ret;
 544         }
 545         VHOST_LOG(INFO, "Enable interrupt for rxq%d\n", qid);
 546         rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1);
 547         rte_wmb();
 548
 549         return ret;
 550 }
 551
 552 static int
 553 eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
 554 {
 555         struct rte_vhost_vring vring;
 556         struct vhost_queue *vq;
 557         int ret = 0;
 558
 559         vq = dev->data->rx_queues[qid];
 560         if (!vq) {
 561                 VHOST_LOG(ERR, "rxq%d is not setup yet\n", qid);
 562                 return -1;
 563         }
 564
 565         ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
 566         if (ret < 0) {
 567                 VHOST_LOG(ERR, "Failed to get rxq%d's vring", qid);
 568                 return ret;
 569         }
 570         VHOST_LOG(INFO, "Disable interrupt for rxq%d\n", qid);
 571         rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
 572         rte_wmb();
 573
 574         return 0;
 575 }
 576
 577 static void
 578 eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
 579 {
 580         struct rte_intr_handle *intr_handle = dev->intr_handle;
 581
 582         if (intr_handle) {
 583                 if (intr_handle->intr_vec)
 584                         free(intr_handle->intr_vec);
 585                 free(intr_handle);
 586         }
 587
 588         dev->intr_handle = NULL;
 589 }
 590
 591 static int
 592 eth_vhost_install_intr(struct rte_eth_dev *dev)
 593 {
 594         struct rte_vhost_vring vring;
 595         struct vhost_queue *vq;
 596         int count = 0;
 597         int nb_rxq = dev->data->nb_rx_queues;
 598         int i;
 599         int ret;
 600
 601         /* uninstall firstly if we are reconnecting */
 602         if (dev->intr_handle)
 603                 eth_vhost_uninstall_intr(dev);
 604
 605         dev->intr_handle = malloc(sizeof(*dev->intr_handle));
 606         if (!dev->intr_handle) {
 607                 VHOST_LOG(ERR, "Fail to allocate intr_handle\n");
 608                 return -ENOMEM;
 609         }
 610         memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
 611
 612         dev->intr_handle->efd_counter_size = sizeof(uint64_t);
 613
 614         dev->intr_handle->intr_vec =
 615                 malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
 616
 617         if (!dev->intr_handle->intr_vec) {
 618                 VHOST_LOG(ERR,
 619                         "Failed to allocate memory for interrupt vector\n");
 620                 free(dev->intr_handle);
 621                 return -ENOMEM;
 622         }
 623
 624         VHOST_LOG(INFO, "Prepare intr vec\n");
 625         for (i = 0; i < nb_rxq; i++) {
 626                 vq = dev->data->rx_queues[i];
 627                 if (!vq) {
 628                         VHOST_LOG(INFO, "rxq-%d not setup yet, skip!\n", i);
 629                         continue;
 630                 }
 631
 632                 ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
 633                 if (ret < 0) {
 634                         VHOST_LOG(INFO,
 635                                 "Failed to get rxq-%d's vring, skip!\n", i);
 636                         continue;
 637                 }
 638
 639                 if (vring.kickfd < 0) {
 640                         VHOST_LOG(INFO,
 641                                 "rxq-%d's kickfd is invalid, skip!\n", i);
 642                         continue;
 643                 }
 644                 dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
 645                 dev->intr_handle->efds[i] = vring.kickfd;
 646                 count++;
 647                 VHOST_LOG(INFO, "Installed intr vec for rxq-%d\n", i);
 648         }
 649
 650         dev->intr_handle->nb_efd = count;
 651         dev->intr_handle->max_intr = count + 1;
 652         dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
 653
 654         return 0;
 655 }
 656
 657 static void
 658 update_queuing_status(struct rte_eth_dev *dev)
 659 {
 660         struct pmd_internal *internal = dev->data->dev_private;
 661         struct vhost_queue *vq;
 662         unsigned int i;
 663         int allow_queuing = 1;
 664
 665         if (!dev->data->rx_queues || !dev->data->tx_queues)
 666                 return;
 667
 668         if (rte_atomic32_read(&internal->started) == 0 ||
 669             rte_atomic32_read(&internal->dev_attached) == 0)
 670                 allow_queuing = 0;
 671
 672         /* Wait until rx/tx_pkt_burst stops accessing vhost device */
 673         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 674                 vq = dev->data->rx_queues[i];
 675                 if (vq == NULL)
 676                         continue;
 677                 rte_atomic32_set(&vq->allow_queuing, allow_queuing);
 678                 while (rte_atomic32_read(&vq->while_queuing))
 679                         rte_pause();
 680         }
 681
 682         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 683                 vq = dev->data->tx_queues[i];
 684                 if (vq == NULL)
 685                         continue;
 686                 rte_atomic32_set(&vq->allow_queuing, allow_queuing);
 687                 while (rte_atomic32_read(&vq->while_queuing))
 688                         rte_pause();
 689         }
 690 }
 691
 692 static void
 693 queue_setup(struct rte_eth_dev *eth_dev, struct pmd_internal *internal)
 694 {
 695         struct vhost_queue *vq;
 696         int i;
 697
 698         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 699                 vq = eth_dev->data->rx_queues[i];
 700                 if (!vq)
 701                         continue;
 702                 vq->vid = internal->vid;
 703                 vq->internal = internal;
 704                 vq->port = eth_dev->data->port_id;
 705         }
 706         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 707                 vq = eth_dev->data->tx_queues[i];
 708                 if (!vq)
 709                         continue;
 710                 vq->vid = internal->vid;
 711                 vq->internal = internal;
 712                 vq->port = eth_dev->data->port_id;
 713         }
 714 }
 715
 716 static int
 717 new_device(int vid)
 718 {
 719         struct rte_eth_dev *eth_dev;
 720         struct internal_list *list;
 721         struct pmd_internal *internal;
 722         struct rte_eth_conf *dev_conf;
 723         unsigned i;
 724         char ifname[PATH_MAX];
 725 #ifdef RTE_LIBRTE_VHOST_NUMA
 726         int newnode;
 727 #endif
 728
 729         rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
 730         list = find_internal_resource(ifname);
 731         if (list == NULL) {
 732                 VHOST_LOG(INFO, "Invalid device name: %s\n", ifname);
 733                 return -1;
 734         }
 735
 736         eth_dev = list->eth_dev;
 737         internal = eth_dev->data->dev_private;
 738         dev_conf = &eth_dev->data->dev_conf;
 739
 740 #ifdef RTE_LIBRTE_VHOST_NUMA
 741         newnode = rte_vhost_get_numa_node(vid);
 742         if (newnode >= 0)
 743                 eth_dev->data->numa_node = newnode;
 744 #endif
 745
 746         internal->vid = vid;
 747         if (rte_atomic32_read(&internal->started) == 1) {
 748                 queue_setup(eth_dev, internal);
 749
 750                 if (dev_conf->intr_conf.rxq) {
 751                         if (eth_vhost_install_intr(eth_dev) < 0) {
 752                                 VHOST_LOG(INFO,
 753                                         "Failed to install interrupt handler.");
 754                                         return -1;
 755                         }
 756                 }
 757         } else {
 758                 VHOST_LOG(INFO, "RX/TX queues not exist yet\n");
 759         }
 760
 761         for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
 762                 rte_vhost_enable_guest_notification(vid, i, 0);
 763
 764         rte_vhost_get_mtu(vid, &eth_dev->data->mtu);
 765
 766         eth_dev->data->dev_link.link_status = ETH_LINK_UP;
 767
 768         rte_atomic32_set(&internal->dev_attached, 1);
 769         update_queuing_status(eth_dev);
 770
 771         VHOST_LOG(INFO, "Vhost device %d created\n", vid);
 772
 773         _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 774
 775         return 0;
 776 }
 777
 778 static void
 779 destroy_device(int vid)
 780 {
 781         struct rte_eth_dev *eth_dev;
 782         struct pmd_internal *internal;
 783         struct vhost_queue *vq;
 784         struct internal_list *list;
 785         char ifname[PATH_MAX];
 786         unsigned i;
 787         struct rte_vhost_vring_state *state;
 788
 789         rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
 790         list = find_internal_resource(ifname);
 791         if (list == NULL) {
 792                 VHOST_LOG(ERR, "Invalid interface name: %s\n", ifname);
 793                 return;
 794         }
 795         eth_dev = list->eth_dev;
 796         internal = eth_dev->data->dev_private;
 797
 798         rte_atomic32_set(&internal->dev_attached, 0);
 799         update_queuing_status(eth_dev);
 800
 801         eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
 802
 803         if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) {
 804                 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 805                         vq = eth_dev->data->rx_queues[i];
 806                         if (!vq)
 807                                 continue;
 808                         vq->vid = -1;
 809                 }
 810                 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 811                         vq = eth_dev->data->tx_queues[i];
 812                         if (!vq)
 813                                 continue;
 814                         vq->vid = -1;
 815                 }
 816         }
 817
 818         state = vring_states[eth_dev->data->port_id];
 819         rte_spinlock_lock(&state->lock);
 820         for (i = 0; i <= state->max_vring; i++) {
 821                 state->cur[i] = false;
 822                 state->seen[i] = false;
 823         }
 824         state->max_vring = 0;
 825         rte_spinlock_unlock(&state->lock);
 826
 827         VHOST_LOG(INFO, "Vhost device %d destroyed\n", vid);
 828         eth_vhost_uninstall_intr(eth_dev);
 829
 830         _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 831 }
 832
 833 static int
 834 vring_state_changed(int vid, uint16_t vring, int enable)
 835 {
 836         struct rte_vhost_vring_state *state;
 837         struct rte_eth_dev *eth_dev;
 838         struct internal_list *list;
 839         char ifname[PATH_MAX];
 840
 841         rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
 842         list = find_internal_resource(ifname);
 843         if (list == NULL) {
 844                 VHOST_LOG(ERR, "Invalid interface name: %s\n", ifname);
 845                 return -1;
 846         }
 847
 848         eth_dev = list->eth_dev;
 849         /* won't be NULL */
 850         state = vring_states[eth_dev->data->port_id];
 851         rte_spinlock_lock(&state->lock);
 852         if (state->cur[vring] == enable) {
 853                 rte_spinlock_unlock(&state->lock);
 854                 return 0;
 855         }
 856         state->cur[vring] = enable;
 857         state->max_vring = RTE_MAX(vring, state->max_vring);
 858         rte_spinlock_unlock(&state->lock);
 859
 860         VHOST_LOG(INFO, "vring%u is %s\n",
 861                         vring, enable ? "enabled" : "disabled");
 862
 863         _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE, NULL);
 864
 865         return 0;
 866 }
 867
 868 static struct vhost_device_ops vhost_ops = {
 869         .new_device          = new_device,
 870         .destroy_device      = destroy_device,
 871         .vring_state_changed = vring_state_changed,
 872 };
 873
 874 static int
 875 vhost_driver_setup(struct rte_eth_dev *eth_dev)
 876 {
 877         struct pmd_internal *internal = eth_dev->data->dev_private;
 878         struct internal_list *list = NULL;
 879         struct rte_vhost_vring_state *vring_state = NULL;
 880         unsigned int numa_node = eth_dev->device->numa_node;
 881         const char *name = eth_dev->device->name;
 882
 883         /* Don't try to setup again if it has already been done. */
 884         list = find_internal_resource(internal->iface_name);
 885         if (list)
 886                 return 0;
 887
 888         list = rte_zmalloc_socket(name, sizeof(*list), 0, numa_node);
 889         if (list == NULL)
 890                 return -1;
 891
 892         vring_state = rte_zmalloc_socket(name, sizeof(*vring_state),
 893                                          0, numa_node);
 894         if (vring_state == NULL)
 895                 goto free_list;
 896
 897         list->eth_dev = eth_dev;
 898         pthread_mutex_lock(&internal_list_lock);
 899         TAILQ_INSERT_TAIL(&internal_list, list, next);
 900         pthread_mutex_unlock(&internal_list_lock);
 901
 902         rte_spinlock_init(&vring_state->lock);
 903         vring_states[eth_dev->data->port_id] = vring_state;
 904
 905         if (rte_vhost_driver_register(internal->iface_name, internal->flags))
 906                 goto list_remove;
 907
 908         if (internal->disable_flags) {
 909                 if (rte_vhost_driver_disable_features(internal->iface_name,
 910                                                       internal->disable_flags))
 911                         goto drv_unreg;
 912         }
 913
 914         if (rte_vhost_driver_callback_register(internal->iface_name,
 915                                                &vhost_ops) < 0) {
 916                 VHOST_LOG(ERR, "Can't register callbacks\n");
 917                 goto drv_unreg;
 918         }
 919
 920         if (rte_vhost_driver_start(internal->iface_name) < 0) {
 921                 VHOST_LOG(ERR, "Failed to start driver for %s\n",
 922                           internal->iface_name);
 923                 goto drv_unreg;
 924         }
 925
 926         return 0;
 927
 928 drv_unreg:
 929         rte_vhost_driver_unregister(internal->iface_name);
 930 list_remove:
 931         vring_states[eth_dev->data->port_id] = NULL;
 932         pthread_mutex_lock(&internal_list_lock);
 933         TAILQ_REMOVE(&internal_list, list, next);
 934         pthread_mutex_unlock(&internal_list_lock);
 935         rte_free(vring_state);
 936 free_list:
 937         rte_free(list);
 938
 939         return -1;
 940 }
 941
 942 int
 943 rte_eth_vhost_get_queue_event(uint16_t port_id,
 944                 struct rte_eth_vhost_queue_event *event)
 945 {
 946         struct rte_vhost_vring_state *state;
 947         unsigned int i;
 948         int idx;
 949
 950         if (port_id >= RTE_MAX_ETHPORTS) {
 951                 VHOST_LOG(ERR, "Invalid port id\n");
 952                 return -1;
 953         }
 954
 955         state = vring_states[port_id];
 956         if (!state) {
 957                 VHOST_LOG(ERR, "Unused port\n");
 958                 return -1;
 959         }
 960
 961         rte_spinlock_lock(&state->lock);
 962         for (i = 0; i <= state->max_vring; i++) {
 963                 idx = state->index++ % (state->max_vring + 1);
 964
 965                 if (state->cur[idx] != state->seen[idx]) {
 966                         state->seen[idx] = state->cur[idx];
 967                         event->queue_id = idx / 2;
 968                         event->rx = idx & 1;
 969                         event->enable = state->cur[idx];
 970                         rte_spinlock_unlock(&state->lock);
 971                         return 0;
 972                 }
 973         }
 974         rte_spinlock_unlock(&state->lock);
 975
 976         return -1;
 977 }
 978
 979 int
 980 rte_eth_vhost_get_vid_from_port_id(uint16_t port_id)
 981 {
 982         struct internal_list *list;
 983         struct rte_eth_dev *eth_dev;
 984         struct vhost_queue *vq;
 985         int vid = -1;
 986
 987         if (!rte_eth_dev_is_valid_port(port_id))
 988                 return -1;
 989
 990         pthread_mutex_lock(&internal_list_lock);
 991
 992         TAILQ_FOREACH(list, &internal_list, next) {
 993                 eth_dev = list->eth_dev;
 994                 if (eth_dev->data->port_id == port_id) {
 995                         vq = eth_dev->data->rx_queues[0];
 996                         if (vq) {
 997                                 vid = vq->vid;
 998                         }
 999                         break;
1000                 }
1001         }
1002
1003         pthread_mutex_unlock(&internal_list_lock);
1004
1005         return vid;
1006 }
1007
1008 static int
1009 eth_dev_configure(struct rte_eth_dev *dev)
1010 {
1011         struct pmd_internal *internal = dev->data->dev_private;
1012         const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
1013
1014         /* NOTE: the same process has to operate a vhost interface
1015          * from beginning to end (from eth_dev configure to eth_dev close).
1016          * It is user's responsibility at the moment.
1017          */
1018         if (vhost_driver_setup(dev) < 0)
1019                 return -1;
1020
1021         internal->vlan_strip = !!(rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
1022
1023         return 0;
1024 }
1025
1026 static int
1027 eth_dev_start(struct rte_eth_dev *eth_dev)
1028 {
1029         struct pmd_internal *internal = eth_dev->data->dev_private;
1030         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
1031
1032         queue_setup(eth_dev, internal);
1033
1034         if (rte_atomic32_read(&internal->dev_attached) == 1) {
1035                 if (dev_conf->intr_conf.rxq) {
1036                         if (eth_vhost_install_intr(eth_dev) < 0) {
1037                                 VHOST_LOG(INFO,
1038                                         "Failed to install interrupt handler.");
1039                                         return -1;
1040                         }
1041                 }
1042         }
1043
1044         rte_atomic32_set(&internal->started, 1);
1045         update_queuing_status(eth_dev);
1046
1047         return 0;
1048 }
1049
1050 static void
1051 eth_dev_stop(struct rte_eth_dev *dev)
1052 {
1053         struct pmd_internal *internal = dev->data->dev_private;
1054
1055         rte_atomic32_set(&internal->started, 0);
1056         update_queuing_status(dev);
1057 }
1058
1059 static void
1060 eth_dev_close(struct rte_eth_dev *dev)
1061 {
1062         struct pmd_internal *internal;
1063         struct internal_list *list;
1064         unsigned int i;
1065
1066         internal = dev->data->dev_private;
1067         if (!internal)
1068                 return;
1069
1070         eth_dev_stop(dev);
1071
1072         list = find_internal_resource(internal->iface_name);
1073         if (list) {
1074                 rte_vhost_driver_unregister(internal->iface_name);
1075                 pthread_mutex_lock(&internal_list_lock);
1076                 TAILQ_REMOVE(&internal_list, list, next);
1077                 pthread_mutex_unlock(&internal_list_lock);
1078                 rte_free(list);
1079         }
1080
1081         if (dev->data->rx_queues)
1082                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1083                         rte_free(dev->data->rx_queues[i]);
1084
1085         if (dev->data->tx_queues)
1086                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1087                         rte_free(dev->data->tx_queues[i]);
1088
1089         rte_free(internal->iface_name);
1090         rte_free(internal);
1091
1092         dev->data->dev_private = NULL;
1093
1094         rte_free(vring_states[dev->data->port_id]);
1095         vring_states[dev->data->port_id] = NULL;
1096 }
1097
1098 static int
1099 eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1100                    uint16_t nb_rx_desc __rte_unused,
1101                    unsigned int socket_id,
1102                    const struct rte_eth_rxconf *rx_conf __rte_unused,
1103                    struct rte_mempool *mb_pool)
1104 {
1105         struct vhost_queue *vq;
1106
1107         vq = rte_zmalloc_socket(NULL, sizeof(struct vhost_queue),
1108                         RTE_CACHE_LINE_SIZE, socket_id);
1109         if (vq == NULL) {
1110                 VHOST_LOG(ERR, "Failed to allocate memory for rx queue\n");
1111                 return -ENOMEM;
1112         }
1113
1114         vq->mb_pool = mb_pool;
1115         vq->virtqueue_id = rx_queue_id * VIRTIO_QNUM + VIRTIO_TXQ;
1116         dev->data->rx_queues[rx_queue_id] = vq;
1117
1118         return 0;
1119 }
1120
1121 static int
1122 eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1123                    uint16_t nb_tx_desc __rte_unused,
1124                    unsigned int socket_id,
1125                    const struct rte_eth_txconf *tx_conf __rte_unused)
1126 {
1127         struct vhost_queue *vq;
1128
1129         vq = rte_zmalloc_socket(NULL, sizeof(struct vhost_queue),
1130                         RTE_CACHE_LINE_SIZE, socket_id);
1131         if (vq == NULL) {
1132                 VHOST_LOG(ERR, "Failed to allocate memory for tx queue\n");
1133                 return -ENOMEM;
1134         }
1135
1136         vq->virtqueue_id = tx_queue_id * VIRTIO_QNUM + VIRTIO_RXQ;
1137         dev->data->tx_queues[tx_queue_id] = vq;
1138
1139         return 0;
1140 }
1141
1142 static int
1143 eth_dev_info(struct rte_eth_dev *dev,
1144              struct rte_eth_dev_info *dev_info)
1145 {
1146         struct pmd_internal *internal;
1147
1148         internal = dev->data->dev_private;
1149         if (internal == NULL) {
1150                 VHOST_LOG(ERR, "Invalid device specified\n");
1151                 return -ENODEV;
1152         }
1153
1154         dev_info->max_mac_addrs = 1;
1155         dev_info->max_rx_pktlen = (uint32_t)-1;
1156         dev_info->max_rx_queues = internal->max_queues;
1157         dev_info->max_tx_queues = internal->max_queues;
1158         dev_info->min_rx_bufsize = 0;
1159
1160         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
1161                                 DEV_TX_OFFLOAD_VLAN_INSERT;
1162         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1163
1164         return 0;
1165 }
1166
1167 static int
1168 eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1169 {
1170         unsigned i;
1171         unsigned long rx_total = 0, tx_total = 0;
1172         unsigned long rx_total_bytes = 0, tx_total_bytes = 0;
1173         struct vhost_queue *vq;
1174
1175         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
1176                         i < dev->data->nb_rx_queues; i++) {
1177                 if (dev->data->rx_queues[i] == NULL)
1178                         continue;
1179                 vq = dev->data->rx_queues[i];
1180                 stats->q_ipackets[i] = vq->stats.pkts;
1181                 rx_total += stats->q_ipackets[i];
1182
1183                 stats->q_ibytes[i] = vq->stats.bytes;
1184                 rx_total_bytes += stats->q_ibytes[i];
1185         }
1186
1187         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
1188                         i < dev->data->nb_tx_queues; i++) {
1189                 if (dev->data->tx_queues[i] == NULL)
1190                         continue;
1191                 vq = dev->data->tx_queues[i];
1192                 stats->q_opackets[i] = vq->stats.pkts;
1193                 tx_total += stats->q_opackets[i];
1194
1195                 stats->q_obytes[i] = vq->stats.bytes;
1196                 tx_total_bytes += stats->q_obytes[i];
1197         }
1198
1199         stats->ipackets = rx_total;
1200         stats->opackets = tx_total;
1201         stats->ibytes = rx_total_bytes;
1202         stats->obytes = tx_total_bytes;
1203
1204         return 0;
1205 }
1206
1207 static int
1208 eth_stats_reset(struct rte_eth_dev *dev)
1209 {
1210         struct vhost_queue *vq;
1211         unsigned i;
1212
1213         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1214                 if (dev->data->rx_queues[i] == NULL)
1215                         continue;
1216                 vq = dev->data->rx_queues[i];
1217                 vq->stats.pkts = 0;
1218                 vq->stats.bytes = 0;
1219         }
1220         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                 if (dev->data->tx_queues[i] == NULL)
1222                         continue;
1223                 vq = dev->data->tx_queues[i];
1224                 vq->stats.pkts = 0;
1225                 vq->stats.bytes = 0;
1226                 vq->stats.missed_pkts = 0;
1227         }
1228
1229         return 0;
1230 }
1231
1232 static void
1233 eth_queue_release(void *q)
1234 {
1235         rte_free(q);
1236 }
1237
1238 static int
1239 eth_tx_done_cleanup(void *txq __rte_unused, uint32_t free_cnt __rte_unused)
1240 {
1241         /*
1242          * vHost does not hang onto mbuf. eth_vhost_tx() copies packet data
1243          * and releases mbuf, so nothing to cleanup.
1244          */
1245         return 0;
1246 }
1247
1248 static int
1249 eth_link_update(struct rte_eth_dev *dev __rte_unused,
1250                 int wait_to_complete __rte_unused)
1251 {
1252         return 0;
1253 }
1254
1255 static uint32_t
1256 eth_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1257 {
1258         struct vhost_queue *vq;
1259
1260         vq = dev->data->rx_queues[rx_queue_id];
1261         if (vq == NULL)
1262                 return 0;
1263
1264         return rte_vhost_rx_queue_count(vq->vid, vq->virtqueue_id);
1265 }
1266
1267 static const struct eth_dev_ops ops = {
1268         .dev_start = eth_dev_start,
1269         .dev_stop = eth_dev_stop,
1270         .dev_close = eth_dev_close,
1271         .dev_configure = eth_dev_configure,
1272         .dev_infos_get = eth_dev_info,
1273         .rx_queue_setup = eth_rx_queue_setup,
1274         .tx_queue_setup = eth_tx_queue_setup,
1275         .rx_queue_release = eth_queue_release,
1276         .tx_queue_release = eth_queue_release,
1277         .tx_done_cleanup = eth_tx_done_cleanup,
1278         .rx_queue_count = eth_rx_queue_count,
1279         .link_update = eth_link_update,
1280         .stats_get = eth_stats_get,
1281         .stats_reset = eth_stats_reset,
1282         .xstats_reset = vhost_dev_xstats_reset,
1283         .xstats_get = vhost_dev_xstats_get,
1284         .xstats_get_names = vhost_dev_xstats_get_names,
1285         .rx_queue_intr_enable = eth_rxq_intr_enable,
1286         .rx_queue_intr_disable = eth_rxq_intr_disable,
1287 };
1288
1289 static int
1290 eth_dev_vhost_create(struct rte_vdev_device *dev, char *iface_name,
1291         int16_t queues, const unsigned int numa_node, uint64_t flags,
1292         uint64_t disable_flags)
1293 {
1294         const char *name = rte_vdev_device_name(dev);
1295         struct rte_eth_dev_data *data;
1296         struct pmd_internal *internal = NULL;
1297         struct rte_eth_dev *eth_dev = NULL;
1298         struct rte_ether_addr *eth_addr = NULL;
1299
1300         VHOST_LOG(INFO, "Creating VHOST-USER backend on numa socket %u\n",
1301                 numa_node);
1302
1303         /* reserve an ethdev entry */
1304         eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internal));
1305         if (eth_dev == NULL)
1306                 goto error;
1307         data = eth_dev->data;
1308
1309         eth_addr = rte_zmalloc_socket(name, sizeof(*eth_addr), 0, numa_node);
1310         if (eth_addr == NULL)
1311                 goto error;
1312         data->mac_addrs = eth_addr;
1313         *eth_addr = base_eth_addr;
1314         eth_addr->addr_bytes[5] = eth_dev->data->port_id;
1315
1316         /* now put it all together
1317          * - store queue data in internal,
1318          * - point eth_dev_data to internals
1319          * - and point eth_dev structure to new eth_dev_data structure
1320          */
1321         internal = eth_dev->data->dev_private;
1322         internal->iface_name = rte_malloc_socket(name, strlen(iface_name) + 1,
1323                                                  0, numa_node);
1324         if (internal->iface_name == NULL)
1325                 goto error;
1326         strcpy(internal->iface_name, iface_name);
1327
1328         data->nb_rx_queues = queues;
1329         data->nb_tx_queues = queues;
1330         internal->max_queues = queues;
1331         internal->vid = -1;
1332         internal->flags = flags;
1333         internal->disable_flags = disable_flags;
1334         data->dev_link = pmd_link;
1335         data->dev_flags = RTE_ETH_DEV_INTR_LSC | RTE_ETH_DEV_CLOSE_REMOVE;
1336         data->promiscuous = 1;
1337         data->all_multicast = 1;
1338
1339         eth_dev->dev_ops = &ops;
1340
1341         /* finally assign rx and tx ops */
1342         eth_dev->rx_pkt_burst = eth_vhost_rx;
1343         eth_dev->tx_pkt_burst = eth_vhost_tx;
1344
1345         rte_eth_dev_probing_finish(eth_dev);
1346         return 0;
1347
1348 error:
1349         if (internal)
1350                 rte_free(internal->iface_name);
1351         rte_eth_dev_release_port(eth_dev);
1352
1353         return -1;
1354 }
1355
1356 static inline int
1357 open_iface(const char *key __rte_unused, const char *value, void *extra_args)
1358 {
1359         const char **iface_name = extra_args;
1360
1361         if (value == NULL)
1362                 return -1;
1363
1364         *iface_name = value;
1365
1366         return 0;
1367 }
1368
1369 static inline int
1370 open_int(const char *key __rte_unused, const char *value, void *extra_args)
1371 {
1372         uint16_t *n = extra_args;
1373
1374         if (value == NULL || extra_args == NULL)
1375                 return -EINVAL;
1376
1377         *n = (uint16_t)strtoul(value, NULL, 0);
1378         if (*n == USHRT_MAX && errno == ERANGE)
1379                 return -1;
1380
1381         return 0;
1382 }
1383
1384 static int
1385 rte_pmd_vhost_probe(struct rte_vdev_device *dev)
1386 {
1387         struct rte_kvargs *kvlist = NULL;
1388         int ret = 0;
1389         char *iface_name;
1390         uint16_t queues;
1391         uint64_t flags = 0;
1392         uint64_t disable_flags = 0;
1393         int client_mode = 0;
1394         int dequeue_zero_copy = 0;
1395         int iommu_support = 0;
1396         int postcopy_support = 0;
1397         int tso = 0;
1398         int linear_buf = 0;
1399         int ext_buf = 0;
1400         struct rte_eth_dev *eth_dev;
1401         const char *name = rte_vdev_device_name(dev);
1402
1403         VHOST_LOG(INFO, "Initializing pmd_vhost for %s\n", name);
1404
1405         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1406                 eth_dev = rte_eth_dev_attach_secondary(name);
1407                 if (!eth_dev) {
1408                         VHOST_LOG(ERR, "Failed to probe %s\n", name);
1409                         return -1;
1410                 }
1411                 eth_dev->rx_pkt_burst = eth_vhost_rx;
1412                 eth_dev->tx_pkt_burst = eth_vhost_tx;
1413                 eth_dev->dev_ops = &ops;
1414                 if (dev->device.numa_node == SOCKET_ID_ANY)
1415                         dev->device.numa_node = rte_socket_id();
1416                 eth_dev->device = &dev->device;
1417                 rte_eth_dev_probing_finish(eth_dev);
1418                 return 0;
1419         }
1420
1421         kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments);
1422         if (kvlist == NULL)
1423                 return -1;
1424
1425         if (rte_kvargs_count(kvlist, ETH_VHOST_IFACE_ARG) == 1) {
1426                 ret = rte_kvargs_process(kvlist, ETH_VHOST_IFACE_ARG,
1427                                          &open_iface, &iface_name);
1428                 if (ret < 0)
1429                         goto out_free;
1430         } else {
1431                 ret = -1;
1432                 goto out_free;
1433         }
1434
1435         if (rte_kvargs_count(kvlist, ETH_VHOST_QUEUES_ARG) == 1) {
1436                 ret = rte_kvargs_process(kvlist, ETH_VHOST_QUEUES_ARG,
1437                                          &open_int, &queues);
1438                 if (ret < 0 || queues > RTE_MAX_QUEUES_PER_PORT)
1439                         goto out_free;
1440
1441         } else
1442                 queues = 1;
1443
1444         if (rte_kvargs_count(kvlist, ETH_VHOST_CLIENT_ARG) == 1) {
1445                 ret = rte_kvargs_process(kvlist, ETH_VHOST_CLIENT_ARG,
1446                                          &open_int, &client_mode);
1447                 if (ret < 0)
1448                         goto out_free;
1449
1450                 if (client_mode)
1451                         flags |= RTE_VHOST_USER_CLIENT;
1452         }
1453
1454         if (rte_kvargs_count(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY) == 1) {
1455                 ret = rte_kvargs_process(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY,
1456                                          &open_int, &dequeue_zero_copy);
1457                 if (ret < 0)
1458                         goto out_free;
1459
1460                 if (dequeue_zero_copy)
1461                         flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
1462         }
1463
1464         if (rte_kvargs_count(kvlist, ETH_VHOST_IOMMU_SUPPORT) == 1) {
1465                 ret = rte_kvargs_process(kvlist, ETH_VHOST_IOMMU_SUPPORT,
1466                                          &open_int, &iommu_support);
1467                 if (ret < 0)
1468                         goto out_free;
1469
1470                 if (iommu_support)
1471                         flags |= RTE_VHOST_USER_IOMMU_SUPPORT;
1472         }
1473
1474         if (rte_kvargs_count(kvlist, ETH_VHOST_POSTCOPY_SUPPORT) == 1) {
1475                 ret = rte_kvargs_process(kvlist, ETH_VHOST_POSTCOPY_SUPPORT,
1476                                          &open_int, &postcopy_support);
1477                 if (ret < 0)
1478                         goto out_free;
1479
1480                 if (postcopy_support)
1481                         flags |= RTE_VHOST_USER_POSTCOPY_SUPPORT;
1482         }
1483
1484         if (rte_kvargs_count(kvlist, ETH_VHOST_VIRTIO_NET_F_HOST_TSO) == 1) {
1485                 ret = rte_kvargs_process(kvlist,
1486                                 ETH_VHOST_VIRTIO_NET_F_HOST_TSO,
1487                                 &open_int, &tso);
1488                 if (ret < 0)
1489                         goto out_free;
1490
1491                 if (tso == 0) {
1492                         disable_flags |= (1ULL << VIRTIO_NET_F_HOST_TSO4);
1493                         disable_flags |= (1ULL << VIRTIO_NET_F_HOST_TSO6);
1494                 }
1495         }
1496
1497         if (rte_kvargs_count(kvlist, ETH_VHOST_LINEAR_BUF) == 1) {
1498                 ret = rte_kvargs_process(kvlist,
1499                                 ETH_VHOST_LINEAR_BUF,
1500                                 &open_int, &linear_buf);
1501                 if (ret < 0)
1502                         goto out_free;
1503
1504                 if (linear_buf == 1)
1505                         flags |= RTE_VHOST_USER_LINEARBUF_SUPPORT;
1506         }
1507
1508         if (rte_kvargs_count(kvlist, ETH_VHOST_EXT_BUF) == 1) {
1509                 ret = rte_kvargs_process(kvlist,
1510                                 ETH_VHOST_EXT_BUF,
1511                                 &open_int, &ext_buf);
1512                 if (ret < 0)
1513                         goto out_free;
1514
1515                 if (ext_buf == 1)
1516                         flags |= RTE_VHOST_USER_EXTBUF_SUPPORT;
1517         }
1518
1519         if (dev->device.numa_node == SOCKET_ID_ANY)
1520                 dev->device.numa_node = rte_socket_id();
1521
1522         ret = eth_dev_vhost_create(dev, iface_name, queues,
1523                                    dev->device.numa_node, flags, disable_flags);
1524         if (ret == -1)
1525                 VHOST_LOG(ERR, "Failed to create %s\n", name);
1526
1527 out_free:
1528         rte_kvargs_free(kvlist);
1529         return ret;
1530 }
1531
1532 static int
1533 rte_pmd_vhost_remove(struct rte_vdev_device *dev)
1534 {
1535         const char *name;
1536         struct rte_eth_dev *eth_dev = NULL;
1537
1538         name = rte_vdev_device_name(dev);
1539         VHOST_LOG(INFO, "Un-Initializing pmd_vhost for %s\n", name);
1540
1541         /* find an ethdev entry */
1542         eth_dev = rte_eth_dev_allocated(name);
1543         if (eth_dev == NULL)
1544                 return 0;
1545
1546         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1547                 return rte_eth_dev_release_port(eth_dev);
1548
1549         eth_dev_close(eth_dev);
1550
1551         rte_eth_dev_release_port(eth_dev);
1552
1553         return 0;
1554 }
1555
1556 static struct rte_vdev_driver pmd_vhost_drv = {
1557         .probe = rte_pmd_vhost_probe,
1558         .remove = rte_pmd_vhost_remove,
1559 };
1560
1561 RTE_PMD_REGISTER_VDEV(net_vhost, pmd_vhost_drv);
1562 RTE_PMD_REGISTER_ALIAS(net_vhost, eth_vhost);
1563 RTE_PMD_REGISTER_PARAM_STRING(net_vhost,
1564         "iface=<ifc> "
1565         "queues=<int> "
1566         "client=<0|1> "
1567         "dequeue-zero-copy=<0|1> "
1568         "iommu-support=<0|1> "
1569         "postcopy-support=<0|1> "
1570         "tso=<0|1> "
1571         "linear-buffer=<0|1> "
1572         "ext-buffer=<0|1>");
1573
1574 RTE_INIT(vhost_init_log)
1575 {
1576         vhost_logtype = rte_log_register("pmd.net.vhost");
1577         if (vhost_logtype >= 0)
1578                 rte_log_set_level(vhost_logtype, RTE_LOG_NOTICE);
1579 }