examples/vhost/main.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <arpa/inet.h>
  35 #include <getopt.h>
  36 #include <linux/if_ether.h>
  37 #include <linux/if_vlan.h>
  38 #include <linux/virtio_net.h>
  39 #include <linux/virtio_ring.h>
  40 #include <signal.h>
  41 #include <stdint.h>
  42 #include <sys/eventfd.h>
  43 #include <sys/param.h>
  44 #include <unistd.h>
  45
  46 #include <rte_atomic.h>
  47 #include <rte_cycles.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_log.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_malloc.h>
  52 #include <rte_virtio_net.h>
  53 #include <rte_ip.h>
  54 #include <rte_tcp.h>
  55 #include <rte_udp.h>
  56 #include <rte_sctp.h>
  57
  58 #include "main.h"
  59
  60 #ifndef MAX_QUEUES
  61 #define MAX_QUEUES 128
  62 #endif
  63
  64 /* the maximum number of external ports supported */
  65 #define MAX_SUP_PORTS 1
  66
  67 /*
  68  * Calculate the number of buffers needed per port
  69  */
  70 #define NUM_MBUFS_PER_PORT ((MAX_QUEUES*RTE_TEST_RX_DESC_DEFAULT) +             \
  71                                                         (num_switching_cores*MAX_PKT_BURST) +                   \
  72                                                         (num_switching_cores*RTE_TEST_TX_DESC_DEFAULT) +\
  73                                                         (num_switching_cores*MBUF_CACHE_SIZE))
  74
  75 #define MBUF_CACHE_SIZE 128
  76 #define MBUF_DATA_SIZE  RTE_MBUF_DEFAULT_BUF_SIZE
  77
  78 /*
  79  * No frame data buffer allocated from host are required for zero copy
  80  * implementation, guest will allocate the frame data buffer, and vhost
  81  * directly use it.
  82  */
  83 #define VIRTIO_DESCRIPTOR_LEN_ZCP       RTE_MBUF_DEFAULT_DATAROOM
  84 #define MBUF_DATA_SIZE_ZCP              RTE_MBUF_DEFAULT_BUF_SIZE
  85 #define MBUF_CACHE_SIZE_ZCP 0
  86
  87 #define MAX_PKT_BURST 32                /* Max burst size for RX/TX */
  88 #define BURST_TX_DRAIN_US 100   /* TX drain every ~100us */
  89
  90 #define BURST_RX_WAIT_US 15     /* Defines how long we wait between retries on RX */
  91 #define BURST_RX_RETRIES 4              /* Number of retries on RX. */
  92
  93 #define JUMBO_FRAME_MAX_SIZE    0x2600
  94
  95 /* State of virtio device. */
  96 #define DEVICE_MAC_LEARNING 0
  97 #define DEVICE_RX                       1
  98 #define DEVICE_SAFE_REMOVE      2
  99
 100 /* Config_core_flag status definitions. */
 101 #define REQUEST_DEV_REMOVAL 1
 102 #define ACK_DEV_REMOVAL 0
 103
 104 /* Configurable number of RX/TX ring descriptors */
 105 #define RTE_TEST_RX_DESC_DEFAULT 1024
 106 #define RTE_TEST_TX_DESC_DEFAULT 512
 107
 108 /*
 109  * Need refine these 2 macros for legacy and DPDK based front end:
 110  * Max vring avail descriptor/entries from guest - MAX_PKT_BURST
 111  * And then adjust power 2.
 112  */
 113 /*
 114  * For legacy front end, 128 descriptors,
 115  * half for virtio header, another half for mbuf.
 116  */
 117 #define RTE_TEST_RX_DESC_DEFAULT_ZCP 32   /* legacy: 32, DPDK virt FE: 128. */
 118 #define RTE_TEST_TX_DESC_DEFAULT_ZCP 64   /* legacy: 64, DPDK virt FE: 64.  */
 119
 120 /* Get first 4 bytes in mbuf headroom. */
 121 #define MBUF_HEADROOM_UINT32(mbuf) (*(uint32_t *)((uint8_t *)(mbuf) \
 122                 + sizeof(struct rte_mbuf)))
 123
 124 /* true if x is a power of 2 */
 125 #define POWEROF2(x) ((((x)-1) & (x)) == 0)
 126
 127 #define INVALID_PORT_ID 0xFF
 128
 129 /* Max number of devices. Limited by vmdq. */
 130 #define MAX_DEVICES 64
 131
 132 /* Size of buffers used for snprintfs. */
 133 #define MAX_PRINT_BUFF 6072
 134
 135 /* Maximum character device basename size. */
 136 #define MAX_BASENAME_SZ 10
 137
 138 /* Maximum long option length for option parsing. */
 139 #define MAX_LONG_OPT_SZ 64
 140
 141 /* Used to compare MAC addresses. */
 142 #define MAC_ADDR_CMP 0xFFFFFFFFFFFFULL
 143
 144 /* Number of descriptors per cacheline. */
 145 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 146
 147 #define MBUF_EXT_MEM(mb)   (rte_mbuf_from_indirect(mb) != (mb))
 148
 149 /* mask of enabled ports */
 150 static uint32_t enabled_port_mask = 0;
 151
 152 /* Promiscuous mode */
 153 static uint32_t promiscuous;
 154
 155 /*Number of switching cores enabled*/
 156 static uint32_t num_switching_cores = 0;
 157
 158 /* number of devices/queues to support*/
 159 static uint32_t num_queues = 0;
 160 static uint32_t num_devices;
 161
 162 /*
 163  * Enable zero copy, pkts buffer will directly dma to hw descriptor,
 164  * disabled on default.
 165  */
 166 static uint32_t zero_copy;
 167 static int mergeable;
 168
 169 /* Do vlan strip on host, enabled on default */
 170 static uint32_t vlan_strip = 1;
 171
 172 /* number of descriptors to apply*/
 173 static uint32_t num_rx_descriptor = RTE_TEST_RX_DESC_DEFAULT_ZCP;
 174 static uint32_t num_tx_descriptor = RTE_TEST_TX_DESC_DEFAULT_ZCP;
 175
 176 /* max ring descriptor, ixgbe, i40e, e1000 all are 4096. */
 177 #define MAX_RING_DESC 4096
 178
 179 struct vpool {
 180         struct rte_mempool *pool;
 181         struct rte_ring *ring;
 182         uint32_t buf_size;
 183 } vpool_array[MAX_QUEUES+MAX_QUEUES];
 184
 185 /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */
 186 typedef enum {
 187         VM2VM_DISABLED = 0,
 188         VM2VM_SOFTWARE = 1,
 189         VM2VM_HARDWARE = 2,
 190         VM2VM_LAST
 191 } vm2vm_type;
 192 static vm2vm_type vm2vm_mode = VM2VM_SOFTWARE;
 193
 194 /* The type of host physical address translated from guest physical address. */
 195 typedef enum {
 196         PHYS_ADDR_CONTINUOUS = 0,
 197         PHYS_ADDR_CROSS_SUBREG = 1,
 198         PHYS_ADDR_INVALID = 2,
 199         PHYS_ADDR_LAST
 200 } hpa_type;
 201
 202 /* Enable stats. */
 203 static uint32_t enable_stats = 0;
 204 /* Enable retries on RX. */
 205 static uint32_t enable_retry = 1;
 206
 207 /* Disable TX checksum offload */
 208 static uint32_t enable_tx_csum;
 209
 210 /* Disable TSO offload */
 211 static uint32_t enable_tso;
 212
 213 /* Specify timeout (in useconds) between retries on RX. */
 214 static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US;
 215 /* Specify the number of retries on RX. */
 216 static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;
 217
 218 /* Character device basename. Can be set by user. */
 219 static char dev_basename[MAX_BASENAME_SZ] = "vhost-net";
 220
 221 /* empty vmdq configuration structure. Filled in programatically */
 222 static struct rte_eth_conf vmdq_conf_default = {
 223         .rxmode = {
 224                 .mq_mode        = ETH_MQ_RX_VMDQ_ONLY,
 225                 .split_hdr_size = 0,
 226                 .header_split   = 0, /**< Header Split disabled */
 227                 .hw_ip_checksum = 0, /**< IP checksum offload disabled */
 228                 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
 229                 /*
 230                  * It is necessary for 1G NIC such as I350,
 231                  * this fixes bug of ipv4 forwarding in guest can't
 232                  * forward pakets from one virtio dev to another virtio dev.
 233                  */
 234                 .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 235                 .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
 236                 .hw_strip_crc   = 0, /**< CRC stripped by hardware */
 237         },
 238
 239         .txmode = {
 240                 .mq_mode = ETH_MQ_TX_NONE,
 241         },
 242         .rx_adv_conf = {
 243                 /*
 244                  * should be overridden separately in code with
 245                  * appropriate values
 246                  */
 247                 .vmdq_rx_conf = {
 248                         .nb_queue_pools = ETH_8_POOLS,
 249                         .enable_default_pool = 0,
 250                         .default_pool = 0,
 251                         .nb_pool_maps = 0,
 252                         .pool_map = {{0, 0},},
 253                 },
 254         },
 255 };
 256
 257 static unsigned lcore_ids[RTE_MAX_LCORE];
 258 static uint8_t ports[RTE_MAX_ETHPORTS];
 259 static unsigned num_ports = 0; /**< The number of ports specified in command line */
 260 static uint16_t num_pf_queues, num_vmdq_queues;
 261 static uint16_t vmdq_pool_base, vmdq_queue_base;
 262 static uint16_t queues_per_pool;
 263
 264 static const uint16_t external_pkt_default_vlan_tag = 2000;
 265 const uint16_t vlan_tags[] = {
 266         1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007,
 267         1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015,
 268         1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023,
 269         1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031,
 270         1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039,
 271         1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047,
 272         1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055,
 273         1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063,
 274 };
 275
 276 /* ethernet addresses of ports */
 277 static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
 278
 279 /* heads for the main used and free linked lists for the data path. */
 280 static struct virtio_net_data_ll *ll_root_used = NULL;
 281 static struct virtio_net_data_ll *ll_root_free = NULL;
 282
 283 /* Array of data core structures containing information on individual core linked lists. */
 284 static struct lcore_info lcore_info[RTE_MAX_LCORE];
 285
 286 /* Used for queueing bursts of TX packets. */
 287 struct mbuf_table {
 288         unsigned len;
 289         unsigned txq_id;
 290         struct rte_mbuf *m_table[MAX_PKT_BURST];
 291 };
 292
 293 /* TX queue for each data core. */
 294 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE];
 295
 296 /* TX queue fori each virtio device for zero copy. */
 297 struct mbuf_table tx_queue_zcp[MAX_QUEUES];
 298
 299 /* Vlan header struct used to insert vlan tags on TX. */
 300 struct vlan_ethhdr {
 301         unsigned char   h_dest[ETH_ALEN];
 302         unsigned char   h_source[ETH_ALEN];
 303         __be16          h_vlan_proto;
 304         __be16          h_vlan_TCI;
 305         __be16          h_vlan_encapsulated_proto;
 306 };
 307
 308 /* Header lengths. */
 309 #define VLAN_HLEN       4
 310 #define VLAN_ETH_HLEN   18
 311
 312 /* Per-device statistics struct */
 313 struct device_statistics {
 314         uint64_t tx_total;
 315         rte_atomic64_t rx_total_atomic;
 316         uint64_t rx_total;
 317         uint64_t tx;
 318         rte_atomic64_t rx_atomic;
 319         uint64_t rx;
 320 } __rte_cache_aligned;
 321 struct device_statistics dev_statistics[MAX_DEVICES];
 322
 323 /*
 324  * Builds up the correct configuration for VMDQ VLAN pool map
 325  * according to the pool & queue limits.
 326  */
 327 static inline int
 328 get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices)
 329 {
 330         struct rte_eth_vmdq_rx_conf conf;
 331         struct rte_eth_vmdq_rx_conf *def_conf =
 332                 &vmdq_conf_default.rx_adv_conf.vmdq_rx_conf;
 333         unsigned i;
 334
 335         memset(&conf, 0, sizeof(conf));
 336         conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices;
 337         conf.nb_pool_maps = num_devices;
 338         conf.enable_loop_back = def_conf->enable_loop_back;
 339         conf.rx_mode = def_conf->rx_mode;
 340
 341         for (i = 0; i < conf.nb_pool_maps; i++) {
 342                 conf.pool_map[i].vlan_id = vlan_tags[ i ];
 343                 conf.pool_map[i].pools = (1UL << i);
 344         }
 345
 346         (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf)));
 347         (void)(rte_memcpy(&eth_conf->rx_adv_conf.vmdq_rx_conf, &conf,
 348                    sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf)));
 349         return 0;
 350 }
 351
 352 /*
 353  * Validate the device number according to the max pool number gotten form
 354  * dev_info. If the device number is invalid, give the error message and
 355  * return -1. Each device must have its own pool.
 356  */
 357 static inline int
 358 validate_num_devices(uint32_t max_nb_devices)
 359 {
 360         if (num_devices > max_nb_devices) {
 361                 RTE_LOG(ERR, VHOST_PORT, "invalid number of devices\n");
 362                 return -1;
 363         }
 364         return 0;
 365 }
 366
 367 /*
 368  * Initialises a given port using global settings and with the rx buffers
 369  * coming from the mbuf_pool passed as parameter
 370  */
 371 static inline int
 372 port_init(uint8_t port)
 373 {
 374         struct rte_eth_dev_info dev_info;
 375         struct rte_eth_conf port_conf;
 376         struct rte_eth_rxconf *rxconf;
 377         struct rte_eth_txconf *txconf;
 378         int16_t rx_rings, tx_rings;
 379         uint16_t rx_ring_size, tx_ring_size;
 380         int retval;
 381         uint16_t q;
 382
 383         /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
 384         rte_eth_dev_info_get (port, &dev_info);
 385
 386         if (dev_info.max_rx_queues > MAX_QUEUES) {
 387                 rte_exit(EXIT_FAILURE,
 388                         "please define MAX_QUEUES no less than %u in %s\n",
 389                         dev_info.max_rx_queues, __FILE__);
 390         }
 391
 392         rxconf = &dev_info.default_rxconf;
 393         txconf = &dev_info.default_txconf;
 394         rxconf->rx_drop_en = 1;
 395
 396         /* Enable vlan offload */
 397         txconf->txq_flags &= ~ETH_TXQ_FLAGS_NOVLANOFFL;
 398
 399         /*
 400          * Zero copy defers queue RX/TX start to the time when guest
 401          * finishes its startup and packet buffers from that guest are
 402          * available.
 403          */
 404         if (zero_copy) {
 405                 rxconf->rx_deferred_start = 1;
 406                 rxconf->rx_drop_en = 0;
 407                 txconf->tx_deferred_start = 1;
 408         }
 409
 410         /*configure the number of supported virtio devices based on VMDQ limits */
 411         num_devices = dev_info.max_vmdq_pools;
 412
 413         if (zero_copy) {
 414                 rx_ring_size = num_rx_descriptor;
 415                 tx_ring_size = num_tx_descriptor;
 416                 tx_rings = dev_info.max_tx_queues;
 417         } else {
 418                 rx_ring_size = RTE_TEST_RX_DESC_DEFAULT;
 419                 tx_ring_size = RTE_TEST_TX_DESC_DEFAULT;
 420                 tx_rings = (uint16_t)rte_lcore_count();
 421         }
 422
 423         retval = validate_num_devices(MAX_DEVICES);
 424         if (retval < 0)
 425                 return retval;
 426
 427         /* Get port configuration. */
 428         retval = get_eth_conf(&port_conf, num_devices);
 429         if (retval < 0)
 430                 return retval;
 431         /* NIC queues are divided into pf queues and vmdq queues.  */
 432         num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;
 433         queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;
 434         num_vmdq_queues = num_devices * queues_per_pool;
 435         num_queues = num_pf_queues + num_vmdq_queues;
 436         vmdq_queue_base = dev_info.vmdq_queue_base;
 437         vmdq_pool_base  = dev_info.vmdq_pool_base;
 438         printf("pf queue num: %u, configured vmdq pool num: %u, each vmdq pool has %u queues\n",
 439                 num_pf_queues, num_devices, queues_per_pool);
 440
 441         if (port >= rte_eth_dev_count()) return -1;
 442
 443         if (enable_tx_csum == 0)
 444                 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_CSUM);
 445
 446         if (enable_tso == 0) {
 447                 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_HOST_TSO4);
 448                 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_HOST_TSO6);
 449         }
 450
 451         rx_rings = (uint16_t)dev_info.max_rx_queues;
 452         /* Configure ethernet device. */
 453         retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
 454         if (retval != 0)
 455                 return retval;
 456
 457         /* Setup the queues. */
 458         for (q = 0; q < rx_rings; q ++) {
 459                 retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
 460                                                 rte_eth_dev_socket_id(port),
 461                                                 rxconf,
 462                                                 vpool_array[q].pool);
 463                 if (retval < 0)
 464                         return retval;
 465         }
 466         for (q = 0; q < tx_rings; q ++) {
 467                 retval = rte_eth_tx_queue_setup(port, q, tx_ring_size,
 468                                                 rte_eth_dev_socket_id(port),
 469                                                 txconf);
 470                 if (retval < 0)
 471                         return retval;
 472         }
 473
 474         /* Start the device. */
 475         retval  = rte_eth_dev_start(port);
 476         if (retval < 0) {
 477                 RTE_LOG(ERR, VHOST_DATA, "Failed to start the device.\n");
 478                 return retval;
 479         }
 480
 481         if (promiscuous)
 482                 rte_eth_promiscuous_enable(port);
 483
 484         rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
 485         RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices);
 486         RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
 487                         " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
 488                         (unsigned)port,
 489                         vmdq_ports_eth_addr[port].addr_bytes[0],
 490                         vmdq_ports_eth_addr[port].addr_bytes[1],
 491                         vmdq_ports_eth_addr[port].addr_bytes[2],
 492                         vmdq_ports_eth_addr[port].addr_bytes[3],
 493                         vmdq_ports_eth_addr[port].addr_bytes[4],
 494                         vmdq_ports_eth_addr[port].addr_bytes[5]);
 495
 496         return 0;
 497 }
 498
 499 /*
 500  * Set character device basename.
 501  */
 502 static int
 503 us_vhost_parse_basename(const char *q_arg)
 504 {
 505         /* parse number string */
 506
 507         if (strnlen(q_arg, MAX_BASENAME_SZ) > MAX_BASENAME_SZ)
 508                 return -1;
 509         else
 510                 snprintf((char*)&dev_basename, MAX_BASENAME_SZ, "%s", q_arg);
 511
 512         return 0;
 513 }
 514
 515 /*
 516  * Parse the portmask provided at run time.
 517  */
 518 static int
 519 parse_portmask(const char *portmask)
 520 {
 521         char *end = NULL;
 522         unsigned long pm;
 523
 524         errno = 0;
 525
 526         /* parse hexadecimal string */
 527         pm = strtoul(portmask, &end, 16);
 528         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 529                 return -1;
 530
 531         if (pm == 0)
 532                 return -1;
 533
 534         return pm;
 535
 536 }
 537
 538 /*
 539  * Parse num options at run time.
 540  */
 541 static int
 542 parse_num_opt(const char *q_arg, uint32_t max_valid_value)
 543 {
 544         char *end = NULL;
 545         unsigned long num;
 546
 547         errno = 0;
 548
 549         /* parse unsigned int string */
 550         num = strtoul(q_arg, &end, 10);
 551         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 552                 return -1;
 553
 554         if (num > max_valid_value)
 555                 return -1;
 556
 557         return num;
 558
 559 }
 560
 561 /*
 562  * Display usage
 563  */
 564 static void
 565 us_vhost_usage(const char *prgname)
 566 {
 567         RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n"
 568         "               --vm2vm [0|1|2]\n"
 569         "               --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n"
 570         "               --dev-basename <name>\n"
 571         "               --nb-devices ND\n"
 572         "               -p PORTMASK: Set mask for ports to be used by application\n"
 573         "               --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n"
 574         "               --rx-retry [0|1]: disable/enable(default) retries on rx. Enable retry if destintation queue is full\n"
 575         "               --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX. This makes effect only if retries on rx enabled\n"
 576         "               --rx-retry-num [0-N]: the number of retries on rx. This makes effect only if retries on rx enabled\n"
 577         "               --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n"
 578         "               --vlan-strip [0|1]: disable/enable(default) RX VLAN strip on host\n"
 579         "               --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
 580         "               --dev-basename: The basename to be used for the character device.\n"
 581         "               --zero-copy [0|1]: disable(default)/enable rx/tx "
 582                         "zero copy\n"
 583         "               --rx-desc-num [0-N]: the number of descriptors on rx, "
 584                         "used only when zero copy is enabled.\n"
 585         "               --tx-desc-num [0-N]: the number of descriptors on tx, "
 586                         "used only when zero copy is enabled.\n"
 587         "               --tx-csum [0|1] disable/enable TX checksum offload.\n"
 588         "               --tso [0|1] disable/enable TCP segment offload.\n",
 589                prgname);
 590 }
 591
 592 /*
 593  * Parse the arguments given in the command line of the application.
 594  */
 595 static int
 596 us_vhost_parse_args(int argc, char **argv)
 597 {
 598         int opt, ret;
 599         int option_index;
 600         unsigned i;
 601         const char *prgname = argv[0];
 602         static struct option long_option[] = {
 603                 {"vm2vm", required_argument, NULL, 0},
 604                 {"rx-retry", required_argument, NULL, 0},
 605                 {"rx-retry-delay", required_argument, NULL, 0},
 606                 {"rx-retry-num", required_argument, NULL, 0},
 607                 {"mergeable", required_argument, NULL, 0},
 608                 {"vlan-strip", required_argument, NULL, 0},
 609                 {"stats", required_argument, NULL, 0},
 610                 {"dev-basename", required_argument, NULL, 0},
 611                 {"zero-copy", required_argument, NULL, 0},
 612                 {"rx-desc-num", required_argument, NULL, 0},
 613                 {"tx-desc-num", required_argument, NULL, 0},
 614                 {"tx-csum", required_argument, NULL, 0},
 615                 {"tso", required_argument, NULL, 0},
 616                 {NULL, 0, 0, 0},
 617         };
 618
 619         /* Parse command line */
 620         while ((opt = getopt_long(argc, argv, "p:P",
 621                         long_option, &option_index)) != EOF) {
 622                 switch (opt) {
 623                 /* Portmask */
 624                 case 'p':
 625                         enabled_port_mask = parse_portmask(optarg);
 626                         if (enabled_port_mask == 0) {
 627                                 RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n");
 628                                 us_vhost_usage(prgname);
 629                                 return -1;
 630                         }
 631                         break;
 632
 633                 case 'P':
 634                         promiscuous = 1;
 635                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.rx_mode =
 636                                 ETH_VMDQ_ACCEPT_BROADCAST |
 637                                 ETH_VMDQ_ACCEPT_MULTICAST;
 638                         rte_vhost_feature_enable(1ULL << VIRTIO_NET_F_CTRL_RX);
 639
 640                         break;
 641
 642                 case 0:
 643                         /* Enable/disable vm2vm comms. */
 644                         if (!strncmp(long_option[option_index].name, "vm2vm",
 645                                 MAX_LONG_OPT_SZ)) {
 646                                 ret = parse_num_opt(optarg, (VM2VM_LAST - 1));
 647                                 if (ret == -1) {
 648                                         RTE_LOG(INFO, VHOST_CONFIG,
 649                                                 "Invalid argument for "
 650                                                 "vm2vm [0|1|2]\n");
 651                                         us_vhost_usage(prgname);
 652                                         return -1;
 653                                 } else {
 654                                         vm2vm_mode = (vm2vm_type)ret;
 655                                 }
 656                         }
 657
 658                         /* Enable/disable retries on RX. */
 659                         if (!strncmp(long_option[option_index].name, "rx-retry", MAX_LONG_OPT_SZ)) {
 660                                 ret = parse_num_opt(optarg, 1);
 661                                 if (ret == -1) {
 662                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry [0|1]\n");
 663                                         us_vhost_usage(prgname);
 664                                         return -1;
 665                                 } else {
 666                                         enable_retry = ret;
 667                                 }
 668                         }
 669
 670                         /* Enable/disable TX checksum offload. */
 671                         if (!strncmp(long_option[option_index].name, "tx-csum", MAX_LONG_OPT_SZ)) {
 672                                 ret = parse_num_opt(optarg, 1);
 673                                 if (ret == -1) {
 674                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tx-csum [0|1]\n");
 675                                         us_vhost_usage(prgname);
 676                                         return -1;
 677                                 } else
 678                                         enable_tx_csum = ret;
 679                         }
 680
 681                         /* Enable/disable TSO offload. */
 682                         if (!strncmp(long_option[option_index].name, "tso", MAX_LONG_OPT_SZ)) {
 683                                 ret = parse_num_opt(optarg, 1);
 684                                 if (ret == -1) {
 685                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tso [0|1]\n");
 686                                         us_vhost_usage(prgname);
 687                                         return -1;
 688                                 } else
 689                                         enable_tso = ret;
 690                         }
 691
 692                         /* Specify the retries delay time (in useconds) on RX. */
 693                         if (!strncmp(long_option[option_index].name, "rx-retry-delay", MAX_LONG_OPT_SZ)) {
 694                                 ret = parse_num_opt(optarg, INT32_MAX);
 695                                 if (ret == -1) {
 696                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-delay [0-N]\n");
 697                                         us_vhost_usage(prgname);
 698                                         return -1;
 699                                 } else {
 700                                         burst_rx_delay_time = ret;
 701                                 }
 702                         }
 703
 704                         /* Specify the retries number on RX. */
 705                         if (!strncmp(long_option[option_index].name, "rx-retry-num", MAX_LONG_OPT_SZ)) {
 706                                 ret = parse_num_opt(optarg, INT32_MAX);
 707                                 if (ret == -1) {
 708                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-num [0-N]\n");
 709                                         us_vhost_usage(prgname);
 710                                         return -1;
 711                                 } else {
 712                                         burst_rx_retry_num = ret;
 713                                 }
 714                         }
 715
 716                         /* Enable/disable RX mergeable buffers. */
 717                         if (!strncmp(long_option[option_index].name, "mergeable", MAX_LONG_OPT_SZ)) {
 718                                 ret = parse_num_opt(optarg, 1);
 719                                 if (ret == -1) {
 720                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for mergeable [0|1]\n");
 721                                         us_vhost_usage(prgname);
 722                                         return -1;
 723                                 } else {
 724                                         mergeable = !!ret;
 725                                         if (ret) {
 726                                                 vmdq_conf_default.rxmode.jumbo_frame = 1;
 727                                                 vmdq_conf_default.rxmode.max_rx_pkt_len
 728                                                         = JUMBO_FRAME_MAX_SIZE;
 729                                         }
 730                                 }
 731                         }
 732
 733                         /* Enable/disable RX VLAN strip on host. */
 734                         if (!strncmp(long_option[option_index].name,
 735                                 "vlan-strip", MAX_LONG_OPT_SZ)) {
 736                                 ret = parse_num_opt(optarg, 1);
 737                                 if (ret == -1) {
 738                                         RTE_LOG(INFO, VHOST_CONFIG,
 739                                                 "Invalid argument for VLAN strip [0|1]\n");
 740                                         us_vhost_usage(prgname);
 741                                         return -1;
 742                                 } else {
 743                                         vlan_strip = !!ret;
 744                                         vmdq_conf_default.rxmode.hw_vlan_strip =
 745                                                 vlan_strip;
 746                                 }
 747                         }
 748
 749                         /* Enable/disable stats. */
 750                         if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) {
 751                                 ret = parse_num_opt(optarg, INT32_MAX);
 752                                 if (ret == -1) {
 753                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for stats [0..N]\n");
 754                                         us_vhost_usage(prgname);
 755                                         return -1;
 756                                 } else {
 757                                         enable_stats = ret;
 758                                 }
 759                         }
 760
 761                         /* Set character device basename. */
 762                         if (!strncmp(long_option[option_index].name, "dev-basename", MAX_LONG_OPT_SZ)) {
 763                                 if (us_vhost_parse_basename(optarg) == -1) {
 764                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for character device basename (Max %d characters)\n", MAX_BASENAME_SZ);
 765                                         us_vhost_usage(prgname);
 766                                         return -1;
 767                                 }
 768                         }
 769
 770                         /* Enable/disable rx/tx zero copy. */
 771                         if (!strncmp(long_option[option_index].name,
 772                                 "zero-copy", MAX_LONG_OPT_SZ)) {
 773                                 ret = parse_num_opt(optarg, 1);
 774                                 if (ret == -1) {
 775                                         RTE_LOG(INFO, VHOST_CONFIG,
 776                                                 "Invalid argument"
 777                                                 " for zero-copy [0|1]\n");
 778                                         us_vhost_usage(prgname);
 779                                         return -1;
 780                                 } else
 781                                         zero_copy = ret;
 782                         }
 783
 784                         /* Specify the descriptor number on RX. */
 785                         if (!strncmp(long_option[option_index].name,
 786                                 "rx-desc-num", MAX_LONG_OPT_SZ)) {
 787                                 ret = parse_num_opt(optarg, MAX_RING_DESC);
 788                                 if ((ret == -1) || (!POWEROF2(ret))) {
 789                                         RTE_LOG(INFO, VHOST_CONFIG,
 790                                         "Invalid argument for rx-desc-num[0-N],"
 791                                         "power of 2 required.\n");
 792                                         us_vhost_usage(prgname);
 793                                         return -1;
 794                                 } else {
 795                                         num_rx_descriptor = ret;
 796                                 }
 797                         }
 798
 799                         /* Specify the descriptor number on TX. */
 800                         if (!strncmp(long_option[option_index].name,
 801                                 "tx-desc-num", MAX_LONG_OPT_SZ)) {
 802                                 ret = parse_num_opt(optarg, MAX_RING_DESC);
 803                                 if ((ret == -1) || (!POWEROF2(ret))) {
 804                                         RTE_LOG(INFO, VHOST_CONFIG,
 805                                         "Invalid argument for tx-desc-num [0-N],"
 806                                         "power of 2 required.\n");
 807                                         us_vhost_usage(prgname);
 808                                         return -1;
 809                                 } else {
 810                                         num_tx_descriptor = ret;
 811                                 }
 812                         }
 813
 814                         break;
 815
 816                         /* Invalid option - print options. */
 817                 default:
 818                         us_vhost_usage(prgname);
 819                         return -1;
 820                 }
 821         }
 822
 823         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 824                 if (enabled_port_mask & (1 << i))
 825                         ports[num_ports++] = (uint8_t)i;
 826         }
 827
 828         if ((num_ports ==  0) || (num_ports > MAX_SUP_PORTS)) {
 829                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
 830                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
 831                 return -1;
 832         }
 833
 834         if ((zero_copy == 1) && (vm2vm_mode == VM2VM_SOFTWARE)) {
 835                 RTE_LOG(INFO, VHOST_PORT,
 836                         "Vhost zero copy doesn't support software vm2vm,"
 837                         "please specify 'vm2vm 2' to use hardware vm2vm.\n");
 838                 return -1;
 839         }
 840
 841         if ((zero_copy == 1) && (vmdq_conf_default.rxmode.jumbo_frame == 1)) {
 842                 RTE_LOG(INFO, VHOST_PORT,
 843                         "Vhost zero copy doesn't support jumbo frame,"
 844                         "please specify '--mergeable 0' to disable the "
 845                         "mergeable feature.\n");
 846                 return -1;
 847         }
 848
 849         return 0;
 850 }
 851
 852 /*
 853  * Update the global var NUM_PORTS and array PORTS according to system ports number
 854  * and return valid ports number
 855  */
 856 static unsigned check_ports_num(unsigned nb_ports)
 857 {
 858         unsigned valid_num_ports = num_ports;
 859         unsigned portid;
 860
 861         if (num_ports > nb_ports) {
 862                 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n",
 863                         num_ports, nb_ports);
 864                 num_ports = nb_ports;
 865         }
 866
 867         for (portid = 0; portid < num_ports; portid ++) {
 868                 if (ports[portid] >= nb_ports) {
 869                         RTE_LOG(INFO, VHOST_PORT, "\nSpecified port ID(%u) exceeds max system port ID(%u)\n",
 870                                 ports[portid], (nb_ports - 1));
 871                         ports[portid] = INVALID_PORT_ID;
 872                         valid_num_ports--;
 873                 }
 874         }
 875         return valid_num_ports;
 876 }
 877
 878 /*
 879  * Macro to print out packet contents. Wrapped in debug define so that the
 880  * data path is not effected when debug is disabled.
 881  */
 882 #ifdef DEBUG
 883 #define PRINT_PACKET(device, addr, size, header) do {                                                                                                                           \
 884         char *pkt_addr = (char*)(addr);                                                                                                                                                                 \
 885         unsigned int index;                                                                                                                                                                                             \
 886         char packet[MAX_PRINT_BUFF];                                                                                                                                                                    \
 887                                                                                                                                                                                                                                         \
 888         if ((header))                                                                                                                                                                                                   \
 889                 snprintf(packet, MAX_PRINT_BUFF, "(%"PRIu64") Header size %d: ", (device->device_fh), (size));                          \
 890         else                                                                                                                                                                                                                    \
 891                 snprintf(packet, MAX_PRINT_BUFF, "(%"PRIu64") Packet size %d: ", (device->device_fh), (size));                          \
 892         for (index = 0; index < (size); index++) {                                                                                                                                              \
 893                 snprintf(packet + strnlen(packet, MAX_PRINT_BUFF), MAX_PRINT_BUFF - strnlen(packet, MAX_PRINT_BUFF),    \
 894                         "%02hhx ", pkt_addr[index]);                                                                                                                                                    \
 895         }                                                                                                                                                                                                                               \
 896         snprintf(packet + strnlen(packet, MAX_PRINT_BUFF), MAX_PRINT_BUFF - strnlen(packet, MAX_PRINT_BUFF), "\n");     \
 897                                                                                                                                                                                                                                         \
 898         LOG_DEBUG(VHOST_DATA, "%s", packet);                                                                                                                                                                    \
 899 } while(0)
 900 #else
 901 #define PRINT_PACKET(device, addr, size, header) do{} while(0)
 902 #endif
 903
 904 /*
 905  * Function to convert guest physical addresses to vhost physical addresses.
 906  * This is used to convert virtio buffer addresses.
 907  */
 908 static inline uint64_t __attribute__((always_inline))
 909 gpa_to_hpa(struct vhost_dev  *vdev, uint64_t guest_pa,
 910         uint32_t buf_len, hpa_type *addr_type)
 911 {
 912         struct virtio_memory_regions_hpa *region;
 913         uint32_t regionidx;
 914         uint64_t vhost_pa = 0;
 915
 916         *addr_type = PHYS_ADDR_INVALID;
 917
 918         for (regionidx = 0; regionidx < vdev->nregions_hpa; regionidx++) {
 919                 region = &vdev->regions_hpa[regionidx];
 920                 if ((guest_pa >= region->guest_phys_address) &&
 921                         (guest_pa <= region->guest_phys_address_end)) {
 922                         vhost_pa = region->host_phys_addr_offset + guest_pa;
 923                         if (likely((guest_pa + buf_len - 1)
 924                                 <= region->guest_phys_address_end))
 925                                 *addr_type = PHYS_ADDR_CONTINUOUS;
 926                         else
 927                                 *addr_type = PHYS_ADDR_CROSS_SUBREG;
 928                         break;
 929                 }
 930         }
 931
 932         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") GPA %p| HPA %p\n",
 933                 vdev->dev->device_fh, (void *)(uintptr_t)guest_pa,
 934                 (void *)(uintptr_t)vhost_pa);
 935
 936         return vhost_pa;
 937 }
 938
 939 /*
 940  * Compares a packet destination MAC address to a device MAC address.
 941  */
 942 static inline int __attribute__((always_inline))
 943 ether_addr_cmp(struct ether_addr *ea, struct ether_addr *eb)
 944 {
 945         return ((*(uint64_t *)ea ^ *(uint64_t *)eb) & MAC_ADDR_CMP) == 0;
 946 }
 947
 948 /*
 949  * This function learns the MAC address of the device and registers this along with a
 950  * vlan tag to a VMDQ.
 951  */
 952 static int
 953 link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
 954 {
 955         struct ether_hdr *pkt_hdr;
 956         struct virtio_net_data_ll *dev_ll;
 957         struct virtio_net *dev = vdev->dev;
 958         int i, ret;
 959
 960         /* Learn MAC address of guest device from packet */
 961         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 962
 963         dev_ll = ll_root_used;
 964
 965         while (dev_ll != NULL) {
 966                 if (ether_addr_cmp(&(pkt_hdr->s_addr), &dev_ll->vdev->mac_address)) {
 967                         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") WARNING: This device is using an existing MAC address and has not been registered.\n", dev->device_fh);
 968                         return -1;
 969                 }
 970                 dev_ll = dev_ll->next;
 971         }
 972
 973         for (i = 0; i < ETHER_ADDR_LEN; i++)
 974                 vdev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i];
 975
 976         /* vlan_tag currently uses the device_id. */
 977         vdev->vlan_tag = vlan_tags[dev->device_fh];
 978
 979         /* Print out VMDQ registration info. */
 980         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") MAC_ADDRESS %02x:%02x:%02x:%02x:%02x:%02x and VLAN_TAG %d registered\n",
 981                 dev->device_fh,
 982                 vdev->mac_address.addr_bytes[0], vdev->mac_address.addr_bytes[1],
 983                 vdev->mac_address.addr_bytes[2], vdev->mac_address.addr_bytes[3],
 984                 vdev->mac_address.addr_bytes[4], vdev->mac_address.addr_bytes[5],
 985                 vdev->vlan_tag);
 986
 987         /* Register the MAC address. */
 988         ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address,
 989                                 (uint32_t)dev->device_fh + vmdq_pool_base);
 990         if (ret)
 991                 RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Failed to add device MAC address to VMDQ\n",
 992                                         dev->device_fh);
 993
 994         /* Enable stripping of the vlan tag as we handle routing. */
 995         if (vlan_strip)
 996                 rte_eth_dev_set_vlan_strip_on_queue(ports[0],
 997                         (uint16_t)vdev->vmdq_rx_q, 1);
 998
 999         /* Set device as ready for RX. */
1000         vdev->ready = DEVICE_RX;
1001
1002         return 0;
1003 }
1004
1005 /*
1006  * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX
1007  * queue before disabling RX on the device.
1008  */
1009 static inline void
1010 unlink_vmdq(struct vhost_dev *vdev)
1011 {
1012         unsigned i = 0;
1013         unsigned rx_count;
1014         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1015
1016         if (vdev->ready == DEVICE_RX) {
1017                 /*clear MAC and VLAN settings*/
1018                 rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address);
1019                 for (i = 0; i < 6; i++)
1020                         vdev->mac_address.addr_bytes[i] = 0;
1021
1022                 vdev->vlan_tag = 0;
1023
1024                 /*Clear out the receive buffers*/
1025                 rx_count = rte_eth_rx_burst(ports[0],
1026                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
1027
1028                 while (rx_count) {
1029                         for (i = 0; i < rx_count; i++)
1030                                 rte_pktmbuf_free(pkts_burst[i]);
1031
1032                         rx_count = rte_eth_rx_burst(ports[0],
1033                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
1034                 }
1035
1036                 vdev->ready = DEVICE_MAC_LEARNING;
1037         }
1038 }
1039
1040 /*
1041  * Check if the packet destination MAC address is for a local device. If so then put
1042  * the packet on that devices RX queue. If not then return.
1043  */
1044 static inline int __attribute__((always_inline))
1045 virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
1046 {
1047         struct virtio_net_data_ll *dev_ll;
1048         struct ether_hdr *pkt_hdr;
1049         uint64_t ret = 0;
1050         struct virtio_net *dev = vdev->dev;
1051         struct virtio_net *tdev; /* destination virito device */
1052
1053         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1054
1055         /*get the used devices list*/
1056         dev_ll = ll_root_used;
1057
1058         while (dev_ll != NULL) {
1059                 if ((dev_ll->vdev->ready == DEVICE_RX) && ether_addr_cmp(&(pkt_hdr->d_addr),
1060                                           &dev_ll->vdev->mac_address)) {
1061
1062                         /* Drop the packet if the TX packet is destined for the TX device. */
1063                         if (dev_ll->vdev->dev->device_fh == dev->device_fh) {
1064                                 LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: Source and destination MAC addresses are the same. Dropping packet.\n",
1065                                                         dev->device_fh);
1066                                 return 0;
1067                         }
1068                         tdev = dev_ll->vdev->dev;
1069
1070
1071                         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: MAC address is local\n", tdev->device_fh);
1072
1073                         if (unlikely(dev_ll->vdev->remove)) {
1074                                 /*drop the packet if the device is marked for removal*/
1075                                 LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Device is marked for removal\n", tdev->device_fh);
1076                         } else {
1077                                 /*send the packet to the local virtio device*/
1078                                 ret = rte_vhost_enqueue_burst(tdev, VIRTIO_RXQ, &m, 1);
1079                                 if (enable_stats) {
1080                                         rte_atomic64_add(
1081                                         &dev_statistics[tdev->device_fh].rx_total_atomic,
1082                                         1);
1083                                         rte_atomic64_add(
1084                                         &dev_statistics[tdev->device_fh].rx_atomic,
1085                                         ret);
1086                                         dev_statistics[dev->device_fh].tx_total++;
1087                                         dev_statistics[dev->device_fh].tx += ret;
1088                                 }
1089                         }
1090
1091                         return 0;
1092                 }
1093                 dev_ll = dev_ll->next;
1094         }
1095
1096         return -1;
1097 }
1098
1099 /*
1100  * Check if the destination MAC of a packet is one local VM,
1101  * and get its vlan tag, and offset if it is.
1102  */
1103 static inline int __attribute__((always_inline))
1104 find_local_dest(struct virtio_net *dev, struct rte_mbuf *m,
1105         uint32_t *offset, uint16_t *vlan_tag)
1106 {
1107         struct virtio_net_data_ll *dev_ll = ll_root_used;
1108         struct ether_hdr *pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1109
1110         while (dev_ll != NULL) {
1111                 if ((dev_ll->vdev->ready == DEVICE_RX)
1112                         && ether_addr_cmp(&(pkt_hdr->d_addr),
1113                 &dev_ll->vdev->mac_address)) {
1114                         /*
1115                          * Drop the packet if the TX packet is
1116                          * destined for the TX device.
1117                          */
1118                         if (dev_ll->vdev->dev->device_fh == dev->device_fh) {
1119                                 LOG_DEBUG(VHOST_DATA,
1120                                 "(%"PRIu64") TX: Source and destination"
1121                                 " MAC addresses are the same. Dropping "
1122                                 "packet.\n",
1123                                 dev_ll->vdev->dev->device_fh);
1124                                 return -1;
1125                         }
1126
1127                         /*
1128                          * HW vlan strip will reduce the packet length
1129                          * by minus length of vlan tag, so need restore
1130                          * the packet length by plus it.
1131                          */
1132                         *offset = VLAN_HLEN;
1133                         *vlan_tag =
1134                         (uint16_t)
1135                         vlan_tags[(uint16_t)dev_ll->vdev->dev->device_fh];
1136
1137                         LOG_DEBUG(VHOST_DATA,
1138                         "(%"PRIu64") TX: pkt to local VM device id:"
1139                         "(%"PRIu64") vlan tag: %d.\n",
1140                         dev->device_fh, dev_ll->vdev->dev->device_fh,
1141                         (int)*vlan_tag);
1142
1143                         break;
1144                 }
1145                 dev_ll = dev_ll->next;
1146         }
1147         return 0;
1148 }
1149
1150 static uint16_t
1151 get_psd_sum(void *l3_hdr, uint64_t ol_flags)
1152 {
1153         if (ol_flags & PKT_TX_IPV4)
1154                 return rte_ipv4_phdr_cksum(l3_hdr, ol_flags);
1155         else /* assume ethertype == ETHER_TYPE_IPv6 */
1156                 return rte_ipv6_phdr_cksum(l3_hdr, ol_flags);
1157 }
1158
1159 static void virtio_tx_offload(struct rte_mbuf *m)
1160 {
1161         void *l3_hdr;
1162         struct ipv4_hdr *ipv4_hdr = NULL;
1163         struct tcp_hdr *tcp_hdr = NULL;
1164         struct udp_hdr *udp_hdr = NULL;
1165         struct sctp_hdr *sctp_hdr = NULL;
1166         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1167
1168         l3_hdr = (char *)eth_hdr + m->l2_len;
1169
1170         if (m->tso_segsz != 0) {
1171                 ipv4_hdr = (struct ipv4_hdr *)l3_hdr;
1172                 tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + m->l3_len);
1173                 m->ol_flags |= PKT_TX_IP_CKSUM;
1174                 ipv4_hdr->hdr_checksum = 0;
1175                 tcp_hdr->cksum = get_psd_sum(l3_hdr, m->ol_flags);
1176                 return;
1177         }
1178
1179         if (m->ol_flags & PKT_TX_L4_MASK) {
1180                 switch (m->ol_flags & PKT_TX_L4_MASK) {
1181                 case PKT_TX_TCP_CKSUM:
1182                         tcp_hdr = (struct tcp_hdr *)
1183                                         ((char *)l3_hdr + m->l3_len);
1184                         tcp_hdr->cksum = get_psd_sum(l3_hdr, m->ol_flags);
1185                         break;
1186                 case PKT_TX_UDP_CKSUM:
1187                         udp_hdr = (struct udp_hdr *)
1188                                         ((char *)l3_hdr + m->l3_len);
1189                         udp_hdr->dgram_cksum = get_psd_sum(l3_hdr, m->ol_flags);
1190                         break;
1191                 case PKT_TX_SCTP_CKSUM:
1192                         sctp_hdr = (struct sctp_hdr *)
1193                                         ((char *)l3_hdr + m->l3_len);
1194                         sctp_hdr->cksum = 0;
1195                         break;
1196                 default:
1197                         break;
1198                 }
1199         }
1200 }
1201
1202 /*
1203  * This function routes the TX packet to the correct interface. This may be a local device
1204  * or the physical port.
1205  */
1206 static inline void __attribute__((always_inline))
1207 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)
1208 {
1209         struct mbuf_table *tx_q;
1210         struct rte_mbuf **m_table;
1211         unsigned len, ret, offset = 0;
1212         const uint16_t lcore_id = rte_lcore_id();
1213         struct virtio_net *dev = vdev->dev;
1214         struct ether_hdr *nh;
1215
1216         /*check if destination is local VM*/
1217         if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) {
1218                 rte_pktmbuf_free(m);
1219                 return;
1220         }
1221
1222         if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
1223                 if (unlikely(find_local_dest(dev, m, &offset, &vlan_tag) != 0)) {
1224                         rte_pktmbuf_free(m);
1225                         return;
1226                 }
1227         }
1228
1229         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: MAC address is external\n", dev->device_fh);
1230
1231         /*Add packet to the port tx queue*/
1232         tx_q = &lcore_tx_queue[lcore_id];
1233         len = tx_q->len;
1234
1235         nh = rte_pktmbuf_mtod(m, struct ether_hdr *);
1236         if (unlikely(nh->ether_type == rte_cpu_to_be_16(ETHER_TYPE_VLAN))) {
1237                 /* Guest has inserted the vlan tag. */
1238                 struct vlan_hdr *vh = (struct vlan_hdr *) (nh + 1);
1239                 uint16_t vlan_tag_be = rte_cpu_to_be_16(vlan_tag);
1240                 if ((vm2vm_mode == VM2VM_HARDWARE) &&
1241                         (vh->vlan_tci != vlan_tag_be))
1242                         vh->vlan_tci = vlan_tag_be;
1243         } else {
1244                 m->ol_flags |= PKT_TX_VLAN_PKT;
1245
1246                 /*
1247                  * Find the right seg to adjust the data len when offset is
1248                  * bigger than tail room size.
1249                  */
1250                 if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
1251                         if (likely(offset <= rte_pktmbuf_tailroom(m)))
1252                                 m->data_len += offset;
1253                         else {
1254                                 struct rte_mbuf *seg = m;
1255
1256                                 while ((seg->next != NULL) &&
1257                                         (offset > rte_pktmbuf_tailroom(seg)))
1258                                         seg = seg->next;
1259
1260                                 seg->data_len += offset;
1261                         }
1262                         m->pkt_len += offset;
1263                 }
1264
1265                 m->vlan_tci = vlan_tag;
1266         }
1267
1268         if ((m->ol_flags & PKT_TX_L4_MASK) || (m->ol_flags & PKT_TX_TCP_SEG))
1269                 virtio_tx_offload(m);
1270
1271         tx_q->m_table[len] = m;
1272         len++;
1273         if (enable_stats) {
1274                 dev_statistics[dev->device_fh].tx_total++;
1275                 dev_statistics[dev->device_fh].tx++;
1276         }
1277
1278         if (unlikely(len == MAX_PKT_BURST)) {
1279                 m_table = (struct rte_mbuf **)tx_q->m_table;
1280                 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id, m_table, (uint16_t) len);
1281                 /* Free any buffers not handled by TX and update the port stats. */
1282                 if (unlikely(ret < len)) {
1283                         do {
1284                                 rte_pktmbuf_free(m_table[ret]);
1285                         } while (++ret < len);
1286                 }
1287
1288                 len = 0;
1289         }
1290
1291         tx_q->len = len;
1292         return;
1293 }
1294 /*
1295  * This function is called by each data core. It handles all RX/TX registered with the
1296  * core. For TX the specific lcore linked list is used. For RX, MAC addresses are compared
1297  * with all devices in the main linked list.
1298  */
1299 static int
1300 switch_worker(__attribute__((unused)) void *arg)
1301 {
1302         struct rte_mempool *mbuf_pool = arg;
1303         struct virtio_net *dev = NULL;
1304         struct vhost_dev *vdev = NULL;
1305         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1306         struct virtio_net_data_ll *dev_ll;
1307         struct mbuf_table *tx_q;
1308         volatile struct lcore_ll_info *lcore_ll;
1309         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US;
1310         uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0;
1311         unsigned ret, i;
1312         const uint16_t lcore_id = rte_lcore_id();
1313         const uint16_t num_cores = (uint16_t)rte_lcore_count();
1314         uint16_t rx_count = 0;
1315         uint16_t tx_count;
1316         uint32_t retry = 0;
1317
1318         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
1319         lcore_ll = lcore_info[lcore_id].lcore_ll;
1320         prev_tsc = 0;
1321
1322         tx_q = &lcore_tx_queue[lcore_id];
1323         for (i = 0; i < num_cores; i ++) {
1324                 if (lcore_ids[i] == lcore_id) {
1325                         tx_q->txq_id = i;
1326                         break;
1327                 }
1328         }
1329
1330         while(1) {
1331                 cur_tsc = rte_rdtsc();
1332                 /*
1333                  * TX burst queue drain
1334                  */
1335                 diff_tsc = cur_tsc - prev_tsc;
1336                 if (unlikely(diff_tsc > drain_tsc)) {
1337
1338                         if (tx_q->len) {
1339                                 LOG_DEBUG(VHOST_DATA, "TX queue drained after timeout with burst size %u \n", tx_q->len);
1340
1341                                 /*Tx any packets in the queue*/
1342                                 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id,
1343                                                                            (struct rte_mbuf **)tx_q->m_table,
1344                                                                            (uint16_t)tx_q->len);
1345                                 if (unlikely(ret < tx_q->len)) {
1346                                         do {
1347                                                 rte_pktmbuf_free(tx_q->m_table[ret]);
1348                                         } while (++ret < tx_q->len);
1349                                 }
1350
1351                                 tx_q->len = 0;
1352                         }
1353
1354                         prev_tsc = cur_tsc;
1355
1356                 }
1357
1358                 rte_prefetch0(lcore_ll->ll_root_used);
1359                 /*
1360                  * Inform the configuration core that we have exited the linked list and that no devices are
1361                  * in use if requested.
1362                  */
1363                 if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL)
1364                         lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
1365
1366                 /*
1367                  * Process devices
1368                  */
1369                 dev_ll = lcore_ll->ll_root_used;
1370
1371                 while (dev_ll != NULL) {
1372                         /*get virtio device ID*/
1373                         vdev = dev_ll->vdev;
1374                         dev = vdev->dev;
1375
1376                         if (unlikely(vdev->remove)) {
1377                                 dev_ll = dev_ll->next;
1378                                 unlink_vmdq(vdev);
1379                                 vdev->ready = DEVICE_SAFE_REMOVE;
1380                                 continue;
1381                         }
1382                         if (likely(vdev->ready == DEVICE_RX)) {
1383                                 /*Handle guest RX*/
1384                                 rx_count = rte_eth_rx_burst(ports[0],
1385                                         vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
1386
1387                                 if (rx_count) {
1388                                         /*
1389                                         * Retry is enabled and the queue is full then we wait and retry to avoid packet loss
1390                                         * Here MAX_PKT_BURST must be less than virtio queue size
1391                                         */
1392                                         if (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev, VIRTIO_RXQ))) {
1393                                                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
1394                                                         rte_delay_us(burst_rx_delay_time);
1395                                                         if (rx_count <= rte_vring_available_entries(dev, VIRTIO_RXQ))
1396                                                                 break;
1397                                                 }
1398                                         }
1399                                         ret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ, pkts_burst, rx_count);
1400                                         if (enable_stats) {
1401                                                 rte_atomic64_add(
1402                                                 &dev_statistics[dev_ll->vdev->dev->device_fh].rx_total_atomic,
1403                                                 rx_count);
1404                                                 rte_atomic64_add(
1405                                                 &dev_statistics[dev_ll->vdev->dev->device_fh].rx_atomic, ret_count);
1406                                         }
1407                                         while (likely(rx_count)) {
1408                                                 rx_count--;
1409                                                 rte_pktmbuf_free(pkts_burst[rx_count]);
1410                                         }
1411
1412                                 }
1413                         }
1414
1415                         if (likely(!vdev->remove)) {
1416                                 /* Handle guest TX*/
1417                                 tx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ, mbuf_pool, pkts_burst, MAX_PKT_BURST);
1418                                 /* If this is the first received packet we need to learn the MAC and setup VMDQ */
1419                                 if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) {
1420                                         if (vdev->remove || (link_vmdq(vdev, pkts_burst[0]) == -1)) {
1421                                                 while (tx_count)
1422                                                         rte_pktmbuf_free(pkts_burst[--tx_count]);
1423                                         }
1424                                 }
1425                                 while (tx_count)
1426                                         virtio_tx_route(vdev, pkts_burst[--tx_count], (uint16_t)dev->device_fh);
1427                         }
1428
1429                         /*move to the next device in the list*/
1430                         dev_ll = dev_ll->next;
1431                 }
1432         }
1433
1434         return 0;
1435 }
1436
1437 /*
1438  * This function gets available ring number for zero copy rx.
1439  * Only one thread will call this funciton for a paticular virtio device,
1440  * so, it is designed as non-thread-safe function.
1441  */
1442 static inline uint32_t __attribute__((always_inline))
1443 get_available_ring_num_zcp(struct virtio_net *dev)
1444 {
1445         struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_RXQ];
1446         uint16_t avail_idx;
1447
1448         avail_idx = *((volatile uint16_t *)&vq->avail->idx);
1449         return (uint32_t)(avail_idx - vq->last_used_idx_res);
1450 }
1451
1452 /*
1453  * This function gets available ring index for zero copy rx,
1454  * it will retry 'burst_rx_retry_num' times till it get enough ring index.
1455  * Only one thread will call this funciton for a paticular virtio device,
1456  * so, it is designed as non-thread-safe function.
1457  */
1458 static inline uint32_t __attribute__((always_inline))
1459 get_available_ring_index_zcp(struct virtio_net *dev,
1460         uint16_t *res_base_idx, uint32_t count)
1461 {
1462         struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_RXQ];
1463         uint16_t avail_idx;
1464         uint32_t retry = 0;
1465         uint16_t free_entries;
1466
1467         *res_base_idx = vq->last_used_idx_res;
1468         avail_idx = *((volatile uint16_t *)&vq->avail->idx);
1469         free_entries = (avail_idx - *res_base_idx);
1470
1471         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") in get_available_ring_index_zcp: "
1472                         "avail idx: %d, "
1473                         "res base idx:%d, free entries:%d\n",
1474                         dev->device_fh, avail_idx, *res_base_idx,
1475                         free_entries);
1476
1477         /*
1478          * If retry is enabled and the queue is full then we wait
1479          * and retry to avoid packet loss.
1480          */
1481         if (enable_retry && unlikely(count > free_entries)) {
1482                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
1483                         rte_delay_us(burst_rx_delay_time);
1484                         avail_idx = *((volatile uint16_t *)&vq->avail->idx);
1485                         free_entries = (avail_idx - *res_base_idx);
1486                         if (count <= free_entries)
1487                                 break;
1488                 }
1489         }
1490
1491         /*check that we have enough buffers*/
1492         if (unlikely(count > free_entries))
1493                 count = free_entries;
1494
1495         if (unlikely(count == 0)) {
1496                 LOG_DEBUG(VHOST_DATA,
1497                         "(%"PRIu64") Fail in get_available_ring_index_zcp: "
1498                         "avail idx: %d, res base idx:%d, free entries:%d\n",
1499                         dev->device_fh, avail_idx,
1500                         *res_base_idx, free_entries);
1501                 return 0;
1502         }
1503
1504         vq->last_used_idx_res = *res_base_idx + count;
1505
1506         return count;
1507 }
1508
1509 /*
1510  * This function put descriptor back to used list.
1511  */
1512 static inline void __attribute__((always_inline))
1513 put_desc_to_used_list_zcp(struct vhost_virtqueue *vq, uint16_t desc_idx)
1514 {
1515         uint16_t res_cur_idx = vq->last_used_idx;
1516         vq->used->ring[res_cur_idx & (vq->size - 1)].id = (uint32_t)desc_idx;
1517         vq->used->ring[res_cur_idx & (vq->size - 1)].len = 0;
1518         rte_compiler_barrier();
1519         *(volatile uint16_t *)&vq->used->idx += 1;
1520         vq->last_used_idx += 1;
1521
1522         /* Kick the guest if necessary. */
1523         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
1524                 eventfd_write(vq->callfd, (eventfd_t)1);
1525 }
1526
1527 /*
1528  * This function get available descriptor from vitio vring and un-attached mbuf
1529  * from vpool->ring, and then attach them together. It needs adjust the offset
1530  * for buff_addr and phys_addr accroding to PMD implementation, otherwise the
1531  * frame data may be put to wrong location in mbuf.
1532  */
1533 static inline void __attribute__((always_inline))
1534 attach_rxmbuf_zcp(struct virtio_net *dev)
1535 {
1536         uint16_t res_base_idx, desc_idx;
1537         uint64_t buff_addr, phys_addr;
1538         struct vhost_virtqueue *vq;
1539         struct vring_desc *desc;
1540         void *obj = NULL;
1541         struct rte_mbuf *mbuf;
1542         struct vpool *vpool;
1543         hpa_type addr_type;
1544         struct vhost_dev *vdev = (struct vhost_dev *)dev->priv;
1545
1546         vpool = &vpool_array[vdev->vmdq_rx_q];
1547         vq = dev->virtqueue[VIRTIO_RXQ];
1548
1549         do {
1550                 if (unlikely(get_available_ring_index_zcp(vdev->dev, &res_base_idx,
1551                                 1) != 1))
1552                         return;
1553                 desc_idx = vq->avail->ring[(res_base_idx) & (vq->size - 1)];
1554
1555                 desc = &vq->desc[desc_idx];
1556                 if (desc->flags & VRING_DESC_F_NEXT) {
1557                         desc = &vq->desc[desc->next];
1558                         buff_addr = gpa_to_vva(dev, desc->addr);
1559                         phys_addr = gpa_to_hpa(vdev, desc->addr, desc->len,
1560                                         &addr_type);
1561                 } else {
1562                         buff_addr = gpa_to_vva(dev,
1563                                         desc->addr + vq->vhost_hlen);
1564                         phys_addr = gpa_to_hpa(vdev,
1565                                         desc->addr + vq->vhost_hlen,
1566                                         desc->len, &addr_type);
1567                 }
1568
1569                 if (unlikely(addr_type == PHYS_ADDR_INVALID)) {
1570                         RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Invalid frame buffer"
1571                                 " address found when attaching RX frame buffer"
1572                                 " address!\n", dev->device_fh);
1573                         put_desc_to_used_list_zcp(vq, desc_idx);
1574                         continue;
1575                 }
1576
1577                 /*
1578                  * Check if the frame buffer address from guest crosses
1579                  * sub-region or not.
1580                  */
1581                 if (unlikely(addr_type == PHYS_ADDR_CROSS_SUBREG)) {
1582                         RTE_LOG(ERR, VHOST_DATA,
1583                                 "(%"PRIu64") Frame buffer address cross "
1584                                 "sub-regioin found when attaching RX frame "
1585                                 "buffer address!\n",
1586                                 dev->device_fh);
1587                         put_desc_to_used_list_zcp(vq, desc_idx);
1588                         continue;
1589                 }
1590         } while (unlikely(phys_addr == 0));
1591
1592         rte_ring_sc_dequeue(vpool->ring, &obj);
1593         mbuf = obj;
1594         if (unlikely(mbuf == NULL)) {
1595                 LOG_DEBUG(VHOST_DATA,
1596                         "(%"PRIu64") in attach_rxmbuf_zcp: "
1597                         "ring_sc_dequeue fail.\n",
1598                         dev->device_fh);
1599                 put_desc_to_used_list_zcp(vq, desc_idx);
1600                 return;
1601         }
1602
1603         if (unlikely(vpool->buf_size > desc->len)) {
1604                 LOG_DEBUG(VHOST_DATA,
1605                         "(%"PRIu64") in attach_rxmbuf_zcp: frame buffer "
1606                         "length(%d) of descriptor idx: %d less than room "
1607                         "size required: %d\n",
1608                         dev->device_fh, desc->len, desc_idx, vpool->buf_size);
1609                 put_desc_to_used_list_zcp(vq, desc_idx);
1610                 rte_ring_sp_enqueue(vpool->ring, obj);
1611                 return;
1612         }
1613
1614         mbuf->buf_addr = (void *)(uintptr_t)(buff_addr - RTE_PKTMBUF_HEADROOM);
1615         mbuf->data_off = RTE_PKTMBUF_HEADROOM;
1616         mbuf->buf_physaddr = phys_addr - RTE_PKTMBUF_HEADROOM;
1617         mbuf->data_len = desc->len;
1618         MBUF_HEADROOM_UINT32(mbuf) = (uint32_t)desc_idx;
1619
1620         LOG_DEBUG(VHOST_DATA,
1621                 "(%"PRIu64") in attach_rxmbuf_zcp: res base idx:%d, "
1622                 "descriptor idx:%d\n",
1623                 dev->device_fh, res_base_idx, desc_idx);
1624
1625         __rte_mbuf_raw_free(mbuf);
1626
1627         return;
1628 }
1629
1630 /*
1631  * Detach an attched packet mbuf -
1632  *  - restore original mbuf address and length values.
1633  *  - reset pktmbuf data and data_len to their default values.
1634  *  All other fields of the given packet mbuf will be left intact.
1635  *
1636  * @param m
1637  *   The attached packet mbuf.
1638  */
1639 static inline void pktmbuf_detach_zcp(struct rte_mbuf *m)
1640 {
1641         const struct rte_mempool *mp = m->pool;
1642         void *buf = rte_mbuf_to_baddr(m);
1643         uint32_t buf_ofs;
1644         uint32_t buf_len = mp->elt_size - sizeof(*m);
1645         m->buf_physaddr = rte_mempool_virt2phy(mp, m) + sizeof(*m);
1646
1647         m->buf_addr = buf;
1648         m->buf_len = (uint16_t)buf_len;
1649
1650         buf_ofs = (RTE_PKTMBUF_HEADROOM <= m->buf_len) ?
1651                         RTE_PKTMBUF_HEADROOM : m->buf_len;
1652         m->data_off = buf_ofs;
1653
1654         m->data_len = 0;
1655 }
1656
1657 /*
1658  * This function is called after packets have been transimited. It fetchs mbuf
1659  * from vpool->pool, detached it and put into vpool->ring. It also update the
1660  * used index and kick the guest if necessary.
1661  */
1662 static inline uint32_t __attribute__((always_inline))
1663 txmbuf_clean_zcp(struct virtio_net *dev, struct vpool *vpool)
1664 {
1665         struct rte_mbuf *mbuf;
1666         struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_TXQ];
1667         uint32_t used_idx = vq->last_used_idx & (vq->size - 1);
1668         uint32_t index = 0;
1669         uint32_t mbuf_count = rte_mempool_count(vpool->pool);
1670
1671         LOG_DEBUG(VHOST_DATA,
1672                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in mempool before "
1673                 "clean is: %d\n",
1674                 dev->device_fh, mbuf_count);
1675         LOG_DEBUG(VHOST_DATA,
1676                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in  ring before "
1677                 "clean  is : %d\n",
1678                 dev->device_fh, rte_ring_count(vpool->ring));
1679
1680         for (index = 0; index < mbuf_count; index++) {
1681                 mbuf = __rte_mbuf_raw_alloc(vpool->pool);
1682                 if (likely(MBUF_EXT_MEM(mbuf)))
1683                         pktmbuf_detach_zcp(mbuf);
1684                 rte_ring_sp_enqueue(vpool->ring, mbuf);
1685
1686                 /* Update used index buffer information. */
1687                 vq->used->ring[used_idx].id = MBUF_HEADROOM_UINT32(mbuf);
1688                 vq->used->ring[used_idx].len = 0;
1689
1690                 used_idx = (used_idx + 1) & (vq->size - 1);
1691         }
1692
1693         LOG_DEBUG(VHOST_DATA,
1694                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in mempool after "
1695                 "clean is: %d\n",
1696                 dev->device_fh, rte_mempool_count(vpool->pool));
1697         LOG_DEBUG(VHOST_DATA,
1698                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in  ring after "
1699                 "clean  is : %d\n",
1700                 dev->device_fh, rte_ring_count(vpool->ring));
1701         LOG_DEBUG(VHOST_DATA,
1702                 "(%"PRIu64") in txmbuf_clean_zcp: before updated "
1703                 "vq->last_used_idx:%d\n",
1704                 dev->device_fh, vq->last_used_idx);
1705
1706         vq->last_used_idx += mbuf_count;
1707
1708         LOG_DEBUG(VHOST_DATA,
1709                 "(%"PRIu64") in txmbuf_clean_zcp: after updated "
1710                 "vq->last_used_idx:%d\n",
1711                 dev->device_fh, vq->last_used_idx);
1712
1713         rte_compiler_barrier();
1714
1715         *(volatile uint16_t *)&vq->used->idx += mbuf_count;
1716
1717         /* Kick guest if required. */
1718         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
1719                 eventfd_write(vq->callfd, (eventfd_t)1);
1720
1721         return 0;
1722 }
1723
1724 /*
1725  * This function is called when a virtio device is destroy.
1726  * It fetchs mbuf from vpool->pool, and detached it, and put into vpool->ring.
1727  */
1728 static void mbuf_destroy_zcp(struct vpool *vpool)
1729 {
1730         struct rte_mbuf *mbuf = NULL;
1731         uint32_t index, mbuf_count = rte_mempool_count(vpool->pool);
1732
1733         LOG_DEBUG(VHOST_CONFIG,
1734                 "in mbuf_destroy_zcp: mbuf count in mempool before "
1735                 "mbuf_destroy_zcp is: %d\n",
1736                 mbuf_count);
1737         LOG_DEBUG(VHOST_CONFIG,
1738                 "in mbuf_destroy_zcp: mbuf count in  ring before "
1739                 "mbuf_destroy_zcp  is : %d\n",
1740                 rte_ring_count(vpool->ring));
1741
1742         for (index = 0; index < mbuf_count; index++) {
1743                 mbuf = __rte_mbuf_raw_alloc(vpool->pool);
1744                 if (likely(mbuf != NULL)) {
1745                         if (likely(MBUF_EXT_MEM(mbuf)))
1746                                 pktmbuf_detach_zcp(mbuf);
1747                         rte_ring_sp_enqueue(vpool->ring, (void *)mbuf);
1748                 }
1749         }
1750
1751         LOG_DEBUG(VHOST_CONFIG,
1752                 "in mbuf_destroy_zcp: mbuf count in mempool after "
1753                 "mbuf_destroy_zcp is: %d\n",
1754                 rte_mempool_count(vpool->pool));
1755         LOG_DEBUG(VHOST_CONFIG,
1756                 "in mbuf_destroy_zcp: mbuf count in ring after "
1757                 "mbuf_destroy_zcp is : %d\n",
1758                 rte_ring_count(vpool->ring));
1759 }
1760
1761 /*
1762  * This function update the use flag and counter.
1763  */
1764 static inline uint32_t __attribute__((always_inline))
1765 virtio_dev_rx_zcp(struct virtio_net *dev, struct rte_mbuf **pkts,
1766         uint32_t count)
1767 {
1768         struct vhost_virtqueue *vq;
1769         struct vring_desc *desc;
1770         struct rte_mbuf *buff;
1771         /* The virtio_hdr is initialised to 0. */
1772         struct virtio_net_hdr_mrg_rxbuf virtio_hdr
1773                 = {{0, 0, 0, 0, 0, 0}, 0};
1774         uint64_t buff_hdr_addr = 0;
1775         uint32_t head[MAX_PKT_BURST], packet_len = 0;
1776         uint32_t head_idx, packet_success = 0;
1777         uint16_t res_cur_idx;
1778
1779         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh);
1780
1781         if (count == 0)
1782                 return 0;
1783
1784         vq = dev->virtqueue[VIRTIO_RXQ];
1785         count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
1786
1787         res_cur_idx = vq->last_used_idx;
1788         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n",
1789                 dev->device_fh, res_cur_idx, res_cur_idx + count);
1790
1791         /* Retrieve all of the head indexes first to avoid caching issues. */
1792         for (head_idx = 0; head_idx < count; head_idx++)
1793                 head[head_idx] = MBUF_HEADROOM_UINT32(pkts[head_idx]);
1794
1795         /*Prefetch descriptor index. */
1796         rte_prefetch0(&vq->desc[head[packet_success]]);
1797
1798         while (packet_success != count) {
1799                 /* Get descriptor from available ring */
1800                 desc = &vq->desc[head[packet_success]];
1801
1802                 buff = pkts[packet_success];
1803                 LOG_DEBUG(VHOST_DATA,
1804                         "(%"PRIu64") in dev_rx_zcp: update the used idx for "
1805                         "pkt[%d] descriptor idx: %d\n",
1806                         dev->device_fh, packet_success,
1807                         MBUF_HEADROOM_UINT32(buff));
1808
1809                 PRINT_PACKET(dev,
1810                         (uintptr_t)(((uint64_t)(uintptr_t)buff->buf_addr)
1811                         + RTE_PKTMBUF_HEADROOM),
1812                         rte_pktmbuf_data_len(buff), 0);
1813
1814                 /* Buffer address translation for virtio header. */
1815                 buff_hdr_addr = gpa_to_vva(dev, desc->addr);
1816                 packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen;
1817
1818                 /*
1819                  * If the descriptors are chained the header and data are
1820                  * placed in separate buffers.
1821                  */
1822                 if (desc->flags & VRING_DESC_F_NEXT) {
1823                         desc->len = vq->vhost_hlen;
1824                         desc = &vq->desc[desc->next];
1825                         desc->len = rte_pktmbuf_data_len(buff);
1826                 } else {
1827                         desc->len = packet_len;
1828                 }
1829
1830                 /* Update used ring with desc information */
1831                 vq->used->ring[res_cur_idx & (vq->size - 1)].id
1832                         = head[packet_success];
1833                 vq->used->ring[res_cur_idx & (vq->size - 1)].len
1834                         = packet_len;
1835                 res_cur_idx++;
1836                 packet_success++;
1837
1838                 /* A header is required per buffer. */
1839                 rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
1840                         (const void *)&virtio_hdr, vq->vhost_hlen);
1841
1842                 PRINT_PACKET(dev, (uintptr_t)buff_hdr_addr, vq->vhost_hlen, 1);
1843
1844                 if (likely(packet_success < count)) {
1845                         /* Prefetch descriptor index. */
1846                         rte_prefetch0(&vq->desc[head[packet_success]]);
1847                 }
1848         }
1849
1850         rte_compiler_barrier();
1851
1852         LOG_DEBUG(VHOST_DATA,
1853                 "(%"PRIu64") in dev_rx_zcp: before update used idx: "
1854                 "vq.last_used_idx: %d, vq->used->idx: %d\n",
1855                 dev->device_fh, vq->last_used_idx, vq->used->idx);
1856
1857         *(volatile uint16_t *)&vq->used->idx += count;
1858         vq->last_used_idx += count;
1859
1860         LOG_DEBUG(VHOST_DATA,
1861                 "(%"PRIu64") in dev_rx_zcp: after  update used idx: "
1862                 "vq.last_used_idx: %d, vq->used->idx: %d\n",
1863                 dev->device_fh, vq->last_used_idx, vq->used->idx);
1864
1865         /* Kick the guest if necessary. */
1866         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
1867                 eventfd_write(vq->callfd, (eventfd_t)1);
1868
1869         return count;
1870 }
1871
1872 /*
1873  * This function routes the TX packet to the correct interface.
1874  * This may be a local device or the physical port.
1875  */
1876 static inline void __attribute__((always_inline))
1877 virtio_tx_route_zcp(struct virtio_net *dev, struct rte_mbuf *m,
1878         uint32_t desc_idx, uint8_t need_copy)
1879 {
1880         struct mbuf_table *tx_q;
1881         struct rte_mbuf **m_table;
1882         void *obj = NULL;
1883         struct rte_mbuf *mbuf;
1884         unsigned len, ret, offset = 0;
1885         struct vpool *vpool;
1886         uint16_t vlan_tag = (uint16_t)vlan_tags[(uint16_t)dev->device_fh];
1887         uint16_t vmdq_rx_q = ((struct vhost_dev *)dev->priv)->vmdq_rx_q;
1888
1889         /*Add packet to the port tx queue*/
1890         tx_q = &tx_queue_zcp[vmdq_rx_q];
1891         len = tx_q->len;
1892
1893         /* Allocate an mbuf and populate the structure. */
1894         vpool = &vpool_array[MAX_QUEUES + vmdq_rx_q];
1895         rte_ring_sc_dequeue(vpool->ring, &obj);
1896         mbuf = obj;
1897         if (unlikely(mbuf == NULL)) {
1898                 struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_TXQ];
1899                 RTE_LOG(ERR, VHOST_DATA,
1900                         "(%"PRIu64") Failed to allocate memory for mbuf.\n",
1901                         dev->device_fh);
1902                 put_desc_to_used_list_zcp(vq, desc_idx);
1903                 return;
1904         }
1905
1906         if (vm2vm_mode == VM2VM_HARDWARE) {
1907                 /* Avoid using a vlan tag from any vm for external pkt, such as
1908                  * vlan_tags[dev->device_fh], oterwise, it conflicts when pool
1909                  * selection, MAC address determines it as an external pkt
1910                  * which should go to network, while vlan tag determine it as
1911                  * a vm2vm pkt should forward to another vm. Hardware confuse
1912                  * such a ambiguous situation, so pkt will lost.
1913                  */
1914                 vlan_tag = external_pkt_default_vlan_tag;
1915                 if (find_local_dest(dev, m, &offset, &vlan_tag) != 0) {
1916                         MBUF_HEADROOM_UINT32(mbuf) = (uint32_t)desc_idx;
1917                         __rte_mbuf_raw_free(mbuf);
1918                         return;
1919                 }
1920         }
1921
1922         mbuf->nb_segs = m->nb_segs;
1923         mbuf->next = m->next;
1924         mbuf->data_len = m->data_len + offset;
1925         mbuf->pkt_len = mbuf->data_len;
1926         if (unlikely(need_copy)) {
1927                 /* Copy the packet contents to the mbuf. */
1928                 rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
1929                         rte_pktmbuf_mtod(m, void *),
1930                         m->data_len);
1931         } else {
1932                 mbuf->data_off = m->data_off;
1933                 mbuf->buf_physaddr = m->buf_physaddr;
1934                 mbuf->buf_addr = m->buf_addr;
1935         }
1936         mbuf->ol_flags |= PKT_TX_VLAN_PKT;
1937         mbuf->vlan_tci = vlan_tag;
1938         mbuf->l2_len = sizeof(struct ether_hdr);
1939         mbuf->l3_len = sizeof(struct ipv4_hdr);
1940         MBUF_HEADROOM_UINT32(mbuf) = (uint32_t)desc_idx;
1941
1942         tx_q->m_table[len] = mbuf;
1943         len++;
1944
1945         LOG_DEBUG(VHOST_DATA,
1946                 "(%"PRIu64") in tx_route_zcp: pkt: nb_seg: %d, next:%s\n",
1947                 dev->device_fh,
1948                 mbuf->nb_segs,
1949                 (mbuf->next == NULL) ? "null" : "non-null");
1950
1951         if (enable_stats) {
1952                 dev_statistics[dev->device_fh].tx_total++;
1953                 dev_statistics[dev->device_fh].tx++;
1954         }
1955
1956         if (unlikely(len == MAX_PKT_BURST)) {
1957                 m_table = (struct rte_mbuf **)tx_q->m_table;
1958                 ret = rte_eth_tx_burst(ports[0],
1959                         (uint16_t)tx_q->txq_id, m_table, (uint16_t) len);
1960
1961                 /*
1962                  * Free any buffers not handled by TX and update
1963                  * the port stats.
1964                  */
1965                 if (unlikely(ret < len)) {
1966                         do {
1967                                 rte_pktmbuf_free(m_table[ret]);
1968                         } while (++ret < len);
1969                 }
1970
1971                 len = 0;
1972                 txmbuf_clean_zcp(dev, vpool);
1973         }
1974
1975         tx_q->len = len;
1976
1977         return;
1978 }
1979
1980 /*
1981  * This function TX all available packets in virtio TX queue for one
1982  * virtio-net device. If it is first packet, it learns MAC address and
1983  * setup VMDQ.
1984  */
1985 static inline void __attribute__((always_inline))
1986 virtio_dev_tx_zcp(struct virtio_net *dev)
1987 {
1988         struct rte_mbuf m;
1989         struct vhost_virtqueue *vq;
1990         struct vring_desc *desc;
1991         uint64_t buff_addr = 0, phys_addr;
1992         uint32_t head[MAX_PKT_BURST];
1993         uint32_t i;
1994         uint16_t free_entries, packet_success = 0;
1995         uint16_t avail_idx;
1996         uint8_t need_copy = 0;
1997         hpa_type addr_type;
1998         struct vhost_dev *vdev = (struct vhost_dev *)dev->priv;
1999
2000         vq = dev->virtqueue[VIRTIO_TXQ];
2001         avail_idx =  *((volatile uint16_t *)&vq->avail->idx);
2002
2003         /* If there are no available buffers then return. */
2004         if (vq->last_used_idx_res == avail_idx)
2005                 return;
2006
2007         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_tx()\n", dev->device_fh);
2008
2009         /* Prefetch available ring to retrieve head indexes. */
2010         rte_prefetch0(&vq->avail->ring[vq->last_used_idx_res & (vq->size - 1)]);
2011
2012         /* Get the number of free entries in the ring */
2013         free_entries = (avail_idx - vq->last_used_idx_res);
2014
2015         /* Limit to MAX_PKT_BURST. */
2016         free_entries
2017                 = (free_entries > MAX_PKT_BURST) ? MAX_PKT_BURST : free_entries;
2018
2019         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n",
2020                 dev->device_fh, free_entries);
2021
2022         /* Retrieve all of the head indexes first to avoid caching issues. */
2023         for (i = 0; i < free_entries; i++)
2024                 head[i]
2025                         = vq->avail->ring[(vq->last_used_idx_res + i)
2026                         & (vq->size - 1)];
2027
2028         vq->last_used_idx_res += free_entries;
2029
2030         /* Prefetch descriptor index. */
2031         rte_prefetch0(&vq->desc[head[packet_success]]);
2032         rte_prefetch0(&vq->used->ring[vq->last_used_idx & (vq->size - 1)]);
2033
2034         while (packet_success < free_entries) {
2035                 desc = &vq->desc[head[packet_success]];
2036
2037                 /* Discard first buffer as it is the virtio header */
2038                 desc = &vq->desc[desc->next];
2039
2040                 /* Buffer address translation. */
2041                 buff_addr = gpa_to_vva(dev, desc->addr);
2042                 /* Need check extra VLAN_HLEN size for inserting VLAN tag */
2043                 phys_addr = gpa_to_hpa(vdev, desc->addr, desc->len + VLAN_HLEN,
2044                         &addr_type);
2045
2046                 if (likely(packet_success < (free_entries - 1)))
2047                         /* Prefetch descriptor index. */
2048                         rte_prefetch0(&vq->desc[head[packet_success + 1]]);
2049
2050                 if (unlikely(addr_type == PHYS_ADDR_INVALID)) {
2051                         RTE_LOG(ERR, VHOST_DATA,
2052                                 "(%"PRIu64") Invalid frame buffer address found"
2053                                 "when TX packets!\n",
2054                                 dev->device_fh);
2055                         packet_success++;
2056                         continue;
2057                 }
2058
2059                 /* Prefetch buffer address. */
2060                 rte_prefetch0((void *)(uintptr_t)buff_addr);
2061
2062                 /*
2063                  * Setup dummy mbuf. This is copied to a real mbuf if
2064                  * transmitted out the physical port.
2065                  */
2066                 m.data_len = desc->len;
2067                 m.nb_segs = 1;
2068                 m.next = NULL;
2069                 m.data_off = 0;
2070                 m.buf_addr = (void *)(uintptr_t)buff_addr;
2071                 m.buf_physaddr = phys_addr;
2072
2073                 /*
2074                  * Check if the frame buffer address from guest crosses
2075                  * sub-region or not.
2076                  */
2077                 if (unlikely(addr_type == PHYS_ADDR_CROSS_SUBREG)) {
2078                         RTE_LOG(ERR, VHOST_DATA,
2079                                 "(%"PRIu64") Frame buffer address cross "
2080                                 "sub-regioin found when attaching TX frame "
2081                                 "buffer address!\n",
2082                                 dev->device_fh);
2083                         need_copy = 1;
2084                 } else
2085                         need_copy = 0;
2086
2087                 PRINT_PACKET(dev, (uintptr_t)buff_addr, desc->len, 0);
2088
2089                 /*
2090                  * If this is the first received packet we need to learn
2091                  * the MAC and setup VMDQ
2092                  */
2093                 if (unlikely(vdev->ready == DEVICE_MAC_LEARNING)) {
2094                         if (vdev->remove || (link_vmdq(vdev, &m) == -1)) {
2095                                 /*
2096                                  * Discard frame if device is scheduled for
2097                                  * removal or a duplicate MAC address is found.
2098                                  */
2099                                 packet_success += free_entries;
2100                                 vq->last_used_idx += packet_success;
2101                                 break;
2102                         }
2103                 }
2104
2105                 virtio_tx_route_zcp(dev, &m, head[packet_success], need_copy);
2106                 packet_success++;
2107         }
2108 }
2109
2110 /*
2111  * This function is called by each data core. It handles all RX/TX registered
2112  * with the core. For TX the specific lcore linked list is used. For RX, MAC
2113  * addresses are compared with all devices in the main linked list.
2114  */
2115 static int
2116 switch_worker_zcp(__attribute__((unused)) void *arg)
2117 {
2118         struct virtio_net *dev = NULL;
2119         struct vhost_dev  *vdev = NULL;
2120         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2121         struct virtio_net_data_ll *dev_ll;
2122         struct mbuf_table *tx_q;
2123         volatile struct lcore_ll_info *lcore_ll;
2124         const uint64_t drain_tsc
2125                 = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S
2126                 * BURST_TX_DRAIN_US;
2127         uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0;
2128         unsigned ret;
2129         const uint16_t lcore_id = rte_lcore_id();
2130         uint16_t count_in_ring, rx_count = 0;
2131
2132         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
2133
2134         lcore_ll = lcore_info[lcore_id].lcore_ll;
2135         prev_tsc = 0;
2136
2137         while (1) {
2138                 cur_tsc = rte_rdtsc();
2139
2140                 /* TX burst queue drain */
2141                 diff_tsc = cur_tsc - prev_tsc;
2142                 if (unlikely(diff_tsc > drain_tsc)) {
2143                         /*
2144                          * Get mbuf from vpool.pool and detach mbuf and
2145                          * put back into vpool.ring.
2146                          */
2147                         dev_ll = lcore_ll->ll_root_used;
2148                         while ((dev_ll != NULL) && (dev_ll->vdev != NULL)) {
2149                                 /* Get virtio device ID */
2150                                 vdev = dev_ll->vdev;
2151                                 dev = vdev->dev;
2152
2153                                 if (likely(!vdev->remove)) {
2154                                         tx_q = &tx_queue_zcp[(uint16_t)vdev->vmdq_rx_q];
2155                                         if (tx_q->len) {
2156                                                 LOG_DEBUG(VHOST_DATA,
2157                                                 "TX queue drained after timeout"
2158                                                 " with burst size %u\n",
2159                                                 tx_q->len);
2160
2161                                                 /*
2162                                                  * Tx any packets in the queue
2163                                                  */
2164                                                 ret = rte_eth_tx_burst(
2165                                                         ports[0],
2166                                                         (uint16_t)tx_q->txq_id,
2167                                                         (struct rte_mbuf **)
2168                                                         tx_q->m_table,
2169                                                         (uint16_t)tx_q->len);
2170                                                 if (unlikely(ret < tx_q->len)) {
2171                                                         do {
2172                                                                 rte_pktmbuf_free(
2173                                                                         tx_q->m_table[ret]);
2174                                                         } while (++ret < tx_q->len);
2175                                                 }
2176                                                 tx_q->len = 0;
2177
2178                                                 txmbuf_clean_zcp(dev,
2179                                                         &vpool_array[MAX_QUEUES+vdev->vmdq_rx_q]);
2180                                         }
2181                                 }
2182                                 dev_ll = dev_ll->next;
2183                         }
2184                         prev_tsc = cur_tsc;
2185                 }
2186
2187                 rte_prefetch0(lcore_ll->ll_root_used);
2188
2189                 /*
2190                  * Inform the configuration core that we have exited the linked
2191                  * list and that no devices are in use if requested.
2192                  */
2193                 if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL)
2194                         lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
2195
2196                 /* Process devices */
2197                 dev_ll = lcore_ll->ll_root_used;
2198
2199                 while ((dev_ll != NULL) && (dev_ll->vdev != NULL)) {
2200                         vdev = dev_ll->vdev;
2201                         dev  = vdev->dev;
2202                         if (unlikely(vdev->remove)) {
2203                                 dev_ll = dev_ll->next;
2204                                 unlink_vmdq(vdev);
2205                                 vdev->ready = DEVICE_SAFE_REMOVE;
2206                                 continue;
2207                         }
2208
2209                         if (likely(vdev->ready == DEVICE_RX)) {
2210                                 uint32_t index = vdev->vmdq_rx_q;
2211                                 uint16_t i;
2212                                 count_in_ring
2213                                 = rte_ring_count(vpool_array[index].ring);
2214                                 uint16_t free_entries
2215                                 = (uint16_t)get_available_ring_num_zcp(dev);
2216
2217                                 /*
2218                                  * Attach all mbufs in vpool.ring and put back
2219                                  * into vpool.pool.
2220                                  */
2221                                 for (i = 0;
2222                                 i < RTE_MIN(free_entries,
2223                                 RTE_MIN(count_in_ring, MAX_PKT_BURST));
2224                                 i++)
2225                                         attach_rxmbuf_zcp(dev);
2226
2227                                 /* Handle guest RX */
2228                                 rx_count = rte_eth_rx_burst(ports[0],
2229                                         vdev->vmdq_rx_q, pkts_burst,
2230                                         MAX_PKT_BURST);
2231
2232                                 if (rx_count) {
2233                                         ret_count = virtio_dev_rx_zcp(dev,
2234                                                         pkts_burst, rx_count);
2235                                         if (enable_stats) {
2236                                                 dev_statistics[dev->device_fh].rx_total
2237                                                         += rx_count;
2238                                                 dev_statistics[dev->device_fh].rx
2239                                                         += ret_count;
2240                                         }
2241                                         while (likely(rx_count)) {
2242                                                 rx_count--;
2243                                                 pktmbuf_detach_zcp(
2244                                                         pkts_burst[rx_count]);
2245                                                 rte_ring_sp_enqueue(
2246                                                         vpool_array[index].ring,
2247                                                         (void *)pkts_burst[rx_count]);
2248                                         }
2249                                 }
2250                         }
2251
2252                         if (likely(!vdev->remove))
2253                                 /* Handle guest TX */
2254                                 virtio_dev_tx_zcp(dev);
2255
2256                         /* Move to the next device in the list */
2257                         dev_ll = dev_ll->next;
2258                 }
2259         }
2260
2261         return 0;
2262 }
2263
2264
2265 /*
2266  * Add an entry to a used linked list. A free entry must first be found
2267  * in the free linked list using get_data_ll_free_entry();
2268  */
2269 static void
2270 add_data_ll_entry(struct virtio_net_data_ll **ll_root_addr,
2271         struct virtio_net_data_ll *ll_dev)
2272 {
2273         struct virtio_net_data_ll *ll = *ll_root_addr;
2274
2275         /* Set next as NULL and use a compiler barrier to avoid reordering. */
2276         ll_dev->next = NULL;
2277         rte_compiler_barrier();
2278
2279         /* If ll == NULL then this is the first device. */
2280         if (ll) {
2281                 /* Increment to the tail of the linked list. */
2282                 while ((ll->next != NULL) )
2283                         ll = ll->next;
2284
2285                 ll->next = ll_dev;
2286         } else {
2287                 *ll_root_addr = ll_dev;
2288         }
2289 }
2290
2291 /*
2292  * Remove an entry from a used linked list. The entry must then be added to
2293  * the free linked list using put_data_ll_free_entry().
2294  */
2295 static void
2296 rm_data_ll_entry(struct virtio_net_data_ll **ll_root_addr,
2297         struct virtio_net_data_ll *ll_dev,
2298         struct virtio_net_data_ll *ll_dev_last)
2299 {
2300         struct virtio_net_data_ll *ll = *ll_root_addr;
2301
2302         if (unlikely((ll == NULL) || (ll_dev == NULL)))
2303                 return;
2304
2305         if (ll_dev == ll)
2306                 *ll_root_addr = ll_dev->next;
2307         else
2308                 if (likely(ll_dev_last != NULL))
2309                         ll_dev_last->next = ll_dev->next;
2310                 else
2311                         RTE_LOG(ERR, VHOST_CONFIG, "Remove entry form ll failed.\n");
2312 }
2313
2314 /*
2315  * Find and return an entry from the free linked list.
2316  */
2317 static struct virtio_net_data_ll *
2318 get_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr)
2319 {
2320         struct virtio_net_data_ll *ll_free = *ll_root_addr;
2321         struct virtio_net_data_ll *ll_dev;
2322
2323         if (ll_free == NULL)
2324                 return NULL;
2325
2326         ll_dev = ll_free;
2327         *ll_root_addr = ll_free->next;
2328
2329         return ll_dev;
2330 }
2331
2332 /*
2333  * Place an entry back on to the free linked list.
2334  */
2335 static void
2336 put_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr,
2337         struct virtio_net_data_ll *ll_dev)
2338 {
2339         struct virtio_net_data_ll *ll_free = *ll_root_addr;
2340
2341         if (ll_dev == NULL)
2342                 return;
2343
2344         ll_dev->next = ll_free;
2345         *ll_root_addr = ll_dev;
2346 }
2347
2348 /*
2349  * Creates a linked list of a given size.
2350  */
2351 static struct virtio_net_data_ll *
2352 alloc_data_ll(uint32_t size)
2353 {
2354         struct virtio_net_data_ll *ll_new;
2355         uint32_t i;
2356
2357         /* Malloc and then chain the linked list. */
2358         ll_new = malloc(size * sizeof(struct virtio_net_data_ll));
2359         if (ll_new == NULL) {
2360                 RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for ll_new.\n");
2361                 return NULL;
2362         }
2363
2364         for (i = 0; i < size - 1; i++) {
2365                 ll_new[i].vdev = NULL;
2366                 ll_new[i].next = &ll_new[i+1];
2367         }
2368         ll_new[i].next = NULL;
2369
2370         return ll_new;
2371 }
2372
2373 /*
2374  * Create the main linked list along with each individual cores linked list. A used and a free list
2375  * are created to manage entries.
2376  */
2377 static int
2378 init_data_ll (void)
2379 {
2380         int lcore;
2381
2382         RTE_LCORE_FOREACH_SLAVE(lcore) {
2383                 lcore_info[lcore].lcore_ll = malloc(sizeof(struct lcore_ll_info));
2384                 if (lcore_info[lcore].lcore_ll == NULL) {
2385                         RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for lcore_ll.\n");
2386                         return -1;
2387                 }
2388
2389                 lcore_info[lcore].lcore_ll->device_num = 0;
2390                 lcore_info[lcore].lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
2391                 lcore_info[lcore].lcore_ll->ll_root_used = NULL;
2392                 if (num_devices % num_switching_cores)
2393                         lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll((num_devices / num_switching_cores) + 1);
2394                 else
2395                         lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll(num_devices / num_switching_cores);
2396         }
2397
2398         /* Allocate devices up to a maximum of MAX_DEVICES. */
2399         ll_root_free = alloc_data_ll(MIN((num_devices), MAX_DEVICES));
2400
2401         return 0;
2402 }
2403
2404 /*
2405  * Remove a device from the specific data core linked list and from the main linked list. Synchonization
2406  * occurs through the use of the lcore dev_removal_flag. Device is made volatile here to avoid re-ordering
2407  * of dev->remove=1 which can cause an infinite loop in the rte_pause loop.
2408  */
2409 static void
2410 destroy_device (volatile struct virtio_net *dev)
2411 {
2412         struct virtio_net_data_ll *ll_lcore_dev_cur;
2413         struct virtio_net_data_ll *ll_main_dev_cur;
2414         struct virtio_net_data_ll *ll_lcore_dev_last = NULL;
2415         struct virtio_net_data_ll *ll_main_dev_last = NULL;
2416         struct vhost_dev *vdev;
2417         int lcore;
2418
2419         dev->flags &= ~VIRTIO_DEV_RUNNING;
2420
2421         vdev = (struct vhost_dev *)dev->priv;
2422         /*set the remove flag. */
2423         vdev->remove = 1;
2424         while(vdev->ready != DEVICE_SAFE_REMOVE) {
2425                 rte_pause();
2426         }
2427
2428         /* Search for entry to be removed from lcore ll */
2429         ll_lcore_dev_cur = lcore_info[vdev->coreid].lcore_ll->ll_root_used;
2430         while (ll_lcore_dev_cur != NULL) {
2431                 if (ll_lcore_dev_cur->vdev == vdev) {
2432                         break;
2433                 } else {
2434                         ll_lcore_dev_last = ll_lcore_dev_cur;
2435                         ll_lcore_dev_cur = ll_lcore_dev_cur->next;
2436                 }
2437         }
2438
2439         if (ll_lcore_dev_cur == NULL) {
2440                 RTE_LOG(ERR, VHOST_CONFIG,
2441                         "(%"PRIu64") Failed to find the dev to be destroy.\n",
2442                         dev->device_fh);
2443                 return;
2444         }
2445
2446         /* Search for entry to be removed from main ll */
2447         ll_main_dev_cur = ll_root_used;
2448         ll_main_dev_last = NULL;
2449         while (ll_main_dev_cur != NULL) {
2450                 if (ll_main_dev_cur->vdev == vdev) {
2451                         break;
2452                 } else {
2453                         ll_main_dev_last = ll_main_dev_cur;
2454                         ll_main_dev_cur = ll_main_dev_cur->next;
2455                 }
2456         }
2457
2458         /* Remove entries from the lcore and main ll. */
2459         rm_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
2460         rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
2461
2462         /* Set the dev_removal_flag on each lcore. */
2463         RTE_LCORE_FOREACH_SLAVE(lcore) {
2464                 lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;
2465         }
2466
2467         /*
2468          * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that
2469          * they can no longer access the device removed from the linked lists and that the devices
2470          * are no longer in use.
2471          */
2472         RTE_LCORE_FOREACH_SLAVE(lcore) {
2473                 while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL) {
2474                         rte_pause();
2475                 }
2476         }
2477
2478         /* Add the entries back to the lcore and main free ll.*/
2479         put_data_ll_free_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);
2480         put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
2481
2482         /* Decrement number of device on the lcore. */
2483         lcore_info[vdev->coreid].lcore_ll->device_num--;
2484
2485         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been removed from data core\n", dev->device_fh);
2486
2487         if (zero_copy) {
2488                 struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
2489
2490                 /* Stop the RX queue. */
2491                 if (rte_eth_dev_rx_queue_stop(ports[0], vdev->vmdq_rx_q) != 0) {
2492                         LOG_DEBUG(VHOST_CONFIG,
2493                                 "(%"PRIu64") In destroy_device: Failed to stop "
2494                                 "rx queue:%d\n",
2495                                 dev->device_fh,
2496                                 vdev->vmdq_rx_q);
2497                 }
2498
2499                 LOG_DEBUG(VHOST_CONFIG,
2500                         "(%"PRIu64") in destroy_device: Start put mbuf in "
2501                         "mempool back to ring for RX queue: %d\n",
2502                         dev->device_fh, vdev->vmdq_rx_q);
2503
2504                 mbuf_destroy_zcp(vpool);
2505
2506                 /* Stop the TX queue. */
2507                 if (rte_eth_dev_tx_queue_stop(ports[0], vdev->vmdq_rx_q) != 0) {
2508                         LOG_DEBUG(VHOST_CONFIG,
2509                                 "(%"PRIu64") In destroy_device: Failed to "
2510                                 "stop tx queue:%d\n",
2511                                 dev->device_fh, vdev->vmdq_rx_q);
2512                 }
2513
2514                 vpool = &vpool_array[vdev->vmdq_rx_q + MAX_QUEUES];
2515
2516                 LOG_DEBUG(VHOST_CONFIG,
2517                         "(%"PRIu64") destroy_device: Start put mbuf in mempool "
2518                         "back to ring for TX queue: %d, dev:(%"PRIu64")\n",
2519                         dev->device_fh, (vdev->vmdq_rx_q + MAX_QUEUES),
2520                         dev->device_fh);
2521
2522                 mbuf_destroy_zcp(vpool);
2523                 rte_free(vdev->regions_hpa);
2524         }
2525         rte_free(vdev);
2526
2527 }
2528
2529 /*
2530  * Calculate the region count of physical continous regions for one particular
2531  * region of whose vhost virtual address is continous. The particular region
2532  * start from vva_start, with size of 'size' in argument.
2533  */
2534 static uint32_t
2535 check_hpa_regions(uint64_t vva_start, uint64_t size)
2536 {
2537         uint32_t i, nregions = 0, page_size = getpagesize();
2538         uint64_t cur_phys_addr = 0, next_phys_addr = 0;
2539         if (vva_start % page_size) {
2540                 LOG_DEBUG(VHOST_CONFIG,
2541                         "in check_countinous: vva start(%p) mod page_size(%d) "
2542                         "has remainder\n",
2543                         (void *)(uintptr_t)vva_start, page_size);
2544                 return 0;
2545         }
2546         if (size % page_size) {
2547                 LOG_DEBUG(VHOST_CONFIG,
2548                         "in check_countinous: "
2549                         "size((%"PRIu64")) mod page_size(%d) has remainder\n",
2550                         size, page_size);
2551                 return 0;
2552         }
2553         for (i = 0; i < size - page_size; i = i + page_size) {
2554                 cur_phys_addr
2555                         = rte_mem_virt2phy((void *)(uintptr_t)(vva_start + i));
2556                 next_phys_addr = rte_mem_virt2phy(
2557                         (void *)(uintptr_t)(vva_start + i + page_size));
2558                 if ((cur_phys_addr + page_size) != next_phys_addr) {
2559                         ++nregions;
2560                         LOG_DEBUG(VHOST_CONFIG,
2561                                 "in check_continuous: hva addr:(%p) is not "
2562                                 "continuous with hva addr:(%p), diff:%d\n",
2563                                 (void *)(uintptr_t)(vva_start + (uint64_t)i),
2564                                 (void *)(uintptr_t)(vva_start + (uint64_t)i
2565                                 + page_size), page_size);
2566                         LOG_DEBUG(VHOST_CONFIG,
2567                                 "in check_continuous: hpa addr:(%p) is not "
2568                                 "continuous with hpa addr:(%p), "
2569                                 "diff:(%"PRIu64")\n",
2570                                 (void *)(uintptr_t)cur_phys_addr,
2571                                 (void *)(uintptr_t)next_phys_addr,
2572                                 (next_phys_addr-cur_phys_addr));
2573                 }
2574         }
2575         return nregions;
2576 }
2577
2578 /*
2579  * Divide each region whose vhost virtual address is continous into a few
2580  * sub-regions, make sure the physical address within each sub-region are
2581  * continous. And fill offset(to GPA) and size etc. information of each
2582  * sub-region into regions_hpa.
2583  */
2584 static uint32_t
2585 fill_hpa_memory_regions(struct virtio_memory_regions_hpa *mem_region_hpa, struct virtio_memory *virtio_memory)
2586 {
2587         uint32_t regionidx, regionidx_hpa = 0, i, k, page_size = getpagesize();
2588         uint64_t cur_phys_addr = 0, next_phys_addr = 0, vva_start;
2589
2590         if (mem_region_hpa == NULL)
2591                 return 0;
2592
2593         for (regionidx = 0; regionidx < virtio_memory->nregions; regionidx++) {
2594                 vva_start = virtio_memory->regions[regionidx].guest_phys_address +
2595                         virtio_memory->regions[regionidx].address_offset;
2596                 mem_region_hpa[regionidx_hpa].guest_phys_address
2597                         = virtio_memory->regions[regionidx].guest_phys_address;
2598                 mem_region_hpa[regionidx_hpa].host_phys_addr_offset =
2599                         rte_mem_virt2phy((void *)(uintptr_t)(vva_start)) -
2600                         mem_region_hpa[regionidx_hpa].guest_phys_address;
2601                 LOG_DEBUG(VHOST_CONFIG,
2602                         "in fill_hpa_regions: guest phys addr start[%d]:(%p)\n",
2603                         regionidx_hpa,
2604                         (void *)(uintptr_t)
2605                         (mem_region_hpa[regionidx_hpa].guest_phys_address));
2606                 LOG_DEBUG(VHOST_CONFIG,
2607                         "in fill_hpa_regions: host  phys addr start[%d]:(%p)\n",
2608                         regionidx_hpa,
2609                         (void *)(uintptr_t)
2610                         (mem_region_hpa[regionidx_hpa].host_phys_addr_offset));
2611                 for (i = 0, k = 0;
2612                         i < virtio_memory->regions[regionidx].memory_size -
2613                                 page_size;
2614                         i += page_size) {
2615                         cur_phys_addr = rte_mem_virt2phy(
2616                                         (void *)(uintptr_t)(vva_start + i));
2617                         next_phys_addr = rte_mem_virt2phy(
2618                                         (void *)(uintptr_t)(vva_start +
2619                                         i + page_size));
2620                         if ((cur_phys_addr + page_size) != next_phys_addr) {
2621                                 mem_region_hpa[regionidx_hpa].guest_phys_address_end =
2622                                         mem_region_hpa[regionidx_hpa].guest_phys_address +
2623                                         k + page_size;
2624                                 mem_region_hpa[regionidx_hpa].memory_size
2625                                         = k + page_size;
2626                                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest "
2627                                         "phys addr end  [%d]:(%p)\n",
2628                                         regionidx_hpa,
2629                                         (void *)(uintptr_t)
2630                                         (mem_region_hpa[regionidx_hpa].guest_phys_address_end));
2631                                 LOG_DEBUG(VHOST_CONFIG,
2632                                         "in fill_hpa_regions: guest phys addr "
2633                                         "size [%d]:(%p)\n",
2634                                         regionidx_hpa,
2635                                         (void *)(uintptr_t)
2636                                         (mem_region_hpa[regionidx_hpa].memory_size));
2637                                 mem_region_hpa[regionidx_hpa + 1].guest_phys_address
2638                                         = mem_region_hpa[regionidx_hpa].guest_phys_address_end;
2639                                 ++regionidx_hpa;
2640                                 mem_region_hpa[regionidx_hpa].host_phys_addr_offset =
2641                                         next_phys_addr -
2642                                         mem_region_hpa[regionidx_hpa].guest_phys_address;
2643                                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest"
2644                                         " phys addr start[%d]:(%p)\n",
2645                                         regionidx_hpa,
2646                                         (void *)(uintptr_t)
2647                                         (mem_region_hpa[regionidx_hpa].guest_phys_address));
2648                                 LOG_DEBUG(VHOST_CONFIG,
2649                                         "in fill_hpa_regions: host  phys addr "
2650                                         "start[%d]:(%p)\n",
2651                                         regionidx_hpa,
2652                                         (void *)(uintptr_t)
2653                                         (mem_region_hpa[regionidx_hpa].host_phys_addr_offset));
2654                                 k = 0;
2655                         } else {
2656                                 k += page_size;
2657                         }
2658                 }
2659                 mem_region_hpa[regionidx_hpa].guest_phys_address_end
2660                         = mem_region_hpa[regionidx_hpa].guest_phys_address
2661                         + k + page_size;
2662                 mem_region_hpa[regionidx_hpa].memory_size = k + page_size;
2663                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest phys addr end  "
2664                         "[%d]:(%p)\n", regionidx_hpa,
2665                         (void *)(uintptr_t)
2666                         (mem_region_hpa[regionidx_hpa].guest_phys_address_end));
2667                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest phys addr size "
2668                         "[%d]:(%p)\n", regionidx_hpa,
2669                         (void *)(uintptr_t)
2670                         (mem_region_hpa[regionidx_hpa].memory_size));
2671                 ++regionidx_hpa;
2672         }
2673         return regionidx_hpa;
2674 }
2675
2676 /*
2677  * A new device is added to a data core. First the device is added to the main linked list
2678  * and the allocated to a specific data core.
2679  */
2680 static int
2681 new_device (struct virtio_net *dev)
2682 {
2683         struct virtio_net_data_ll *ll_dev;
2684         int lcore, core_add = 0;
2685         uint32_t device_num_min = num_devices;
2686         struct vhost_dev *vdev;
2687         uint32_t regionidx;
2688
2689         vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
2690         if (vdev == NULL) {
2691                 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Couldn't allocate memory for vhost dev\n",
2692                         dev->device_fh);
2693                 return -1;
2694         }
2695         vdev->dev = dev;
2696         dev->priv = vdev;
2697
2698         if (zero_copy) {
2699                 vdev->nregions_hpa = dev->mem->nregions;
2700                 for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
2701                         vdev->nregions_hpa
2702                                 += check_hpa_regions(
2703                                         dev->mem->regions[regionidx].guest_phys_address
2704                                         + dev->mem->regions[regionidx].address_offset,
2705                                         dev->mem->regions[regionidx].memory_size);
2706
2707                 }
2708
2709                 vdev->regions_hpa = rte_calloc("vhost hpa region",
2710                                                vdev->nregions_hpa,
2711                                                sizeof(struct virtio_memory_regions_hpa),
2712                                                RTE_CACHE_LINE_SIZE);
2713                 if (vdev->regions_hpa == NULL) {
2714                         RTE_LOG(ERR, VHOST_CONFIG, "Cannot allocate memory for hpa region\n");
2715                         rte_free(vdev);
2716                         return -1;
2717                 }
2718
2719
2720                 if (fill_hpa_memory_regions(
2721                         vdev->regions_hpa, dev->mem
2722                         ) != vdev->nregions_hpa) {
2723
2724                         RTE_LOG(ERR, VHOST_CONFIG,
2725                                 "hpa memory regions number mismatch: "
2726                                 "[%d]\n", vdev->nregions_hpa);
2727                         rte_free(vdev->regions_hpa);
2728                         rte_free(vdev);
2729                         return -1;
2730                 }
2731         }
2732
2733
2734         /* Add device to main ll */
2735         ll_dev = get_data_ll_free_entry(&ll_root_free);
2736         if (ll_dev == NULL) {
2737                 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") No free entry found in linked list. Device limit "
2738                         "of %d devices per core has been reached\n",
2739                         dev->device_fh, num_devices);
2740                 if (vdev->regions_hpa)
2741                         rte_free(vdev->regions_hpa);
2742                 rte_free(vdev);
2743                 return -1;
2744         }
2745         ll_dev->vdev = vdev;
2746         add_data_ll_entry(&ll_root_used, ll_dev);
2747         vdev->vmdq_rx_q
2748                 = dev->device_fh * queues_per_pool + vmdq_queue_base;
2749
2750         if (zero_copy) {
2751                 uint32_t index = vdev->vmdq_rx_q;
2752                 uint32_t count_in_ring, i;
2753                 struct mbuf_table *tx_q;
2754
2755                 count_in_ring = rte_ring_count(vpool_array[index].ring);
2756
2757                 LOG_DEBUG(VHOST_CONFIG,
2758                         "(%"PRIu64") in new_device: mbuf count in mempool "
2759                         "before attach is: %d\n",
2760                         dev->device_fh,
2761                         rte_mempool_count(vpool_array[index].pool));
2762                 LOG_DEBUG(VHOST_CONFIG,
2763                         "(%"PRIu64") in new_device: mbuf count in  ring "
2764                         "before attach  is : %d\n",
2765                         dev->device_fh, count_in_ring);
2766
2767                 /*
2768                  * Attach all mbufs in vpool.ring and put back intovpool.pool.
2769                  */
2770                 for (i = 0; i < count_in_ring; i++)
2771                         attach_rxmbuf_zcp(dev);
2772
2773                 LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") in new_device: mbuf count in "
2774                         "mempool after attach is: %d\n",
2775                         dev->device_fh,
2776                         rte_mempool_count(vpool_array[index].pool));
2777                 LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") in new_device: mbuf count in "
2778                         "ring after attach  is : %d\n",
2779                         dev->device_fh,
2780                         rte_ring_count(vpool_array[index].ring));
2781
2782                 tx_q = &tx_queue_zcp[(uint16_t)vdev->vmdq_rx_q];
2783                 tx_q->txq_id = vdev->vmdq_rx_q;
2784
2785                 if (rte_eth_dev_tx_queue_start(ports[0], vdev->vmdq_rx_q) != 0) {
2786                         struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
2787
2788                         LOG_DEBUG(VHOST_CONFIG,
2789                                 "(%"PRIu64") In new_device: Failed to start "
2790                                 "tx queue:%d\n",
2791                                 dev->device_fh, vdev->vmdq_rx_q);
2792
2793                         mbuf_destroy_zcp(vpool);
2794                         rte_free(vdev->regions_hpa);
2795                         rte_free(vdev);
2796                         return -1;
2797                 }
2798
2799                 if (rte_eth_dev_rx_queue_start(ports[0], vdev->vmdq_rx_q) != 0) {
2800                         struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
2801
2802                         LOG_DEBUG(VHOST_CONFIG,
2803                                 "(%"PRIu64") In new_device: Failed to start "
2804                                 "rx queue:%d\n",
2805                                 dev->device_fh, vdev->vmdq_rx_q);
2806
2807                         /* Stop the TX queue. */
2808                         if (rte_eth_dev_tx_queue_stop(ports[0],
2809                                 vdev->vmdq_rx_q) != 0) {
2810                                 LOG_DEBUG(VHOST_CONFIG,
2811                                         "(%"PRIu64") In new_device: Failed to "
2812                                         "stop tx queue:%d\n",
2813                                         dev->device_fh, vdev->vmdq_rx_q);
2814                         }
2815
2816                         mbuf_destroy_zcp(vpool);
2817                         rte_free(vdev->regions_hpa);
2818                         rte_free(vdev);
2819                         return -1;
2820                 }
2821
2822         }
2823
2824         /*reset ready flag*/
2825         vdev->ready = DEVICE_MAC_LEARNING;
2826         vdev->remove = 0;
2827
2828         /* Find a suitable lcore to add the device. */
2829         RTE_LCORE_FOREACH_SLAVE(lcore) {
2830                 if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
2831                         device_num_min = lcore_info[lcore].lcore_ll->device_num;
2832                         core_add = lcore;
2833                 }
2834         }
2835         /* Add device to lcore ll */
2836         ll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);
2837         if (ll_dev == NULL) {
2838                 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
2839                 vdev->ready = DEVICE_SAFE_REMOVE;
2840                 destroy_device(dev);
2841                 rte_free(vdev->regions_hpa);
2842                 rte_free(vdev);
2843                 return -1;
2844         }
2845         ll_dev->vdev = vdev;
2846         vdev->coreid = core_add;
2847
2848         add_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_dev);
2849
2850         /* Initialize device stats */
2851         memset(&dev_statistics[dev->device_fh], 0, sizeof(struct device_statistics));
2852
2853         /* Disable notifications. */
2854         rte_vhost_enable_guest_notification(dev, VIRTIO_RXQ, 0);
2855         rte_vhost_enable_guest_notification(dev, VIRTIO_TXQ, 0);
2856         lcore_info[vdev->coreid].lcore_ll->device_num++;
2857         dev->flags |= VIRTIO_DEV_RUNNING;
2858
2859         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, vdev->coreid);
2860
2861         return 0;
2862 }
2863
2864 /*
2865  * These callback allow devices to be added to the data core when configuration
2866  * has been fully complete.
2867  */
2868 static const struct virtio_net_device_ops virtio_net_device_ops =
2869 {
2870         .new_device =  new_device,
2871         .destroy_device = destroy_device,
2872 };
2873
2874 /*
2875  * This is a thread will wake up after a period to print stats if the user has
2876  * enabled them.
2877  */
2878 static void
2879 print_stats(void)
2880 {
2881         struct virtio_net_data_ll *dev_ll;
2882         uint64_t tx_dropped, rx_dropped;
2883         uint64_t tx, tx_total, rx, rx_total;
2884         uint32_t device_fh;
2885         const char clr[] = { 27, '[', '2', 'J', '\0' };
2886         const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' };
2887
2888         while(1) {
2889                 sleep(enable_stats);
2890
2891                 /* Clear screen and move to top left */
2892                 printf("%s%s", clr, top_left);
2893
2894                 printf("\nDevice statistics ====================================");
2895
2896                 dev_ll = ll_root_used;
2897                 while (dev_ll != NULL) {
2898                         device_fh = (uint32_t)dev_ll->vdev->dev->device_fh;
2899                         tx_total = dev_statistics[device_fh].tx_total;
2900                         tx = dev_statistics[device_fh].tx;
2901                         tx_dropped = tx_total - tx;
2902                         if (zero_copy == 0) {
2903                                 rx_total = rte_atomic64_read(
2904                                         &dev_statistics[device_fh].rx_total_atomic);
2905                                 rx = rte_atomic64_read(
2906                                         &dev_statistics[device_fh].rx_atomic);
2907                         } else {
2908                                 rx_total = dev_statistics[device_fh].rx_total;
2909                                 rx = dev_statistics[device_fh].rx;
2910                         }
2911                         rx_dropped = rx_total - rx;
2912
2913                         printf("\nStatistics for device %"PRIu32" ------------------------------"
2914                                         "\nTX total:            %"PRIu64""
2915                                         "\nTX dropped:          %"PRIu64""
2916                                         "\nTX successful:               %"PRIu64""
2917                                         "\nRX total:            %"PRIu64""
2918                                         "\nRX dropped:          %"PRIu64""
2919                                         "\nRX successful:               %"PRIu64"",
2920                                         device_fh,
2921                                         tx_total,
2922                                         tx_dropped,
2923                                         tx,
2924                                         rx_total,
2925                                         rx_dropped,
2926                                         rx);
2927
2928                         dev_ll = dev_ll->next;
2929                 }
2930                 printf("\n======================================================\n");
2931         }
2932 }
2933
2934 static void
2935 setup_mempool_tbl(int socket, uint32_t index, char *pool_name,
2936         char *ring_name, uint32_t nb_mbuf)
2937 {
2938         vpool_array[index].pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
2939                 MBUF_CACHE_SIZE_ZCP, 0, MBUF_DATA_SIZE_ZCP, socket);
2940         if (vpool_array[index].pool != NULL) {
2941                 vpool_array[index].ring
2942                         = rte_ring_create(ring_name,
2943                                 rte_align32pow2(nb_mbuf + 1),
2944                                 socket, RING_F_SP_ENQ | RING_F_SC_DEQ);
2945                 if (likely(vpool_array[index].ring != NULL)) {
2946                         LOG_DEBUG(VHOST_CONFIG,
2947                                 "in setup_mempool_tbl: mbuf count in "
2948                                 "mempool is: %d\n",
2949                                 rte_mempool_count(vpool_array[index].pool));
2950                         LOG_DEBUG(VHOST_CONFIG,
2951                                 "in setup_mempool_tbl: mbuf count in "
2952                                 "ring   is: %d\n",
2953                                 rte_ring_count(vpool_array[index].ring));
2954                 } else {
2955                         rte_exit(EXIT_FAILURE, "ring_create(%s) failed",
2956                                 ring_name);
2957                 }
2958
2959                 /* Need consider head room. */
2960                 vpool_array[index].buf_size = VIRTIO_DESCRIPTOR_LEN_ZCP;
2961         } else {
2962                 rte_exit(EXIT_FAILURE, "mempool_create(%s) failed", pool_name);
2963         }
2964 }
2965
2966 /* When we receive a INT signal, unregister vhost driver */
2967 static void
2968 sigint_handler(__rte_unused int signum)
2969 {
2970         /* Unregister vhost driver. */
2971         int ret = rte_vhost_driver_unregister((char *)&dev_basename);
2972         if (ret != 0)
2973                 rte_exit(EXIT_FAILURE, "vhost driver unregister failure.\n");
2974         exit(0);
2975 }
2976
2977 /*
2978  * Main function, does initialisation and calls the per-lcore functions. The CUSE
2979  * device is also registered here to handle the IOCTLs.
2980  */
2981 int
2982 main(int argc, char *argv[])
2983 {
2984         struct rte_mempool *mbuf_pool = NULL;
2985         unsigned lcore_id, core_id = 0;
2986         unsigned nb_ports, valid_num_ports;
2987         int ret;
2988         uint8_t portid;
2989         uint16_t queue_id;
2990         static pthread_t tid;
2991         char thread_name[RTE_MAX_THREAD_NAME_LEN];
2992
2993         signal(SIGINT, sigint_handler);
2994
2995         /* init EAL */
2996         ret = rte_eal_init(argc, argv);
2997         if (ret < 0)
2998                 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
2999         argc -= ret;
3000         argv += ret;
3001
3002         /* parse app arguments */
3003         ret = us_vhost_parse_args(argc, argv);
3004         if (ret < 0)
3005                 rte_exit(EXIT_FAILURE, "Invalid argument\n");
3006
3007         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++)
3008                 if (rte_lcore_is_enabled(lcore_id))
3009                         lcore_ids[core_id ++] = lcore_id;
3010
3011         if (rte_lcore_count() > RTE_MAX_LCORE)
3012                 rte_exit(EXIT_FAILURE,"Not enough cores\n");
3013
3014         /*set the number of swithcing cores available*/
3015         num_switching_cores = rte_lcore_count()-1;
3016
3017         /* Get the number of physical ports. */
3018         nb_ports = rte_eth_dev_count();
3019         if (nb_ports > RTE_MAX_ETHPORTS)
3020                 nb_ports = RTE_MAX_ETHPORTS;
3021
3022         /*
3023          * Update the global var NUM_PORTS and global array PORTS
3024          * and get value of var VALID_NUM_PORTS according to system ports number
3025          */
3026         valid_num_ports = check_ports_num(nb_ports);
3027
3028         if ((valid_num_ports ==  0) || (valid_num_ports > MAX_SUP_PORTS)) {
3029                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
3030                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
3031                 return -1;
3032         }
3033
3034         if (zero_copy == 0) {
3035                 /* Create the mbuf pool. */
3036                 mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL",
3037                         NUM_MBUFS_PER_PORT * valid_num_ports, MBUF_CACHE_SIZE,
3038                         0, MBUF_DATA_SIZE, rte_socket_id());
3039                 if (mbuf_pool == NULL)
3040                         rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
3041
3042                 for (queue_id = 0; queue_id < MAX_QUEUES + 1; queue_id++)
3043                         vpool_array[queue_id].pool = mbuf_pool;
3044
3045                 if (vm2vm_mode == VM2VM_HARDWARE) {
3046                         /* Enable VT loop back to let L2 switch to do it. */
3047                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1;
3048                         LOG_DEBUG(VHOST_CONFIG,
3049                                 "Enable loop back for L2 switch in vmdq.\n");
3050                 }
3051         } else {
3052                 uint32_t nb_mbuf;
3053                 char pool_name[RTE_MEMPOOL_NAMESIZE];
3054                 char ring_name[RTE_MEMPOOL_NAMESIZE];
3055
3056                 nb_mbuf = num_rx_descriptor
3057                         + num_switching_cores * MBUF_CACHE_SIZE_ZCP
3058                         + num_switching_cores * MAX_PKT_BURST;
3059
3060                 for (queue_id = 0; queue_id < MAX_QUEUES; queue_id++) {
3061                         snprintf(pool_name, sizeof(pool_name),
3062                                 "rxmbuf_pool_%u", queue_id);
3063                         snprintf(ring_name, sizeof(ring_name),
3064                                 "rxmbuf_ring_%u", queue_id);
3065                         setup_mempool_tbl(rte_socket_id(), queue_id,
3066                                 pool_name, ring_name, nb_mbuf);
3067                 }
3068
3069                 nb_mbuf = num_tx_descriptor
3070                                 + num_switching_cores * MBUF_CACHE_SIZE_ZCP
3071                                 + num_switching_cores * MAX_PKT_BURST;
3072
3073                 for (queue_id = 0; queue_id < MAX_QUEUES; queue_id++) {
3074                         snprintf(pool_name, sizeof(pool_name),
3075                                 "txmbuf_pool_%u", queue_id);
3076                         snprintf(ring_name, sizeof(ring_name),
3077                                 "txmbuf_ring_%u", queue_id);
3078                         setup_mempool_tbl(rte_socket_id(),
3079                                 (queue_id + MAX_QUEUES),
3080                                 pool_name, ring_name, nb_mbuf);
3081                 }
3082
3083                 if (vm2vm_mode == VM2VM_HARDWARE) {
3084                         /* Enable VT loop back to let L2 switch to do it. */
3085                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1;
3086                         LOG_DEBUG(VHOST_CONFIG,
3087                                 "Enable loop back for L2 switch in vmdq.\n");
3088                 }
3089         }
3090         /* Set log level. */
3091         rte_set_log_level(LOG_LEVEL);
3092
3093         /* initialize all ports */
3094         for (portid = 0; portid < nb_ports; portid++) {
3095                 /* skip ports that are not enabled */
3096                 if ((enabled_port_mask & (1 << portid)) == 0) {
3097                         RTE_LOG(INFO, VHOST_PORT,
3098                                 "Skipping disabled port %d\n", portid);
3099                         continue;
3100                 }
3101                 if (port_init(portid) != 0)
3102                         rte_exit(EXIT_FAILURE,
3103                                 "Cannot initialize network ports\n");
3104         }
3105
3106         /* Initialise all linked lists. */
3107         if (init_data_ll() == -1)
3108                 rte_exit(EXIT_FAILURE, "Failed to initialize linked list\n");
3109
3110         /* Initialize device stats */
3111         memset(&dev_statistics, 0, sizeof(dev_statistics));
3112
3113         /* Enable stats if the user option is set. */
3114         if (enable_stats) {
3115                 ret = pthread_create(&tid, NULL, (void *)print_stats, NULL);
3116                 if (ret != 0)
3117                         rte_exit(EXIT_FAILURE,
3118                                 "Cannot create print-stats thread\n");
3119
3120                 /* Set thread_name for aid in debugging.  */
3121                 snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, "print-stats");
3122                 ret = rte_thread_setname(tid, thread_name);
3123                 if (ret != 0)
3124                         RTE_LOG(ERR, VHOST_CONFIG,
3125                                 "Cannot set print-stats name\n");
3126         }
3127
3128         /* Launch all data cores. */
3129         if (zero_copy == 0) {
3130                 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3131                         rte_eal_remote_launch(switch_worker,
3132                                 mbuf_pool, lcore_id);
3133                 }
3134         } else {
3135                 uint32_t count_in_mempool, index, i;
3136                 for (index = 0; index < 2*MAX_QUEUES; index++) {
3137                         /* For all RX and TX queues. */
3138                         count_in_mempool
3139                                 = rte_mempool_count(vpool_array[index].pool);
3140
3141                         /*
3142                          * Transfer all un-attached mbufs from vpool.pool
3143                          * to vpoo.ring.
3144                          */
3145                         for (i = 0; i < count_in_mempool; i++) {
3146                                 struct rte_mbuf *mbuf
3147                                         = __rte_mbuf_raw_alloc(
3148                                                 vpool_array[index].pool);
3149                                 rte_ring_sp_enqueue(vpool_array[index].ring,
3150                                                 (void *)mbuf);
3151                         }
3152
3153                         LOG_DEBUG(VHOST_CONFIG,
3154                                 "in main: mbuf count in mempool at initial "
3155                                 "is: %d\n", count_in_mempool);
3156                         LOG_DEBUG(VHOST_CONFIG,
3157                                 "in main: mbuf count in  ring at initial  is :"
3158                                 " %d\n",
3159                                 rte_ring_count(vpool_array[index].ring));
3160                 }
3161
3162                 RTE_LCORE_FOREACH_SLAVE(lcore_id)
3163                         rte_eal_remote_launch(switch_worker_zcp, NULL,
3164                                 lcore_id);
3165         }
3166
3167         if (mergeable == 0)
3168                 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_MRG_RXBUF);
3169
3170         /* Register vhost(cuse or user) driver to handle vhost messages. */
3171         ret = rte_vhost_driver_register((char *)&dev_basename);
3172         if (ret != 0)
3173                 rte_exit(EXIT_FAILURE, "vhost driver register failure.\n");
3174
3175         rte_vhost_driver_callback_register(&virtio_net_device_ops);
3176
3177         /* Start CUSE session. */
3178         rte_vhost_driver_session_start();
3179         return 0;
3180
3181 }