examples/vhost/main.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <arpa/inet.h>
  35 #include <getopt.h>
  36 #include <linux/if_ether.h>
  37 #include <linux/if_vlan.h>
  38 #include <linux/virtio_net.h>
  39 #include <linux/virtio_ring.h>
  40 #include <signal.h>
  41 #include <stdint.h>
  42 #include <sys/eventfd.h>
  43 #include <sys/param.h>
  44 #include <unistd.h>
  45
  46 #include <rte_atomic.h>
  47 #include <rte_cycles.h>
  48 #include <rte_ethdev.h>
  49 #include <rte_log.h>
  50 #include <rte_string_fns.h>
  51 #include <rte_malloc.h>
  52 #include <rte_virtio_net.h>
  53
  54 #include "main.h"
  55
  56 #define MAX_QUEUES 512
  57
  58 /* the maximum number of external ports supported */
  59 #define MAX_SUP_PORTS 1
  60
  61 /*
  62  * Calculate the number of buffers needed per port
  63  */
  64 #define NUM_MBUFS_PER_PORT ((MAX_QUEUES*RTE_TEST_RX_DESC_DEFAULT) +             \
  65                                                         (num_switching_cores*MAX_PKT_BURST) +                   \
  66                                                         (num_switching_cores*RTE_TEST_TX_DESC_DEFAULT) +\
  67                                                         (num_switching_cores*MBUF_CACHE_SIZE))
  68
  69 #define MBUF_CACHE_SIZE 128
  70 #define MBUF_DATA_SIZE  RTE_MBUF_DEFAULT_BUF_SIZE
  71
  72 /*
  73  * No frame data buffer allocated from host are required for zero copy
  74  * implementation, guest will allocate the frame data buffer, and vhost
  75  * directly use it.
  76  */
  77 #define VIRTIO_DESCRIPTOR_LEN_ZCP       RTE_MBUF_DEFAULT_DATAROOM
  78 #define MBUF_DATA_SIZE_ZCP              RTE_MBUF_DEFAULT_BUF_SIZE
  79 #define MBUF_CACHE_SIZE_ZCP 0
  80
  81 #define MAX_PKT_BURST 32                /* Max burst size for RX/TX */
  82 #define BURST_TX_DRAIN_US 100   /* TX drain every ~100us */
  83
  84 #define BURST_RX_WAIT_US 15     /* Defines how long we wait between retries on RX */
  85 #define BURST_RX_RETRIES 4              /* Number of retries on RX. */
  86
  87 #define JUMBO_FRAME_MAX_SIZE    0x2600
  88
  89 /* State of virtio device. */
  90 #define DEVICE_MAC_LEARNING 0
  91 #define DEVICE_RX                       1
  92 #define DEVICE_SAFE_REMOVE      2
  93
  94 /* Config_core_flag status definitions. */
  95 #define REQUEST_DEV_REMOVAL 1
  96 #define ACK_DEV_REMOVAL 0
  97
  98 /* Configurable number of RX/TX ring descriptors */
  99 #define RTE_TEST_RX_DESC_DEFAULT 1024
 100 #define RTE_TEST_TX_DESC_DEFAULT 512
 101
 102 /*
 103  * Need refine these 2 macros for legacy and DPDK based front end:
 104  * Max vring avail descriptor/entries from guest - MAX_PKT_BURST
 105  * And then adjust power 2.
 106  */
 107 /*
 108  * For legacy front end, 128 descriptors,
 109  * half for virtio header, another half for mbuf.
 110  */
 111 #define RTE_TEST_RX_DESC_DEFAULT_ZCP 32   /* legacy: 32, DPDK virt FE: 128. */
 112 #define RTE_TEST_TX_DESC_DEFAULT_ZCP 64   /* legacy: 64, DPDK virt FE: 64.  */
 113
 114 /* Get first 4 bytes in mbuf headroom. */
 115 #define MBUF_HEADROOM_UINT32(mbuf) (*(uint32_t *)((uint8_t *)(mbuf) \
 116                 + sizeof(struct rte_mbuf)))
 117
 118 /* true if x is a power of 2 */
 119 #define POWEROF2(x) ((((x)-1) & (x)) == 0)
 120
 121 #define INVALID_PORT_ID 0xFF
 122
 123 /* Max number of devices. Limited by vmdq. */
 124 #define MAX_DEVICES 64
 125
 126 /* Size of buffers used for snprintfs. */
 127 #define MAX_PRINT_BUFF 6072
 128
 129 /* Maximum character device basename size. */
 130 #define MAX_BASENAME_SZ 10
 131
 132 /* Maximum long option length for option parsing. */
 133 #define MAX_LONG_OPT_SZ 64
 134
 135 /* Used to compare MAC addresses. */
 136 #define MAC_ADDR_CMP 0xFFFFFFFFFFFFULL
 137
 138 /* Number of descriptors per cacheline. */
 139 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 140
 141 #define MBUF_EXT_MEM(mb)   (rte_mbuf_from_indirect(mb) != (mb))
 142
 143 /* mask of enabled ports */
 144 static uint32_t enabled_port_mask = 0;
 145
 146 /* Promiscuous mode */
 147 static uint32_t promiscuous;
 148
 149 /*Number of switching cores enabled*/
 150 static uint32_t num_switching_cores = 0;
 151
 152 /* number of devices/queues to support*/
 153 static uint32_t num_queues = 0;
 154 static uint32_t num_devices;
 155
 156 /*
 157  * Enable zero copy, pkts buffer will directly dma to hw descriptor,
 158  * disabled on default.
 159  */
 160 static uint32_t zero_copy;
 161 static int mergeable;
 162
 163 /* Do vlan strip on host, enabled on default */
 164 static uint32_t vlan_strip = 1;
 165
 166 /* number of descriptors to apply*/
 167 static uint32_t num_rx_descriptor = RTE_TEST_RX_DESC_DEFAULT_ZCP;
 168 static uint32_t num_tx_descriptor = RTE_TEST_TX_DESC_DEFAULT_ZCP;
 169
 170 /* max ring descriptor, ixgbe, i40e, e1000 all are 4096. */
 171 #define MAX_RING_DESC 4096
 172
 173 struct vpool {
 174         struct rte_mempool *pool;
 175         struct rte_ring *ring;
 176         uint32_t buf_size;
 177 } vpool_array[MAX_QUEUES+MAX_QUEUES];
 178
 179 /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */
 180 typedef enum {
 181         VM2VM_DISABLED = 0,
 182         VM2VM_SOFTWARE = 1,
 183         VM2VM_HARDWARE = 2,
 184         VM2VM_LAST
 185 } vm2vm_type;
 186 static vm2vm_type vm2vm_mode = VM2VM_SOFTWARE;
 187
 188 /* The type of host physical address translated from guest physical address. */
 189 typedef enum {
 190         PHYS_ADDR_CONTINUOUS = 0,
 191         PHYS_ADDR_CROSS_SUBREG = 1,
 192         PHYS_ADDR_INVALID = 2,
 193         PHYS_ADDR_LAST
 194 } hpa_type;
 195
 196 /* Enable stats. */
 197 static uint32_t enable_stats = 0;
 198 /* Enable retries on RX. */
 199 static uint32_t enable_retry = 1;
 200 /* Specify timeout (in useconds) between retries on RX. */
 201 static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US;
 202 /* Specify the number of retries on RX. */
 203 static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;
 204
 205 /* Character device basename. Can be set by user. */
 206 static char dev_basename[MAX_BASENAME_SZ] = "vhost-net";
 207
 208 /* empty vmdq configuration structure. Filled in programatically */
 209 static struct rte_eth_conf vmdq_conf_default = {
 210         .rxmode = {
 211                 .mq_mode        = ETH_MQ_RX_VMDQ_ONLY,
 212                 .split_hdr_size = 0,
 213                 .header_split   = 0, /**< Header Split disabled */
 214                 .hw_ip_checksum = 0, /**< IP checksum offload disabled */
 215                 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
 216                 /*
 217                  * It is necessary for 1G NIC such as I350,
 218                  * this fixes bug of ipv4 forwarding in guest can't
 219                  * forward pakets from one virtio dev to another virtio dev.
 220                  */
 221                 .hw_vlan_strip  = 1, /**< VLAN strip enabled. */
 222                 .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
 223                 .hw_strip_crc   = 0, /**< CRC stripped by hardware */
 224         },
 225
 226         .txmode = {
 227                 .mq_mode = ETH_MQ_TX_NONE,
 228         },
 229         .rx_adv_conf = {
 230                 /*
 231                  * should be overridden separately in code with
 232                  * appropriate values
 233                  */
 234                 .vmdq_rx_conf = {
 235                         .nb_queue_pools = ETH_8_POOLS,
 236                         .enable_default_pool = 0,
 237                         .default_pool = 0,
 238                         .nb_pool_maps = 0,
 239                         .pool_map = {{0, 0},},
 240                 },
 241         },
 242 };
 243
 244 static unsigned lcore_ids[RTE_MAX_LCORE];
 245 static uint8_t ports[RTE_MAX_ETHPORTS];
 246 static unsigned num_ports = 0; /**< The number of ports specified in command line */
 247 static uint16_t num_pf_queues, num_vmdq_queues;
 248 static uint16_t vmdq_pool_base, vmdq_queue_base;
 249 static uint16_t queues_per_pool;
 250
 251 static const uint16_t external_pkt_default_vlan_tag = 2000;
 252 const uint16_t vlan_tags[] = {
 253         1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007,
 254         1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015,
 255         1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023,
 256         1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031,
 257         1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039,
 258         1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047,
 259         1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055,
 260         1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063,
 261 };
 262
 263 /* ethernet addresses of ports */
 264 static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
 265
 266 /* heads for the main used and free linked lists for the data path. */
 267 static struct virtio_net_data_ll *ll_root_used = NULL;
 268 static struct virtio_net_data_ll *ll_root_free = NULL;
 269
 270 /* Array of data core structures containing information on individual core linked lists. */
 271 static struct lcore_info lcore_info[RTE_MAX_LCORE];
 272
 273 /* Used for queueing bursts of TX packets. */
 274 struct mbuf_table {
 275         unsigned len;
 276         unsigned txq_id;
 277         struct rte_mbuf *m_table[MAX_PKT_BURST];
 278 };
 279
 280 /* TX queue for each data core. */
 281 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE];
 282
 283 /* TX queue fori each virtio device for zero copy. */
 284 struct mbuf_table tx_queue_zcp[MAX_QUEUES];
 285
 286 /* Vlan header struct used to insert vlan tags on TX. */
 287 struct vlan_ethhdr {
 288         unsigned char   h_dest[ETH_ALEN];
 289         unsigned char   h_source[ETH_ALEN];
 290         __be16          h_vlan_proto;
 291         __be16          h_vlan_TCI;
 292         __be16          h_vlan_encapsulated_proto;
 293 };
 294
 295 /* IPv4 Header */
 296 struct ipv4_hdr {
 297         uint8_t  version_ihl;           /**< version and header length */
 298         uint8_t  type_of_service;       /**< type of service */
 299         uint16_t total_length;          /**< length of packet */
 300         uint16_t packet_id;             /**< packet ID */
 301         uint16_t fragment_offset;       /**< fragmentation offset */
 302         uint8_t  time_to_live;          /**< time to live */
 303         uint8_t  next_proto_id;         /**< protocol ID */
 304         uint16_t hdr_checksum;          /**< header checksum */
 305         uint32_t src_addr;              /**< source address */
 306         uint32_t dst_addr;              /**< destination address */
 307 } __attribute__((__packed__));
 308
 309 /* Header lengths. */
 310 #define VLAN_HLEN       4
 311 #define VLAN_ETH_HLEN   18
 312
 313 /* Per-device statistics struct */
 314 struct device_statistics {
 315         uint64_t tx_total;
 316         rte_atomic64_t rx_total_atomic;
 317         uint64_t rx_total;
 318         uint64_t tx;
 319         rte_atomic64_t rx_atomic;
 320         uint64_t rx;
 321 } __rte_cache_aligned;
 322 struct device_statistics dev_statistics[MAX_DEVICES];
 323
 324 /*
 325  * Builds up the correct configuration for VMDQ VLAN pool map
 326  * according to the pool & queue limits.
 327  */
 328 static inline int
 329 get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices)
 330 {
 331         struct rte_eth_vmdq_rx_conf conf;
 332         struct rte_eth_vmdq_rx_conf *def_conf =
 333                 &vmdq_conf_default.rx_adv_conf.vmdq_rx_conf;
 334         unsigned i;
 335
 336         memset(&conf, 0, sizeof(conf));
 337         conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices;
 338         conf.nb_pool_maps = num_devices;
 339         conf.enable_loop_back = def_conf->enable_loop_back;
 340         conf.rx_mode = def_conf->rx_mode;
 341
 342         for (i = 0; i < conf.nb_pool_maps; i++) {
 343                 conf.pool_map[i].vlan_id = vlan_tags[ i ];
 344                 conf.pool_map[i].pools = (1UL << i);
 345         }
 346
 347         (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf)));
 348         (void)(rte_memcpy(&eth_conf->rx_adv_conf.vmdq_rx_conf, &conf,
 349                    sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf)));
 350         return 0;
 351 }
 352
 353 /*
 354  * Validate the device number according to the max pool number gotten form
 355  * dev_info. If the device number is invalid, give the error message and
 356  * return -1. Each device must have its own pool.
 357  */
 358 static inline int
 359 validate_num_devices(uint32_t max_nb_devices)
 360 {
 361         if (num_devices > max_nb_devices) {
 362                 RTE_LOG(ERR, VHOST_PORT, "invalid number of devices\n");
 363                 return -1;
 364         }
 365         return 0;
 366 }
 367
 368 /*
 369  * Initialises a given port using global settings and with the rx buffers
 370  * coming from the mbuf_pool passed as parameter
 371  */
 372 static inline int
 373 port_init(uint8_t port)
 374 {
 375         struct rte_eth_dev_info dev_info;
 376         struct rte_eth_conf port_conf;
 377         struct rte_eth_rxconf *rxconf;
 378         struct rte_eth_txconf *txconf;
 379         int16_t rx_rings, tx_rings;
 380         uint16_t rx_ring_size, tx_ring_size;
 381         int retval;
 382         uint16_t q;
 383
 384         /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
 385         rte_eth_dev_info_get (port, &dev_info);
 386
 387         if (dev_info.max_rx_queues > MAX_QUEUES) {
 388                 rte_exit(EXIT_FAILURE,
 389                         "please define MAX_QUEUES no less than %u in %s\n",
 390                         dev_info.max_rx_queues, __FILE__);
 391         }
 392
 393         rxconf = &dev_info.default_rxconf;
 394         txconf = &dev_info.default_txconf;
 395         rxconf->rx_drop_en = 1;
 396
 397         /* Enable vlan offload */
 398         txconf->txq_flags &= ~ETH_TXQ_FLAGS_NOVLANOFFL;
 399
 400         /*
 401          * Zero copy defers queue RX/TX start to the time when guest
 402          * finishes its startup and packet buffers from that guest are
 403          * available.
 404          */
 405         if (zero_copy) {
 406                 rxconf->rx_deferred_start = 1;
 407                 rxconf->rx_drop_en = 0;
 408                 txconf->tx_deferred_start = 1;
 409         }
 410
 411         /*configure the number of supported virtio devices based on VMDQ limits */
 412         num_devices = dev_info.max_vmdq_pools;
 413
 414         if (zero_copy) {
 415                 rx_ring_size = num_rx_descriptor;
 416                 tx_ring_size = num_tx_descriptor;
 417                 tx_rings = dev_info.max_tx_queues;
 418         } else {
 419                 rx_ring_size = RTE_TEST_RX_DESC_DEFAULT;
 420                 tx_ring_size = RTE_TEST_TX_DESC_DEFAULT;
 421                 tx_rings = (uint16_t)rte_lcore_count();
 422         }
 423
 424         retval = validate_num_devices(MAX_DEVICES);
 425         if (retval < 0)
 426                 return retval;
 427
 428         /* Get port configuration. */
 429         retval = get_eth_conf(&port_conf, num_devices);
 430         if (retval < 0)
 431                 return retval;
 432         /* NIC queues are divided into pf queues and vmdq queues.  */
 433         num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;
 434         queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;
 435         num_vmdq_queues = num_devices * queues_per_pool;
 436         num_queues = num_pf_queues + num_vmdq_queues;
 437         vmdq_queue_base = dev_info.vmdq_queue_base;
 438         vmdq_pool_base  = dev_info.vmdq_pool_base;
 439         printf("pf queue num: %u, configured vmdq pool num: %u, each vmdq pool has %u queues\n",
 440                 num_pf_queues, num_devices, queues_per_pool);
 441
 442         if (port >= rte_eth_dev_count()) return -1;
 443
 444         rx_rings = (uint16_t)dev_info.max_rx_queues;
 445         /* Configure ethernet device. */
 446         retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
 447         if (retval != 0)
 448                 return retval;
 449
 450         /* Setup the queues. */
 451         for (q = 0; q < rx_rings; q ++) {
 452                 retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
 453                                                 rte_eth_dev_socket_id(port),
 454                                                 rxconf,
 455                                                 vpool_array[q].pool);
 456                 if (retval < 0)
 457                         return retval;
 458         }
 459         for (q = 0; q < tx_rings; q ++) {
 460                 retval = rte_eth_tx_queue_setup(port, q, tx_ring_size,
 461                                                 rte_eth_dev_socket_id(port),
 462                                                 txconf);
 463                 if (retval < 0)
 464                         return retval;
 465         }
 466
 467         /* Start the device. */
 468         retval  = rte_eth_dev_start(port);
 469         if (retval < 0) {
 470                 RTE_LOG(ERR, VHOST_DATA, "Failed to start the device.\n");
 471                 return retval;
 472         }
 473
 474         if (promiscuous)
 475                 rte_eth_promiscuous_enable(port);
 476
 477         rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
 478         RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices);
 479         RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
 480                         " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
 481                         (unsigned)port,
 482                         vmdq_ports_eth_addr[port].addr_bytes[0],
 483                         vmdq_ports_eth_addr[port].addr_bytes[1],
 484                         vmdq_ports_eth_addr[port].addr_bytes[2],
 485                         vmdq_ports_eth_addr[port].addr_bytes[3],
 486                         vmdq_ports_eth_addr[port].addr_bytes[4],
 487                         vmdq_ports_eth_addr[port].addr_bytes[5]);
 488
 489         return 0;
 490 }
 491
 492 /*
 493  * Set character device basename.
 494  */
 495 static int
 496 us_vhost_parse_basename(const char *q_arg)
 497 {
 498         /* parse number string */
 499
 500         if (strnlen(q_arg, MAX_BASENAME_SZ) > MAX_BASENAME_SZ)
 501                 return -1;
 502         else
 503                 snprintf((char*)&dev_basename, MAX_BASENAME_SZ, "%s", q_arg);
 504
 505         return 0;
 506 }
 507
 508 /*
 509  * Parse the portmask provided at run time.
 510  */
 511 static int
 512 parse_portmask(const char *portmask)
 513 {
 514         char *end = NULL;
 515         unsigned long pm;
 516
 517         errno = 0;
 518
 519         /* parse hexadecimal string */
 520         pm = strtoul(portmask, &end, 16);
 521         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 522                 return -1;
 523
 524         if (pm == 0)
 525                 return -1;
 526
 527         return pm;
 528
 529 }
 530
 531 /*
 532  * Parse num options at run time.
 533  */
 534 static int
 535 parse_num_opt(const char *q_arg, uint32_t max_valid_value)
 536 {
 537         char *end = NULL;
 538         unsigned long num;
 539
 540         errno = 0;
 541
 542         /* parse unsigned int string */
 543         num = strtoul(q_arg, &end, 10);
 544         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 545                 return -1;
 546
 547         if (num > max_valid_value)
 548                 return -1;
 549
 550         return num;
 551
 552 }
 553
 554 /*
 555  * Display usage
 556  */
 557 static void
 558 us_vhost_usage(const char *prgname)
 559 {
 560         RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n"
 561         "               --vm2vm [0|1|2]\n"
 562         "               --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n"
 563         "               --dev-basename <name>\n"
 564         "               --nb-devices ND\n"
 565         "               -p PORTMASK: Set mask for ports to be used by application\n"
 566         "               --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n"
 567         "               --rx-retry [0|1]: disable/enable(default) retries on rx. Enable retry if destintation queue is full\n"
 568         "               --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX. This makes effect only if retries on rx enabled\n"
 569         "               --rx-retry-num [0-N]: the number of retries on rx. This makes effect only if retries on rx enabled\n"
 570         "               --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n"
 571         "               --vlan-strip [0|1]: disable/enable(default) RX VLAN strip on host\n"
 572         "               --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
 573         "               --dev-basename: The basename to be used for the character device.\n"
 574         "               --zero-copy [0|1]: disable(default)/enable rx/tx "
 575                         "zero copy\n"
 576         "               --rx-desc-num [0-N]: the number of descriptors on rx, "
 577                         "used only when zero copy is enabled.\n"
 578         "               --tx-desc-num [0-N]: the number of descriptors on tx, "
 579                         "used only when zero copy is enabled.\n",
 580                prgname);
 581 }
 582
 583 /*
 584  * Parse the arguments given in the command line of the application.
 585  */
 586 static int
 587 us_vhost_parse_args(int argc, char **argv)
 588 {
 589         int opt, ret;
 590         int option_index;
 591         unsigned i;
 592         const char *prgname = argv[0];
 593         static struct option long_option[] = {
 594                 {"vm2vm", required_argument, NULL, 0},
 595                 {"rx-retry", required_argument, NULL, 0},
 596                 {"rx-retry-delay", required_argument, NULL, 0},
 597                 {"rx-retry-num", required_argument, NULL, 0},
 598                 {"mergeable", required_argument, NULL, 0},
 599                 {"vlan-strip", required_argument, NULL, 0},
 600                 {"stats", required_argument, NULL, 0},
 601                 {"dev-basename", required_argument, NULL, 0},
 602                 {"zero-copy", required_argument, NULL, 0},
 603                 {"rx-desc-num", required_argument, NULL, 0},
 604                 {"tx-desc-num", required_argument, NULL, 0},
 605                 {NULL, 0, 0, 0},
 606         };
 607
 608         /* Parse command line */
 609         while ((opt = getopt_long(argc, argv, "p:P",
 610                         long_option, &option_index)) != EOF) {
 611                 switch (opt) {
 612                 /* Portmask */
 613                 case 'p':
 614                         enabled_port_mask = parse_portmask(optarg);
 615                         if (enabled_port_mask == 0) {
 616                                 RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n");
 617                                 us_vhost_usage(prgname);
 618                                 return -1;
 619                         }
 620                         break;
 621
 622                 case 'P':
 623                         promiscuous = 1;
 624                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.rx_mode =
 625                                 ETH_VMDQ_ACCEPT_BROADCAST |
 626                                 ETH_VMDQ_ACCEPT_MULTICAST;
 627                         rte_vhost_feature_enable(1ULL << VIRTIO_NET_F_CTRL_RX);
 628
 629                         break;
 630
 631                 case 0:
 632                         /* Enable/disable vm2vm comms. */
 633                         if (!strncmp(long_option[option_index].name, "vm2vm",
 634                                 MAX_LONG_OPT_SZ)) {
 635                                 ret = parse_num_opt(optarg, (VM2VM_LAST - 1));
 636                                 if (ret == -1) {
 637                                         RTE_LOG(INFO, VHOST_CONFIG,
 638                                                 "Invalid argument for "
 639                                                 "vm2vm [0|1|2]\n");
 640                                         us_vhost_usage(prgname);
 641                                         return -1;
 642                                 } else {
 643                                         vm2vm_mode = (vm2vm_type)ret;
 644                                 }
 645                         }
 646
 647                         /* Enable/disable retries on RX. */
 648                         if (!strncmp(long_option[option_index].name, "rx-retry", MAX_LONG_OPT_SZ)) {
 649                                 ret = parse_num_opt(optarg, 1);
 650                                 if (ret == -1) {
 651                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry [0|1]\n");
 652                                         us_vhost_usage(prgname);
 653                                         return -1;
 654                                 } else {
 655                                         enable_retry = ret;
 656                                 }
 657                         }
 658
 659                         /* Specify the retries delay time (in useconds) on RX. */
 660                         if (!strncmp(long_option[option_index].name, "rx-retry-delay", MAX_LONG_OPT_SZ)) {
 661                                 ret = parse_num_opt(optarg, INT32_MAX);
 662                                 if (ret == -1) {
 663                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-delay [0-N]\n");
 664                                         us_vhost_usage(prgname);
 665                                         return -1;
 666                                 } else {
 667                                         burst_rx_delay_time = ret;
 668                                 }
 669                         }
 670
 671                         /* Specify the retries number on RX. */
 672                         if (!strncmp(long_option[option_index].name, "rx-retry-num", MAX_LONG_OPT_SZ)) {
 673                                 ret = parse_num_opt(optarg, INT32_MAX);
 674                                 if (ret == -1) {
 675                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-num [0-N]\n");
 676                                         us_vhost_usage(prgname);
 677                                         return -1;
 678                                 } else {
 679                                         burst_rx_retry_num = ret;
 680                                 }
 681                         }
 682
 683                         /* Enable/disable RX mergeable buffers. */
 684                         if (!strncmp(long_option[option_index].name, "mergeable", MAX_LONG_OPT_SZ)) {
 685                                 ret = parse_num_opt(optarg, 1);
 686                                 if (ret == -1) {
 687                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for mergeable [0|1]\n");
 688                                         us_vhost_usage(prgname);
 689                                         return -1;
 690                                 } else {
 691                                         mergeable = !!ret;
 692                                         if (ret) {
 693                                                 vmdq_conf_default.rxmode.jumbo_frame = 1;
 694                                                 vmdq_conf_default.rxmode.max_rx_pkt_len
 695                                                         = JUMBO_FRAME_MAX_SIZE;
 696                                         }
 697                                 }
 698                         }
 699
 700                         /* Enable/disable RX VLAN strip on host. */
 701                         if (!strncmp(long_option[option_index].name,
 702                                 "vlan-strip", MAX_LONG_OPT_SZ)) {
 703                                 ret = parse_num_opt(optarg, 1);
 704                                 if (ret == -1) {
 705                                         RTE_LOG(INFO, VHOST_CONFIG,
 706                                                 "Invalid argument for VLAN strip [0|1]\n");
 707                                         us_vhost_usage(prgname);
 708                                         return -1;
 709                                 } else {
 710                                         vlan_strip = !!ret;
 711                                         vmdq_conf_default.rxmode.hw_vlan_strip =
 712                                                 vlan_strip;
 713                                 }
 714                         }
 715
 716                         /* Enable/disable stats. */
 717                         if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) {
 718                                 ret = parse_num_opt(optarg, INT32_MAX);
 719                                 if (ret == -1) {
 720                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for stats [0..N]\n");
 721                                         us_vhost_usage(prgname);
 722                                         return -1;
 723                                 } else {
 724                                         enable_stats = ret;
 725                                 }
 726                         }
 727
 728                         /* Set character device basename. */
 729                         if (!strncmp(long_option[option_index].name, "dev-basename", MAX_LONG_OPT_SZ)) {
 730                                 if (us_vhost_parse_basename(optarg) == -1) {
 731                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for character device basename (Max %d characters)\n", MAX_BASENAME_SZ);
 732                                         us_vhost_usage(prgname);
 733                                         return -1;
 734                                 }
 735                         }
 736
 737                         /* Enable/disable rx/tx zero copy. */
 738                         if (!strncmp(long_option[option_index].name,
 739                                 "zero-copy", MAX_LONG_OPT_SZ)) {
 740                                 ret = parse_num_opt(optarg, 1);
 741                                 if (ret == -1) {
 742                                         RTE_LOG(INFO, VHOST_CONFIG,
 743                                                 "Invalid argument"
 744                                                 " for zero-copy [0|1]\n");
 745                                         us_vhost_usage(prgname);
 746                                         return -1;
 747                                 } else
 748                                         zero_copy = ret;
 749                         }
 750
 751                         /* Specify the descriptor number on RX. */
 752                         if (!strncmp(long_option[option_index].name,
 753                                 "rx-desc-num", MAX_LONG_OPT_SZ)) {
 754                                 ret = parse_num_opt(optarg, MAX_RING_DESC);
 755                                 if ((ret == -1) || (!POWEROF2(ret))) {
 756                                         RTE_LOG(INFO, VHOST_CONFIG,
 757                                         "Invalid argument for rx-desc-num[0-N],"
 758                                         "power of 2 required.\n");
 759                                         us_vhost_usage(prgname);
 760                                         return -1;
 761                                 } else {
 762                                         num_rx_descriptor = ret;
 763                                 }
 764                         }
 765
 766                         /* Specify the descriptor number on TX. */
 767                         if (!strncmp(long_option[option_index].name,
 768                                 "tx-desc-num", MAX_LONG_OPT_SZ)) {
 769                                 ret = parse_num_opt(optarg, MAX_RING_DESC);
 770                                 if ((ret == -1) || (!POWEROF2(ret))) {
 771                                         RTE_LOG(INFO, VHOST_CONFIG,
 772                                         "Invalid argument for tx-desc-num [0-N],"
 773                                         "power of 2 required.\n");
 774                                         us_vhost_usage(prgname);
 775                                         return -1;
 776                                 } else {
 777                                         num_tx_descriptor = ret;
 778                                 }
 779                         }
 780
 781                         break;
 782
 783                         /* Invalid option - print options. */
 784                 default:
 785                         us_vhost_usage(prgname);
 786                         return -1;
 787                 }
 788         }
 789
 790         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 791                 if (enabled_port_mask & (1 << i))
 792                         ports[num_ports++] = (uint8_t)i;
 793         }
 794
 795         if ((num_ports ==  0) || (num_ports > MAX_SUP_PORTS)) {
 796                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
 797                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
 798                 return -1;
 799         }
 800
 801         if ((zero_copy == 1) && (vm2vm_mode == VM2VM_SOFTWARE)) {
 802                 RTE_LOG(INFO, VHOST_PORT,
 803                         "Vhost zero copy doesn't support software vm2vm,"
 804                         "please specify 'vm2vm 2' to use hardware vm2vm.\n");
 805                 return -1;
 806         }
 807
 808         if ((zero_copy == 1) && (vmdq_conf_default.rxmode.jumbo_frame == 1)) {
 809                 RTE_LOG(INFO, VHOST_PORT,
 810                         "Vhost zero copy doesn't support jumbo frame,"
 811                         "please specify '--mergeable 0' to disable the "
 812                         "mergeable feature.\n");
 813                 return -1;
 814         }
 815
 816         return 0;
 817 }
 818
 819 /*
 820  * Update the global var NUM_PORTS and array PORTS according to system ports number
 821  * and return valid ports number
 822  */
 823 static unsigned check_ports_num(unsigned nb_ports)
 824 {
 825         unsigned valid_num_ports = num_ports;
 826         unsigned portid;
 827
 828         if (num_ports > nb_ports) {
 829                 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n",
 830                         num_ports, nb_ports);
 831                 num_ports = nb_ports;
 832         }
 833
 834         for (portid = 0; portid < num_ports; portid ++) {
 835                 if (ports[portid] >= nb_ports) {
 836                         RTE_LOG(INFO, VHOST_PORT, "\nSpecified port ID(%u) exceeds max system port ID(%u)\n",
 837                                 ports[portid], (nb_ports - 1));
 838                         ports[portid] = INVALID_PORT_ID;
 839                         valid_num_ports--;
 840                 }
 841         }
 842         return valid_num_ports;
 843 }
 844
 845 /*
 846  * Macro to print out packet contents. Wrapped in debug define so that the
 847  * data path is not effected when debug is disabled.
 848  */
 849 #ifdef DEBUG
 850 #define PRINT_PACKET(device, addr, size, header) do {                                                                                                                           \
 851         char *pkt_addr = (char*)(addr);                                                                                                                                                                 \
 852         unsigned int index;                                                                                                                                                                                             \
 853         char packet[MAX_PRINT_BUFF];                                                                                                                                                                    \
 854                                                                                                                                                                                                                                         \
 855         if ((header))                                                                                                                                                                                                   \
 856                 snprintf(packet, MAX_PRINT_BUFF, "(%"PRIu64") Header size %d: ", (device->device_fh), (size));                          \
 857         else                                                                                                                                                                                                                    \
 858                 snprintf(packet, MAX_PRINT_BUFF, "(%"PRIu64") Packet size %d: ", (device->device_fh), (size));                          \
 859         for (index = 0; index < (size); index++) {                                                                                                                                              \
 860                 snprintf(packet + strnlen(packet, MAX_PRINT_BUFF), MAX_PRINT_BUFF - strnlen(packet, MAX_PRINT_BUFF),    \
 861                         "%02hhx ", pkt_addr[index]);                                                                                                                                                    \
 862         }                                                                                                                                                                                                                               \
 863         snprintf(packet + strnlen(packet, MAX_PRINT_BUFF), MAX_PRINT_BUFF - strnlen(packet, MAX_PRINT_BUFF), "\n");     \
 864                                                                                                                                                                                                                                         \
 865         LOG_DEBUG(VHOST_DATA, "%s", packet);                                                                                                                                                                    \
 866 } while(0)
 867 #else
 868 #define PRINT_PACKET(device, addr, size, header) do{} while(0)
 869 #endif
 870
 871 /*
 872  * Function to convert guest physical addresses to vhost physical addresses.
 873  * This is used to convert virtio buffer addresses.
 874  */
 875 static inline uint64_t __attribute__((always_inline))
 876 gpa_to_hpa(struct vhost_dev  *vdev, uint64_t guest_pa,
 877         uint32_t buf_len, hpa_type *addr_type)
 878 {
 879         struct virtio_memory_regions_hpa *region;
 880         uint32_t regionidx;
 881         uint64_t vhost_pa = 0;
 882
 883         *addr_type = PHYS_ADDR_INVALID;
 884
 885         for (regionidx = 0; regionidx < vdev->nregions_hpa; regionidx++) {
 886                 region = &vdev->regions_hpa[regionidx];
 887                 if ((guest_pa >= region->guest_phys_address) &&
 888                         (guest_pa <= region->guest_phys_address_end)) {
 889                         vhost_pa = region->host_phys_addr_offset + guest_pa;
 890                         if (likely((guest_pa + buf_len - 1)
 891                                 <= region->guest_phys_address_end))
 892                                 *addr_type = PHYS_ADDR_CONTINUOUS;
 893                         else
 894                                 *addr_type = PHYS_ADDR_CROSS_SUBREG;
 895                         break;
 896                 }
 897         }
 898
 899         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") GPA %p| HPA %p\n",
 900                 vdev->dev->device_fh, (void *)(uintptr_t)guest_pa,
 901                 (void *)(uintptr_t)vhost_pa);
 902
 903         return vhost_pa;
 904 }
 905
 906 /*
 907  * Compares a packet destination MAC address to a device MAC address.
 908  */
 909 static inline int __attribute__((always_inline))
 910 ether_addr_cmp(struct ether_addr *ea, struct ether_addr *eb)
 911 {
 912         return (((*(uint64_t *)ea ^ *(uint64_t *)eb) & MAC_ADDR_CMP) == 0);
 913 }
 914
 915 /*
 916  * This function learns the MAC address of the device and registers this along with a
 917  * vlan tag to a VMDQ.
 918  */
 919 static int
 920 link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
 921 {
 922         struct ether_hdr *pkt_hdr;
 923         struct virtio_net_data_ll *dev_ll;
 924         struct virtio_net *dev = vdev->dev;
 925         int i, ret;
 926
 927         /* Learn MAC address of guest device from packet */
 928         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 929
 930         dev_ll = ll_root_used;
 931
 932         while (dev_ll != NULL) {
 933                 if (ether_addr_cmp(&(pkt_hdr->s_addr), &dev_ll->vdev->mac_address)) {
 934                         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") WARNING: This device is using an existing MAC address and has not been registered.\n", dev->device_fh);
 935                         return -1;
 936                 }
 937                 dev_ll = dev_ll->next;
 938         }
 939
 940         for (i = 0; i < ETHER_ADDR_LEN; i++)
 941                 vdev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i];
 942
 943         /* vlan_tag currently uses the device_id. */
 944         vdev->vlan_tag = vlan_tags[dev->device_fh];
 945
 946         /* Print out VMDQ registration info. */
 947         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") MAC_ADDRESS %02x:%02x:%02x:%02x:%02x:%02x and VLAN_TAG %d registered\n",
 948                 dev->device_fh,
 949                 vdev->mac_address.addr_bytes[0], vdev->mac_address.addr_bytes[1],
 950                 vdev->mac_address.addr_bytes[2], vdev->mac_address.addr_bytes[3],
 951                 vdev->mac_address.addr_bytes[4], vdev->mac_address.addr_bytes[5],
 952                 vdev->vlan_tag);
 953
 954         /* Register the MAC address. */
 955         ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address,
 956                                 (uint32_t)dev->device_fh + vmdq_pool_base);
 957         if (ret)
 958                 RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Failed to add device MAC address to VMDQ\n",
 959                                         dev->device_fh);
 960
 961         /* Enable stripping of the vlan tag as we handle routing. */
 962         if (vlan_strip)
 963                 rte_eth_dev_set_vlan_strip_on_queue(ports[0],
 964                         (uint16_t)vdev->vmdq_rx_q, 1);
 965
 966         /* Set device as ready for RX. */
 967         vdev->ready = DEVICE_RX;
 968
 969         return 0;
 970 }
 971
 972 /*
 973  * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX
 974  * queue before disabling RX on the device.
 975  */
 976 static inline void
 977 unlink_vmdq(struct vhost_dev *vdev)
 978 {
 979         unsigned i = 0;
 980         unsigned rx_count;
 981         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 982
 983         if (vdev->ready == DEVICE_RX) {
 984                 /*clear MAC and VLAN settings*/
 985                 rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address);
 986                 for (i = 0; i < 6; i++)
 987                         vdev->mac_address.addr_bytes[i] = 0;
 988
 989                 vdev->vlan_tag = 0;
 990
 991                 /*Clear out the receive buffers*/
 992                 rx_count = rte_eth_rx_burst(ports[0],
 993                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
 994
 995                 while (rx_count) {
 996                         for (i = 0; i < rx_count; i++)
 997                                 rte_pktmbuf_free(pkts_burst[i]);
 998
 999                         rx_count = rte_eth_rx_burst(ports[0],
1000                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
1001                 }
1002
1003                 vdev->ready = DEVICE_MAC_LEARNING;
1004         }
1005 }
1006
1007 /*
1008  * Check if the packet destination MAC address is for a local device. If so then put
1009  * the packet on that devices RX queue. If not then return.
1010  */
1011 static inline int __attribute__((always_inline))
1012 virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
1013 {
1014         struct virtio_net_data_ll *dev_ll;
1015         struct ether_hdr *pkt_hdr;
1016         uint64_t ret = 0;
1017         struct virtio_net *dev = vdev->dev;
1018         struct virtio_net *tdev; /* destination virito device */
1019
1020         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1021
1022         /*get the used devices list*/
1023         dev_ll = ll_root_used;
1024
1025         while (dev_ll != NULL) {
1026                 if ((dev_ll->vdev->ready == DEVICE_RX) && ether_addr_cmp(&(pkt_hdr->d_addr),
1027                                           &dev_ll->vdev->mac_address)) {
1028
1029                         /* Drop the packet if the TX packet is destined for the TX device. */
1030                         if (dev_ll->vdev->dev->device_fh == dev->device_fh) {
1031                                 LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: Source and destination MAC addresses are the same. Dropping packet.\n",
1032                                                         dev->device_fh);
1033                                 return 0;
1034                         }
1035                         tdev = dev_ll->vdev->dev;
1036
1037
1038                         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: MAC address is local\n", tdev->device_fh);
1039
1040                         if (unlikely(dev_ll->vdev->remove)) {
1041                                 /*drop the packet if the device is marked for removal*/
1042                                 LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Device is marked for removal\n", tdev->device_fh);
1043                         } else {
1044                                 /*send the packet to the local virtio device*/
1045                                 ret = rte_vhost_enqueue_burst(tdev, VIRTIO_RXQ, &m, 1);
1046                                 if (enable_stats) {
1047                                         rte_atomic64_add(
1048                                         &dev_statistics[tdev->device_fh].rx_total_atomic,
1049                                         1);
1050                                         rte_atomic64_add(
1051                                         &dev_statistics[tdev->device_fh].rx_atomic,
1052                                         ret);
1053                                         dev_statistics[tdev->device_fh].tx_total++;
1054                                         dev_statistics[tdev->device_fh].tx += ret;
1055                                 }
1056                         }
1057
1058                         return 0;
1059                 }
1060                 dev_ll = dev_ll->next;
1061         }
1062
1063         return -1;
1064 }
1065
1066 /*
1067  * Check if the destination MAC of a packet is one local VM,
1068  * and get its vlan tag, and offset if it is.
1069  */
1070 static inline int __attribute__((always_inline))
1071 find_local_dest(struct virtio_net *dev, struct rte_mbuf *m,
1072         uint32_t *offset, uint16_t *vlan_tag)
1073 {
1074         struct virtio_net_data_ll *dev_ll = ll_root_used;
1075         struct ether_hdr *pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1076
1077         while (dev_ll != NULL) {
1078                 if ((dev_ll->vdev->ready == DEVICE_RX)
1079                         && ether_addr_cmp(&(pkt_hdr->d_addr),
1080                 &dev_ll->vdev->mac_address)) {
1081                         /*
1082                          * Drop the packet if the TX packet is
1083                          * destined for the TX device.
1084                          */
1085                         if (dev_ll->vdev->dev->device_fh == dev->device_fh) {
1086                                 LOG_DEBUG(VHOST_DATA,
1087                                 "(%"PRIu64") TX: Source and destination"
1088                                 " MAC addresses are the same. Dropping "
1089                                 "packet.\n",
1090                                 dev_ll->vdev->dev->device_fh);
1091                                 return -1;
1092                         }
1093
1094                         /*
1095                          * HW vlan strip will reduce the packet length
1096                          * by minus length of vlan tag, so need restore
1097                          * the packet length by plus it.
1098                          */
1099                         *offset = VLAN_HLEN;
1100                         *vlan_tag =
1101                         (uint16_t)
1102                         vlan_tags[(uint16_t)dev_ll->vdev->dev->device_fh];
1103
1104                         LOG_DEBUG(VHOST_DATA,
1105                         "(%"PRIu64") TX: pkt to local VM device id:"
1106                         "(%"PRIu64") vlan tag: %d.\n",
1107                         dev->device_fh, dev_ll->vdev->dev->device_fh,
1108                         (int)*vlan_tag);
1109
1110                         break;
1111                 }
1112                 dev_ll = dev_ll->next;
1113         }
1114         return 0;
1115 }
1116
1117 /*
1118  * This function routes the TX packet to the correct interface. This may be a local device
1119  * or the physical port.
1120  */
1121 static inline void __attribute__((always_inline))
1122 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)
1123 {
1124         struct mbuf_table *tx_q;
1125         struct rte_mbuf **m_table;
1126         unsigned len, ret, offset = 0;
1127         const uint16_t lcore_id = rte_lcore_id();
1128         struct virtio_net *dev = vdev->dev;
1129         struct ether_hdr *nh;
1130
1131         /*check if destination is local VM*/
1132         if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) {
1133                 rte_pktmbuf_free(m);
1134                 return;
1135         }
1136
1137         if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
1138                 if (unlikely(find_local_dest(dev, m, &offset, &vlan_tag) != 0)) {
1139                         rte_pktmbuf_free(m);
1140                         return;
1141                 }
1142         }
1143
1144         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: MAC address is external\n", dev->device_fh);
1145
1146         /*Add packet to the port tx queue*/
1147         tx_q = &lcore_tx_queue[lcore_id];
1148         len = tx_q->len;
1149
1150         nh = rte_pktmbuf_mtod(m, struct ether_hdr *);
1151         if (unlikely(nh->ether_type == rte_cpu_to_be_16(ETHER_TYPE_VLAN))) {
1152                 /* Guest has inserted the vlan tag. */
1153                 struct vlan_hdr *vh = (struct vlan_hdr *) (nh + 1);
1154                 uint16_t vlan_tag_be = rte_cpu_to_be_16(vlan_tag);
1155                 if ((vm2vm_mode == VM2VM_HARDWARE) &&
1156                         (vh->vlan_tci != vlan_tag_be))
1157                         vh->vlan_tci = vlan_tag_be;
1158         } else {
1159                 m->ol_flags = PKT_TX_VLAN_PKT;
1160
1161                 /*
1162                  * Find the right seg to adjust the data len when offset is
1163                  * bigger than tail room size.
1164                  */
1165                 if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
1166                         if (likely(offset <= rte_pktmbuf_tailroom(m)))
1167                                 m->data_len += offset;
1168                         else {
1169                                 struct rte_mbuf *seg = m;
1170
1171                                 while ((seg->next != NULL) &&
1172                                         (offset > rte_pktmbuf_tailroom(seg)))
1173                                         seg = seg->next;
1174
1175                                 seg->data_len += offset;
1176                         }
1177                         m->pkt_len += offset;
1178                 }
1179
1180                 m->vlan_tci = vlan_tag;
1181         }
1182
1183         tx_q->m_table[len] = m;
1184         len++;
1185         if (enable_stats) {
1186                 dev_statistics[dev->device_fh].tx_total++;
1187                 dev_statistics[dev->device_fh].tx++;
1188         }
1189
1190         if (unlikely(len == MAX_PKT_BURST)) {
1191                 m_table = (struct rte_mbuf **)tx_q->m_table;
1192                 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id, m_table, (uint16_t) len);
1193                 /* Free any buffers not handled by TX and update the port stats. */
1194                 if (unlikely(ret < len)) {
1195                         do {
1196                                 rte_pktmbuf_free(m_table[ret]);
1197                         } while (++ret < len);
1198                 }
1199
1200                 len = 0;
1201         }
1202
1203         tx_q->len = len;
1204         return;
1205 }
1206 /*
1207  * This function is called by each data core. It handles all RX/TX registered with the
1208  * core. For TX the specific lcore linked list is used. For RX, MAC addresses are compared
1209  * with all devices in the main linked list.
1210  */
1211 static int
1212 switch_worker(__attribute__((unused)) void *arg)
1213 {
1214         struct rte_mempool *mbuf_pool = arg;
1215         struct virtio_net *dev = NULL;
1216         struct vhost_dev *vdev = NULL;
1217         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1218         struct virtio_net_data_ll *dev_ll;
1219         struct mbuf_table *tx_q;
1220         volatile struct lcore_ll_info *lcore_ll;
1221         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US;
1222         uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0;
1223         unsigned ret, i;
1224         const uint16_t lcore_id = rte_lcore_id();
1225         const uint16_t num_cores = (uint16_t)rte_lcore_count();
1226         uint16_t rx_count = 0;
1227         uint16_t tx_count;
1228         uint32_t retry = 0;
1229
1230         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
1231         lcore_ll = lcore_info[lcore_id].lcore_ll;
1232         prev_tsc = 0;
1233
1234         tx_q = &lcore_tx_queue[lcore_id];
1235         for (i = 0; i < num_cores; i ++) {
1236                 if (lcore_ids[i] == lcore_id) {
1237                         tx_q->txq_id = i;
1238                         break;
1239                 }
1240         }
1241
1242         while(1) {
1243                 cur_tsc = rte_rdtsc();
1244                 /*
1245                  * TX burst queue drain
1246                  */
1247                 diff_tsc = cur_tsc - prev_tsc;
1248                 if (unlikely(diff_tsc > drain_tsc)) {
1249
1250                         if (tx_q->len) {
1251                                 LOG_DEBUG(VHOST_DATA, "TX queue drained after timeout with burst size %u \n", tx_q->len);
1252
1253                                 /*Tx any packets in the queue*/
1254                                 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id,
1255                                                                            (struct rte_mbuf **)tx_q->m_table,
1256                                                                            (uint16_t)tx_q->len);
1257                                 if (unlikely(ret < tx_q->len)) {
1258                                         do {
1259                                                 rte_pktmbuf_free(tx_q->m_table[ret]);
1260                                         } while (++ret < tx_q->len);
1261                                 }
1262
1263                                 tx_q->len = 0;
1264                         }
1265
1266                         prev_tsc = cur_tsc;
1267
1268                 }
1269
1270                 rte_prefetch0(lcore_ll->ll_root_used);
1271                 /*
1272                  * Inform the configuration core that we have exited the linked list and that no devices are
1273                  * in use if requested.
1274                  */
1275                 if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL)
1276                         lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
1277
1278                 /*
1279                  * Process devices
1280                  */
1281                 dev_ll = lcore_ll->ll_root_used;
1282
1283                 while (dev_ll != NULL) {
1284                         /*get virtio device ID*/
1285                         vdev = dev_ll->vdev;
1286                         dev = vdev->dev;
1287
1288                         if (unlikely(vdev->remove)) {
1289                                 dev_ll = dev_ll->next;
1290                                 unlink_vmdq(vdev);
1291                                 vdev->ready = DEVICE_SAFE_REMOVE;
1292                                 continue;
1293                         }
1294                         if (likely(vdev->ready == DEVICE_RX)) {
1295                                 /*Handle guest RX*/
1296                                 rx_count = rte_eth_rx_burst(ports[0],
1297                                         vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
1298
1299                                 if (rx_count) {
1300                                         /*
1301                                         * Retry is enabled and the queue is full then we wait and retry to avoid packet loss
1302                                         * Here MAX_PKT_BURST must be less than virtio queue size
1303                                         */
1304                                         if (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev, VIRTIO_RXQ))) {
1305                                                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
1306                                                         rte_delay_us(burst_rx_delay_time);
1307                                                         if (rx_count <= rte_vring_available_entries(dev, VIRTIO_RXQ))
1308                                                                 break;
1309                                                 }
1310                                         }
1311                                         ret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ, pkts_burst, rx_count);
1312                                         if (enable_stats) {
1313                                                 rte_atomic64_add(
1314                                                 &dev_statistics[dev_ll->vdev->dev->device_fh].rx_total_atomic,
1315                                                 rx_count);
1316                                                 rte_atomic64_add(
1317                                                 &dev_statistics[dev_ll->vdev->dev->device_fh].rx_atomic, ret_count);
1318                                         }
1319                                         while (likely(rx_count)) {
1320                                                 rx_count--;
1321                                                 rte_pktmbuf_free(pkts_burst[rx_count]);
1322                                         }
1323
1324                                 }
1325                         }
1326
1327                         if (likely(!vdev->remove)) {
1328                                 /* Handle guest TX*/
1329                                 tx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ, mbuf_pool, pkts_burst, MAX_PKT_BURST);
1330                                 /* If this is the first received packet we need to learn the MAC and setup VMDQ */
1331                                 if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) {
1332                                         if (vdev->remove || (link_vmdq(vdev, pkts_burst[0]) == -1)) {
1333                                                 while (tx_count)
1334                                                         rte_pktmbuf_free(pkts_burst[--tx_count]);
1335                                         }
1336                                 }
1337                                 while (tx_count)
1338                                         virtio_tx_route(vdev, pkts_burst[--tx_count], (uint16_t)dev->device_fh);
1339                         }
1340
1341                         /*move to the next device in the list*/
1342                         dev_ll = dev_ll->next;
1343                 }
1344         }
1345
1346         return 0;
1347 }
1348
1349 /*
1350  * This function gets available ring number for zero copy rx.
1351  * Only one thread will call this funciton for a paticular virtio device,
1352  * so, it is designed as non-thread-safe function.
1353  */
1354 static inline uint32_t __attribute__((always_inline))
1355 get_available_ring_num_zcp(struct virtio_net *dev)
1356 {
1357         struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_RXQ];
1358         uint16_t avail_idx;
1359
1360         avail_idx = *((volatile uint16_t *)&vq->avail->idx);
1361         return (uint32_t)(avail_idx - vq->last_used_idx_res);
1362 }
1363
1364 /*
1365  * This function gets available ring index for zero copy rx,
1366  * it will retry 'burst_rx_retry_num' times till it get enough ring index.
1367  * Only one thread will call this funciton for a paticular virtio device,
1368  * so, it is designed as non-thread-safe function.
1369  */
1370 static inline uint32_t __attribute__((always_inline))
1371 get_available_ring_index_zcp(struct virtio_net *dev,
1372         uint16_t *res_base_idx, uint32_t count)
1373 {
1374         struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_RXQ];
1375         uint16_t avail_idx;
1376         uint32_t retry = 0;
1377         uint16_t free_entries;
1378
1379         *res_base_idx = vq->last_used_idx_res;
1380         avail_idx = *((volatile uint16_t *)&vq->avail->idx);
1381         free_entries = (avail_idx - *res_base_idx);
1382
1383         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") in get_available_ring_index_zcp: "
1384                         "avail idx: %d, "
1385                         "res base idx:%d, free entries:%d\n",
1386                         dev->device_fh, avail_idx, *res_base_idx,
1387                         free_entries);
1388
1389         /*
1390          * If retry is enabled and the queue is full then we wait
1391          * and retry to avoid packet loss.
1392          */
1393         if (enable_retry && unlikely(count > free_entries)) {
1394                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
1395                         rte_delay_us(burst_rx_delay_time);
1396                         avail_idx = *((volatile uint16_t *)&vq->avail->idx);
1397                         free_entries = (avail_idx - *res_base_idx);
1398                         if (count <= free_entries)
1399                                 break;
1400                 }
1401         }
1402
1403         /*check that we have enough buffers*/
1404         if (unlikely(count > free_entries))
1405                 count = free_entries;
1406
1407         if (unlikely(count == 0)) {
1408                 LOG_DEBUG(VHOST_DATA,
1409                         "(%"PRIu64") Fail in get_available_ring_index_zcp: "
1410                         "avail idx: %d, res base idx:%d, free entries:%d\n",
1411                         dev->device_fh, avail_idx,
1412                         *res_base_idx, free_entries);
1413                 return 0;
1414         }
1415
1416         vq->last_used_idx_res = *res_base_idx + count;
1417
1418         return count;
1419 }
1420
1421 /*
1422  * This function put descriptor back to used list.
1423  */
1424 static inline void __attribute__((always_inline))
1425 put_desc_to_used_list_zcp(struct vhost_virtqueue *vq, uint16_t desc_idx)
1426 {
1427         uint16_t res_cur_idx = vq->last_used_idx;
1428         vq->used->ring[res_cur_idx & (vq->size - 1)].id = (uint32_t)desc_idx;
1429         vq->used->ring[res_cur_idx & (vq->size - 1)].len = 0;
1430         rte_compiler_barrier();
1431         *(volatile uint16_t *)&vq->used->idx += 1;
1432         vq->last_used_idx += 1;
1433
1434         /* Kick the guest if necessary. */
1435         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
1436                 eventfd_write((int)vq->callfd, 1);
1437 }
1438
1439 /*
1440  * This function get available descriptor from vitio vring and un-attached mbuf
1441  * from vpool->ring, and then attach them together. It needs adjust the offset
1442  * for buff_addr and phys_addr accroding to PMD implementation, otherwise the
1443  * frame data may be put to wrong location in mbuf.
1444  */
1445 static inline void __attribute__((always_inline))
1446 attach_rxmbuf_zcp(struct virtio_net *dev)
1447 {
1448         uint16_t res_base_idx, desc_idx;
1449         uint64_t buff_addr, phys_addr;
1450         struct vhost_virtqueue *vq;
1451         struct vring_desc *desc;
1452         struct rte_mbuf *mbuf = NULL;
1453         struct vpool *vpool;
1454         hpa_type addr_type;
1455         struct vhost_dev *vdev = (struct vhost_dev *)dev->priv;
1456
1457         vpool = &vpool_array[vdev->vmdq_rx_q];
1458         vq = dev->virtqueue[VIRTIO_RXQ];
1459
1460         do {
1461                 if (unlikely(get_available_ring_index_zcp(vdev->dev, &res_base_idx,
1462                                 1) != 1))
1463                         return;
1464                 desc_idx = vq->avail->ring[(res_base_idx) & (vq->size - 1)];
1465
1466                 desc = &vq->desc[desc_idx];
1467                 if (desc->flags & VRING_DESC_F_NEXT) {
1468                         desc = &vq->desc[desc->next];
1469                         buff_addr = gpa_to_vva(dev, desc->addr);
1470                         phys_addr = gpa_to_hpa(vdev, desc->addr, desc->len,
1471                                         &addr_type);
1472                 } else {
1473                         buff_addr = gpa_to_vva(dev,
1474                                         desc->addr + vq->vhost_hlen);
1475                         phys_addr = gpa_to_hpa(vdev,
1476                                         desc->addr + vq->vhost_hlen,
1477                                         desc->len, &addr_type);
1478                 }
1479
1480                 if (unlikely(addr_type == PHYS_ADDR_INVALID)) {
1481                         RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Invalid frame buffer"
1482                                 " address found when attaching RX frame buffer"
1483                                 " address!\n", dev->device_fh);
1484                         put_desc_to_used_list_zcp(vq, desc_idx);
1485                         continue;
1486                 }
1487
1488                 /*
1489                  * Check if the frame buffer address from guest crosses
1490                  * sub-region or not.
1491                  */
1492                 if (unlikely(addr_type == PHYS_ADDR_CROSS_SUBREG)) {
1493                         RTE_LOG(ERR, VHOST_DATA,
1494                                 "(%"PRIu64") Frame buffer address cross "
1495                                 "sub-regioin found when attaching RX frame "
1496                                 "buffer address!\n",
1497                                 dev->device_fh);
1498                         put_desc_to_used_list_zcp(vq, desc_idx);
1499                         continue;
1500                 }
1501         } while (unlikely(phys_addr == 0));
1502
1503         rte_ring_sc_dequeue(vpool->ring, (void **)&mbuf);
1504         if (unlikely(mbuf == NULL)) {
1505                 LOG_DEBUG(VHOST_DATA,
1506                         "(%"PRIu64") in attach_rxmbuf_zcp: "
1507                         "ring_sc_dequeue fail.\n",
1508                         dev->device_fh);
1509                 put_desc_to_used_list_zcp(vq, desc_idx);
1510                 return;
1511         }
1512
1513         if (unlikely(vpool->buf_size > desc->len)) {
1514                 LOG_DEBUG(VHOST_DATA,
1515                         "(%"PRIu64") in attach_rxmbuf_zcp: frame buffer "
1516                         "length(%d) of descriptor idx: %d less than room "
1517                         "size required: %d\n",
1518                         dev->device_fh, desc->len, desc_idx, vpool->buf_size);
1519                 put_desc_to_used_list_zcp(vq, desc_idx);
1520                 rte_ring_sp_enqueue(vpool->ring, (void *)mbuf);
1521                 return;
1522         }
1523
1524         mbuf->buf_addr = (void *)(uintptr_t)(buff_addr - RTE_PKTMBUF_HEADROOM);
1525         mbuf->data_off = RTE_PKTMBUF_HEADROOM;
1526         mbuf->buf_physaddr = phys_addr - RTE_PKTMBUF_HEADROOM;
1527         mbuf->data_len = desc->len;
1528         MBUF_HEADROOM_UINT32(mbuf) = (uint32_t)desc_idx;
1529
1530         LOG_DEBUG(VHOST_DATA,
1531                 "(%"PRIu64") in attach_rxmbuf_zcp: res base idx:%d, "
1532                 "descriptor idx:%d\n",
1533                 dev->device_fh, res_base_idx, desc_idx);
1534
1535         __rte_mbuf_raw_free(mbuf);
1536
1537         return;
1538 }
1539
1540 /*
1541  * Detach an attched packet mbuf -
1542  *  - restore original mbuf address and length values.
1543  *  - reset pktmbuf data and data_len to their default values.
1544  *  All other fields of the given packet mbuf will be left intact.
1545  *
1546  * @param m
1547  *   The attached packet mbuf.
1548  */
1549 static inline void pktmbuf_detach_zcp(struct rte_mbuf *m)
1550 {
1551         const struct rte_mempool *mp = m->pool;
1552         void *buf = rte_mbuf_to_baddr(m);
1553         uint32_t buf_ofs;
1554         uint32_t buf_len = mp->elt_size - sizeof(*m);
1555         m->buf_physaddr = rte_mempool_virt2phy(mp, m) + sizeof(*m);
1556
1557         m->buf_addr = buf;
1558         m->buf_len = (uint16_t)buf_len;
1559
1560         buf_ofs = (RTE_PKTMBUF_HEADROOM <= m->buf_len) ?
1561                         RTE_PKTMBUF_HEADROOM : m->buf_len;
1562         m->data_off = buf_ofs;
1563
1564         m->data_len = 0;
1565 }
1566
1567 /*
1568  * This function is called after packets have been transimited. It fetchs mbuf
1569  * from vpool->pool, detached it and put into vpool->ring. It also update the
1570  * used index and kick the guest if necessary.
1571  */
1572 static inline uint32_t __attribute__((always_inline))
1573 txmbuf_clean_zcp(struct virtio_net *dev, struct vpool *vpool)
1574 {
1575         struct rte_mbuf *mbuf;
1576         struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_TXQ];
1577         uint32_t used_idx = vq->last_used_idx & (vq->size - 1);
1578         uint32_t index = 0;
1579         uint32_t mbuf_count = rte_mempool_count(vpool->pool);
1580
1581         LOG_DEBUG(VHOST_DATA,
1582                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in mempool before "
1583                 "clean is: %d\n",
1584                 dev->device_fh, mbuf_count);
1585         LOG_DEBUG(VHOST_DATA,
1586                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in  ring before "
1587                 "clean  is : %d\n",
1588                 dev->device_fh, rte_ring_count(vpool->ring));
1589
1590         for (index = 0; index < mbuf_count; index++) {
1591                 mbuf = __rte_mbuf_raw_alloc(vpool->pool);
1592                 if (likely(MBUF_EXT_MEM(mbuf)))
1593                         pktmbuf_detach_zcp(mbuf);
1594                 rte_ring_sp_enqueue(vpool->ring, mbuf);
1595
1596                 /* Update used index buffer information. */
1597                 vq->used->ring[used_idx].id = MBUF_HEADROOM_UINT32(mbuf);
1598                 vq->used->ring[used_idx].len = 0;
1599
1600                 used_idx = (used_idx + 1) & (vq->size - 1);
1601         }
1602
1603         LOG_DEBUG(VHOST_DATA,
1604                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in mempool after "
1605                 "clean is: %d\n",
1606                 dev->device_fh, rte_mempool_count(vpool->pool));
1607         LOG_DEBUG(VHOST_DATA,
1608                 "(%"PRIu64") in txmbuf_clean_zcp: mbuf count in  ring after "
1609                 "clean  is : %d\n",
1610                 dev->device_fh, rte_ring_count(vpool->ring));
1611         LOG_DEBUG(VHOST_DATA,
1612                 "(%"PRIu64") in txmbuf_clean_zcp: before updated "
1613                 "vq->last_used_idx:%d\n",
1614                 dev->device_fh, vq->last_used_idx);
1615
1616         vq->last_used_idx += mbuf_count;
1617
1618         LOG_DEBUG(VHOST_DATA,
1619                 "(%"PRIu64") in txmbuf_clean_zcp: after updated "
1620                 "vq->last_used_idx:%d\n",
1621                 dev->device_fh, vq->last_used_idx);
1622
1623         rte_compiler_barrier();
1624
1625         *(volatile uint16_t *)&vq->used->idx += mbuf_count;
1626
1627         /* Kick guest if required. */
1628         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
1629                 eventfd_write((int)vq->callfd, 1);
1630
1631         return 0;
1632 }
1633
1634 /*
1635  * This function is called when a virtio device is destroy.
1636  * It fetchs mbuf from vpool->pool, and detached it, and put into vpool->ring.
1637  */
1638 static void mbuf_destroy_zcp(struct vpool *vpool)
1639 {
1640         struct rte_mbuf *mbuf = NULL;
1641         uint32_t index, mbuf_count = rte_mempool_count(vpool->pool);
1642
1643         LOG_DEBUG(VHOST_CONFIG,
1644                 "in mbuf_destroy_zcp: mbuf count in mempool before "
1645                 "mbuf_destroy_zcp is: %d\n",
1646                 mbuf_count);
1647         LOG_DEBUG(VHOST_CONFIG,
1648                 "in mbuf_destroy_zcp: mbuf count in  ring before "
1649                 "mbuf_destroy_zcp  is : %d\n",
1650                 rte_ring_count(vpool->ring));
1651
1652         for (index = 0; index < mbuf_count; index++) {
1653                 mbuf = __rte_mbuf_raw_alloc(vpool->pool);
1654                 if (likely(mbuf != NULL)) {
1655                         if (likely(MBUF_EXT_MEM(mbuf)))
1656                                 pktmbuf_detach_zcp(mbuf);
1657                         rte_ring_sp_enqueue(vpool->ring, (void *)mbuf);
1658                 }
1659         }
1660
1661         LOG_DEBUG(VHOST_CONFIG,
1662                 "in mbuf_destroy_zcp: mbuf count in mempool after "
1663                 "mbuf_destroy_zcp is: %d\n",
1664                 rte_mempool_count(vpool->pool));
1665         LOG_DEBUG(VHOST_CONFIG,
1666                 "in mbuf_destroy_zcp: mbuf count in ring after "
1667                 "mbuf_destroy_zcp is : %d\n",
1668                 rte_ring_count(vpool->ring));
1669 }
1670
1671 /*
1672  * This function update the use flag and counter.
1673  */
1674 static inline uint32_t __attribute__((always_inline))
1675 virtio_dev_rx_zcp(struct virtio_net *dev, struct rte_mbuf **pkts,
1676         uint32_t count)
1677 {
1678         struct vhost_virtqueue *vq;
1679         struct vring_desc *desc;
1680         struct rte_mbuf *buff;
1681         /* The virtio_hdr is initialised to 0. */
1682         struct virtio_net_hdr_mrg_rxbuf virtio_hdr
1683                 = {{0, 0, 0, 0, 0, 0}, 0};
1684         uint64_t buff_hdr_addr = 0;
1685         uint32_t head[MAX_PKT_BURST], packet_len = 0;
1686         uint32_t head_idx, packet_success = 0;
1687         uint16_t res_cur_idx;
1688
1689         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_rx()\n", dev->device_fh);
1690
1691         if (count == 0)
1692                 return 0;
1693
1694         vq = dev->virtqueue[VIRTIO_RXQ];
1695         count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
1696
1697         res_cur_idx = vq->last_used_idx;
1698         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Current Index %d| End Index %d\n",
1699                 dev->device_fh, res_cur_idx, res_cur_idx + count);
1700
1701         /* Retrieve all of the head indexes first to avoid caching issues. */
1702         for (head_idx = 0; head_idx < count; head_idx++)
1703                 head[head_idx] = MBUF_HEADROOM_UINT32(pkts[head_idx]);
1704
1705         /*Prefetch descriptor index. */
1706         rte_prefetch0(&vq->desc[head[packet_success]]);
1707
1708         while (packet_success != count) {
1709                 /* Get descriptor from available ring */
1710                 desc = &vq->desc[head[packet_success]];
1711
1712                 buff = pkts[packet_success];
1713                 LOG_DEBUG(VHOST_DATA,
1714                         "(%"PRIu64") in dev_rx_zcp: update the used idx for "
1715                         "pkt[%d] descriptor idx: %d\n",
1716                         dev->device_fh, packet_success,
1717                         MBUF_HEADROOM_UINT32(buff));
1718
1719                 PRINT_PACKET(dev,
1720                         (uintptr_t)(((uint64_t)(uintptr_t)buff->buf_addr)
1721                         + RTE_PKTMBUF_HEADROOM),
1722                         rte_pktmbuf_data_len(buff), 0);
1723
1724                 /* Buffer address translation for virtio header. */
1725                 buff_hdr_addr = gpa_to_vva(dev, desc->addr);
1726                 packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen;
1727
1728                 /*
1729                  * If the descriptors are chained the header and data are
1730                  * placed in separate buffers.
1731                  */
1732                 if (desc->flags & VRING_DESC_F_NEXT) {
1733                         desc->len = vq->vhost_hlen;
1734                         desc = &vq->desc[desc->next];
1735                         desc->len = rte_pktmbuf_data_len(buff);
1736                 } else {
1737                         desc->len = packet_len;
1738                 }
1739
1740                 /* Update used ring with desc information */
1741                 vq->used->ring[res_cur_idx & (vq->size - 1)].id
1742                         = head[packet_success];
1743                 vq->used->ring[res_cur_idx & (vq->size - 1)].len
1744                         = packet_len;
1745                 res_cur_idx++;
1746                 packet_success++;
1747
1748                 /* A header is required per buffer. */
1749                 rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
1750                         (const void *)&virtio_hdr, vq->vhost_hlen);
1751
1752                 PRINT_PACKET(dev, (uintptr_t)buff_hdr_addr, vq->vhost_hlen, 1);
1753
1754                 if (likely(packet_success < count)) {
1755                         /* Prefetch descriptor index. */
1756                         rte_prefetch0(&vq->desc[head[packet_success]]);
1757                 }
1758         }
1759
1760         rte_compiler_barrier();
1761
1762         LOG_DEBUG(VHOST_DATA,
1763                 "(%"PRIu64") in dev_rx_zcp: before update used idx: "
1764                 "vq.last_used_idx: %d, vq->used->idx: %d\n",
1765                 dev->device_fh, vq->last_used_idx, vq->used->idx);
1766
1767         *(volatile uint16_t *)&vq->used->idx += count;
1768         vq->last_used_idx += count;
1769
1770         LOG_DEBUG(VHOST_DATA,
1771                 "(%"PRIu64") in dev_rx_zcp: after  update used idx: "
1772                 "vq.last_used_idx: %d, vq->used->idx: %d\n",
1773                 dev->device_fh, vq->last_used_idx, vq->used->idx);
1774
1775         /* Kick the guest if necessary. */
1776         if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
1777                 eventfd_write((int)vq->callfd, 1);
1778
1779         return count;
1780 }
1781
1782 /*
1783  * This function routes the TX packet to the correct interface.
1784  * This may be a local device or the physical port.
1785  */
1786 static inline void __attribute__((always_inline))
1787 virtio_tx_route_zcp(struct virtio_net *dev, struct rte_mbuf *m,
1788         uint32_t desc_idx, uint8_t need_copy)
1789 {
1790         struct mbuf_table *tx_q;
1791         struct rte_mbuf **m_table;
1792         struct rte_mbuf *mbuf = NULL;
1793         unsigned len, ret, offset = 0;
1794         struct vpool *vpool;
1795         uint16_t vlan_tag = (uint16_t)vlan_tags[(uint16_t)dev->device_fh];
1796         uint16_t vmdq_rx_q = ((struct vhost_dev *)dev->priv)->vmdq_rx_q;
1797
1798         /*Add packet to the port tx queue*/
1799         tx_q = &tx_queue_zcp[vmdq_rx_q];
1800         len = tx_q->len;
1801
1802         /* Allocate an mbuf and populate the structure. */
1803         vpool = &vpool_array[MAX_QUEUES + vmdq_rx_q];
1804         rte_ring_sc_dequeue(vpool->ring, (void **)&mbuf);
1805         if (unlikely(mbuf == NULL)) {
1806                 struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_TXQ];
1807                 RTE_LOG(ERR, VHOST_DATA,
1808                         "(%"PRIu64") Failed to allocate memory for mbuf.\n",
1809                         dev->device_fh);
1810                 put_desc_to_used_list_zcp(vq, desc_idx);
1811                 return;
1812         }
1813
1814         if (vm2vm_mode == VM2VM_HARDWARE) {
1815                 /* Avoid using a vlan tag from any vm for external pkt, such as
1816                  * vlan_tags[dev->device_fh], oterwise, it conflicts when pool
1817                  * selection, MAC address determines it as an external pkt
1818                  * which should go to network, while vlan tag determine it as
1819                  * a vm2vm pkt should forward to another vm. Hardware confuse
1820                  * such a ambiguous situation, so pkt will lost.
1821                  */
1822                 vlan_tag = external_pkt_default_vlan_tag;
1823                 if (find_local_dest(dev, m, &offset, &vlan_tag) != 0) {
1824                         MBUF_HEADROOM_UINT32(mbuf) = (uint32_t)desc_idx;
1825                         __rte_mbuf_raw_free(mbuf);
1826                         return;
1827                 }
1828         }
1829
1830         mbuf->nb_segs = m->nb_segs;
1831         mbuf->next = m->next;
1832         mbuf->data_len = m->data_len + offset;
1833         mbuf->pkt_len = mbuf->data_len;
1834         if (unlikely(need_copy)) {
1835                 /* Copy the packet contents to the mbuf. */
1836                 rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
1837                         rte_pktmbuf_mtod(m, void *),
1838                         m->data_len);
1839         } else {
1840                 mbuf->data_off = m->data_off;
1841                 mbuf->buf_physaddr = m->buf_physaddr;
1842                 mbuf->buf_addr = m->buf_addr;
1843         }
1844         mbuf->ol_flags = PKT_TX_VLAN_PKT;
1845         mbuf->vlan_tci = vlan_tag;
1846         mbuf->l2_len = sizeof(struct ether_hdr);
1847         mbuf->l3_len = sizeof(struct ipv4_hdr);
1848         MBUF_HEADROOM_UINT32(mbuf) = (uint32_t)desc_idx;
1849
1850         tx_q->m_table[len] = mbuf;
1851         len++;
1852
1853         LOG_DEBUG(VHOST_DATA,
1854                 "(%"PRIu64") in tx_route_zcp: pkt: nb_seg: %d, next:%s\n",
1855                 dev->device_fh,
1856                 mbuf->nb_segs,
1857                 (mbuf->next == NULL) ? "null" : "non-null");
1858
1859         if (enable_stats) {
1860                 dev_statistics[dev->device_fh].tx_total++;
1861                 dev_statistics[dev->device_fh].tx++;
1862         }
1863
1864         if (unlikely(len == MAX_PKT_BURST)) {
1865                 m_table = (struct rte_mbuf **)tx_q->m_table;
1866                 ret = rte_eth_tx_burst(ports[0],
1867                         (uint16_t)tx_q->txq_id, m_table, (uint16_t) len);
1868
1869                 /*
1870                  * Free any buffers not handled by TX and update
1871                  * the port stats.
1872                  */
1873                 if (unlikely(ret < len)) {
1874                         do {
1875                                 rte_pktmbuf_free(m_table[ret]);
1876                         } while (++ret < len);
1877                 }
1878
1879                 len = 0;
1880                 txmbuf_clean_zcp(dev, vpool);
1881         }
1882
1883         tx_q->len = len;
1884
1885         return;
1886 }
1887
1888 /*
1889  * This function TX all available packets in virtio TX queue for one
1890  * virtio-net device. If it is first packet, it learns MAC address and
1891  * setup VMDQ.
1892  */
1893 static inline void __attribute__((always_inline))
1894 virtio_dev_tx_zcp(struct virtio_net *dev)
1895 {
1896         struct rte_mbuf m;
1897         struct vhost_virtqueue *vq;
1898         struct vring_desc *desc;
1899         uint64_t buff_addr = 0, phys_addr;
1900         uint32_t head[MAX_PKT_BURST];
1901         uint32_t i;
1902         uint16_t free_entries, packet_success = 0;
1903         uint16_t avail_idx;
1904         uint8_t need_copy = 0;
1905         hpa_type addr_type;
1906         struct vhost_dev *vdev = (struct vhost_dev *)dev->priv;
1907
1908         vq = dev->virtqueue[VIRTIO_TXQ];
1909         avail_idx =  *((volatile uint16_t *)&vq->avail->idx);
1910
1911         /* If there are no available buffers then return. */
1912         if (vq->last_used_idx_res == avail_idx)
1913                 return;
1914
1915         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") virtio_dev_tx()\n", dev->device_fh);
1916
1917         /* Prefetch available ring to retrieve head indexes. */
1918         rte_prefetch0(&vq->avail->ring[vq->last_used_idx_res & (vq->size - 1)]);
1919
1920         /* Get the number of free entries in the ring */
1921         free_entries = (avail_idx - vq->last_used_idx_res);
1922
1923         /* Limit to MAX_PKT_BURST. */
1924         free_entries
1925                 = (free_entries > MAX_PKT_BURST) ? MAX_PKT_BURST : free_entries;
1926
1927         LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Buffers available %d\n",
1928                 dev->device_fh, free_entries);
1929
1930         /* Retrieve all of the head indexes first to avoid caching issues. */
1931         for (i = 0; i < free_entries; i++)
1932                 head[i]
1933                         = vq->avail->ring[(vq->last_used_idx_res + i)
1934                         & (vq->size - 1)];
1935
1936         vq->last_used_idx_res += free_entries;
1937
1938         /* Prefetch descriptor index. */
1939         rte_prefetch0(&vq->desc[head[packet_success]]);
1940         rte_prefetch0(&vq->used->ring[vq->last_used_idx & (vq->size - 1)]);
1941
1942         while (packet_success < free_entries) {
1943                 desc = &vq->desc[head[packet_success]];
1944
1945                 /* Discard first buffer as it is the virtio header */
1946                 desc = &vq->desc[desc->next];
1947
1948                 /* Buffer address translation. */
1949                 buff_addr = gpa_to_vva(dev, desc->addr);
1950                 /* Need check extra VLAN_HLEN size for inserting VLAN tag */
1951                 phys_addr = gpa_to_hpa(vdev, desc->addr, desc->len + VLAN_HLEN,
1952                         &addr_type);
1953
1954                 if (likely(packet_success < (free_entries - 1)))
1955                         /* Prefetch descriptor index. */
1956                         rte_prefetch0(&vq->desc[head[packet_success + 1]]);
1957
1958                 if (unlikely(addr_type == PHYS_ADDR_INVALID)) {
1959                         RTE_LOG(ERR, VHOST_DATA,
1960                                 "(%"PRIu64") Invalid frame buffer address found"
1961                                 "when TX packets!\n",
1962                                 dev->device_fh);
1963                         packet_success++;
1964                         continue;
1965                 }
1966
1967                 /* Prefetch buffer address. */
1968                 rte_prefetch0((void *)(uintptr_t)buff_addr);
1969
1970                 /*
1971                  * Setup dummy mbuf. This is copied to a real mbuf if
1972                  * transmitted out the physical port.
1973                  */
1974                 m.data_len = desc->len;
1975                 m.nb_segs = 1;
1976                 m.next = NULL;
1977                 m.data_off = 0;
1978                 m.buf_addr = (void *)(uintptr_t)buff_addr;
1979                 m.buf_physaddr = phys_addr;
1980
1981                 /*
1982                  * Check if the frame buffer address from guest crosses
1983                  * sub-region or not.
1984                  */
1985                 if (unlikely(addr_type == PHYS_ADDR_CROSS_SUBREG)) {
1986                         RTE_LOG(ERR, VHOST_DATA,
1987                                 "(%"PRIu64") Frame buffer address cross "
1988                                 "sub-regioin found when attaching TX frame "
1989                                 "buffer address!\n",
1990                                 dev->device_fh);
1991                         need_copy = 1;
1992                 } else
1993                         need_copy = 0;
1994
1995                 PRINT_PACKET(dev, (uintptr_t)buff_addr, desc->len, 0);
1996
1997                 /*
1998                  * If this is the first received packet we need to learn
1999                  * the MAC and setup VMDQ
2000                  */
2001                 if (unlikely(vdev->ready == DEVICE_MAC_LEARNING)) {
2002                         if (vdev->remove || (link_vmdq(vdev, &m) == -1)) {
2003                                 /*
2004                                  * Discard frame if device is scheduled for
2005                                  * removal or a duplicate MAC address is found.
2006                                  */
2007                                 packet_success += free_entries;
2008                                 vq->last_used_idx += packet_success;
2009                                 break;
2010                         }
2011                 }
2012
2013                 virtio_tx_route_zcp(dev, &m, head[packet_success], need_copy);
2014                 packet_success++;
2015         }
2016 }
2017
2018 /*
2019  * This function is called by each data core. It handles all RX/TX registered
2020  * with the core. For TX the specific lcore linked list is used. For RX, MAC
2021  * addresses are compared with all devices in the main linked list.
2022  */
2023 static int
2024 switch_worker_zcp(__attribute__((unused)) void *arg)
2025 {
2026         struct virtio_net *dev = NULL;
2027         struct vhost_dev  *vdev = NULL;
2028         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2029         struct virtio_net_data_ll *dev_ll;
2030         struct mbuf_table *tx_q;
2031         volatile struct lcore_ll_info *lcore_ll;
2032         const uint64_t drain_tsc
2033                 = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S
2034                 * BURST_TX_DRAIN_US;
2035         uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0;
2036         unsigned ret;
2037         const uint16_t lcore_id = rte_lcore_id();
2038         uint16_t count_in_ring, rx_count = 0;
2039
2040         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
2041
2042         lcore_ll = lcore_info[lcore_id].lcore_ll;
2043         prev_tsc = 0;
2044
2045         while (1) {
2046                 cur_tsc = rte_rdtsc();
2047
2048                 /* TX burst queue drain */
2049                 diff_tsc = cur_tsc - prev_tsc;
2050                 if (unlikely(diff_tsc > drain_tsc)) {
2051                         /*
2052                          * Get mbuf from vpool.pool and detach mbuf and
2053                          * put back into vpool.ring.
2054                          */
2055                         dev_ll = lcore_ll->ll_root_used;
2056                         while ((dev_ll != NULL) && (dev_ll->vdev != NULL)) {
2057                                 /* Get virtio device ID */
2058                                 vdev = dev_ll->vdev;
2059                                 dev = vdev->dev;
2060
2061                                 if (likely(!vdev->remove)) {
2062                                         tx_q = &tx_queue_zcp[(uint16_t)vdev->vmdq_rx_q];
2063                                         if (tx_q->len) {
2064                                                 LOG_DEBUG(VHOST_DATA,
2065                                                 "TX queue drained after timeout"
2066                                                 " with burst size %u\n",
2067                                                 tx_q->len);
2068
2069                                                 /*
2070                                                  * Tx any packets in the queue
2071                                                  */
2072                                                 ret = rte_eth_tx_burst(
2073                                                         ports[0],
2074                                                         (uint16_t)tx_q->txq_id,
2075                                                         (struct rte_mbuf **)
2076                                                         tx_q->m_table,
2077                                                         (uint16_t)tx_q->len);
2078                                                 if (unlikely(ret < tx_q->len)) {
2079                                                         do {
2080                                                                 rte_pktmbuf_free(
2081                                                                         tx_q->m_table[ret]);
2082                                                         } while (++ret < tx_q->len);
2083                                                 }
2084                                                 tx_q->len = 0;
2085
2086                                                 txmbuf_clean_zcp(dev,
2087                                                         &vpool_array[MAX_QUEUES+vdev->vmdq_rx_q]);
2088                                         }
2089                                 }
2090                                 dev_ll = dev_ll->next;
2091                         }
2092                         prev_tsc = cur_tsc;
2093                 }
2094
2095                 rte_prefetch0(lcore_ll->ll_root_used);
2096
2097                 /*
2098                  * Inform the configuration core that we have exited the linked
2099                  * list and that no devices are in use if requested.
2100                  */
2101                 if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL)
2102                         lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
2103
2104                 /* Process devices */
2105                 dev_ll = lcore_ll->ll_root_used;
2106
2107                 while ((dev_ll != NULL) && (dev_ll->vdev != NULL)) {
2108                         vdev = dev_ll->vdev;
2109                         dev  = vdev->dev;
2110                         if (unlikely(vdev->remove)) {
2111                                 dev_ll = dev_ll->next;
2112                                 unlink_vmdq(vdev);
2113                                 vdev->ready = DEVICE_SAFE_REMOVE;
2114                                 continue;
2115                         }
2116
2117                         if (likely(vdev->ready == DEVICE_RX)) {
2118                                 uint32_t index = vdev->vmdq_rx_q;
2119                                 uint16_t i;
2120                                 count_in_ring
2121                                 = rte_ring_count(vpool_array[index].ring);
2122                                 uint16_t free_entries
2123                                 = (uint16_t)get_available_ring_num_zcp(dev);
2124
2125                                 /*
2126                                  * Attach all mbufs in vpool.ring and put back
2127                                  * into vpool.pool.
2128                                  */
2129                                 for (i = 0;
2130                                 i < RTE_MIN(free_entries,
2131                                 RTE_MIN(count_in_ring, MAX_PKT_BURST));
2132                                 i++)
2133                                         attach_rxmbuf_zcp(dev);
2134
2135                                 /* Handle guest RX */
2136                                 rx_count = rte_eth_rx_burst(ports[0],
2137                                         vdev->vmdq_rx_q, pkts_burst,
2138                                         MAX_PKT_BURST);
2139
2140                                 if (rx_count) {
2141                                         ret_count = virtio_dev_rx_zcp(dev,
2142                                                         pkts_burst, rx_count);
2143                                         if (enable_stats) {
2144                                                 dev_statistics[dev->device_fh].rx_total
2145                                                         += rx_count;
2146                                                 dev_statistics[dev->device_fh].rx
2147                                                         += ret_count;
2148                                         }
2149                                         while (likely(rx_count)) {
2150                                                 rx_count--;
2151                                                 pktmbuf_detach_zcp(
2152                                                         pkts_burst[rx_count]);
2153                                                 rte_ring_sp_enqueue(
2154                                                         vpool_array[index].ring,
2155                                                         (void *)pkts_burst[rx_count]);
2156                                         }
2157                                 }
2158                         }
2159
2160                         if (likely(!vdev->remove))
2161                                 /* Handle guest TX */
2162                                 virtio_dev_tx_zcp(dev);
2163
2164                         /* Move to the next device in the list */
2165                         dev_ll = dev_ll->next;
2166                 }
2167         }
2168
2169         return 0;
2170 }
2171
2172
2173 /*
2174  * Add an entry to a used linked list. A free entry must first be found
2175  * in the free linked list using get_data_ll_free_entry();
2176  */
2177 static void
2178 add_data_ll_entry(struct virtio_net_data_ll **ll_root_addr,
2179         struct virtio_net_data_ll *ll_dev)
2180 {
2181         struct virtio_net_data_ll *ll = *ll_root_addr;
2182
2183         /* Set next as NULL and use a compiler barrier to avoid reordering. */
2184         ll_dev->next = NULL;
2185         rte_compiler_barrier();
2186
2187         /* If ll == NULL then this is the first device. */
2188         if (ll) {
2189                 /* Increment to the tail of the linked list. */
2190                 while ((ll->next != NULL) )
2191                         ll = ll->next;
2192
2193                 ll->next = ll_dev;
2194         } else {
2195                 *ll_root_addr = ll_dev;
2196         }
2197 }
2198
2199 /*
2200  * Remove an entry from a used linked list. The entry must then be added to
2201  * the free linked list using put_data_ll_free_entry().
2202  */
2203 static void
2204 rm_data_ll_entry(struct virtio_net_data_ll **ll_root_addr,
2205         struct virtio_net_data_ll *ll_dev,
2206         struct virtio_net_data_ll *ll_dev_last)
2207 {
2208         struct virtio_net_data_ll *ll = *ll_root_addr;
2209
2210         if (unlikely((ll == NULL) || (ll_dev == NULL)))
2211                 return;
2212
2213         if (ll_dev == ll)
2214                 *ll_root_addr = ll_dev->next;
2215         else
2216                 if (likely(ll_dev_last != NULL))
2217                         ll_dev_last->next = ll_dev->next;
2218                 else
2219                         RTE_LOG(ERR, VHOST_CONFIG, "Remove entry form ll failed.\n");
2220 }
2221
2222 /*
2223  * Find and return an entry from the free linked list.
2224  */
2225 static struct virtio_net_data_ll *
2226 get_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr)
2227 {
2228         struct virtio_net_data_ll *ll_free = *ll_root_addr;
2229         struct virtio_net_data_ll *ll_dev;
2230
2231         if (ll_free == NULL)
2232                 return NULL;
2233
2234         ll_dev = ll_free;
2235         *ll_root_addr = ll_free->next;
2236
2237         return ll_dev;
2238 }
2239
2240 /*
2241  * Place an entry back on to the free linked list.
2242  */
2243 static void
2244 put_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr,
2245         struct virtio_net_data_ll *ll_dev)
2246 {
2247         struct virtio_net_data_ll *ll_free = *ll_root_addr;
2248
2249         if (ll_dev == NULL)
2250                 return;
2251
2252         ll_dev->next = ll_free;
2253         *ll_root_addr = ll_dev;
2254 }
2255
2256 /*
2257  * Creates a linked list of a given size.
2258  */
2259 static struct virtio_net_data_ll *
2260 alloc_data_ll(uint32_t size)
2261 {
2262         struct virtio_net_data_ll *ll_new;
2263         uint32_t i;
2264
2265         /* Malloc and then chain the linked list. */
2266         ll_new = malloc(size * sizeof(struct virtio_net_data_ll));
2267         if (ll_new == NULL) {
2268                 RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for ll_new.\n");
2269                 return NULL;
2270         }
2271
2272         for (i = 0; i < size - 1; i++) {
2273                 ll_new[i].vdev = NULL;
2274                 ll_new[i].next = &ll_new[i+1];
2275         }
2276         ll_new[i].next = NULL;
2277
2278         return (ll_new);
2279 }
2280
2281 /*
2282  * Create the main linked list along with each individual cores linked list. A used and a free list
2283  * are created to manage entries.
2284  */
2285 static int
2286 init_data_ll (void)
2287 {
2288         int lcore;
2289
2290         RTE_LCORE_FOREACH_SLAVE(lcore) {
2291                 lcore_info[lcore].lcore_ll = malloc(sizeof(struct lcore_ll_info));
2292                 if (lcore_info[lcore].lcore_ll == NULL) {
2293                         RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for lcore_ll.\n");
2294                         return -1;
2295                 }
2296
2297                 lcore_info[lcore].lcore_ll->device_num = 0;
2298                 lcore_info[lcore].lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
2299                 lcore_info[lcore].lcore_ll->ll_root_used = NULL;
2300                 if (num_devices % num_switching_cores)
2301                         lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll((num_devices / num_switching_cores) + 1);
2302                 else
2303                         lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll(num_devices / num_switching_cores);
2304         }
2305
2306         /* Allocate devices up to a maximum of MAX_DEVICES. */
2307         ll_root_free = alloc_data_ll(MIN((num_devices), MAX_DEVICES));
2308
2309         return 0;
2310 }
2311
2312 /*
2313  * Remove a device from the specific data core linked list and from the main linked list. Synchonization
2314  * occurs through the use of the lcore dev_removal_flag. Device is made volatile here to avoid re-ordering
2315  * of dev->remove=1 which can cause an infinite loop in the rte_pause loop.
2316  */
2317 static void
2318 destroy_device (volatile struct virtio_net *dev)
2319 {
2320         struct virtio_net_data_ll *ll_lcore_dev_cur;
2321         struct virtio_net_data_ll *ll_main_dev_cur;
2322         struct virtio_net_data_ll *ll_lcore_dev_last = NULL;
2323         struct virtio_net_data_ll *ll_main_dev_last = NULL;
2324         struct vhost_dev *vdev;
2325         int lcore;
2326
2327         dev->flags &= ~VIRTIO_DEV_RUNNING;
2328
2329         vdev = (struct vhost_dev *)dev->priv;
2330         /*set the remove flag. */
2331         vdev->remove = 1;
2332         while(vdev->ready != DEVICE_SAFE_REMOVE) {
2333                 rte_pause();
2334         }
2335
2336         /* Search for entry to be removed from lcore ll */
2337         ll_lcore_dev_cur = lcore_info[vdev->coreid].lcore_ll->ll_root_used;
2338         while (ll_lcore_dev_cur != NULL) {
2339                 if (ll_lcore_dev_cur->vdev == vdev) {
2340                         break;
2341                 } else {
2342                         ll_lcore_dev_last = ll_lcore_dev_cur;
2343                         ll_lcore_dev_cur = ll_lcore_dev_cur->next;
2344                 }
2345         }
2346
2347         if (ll_lcore_dev_cur == NULL) {
2348                 RTE_LOG(ERR, VHOST_CONFIG,
2349                         "(%"PRIu64") Failed to find the dev to be destroy.\n",
2350                         dev->device_fh);
2351                 return;
2352         }
2353
2354         /* Search for entry to be removed from main ll */
2355         ll_main_dev_cur = ll_root_used;
2356         ll_main_dev_last = NULL;
2357         while (ll_main_dev_cur != NULL) {
2358                 if (ll_main_dev_cur->vdev == vdev) {
2359                         break;
2360                 } else {
2361                         ll_main_dev_last = ll_main_dev_cur;
2362                         ll_main_dev_cur = ll_main_dev_cur->next;
2363                 }
2364         }
2365
2366         /* Remove entries from the lcore and main ll. */
2367         rm_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
2368         rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
2369
2370         /* Set the dev_removal_flag on each lcore. */
2371         RTE_LCORE_FOREACH_SLAVE(lcore) {
2372                 lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;
2373         }
2374
2375         /*
2376          * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that
2377          * they can no longer access the device removed from the linked lists and that the devices
2378          * are no longer in use.
2379          */
2380         RTE_LCORE_FOREACH_SLAVE(lcore) {
2381                 while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL) {
2382                         rte_pause();
2383                 }
2384         }
2385
2386         /* Add the entries back to the lcore and main free ll.*/
2387         put_data_ll_free_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);
2388         put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
2389
2390         /* Decrement number of device on the lcore. */
2391         lcore_info[vdev->coreid].lcore_ll->device_num--;
2392
2393         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been removed from data core\n", dev->device_fh);
2394
2395         if (zero_copy) {
2396                 struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
2397
2398                 /* Stop the RX queue. */
2399                 if (rte_eth_dev_rx_queue_stop(ports[0], vdev->vmdq_rx_q) != 0) {
2400                         LOG_DEBUG(VHOST_CONFIG,
2401                                 "(%"PRIu64") In destroy_device: Failed to stop "
2402                                 "rx queue:%d\n",
2403                                 dev->device_fh,
2404                                 vdev->vmdq_rx_q);
2405                 }
2406
2407                 LOG_DEBUG(VHOST_CONFIG,
2408                         "(%"PRIu64") in destroy_device: Start put mbuf in "
2409                         "mempool back to ring for RX queue: %d\n",
2410                         dev->device_fh, vdev->vmdq_rx_q);
2411
2412                 mbuf_destroy_zcp(vpool);
2413
2414                 /* Stop the TX queue. */
2415                 if (rte_eth_dev_tx_queue_stop(ports[0], vdev->vmdq_rx_q) != 0) {
2416                         LOG_DEBUG(VHOST_CONFIG,
2417                                 "(%"PRIu64") In destroy_device: Failed to "
2418                                 "stop tx queue:%d\n",
2419                                 dev->device_fh, vdev->vmdq_rx_q);
2420                 }
2421
2422                 vpool = &vpool_array[vdev->vmdq_rx_q + MAX_QUEUES];
2423
2424                 LOG_DEBUG(VHOST_CONFIG,
2425                         "(%"PRIu64") destroy_device: Start put mbuf in mempool "
2426                         "back to ring for TX queue: %d, dev:(%"PRIu64")\n",
2427                         dev->device_fh, (vdev->vmdq_rx_q + MAX_QUEUES),
2428                         dev->device_fh);
2429
2430                 mbuf_destroy_zcp(vpool);
2431                 rte_free(vdev->regions_hpa);
2432         }
2433         rte_free(vdev);
2434
2435 }
2436
2437 /*
2438  * Calculate the region count of physical continous regions for one particular
2439  * region of whose vhost virtual address is continous. The particular region
2440  * start from vva_start, with size of 'size' in argument.
2441  */
2442 static uint32_t
2443 check_hpa_regions(uint64_t vva_start, uint64_t size)
2444 {
2445         uint32_t i, nregions = 0, page_size = getpagesize();
2446         uint64_t cur_phys_addr = 0, next_phys_addr = 0;
2447         if (vva_start % page_size) {
2448                 LOG_DEBUG(VHOST_CONFIG,
2449                         "in check_countinous: vva start(%p) mod page_size(%d) "
2450                         "has remainder\n",
2451                         (void *)(uintptr_t)vva_start, page_size);
2452                 return 0;
2453         }
2454         if (size % page_size) {
2455                 LOG_DEBUG(VHOST_CONFIG,
2456                         "in check_countinous: "
2457                         "size((%"PRIu64")) mod page_size(%d) has remainder\n",
2458                         size, page_size);
2459                 return 0;
2460         }
2461         for (i = 0; i < size - page_size; i = i + page_size) {
2462                 cur_phys_addr
2463                         = rte_mem_virt2phy((void *)(uintptr_t)(vva_start + i));
2464                 next_phys_addr = rte_mem_virt2phy(
2465                         (void *)(uintptr_t)(vva_start + i + page_size));
2466                 if ((cur_phys_addr + page_size) != next_phys_addr) {
2467                         ++nregions;
2468                         LOG_DEBUG(VHOST_CONFIG,
2469                                 "in check_continuous: hva addr:(%p) is not "
2470                                 "continuous with hva addr:(%p), diff:%d\n",
2471                                 (void *)(uintptr_t)(vva_start + (uint64_t)i),
2472                                 (void *)(uintptr_t)(vva_start + (uint64_t)i
2473                                 + page_size), page_size);
2474                         LOG_DEBUG(VHOST_CONFIG,
2475                                 "in check_continuous: hpa addr:(%p) is not "
2476                                 "continuous with hpa addr:(%p), "
2477                                 "diff:(%"PRIu64")\n",
2478                                 (void *)(uintptr_t)cur_phys_addr,
2479                                 (void *)(uintptr_t)next_phys_addr,
2480                                 (next_phys_addr-cur_phys_addr));
2481                 }
2482         }
2483         return nregions;
2484 }
2485
2486 /*
2487  * Divide each region whose vhost virtual address is continous into a few
2488  * sub-regions, make sure the physical address within each sub-region are
2489  * continous. And fill offset(to GPA) and size etc. information of each
2490  * sub-region into regions_hpa.
2491  */
2492 static uint32_t
2493 fill_hpa_memory_regions(struct virtio_memory_regions_hpa *mem_region_hpa, struct virtio_memory *virtio_memory)
2494 {
2495         uint32_t regionidx, regionidx_hpa = 0, i, k, page_size = getpagesize();
2496         uint64_t cur_phys_addr = 0, next_phys_addr = 0, vva_start;
2497
2498         if (mem_region_hpa == NULL)
2499                 return 0;
2500
2501         for (regionidx = 0; regionidx < virtio_memory->nregions; regionidx++) {
2502                 vva_start = virtio_memory->regions[regionidx].guest_phys_address +
2503                         virtio_memory->regions[regionidx].address_offset;
2504                 mem_region_hpa[regionidx_hpa].guest_phys_address
2505                         = virtio_memory->regions[regionidx].guest_phys_address;
2506                 mem_region_hpa[regionidx_hpa].host_phys_addr_offset =
2507                         rte_mem_virt2phy((void *)(uintptr_t)(vva_start)) -
2508                         mem_region_hpa[regionidx_hpa].guest_phys_address;
2509                 LOG_DEBUG(VHOST_CONFIG,
2510                         "in fill_hpa_regions: guest phys addr start[%d]:(%p)\n",
2511                         regionidx_hpa,
2512                         (void *)(uintptr_t)
2513                         (mem_region_hpa[regionidx_hpa].guest_phys_address));
2514                 LOG_DEBUG(VHOST_CONFIG,
2515                         "in fill_hpa_regions: host  phys addr start[%d]:(%p)\n",
2516                         regionidx_hpa,
2517                         (void *)(uintptr_t)
2518                         (mem_region_hpa[regionidx_hpa].host_phys_addr_offset));
2519                 for (i = 0, k = 0;
2520                         i < virtio_memory->regions[regionidx].memory_size -
2521                                 page_size;
2522                         i += page_size) {
2523                         cur_phys_addr = rte_mem_virt2phy(
2524                                         (void *)(uintptr_t)(vva_start + i));
2525                         next_phys_addr = rte_mem_virt2phy(
2526                                         (void *)(uintptr_t)(vva_start +
2527                                         i + page_size));
2528                         if ((cur_phys_addr + page_size) != next_phys_addr) {
2529                                 mem_region_hpa[regionidx_hpa].guest_phys_address_end =
2530                                         mem_region_hpa[regionidx_hpa].guest_phys_address +
2531                                         k + page_size;
2532                                 mem_region_hpa[regionidx_hpa].memory_size
2533                                         = k + page_size;
2534                                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest "
2535                                         "phys addr end  [%d]:(%p)\n",
2536                                         regionidx_hpa,
2537                                         (void *)(uintptr_t)
2538                                         (mem_region_hpa[regionidx_hpa].guest_phys_address_end));
2539                                 LOG_DEBUG(VHOST_CONFIG,
2540                                         "in fill_hpa_regions: guest phys addr "
2541                                         "size [%d]:(%p)\n",
2542                                         regionidx_hpa,
2543                                         (void *)(uintptr_t)
2544                                         (mem_region_hpa[regionidx_hpa].memory_size));
2545                                 mem_region_hpa[regionidx_hpa + 1].guest_phys_address
2546                                         = mem_region_hpa[regionidx_hpa].guest_phys_address_end;
2547                                 ++regionidx_hpa;
2548                                 mem_region_hpa[regionidx_hpa].host_phys_addr_offset =
2549                                         next_phys_addr -
2550                                         mem_region_hpa[regionidx_hpa].guest_phys_address;
2551                                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest"
2552                                         " phys addr start[%d]:(%p)\n",
2553                                         regionidx_hpa,
2554                                         (void *)(uintptr_t)
2555                                         (mem_region_hpa[regionidx_hpa].guest_phys_address));
2556                                 LOG_DEBUG(VHOST_CONFIG,
2557                                         "in fill_hpa_regions: host  phys addr "
2558                                         "start[%d]:(%p)\n",
2559                                         regionidx_hpa,
2560                                         (void *)(uintptr_t)
2561                                         (mem_region_hpa[regionidx_hpa].host_phys_addr_offset));
2562                                 k = 0;
2563                         } else {
2564                                 k += page_size;
2565                         }
2566                 }
2567                 mem_region_hpa[regionidx_hpa].guest_phys_address_end
2568                         = mem_region_hpa[regionidx_hpa].guest_phys_address
2569                         + k + page_size;
2570                 mem_region_hpa[regionidx_hpa].memory_size = k + page_size;
2571                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest phys addr end  "
2572                         "[%d]:(%p)\n", regionidx_hpa,
2573                         (void *)(uintptr_t)
2574                         (mem_region_hpa[regionidx_hpa].guest_phys_address_end));
2575                 LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest phys addr size "
2576                         "[%d]:(%p)\n", regionidx_hpa,
2577                         (void *)(uintptr_t)
2578                         (mem_region_hpa[regionidx_hpa].memory_size));
2579                 ++regionidx_hpa;
2580         }
2581         return regionidx_hpa;
2582 }
2583
2584 /*
2585  * A new device is added to a data core. First the device is added to the main linked list
2586  * and the allocated to a specific data core.
2587  */
2588 static int
2589 new_device (struct virtio_net *dev)
2590 {
2591         struct virtio_net_data_ll *ll_dev;
2592         int lcore, core_add = 0;
2593         uint32_t device_num_min = num_devices;
2594         struct vhost_dev *vdev;
2595         uint32_t regionidx;
2596
2597         vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
2598         if (vdev == NULL) {
2599                 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Couldn't allocate memory for vhost dev\n",
2600                         dev->device_fh);
2601                 return -1;
2602         }
2603         vdev->dev = dev;
2604         dev->priv = vdev;
2605
2606         if (zero_copy) {
2607                 vdev->nregions_hpa = dev->mem->nregions;
2608                 for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
2609                         vdev->nregions_hpa
2610                                 += check_hpa_regions(
2611                                         dev->mem->regions[regionidx].guest_phys_address
2612                                         + dev->mem->regions[regionidx].address_offset,
2613                                         dev->mem->regions[regionidx].memory_size);
2614
2615                 }
2616
2617                 vdev->regions_hpa = rte_calloc("vhost hpa region",
2618                                                vdev->nregions_hpa,
2619                                                sizeof(struct virtio_memory_regions_hpa),
2620                                                RTE_CACHE_LINE_SIZE);
2621                 if (vdev->regions_hpa == NULL) {
2622                         RTE_LOG(ERR, VHOST_CONFIG, "Cannot allocate memory for hpa region\n");
2623                         rte_free(vdev);
2624                         return -1;
2625                 }
2626
2627
2628                 if (fill_hpa_memory_regions(
2629                         vdev->regions_hpa, dev->mem
2630                         ) != vdev->nregions_hpa) {
2631
2632                         RTE_LOG(ERR, VHOST_CONFIG,
2633                                 "hpa memory regions number mismatch: "
2634                                 "[%d]\n", vdev->nregions_hpa);
2635                         rte_free(vdev->regions_hpa);
2636                         rte_free(vdev);
2637                         return -1;
2638                 }
2639         }
2640
2641
2642         /* Add device to main ll */
2643         ll_dev = get_data_ll_free_entry(&ll_root_free);
2644         if (ll_dev == NULL) {
2645                 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") No free entry found in linked list. Device limit "
2646                         "of %d devices per core has been reached\n",
2647                         dev->device_fh, num_devices);
2648                 if (vdev->regions_hpa)
2649                         rte_free(vdev->regions_hpa);
2650                 rte_free(vdev);
2651                 return -1;
2652         }
2653         ll_dev->vdev = vdev;
2654         add_data_ll_entry(&ll_root_used, ll_dev);
2655         vdev->vmdq_rx_q
2656                 = dev->device_fh * queues_per_pool + vmdq_queue_base;
2657
2658         if (zero_copy) {
2659                 uint32_t index = vdev->vmdq_rx_q;
2660                 uint32_t count_in_ring, i;
2661                 struct mbuf_table *tx_q;
2662
2663                 count_in_ring = rte_ring_count(vpool_array[index].ring);
2664
2665                 LOG_DEBUG(VHOST_CONFIG,
2666                         "(%"PRIu64") in new_device: mbuf count in mempool "
2667                         "before attach is: %d\n",
2668                         dev->device_fh,
2669                         rte_mempool_count(vpool_array[index].pool));
2670                 LOG_DEBUG(VHOST_CONFIG,
2671                         "(%"PRIu64") in new_device: mbuf count in  ring "
2672                         "before attach  is : %d\n",
2673                         dev->device_fh, count_in_ring);
2674
2675                 /*
2676                  * Attach all mbufs in vpool.ring and put back intovpool.pool.
2677                  */
2678                 for (i = 0; i < count_in_ring; i++)
2679                         attach_rxmbuf_zcp(dev);
2680
2681                 LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") in new_device: mbuf count in "
2682                         "mempool after attach is: %d\n",
2683                         dev->device_fh,
2684                         rte_mempool_count(vpool_array[index].pool));
2685                 LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") in new_device: mbuf count in "
2686                         "ring after attach  is : %d\n",
2687                         dev->device_fh,
2688                         rte_ring_count(vpool_array[index].ring));
2689
2690                 tx_q = &tx_queue_zcp[(uint16_t)vdev->vmdq_rx_q];
2691                 tx_q->txq_id = vdev->vmdq_rx_q;
2692
2693                 if (rte_eth_dev_tx_queue_start(ports[0], vdev->vmdq_rx_q) != 0) {
2694                         struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
2695
2696                         LOG_DEBUG(VHOST_CONFIG,
2697                                 "(%"PRIu64") In new_device: Failed to start "
2698                                 "tx queue:%d\n",
2699                                 dev->device_fh, vdev->vmdq_rx_q);
2700
2701                         mbuf_destroy_zcp(vpool);
2702                         rte_free(vdev->regions_hpa);
2703                         rte_free(vdev);
2704                         return -1;
2705                 }
2706
2707                 if (rte_eth_dev_rx_queue_start(ports[0], vdev->vmdq_rx_q) != 0) {
2708                         struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
2709
2710                         LOG_DEBUG(VHOST_CONFIG,
2711                                 "(%"PRIu64") In new_device: Failed to start "
2712                                 "rx queue:%d\n",
2713                                 dev->device_fh, vdev->vmdq_rx_q);
2714
2715                         /* Stop the TX queue. */
2716                         if (rte_eth_dev_tx_queue_stop(ports[0],
2717                                 vdev->vmdq_rx_q) != 0) {
2718                                 LOG_DEBUG(VHOST_CONFIG,
2719                                         "(%"PRIu64") In new_device: Failed to "
2720                                         "stop tx queue:%d\n",
2721                                         dev->device_fh, vdev->vmdq_rx_q);
2722                         }
2723
2724                         mbuf_destroy_zcp(vpool);
2725                         rte_free(vdev->regions_hpa);
2726                         rte_free(vdev);
2727                         return -1;
2728                 }
2729
2730         }
2731
2732         /*reset ready flag*/
2733         vdev->ready = DEVICE_MAC_LEARNING;
2734         vdev->remove = 0;
2735
2736         /* Find a suitable lcore to add the device. */
2737         RTE_LCORE_FOREACH_SLAVE(lcore) {
2738                 if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
2739                         device_num_min = lcore_info[lcore].lcore_ll->device_num;
2740                         core_add = lcore;
2741                 }
2742         }
2743         /* Add device to lcore ll */
2744         ll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);
2745         if (ll_dev == NULL) {
2746                 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
2747                 vdev->ready = DEVICE_SAFE_REMOVE;
2748                 destroy_device(dev);
2749                 rte_free(vdev->regions_hpa);
2750                 rte_free(vdev);
2751                 return -1;
2752         }
2753         ll_dev->vdev = vdev;
2754         vdev->coreid = core_add;
2755
2756         add_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_dev);
2757
2758         /* Initialize device stats */
2759         memset(&dev_statistics[dev->device_fh], 0, sizeof(struct device_statistics));
2760
2761         /* Disable notifications. */
2762         rte_vhost_enable_guest_notification(dev, VIRTIO_RXQ, 0);
2763         rte_vhost_enable_guest_notification(dev, VIRTIO_TXQ, 0);
2764         lcore_info[vdev->coreid].lcore_ll->device_num++;
2765         dev->flags |= VIRTIO_DEV_RUNNING;
2766
2767         RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, vdev->coreid);
2768
2769         return 0;
2770 }
2771
2772 /*
2773  * These callback allow devices to be added to the data core when configuration
2774  * has been fully complete.
2775  */
2776 static const struct virtio_net_device_ops virtio_net_device_ops =
2777 {
2778         .new_device =  new_device,
2779         .destroy_device = destroy_device,
2780 };
2781
2782 /*
2783  * This is a thread will wake up after a period to print stats if the user has
2784  * enabled them.
2785  */
2786 static void
2787 print_stats(void)
2788 {
2789         struct virtio_net_data_ll *dev_ll;
2790         uint64_t tx_dropped, rx_dropped;
2791         uint64_t tx, tx_total, rx, rx_total;
2792         uint32_t device_fh;
2793         const char clr[] = { 27, '[', '2', 'J', '\0' };
2794         const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' };
2795
2796         while(1) {
2797                 sleep(enable_stats);
2798
2799                 /* Clear screen and move to top left */
2800                 printf("%s%s", clr, top_left);
2801
2802                 printf("\nDevice statistics ====================================");
2803
2804                 dev_ll = ll_root_used;
2805                 while (dev_ll != NULL) {
2806                         device_fh = (uint32_t)dev_ll->vdev->dev->device_fh;
2807                         tx_total = dev_statistics[device_fh].tx_total;
2808                         tx = dev_statistics[device_fh].tx;
2809                         tx_dropped = tx_total - tx;
2810                         if (zero_copy == 0) {
2811                                 rx_total = rte_atomic64_read(
2812                                         &dev_statistics[device_fh].rx_total_atomic);
2813                                 rx = rte_atomic64_read(
2814                                         &dev_statistics[device_fh].rx_atomic);
2815                         } else {
2816                                 rx_total = dev_statistics[device_fh].rx_total;
2817                                 rx = dev_statistics[device_fh].rx;
2818                         }
2819                         rx_dropped = rx_total - rx;
2820
2821                         printf("\nStatistics for device %"PRIu32" ------------------------------"
2822                                         "\nTX total:            %"PRIu64""
2823                                         "\nTX dropped:          %"PRIu64""
2824                                         "\nTX successful:               %"PRIu64""
2825                                         "\nRX total:            %"PRIu64""
2826                                         "\nRX dropped:          %"PRIu64""
2827                                         "\nRX successful:               %"PRIu64"",
2828                                         device_fh,
2829                                         tx_total,
2830                                         tx_dropped,
2831                                         tx,
2832                                         rx_total,
2833                                         rx_dropped,
2834                                         rx);
2835
2836                         dev_ll = dev_ll->next;
2837                 }
2838                 printf("\n======================================================\n");
2839         }
2840 }
2841
2842 static void
2843 setup_mempool_tbl(int socket, uint32_t index, char *pool_name,
2844         char *ring_name, uint32_t nb_mbuf)
2845 {
2846         vpool_array[index].pool = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
2847                 MBUF_CACHE_SIZE_ZCP, 0, MBUF_DATA_SIZE_ZCP, socket);
2848         if (vpool_array[index].pool != NULL) {
2849                 vpool_array[index].ring
2850                         = rte_ring_create(ring_name,
2851                                 rte_align32pow2(nb_mbuf + 1),
2852                                 socket, RING_F_SP_ENQ | RING_F_SC_DEQ);
2853                 if (likely(vpool_array[index].ring != NULL)) {
2854                         LOG_DEBUG(VHOST_CONFIG,
2855                                 "in setup_mempool_tbl: mbuf count in "
2856                                 "mempool is: %d\n",
2857                                 rte_mempool_count(vpool_array[index].pool));
2858                         LOG_DEBUG(VHOST_CONFIG,
2859                                 "in setup_mempool_tbl: mbuf count in "
2860                                 "ring   is: %d\n",
2861                                 rte_ring_count(vpool_array[index].ring));
2862                 } else {
2863                         rte_exit(EXIT_FAILURE, "ring_create(%s) failed",
2864                                 ring_name);
2865                 }
2866
2867                 /* Need consider head room. */
2868                 vpool_array[index].buf_size = VIRTIO_DESCRIPTOR_LEN_ZCP;
2869         } else {
2870                 rte_exit(EXIT_FAILURE, "mempool_create(%s) failed", pool_name);
2871         }
2872 }
2873
2874
2875 /*
2876  * Main function, does initialisation and calls the per-lcore functions. The CUSE
2877  * device is also registered here to handle the IOCTLs.
2878  */
2879 int
2880 main(int argc, char *argv[])
2881 {
2882         struct rte_mempool *mbuf_pool = NULL;
2883         unsigned lcore_id, core_id = 0;
2884         unsigned nb_ports, valid_num_ports;
2885         int ret;
2886         uint8_t portid;
2887         uint16_t queue_id;
2888         static pthread_t tid;
2889
2890         /* init EAL */
2891         ret = rte_eal_init(argc, argv);
2892         if (ret < 0)
2893                 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
2894         argc -= ret;
2895         argv += ret;
2896
2897         /* parse app arguments */
2898         ret = us_vhost_parse_args(argc, argv);
2899         if (ret < 0)
2900                 rte_exit(EXIT_FAILURE, "Invalid argument\n");
2901
2902         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++)
2903                 if (rte_lcore_is_enabled(lcore_id))
2904                         lcore_ids[core_id ++] = lcore_id;
2905
2906         if (rte_lcore_count() > RTE_MAX_LCORE)
2907                 rte_exit(EXIT_FAILURE,"Not enough cores\n");
2908
2909         /*set the number of swithcing cores available*/
2910         num_switching_cores = rte_lcore_count()-1;
2911
2912         /* Get the number of physical ports. */
2913         nb_ports = rte_eth_dev_count();
2914         if (nb_ports > RTE_MAX_ETHPORTS)
2915                 nb_ports = RTE_MAX_ETHPORTS;
2916
2917         /*
2918          * Update the global var NUM_PORTS and global array PORTS
2919          * and get value of var VALID_NUM_PORTS according to system ports number
2920          */
2921         valid_num_ports = check_ports_num(nb_ports);
2922
2923         if ((valid_num_ports ==  0) || (valid_num_ports > MAX_SUP_PORTS)) {
2924                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
2925                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
2926                 return -1;
2927         }
2928
2929         if (zero_copy == 0) {
2930                 /* Create the mbuf pool. */
2931                 mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL",
2932                         NUM_MBUFS_PER_PORT * valid_num_ports, MBUF_CACHE_SIZE,
2933                         0, MBUF_DATA_SIZE, rte_socket_id());
2934                 if (mbuf_pool == NULL)
2935                         rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
2936
2937                 for (queue_id = 0; queue_id < MAX_QUEUES + 1; queue_id++)
2938                         vpool_array[queue_id].pool = mbuf_pool;
2939
2940                 if (vm2vm_mode == VM2VM_HARDWARE) {
2941                         /* Enable VT loop back to let L2 switch to do it. */
2942                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1;
2943                         LOG_DEBUG(VHOST_CONFIG,
2944                                 "Enable loop back for L2 switch in vmdq.\n");
2945                 }
2946         } else {
2947                 uint32_t nb_mbuf;
2948                 char pool_name[RTE_MEMPOOL_NAMESIZE];
2949                 char ring_name[RTE_MEMPOOL_NAMESIZE];
2950
2951                 nb_mbuf = num_rx_descriptor
2952                         + num_switching_cores * MBUF_CACHE_SIZE_ZCP
2953                         + num_switching_cores * MAX_PKT_BURST;
2954
2955                 for (queue_id = 0; queue_id < MAX_QUEUES; queue_id++) {
2956                         snprintf(pool_name, sizeof(pool_name),
2957                                 "rxmbuf_pool_%u", queue_id);
2958                         snprintf(ring_name, sizeof(ring_name),
2959                                 "rxmbuf_ring_%u", queue_id);
2960                         setup_mempool_tbl(rte_socket_id(), queue_id,
2961                                 pool_name, ring_name, nb_mbuf);
2962                 }
2963
2964                 nb_mbuf = num_tx_descriptor
2965                                 + num_switching_cores * MBUF_CACHE_SIZE_ZCP
2966                                 + num_switching_cores * MAX_PKT_BURST;
2967
2968                 for (queue_id = 0; queue_id < MAX_QUEUES; queue_id++) {
2969                         snprintf(pool_name, sizeof(pool_name),
2970                                 "txmbuf_pool_%u", queue_id);
2971                         snprintf(ring_name, sizeof(ring_name),
2972                                 "txmbuf_ring_%u", queue_id);
2973                         setup_mempool_tbl(rte_socket_id(),
2974                                 (queue_id + MAX_QUEUES),
2975                                 pool_name, ring_name, nb_mbuf);
2976                 }
2977
2978                 if (vm2vm_mode == VM2VM_HARDWARE) {
2979                         /* Enable VT loop back to let L2 switch to do it. */
2980                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1;
2981                         LOG_DEBUG(VHOST_CONFIG,
2982                                 "Enable loop back for L2 switch in vmdq.\n");
2983                 }
2984         }
2985         /* Set log level. */
2986         rte_set_log_level(LOG_LEVEL);
2987
2988         /* initialize all ports */
2989         for (portid = 0; portid < nb_ports; portid++) {
2990                 /* skip ports that are not enabled */
2991                 if ((enabled_port_mask & (1 << portid)) == 0) {
2992                         RTE_LOG(INFO, VHOST_PORT,
2993                                 "Skipping disabled port %d\n", portid);
2994                         continue;
2995                 }
2996                 if (port_init(portid) != 0)
2997                         rte_exit(EXIT_FAILURE,
2998                                 "Cannot initialize network ports\n");
2999         }
3000
3001         /* Initialise all linked lists. */
3002         if (init_data_ll() == -1)
3003                 rte_exit(EXIT_FAILURE, "Failed to initialize linked list\n");
3004
3005         /* Initialize device stats */
3006         memset(&dev_statistics, 0, sizeof(dev_statistics));
3007
3008         /* Enable stats if the user option is set. */
3009         if (enable_stats)
3010                 pthread_create(&tid, NULL, (void*)print_stats, NULL );
3011
3012         /* Launch all data cores. */
3013         if (zero_copy == 0) {
3014                 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
3015                         rte_eal_remote_launch(switch_worker,
3016                                 mbuf_pool, lcore_id);
3017                 }
3018         } else {
3019                 uint32_t count_in_mempool, index, i;
3020                 for (index = 0; index < 2*MAX_QUEUES; index++) {
3021                         /* For all RX and TX queues. */
3022                         count_in_mempool
3023                                 = rte_mempool_count(vpool_array[index].pool);
3024
3025                         /*
3026                          * Transfer all un-attached mbufs from vpool.pool
3027                          * to vpoo.ring.
3028                          */
3029                         for (i = 0; i < count_in_mempool; i++) {
3030                                 struct rte_mbuf *mbuf
3031                                         = __rte_mbuf_raw_alloc(
3032                                                 vpool_array[index].pool);
3033                                 rte_ring_sp_enqueue(vpool_array[index].ring,
3034                                                 (void *)mbuf);
3035                         }
3036
3037                         LOG_DEBUG(VHOST_CONFIG,
3038                                 "in main: mbuf count in mempool at initial "
3039                                 "is: %d\n", count_in_mempool);
3040                         LOG_DEBUG(VHOST_CONFIG,
3041                                 "in main: mbuf count in  ring at initial  is :"
3042                                 " %d\n",
3043                                 rte_ring_count(vpool_array[index].ring));
3044                 }
3045
3046                 RTE_LCORE_FOREACH_SLAVE(lcore_id)
3047                         rte_eal_remote_launch(switch_worker_zcp, NULL,
3048                                 lcore_id);
3049         }
3050
3051         if (mergeable == 0)
3052                 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_MRG_RXBUF);
3053
3054         /* Register CUSE device to handle IOCTLs. */
3055         ret = rte_vhost_driver_register((char *)&dev_basename);
3056         if (ret != 0)
3057                 rte_exit(EXIT_FAILURE,"CUSE device setup failure.\n");
3058
3059         rte_vhost_driver_callback_register(&virtio_net_device_ops);
3060
3061         /* Start CUSE session. */
3062         rte_vhost_driver_session_start();
3063         return 0;
3064
3065 }