examples/vhost/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <arpa/inet.h>
   6 #include <getopt.h>
   7 #include <linux/if_ether.h>
   8 #include <linux/if_vlan.h>
   9 #include <linux/virtio_net.h>
  10 #include <linux/virtio_ring.h>
  11 #include <signal.h>
  12 #include <stdint.h>
  13 #include <sys/eventfd.h>
  14 #include <sys/param.h>
  15 #include <unistd.h>
  16
  17 #include <rte_atomic.h>
  18 #include <rte_cycles.h>
  19 #include <rte_ethdev.h>
  20 #include <rte_log.h>
  21 #include <rte_string_fns.h>
  22 #include <rte_malloc.h>
  23 #include <rte_vhost.h>
  24 #include <rte_ip.h>
  25 #include <rte_tcp.h>
  26 #include <rte_pause.h>
  27
  28 #include "main.h"
  29
  30 #ifndef MAX_QUEUES
  31 #define MAX_QUEUES 128
  32 #endif
  33
  34 /* the maximum number of external ports supported */
  35 #define MAX_SUP_PORTS 1
  36
  37 #define MBUF_CACHE_SIZE 128
  38 #define MBUF_DATA_SIZE  RTE_MBUF_DEFAULT_BUF_SIZE
  39
  40 #define BURST_TX_DRAIN_US 100   /* TX drain every ~100us */
  41
  42 #define BURST_RX_WAIT_US 15     /* Defines how long we wait between retries on RX */
  43 #define BURST_RX_RETRIES 4              /* Number of retries on RX. */
  44
  45 #define JUMBO_FRAME_MAX_SIZE    0x2600
  46
  47 /* State of virtio device. */
  48 #define DEVICE_MAC_LEARNING 0
  49 #define DEVICE_RX                       1
  50 #define DEVICE_SAFE_REMOVE      2
  51
  52 /* Configurable number of RX/TX ring descriptors */
  53 #define RTE_TEST_RX_DESC_DEFAULT 1024
  54 #define RTE_TEST_TX_DESC_DEFAULT 512
  55
  56 #define INVALID_PORT_ID 0xFF
  57
  58 /* Maximum long option length for option parsing. */
  59 #define MAX_LONG_OPT_SZ 64
  60
  61 /* mask of enabled ports */
  62 static uint32_t enabled_port_mask = 0;
  63
  64 /* Promiscuous mode */
  65 static uint32_t promiscuous;
  66
  67 /* number of devices/queues to support*/
  68 static uint32_t num_queues = 0;
  69 static uint32_t num_devices;
  70
  71 static struct rte_mempool *mbuf_pool;
  72 static int mergeable;
  73
  74 /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */
  75 typedef enum {
  76         VM2VM_DISABLED = 0,
  77         VM2VM_SOFTWARE = 1,
  78         VM2VM_HARDWARE = 2,
  79         VM2VM_LAST
  80 } vm2vm_type;
  81 static vm2vm_type vm2vm_mode = VM2VM_SOFTWARE;
  82
  83 /* Enable stats. */
  84 static uint32_t enable_stats = 0;
  85 /* Enable retries on RX. */
  86 static uint32_t enable_retry = 1;
  87
  88 /* Disable TX checksum offload */
  89 static uint32_t enable_tx_csum;
  90
  91 /* Disable TSO offload */
  92 static uint32_t enable_tso;
  93
  94 static int client_mode;
  95 static int dequeue_zero_copy;
  96
  97 static int builtin_net_driver;
  98
  99 /* Specify timeout (in useconds) between retries on RX. */
 100 static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US;
 101 /* Specify the number of retries on RX. */
 102 static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;
 103
 104 /* Socket file paths. Can be set by user */
 105 static char *socket_files;
 106 static int nb_sockets;
 107
 108 /* empty vmdq configuration structure. Filled in programatically */
 109 static struct rte_eth_conf vmdq_conf_default = {
 110         .rxmode = {
 111                 .mq_mode        = ETH_MQ_RX_VMDQ_ONLY,
 112                 .split_hdr_size = 0,
 113                 /*
 114                  * VLAN strip is necessary for 1G NIC such as I350,
 115                  * this fixes bug of ipv4 forwarding in guest can't
 116                  * forward pakets from one virtio dev to another virtio dev.
 117                  */
 118                 .offloads = DEV_RX_OFFLOAD_VLAN_STRIP,
 119         },
 120
 121         .txmode = {
 122                 .mq_mode = ETH_MQ_TX_NONE,
 123                 .offloads = (DEV_TX_OFFLOAD_IPV4_CKSUM |
 124                              DEV_TX_OFFLOAD_TCP_CKSUM |
 125                              DEV_TX_OFFLOAD_VLAN_INSERT |
 126                              DEV_TX_OFFLOAD_MULTI_SEGS |
 127                              DEV_TX_OFFLOAD_TCP_TSO),
 128         },
 129         .rx_adv_conf = {
 130                 /*
 131                  * should be overridden separately in code with
 132                  * appropriate values
 133                  */
 134                 .vmdq_rx_conf = {
 135                         .nb_queue_pools = ETH_8_POOLS,
 136                         .enable_default_pool = 0,
 137                         .default_pool = 0,
 138                         .nb_pool_maps = 0,
 139                         .pool_map = {{0, 0},},
 140                 },
 141         },
 142 };
 143
 144
 145 static unsigned lcore_ids[RTE_MAX_LCORE];
 146 static uint16_t ports[RTE_MAX_ETHPORTS];
 147 static unsigned num_ports = 0; /**< The number of ports specified in command line */
 148 static uint16_t num_pf_queues, num_vmdq_queues;
 149 static uint16_t vmdq_pool_base, vmdq_queue_base;
 150 static uint16_t queues_per_pool;
 151
 152 const uint16_t vlan_tags[] = {
 153         1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007,
 154         1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015,
 155         1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023,
 156         1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031,
 157         1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039,
 158         1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047,
 159         1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055,
 160         1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063,
 161 };
 162
 163 /* ethernet addresses of ports */
 164 static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
 165
 166 static struct vhost_dev_tailq_list vhost_dev_list =
 167         TAILQ_HEAD_INITIALIZER(vhost_dev_list);
 168
 169 static struct lcore_info lcore_info[RTE_MAX_LCORE];
 170
 171 /* Used for queueing bursts of TX packets. */
 172 struct mbuf_table {
 173         unsigned len;
 174         unsigned txq_id;
 175         struct rte_mbuf *m_table[MAX_PKT_BURST];
 176 };
 177
 178 /* TX queue for each data core. */
 179 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE];
 180
 181 #define MBUF_TABLE_DRAIN_TSC    ((rte_get_tsc_hz() + US_PER_S - 1) \
 182                                  / US_PER_S * BURST_TX_DRAIN_US)
 183 #define VLAN_HLEN       4
 184
 185 /*
 186  * Builds up the correct configuration for VMDQ VLAN pool map
 187  * according to the pool & queue limits.
 188  */
 189 static inline int
 190 get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices)
 191 {
 192         struct rte_eth_vmdq_rx_conf conf;
 193         struct rte_eth_vmdq_rx_conf *def_conf =
 194                 &vmdq_conf_default.rx_adv_conf.vmdq_rx_conf;
 195         unsigned i;
 196
 197         memset(&conf, 0, sizeof(conf));
 198         conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices;
 199         conf.nb_pool_maps = num_devices;
 200         conf.enable_loop_back = def_conf->enable_loop_back;
 201         conf.rx_mode = def_conf->rx_mode;
 202
 203         for (i = 0; i < conf.nb_pool_maps; i++) {
 204                 conf.pool_map[i].vlan_id = vlan_tags[ i ];
 205                 conf.pool_map[i].pools = (1UL << i);
 206         }
 207
 208         (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf)));
 209         (void)(rte_memcpy(&eth_conf->rx_adv_conf.vmdq_rx_conf, &conf,
 210                    sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf)));
 211         return 0;
 212 }
 213
 214 /*
 215  * Initialises a given port using global settings and with the rx buffers
 216  * coming from the mbuf_pool passed as parameter
 217  */
 218 static inline int
 219 port_init(uint16_t port)
 220 {
 221         struct rte_eth_dev_info dev_info;
 222         struct rte_eth_conf port_conf;
 223         struct rte_eth_rxconf *rxconf;
 224         struct rte_eth_txconf *txconf;
 225         int16_t rx_rings, tx_rings;
 226         uint16_t rx_ring_size, tx_ring_size;
 227         int retval;
 228         uint16_t q;
 229
 230         /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
 231         rte_eth_dev_info_get (port, &dev_info);
 232
 233         rxconf = &dev_info.default_rxconf;
 234         txconf = &dev_info.default_txconf;
 235         rxconf->rx_drop_en = 1;
 236
 237         /*configure the number of supported virtio devices based on VMDQ limits */
 238         num_devices = dev_info.max_vmdq_pools;
 239
 240         rx_ring_size = RTE_TEST_RX_DESC_DEFAULT;
 241         tx_ring_size = RTE_TEST_TX_DESC_DEFAULT;
 242
 243         /*
 244          * When dequeue zero copy is enabled, guest Tx used vring will be
 245          * updated only when corresponding mbuf is freed. Thus, the nb_tx_desc
 246          * (tx_ring_size here) must be small enough so that the driver will
 247          * hit the free threshold easily and free mbufs timely. Otherwise,
 248          * guest Tx vring would be starved.
 249          */
 250         if (dequeue_zero_copy)
 251                 tx_ring_size = 64;
 252
 253         tx_rings = (uint16_t)rte_lcore_count();
 254
 255         /* Get port configuration. */
 256         retval = get_eth_conf(&port_conf, num_devices);
 257         if (retval < 0)
 258                 return retval;
 259         /* NIC queues are divided into pf queues and vmdq queues.  */
 260         num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;
 261         queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;
 262         num_vmdq_queues = num_devices * queues_per_pool;
 263         num_queues = num_pf_queues + num_vmdq_queues;
 264         vmdq_queue_base = dev_info.vmdq_queue_base;
 265         vmdq_pool_base  = dev_info.vmdq_pool_base;
 266         printf("pf queue num: %u, configured vmdq pool num: %u, each vmdq pool has %u queues\n",
 267                 num_pf_queues, num_devices, queues_per_pool);
 268
 269         if (!rte_eth_dev_is_valid_port(port))
 270                 return -1;
 271
 272         rx_rings = (uint16_t)dev_info.max_rx_queues;
 273         if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 274                 port_conf.txmode.offloads |=
 275                         DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 276         /* Configure ethernet device. */
 277         retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
 278         if (retval != 0) {
 279                 RTE_LOG(ERR, VHOST_PORT, "Failed to configure port %u: %s.\n",
 280                         port, strerror(-retval));
 281                 return retval;
 282         }
 283
 284         retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &rx_ring_size,
 285                 &tx_ring_size);
 286         if (retval != 0) {
 287                 RTE_LOG(ERR, VHOST_PORT, "Failed to adjust number of descriptors "
 288                         "for port %u: %s.\n", port, strerror(-retval));
 289                 return retval;
 290         }
 291         if (rx_ring_size > RTE_TEST_RX_DESC_DEFAULT) {
 292                 RTE_LOG(ERR, VHOST_PORT, "Mbuf pool has an insufficient size "
 293                         "for Rx queues on port %u.\n", port);
 294                 return -1;
 295         }
 296
 297         /* Setup the queues. */
 298         rxconf->offloads = port_conf.rxmode.offloads;
 299         for (q = 0; q < rx_rings; q ++) {
 300                 retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
 301                                                 rte_eth_dev_socket_id(port),
 302                                                 rxconf,
 303                                                 mbuf_pool);
 304                 if (retval < 0) {
 305                         RTE_LOG(ERR, VHOST_PORT,
 306                                 "Failed to setup rx queue %u of port %u: %s.\n",
 307                                 q, port, strerror(-retval));
 308                         return retval;
 309                 }
 310         }
 311         txconf->offloads = port_conf.txmode.offloads;
 312         for (q = 0; q < tx_rings; q ++) {
 313                 retval = rte_eth_tx_queue_setup(port, q, tx_ring_size,
 314                                                 rte_eth_dev_socket_id(port),
 315                                                 txconf);
 316                 if (retval < 0) {
 317                         RTE_LOG(ERR, VHOST_PORT,
 318                                 "Failed to setup tx queue %u of port %u: %s.\n",
 319                                 q, port, strerror(-retval));
 320                         return retval;
 321                 }
 322         }
 323
 324         /* Start the device. */
 325         retval  = rte_eth_dev_start(port);
 326         if (retval < 0) {
 327                 RTE_LOG(ERR, VHOST_PORT, "Failed to start port %u: %s\n",
 328                         port, strerror(-retval));
 329                 return retval;
 330         }
 331
 332         if (promiscuous)
 333                 rte_eth_promiscuous_enable(port);
 334
 335         rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
 336         RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices);
 337         RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
 338                         " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
 339                         port,
 340                         vmdq_ports_eth_addr[port].addr_bytes[0],
 341                         vmdq_ports_eth_addr[port].addr_bytes[1],
 342                         vmdq_ports_eth_addr[port].addr_bytes[2],
 343                         vmdq_ports_eth_addr[port].addr_bytes[3],
 344                         vmdq_ports_eth_addr[port].addr_bytes[4],
 345                         vmdq_ports_eth_addr[port].addr_bytes[5]);
 346
 347         return 0;
 348 }
 349
 350 /*
 351  * Set socket file path.
 352  */
 353 static int
 354 us_vhost_parse_socket_path(const char *q_arg)
 355 {
 356         /* parse number string */
 357         if (strnlen(q_arg, PATH_MAX) == PATH_MAX)
 358                 return -1;
 359
 360         socket_files = realloc(socket_files, PATH_MAX * (nb_sockets + 1));
 361         snprintf(socket_files + nb_sockets * PATH_MAX, PATH_MAX, "%s", q_arg);
 362         nb_sockets++;
 363
 364         return 0;
 365 }
 366
 367 /*
 368  * Parse the portmask provided at run time.
 369  */
 370 static int
 371 parse_portmask(const char *portmask)
 372 {
 373         char *end = NULL;
 374         unsigned long pm;
 375
 376         errno = 0;
 377
 378         /* parse hexadecimal string */
 379         pm = strtoul(portmask, &end, 16);
 380         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 381                 return -1;
 382
 383         if (pm == 0)
 384                 return -1;
 385
 386         return pm;
 387
 388 }
 389
 390 /*
 391  * Parse num options at run time.
 392  */
 393 static int
 394 parse_num_opt(const char *q_arg, uint32_t max_valid_value)
 395 {
 396         char *end = NULL;
 397         unsigned long num;
 398
 399         errno = 0;
 400
 401         /* parse unsigned int string */
 402         num = strtoul(q_arg, &end, 10);
 403         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 404                 return -1;
 405
 406         if (num > max_valid_value)
 407                 return -1;
 408
 409         return num;
 410
 411 }
 412
 413 /*
 414  * Display usage
 415  */
 416 static void
 417 us_vhost_usage(const char *prgname)
 418 {
 419         RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n"
 420         "               --vm2vm [0|1|2]\n"
 421         "               --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n"
 422         "               --socket-file <path>\n"
 423         "               --nb-devices ND\n"
 424         "               -p PORTMASK: Set mask for ports to be used by application\n"
 425         "               --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n"
 426         "               --rx-retry [0|1]: disable/enable(default) retries on rx. Enable retry if destintation queue is full\n"
 427         "               --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX. This makes effect only if retries on rx enabled\n"
 428         "               --rx-retry-num [0-N]: the number of retries on rx. This makes effect only if retries on rx enabled\n"
 429         "               --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n"
 430         "               --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
 431         "               --socket-file: The path of the socket file.\n"
 432         "               --tx-csum [0|1] disable/enable TX checksum offload.\n"
 433         "               --tso [0|1] disable/enable TCP segment offload.\n"
 434         "               --client register a vhost-user socket as client mode.\n"
 435         "               --dequeue-zero-copy enables dequeue zero copy\n",
 436                prgname);
 437 }
 438
 439 /*
 440  * Parse the arguments given in the command line of the application.
 441  */
 442 static int
 443 us_vhost_parse_args(int argc, char **argv)
 444 {
 445         int opt, ret;
 446         int option_index;
 447         unsigned i;
 448         const char *prgname = argv[0];
 449         static struct option long_option[] = {
 450                 {"vm2vm", required_argument, NULL, 0},
 451                 {"rx-retry", required_argument, NULL, 0},
 452                 {"rx-retry-delay", required_argument, NULL, 0},
 453                 {"rx-retry-num", required_argument, NULL, 0},
 454                 {"mergeable", required_argument, NULL, 0},
 455                 {"stats", required_argument, NULL, 0},
 456                 {"socket-file", required_argument, NULL, 0},
 457                 {"tx-csum", required_argument, NULL, 0},
 458                 {"tso", required_argument, NULL, 0},
 459                 {"client", no_argument, &client_mode, 1},
 460                 {"dequeue-zero-copy", no_argument, &dequeue_zero_copy, 1},
 461                 {"builtin-net-driver", no_argument, &builtin_net_driver, 1},
 462                 {NULL, 0, 0, 0},
 463         };
 464
 465         /* Parse command line */
 466         while ((opt = getopt_long(argc, argv, "p:P",
 467                         long_option, &option_index)) != EOF) {
 468                 switch (opt) {
 469                 /* Portmask */
 470                 case 'p':
 471                         enabled_port_mask = parse_portmask(optarg);
 472                         if (enabled_port_mask == 0) {
 473                                 RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n");
 474                                 us_vhost_usage(prgname);
 475                                 return -1;
 476                         }
 477                         break;
 478
 479                 case 'P':
 480                         promiscuous = 1;
 481                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.rx_mode =
 482                                 ETH_VMDQ_ACCEPT_BROADCAST |
 483                                 ETH_VMDQ_ACCEPT_MULTICAST;
 484
 485                         break;
 486
 487                 case 0:
 488                         /* Enable/disable vm2vm comms. */
 489                         if (!strncmp(long_option[option_index].name, "vm2vm",
 490                                 MAX_LONG_OPT_SZ)) {
 491                                 ret = parse_num_opt(optarg, (VM2VM_LAST - 1));
 492                                 if (ret == -1) {
 493                                         RTE_LOG(INFO, VHOST_CONFIG,
 494                                                 "Invalid argument for "
 495                                                 "vm2vm [0|1|2]\n");
 496                                         us_vhost_usage(prgname);
 497                                         return -1;
 498                                 } else {
 499                                         vm2vm_mode = (vm2vm_type)ret;
 500                                 }
 501                         }
 502
 503                         /* Enable/disable retries on RX. */
 504                         if (!strncmp(long_option[option_index].name, "rx-retry", MAX_LONG_OPT_SZ)) {
 505                                 ret = parse_num_opt(optarg, 1);
 506                                 if (ret == -1) {
 507                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry [0|1]\n");
 508                                         us_vhost_usage(prgname);
 509                                         return -1;
 510                                 } else {
 511                                         enable_retry = ret;
 512                                 }
 513                         }
 514
 515                         /* Enable/disable TX checksum offload. */
 516                         if (!strncmp(long_option[option_index].name, "tx-csum", MAX_LONG_OPT_SZ)) {
 517                                 ret = parse_num_opt(optarg, 1);
 518                                 if (ret == -1) {
 519                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tx-csum [0|1]\n");
 520                                         us_vhost_usage(prgname);
 521                                         return -1;
 522                                 } else
 523                                         enable_tx_csum = ret;
 524                         }
 525
 526                         /* Enable/disable TSO offload. */
 527                         if (!strncmp(long_option[option_index].name, "tso", MAX_LONG_OPT_SZ)) {
 528                                 ret = parse_num_opt(optarg, 1);
 529                                 if (ret == -1) {
 530                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tso [0|1]\n");
 531                                         us_vhost_usage(prgname);
 532                                         return -1;
 533                                 } else
 534                                         enable_tso = ret;
 535                         }
 536
 537                         /* Specify the retries delay time (in useconds) on RX. */
 538                         if (!strncmp(long_option[option_index].name, "rx-retry-delay", MAX_LONG_OPT_SZ)) {
 539                                 ret = parse_num_opt(optarg, INT32_MAX);
 540                                 if (ret == -1) {
 541                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-delay [0-N]\n");
 542                                         us_vhost_usage(prgname);
 543                                         return -1;
 544                                 } else {
 545                                         burst_rx_delay_time = ret;
 546                                 }
 547                         }
 548
 549                         /* Specify the retries number on RX. */
 550                         if (!strncmp(long_option[option_index].name, "rx-retry-num", MAX_LONG_OPT_SZ)) {
 551                                 ret = parse_num_opt(optarg, INT32_MAX);
 552                                 if (ret == -1) {
 553                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-num [0-N]\n");
 554                                         us_vhost_usage(prgname);
 555                                         return -1;
 556                                 } else {
 557                                         burst_rx_retry_num = ret;
 558                                 }
 559                         }
 560
 561                         /* Enable/disable RX mergeable buffers. */
 562                         if (!strncmp(long_option[option_index].name, "mergeable", MAX_LONG_OPT_SZ)) {
 563                                 ret = parse_num_opt(optarg, 1);
 564                                 if (ret == -1) {
 565                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for mergeable [0|1]\n");
 566                                         us_vhost_usage(prgname);
 567                                         return -1;
 568                                 } else {
 569                                         mergeable = !!ret;
 570                                         if (ret) {
 571                                                 vmdq_conf_default.rxmode.offloads |=
 572                                                         DEV_RX_OFFLOAD_JUMBO_FRAME;
 573                                                 vmdq_conf_default.rxmode.max_rx_pkt_len
 574                                                         = JUMBO_FRAME_MAX_SIZE;
 575                                         }
 576                                 }
 577                         }
 578
 579                         /* Enable/disable stats. */
 580                         if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) {
 581                                 ret = parse_num_opt(optarg, INT32_MAX);
 582                                 if (ret == -1) {
 583                                         RTE_LOG(INFO, VHOST_CONFIG,
 584                                                 "Invalid argument for stats [0..N]\n");
 585                                         us_vhost_usage(prgname);
 586                                         return -1;
 587                                 } else {
 588                                         enable_stats = ret;
 589                                 }
 590                         }
 591
 592                         /* Set socket file path. */
 593                         if (!strncmp(long_option[option_index].name,
 594                                                 "socket-file", MAX_LONG_OPT_SZ)) {
 595                                 if (us_vhost_parse_socket_path(optarg) == -1) {
 596                                         RTE_LOG(INFO, VHOST_CONFIG,
 597                                         "Invalid argument for socket name (Max %d characters)\n",
 598                                         PATH_MAX);
 599                                         us_vhost_usage(prgname);
 600                                         return -1;
 601                                 }
 602                         }
 603
 604                         break;
 605
 606                         /* Invalid option - print options. */
 607                 default:
 608                         us_vhost_usage(prgname);
 609                         return -1;
 610                 }
 611         }
 612
 613         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 614                 if (enabled_port_mask & (1 << i))
 615                         ports[num_ports++] = i;
 616         }
 617
 618         if ((num_ports ==  0) || (num_ports > MAX_SUP_PORTS)) {
 619                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
 620                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
 621                 return -1;
 622         }
 623
 624         return 0;
 625 }
 626
 627 /*
 628  * Update the global var NUM_PORTS and array PORTS according to system ports number
 629  * and return valid ports number
 630  */
 631 static unsigned check_ports_num(unsigned nb_ports)
 632 {
 633         unsigned valid_num_ports = num_ports;
 634         unsigned portid;
 635
 636         if (num_ports > nb_ports) {
 637                 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n",
 638                         num_ports, nb_ports);
 639                 num_ports = nb_ports;
 640         }
 641
 642         for (portid = 0; portid < num_ports; portid ++) {
 643                 if (!rte_eth_dev_is_valid_port(ports[portid])) {
 644                         RTE_LOG(INFO, VHOST_PORT,
 645                                 "\nSpecified port ID(%u) is not valid\n",
 646                                 ports[portid]);
 647                         ports[portid] = INVALID_PORT_ID;
 648                         valid_num_ports--;
 649                 }
 650         }
 651         return valid_num_ports;
 652 }
 653
 654 static __rte_always_inline struct vhost_dev *
 655 find_vhost_dev(struct ether_addr *mac)
 656 {
 657         struct vhost_dev *vdev;
 658
 659         TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
 660                 if (vdev->ready == DEVICE_RX &&
 661                     is_same_ether_addr(mac, &vdev->mac_address))
 662                         return vdev;
 663         }
 664
 665         return NULL;
 666 }
 667
 668 /*
 669  * This function learns the MAC address of the device and registers this along with a
 670  * vlan tag to a VMDQ.
 671  */
 672 static int
 673 link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
 674 {
 675         struct ether_hdr *pkt_hdr;
 676         int i, ret;
 677
 678         /* Learn MAC address of guest device from packet */
 679         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 680
 681         if (find_vhost_dev(&pkt_hdr->s_addr)) {
 682                 RTE_LOG(ERR, VHOST_DATA,
 683                         "(%d) device is using a registered MAC!\n",
 684                         vdev->vid);
 685                 return -1;
 686         }
 687
 688         for (i = 0; i < ETHER_ADDR_LEN; i++)
 689                 vdev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i];
 690
 691         /* vlan_tag currently uses the device_id. */
 692         vdev->vlan_tag = vlan_tags[vdev->vid];
 693
 694         /* Print out VMDQ registration info. */
 695         RTE_LOG(INFO, VHOST_DATA,
 696                 "(%d) mac %02x:%02x:%02x:%02x:%02x:%02x and vlan %d registered\n",
 697                 vdev->vid,
 698                 vdev->mac_address.addr_bytes[0], vdev->mac_address.addr_bytes[1],
 699                 vdev->mac_address.addr_bytes[2], vdev->mac_address.addr_bytes[3],
 700                 vdev->mac_address.addr_bytes[4], vdev->mac_address.addr_bytes[5],
 701                 vdev->vlan_tag);
 702
 703         /* Register the MAC address. */
 704         ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address,
 705                                 (uint32_t)vdev->vid + vmdq_pool_base);
 706         if (ret)
 707                 RTE_LOG(ERR, VHOST_DATA,
 708                         "(%d) failed to add device MAC address to VMDQ\n",
 709                         vdev->vid);
 710
 711         rte_eth_dev_set_vlan_strip_on_queue(ports[0], vdev->vmdq_rx_q, 1);
 712
 713         /* Set device as ready for RX. */
 714         vdev->ready = DEVICE_RX;
 715
 716         return 0;
 717 }
 718
 719 /*
 720  * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX
 721  * queue before disabling RX on the device.
 722  */
 723 static inline void
 724 unlink_vmdq(struct vhost_dev *vdev)
 725 {
 726         unsigned i = 0;
 727         unsigned rx_count;
 728         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 729
 730         if (vdev->ready == DEVICE_RX) {
 731                 /*clear MAC and VLAN settings*/
 732                 rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address);
 733                 for (i = 0; i < 6; i++)
 734                         vdev->mac_address.addr_bytes[i] = 0;
 735
 736                 vdev->vlan_tag = 0;
 737
 738                 /*Clear out the receive buffers*/
 739                 rx_count = rte_eth_rx_burst(ports[0],
 740                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
 741
 742                 while (rx_count) {
 743                         for (i = 0; i < rx_count; i++)
 744                                 rte_pktmbuf_free(pkts_burst[i]);
 745
 746                         rx_count = rte_eth_rx_burst(ports[0],
 747                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
 748                 }
 749
 750                 vdev->ready = DEVICE_MAC_LEARNING;
 751         }
 752 }
 753
 754 static __rte_always_inline void
 755 virtio_xmit(struct vhost_dev *dst_vdev, struct vhost_dev *src_vdev,
 756             struct rte_mbuf *m)
 757 {
 758         uint16_t ret;
 759
 760         if (builtin_net_driver) {
 761                 ret = vs_enqueue_pkts(dst_vdev, VIRTIO_RXQ, &m, 1);
 762         } else {
 763                 ret = rte_vhost_enqueue_burst(dst_vdev->vid, VIRTIO_RXQ, &m, 1);
 764         }
 765
 766         if (enable_stats) {
 767                 rte_atomic64_inc(&dst_vdev->stats.rx_total_atomic);
 768                 rte_atomic64_add(&dst_vdev->stats.rx_atomic, ret);
 769                 src_vdev->stats.tx_total++;
 770                 src_vdev->stats.tx += ret;
 771         }
 772 }
 773
 774 /*
 775  * Check if the packet destination MAC address is for a local device. If so then put
 776  * the packet on that devices RX queue. If not then return.
 777  */
 778 static __rte_always_inline int
 779 virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
 780 {
 781         struct ether_hdr *pkt_hdr;
 782         struct vhost_dev *dst_vdev;
 783
 784         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 785
 786         dst_vdev = find_vhost_dev(&pkt_hdr->d_addr);
 787         if (!dst_vdev)
 788                 return -1;
 789
 790         if (vdev->vid == dst_vdev->vid) {
 791                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 792                         "(%d) TX: src and dst MAC is same. Dropping packet.\n",
 793                         vdev->vid);
 794                 return 0;
 795         }
 796
 797         RTE_LOG_DP(DEBUG, VHOST_DATA,
 798                 "(%d) TX: MAC address is local\n", dst_vdev->vid);
 799
 800         if (unlikely(dst_vdev->remove)) {
 801                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 802                         "(%d) device is marked for removal\n", dst_vdev->vid);
 803                 return 0;
 804         }
 805
 806         virtio_xmit(dst_vdev, vdev, m);
 807         return 0;
 808 }
 809
 810 /*
 811  * Check if the destination MAC of a packet is one local VM,
 812  * and get its vlan tag, and offset if it is.
 813  */
 814 static __rte_always_inline int
 815 find_local_dest(struct vhost_dev *vdev, struct rte_mbuf *m,
 816         uint32_t *offset, uint16_t *vlan_tag)
 817 {
 818         struct vhost_dev *dst_vdev;
 819         struct ether_hdr *pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 820
 821         dst_vdev = find_vhost_dev(&pkt_hdr->d_addr);
 822         if (!dst_vdev)
 823                 return 0;
 824
 825         if (vdev->vid == dst_vdev->vid) {
 826                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 827                         "(%d) TX: src and dst MAC is same. Dropping packet.\n",
 828                         vdev->vid);
 829                 return -1;
 830         }
 831
 832         /*
 833          * HW vlan strip will reduce the packet length
 834          * by minus length of vlan tag, so need restore
 835          * the packet length by plus it.
 836          */
 837         *offset  = VLAN_HLEN;
 838         *vlan_tag = vlan_tags[vdev->vid];
 839
 840         RTE_LOG_DP(DEBUG, VHOST_DATA,
 841                 "(%d) TX: pkt to local VM device id: (%d), vlan tag: %u.\n",
 842                 vdev->vid, dst_vdev->vid, *vlan_tag);
 843
 844         return 0;
 845 }
 846
 847 static uint16_t
 848 get_psd_sum(void *l3_hdr, uint64_t ol_flags)
 849 {
 850         if (ol_flags & PKT_TX_IPV4)
 851                 return rte_ipv4_phdr_cksum(l3_hdr, ol_flags);
 852         else /* assume ethertype == ETHER_TYPE_IPv6 */
 853                 return rte_ipv6_phdr_cksum(l3_hdr, ol_flags);
 854 }
 855
 856 static void virtio_tx_offload(struct rte_mbuf *m)
 857 {
 858         void *l3_hdr;
 859         struct ipv4_hdr *ipv4_hdr = NULL;
 860         struct tcp_hdr *tcp_hdr = NULL;
 861         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 862
 863         l3_hdr = (char *)eth_hdr + m->l2_len;
 864
 865         if (m->ol_flags & PKT_TX_IPV4) {
 866                 ipv4_hdr = l3_hdr;
 867                 ipv4_hdr->hdr_checksum = 0;
 868                 m->ol_flags |= PKT_TX_IP_CKSUM;
 869         }
 870
 871         tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + m->l3_len);
 872         tcp_hdr->cksum = get_psd_sum(l3_hdr, m->ol_flags);
 873 }
 874
 875 static inline void
 876 free_pkts(struct rte_mbuf **pkts, uint16_t n)
 877 {
 878         while (n--)
 879                 rte_pktmbuf_free(pkts[n]);
 880 }
 881
 882 static __rte_always_inline void
 883 do_drain_mbuf_table(struct mbuf_table *tx_q)
 884 {
 885         uint16_t count;
 886
 887         count = rte_eth_tx_burst(ports[0], tx_q->txq_id,
 888                                  tx_q->m_table, tx_q->len);
 889         if (unlikely(count < tx_q->len))
 890                 free_pkts(&tx_q->m_table[count], tx_q->len - count);
 891
 892         tx_q->len = 0;
 893 }
 894
 895 /*
 896  * This function routes the TX packet to the correct interface. This
 897  * may be a local device or the physical port.
 898  */
 899 static __rte_always_inline void
 900 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)
 901 {
 902         struct mbuf_table *tx_q;
 903         unsigned offset = 0;
 904         const uint16_t lcore_id = rte_lcore_id();
 905         struct ether_hdr *nh;
 906
 907
 908         nh = rte_pktmbuf_mtod(m, struct ether_hdr *);
 909         if (unlikely(is_broadcast_ether_addr(&nh->d_addr))) {
 910                 struct vhost_dev *vdev2;
 911
 912                 TAILQ_FOREACH(vdev2, &vhost_dev_list, global_vdev_entry) {
 913                         if (vdev2 != vdev)
 914                                 virtio_xmit(vdev2, vdev, m);
 915                 }
 916                 goto queue2nic;
 917         }
 918
 919         /*check if destination is local VM*/
 920         if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) {
 921                 rte_pktmbuf_free(m);
 922                 return;
 923         }
 924
 925         if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
 926                 if (unlikely(find_local_dest(vdev, m, &offset,
 927                                              &vlan_tag) != 0)) {
 928                         rte_pktmbuf_free(m);
 929                         return;
 930                 }
 931         }
 932
 933         RTE_LOG_DP(DEBUG, VHOST_DATA,
 934                 "(%d) TX: MAC address is external\n", vdev->vid);
 935
 936 queue2nic:
 937
 938         /*Add packet to the port tx queue*/
 939         tx_q = &lcore_tx_queue[lcore_id];
 940
 941         nh = rte_pktmbuf_mtod(m, struct ether_hdr *);
 942         if (unlikely(nh->ether_type == rte_cpu_to_be_16(ETHER_TYPE_VLAN))) {
 943                 /* Guest has inserted the vlan tag. */
 944                 struct vlan_hdr *vh = (struct vlan_hdr *) (nh + 1);
 945                 uint16_t vlan_tag_be = rte_cpu_to_be_16(vlan_tag);
 946                 if ((vm2vm_mode == VM2VM_HARDWARE) &&
 947                         (vh->vlan_tci != vlan_tag_be))
 948                         vh->vlan_tci = vlan_tag_be;
 949         } else {
 950                 m->ol_flags |= PKT_TX_VLAN_PKT;
 951
 952                 /*
 953                  * Find the right seg to adjust the data len when offset is
 954                  * bigger than tail room size.
 955                  */
 956                 if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
 957                         if (likely(offset <= rte_pktmbuf_tailroom(m)))
 958                                 m->data_len += offset;
 959                         else {
 960                                 struct rte_mbuf *seg = m;
 961
 962                                 while ((seg->next != NULL) &&
 963                                         (offset > rte_pktmbuf_tailroom(seg)))
 964                                         seg = seg->next;
 965
 966                                 seg->data_len += offset;
 967                         }
 968                         m->pkt_len += offset;
 969                 }
 970
 971                 m->vlan_tci = vlan_tag;
 972         }
 973
 974         if (m->ol_flags & PKT_TX_TCP_SEG)
 975                 virtio_tx_offload(m);
 976
 977         tx_q->m_table[tx_q->len++] = m;
 978         if (enable_stats) {
 979                 vdev->stats.tx_total++;
 980                 vdev->stats.tx++;
 981         }
 982
 983         if (unlikely(tx_q->len == MAX_PKT_BURST))
 984                 do_drain_mbuf_table(tx_q);
 985 }
 986
 987
 988 static __rte_always_inline void
 989 drain_mbuf_table(struct mbuf_table *tx_q)
 990 {
 991         static uint64_t prev_tsc;
 992         uint64_t cur_tsc;
 993
 994         if (tx_q->len == 0)
 995                 return;
 996
 997         cur_tsc = rte_rdtsc();
 998         if (unlikely(cur_tsc - prev_tsc > MBUF_TABLE_DRAIN_TSC)) {
 999                 prev_tsc = cur_tsc;
1000
1001                 RTE_LOG_DP(DEBUG, VHOST_DATA,
1002                         "TX queue drained after timeout with burst size %u\n",
1003                         tx_q->len);
1004                 do_drain_mbuf_table(tx_q);
1005         }
1006 }
1007
1008 static __rte_always_inline void
1009 drain_eth_rx(struct vhost_dev *vdev)
1010 {
1011         uint16_t rx_count, enqueue_count;
1012         struct rte_mbuf *pkts[MAX_PKT_BURST];
1013
1014         rx_count = rte_eth_rx_burst(ports[0], vdev->vmdq_rx_q,
1015                                     pkts, MAX_PKT_BURST);
1016         if (!rx_count)
1017                 return;
1018
1019         /*
1020          * When "enable_retry" is set, here we wait and retry when there
1021          * is no enough free slots in the queue to hold @rx_count packets,
1022          * to diminish packet loss.
1023          */
1024         if (enable_retry &&
1025             unlikely(rx_count > rte_vhost_avail_entries(vdev->vid,
1026                         VIRTIO_RXQ))) {
1027                 uint32_t retry;
1028
1029                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
1030                         rte_delay_us(burst_rx_delay_time);
1031                         if (rx_count <= rte_vhost_avail_entries(vdev->vid,
1032                                         VIRTIO_RXQ))
1033                                 break;
1034                 }
1035         }
1036
1037         if (builtin_net_driver) {
1038                 enqueue_count = vs_enqueue_pkts(vdev, VIRTIO_RXQ,
1039                                                 pkts, rx_count);
1040         } else {
1041                 enqueue_count = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ,
1042                                                 pkts, rx_count);
1043         }
1044         if (enable_stats) {
1045                 rte_atomic64_add(&vdev->stats.rx_total_atomic, rx_count);
1046                 rte_atomic64_add(&vdev->stats.rx_atomic, enqueue_count);
1047         }
1048
1049         free_pkts(pkts, rx_count);
1050 }
1051
1052 static __rte_always_inline void
1053 drain_virtio_tx(struct vhost_dev *vdev)
1054 {
1055         struct rte_mbuf *pkts[MAX_PKT_BURST];
1056         uint16_t count;
1057         uint16_t i;
1058
1059         if (builtin_net_driver) {
1060                 count = vs_dequeue_pkts(vdev, VIRTIO_TXQ, mbuf_pool,
1061                                         pkts, MAX_PKT_BURST);
1062         } else {
1063                 count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ,
1064                                         mbuf_pool, pkts, MAX_PKT_BURST);
1065         }
1066
1067         /* setup VMDq for the first packet */
1068         if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count) {
1069                 if (vdev->remove || link_vmdq(vdev, pkts[0]) == -1)
1070                         free_pkts(pkts, count);
1071         }
1072
1073         for (i = 0; i < count; ++i)
1074                 virtio_tx_route(vdev, pkts[i], vlan_tags[vdev->vid]);
1075 }
1076
1077 /*
1078  * Main function of vhost-switch. It basically does:
1079  *
1080  * for each vhost device {
1081  *    - drain_eth_rx()
1082  *
1083  *      Which drains the host eth Rx queue linked to the vhost device,
1084  *      and deliver all of them to guest virito Rx ring associated with
1085  *      this vhost device.
1086  *
1087  *    - drain_virtio_tx()
1088  *
1089  *      Which drains the guest virtio Tx queue and deliver all of them
1090  *      to the target, which could be another vhost device, or the
1091  *      physical eth dev. The route is done in function "virtio_tx_route".
1092  * }
1093  */
1094 static int
1095 switch_worker(void *arg __rte_unused)
1096 {
1097         unsigned i;
1098         unsigned lcore_id = rte_lcore_id();
1099         struct vhost_dev *vdev;
1100         struct mbuf_table *tx_q;
1101
1102         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
1103
1104         tx_q = &lcore_tx_queue[lcore_id];
1105         for (i = 0; i < rte_lcore_count(); i++) {
1106                 if (lcore_ids[i] == lcore_id) {
1107                         tx_q->txq_id = i;
1108                         break;
1109                 }
1110         }
1111
1112         while(1) {
1113                 drain_mbuf_table(tx_q);
1114
1115                 /*
1116                  * Inform the configuration core that we have exited the
1117                  * linked list and that no devices are in use if requested.
1118                  */
1119                 if (lcore_info[lcore_id].dev_removal_flag == REQUEST_DEV_REMOVAL)
1120                         lcore_info[lcore_id].dev_removal_flag = ACK_DEV_REMOVAL;
1121
1122                 /*
1123                  * Process vhost devices
1124                  */
1125                 TAILQ_FOREACH(vdev, &lcore_info[lcore_id].vdev_list,
1126                               lcore_vdev_entry) {
1127                         if (unlikely(vdev->remove)) {
1128                                 unlink_vmdq(vdev);
1129                                 vdev->ready = DEVICE_SAFE_REMOVE;
1130                                 continue;
1131                         }
1132
1133                         if (likely(vdev->ready == DEVICE_RX))
1134                                 drain_eth_rx(vdev);
1135
1136                         if (likely(!vdev->remove))
1137                                 drain_virtio_tx(vdev);
1138                 }
1139         }
1140
1141         return 0;
1142 }
1143
1144 /*
1145  * Remove a device from the specific data core linked list and from the
1146  * main linked list. Synchonization  occurs through the use of the
1147  * lcore dev_removal_flag. Device is made volatile here to avoid re-ordering
1148  * of dev->remove=1 which can cause an infinite loop in the rte_pause loop.
1149  */
1150 static void
1151 destroy_device(int vid)
1152 {
1153         struct vhost_dev *vdev = NULL;
1154         int lcore;
1155
1156         TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
1157                 if (vdev->vid == vid)
1158                         break;
1159         }
1160         if (!vdev)
1161                 return;
1162         /*set the remove flag. */
1163         vdev->remove = 1;
1164         while(vdev->ready != DEVICE_SAFE_REMOVE) {
1165                 rte_pause();
1166         }
1167
1168         if (builtin_net_driver)
1169                 vs_vhost_net_remove(vdev);
1170
1171         TAILQ_REMOVE(&lcore_info[vdev->coreid].vdev_list, vdev,
1172                      lcore_vdev_entry);
1173         TAILQ_REMOVE(&vhost_dev_list, vdev, global_vdev_entry);
1174
1175
1176         /* Set the dev_removal_flag on each lcore. */
1177         RTE_LCORE_FOREACH_SLAVE(lcore)
1178                 lcore_info[lcore].dev_removal_flag = REQUEST_DEV_REMOVAL;
1179
1180         /*
1181          * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL
1182          * we can be sure that they can no longer access the device removed
1183          * from the linked lists and that the devices are no longer in use.
1184          */
1185         RTE_LCORE_FOREACH_SLAVE(lcore) {
1186                 while (lcore_info[lcore].dev_removal_flag != ACK_DEV_REMOVAL)
1187                         rte_pause();
1188         }
1189
1190         lcore_info[vdev->coreid].device_num--;
1191
1192         RTE_LOG(INFO, VHOST_DATA,
1193                 "(%d) device has been removed from data core\n",
1194                 vdev->vid);
1195
1196         rte_free(vdev);
1197 }
1198
1199 /*
1200  * A new device is added to a data core. First the device is added to the main linked list
1201  * and the allocated to a specific data core.
1202  */
1203 static int
1204 new_device(int vid)
1205 {
1206         int lcore, core_add = 0;
1207         uint32_t device_num_min = num_devices;
1208         struct vhost_dev *vdev;
1209
1210         vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
1211         if (vdev == NULL) {
1212                 RTE_LOG(INFO, VHOST_DATA,
1213                         "(%d) couldn't allocate memory for vhost dev\n",
1214                         vid);
1215                 return -1;
1216         }
1217         vdev->vid = vid;
1218
1219         if (builtin_net_driver)
1220                 vs_vhost_net_setup(vdev);
1221
1222         TAILQ_INSERT_TAIL(&vhost_dev_list, vdev, global_vdev_entry);
1223         vdev->vmdq_rx_q = vid * queues_per_pool + vmdq_queue_base;
1224
1225         /*reset ready flag*/
1226         vdev->ready = DEVICE_MAC_LEARNING;
1227         vdev->remove = 0;
1228
1229         /* Find a suitable lcore to add the device. */
1230         RTE_LCORE_FOREACH_SLAVE(lcore) {
1231                 if (lcore_info[lcore].device_num < device_num_min) {
1232                         device_num_min = lcore_info[lcore].device_num;
1233                         core_add = lcore;
1234                 }
1235         }
1236         vdev->coreid = core_add;
1237
1238         TAILQ_INSERT_TAIL(&lcore_info[vdev->coreid].vdev_list, vdev,
1239                           lcore_vdev_entry);
1240         lcore_info[vdev->coreid].device_num++;
1241
1242         /* Disable notifications. */
1243         rte_vhost_enable_guest_notification(vid, VIRTIO_RXQ, 0);
1244         rte_vhost_enable_guest_notification(vid, VIRTIO_TXQ, 0);
1245
1246         RTE_LOG(INFO, VHOST_DATA,
1247                 "(%d) device has been added to data core %d\n",
1248                 vid, vdev->coreid);
1249
1250         return 0;
1251 }
1252
1253 /*
1254  * These callback allow devices to be added to the data core when configuration
1255  * has been fully complete.
1256  */
1257 static const struct vhost_device_ops virtio_net_device_ops =
1258 {
1259         .new_device =  new_device,
1260         .destroy_device = destroy_device,
1261 };
1262
1263 /*
1264  * This is a thread will wake up after a period to print stats if the user has
1265  * enabled them.
1266  */
1267 static void *
1268 print_stats(__rte_unused void *arg)
1269 {
1270         struct vhost_dev *vdev;
1271         uint64_t tx_dropped, rx_dropped;
1272         uint64_t tx, tx_total, rx, rx_total;
1273         const char clr[] = { 27, '[', '2', 'J', '\0' };
1274         const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' };
1275
1276         while(1) {
1277                 sleep(enable_stats);
1278
1279                 /* Clear screen and move to top left */
1280                 printf("%s%s\n", clr, top_left);
1281                 printf("Device statistics =================================\n");
1282
1283                 TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
1284                         tx_total   = vdev->stats.tx_total;
1285                         tx         = vdev->stats.tx;
1286                         tx_dropped = tx_total - tx;
1287
1288                         rx_total   = rte_atomic64_read(&vdev->stats.rx_total_atomic);
1289                         rx         = rte_atomic64_read(&vdev->stats.rx_atomic);
1290                         rx_dropped = rx_total - rx;
1291
1292                         printf("Statistics for device %d\n"
1293                                 "-----------------------\n"
1294                                 "TX total:              %" PRIu64 "\n"
1295                                 "TX dropped:            %" PRIu64 "\n"
1296                                 "TX successful:         %" PRIu64 "\n"
1297                                 "RX total:              %" PRIu64 "\n"
1298                                 "RX dropped:            %" PRIu64 "\n"
1299                                 "RX successful:         %" PRIu64 "\n",
1300                                 vdev->vid,
1301                                 tx_total, tx_dropped, tx,
1302                                 rx_total, rx_dropped, rx);
1303                 }
1304
1305                 printf("===================================================\n");
1306         }
1307
1308         return NULL;
1309 }
1310
1311 static void
1312 unregister_drivers(int socket_num)
1313 {
1314         int i, ret;
1315
1316         for (i = 0; i < socket_num; i++) {
1317                 ret = rte_vhost_driver_unregister(socket_files + i * PATH_MAX);
1318                 if (ret != 0)
1319                         RTE_LOG(ERR, VHOST_CONFIG,
1320                                 "Fail to unregister vhost driver for %s.\n",
1321                                 socket_files + i * PATH_MAX);
1322         }
1323 }
1324
1325 /* When we receive a INT signal, unregister vhost driver */
1326 static void
1327 sigint_handler(__rte_unused int signum)
1328 {
1329         /* Unregister vhost driver. */
1330         unregister_drivers(nb_sockets);
1331
1332         exit(0);
1333 }
1334
1335 /*
1336  * While creating an mbuf pool, one key thing is to figure out how
1337  * many mbuf entries is enough for our use. FYI, here are some
1338  * guidelines:
1339  *
1340  * - Each rx queue would reserve @nr_rx_desc mbufs at queue setup stage
1341  *
1342  * - For each switch core (A CPU core does the packet switch), we need
1343  *   also make some reservation for receiving the packets from virtio
1344  *   Tx queue. How many is enough depends on the usage. It's normally
1345  *   a simple calculation like following:
1346  *
1347  *       MAX_PKT_BURST * max packet size / mbuf size
1348  *
1349  *   So, we definitely need allocate more mbufs when TSO is enabled.
1350  *
1351  * - Similarly, for each switching core, we should serve @nr_rx_desc
1352  *   mbufs for receiving the packets from physical NIC device.
1353  *
1354  * - We also need make sure, for each switch core, we have allocated
1355  *   enough mbufs to fill up the mbuf cache.
1356  */
1357 static void
1358 create_mbuf_pool(uint16_t nr_port, uint32_t nr_switch_core, uint32_t mbuf_size,
1359         uint32_t nr_queues, uint32_t nr_rx_desc, uint32_t nr_mbuf_cache)
1360 {
1361         uint32_t nr_mbufs;
1362         uint32_t nr_mbufs_per_core;
1363         uint32_t mtu = 1500;
1364
1365         if (mergeable)
1366                 mtu = 9000;
1367         if (enable_tso)
1368                 mtu = 64 * 1024;
1369
1370         nr_mbufs_per_core  = (mtu + mbuf_size) * MAX_PKT_BURST /
1371                         (mbuf_size - RTE_PKTMBUF_HEADROOM);
1372         nr_mbufs_per_core += nr_rx_desc;
1373         nr_mbufs_per_core  = RTE_MAX(nr_mbufs_per_core, nr_mbuf_cache);
1374
1375         nr_mbufs  = nr_queues * nr_rx_desc;
1376         nr_mbufs += nr_mbufs_per_core * nr_switch_core;
1377         nr_mbufs *= nr_port;
1378
1379         mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", nr_mbufs,
1380                                             nr_mbuf_cache, 0, mbuf_size,
1381                                             rte_socket_id());
1382         if (mbuf_pool == NULL)
1383                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
1384 }
1385
1386 /*
1387  * Main function, does initialisation and calls the per-lcore functions.
1388  */
1389 int
1390 main(int argc, char *argv[])
1391 {
1392         unsigned lcore_id, core_id = 0;
1393         unsigned nb_ports, valid_num_ports;
1394         int ret, i;
1395         uint16_t portid;
1396         static pthread_t tid;
1397         uint64_t flags = 0;
1398
1399         signal(SIGINT, sigint_handler);
1400
1401         /* init EAL */
1402         ret = rte_eal_init(argc, argv);
1403         if (ret < 0)
1404                 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
1405         argc -= ret;
1406         argv += ret;
1407
1408         /* parse app arguments */
1409         ret = us_vhost_parse_args(argc, argv);
1410         if (ret < 0)
1411                 rte_exit(EXIT_FAILURE, "Invalid argument\n");
1412
1413         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1414                 TAILQ_INIT(&lcore_info[lcore_id].vdev_list);
1415
1416                 if (rte_lcore_is_enabled(lcore_id))
1417                         lcore_ids[core_id++] = lcore_id;
1418         }
1419
1420         if (rte_lcore_count() > RTE_MAX_LCORE)
1421                 rte_exit(EXIT_FAILURE,"Not enough cores\n");
1422
1423         /* Get the number of physical ports. */
1424         nb_ports = rte_eth_dev_count_avail();
1425
1426         /*
1427          * Update the global var NUM_PORTS and global array PORTS
1428          * and get value of var VALID_NUM_PORTS according to system ports number
1429          */
1430         valid_num_ports = check_ports_num(nb_ports);
1431
1432         if ((valid_num_ports ==  0) || (valid_num_ports > MAX_SUP_PORTS)) {
1433                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
1434                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
1435                 return -1;
1436         }
1437
1438         /*
1439          * FIXME: here we are trying to allocate mbufs big enough for
1440          * @MAX_QUEUES, but the truth is we're never going to use that
1441          * many queues here. We probably should only do allocation for
1442          * those queues we are going to use.
1443          */
1444         create_mbuf_pool(valid_num_ports, rte_lcore_count() - 1, MBUF_DATA_SIZE,
1445                          MAX_QUEUES, RTE_TEST_RX_DESC_DEFAULT, MBUF_CACHE_SIZE);
1446
1447         if (vm2vm_mode == VM2VM_HARDWARE) {
1448                 /* Enable VT loop back to let L2 switch to do it. */
1449                 vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1;
1450                 RTE_LOG(DEBUG, VHOST_CONFIG,
1451                         "Enable loop back for L2 switch in vmdq.\n");
1452         }
1453
1454         /* initialize all ports */
1455         RTE_ETH_FOREACH_DEV(portid) {
1456                 /* skip ports that are not enabled */
1457                 if ((enabled_port_mask & (1 << portid)) == 0) {
1458                         RTE_LOG(INFO, VHOST_PORT,
1459                                 "Skipping disabled port %d\n", portid);
1460                         continue;
1461                 }
1462                 if (port_init(portid) != 0)
1463                         rte_exit(EXIT_FAILURE,
1464                                 "Cannot initialize network ports\n");
1465         }
1466
1467         /* Enable stats if the user option is set. */
1468         if (enable_stats) {
1469                 ret = rte_ctrl_thread_create(&tid, "print-stats", NULL,
1470                                         print_stats, NULL);
1471                 if (ret < 0)
1472                         rte_exit(EXIT_FAILURE,
1473                                 "Cannot create print-stats thread\n");
1474         }
1475
1476         /* Launch all data cores. */
1477         RTE_LCORE_FOREACH_SLAVE(lcore_id)
1478                 rte_eal_remote_launch(switch_worker, NULL, lcore_id);
1479
1480         if (client_mode)
1481                 flags |= RTE_VHOST_USER_CLIENT;
1482
1483         if (dequeue_zero_copy)
1484                 flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
1485
1486         /* Register vhost user driver to handle vhost messages. */
1487         for (i = 0; i < nb_sockets; i++) {
1488                 char *file = socket_files + i * PATH_MAX;
1489                 ret = rte_vhost_driver_register(file, flags);
1490                 if (ret != 0) {
1491                         unregister_drivers(i);
1492                         rte_exit(EXIT_FAILURE,
1493                                 "vhost driver register failure.\n");
1494                 }
1495
1496                 if (builtin_net_driver)
1497                         rte_vhost_driver_set_features(file, VIRTIO_NET_FEATURES);
1498
1499                 if (mergeable == 0) {
1500                         rte_vhost_driver_disable_features(file,
1501                                 1ULL << VIRTIO_NET_F_MRG_RXBUF);
1502                 }
1503
1504                 if (enable_tx_csum == 0) {
1505                         rte_vhost_driver_disable_features(file,
1506                                 1ULL << VIRTIO_NET_F_CSUM);
1507                 }
1508
1509                 if (enable_tso == 0) {
1510                         rte_vhost_driver_disable_features(file,
1511                                 1ULL << VIRTIO_NET_F_HOST_TSO4);
1512                         rte_vhost_driver_disable_features(file,
1513                                 1ULL << VIRTIO_NET_F_HOST_TSO6);
1514                         rte_vhost_driver_disable_features(file,
1515                                 1ULL << VIRTIO_NET_F_GUEST_TSO4);
1516                         rte_vhost_driver_disable_features(file,
1517                                 1ULL << VIRTIO_NET_F_GUEST_TSO6);
1518                 }
1519
1520                 if (promiscuous) {
1521                         rte_vhost_driver_enable_features(file,
1522                                 1ULL << VIRTIO_NET_F_CTRL_RX);
1523                 }
1524
1525                 ret = rte_vhost_driver_callback_register(file,
1526                         &virtio_net_device_ops);
1527                 if (ret != 0) {
1528                         rte_exit(EXIT_FAILURE,
1529                                 "failed to register vhost driver callbacks.\n");
1530                 }
1531
1532                 if (rte_vhost_driver_start(file) < 0) {
1533                         rte_exit(EXIT_FAILURE,
1534                                 "failed to start vhost driver.\n");
1535                 }
1536         }
1537
1538         RTE_LCORE_FOREACH_SLAVE(lcore_id)
1539                 rte_eal_wait_lcore(lcore_id);
1540
1541         return 0;
1542
1543 }