examples/vhost/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <arpa/inet.h>
   6 #include <getopt.h>
   7 #include <linux/if_ether.h>
   8 #include <linux/if_vlan.h>
   9 #include <linux/virtio_net.h>
  10 #include <linux/virtio_ring.h>
  11 #include <signal.h>
  12 #include <stdint.h>
  13 #include <sys/eventfd.h>
  14 #include <sys/param.h>
  15 #include <unistd.h>
  16
  17 #include <rte_atomic.h>
  18 #include <rte_cycles.h>
  19 #include <rte_ethdev.h>
  20 #include <rte_log.h>
  21 #include <rte_string_fns.h>
  22 #include <rte_malloc.h>
  23 #include <rte_vhost.h>
  24 #include <rte_ip.h>
  25 #include <rte_tcp.h>
  26 #include <rte_pause.h>
  27
  28 #include "main.h"
  29
  30 #ifndef MAX_QUEUES
  31 #define MAX_QUEUES 128
  32 #endif
  33
  34 /* the maximum number of external ports supported */
  35 #define MAX_SUP_PORTS 1
  36
  37 #define MBUF_CACHE_SIZE 128
  38 #define MBUF_DATA_SIZE  RTE_MBUF_DEFAULT_BUF_SIZE
  39
  40 #define BURST_TX_DRAIN_US 100   /* TX drain every ~100us */
  41
  42 #define BURST_RX_WAIT_US 15     /* Defines how long we wait between retries on RX */
  43 #define BURST_RX_RETRIES 4              /* Number of retries on RX. */
  44
  45 #define JUMBO_FRAME_MAX_SIZE    0x2600
  46
  47 /* State of virtio device. */
  48 #define DEVICE_MAC_LEARNING 0
  49 #define DEVICE_RX                       1
  50 #define DEVICE_SAFE_REMOVE      2
  51
  52 /* Configurable number of RX/TX ring descriptors */
  53 #define RTE_TEST_RX_DESC_DEFAULT 1024
  54 #define RTE_TEST_TX_DESC_DEFAULT 512
  55
  56 #define INVALID_PORT_ID 0xFF
  57
  58 /* Maximum long option length for option parsing. */
  59 #define MAX_LONG_OPT_SZ 64
  60
  61 /* mask of enabled ports */
  62 static uint32_t enabled_port_mask = 0;
  63
  64 /* Promiscuous mode */
  65 static uint32_t promiscuous;
  66
  67 /* number of devices/queues to support*/
  68 static uint32_t num_queues = 0;
  69 static uint32_t num_devices;
  70
  71 static struct rte_mempool *mbuf_pool;
  72 static int mergeable;
  73
  74 /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */
  75 typedef enum {
  76         VM2VM_DISABLED = 0,
  77         VM2VM_SOFTWARE = 1,
  78         VM2VM_HARDWARE = 2,
  79         VM2VM_LAST
  80 } vm2vm_type;
  81 static vm2vm_type vm2vm_mode = VM2VM_SOFTWARE;
  82
  83 /* Enable stats. */
  84 static uint32_t enable_stats = 0;
  85 /* Enable retries on RX. */
  86 static uint32_t enable_retry = 1;
  87
  88 /* Disable TX checksum offload */
  89 static uint32_t enable_tx_csum;
  90
  91 /* Disable TSO offload */
  92 static uint32_t enable_tso;
  93
  94 static int client_mode;
  95 static int dequeue_zero_copy;
  96
  97 static int builtin_net_driver;
  98
  99 /* Specify timeout (in useconds) between retries on RX. */
 100 static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US;
 101 /* Specify the number of retries on RX. */
 102 static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;
 103
 104 /* Socket file paths. Can be set by user */
 105 static char *socket_files;
 106 static int nb_sockets;
 107
 108 /* empty vmdq configuration structure. Filled in programatically */
 109 static struct rte_eth_conf vmdq_conf_default = {
 110         .rxmode = {
 111                 .mq_mode        = ETH_MQ_RX_VMDQ_ONLY,
 112                 .split_hdr_size = 0,
 113                 /*
 114                  * VLAN strip is necessary for 1G NIC such as I350,
 115                  * this fixes bug of ipv4 forwarding in guest can't
 116                  * forward pakets from one virtio dev to another virtio dev.
 117                  */
 118                 .offloads = DEV_RX_OFFLOAD_VLAN_STRIP,
 119         },
 120
 121         .txmode = {
 122                 .mq_mode = ETH_MQ_TX_NONE,
 123                 .offloads = (DEV_TX_OFFLOAD_IPV4_CKSUM |
 124                              DEV_TX_OFFLOAD_TCP_CKSUM |
 125                              DEV_TX_OFFLOAD_VLAN_INSERT |
 126                              DEV_TX_OFFLOAD_MULTI_SEGS |
 127                              DEV_TX_OFFLOAD_TCP_TSO),
 128         },
 129         .rx_adv_conf = {
 130                 /*
 131                  * should be overridden separately in code with
 132                  * appropriate values
 133                  */
 134                 .vmdq_rx_conf = {
 135                         .nb_queue_pools = ETH_8_POOLS,
 136                         .enable_default_pool = 0,
 137                         .default_pool = 0,
 138                         .nb_pool_maps = 0,
 139                         .pool_map = {{0, 0},},
 140                 },
 141         },
 142 };
 143
 144
 145 static unsigned lcore_ids[RTE_MAX_LCORE];
 146 static uint16_t ports[RTE_MAX_ETHPORTS];
 147 static unsigned num_ports = 0; /**< The number of ports specified in command line */
 148 static uint16_t num_pf_queues, num_vmdq_queues;
 149 static uint16_t vmdq_pool_base, vmdq_queue_base;
 150 static uint16_t queues_per_pool;
 151
 152 const uint16_t vlan_tags[] = {
 153         1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007,
 154         1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015,
 155         1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023,
 156         1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031,
 157         1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039,
 158         1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047,
 159         1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055,
 160         1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063,
 161 };
 162
 163 /* ethernet addresses of ports */
 164 static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
 165
 166 static struct vhost_dev_tailq_list vhost_dev_list =
 167         TAILQ_HEAD_INITIALIZER(vhost_dev_list);
 168
 169 static struct lcore_info lcore_info[RTE_MAX_LCORE];
 170
 171 /* Used for queueing bursts of TX packets. */
 172 struct mbuf_table {
 173         unsigned len;
 174         unsigned txq_id;
 175         struct rte_mbuf *m_table[MAX_PKT_BURST];
 176 };
 177
 178 /* TX queue for each data core. */
 179 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE];
 180
 181 #define MBUF_TABLE_DRAIN_TSC    ((rte_get_tsc_hz() + US_PER_S - 1) \
 182                                  / US_PER_S * BURST_TX_DRAIN_US)
 183 #define VLAN_HLEN       4
 184
 185 /*
 186  * Builds up the correct configuration for VMDQ VLAN pool map
 187  * according to the pool & queue limits.
 188  */
 189 static inline int
 190 get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices)
 191 {
 192         struct rte_eth_vmdq_rx_conf conf;
 193         struct rte_eth_vmdq_rx_conf *def_conf =
 194                 &vmdq_conf_default.rx_adv_conf.vmdq_rx_conf;
 195         unsigned i;
 196
 197         memset(&conf, 0, sizeof(conf));
 198         conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices;
 199         conf.nb_pool_maps = num_devices;
 200         conf.enable_loop_back = def_conf->enable_loop_back;
 201         conf.rx_mode = def_conf->rx_mode;
 202
 203         for (i = 0; i < conf.nb_pool_maps; i++) {
 204                 conf.pool_map[i].vlan_id = vlan_tags[ i ];
 205                 conf.pool_map[i].pools = (1UL << i);
 206         }
 207
 208         (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf)));
 209         (void)(rte_memcpy(&eth_conf->rx_adv_conf.vmdq_rx_conf, &conf,
 210                    sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf)));
 211         return 0;
 212 }
 213
 214 /*
 215  * Initialises a given port using global settings and with the rx buffers
 216  * coming from the mbuf_pool passed as parameter
 217  */
 218 static inline int
 219 port_init(uint16_t port)
 220 {
 221         struct rte_eth_dev_info dev_info;
 222         struct rte_eth_conf port_conf;
 223         struct rte_eth_rxconf *rxconf;
 224         struct rte_eth_txconf *txconf;
 225         int16_t rx_rings, tx_rings;
 226         uint16_t rx_ring_size, tx_ring_size;
 227         int retval;
 228         uint16_t q;
 229
 230         /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
 231         rte_eth_dev_info_get (port, &dev_info);
 232
 233         rxconf = &dev_info.default_rxconf;
 234         txconf = &dev_info.default_txconf;
 235         rxconf->rx_drop_en = 1;
 236
 237         /*configure the number of supported virtio devices based on VMDQ limits */
 238         num_devices = dev_info.max_vmdq_pools;
 239
 240         rx_ring_size = RTE_TEST_RX_DESC_DEFAULT;
 241         tx_ring_size = RTE_TEST_TX_DESC_DEFAULT;
 242
 243         /*
 244          * When dequeue zero copy is enabled, guest Tx used vring will be
 245          * updated only when corresponding mbuf is freed. Thus, the nb_tx_desc
 246          * (tx_ring_size here) must be small enough so that the driver will
 247          * hit the free threshold easily and free mbufs timely. Otherwise,
 248          * guest Tx vring would be starved.
 249          */
 250         if (dequeue_zero_copy)
 251                 tx_ring_size = 64;
 252
 253         tx_rings = (uint16_t)rte_lcore_count();
 254
 255         /* Get port configuration. */
 256         retval = get_eth_conf(&port_conf, num_devices);
 257         if (retval < 0)
 258                 return retval;
 259         /* NIC queues are divided into pf queues and vmdq queues.  */
 260         num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;
 261         queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;
 262         num_vmdq_queues = num_devices * queues_per_pool;
 263         num_queues = num_pf_queues + num_vmdq_queues;
 264         vmdq_queue_base = dev_info.vmdq_queue_base;
 265         vmdq_pool_base  = dev_info.vmdq_pool_base;
 266         printf("pf queue num: %u, configured vmdq pool num: %u, each vmdq pool has %u queues\n",
 267                 num_pf_queues, num_devices, queues_per_pool);
 268
 269         if (!rte_eth_dev_is_valid_port(port))
 270                 return -1;
 271
 272         rx_rings = (uint16_t)dev_info.max_rx_queues;
 273         if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 274                 port_conf.txmode.offloads |=
 275                         DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 276         /* Configure ethernet device. */
 277         retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
 278         if (retval != 0) {
 279                 RTE_LOG(ERR, VHOST_PORT, "Failed to configure port %u: %s.\n",
 280                         port, strerror(-retval));
 281                 return retval;
 282         }
 283
 284         retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &rx_ring_size,
 285                 &tx_ring_size);
 286         if (retval != 0) {
 287                 RTE_LOG(ERR, VHOST_PORT, "Failed to adjust number of descriptors "
 288                         "for port %u: %s.\n", port, strerror(-retval));
 289                 return retval;
 290         }
 291         if (rx_ring_size > RTE_TEST_RX_DESC_DEFAULT) {
 292                 RTE_LOG(ERR, VHOST_PORT, "Mbuf pool has an insufficient size "
 293                         "for Rx queues on port %u.\n", port);
 294                 return -1;
 295         }
 296
 297         /* Setup the queues. */
 298         rxconf->offloads = port_conf.rxmode.offloads;
 299         for (q = 0; q < rx_rings; q ++) {
 300                 retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
 301                                                 rte_eth_dev_socket_id(port),
 302                                                 rxconf,
 303                                                 mbuf_pool);
 304                 if (retval < 0) {
 305                         RTE_LOG(ERR, VHOST_PORT,
 306                                 "Failed to setup rx queue %u of port %u: %s.\n",
 307                                 q, port, strerror(-retval));
 308                         return retval;
 309                 }
 310         }
 311         txconf->offloads = port_conf.txmode.offloads;
 312         for (q = 0; q < tx_rings; q ++) {
 313                 retval = rte_eth_tx_queue_setup(port, q, tx_ring_size,
 314                                                 rte_eth_dev_socket_id(port),
 315                                                 txconf);
 316                 if (retval < 0) {
 317                         RTE_LOG(ERR, VHOST_PORT,
 318                                 "Failed to setup tx queue %u of port %u: %s.\n",
 319                                 q, port, strerror(-retval));
 320                         return retval;
 321                 }
 322         }
 323
 324         /* Start the device. */
 325         retval  = rte_eth_dev_start(port);
 326         if (retval < 0) {
 327                 RTE_LOG(ERR, VHOST_PORT, "Failed to start port %u: %s\n",
 328                         port, strerror(-retval));
 329                 return retval;
 330         }
 331
 332         if (promiscuous)
 333                 rte_eth_promiscuous_enable(port);
 334
 335         rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
 336         RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices);
 337         RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
 338                         " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
 339                         port,
 340                         vmdq_ports_eth_addr[port].addr_bytes[0],
 341                         vmdq_ports_eth_addr[port].addr_bytes[1],
 342                         vmdq_ports_eth_addr[port].addr_bytes[2],
 343                         vmdq_ports_eth_addr[port].addr_bytes[3],
 344                         vmdq_ports_eth_addr[port].addr_bytes[4],
 345                         vmdq_ports_eth_addr[port].addr_bytes[5]);
 346
 347         return 0;
 348 }
 349
 350 /*
 351  * Set socket file path.
 352  */
 353 static int
 354 us_vhost_parse_socket_path(const char *q_arg)
 355 {
 356         char *old;
 357
 358         /* parse number string */
 359         if (strnlen(q_arg, PATH_MAX) == PATH_MAX)
 360                 return -1;
 361
 362         old = socket_files;
 363         socket_files = realloc(socket_files, PATH_MAX * (nb_sockets + 1));
 364         if (socket_files == NULL) {
 365                 free(old);
 366                 return -1;
 367         }
 368
 369         snprintf(socket_files + nb_sockets * PATH_MAX, PATH_MAX, "%s", q_arg);
 370         nb_sockets++;
 371
 372         return 0;
 373 }
 374
 375 /*
 376  * Parse the portmask provided at run time.
 377  */
 378 static int
 379 parse_portmask(const char *portmask)
 380 {
 381         char *end = NULL;
 382         unsigned long pm;
 383
 384         errno = 0;
 385
 386         /* parse hexadecimal string */
 387         pm = strtoul(portmask, &end, 16);
 388         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 389                 return -1;
 390
 391         if (pm == 0)
 392                 return -1;
 393
 394         return pm;
 395
 396 }
 397
 398 /*
 399  * Parse num options at run time.
 400  */
 401 static int
 402 parse_num_opt(const char *q_arg, uint32_t max_valid_value)
 403 {
 404         char *end = NULL;
 405         unsigned long num;
 406
 407         errno = 0;
 408
 409         /* parse unsigned int string */
 410         num = strtoul(q_arg, &end, 10);
 411         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 412                 return -1;
 413
 414         if (num > max_valid_value)
 415                 return -1;
 416
 417         return num;
 418
 419 }
 420
 421 /*
 422  * Display usage
 423  */
 424 static void
 425 us_vhost_usage(const char *prgname)
 426 {
 427         RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n"
 428         "               --vm2vm [0|1|2]\n"
 429         "               --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n"
 430         "               --socket-file <path>\n"
 431         "               --nb-devices ND\n"
 432         "               -p PORTMASK: Set mask for ports to be used by application\n"
 433         "               --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n"
 434         "               --rx-retry [0|1]: disable/enable(default) retries on rx. Enable retry if destintation queue is full\n"
 435         "               --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX. This makes effect only if retries on rx enabled\n"
 436         "               --rx-retry-num [0-N]: the number of retries on rx. This makes effect only if retries on rx enabled\n"
 437         "               --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n"
 438         "               --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
 439         "               --socket-file: The path of the socket file.\n"
 440         "               --tx-csum [0|1] disable/enable TX checksum offload.\n"
 441         "               --tso [0|1] disable/enable TCP segment offload.\n"
 442         "               --client register a vhost-user socket as client mode.\n"
 443         "               --dequeue-zero-copy enables dequeue zero copy\n",
 444                prgname);
 445 }
 446
 447 /*
 448  * Parse the arguments given in the command line of the application.
 449  */
 450 static int
 451 us_vhost_parse_args(int argc, char **argv)
 452 {
 453         int opt, ret;
 454         int option_index;
 455         unsigned i;
 456         const char *prgname = argv[0];
 457         static struct option long_option[] = {
 458                 {"vm2vm", required_argument, NULL, 0},
 459                 {"rx-retry", required_argument, NULL, 0},
 460                 {"rx-retry-delay", required_argument, NULL, 0},
 461                 {"rx-retry-num", required_argument, NULL, 0},
 462                 {"mergeable", required_argument, NULL, 0},
 463                 {"stats", required_argument, NULL, 0},
 464                 {"socket-file", required_argument, NULL, 0},
 465                 {"tx-csum", required_argument, NULL, 0},
 466                 {"tso", required_argument, NULL, 0},
 467                 {"client", no_argument, &client_mode, 1},
 468                 {"dequeue-zero-copy", no_argument, &dequeue_zero_copy, 1},
 469                 {"builtin-net-driver", no_argument, &builtin_net_driver, 1},
 470                 {NULL, 0, 0, 0},
 471         };
 472
 473         /* Parse command line */
 474         while ((opt = getopt_long(argc, argv, "p:P",
 475                         long_option, &option_index)) != EOF) {
 476                 switch (opt) {
 477                 /* Portmask */
 478                 case 'p':
 479                         enabled_port_mask = parse_portmask(optarg);
 480                         if (enabled_port_mask == 0) {
 481                                 RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n");
 482                                 us_vhost_usage(prgname);
 483                                 return -1;
 484                         }
 485                         break;
 486
 487                 case 'P':
 488                         promiscuous = 1;
 489                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.rx_mode =
 490                                 ETH_VMDQ_ACCEPT_BROADCAST |
 491                                 ETH_VMDQ_ACCEPT_MULTICAST;
 492
 493                         break;
 494
 495                 case 0:
 496                         /* Enable/disable vm2vm comms. */
 497                         if (!strncmp(long_option[option_index].name, "vm2vm",
 498                                 MAX_LONG_OPT_SZ)) {
 499                                 ret = parse_num_opt(optarg, (VM2VM_LAST - 1));
 500                                 if (ret == -1) {
 501                                         RTE_LOG(INFO, VHOST_CONFIG,
 502                                                 "Invalid argument for "
 503                                                 "vm2vm [0|1|2]\n");
 504                                         us_vhost_usage(prgname);
 505                                         return -1;
 506                                 } else {
 507                                         vm2vm_mode = (vm2vm_type)ret;
 508                                 }
 509                         }
 510
 511                         /* Enable/disable retries on RX. */
 512                         if (!strncmp(long_option[option_index].name, "rx-retry", MAX_LONG_OPT_SZ)) {
 513                                 ret = parse_num_opt(optarg, 1);
 514                                 if (ret == -1) {
 515                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry [0|1]\n");
 516                                         us_vhost_usage(prgname);
 517                                         return -1;
 518                                 } else {
 519                                         enable_retry = ret;
 520                                 }
 521                         }
 522
 523                         /* Enable/disable TX checksum offload. */
 524                         if (!strncmp(long_option[option_index].name, "tx-csum", MAX_LONG_OPT_SZ)) {
 525                                 ret = parse_num_opt(optarg, 1);
 526                                 if (ret == -1) {
 527                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tx-csum [0|1]\n");
 528                                         us_vhost_usage(prgname);
 529                                         return -1;
 530                                 } else
 531                                         enable_tx_csum = ret;
 532                         }
 533
 534                         /* Enable/disable TSO offload. */
 535                         if (!strncmp(long_option[option_index].name, "tso", MAX_LONG_OPT_SZ)) {
 536                                 ret = parse_num_opt(optarg, 1);
 537                                 if (ret == -1) {
 538                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tso [0|1]\n");
 539                                         us_vhost_usage(prgname);
 540                                         return -1;
 541                                 } else
 542                                         enable_tso = ret;
 543                         }
 544
 545                         /* Specify the retries delay time (in useconds) on RX. */
 546                         if (!strncmp(long_option[option_index].name, "rx-retry-delay", MAX_LONG_OPT_SZ)) {
 547                                 ret = parse_num_opt(optarg, INT32_MAX);
 548                                 if (ret == -1) {
 549                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-delay [0-N]\n");
 550                                         us_vhost_usage(prgname);
 551                                         return -1;
 552                                 } else {
 553                                         burst_rx_delay_time = ret;
 554                                 }
 555                         }
 556
 557                         /* Specify the retries number on RX. */
 558                         if (!strncmp(long_option[option_index].name, "rx-retry-num", MAX_LONG_OPT_SZ)) {
 559                                 ret = parse_num_opt(optarg, INT32_MAX);
 560                                 if (ret == -1) {
 561                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-num [0-N]\n");
 562                                         us_vhost_usage(prgname);
 563                                         return -1;
 564                                 } else {
 565                                         burst_rx_retry_num = ret;
 566                                 }
 567                         }
 568
 569                         /* Enable/disable RX mergeable buffers. */
 570                         if (!strncmp(long_option[option_index].name, "mergeable", MAX_LONG_OPT_SZ)) {
 571                                 ret = parse_num_opt(optarg, 1);
 572                                 if (ret == -1) {
 573                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for mergeable [0|1]\n");
 574                                         us_vhost_usage(prgname);
 575                                         return -1;
 576                                 } else {
 577                                         mergeable = !!ret;
 578                                         if (ret) {
 579                                                 vmdq_conf_default.rxmode.offloads |=
 580                                                         DEV_RX_OFFLOAD_JUMBO_FRAME;
 581                                                 vmdq_conf_default.rxmode.max_rx_pkt_len
 582                                                         = JUMBO_FRAME_MAX_SIZE;
 583                                         }
 584                                 }
 585                         }
 586
 587                         /* Enable/disable stats. */
 588                         if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) {
 589                                 ret = parse_num_opt(optarg, INT32_MAX);
 590                                 if (ret == -1) {
 591                                         RTE_LOG(INFO, VHOST_CONFIG,
 592                                                 "Invalid argument for stats [0..N]\n");
 593                                         us_vhost_usage(prgname);
 594                                         return -1;
 595                                 } else {
 596                                         enable_stats = ret;
 597                                 }
 598                         }
 599
 600                         /* Set socket file path. */
 601                         if (!strncmp(long_option[option_index].name,
 602                                                 "socket-file", MAX_LONG_OPT_SZ)) {
 603                                 if (us_vhost_parse_socket_path(optarg) == -1) {
 604                                         RTE_LOG(INFO, VHOST_CONFIG,
 605                                         "Invalid argument for socket name (Max %d characters)\n",
 606                                         PATH_MAX);
 607                                         us_vhost_usage(prgname);
 608                                         return -1;
 609                                 }
 610                         }
 611
 612                         break;
 613
 614                         /* Invalid option - print options. */
 615                 default:
 616                         us_vhost_usage(prgname);
 617                         return -1;
 618                 }
 619         }
 620
 621         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 622                 if (enabled_port_mask & (1 << i))
 623                         ports[num_ports++] = i;
 624         }
 625
 626         if ((num_ports ==  0) || (num_ports > MAX_SUP_PORTS)) {
 627                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
 628                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
 629                 return -1;
 630         }
 631
 632         return 0;
 633 }
 634
 635 /*
 636  * Update the global var NUM_PORTS and array PORTS according to system ports number
 637  * and return valid ports number
 638  */
 639 static unsigned check_ports_num(unsigned nb_ports)
 640 {
 641         unsigned valid_num_ports = num_ports;
 642         unsigned portid;
 643
 644         if (num_ports > nb_ports) {
 645                 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n",
 646                         num_ports, nb_ports);
 647                 num_ports = nb_ports;
 648         }
 649
 650         for (portid = 0; portid < num_ports; portid ++) {
 651                 if (!rte_eth_dev_is_valid_port(ports[portid])) {
 652                         RTE_LOG(INFO, VHOST_PORT,
 653                                 "\nSpecified port ID(%u) is not valid\n",
 654                                 ports[portid]);
 655                         ports[portid] = INVALID_PORT_ID;
 656                         valid_num_ports--;
 657                 }
 658         }
 659         return valid_num_ports;
 660 }
 661
 662 static __rte_always_inline struct vhost_dev *
 663 find_vhost_dev(struct ether_addr *mac)
 664 {
 665         struct vhost_dev *vdev;
 666
 667         TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
 668                 if (vdev->ready == DEVICE_RX &&
 669                     is_same_ether_addr(mac, &vdev->mac_address))
 670                         return vdev;
 671         }
 672
 673         return NULL;
 674 }
 675
 676 /*
 677  * This function learns the MAC address of the device and registers this along with a
 678  * vlan tag to a VMDQ.
 679  */
 680 static int
 681 link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
 682 {
 683         struct ether_hdr *pkt_hdr;
 684         int i, ret;
 685
 686         /* Learn MAC address of guest device from packet */
 687         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 688
 689         if (find_vhost_dev(&pkt_hdr->s_addr)) {
 690                 RTE_LOG(ERR, VHOST_DATA,
 691                         "(%d) device is using a registered MAC!\n",
 692                         vdev->vid);
 693                 return -1;
 694         }
 695
 696         for (i = 0; i < ETHER_ADDR_LEN; i++)
 697                 vdev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i];
 698
 699         /* vlan_tag currently uses the device_id. */
 700         vdev->vlan_tag = vlan_tags[vdev->vid];
 701
 702         /* Print out VMDQ registration info. */
 703         RTE_LOG(INFO, VHOST_DATA,
 704                 "(%d) mac %02x:%02x:%02x:%02x:%02x:%02x and vlan %d registered\n",
 705                 vdev->vid,
 706                 vdev->mac_address.addr_bytes[0], vdev->mac_address.addr_bytes[1],
 707                 vdev->mac_address.addr_bytes[2], vdev->mac_address.addr_bytes[3],
 708                 vdev->mac_address.addr_bytes[4], vdev->mac_address.addr_bytes[5],
 709                 vdev->vlan_tag);
 710
 711         /* Register the MAC address. */
 712         ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address,
 713                                 (uint32_t)vdev->vid + vmdq_pool_base);
 714         if (ret)
 715                 RTE_LOG(ERR, VHOST_DATA,
 716                         "(%d) failed to add device MAC address to VMDQ\n",
 717                         vdev->vid);
 718
 719         rte_eth_dev_set_vlan_strip_on_queue(ports[0], vdev->vmdq_rx_q, 1);
 720
 721         /* Set device as ready for RX. */
 722         vdev->ready = DEVICE_RX;
 723
 724         return 0;
 725 }
 726
 727 /*
 728  * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX
 729  * queue before disabling RX on the device.
 730  */
 731 static inline void
 732 unlink_vmdq(struct vhost_dev *vdev)
 733 {
 734         unsigned i = 0;
 735         unsigned rx_count;
 736         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 737
 738         if (vdev->ready == DEVICE_RX) {
 739                 /*clear MAC and VLAN settings*/
 740                 rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address);
 741                 for (i = 0; i < 6; i++)
 742                         vdev->mac_address.addr_bytes[i] = 0;
 743
 744                 vdev->vlan_tag = 0;
 745
 746                 /*Clear out the receive buffers*/
 747                 rx_count = rte_eth_rx_burst(ports[0],
 748                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
 749
 750                 while (rx_count) {
 751                         for (i = 0; i < rx_count; i++)
 752                                 rte_pktmbuf_free(pkts_burst[i]);
 753
 754                         rx_count = rte_eth_rx_burst(ports[0],
 755                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
 756                 }
 757
 758                 vdev->ready = DEVICE_MAC_LEARNING;
 759         }
 760 }
 761
 762 static __rte_always_inline void
 763 virtio_xmit(struct vhost_dev *dst_vdev, struct vhost_dev *src_vdev,
 764             struct rte_mbuf *m)
 765 {
 766         uint16_t ret;
 767
 768         if (builtin_net_driver) {
 769                 ret = vs_enqueue_pkts(dst_vdev, VIRTIO_RXQ, &m, 1);
 770         } else {
 771                 ret = rte_vhost_enqueue_burst(dst_vdev->vid, VIRTIO_RXQ, &m, 1);
 772         }
 773
 774         if (enable_stats) {
 775                 rte_atomic64_inc(&dst_vdev->stats.rx_total_atomic);
 776                 rte_atomic64_add(&dst_vdev->stats.rx_atomic, ret);
 777                 src_vdev->stats.tx_total++;
 778                 src_vdev->stats.tx += ret;
 779         }
 780 }
 781
 782 /*
 783  * Check if the packet destination MAC address is for a local device. If so then put
 784  * the packet on that devices RX queue. If not then return.
 785  */
 786 static __rte_always_inline int
 787 virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
 788 {
 789         struct ether_hdr *pkt_hdr;
 790         struct vhost_dev *dst_vdev;
 791
 792         pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 793
 794         dst_vdev = find_vhost_dev(&pkt_hdr->d_addr);
 795         if (!dst_vdev)
 796                 return -1;
 797
 798         if (vdev->vid == dst_vdev->vid) {
 799                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 800                         "(%d) TX: src and dst MAC is same. Dropping packet.\n",
 801                         vdev->vid);
 802                 return 0;
 803         }
 804
 805         RTE_LOG_DP(DEBUG, VHOST_DATA,
 806                 "(%d) TX: MAC address is local\n", dst_vdev->vid);
 807
 808         if (unlikely(dst_vdev->remove)) {
 809                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 810                         "(%d) device is marked for removal\n", dst_vdev->vid);
 811                 return 0;
 812         }
 813
 814         virtio_xmit(dst_vdev, vdev, m);
 815         return 0;
 816 }
 817
 818 /*
 819  * Check if the destination MAC of a packet is one local VM,
 820  * and get its vlan tag, and offset if it is.
 821  */
 822 static __rte_always_inline int
 823 find_local_dest(struct vhost_dev *vdev, struct rte_mbuf *m,
 824         uint32_t *offset, uint16_t *vlan_tag)
 825 {
 826         struct vhost_dev *dst_vdev;
 827         struct ether_hdr *pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 828
 829         dst_vdev = find_vhost_dev(&pkt_hdr->d_addr);
 830         if (!dst_vdev)
 831                 return 0;
 832
 833         if (vdev->vid == dst_vdev->vid) {
 834                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 835                         "(%d) TX: src and dst MAC is same. Dropping packet.\n",
 836                         vdev->vid);
 837                 return -1;
 838         }
 839
 840         /*
 841          * HW vlan strip will reduce the packet length
 842          * by minus length of vlan tag, so need restore
 843          * the packet length by plus it.
 844          */
 845         *offset  = VLAN_HLEN;
 846         *vlan_tag = vlan_tags[vdev->vid];
 847
 848         RTE_LOG_DP(DEBUG, VHOST_DATA,
 849                 "(%d) TX: pkt to local VM device id: (%d), vlan tag: %u.\n",
 850                 vdev->vid, dst_vdev->vid, *vlan_tag);
 851
 852         return 0;
 853 }
 854
 855 static uint16_t
 856 get_psd_sum(void *l3_hdr, uint64_t ol_flags)
 857 {
 858         if (ol_flags & PKT_TX_IPV4)
 859                 return rte_ipv4_phdr_cksum(l3_hdr, ol_flags);
 860         else /* assume ethertype == ETHER_TYPE_IPv6 */
 861                 return rte_ipv6_phdr_cksum(l3_hdr, ol_flags);
 862 }
 863
 864 static void virtio_tx_offload(struct rte_mbuf *m)
 865 {
 866         void *l3_hdr;
 867         struct ipv4_hdr *ipv4_hdr = NULL;
 868         struct tcp_hdr *tcp_hdr = NULL;
 869         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 870
 871         l3_hdr = (char *)eth_hdr + m->l2_len;
 872
 873         if (m->ol_flags & PKT_TX_IPV4) {
 874                 ipv4_hdr = l3_hdr;
 875                 ipv4_hdr->hdr_checksum = 0;
 876                 m->ol_flags |= PKT_TX_IP_CKSUM;
 877         }
 878
 879         tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + m->l3_len);
 880         tcp_hdr->cksum = get_psd_sum(l3_hdr, m->ol_flags);
 881 }
 882
 883 static inline void
 884 free_pkts(struct rte_mbuf **pkts, uint16_t n)
 885 {
 886         while (n--)
 887                 rte_pktmbuf_free(pkts[n]);
 888 }
 889
 890 static __rte_always_inline void
 891 do_drain_mbuf_table(struct mbuf_table *tx_q)
 892 {
 893         uint16_t count;
 894
 895         count = rte_eth_tx_burst(ports[0], tx_q->txq_id,
 896                                  tx_q->m_table, tx_q->len);
 897         if (unlikely(count < tx_q->len))
 898                 free_pkts(&tx_q->m_table[count], tx_q->len - count);
 899
 900         tx_q->len = 0;
 901 }
 902
 903 /*
 904  * This function routes the TX packet to the correct interface. This
 905  * may be a local device or the physical port.
 906  */
 907 static __rte_always_inline void
 908 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)
 909 {
 910         struct mbuf_table *tx_q;
 911         unsigned offset = 0;
 912         const uint16_t lcore_id = rte_lcore_id();
 913         struct ether_hdr *nh;
 914
 915
 916         nh = rte_pktmbuf_mtod(m, struct ether_hdr *);
 917         if (unlikely(is_broadcast_ether_addr(&nh->d_addr))) {
 918                 struct vhost_dev *vdev2;
 919
 920                 TAILQ_FOREACH(vdev2, &vhost_dev_list, global_vdev_entry) {
 921                         if (vdev2 != vdev)
 922                                 virtio_xmit(vdev2, vdev, m);
 923                 }
 924                 goto queue2nic;
 925         }
 926
 927         /*check if destination is local VM*/
 928         if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) {
 929                 rte_pktmbuf_free(m);
 930                 return;
 931         }
 932
 933         if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
 934                 if (unlikely(find_local_dest(vdev, m, &offset,
 935                                              &vlan_tag) != 0)) {
 936                         rte_pktmbuf_free(m);
 937                         return;
 938                 }
 939         }
 940
 941         RTE_LOG_DP(DEBUG, VHOST_DATA,
 942                 "(%d) TX: MAC address is external\n", vdev->vid);
 943
 944 queue2nic:
 945
 946         /*Add packet to the port tx queue*/
 947         tx_q = &lcore_tx_queue[lcore_id];
 948
 949         nh = rte_pktmbuf_mtod(m, struct ether_hdr *);
 950         if (unlikely(nh->ether_type == rte_cpu_to_be_16(ETHER_TYPE_VLAN))) {
 951                 /* Guest has inserted the vlan tag. */
 952                 struct vlan_hdr *vh = (struct vlan_hdr *) (nh + 1);
 953                 uint16_t vlan_tag_be = rte_cpu_to_be_16(vlan_tag);
 954                 if ((vm2vm_mode == VM2VM_HARDWARE) &&
 955                         (vh->vlan_tci != vlan_tag_be))
 956                         vh->vlan_tci = vlan_tag_be;
 957         } else {
 958                 m->ol_flags |= PKT_TX_VLAN_PKT;
 959
 960                 /*
 961                  * Find the right seg to adjust the data len when offset is
 962                  * bigger than tail room size.
 963                  */
 964                 if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
 965                         if (likely(offset <= rte_pktmbuf_tailroom(m)))
 966                                 m->data_len += offset;
 967                         else {
 968                                 struct rte_mbuf *seg = m;
 969
 970                                 while ((seg->next != NULL) &&
 971                                         (offset > rte_pktmbuf_tailroom(seg)))
 972                                         seg = seg->next;
 973
 974                                 seg->data_len += offset;
 975                         }
 976                         m->pkt_len += offset;
 977                 }
 978
 979                 m->vlan_tci = vlan_tag;
 980         }
 981
 982         if (m->ol_flags & PKT_TX_TCP_SEG)
 983                 virtio_tx_offload(m);
 984
 985         tx_q->m_table[tx_q->len++] = m;
 986         if (enable_stats) {
 987                 vdev->stats.tx_total++;
 988                 vdev->stats.tx++;
 989         }
 990
 991         if (unlikely(tx_q->len == MAX_PKT_BURST))
 992                 do_drain_mbuf_table(tx_q);
 993 }
 994
 995
 996 static __rte_always_inline void
 997 drain_mbuf_table(struct mbuf_table *tx_q)
 998 {
 999         static uint64_t prev_tsc;
1000         uint64_t cur_tsc;
1001
1002         if (tx_q->len == 0)
1003                 return;
1004
1005         cur_tsc = rte_rdtsc();
1006         if (unlikely(cur_tsc - prev_tsc > MBUF_TABLE_DRAIN_TSC)) {
1007                 prev_tsc = cur_tsc;
1008
1009                 RTE_LOG_DP(DEBUG, VHOST_DATA,
1010                         "TX queue drained after timeout with burst size %u\n",
1011                         tx_q->len);
1012                 do_drain_mbuf_table(tx_q);
1013         }
1014 }
1015
1016 static __rte_always_inline void
1017 drain_eth_rx(struct vhost_dev *vdev)
1018 {
1019         uint16_t rx_count, enqueue_count;
1020         struct rte_mbuf *pkts[MAX_PKT_BURST];
1021
1022         rx_count = rte_eth_rx_burst(ports[0], vdev->vmdq_rx_q,
1023                                     pkts, MAX_PKT_BURST);
1024         if (!rx_count)
1025                 return;
1026
1027         /*
1028          * When "enable_retry" is set, here we wait and retry when there
1029          * is no enough free slots in the queue to hold @rx_count packets,
1030          * to diminish packet loss.
1031          */
1032         if (enable_retry &&
1033             unlikely(rx_count > rte_vhost_avail_entries(vdev->vid,
1034                         VIRTIO_RXQ))) {
1035                 uint32_t retry;
1036
1037                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
1038                         rte_delay_us(burst_rx_delay_time);
1039                         if (rx_count <= rte_vhost_avail_entries(vdev->vid,
1040                                         VIRTIO_RXQ))
1041                                 break;
1042                 }
1043         }
1044
1045         if (builtin_net_driver) {
1046                 enqueue_count = vs_enqueue_pkts(vdev, VIRTIO_RXQ,
1047                                                 pkts, rx_count);
1048         } else {
1049                 enqueue_count = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ,
1050                                                 pkts, rx_count);
1051         }
1052         if (enable_stats) {
1053                 rte_atomic64_add(&vdev->stats.rx_total_atomic, rx_count);
1054                 rte_atomic64_add(&vdev->stats.rx_atomic, enqueue_count);
1055         }
1056
1057         free_pkts(pkts, rx_count);
1058 }
1059
1060 static __rte_always_inline void
1061 drain_virtio_tx(struct vhost_dev *vdev)
1062 {
1063         struct rte_mbuf *pkts[MAX_PKT_BURST];
1064         uint16_t count;
1065         uint16_t i;
1066
1067         if (builtin_net_driver) {
1068                 count = vs_dequeue_pkts(vdev, VIRTIO_TXQ, mbuf_pool,
1069                                         pkts, MAX_PKT_BURST);
1070         } else {
1071                 count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ,
1072                                         mbuf_pool, pkts, MAX_PKT_BURST);
1073         }
1074
1075         /* setup VMDq for the first packet */
1076         if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count) {
1077                 if (vdev->remove || link_vmdq(vdev, pkts[0]) == -1)
1078                         free_pkts(pkts, count);
1079         }
1080
1081         for (i = 0; i < count; ++i)
1082                 virtio_tx_route(vdev, pkts[i], vlan_tags[vdev->vid]);
1083 }
1084
1085 /*
1086  * Main function of vhost-switch. It basically does:
1087  *
1088  * for each vhost device {
1089  *    - drain_eth_rx()
1090  *
1091  *      Which drains the host eth Rx queue linked to the vhost device,
1092  *      and deliver all of them to guest virito Rx ring associated with
1093  *      this vhost device.
1094  *
1095  *    - drain_virtio_tx()
1096  *
1097  *      Which drains the guest virtio Tx queue and deliver all of them
1098  *      to the target, which could be another vhost device, or the
1099  *      physical eth dev. The route is done in function "virtio_tx_route".
1100  * }
1101  */
1102 static int
1103 switch_worker(void *arg __rte_unused)
1104 {
1105         unsigned i;
1106         unsigned lcore_id = rte_lcore_id();
1107         struct vhost_dev *vdev;
1108         struct mbuf_table *tx_q;
1109
1110         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
1111
1112         tx_q = &lcore_tx_queue[lcore_id];
1113         for (i = 0; i < rte_lcore_count(); i++) {
1114                 if (lcore_ids[i] == lcore_id) {
1115                         tx_q->txq_id = i;
1116                         break;
1117                 }
1118         }
1119
1120         while(1) {
1121                 drain_mbuf_table(tx_q);
1122
1123                 /*
1124                  * Inform the configuration core that we have exited the
1125                  * linked list and that no devices are in use if requested.
1126                  */
1127                 if (lcore_info[lcore_id].dev_removal_flag == REQUEST_DEV_REMOVAL)
1128                         lcore_info[lcore_id].dev_removal_flag = ACK_DEV_REMOVAL;
1129
1130                 /*
1131                  * Process vhost devices
1132                  */
1133                 TAILQ_FOREACH(vdev, &lcore_info[lcore_id].vdev_list,
1134                               lcore_vdev_entry) {
1135                         if (unlikely(vdev->remove)) {
1136                                 unlink_vmdq(vdev);
1137                                 vdev->ready = DEVICE_SAFE_REMOVE;
1138                                 continue;
1139                         }
1140
1141                         if (likely(vdev->ready == DEVICE_RX))
1142                                 drain_eth_rx(vdev);
1143
1144                         if (likely(!vdev->remove))
1145                                 drain_virtio_tx(vdev);
1146                 }
1147         }
1148
1149         return 0;
1150 }
1151
1152 /*
1153  * Remove a device from the specific data core linked list and from the
1154  * main linked list. Synchonization  occurs through the use of the
1155  * lcore dev_removal_flag. Device is made volatile here to avoid re-ordering
1156  * of dev->remove=1 which can cause an infinite loop in the rte_pause loop.
1157  */
1158 static void
1159 destroy_device(int vid)
1160 {
1161         struct vhost_dev *vdev = NULL;
1162         int lcore;
1163
1164         TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
1165                 if (vdev->vid == vid)
1166                         break;
1167         }
1168         if (!vdev)
1169                 return;
1170         /*set the remove flag. */
1171         vdev->remove = 1;
1172         while(vdev->ready != DEVICE_SAFE_REMOVE) {
1173                 rte_pause();
1174         }
1175
1176         if (builtin_net_driver)
1177                 vs_vhost_net_remove(vdev);
1178
1179         TAILQ_REMOVE(&lcore_info[vdev->coreid].vdev_list, vdev,
1180                      lcore_vdev_entry);
1181         TAILQ_REMOVE(&vhost_dev_list, vdev, global_vdev_entry);
1182
1183
1184         /* Set the dev_removal_flag on each lcore. */
1185         RTE_LCORE_FOREACH_SLAVE(lcore)
1186                 lcore_info[lcore].dev_removal_flag = REQUEST_DEV_REMOVAL;
1187
1188         /*
1189          * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL
1190          * we can be sure that they can no longer access the device removed
1191          * from the linked lists and that the devices are no longer in use.
1192          */
1193         RTE_LCORE_FOREACH_SLAVE(lcore) {
1194                 while (lcore_info[lcore].dev_removal_flag != ACK_DEV_REMOVAL)
1195                         rte_pause();
1196         }
1197
1198         lcore_info[vdev->coreid].device_num--;
1199
1200         RTE_LOG(INFO, VHOST_DATA,
1201                 "(%d) device has been removed from data core\n",
1202                 vdev->vid);
1203
1204         rte_free(vdev);
1205 }
1206
1207 /*
1208  * A new device is added to a data core. First the device is added to the main linked list
1209  * and then allocated to a specific data core.
1210  */
1211 static int
1212 new_device(int vid)
1213 {
1214         int lcore, core_add = 0;
1215         uint32_t device_num_min = num_devices;
1216         struct vhost_dev *vdev;
1217
1218         vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
1219         if (vdev == NULL) {
1220                 RTE_LOG(INFO, VHOST_DATA,
1221                         "(%d) couldn't allocate memory for vhost dev\n",
1222                         vid);
1223                 return -1;
1224         }
1225         vdev->vid = vid;
1226
1227         if (builtin_net_driver)
1228                 vs_vhost_net_setup(vdev);
1229
1230         TAILQ_INSERT_TAIL(&vhost_dev_list, vdev, global_vdev_entry);
1231         vdev->vmdq_rx_q = vid * queues_per_pool + vmdq_queue_base;
1232
1233         /*reset ready flag*/
1234         vdev->ready = DEVICE_MAC_LEARNING;
1235         vdev->remove = 0;
1236
1237         /* Find a suitable lcore to add the device. */
1238         RTE_LCORE_FOREACH_SLAVE(lcore) {
1239                 if (lcore_info[lcore].device_num < device_num_min) {
1240                         device_num_min = lcore_info[lcore].device_num;
1241                         core_add = lcore;
1242                 }
1243         }
1244         vdev->coreid = core_add;
1245
1246         TAILQ_INSERT_TAIL(&lcore_info[vdev->coreid].vdev_list, vdev,
1247                           lcore_vdev_entry);
1248         lcore_info[vdev->coreid].device_num++;
1249
1250         /* Disable notifications. */
1251         rte_vhost_enable_guest_notification(vid, VIRTIO_RXQ, 0);
1252         rte_vhost_enable_guest_notification(vid, VIRTIO_TXQ, 0);
1253
1254         RTE_LOG(INFO, VHOST_DATA,
1255                 "(%d) device has been added to data core %d\n",
1256                 vid, vdev->coreid);
1257
1258         return 0;
1259 }
1260
1261 /*
1262  * These callback allow devices to be added to the data core when configuration
1263  * has been fully complete.
1264  */
1265 static const struct vhost_device_ops virtio_net_device_ops =
1266 {
1267         .new_device =  new_device,
1268         .destroy_device = destroy_device,
1269 };
1270
1271 /*
1272  * This is a thread will wake up after a period to print stats if the user has
1273  * enabled them.
1274  */
1275 static void *
1276 print_stats(__rte_unused void *arg)
1277 {
1278         struct vhost_dev *vdev;
1279         uint64_t tx_dropped, rx_dropped;
1280         uint64_t tx, tx_total, rx, rx_total;
1281         const char clr[] = { 27, '[', '2', 'J', '\0' };
1282         const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' };
1283
1284         while(1) {
1285                 sleep(enable_stats);
1286
1287                 /* Clear screen and move to top left */
1288                 printf("%s%s\n", clr, top_left);
1289                 printf("Device statistics =================================\n");
1290
1291                 TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
1292                         tx_total   = vdev->stats.tx_total;
1293                         tx         = vdev->stats.tx;
1294                         tx_dropped = tx_total - tx;
1295
1296                         rx_total   = rte_atomic64_read(&vdev->stats.rx_total_atomic);
1297                         rx         = rte_atomic64_read(&vdev->stats.rx_atomic);
1298                         rx_dropped = rx_total - rx;
1299
1300                         printf("Statistics for device %d\n"
1301                                 "-----------------------\n"
1302                                 "TX total:              %" PRIu64 "\n"
1303                                 "TX dropped:            %" PRIu64 "\n"
1304                                 "TX successful:         %" PRIu64 "\n"
1305                                 "RX total:              %" PRIu64 "\n"
1306                                 "RX dropped:            %" PRIu64 "\n"
1307                                 "RX successful:         %" PRIu64 "\n",
1308                                 vdev->vid,
1309                                 tx_total, tx_dropped, tx,
1310                                 rx_total, rx_dropped, rx);
1311                 }
1312
1313                 printf("===================================================\n");
1314         }
1315
1316         return NULL;
1317 }
1318
1319 static void
1320 unregister_drivers(int socket_num)
1321 {
1322         int i, ret;
1323
1324         for (i = 0; i < socket_num; i++) {
1325                 ret = rte_vhost_driver_unregister(socket_files + i * PATH_MAX);
1326                 if (ret != 0)
1327                         RTE_LOG(ERR, VHOST_CONFIG,
1328                                 "Fail to unregister vhost driver for %s.\n",
1329                                 socket_files + i * PATH_MAX);
1330         }
1331 }
1332
1333 /* When we receive a INT signal, unregister vhost driver */
1334 static void
1335 sigint_handler(__rte_unused int signum)
1336 {
1337         /* Unregister vhost driver. */
1338         unregister_drivers(nb_sockets);
1339
1340         exit(0);
1341 }
1342
1343 /*
1344  * While creating an mbuf pool, one key thing is to figure out how
1345  * many mbuf entries is enough for our use. FYI, here are some
1346  * guidelines:
1347  *
1348  * - Each rx queue would reserve @nr_rx_desc mbufs at queue setup stage
1349  *
1350  * - For each switch core (A CPU core does the packet switch), we need
1351  *   also make some reservation for receiving the packets from virtio
1352  *   Tx queue. How many is enough depends on the usage. It's normally
1353  *   a simple calculation like following:
1354  *
1355  *       MAX_PKT_BURST * max packet size / mbuf size
1356  *
1357  *   So, we definitely need allocate more mbufs when TSO is enabled.
1358  *
1359  * - Similarly, for each switching core, we should serve @nr_rx_desc
1360  *   mbufs for receiving the packets from physical NIC device.
1361  *
1362  * - We also need make sure, for each switch core, we have allocated
1363  *   enough mbufs to fill up the mbuf cache.
1364  */
1365 static void
1366 create_mbuf_pool(uint16_t nr_port, uint32_t nr_switch_core, uint32_t mbuf_size,
1367         uint32_t nr_queues, uint32_t nr_rx_desc, uint32_t nr_mbuf_cache)
1368 {
1369         uint32_t nr_mbufs;
1370         uint32_t nr_mbufs_per_core;
1371         uint32_t mtu = 1500;
1372
1373         if (mergeable)
1374                 mtu = 9000;
1375         if (enable_tso)
1376                 mtu = 64 * 1024;
1377
1378         nr_mbufs_per_core  = (mtu + mbuf_size) * MAX_PKT_BURST /
1379                         (mbuf_size - RTE_PKTMBUF_HEADROOM);
1380         nr_mbufs_per_core += nr_rx_desc;
1381         nr_mbufs_per_core  = RTE_MAX(nr_mbufs_per_core, nr_mbuf_cache);
1382
1383         nr_mbufs  = nr_queues * nr_rx_desc;
1384         nr_mbufs += nr_mbufs_per_core * nr_switch_core;
1385         nr_mbufs *= nr_port;
1386
1387         mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", nr_mbufs,
1388                                             nr_mbuf_cache, 0, mbuf_size,
1389                                             rte_socket_id());
1390         if (mbuf_pool == NULL)
1391                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
1392 }
1393
1394 /*
1395  * Main function, does initialisation and calls the per-lcore functions.
1396  */
1397 int
1398 main(int argc, char *argv[])
1399 {
1400         unsigned lcore_id, core_id = 0;
1401         unsigned nb_ports, valid_num_ports;
1402         int ret, i;
1403         uint16_t portid;
1404         static pthread_t tid;
1405         uint64_t flags = 0;
1406
1407         signal(SIGINT, sigint_handler);
1408
1409         /* init EAL */
1410         ret = rte_eal_init(argc, argv);
1411         if (ret < 0)
1412                 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
1413         argc -= ret;
1414         argv += ret;
1415
1416         /* parse app arguments */
1417         ret = us_vhost_parse_args(argc, argv);
1418         if (ret < 0)
1419                 rte_exit(EXIT_FAILURE, "Invalid argument\n");
1420
1421         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1422                 TAILQ_INIT(&lcore_info[lcore_id].vdev_list);
1423
1424                 if (rte_lcore_is_enabled(lcore_id))
1425                         lcore_ids[core_id++] = lcore_id;
1426         }
1427
1428         if (rte_lcore_count() > RTE_MAX_LCORE)
1429                 rte_exit(EXIT_FAILURE,"Not enough cores\n");
1430
1431         /* Get the number of physical ports. */
1432         nb_ports = rte_eth_dev_count_avail();
1433
1434         /*
1435          * Update the global var NUM_PORTS and global array PORTS
1436          * and get value of var VALID_NUM_PORTS according to system ports number
1437          */
1438         valid_num_ports = check_ports_num(nb_ports);
1439
1440         if ((valid_num_ports ==  0) || (valid_num_ports > MAX_SUP_PORTS)) {
1441                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
1442                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
1443                 return -1;
1444         }
1445
1446         /*
1447          * FIXME: here we are trying to allocate mbufs big enough for
1448          * @MAX_QUEUES, but the truth is we're never going to use that
1449          * many queues here. We probably should only do allocation for
1450          * those queues we are going to use.
1451          */
1452         create_mbuf_pool(valid_num_ports, rte_lcore_count() - 1, MBUF_DATA_SIZE,
1453                          MAX_QUEUES, RTE_TEST_RX_DESC_DEFAULT, MBUF_CACHE_SIZE);
1454
1455         if (vm2vm_mode == VM2VM_HARDWARE) {
1456                 /* Enable VT loop back to let L2 switch to do it. */
1457                 vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1;
1458                 RTE_LOG(DEBUG, VHOST_CONFIG,
1459                         "Enable loop back for L2 switch in vmdq.\n");
1460         }
1461
1462         /* initialize all ports */
1463         RTE_ETH_FOREACH_DEV(portid) {
1464                 /* skip ports that are not enabled */
1465                 if ((enabled_port_mask & (1 << portid)) == 0) {
1466                         RTE_LOG(INFO, VHOST_PORT,
1467                                 "Skipping disabled port %d\n", portid);
1468                         continue;
1469                 }
1470                 if (port_init(portid) != 0)
1471                         rte_exit(EXIT_FAILURE,
1472                                 "Cannot initialize network ports\n");
1473         }
1474
1475         /* Enable stats if the user option is set. */
1476         if (enable_stats) {
1477                 ret = rte_ctrl_thread_create(&tid, "print-stats", NULL,
1478                                         print_stats, NULL);
1479                 if (ret < 0)
1480                         rte_exit(EXIT_FAILURE,
1481                                 "Cannot create print-stats thread\n");
1482         }
1483
1484         /* Launch all data cores. */
1485         RTE_LCORE_FOREACH_SLAVE(lcore_id)
1486                 rte_eal_remote_launch(switch_worker, NULL, lcore_id);
1487
1488         if (client_mode)
1489                 flags |= RTE_VHOST_USER_CLIENT;
1490
1491         if (dequeue_zero_copy)
1492                 flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
1493
1494         /* Register vhost user driver to handle vhost messages. */
1495         for (i = 0; i < nb_sockets; i++) {
1496                 char *file = socket_files + i * PATH_MAX;
1497                 ret = rte_vhost_driver_register(file, flags);
1498                 if (ret != 0) {
1499                         unregister_drivers(i);
1500                         rte_exit(EXIT_FAILURE,
1501                                 "vhost driver register failure.\n");
1502                 }
1503
1504                 if (builtin_net_driver)
1505                         rte_vhost_driver_set_features(file, VIRTIO_NET_FEATURES);
1506
1507                 if (mergeable == 0) {
1508                         rte_vhost_driver_disable_features(file,
1509                                 1ULL << VIRTIO_NET_F_MRG_RXBUF);
1510                 }
1511
1512                 if (enable_tx_csum == 0) {
1513                         rte_vhost_driver_disable_features(file,
1514                                 1ULL << VIRTIO_NET_F_CSUM);
1515                 }
1516
1517                 if (enable_tso == 0) {
1518                         rte_vhost_driver_disable_features(file,
1519                                 1ULL << VIRTIO_NET_F_HOST_TSO4);
1520                         rte_vhost_driver_disable_features(file,
1521                                 1ULL << VIRTIO_NET_F_HOST_TSO6);
1522                         rte_vhost_driver_disable_features(file,
1523                                 1ULL << VIRTIO_NET_F_GUEST_TSO4);
1524                         rte_vhost_driver_disable_features(file,
1525                                 1ULL << VIRTIO_NET_F_GUEST_TSO6);
1526                 }
1527
1528                 if (promiscuous) {
1529                         rte_vhost_driver_enable_features(file,
1530                                 1ULL << VIRTIO_NET_F_CTRL_RX);
1531                 }
1532
1533                 ret = rte_vhost_driver_callback_register(file,
1534                         &virtio_net_device_ops);
1535                 if (ret != 0) {
1536                         rte_exit(EXIT_FAILURE,
1537                                 "failed to register vhost driver callbacks.\n");
1538                 }
1539
1540                 if (rte_vhost_driver_start(file) < 0) {
1541                         rte_exit(EXIT_FAILURE,
1542                                 "failed to start vhost driver.\n");
1543                 }
1544         }
1545
1546         RTE_LCORE_FOREACH_SLAVE(lcore_id)
1547                 rte_eal_wait_lcore(lcore_id);
1548
1549         return 0;
1550
1551 }