examples/vhost/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <arpa/inet.h>
   6 #include <getopt.h>
   7 #include <linux/if_ether.h>
   8 #include <linux/if_vlan.h>
   9 #include <linux/virtio_net.h>
  10 #include <linux/virtio_ring.h>
  11 #include <signal.h>
  12 #include <stdint.h>
  13 #include <sys/eventfd.h>
  14 #include <sys/param.h>
  15 #include <unistd.h>
  16
  17 #include <rte_atomic.h>
  18 #include <rte_cycles.h>
  19 #include <rte_ethdev.h>
  20 #include <rte_log.h>
  21 #include <rte_string_fns.h>
  22 #include <rte_malloc.h>
  23 #include <rte_vhost.h>
  24 #include <rte_ip.h>
  25 #include <rte_tcp.h>
  26 #include <rte_pause.h>
  27
  28 #include "main.h"
  29
  30 #ifndef MAX_QUEUES
  31 #define MAX_QUEUES 128
  32 #endif
  33
  34 /* the maximum number of external ports supported */
  35 #define MAX_SUP_PORTS 1
  36
  37 #define MBUF_CACHE_SIZE 128
  38 #define MBUF_DATA_SIZE  RTE_MBUF_DEFAULT_BUF_SIZE
  39
  40 #define BURST_TX_DRAIN_US 100   /* TX drain every ~100us */
  41
  42 #define BURST_RX_WAIT_US 15     /* Defines how long we wait between retries on RX */
  43 #define BURST_RX_RETRIES 4              /* Number of retries on RX. */
  44
  45 #define JUMBO_FRAME_MAX_SIZE    0x2600
  46
  47 /* State of virtio device. */
  48 #define DEVICE_MAC_LEARNING 0
  49 #define DEVICE_RX                       1
  50 #define DEVICE_SAFE_REMOVE      2
  51
  52 /* Configurable number of RX/TX ring descriptors */
  53 #define RTE_TEST_RX_DESC_DEFAULT 1024
  54 #define RTE_TEST_TX_DESC_DEFAULT 512
  55
  56 #define INVALID_PORT_ID 0xFF
  57
  58 /* Maximum long option length for option parsing. */
  59 #define MAX_LONG_OPT_SZ 64
  60
  61 /* mask of enabled ports */
  62 static uint32_t enabled_port_mask = 0;
  63
  64 /* Promiscuous mode */
  65 static uint32_t promiscuous;
  66
  67 /* number of devices/queues to support*/
  68 static uint32_t num_queues = 0;
  69 static uint32_t num_devices;
  70
  71 static struct rte_mempool *mbuf_pool;
  72 static int mergeable;
  73
  74 /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */
  75 typedef enum {
  76         VM2VM_DISABLED = 0,
  77         VM2VM_SOFTWARE = 1,
  78         VM2VM_HARDWARE = 2,
  79         VM2VM_LAST
  80 } vm2vm_type;
  81 static vm2vm_type vm2vm_mode = VM2VM_SOFTWARE;
  82
  83 /* Enable stats. */
  84 static uint32_t enable_stats = 0;
  85 /* Enable retries on RX. */
  86 static uint32_t enable_retry = 1;
  87
  88 /* Disable TX checksum offload */
  89 static uint32_t enable_tx_csum;
  90
  91 /* Disable TSO offload */
  92 static uint32_t enable_tso;
  93
  94 static int client_mode;
  95 static int dequeue_zero_copy;
  96
  97 static int builtin_net_driver;
  98
  99 /* Specify timeout (in useconds) between retries on RX. */
 100 static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US;
 101 /* Specify the number of retries on RX. */
 102 static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;
 103
 104 /* Socket file paths. Can be set by user */
 105 static char *socket_files;
 106 static int nb_sockets;
 107
 108 /* empty vmdq configuration structure. Filled in programatically */
 109 static struct rte_eth_conf vmdq_conf_default = {
 110         .rxmode = {
 111                 .mq_mode        = ETH_MQ_RX_VMDQ_ONLY,
 112                 .split_hdr_size = 0,
 113                 /*
 114                  * VLAN strip is necessary for 1G NIC such as I350,
 115                  * this fixes bug of ipv4 forwarding in guest can't
 116                  * forward pakets from one virtio dev to another virtio dev.
 117                  */
 118                 .offloads = DEV_RX_OFFLOAD_VLAN_STRIP,
 119         },
 120
 121         .txmode = {
 122                 .mq_mode = ETH_MQ_TX_NONE,
 123                 .offloads = (DEV_TX_OFFLOAD_IPV4_CKSUM |
 124                              DEV_TX_OFFLOAD_TCP_CKSUM |
 125                              DEV_TX_OFFLOAD_VLAN_INSERT |
 126                              DEV_TX_OFFLOAD_MULTI_SEGS |
 127                              DEV_TX_OFFLOAD_TCP_TSO),
 128         },
 129         .rx_adv_conf = {
 130                 /*
 131                  * should be overridden separately in code with
 132                  * appropriate values
 133                  */
 134                 .vmdq_rx_conf = {
 135                         .nb_queue_pools = ETH_8_POOLS,
 136                         .enable_default_pool = 0,
 137                         .default_pool = 0,
 138                         .nb_pool_maps = 0,
 139                         .pool_map = {{0, 0},},
 140                 },
 141         },
 142 };
 143
 144
 145 static unsigned lcore_ids[RTE_MAX_LCORE];
 146 static uint16_t ports[RTE_MAX_ETHPORTS];
 147 static unsigned num_ports = 0; /**< The number of ports specified in command line */
 148 static uint16_t num_pf_queues, num_vmdq_queues;
 149 static uint16_t vmdq_pool_base, vmdq_queue_base;
 150 static uint16_t queues_per_pool;
 151
 152 const uint16_t vlan_tags[] = {
 153         1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007,
 154         1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015,
 155         1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023,
 156         1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031,
 157         1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039,
 158         1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047,
 159         1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055,
 160         1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063,
 161 };
 162
 163 /* ethernet addresses of ports */
 164 static struct rte_ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
 165
 166 static struct vhost_dev_tailq_list vhost_dev_list =
 167         TAILQ_HEAD_INITIALIZER(vhost_dev_list);
 168
 169 static struct lcore_info lcore_info[RTE_MAX_LCORE];
 170
 171 /* Used for queueing bursts of TX packets. */
 172 struct mbuf_table {
 173         unsigned len;
 174         unsigned txq_id;
 175         struct rte_mbuf *m_table[MAX_PKT_BURST];
 176 };
 177
 178 /* TX queue for each data core. */
 179 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE];
 180
 181 #define MBUF_TABLE_DRAIN_TSC    ((rte_get_tsc_hz() + US_PER_S - 1) \
 182                                  / US_PER_S * BURST_TX_DRAIN_US)
 183 #define VLAN_HLEN       4
 184
 185 /*
 186  * Builds up the correct configuration for VMDQ VLAN pool map
 187  * according to the pool & queue limits.
 188  */
 189 static inline int
 190 get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices)
 191 {
 192         struct rte_eth_vmdq_rx_conf conf;
 193         struct rte_eth_vmdq_rx_conf *def_conf =
 194                 &vmdq_conf_default.rx_adv_conf.vmdq_rx_conf;
 195         unsigned i;
 196
 197         memset(&conf, 0, sizeof(conf));
 198         conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices;
 199         conf.nb_pool_maps = num_devices;
 200         conf.enable_loop_back = def_conf->enable_loop_back;
 201         conf.rx_mode = def_conf->rx_mode;
 202
 203         for (i = 0; i < conf.nb_pool_maps; i++) {
 204                 conf.pool_map[i].vlan_id = vlan_tags[ i ];
 205                 conf.pool_map[i].pools = (1UL << i);
 206         }
 207
 208         (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf)));
 209         (void)(rte_memcpy(&eth_conf->rx_adv_conf.vmdq_rx_conf, &conf,
 210                    sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf)));
 211         return 0;
 212 }
 213
 214 /*
 215  * Initialises a given port using global settings and with the rx buffers
 216  * coming from the mbuf_pool passed as parameter
 217  */
 218 static inline int
 219 port_init(uint16_t port)
 220 {
 221         struct rte_eth_dev_info dev_info;
 222         struct rte_eth_conf port_conf;
 223         struct rte_eth_rxconf *rxconf;
 224         struct rte_eth_txconf *txconf;
 225         int16_t rx_rings, tx_rings;
 226         uint16_t rx_ring_size, tx_ring_size;
 227         int retval;
 228         uint16_t q;
 229
 230         /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
 231         rte_eth_dev_info_get (port, &dev_info);
 232
 233         rxconf = &dev_info.default_rxconf;
 234         txconf = &dev_info.default_txconf;
 235         rxconf->rx_drop_en = 1;
 236
 237         /*configure the number of supported virtio devices based on VMDQ limits */
 238         num_devices = dev_info.max_vmdq_pools;
 239
 240         rx_ring_size = RTE_TEST_RX_DESC_DEFAULT;
 241         tx_ring_size = RTE_TEST_TX_DESC_DEFAULT;
 242
 243         /*
 244          * When dequeue zero copy is enabled, guest Tx used vring will be
 245          * updated only when corresponding mbuf is freed. Thus, the nb_tx_desc
 246          * (tx_ring_size here) must be small enough so that the driver will
 247          * hit the free threshold easily and free mbufs timely. Otherwise,
 248          * guest Tx vring would be starved.
 249          */
 250         if (dequeue_zero_copy)
 251                 tx_ring_size = 64;
 252
 253         tx_rings = (uint16_t)rte_lcore_count();
 254
 255         /* Get port configuration. */
 256         retval = get_eth_conf(&port_conf, num_devices);
 257         if (retval < 0)
 258                 return retval;
 259         /* NIC queues are divided into pf queues and vmdq queues.  */
 260         num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;
 261         queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;
 262         num_vmdq_queues = num_devices * queues_per_pool;
 263         num_queues = num_pf_queues + num_vmdq_queues;
 264         vmdq_queue_base = dev_info.vmdq_queue_base;
 265         vmdq_pool_base  = dev_info.vmdq_pool_base;
 266         printf("pf queue num: %u, configured vmdq pool num: %u, each vmdq pool has %u queues\n",
 267                 num_pf_queues, num_devices, queues_per_pool);
 268
 269         if (!rte_eth_dev_is_valid_port(port))
 270                 return -1;
 271
 272         rx_rings = (uint16_t)dev_info.max_rx_queues;
 273         if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 274                 port_conf.txmode.offloads |=
 275                         DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 276         /* Configure ethernet device. */
 277         retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
 278         if (retval != 0) {
 279                 RTE_LOG(ERR, VHOST_PORT, "Failed to configure port %u: %s.\n",
 280                         port, strerror(-retval));
 281                 return retval;
 282         }
 283
 284         retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &rx_ring_size,
 285                 &tx_ring_size);
 286         if (retval != 0) {
 287                 RTE_LOG(ERR, VHOST_PORT, "Failed to adjust number of descriptors "
 288                         "for port %u: %s.\n", port, strerror(-retval));
 289                 return retval;
 290         }
 291         if (rx_ring_size > RTE_TEST_RX_DESC_DEFAULT) {
 292                 RTE_LOG(ERR, VHOST_PORT, "Mbuf pool has an insufficient size "
 293                         "for Rx queues on port %u.\n", port);
 294                 return -1;
 295         }
 296
 297         /* Setup the queues. */
 298         rxconf->offloads = port_conf.rxmode.offloads;
 299         for (q = 0; q < rx_rings; q ++) {
 300                 retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
 301                                                 rte_eth_dev_socket_id(port),
 302                                                 rxconf,
 303                                                 mbuf_pool);
 304                 if (retval < 0) {
 305                         RTE_LOG(ERR, VHOST_PORT,
 306                                 "Failed to setup rx queue %u of port %u: %s.\n",
 307                                 q, port, strerror(-retval));
 308                         return retval;
 309                 }
 310         }
 311         txconf->offloads = port_conf.txmode.offloads;
 312         for (q = 0; q < tx_rings; q ++) {
 313                 retval = rte_eth_tx_queue_setup(port, q, tx_ring_size,
 314                                                 rte_eth_dev_socket_id(port),
 315                                                 txconf);
 316                 if (retval < 0) {
 317                         RTE_LOG(ERR, VHOST_PORT,
 318                                 "Failed to setup tx queue %u of port %u: %s.\n",
 319                                 q, port, strerror(-retval));
 320                         return retval;
 321                 }
 322         }
 323
 324         /* Start the device. */
 325         retval  = rte_eth_dev_start(port);
 326         if (retval < 0) {
 327                 RTE_LOG(ERR, VHOST_PORT, "Failed to start port %u: %s\n",
 328                         port, strerror(-retval));
 329                 return retval;
 330         }
 331
 332         if (promiscuous)
 333                 rte_eth_promiscuous_enable(port);
 334
 335         rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
 336         RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices);
 337         RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
 338                         " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
 339                         port,
 340                         vmdq_ports_eth_addr[port].addr_bytes[0],
 341                         vmdq_ports_eth_addr[port].addr_bytes[1],
 342                         vmdq_ports_eth_addr[port].addr_bytes[2],
 343                         vmdq_ports_eth_addr[port].addr_bytes[3],
 344                         vmdq_ports_eth_addr[port].addr_bytes[4],
 345                         vmdq_ports_eth_addr[port].addr_bytes[5]);
 346
 347         return 0;
 348 }
 349
 350 /*
 351  * Set socket file path.
 352  */
 353 static int
 354 us_vhost_parse_socket_path(const char *q_arg)
 355 {
 356         char *old;
 357
 358         /* parse number string */
 359         if (strnlen(q_arg, PATH_MAX) == PATH_MAX)
 360                 return -1;
 361
 362         old = socket_files;
 363         socket_files = realloc(socket_files, PATH_MAX * (nb_sockets + 1));
 364         if (socket_files == NULL) {
 365                 free(old);
 366                 return -1;
 367         }
 368
 369         strlcpy(socket_files + nb_sockets * PATH_MAX, q_arg, PATH_MAX);
 370         nb_sockets++;
 371
 372         return 0;
 373 }
 374
 375 /*
 376  * Parse the portmask provided at run time.
 377  */
 378 static int
 379 parse_portmask(const char *portmask)
 380 {
 381         char *end = NULL;
 382         unsigned long pm;
 383
 384         errno = 0;
 385
 386         /* parse hexadecimal string */
 387         pm = strtoul(portmask, &end, 16);
 388         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 389                 return -1;
 390
 391         if (pm == 0)
 392                 return -1;
 393
 394         return pm;
 395
 396 }
 397
 398 /*
 399  * Parse num options at run time.
 400  */
 401 static int
 402 parse_num_opt(const char *q_arg, uint32_t max_valid_value)
 403 {
 404         char *end = NULL;
 405         unsigned long num;
 406
 407         errno = 0;
 408
 409         /* parse unsigned int string */
 410         num = strtoul(q_arg, &end, 10);
 411         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
 412                 return -1;
 413
 414         if (num > max_valid_value)
 415                 return -1;
 416
 417         return num;
 418
 419 }
 420
 421 /*
 422  * Display usage
 423  */
 424 static void
 425 us_vhost_usage(const char *prgname)
 426 {
 427         RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n"
 428         "               --vm2vm [0|1|2]\n"
 429         "               --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n"
 430         "               --socket-file <path>\n"
 431         "               --nb-devices ND\n"
 432         "               -p PORTMASK: Set mask for ports to be used by application\n"
 433         "               --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n"
 434         "               --rx-retry [0|1]: disable/enable(default) retries on rx. Enable retry if destintation queue is full\n"
 435         "               --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX. This makes effect only if retries on rx enabled\n"
 436         "               --rx-retry-num [0-N]: the number of retries on rx. This makes effect only if retries on rx enabled\n"
 437         "               --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n"
 438         "               --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
 439         "               --socket-file: The path of the socket file.\n"
 440         "               --tx-csum [0|1] disable/enable TX checksum offload.\n"
 441         "               --tso [0|1] disable/enable TCP segment offload.\n"
 442         "               --client register a vhost-user socket as client mode.\n"
 443         "               --dequeue-zero-copy enables dequeue zero copy\n",
 444                prgname);
 445 }
 446
 447 /*
 448  * Parse the arguments given in the command line of the application.
 449  */
 450 static int
 451 us_vhost_parse_args(int argc, char **argv)
 452 {
 453         int opt, ret;
 454         int option_index;
 455         unsigned i;
 456         const char *prgname = argv[0];
 457         static struct option long_option[] = {
 458                 {"vm2vm", required_argument, NULL, 0},
 459                 {"rx-retry", required_argument, NULL, 0},
 460                 {"rx-retry-delay", required_argument, NULL, 0},
 461                 {"rx-retry-num", required_argument, NULL, 0},
 462                 {"mergeable", required_argument, NULL, 0},
 463                 {"stats", required_argument, NULL, 0},
 464                 {"socket-file", required_argument, NULL, 0},
 465                 {"tx-csum", required_argument, NULL, 0},
 466                 {"tso", required_argument, NULL, 0},
 467                 {"client", no_argument, &client_mode, 1},
 468                 {"dequeue-zero-copy", no_argument, &dequeue_zero_copy, 1},
 469                 {"builtin-net-driver", no_argument, &builtin_net_driver, 1},
 470                 {NULL, 0, 0, 0},
 471         };
 472
 473         /* Parse command line */
 474         while ((opt = getopt_long(argc, argv, "p:P",
 475                         long_option, &option_index)) != EOF) {
 476                 switch (opt) {
 477                 /* Portmask */
 478                 case 'p':
 479                         enabled_port_mask = parse_portmask(optarg);
 480                         if (enabled_port_mask == 0) {
 481                                 RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n");
 482                                 us_vhost_usage(prgname);
 483                                 return -1;
 484                         }
 485                         break;
 486
 487                 case 'P':
 488                         promiscuous = 1;
 489                         vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.rx_mode =
 490                                 ETH_VMDQ_ACCEPT_BROADCAST |
 491                                 ETH_VMDQ_ACCEPT_MULTICAST;
 492
 493                         break;
 494
 495                 case 0:
 496                         /* Enable/disable vm2vm comms. */
 497                         if (!strncmp(long_option[option_index].name, "vm2vm",
 498                                 MAX_LONG_OPT_SZ)) {
 499                                 ret = parse_num_opt(optarg, (VM2VM_LAST - 1));
 500                                 if (ret == -1) {
 501                                         RTE_LOG(INFO, VHOST_CONFIG,
 502                                                 "Invalid argument for "
 503                                                 "vm2vm [0|1|2]\n");
 504                                         us_vhost_usage(prgname);
 505                                         return -1;
 506                                 } else {
 507                                         vm2vm_mode = (vm2vm_type)ret;
 508                                 }
 509                         }
 510
 511                         /* Enable/disable retries on RX. */
 512                         if (!strncmp(long_option[option_index].name, "rx-retry", MAX_LONG_OPT_SZ)) {
 513                                 ret = parse_num_opt(optarg, 1);
 514                                 if (ret == -1) {
 515                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry [0|1]\n");
 516                                         us_vhost_usage(prgname);
 517                                         return -1;
 518                                 } else {
 519                                         enable_retry = ret;
 520                                 }
 521                         }
 522
 523                         /* Enable/disable TX checksum offload. */
 524                         if (!strncmp(long_option[option_index].name, "tx-csum", MAX_LONG_OPT_SZ)) {
 525                                 ret = parse_num_opt(optarg, 1);
 526                                 if (ret == -1) {
 527                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tx-csum [0|1]\n");
 528                                         us_vhost_usage(prgname);
 529                                         return -1;
 530                                 } else
 531                                         enable_tx_csum = ret;
 532                         }
 533
 534                         /* Enable/disable TSO offload. */
 535                         if (!strncmp(long_option[option_index].name, "tso", MAX_LONG_OPT_SZ)) {
 536                                 ret = parse_num_opt(optarg, 1);
 537                                 if (ret == -1) {
 538                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tso [0|1]\n");
 539                                         us_vhost_usage(prgname);
 540                                         return -1;
 541                                 } else
 542                                         enable_tso = ret;
 543                         }
 544
 545                         /* Specify the retries delay time (in useconds) on RX. */
 546                         if (!strncmp(long_option[option_index].name, "rx-retry-delay", MAX_LONG_OPT_SZ)) {
 547                                 ret = parse_num_opt(optarg, INT32_MAX);
 548                                 if (ret == -1) {
 549                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-delay [0-N]\n");
 550                                         us_vhost_usage(prgname);
 551                                         return -1;
 552                                 } else {
 553                                         burst_rx_delay_time = ret;
 554                                 }
 555                         }
 556
 557                         /* Specify the retries number on RX. */
 558                         if (!strncmp(long_option[option_index].name, "rx-retry-num", MAX_LONG_OPT_SZ)) {
 559                                 ret = parse_num_opt(optarg, INT32_MAX);
 560                                 if (ret == -1) {
 561                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-num [0-N]\n");
 562                                         us_vhost_usage(prgname);
 563                                         return -1;
 564                                 } else {
 565                                         burst_rx_retry_num = ret;
 566                                 }
 567                         }
 568
 569                         /* Enable/disable RX mergeable buffers. */
 570                         if (!strncmp(long_option[option_index].name, "mergeable", MAX_LONG_OPT_SZ)) {
 571                                 ret = parse_num_opt(optarg, 1);
 572                                 if (ret == -1) {
 573                                         RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for mergeable [0|1]\n");
 574                                         us_vhost_usage(prgname);
 575                                         return -1;
 576                                 } else {
 577                                         mergeable = !!ret;
 578                                         if (ret) {
 579                                                 vmdq_conf_default.rxmode.offloads |=
 580                                                         DEV_RX_OFFLOAD_JUMBO_FRAME;
 581                                                 vmdq_conf_default.rxmode.max_rx_pkt_len
 582                                                         = JUMBO_FRAME_MAX_SIZE;
 583                                         }
 584                                 }
 585                         }
 586
 587                         /* Enable/disable stats. */
 588                         if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) {
 589                                 ret = parse_num_opt(optarg, INT32_MAX);
 590                                 if (ret == -1) {
 591                                         RTE_LOG(INFO, VHOST_CONFIG,
 592                                                 "Invalid argument for stats [0..N]\n");
 593                                         us_vhost_usage(prgname);
 594                                         return -1;
 595                                 } else {
 596                                         enable_stats = ret;
 597                                 }
 598                         }
 599
 600                         /* Set socket file path. */
 601                         if (!strncmp(long_option[option_index].name,
 602                                                 "socket-file", MAX_LONG_OPT_SZ)) {
 603                                 if (us_vhost_parse_socket_path(optarg) == -1) {
 604                                         RTE_LOG(INFO, VHOST_CONFIG,
 605                                         "Invalid argument for socket name (Max %d characters)\n",
 606                                         PATH_MAX);
 607                                         us_vhost_usage(prgname);
 608                                         return -1;
 609                                 }
 610                         }
 611
 612                         break;
 613
 614                         /* Invalid option - print options. */
 615                 default:
 616                         us_vhost_usage(prgname);
 617                         return -1;
 618                 }
 619         }
 620
 621         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 622                 if (enabled_port_mask & (1 << i))
 623                         ports[num_ports++] = i;
 624         }
 625
 626         if ((num_ports ==  0) || (num_ports > MAX_SUP_PORTS)) {
 627                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
 628                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
 629                 return -1;
 630         }
 631
 632         return 0;
 633 }
 634
 635 /*
 636  * Update the global var NUM_PORTS and array PORTS according to system ports number
 637  * and return valid ports number
 638  */
 639 static unsigned check_ports_num(unsigned nb_ports)
 640 {
 641         unsigned valid_num_ports = num_ports;
 642         unsigned portid;
 643
 644         if (num_ports > nb_ports) {
 645                 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n",
 646                         num_ports, nb_ports);
 647                 num_ports = nb_ports;
 648         }
 649
 650         for (portid = 0; portid < num_ports; portid ++) {
 651                 if (!rte_eth_dev_is_valid_port(ports[portid])) {
 652                         RTE_LOG(INFO, VHOST_PORT,
 653                                 "\nSpecified port ID(%u) is not valid\n",
 654                                 ports[portid]);
 655                         ports[portid] = INVALID_PORT_ID;
 656                         valid_num_ports--;
 657                 }
 658         }
 659         return valid_num_ports;
 660 }
 661
 662 static __rte_always_inline struct vhost_dev *
 663 find_vhost_dev(struct rte_ether_addr *mac)
 664 {
 665         struct vhost_dev *vdev;
 666
 667         TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
 668                 if (vdev->ready == DEVICE_RX &&
 669                     is_same_ether_addr(mac, &vdev->mac_address))
 670                         return vdev;
 671         }
 672
 673         return NULL;
 674 }
 675
 676 /*
 677  * This function learns the MAC address of the device and registers this along with a
 678  * vlan tag to a VMDQ.
 679  */
 680 static int
 681 link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
 682 {
 683         struct rte_ether_hdr *pkt_hdr;
 684         int i, ret;
 685
 686         /* Learn MAC address of guest device from packet */
 687         pkt_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 688
 689         if (find_vhost_dev(&pkt_hdr->s_addr)) {
 690                 RTE_LOG(ERR, VHOST_DATA,
 691                         "(%d) device is using a registered MAC!\n",
 692                         vdev->vid);
 693                 return -1;
 694         }
 695
 696         for (i = 0; i < ETHER_ADDR_LEN; i++)
 697                 vdev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i];
 698
 699         /* vlan_tag currently uses the device_id. */
 700         vdev->vlan_tag = vlan_tags[vdev->vid];
 701
 702         /* Print out VMDQ registration info. */
 703         RTE_LOG(INFO, VHOST_DATA,
 704                 "(%d) mac %02x:%02x:%02x:%02x:%02x:%02x and vlan %d registered\n",
 705                 vdev->vid,
 706                 vdev->mac_address.addr_bytes[0], vdev->mac_address.addr_bytes[1],
 707                 vdev->mac_address.addr_bytes[2], vdev->mac_address.addr_bytes[3],
 708                 vdev->mac_address.addr_bytes[4], vdev->mac_address.addr_bytes[5],
 709                 vdev->vlan_tag);
 710
 711         /* Register the MAC address. */
 712         ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address,
 713                                 (uint32_t)vdev->vid + vmdq_pool_base);
 714         if (ret)
 715                 RTE_LOG(ERR, VHOST_DATA,
 716                         "(%d) failed to add device MAC address to VMDQ\n",
 717                         vdev->vid);
 718
 719         rte_eth_dev_set_vlan_strip_on_queue(ports[0], vdev->vmdq_rx_q, 1);
 720
 721         /* Set device as ready for RX. */
 722         vdev->ready = DEVICE_RX;
 723
 724         return 0;
 725 }
 726
 727 /*
 728  * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX
 729  * queue before disabling RX on the device.
 730  */
 731 static inline void
 732 unlink_vmdq(struct vhost_dev *vdev)
 733 {
 734         unsigned i = 0;
 735         unsigned rx_count;
 736         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 737
 738         if (vdev->ready == DEVICE_RX) {
 739                 /*clear MAC and VLAN settings*/
 740                 rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address);
 741                 for (i = 0; i < 6; i++)
 742                         vdev->mac_address.addr_bytes[i] = 0;
 743
 744                 vdev->vlan_tag = 0;
 745
 746                 /*Clear out the receive buffers*/
 747                 rx_count = rte_eth_rx_burst(ports[0],
 748                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
 749
 750                 while (rx_count) {
 751                         for (i = 0; i < rx_count; i++)
 752                                 rte_pktmbuf_free(pkts_burst[i]);
 753
 754                         rx_count = rte_eth_rx_burst(ports[0],
 755                                         (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
 756                 }
 757
 758                 vdev->ready = DEVICE_MAC_LEARNING;
 759         }
 760 }
 761
 762 static __rte_always_inline void
 763 virtio_xmit(struct vhost_dev *dst_vdev, struct vhost_dev *src_vdev,
 764             struct rte_mbuf *m)
 765 {
 766         uint16_t ret;
 767
 768         if (builtin_net_driver) {
 769                 ret = vs_enqueue_pkts(dst_vdev, VIRTIO_RXQ, &m, 1);
 770         } else {
 771                 ret = rte_vhost_enqueue_burst(dst_vdev->vid, VIRTIO_RXQ, &m, 1);
 772         }
 773
 774         if (enable_stats) {
 775                 rte_atomic64_inc(&dst_vdev->stats.rx_total_atomic);
 776                 rte_atomic64_add(&dst_vdev->stats.rx_atomic, ret);
 777                 src_vdev->stats.tx_total++;
 778                 src_vdev->stats.tx += ret;
 779         }
 780 }
 781
 782 /*
 783  * Check if the packet destination MAC address is for a local device. If so then put
 784  * the packet on that devices RX queue. If not then return.
 785  */
 786 static __rte_always_inline int
 787 virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
 788 {
 789         struct rte_ether_hdr *pkt_hdr;
 790         struct vhost_dev *dst_vdev;
 791
 792         pkt_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 793
 794         dst_vdev = find_vhost_dev(&pkt_hdr->d_addr);
 795         if (!dst_vdev)
 796                 return -1;
 797
 798         if (vdev->vid == dst_vdev->vid) {
 799                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 800                         "(%d) TX: src and dst MAC is same. Dropping packet.\n",
 801                         vdev->vid);
 802                 return 0;
 803         }
 804
 805         RTE_LOG_DP(DEBUG, VHOST_DATA,
 806                 "(%d) TX: MAC address is local\n", dst_vdev->vid);
 807
 808         if (unlikely(dst_vdev->remove)) {
 809                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 810                         "(%d) device is marked for removal\n", dst_vdev->vid);
 811                 return 0;
 812         }
 813
 814         virtio_xmit(dst_vdev, vdev, m);
 815         return 0;
 816 }
 817
 818 /*
 819  * Check if the destination MAC of a packet is one local VM,
 820  * and get its vlan tag, and offset if it is.
 821  */
 822 static __rte_always_inline int
 823 find_local_dest(struct vhost_dev *vdev, struct rte_mbuf *m,
 824         uint32_t *offset, uint16_t *vlan_tag)
 825 {
 826         struct vhost_dev *dst_vdev;
 827         struct rte_ether_hdr *pkt_hdr =
 828                 rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 829
 830         dst_vdev = find_vhost_dev(&pkt_hdr->d_addr);
 831         if (!dst_vdev)
 832                 return 0;
 833
 834         if (vdev->vid == dst_vdev->vid) {
 835                 RTE_LOG_DP(DEBUG, VHOST_DATA,
 836                         "(%d) TX: src and dst MAC is same. Dropping packet.\n",
 837                         vdev->vid);
 838                 return -1;
 839         }
 840
 841         /*
 842          * HW vlan strip will reduce the packet length
 843          * by minus length of vlan tag, so need restore
 844          * the packet length by plus it.
 845          */
 846         *offset  = VLAN_HLEN;
 847         *vlan_tag = vlan_tags[vdev->vid];
 848
 849         RTE_LOG_DP(DEBUG, VHOST_DATA,
 850                 "(%d) TX: pkt to local VM device id: (%d), vlan tag: %u.\n",
 851                 vdev->vid, dst_vdev->vid, *vlan_tag);
 852
 853         return 0;
 854 }
 855
 856 static uint16_t
 857 get_psd_sum(void *l3_hdr, uint64_t ol_flags)
 858 {
 859         if (ol_flags & PKT_TX_IPV4)
 860                 return rte_ipv4_phdr_cksum(l3_hdr, ol_flags);
 861         else /* assume ethertype == ETHER_TYPE_IPv6 */
 862                 return rte_ipv6_phdr_cksum(l3_hdr, ol_flags);
 863 }
 864
 865 static void virtio_tx_offload(struct rte_mbuf *m)
 866 {
 867         void *l3_hdr;
 868         struct ipv4_hdr *ipv4_hdr = NULL;
 869         struct tcp_hdr *tcp_hdr = NULL;
 870         struct rte_ether_hdr *eth_hdr =
 871                 rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 872
 873         l3_hdr = (char *)eth_hdr + m->l2_len;
 874
 875         if (m->ol_flags & PKT_TX_IPV4) {
 876                 ipv4_hdr = l3_hdr;
 877                 ipv4_hdr->hdr_checksum = 0;
 878                 m->ol_flags |= PKT_TX_IP_CKSUM;
 879         }
 880
 881         tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + m->l3_len);
 882         tcp_hdr->cksum = get_psd_sum(l3_hdr, m->ol_flags);
 883 }
 884
 885 static inline void
 886 free_pkts(struct rte_mbuf **pkts, uint16_t n)
 887 {
 888         while (n--)
 889                 rte_pktmbuf_free(pkts[n]);
 890 }
 891
 892 static __rte_always_inline void
 893 do_drain_mbuf_table(struct mbuf_table *tx_q)
 894 {
 895         uint16_t count;
 896
 897         count = rte_eth_tx_burst(ports[0], tx_q->txq_id,
 898                                  tx_q->m_table, tx_q->len);
 899         if (unlikely(count < tx_q->len))
 900                 free_pkts(&tx_q->m_table[count], tx_q->len - count);
 901
 902         tx_q->len = 0;
 903 }
 904
 905 /*
 906  * This function routes the TX packet to the correct interface. This
 907  * may be a local device or the physical port.
 908  */
 909 static __rte_always_inline void
 910 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)
 911 {
 912         struct mbuf_table *tx_q;
 913         unsigned offset = 0;
 914         const uint16_t lcore_id = rte_lcore_id();
 915         struct rte_ether_hdr *nh;
 916
 917
 918         nh = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 919         if (unlikely(is_broadcast_ether_addr(&nh->d_addr))) {
 920                 struct vhost_dev *vdev2;
 921
 922                 TAILQ_FOREACH(vdev2, &vhost_dev_list, global_vdev_entry) {
 923                         if (vdev2 != vdev)
 924                                 virtio_xmit(vdev2, vdev, m);
 925                 }
 926                 goto queue2nic;
 927         }
 928
 929         /*check if destination is local VM*/
 930         if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) {
 931                 rte_pktmbuf_free(m);
 932                 return;
 933         }
 934
 935         if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
 936                 if (unlikely(find_local_dest(vdev, m, &offset,
 937                                              &vlan_tag) != 0)) {
 938                         rte_pktmbuf_free(m);
 939                         return;
 940                 }
 941         }
 942
 943         RTE_LOG_DP(DEBUG, VHOST_DATA,
 944                 "(%d) TX: MAC address is external\n", vdev->vid);
 945
 946 queue2nic:
 947
 948         /*Add packet to the port tx queue*/
 949         tx_q = &lcore_tx_queue[lcore_id];
 950
 951         nh = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 952         if (unlikely(nh->ether_type == rte_cpu_to_be_16(ETHER_TYPE_VLAN))) {
 953                 /* Guest has inserted the vlan tag. */
 954                 struct rte_vlan_hdr *vh = (struct rte_vlan_hdr *) (nh + 1);
 955                 uint16_t vlan_tag_be = rte_cpu_to_be_16(vlan_tag);
 956                 if ((vm2vm_mode == VM2VM_HARDWARE) &&
 957                         (vh->vlan_tci != vlan_tag_be))
 958                         vh->vlan_tci = vlan_tag_be;
 959         } else {
 960                 m->ol_flags |= PKT_TX_VLAN_PKT;
 961
 962                 /*
 963                  * Find the right seg to adjust the data len when offset is
 964                  * bigger than tail room size.
 965                  */
 966                 if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) {
 967                         if (likely(offset <= rte_pktmbuf_tailroom(m)))
 968                                 m->data_len += offset;
 969                         else {
 970                                 struct rte_mbuf *seg = m;
 971
 972                                 while ((seg->next != NULL) &&
 973                                         (offset > rte_pktmbuf_tailroom(seg)))
 974                                         seg = seg->next;
 975
 976                                 seg->data_len += offset;
 977                         }
 978                         m->pkt_len += offset;
 979                 }
 980
 981                 m->vlan_tci = vlan_tag;
 982         }
 983
 984         if (m->ol_flags & PKT_TX_TCP_SEG)
 985                 virtio_tx_offload(m);
 986
 987         tx_q->m_table[tx_q->len++] = m;
 988         if (enable_stats) {
 989                 vdev->stats.tx_total++;
 990                 vdev->stats.tx++;
 991         }
 992
 993         if (unlikely(tx_q->len == MAX_PKT_BURST))
 994                 do_drain_mbuf_table(tx_q);
 995 }
 996
 997
 998 static __rte_always_inline void
 999 drain_mbuf_table(struct mbuf_table *tx_q)
1000 {
1001         static uint64_t prev_tsc;
1002         uint64_t cur_tsc;
1003
1004         if (tx_q->len == 0)
1005                 return;
1006
1007         cur_tsc = rte_rdtsc();
1008         if (unlikely(cur_tsc - prev_tsc > MBUF_TABLE_DRAIN_TSC)) {
1009                 prev_tsc = cur_tsc;
1010
1011                 RTE_LOG_DP(DEBUG, VHOST_DATA,
1012                         "TX queue drained after timeout with burst size %u\n",
1013                         tx_q->len);
1014                 do_drain_mbuf_table(tx_q);
1015         }
1016 }
1017
1018 static __rte_always_inline void
1019 drain_eth_rx(struct vhost_dev *vdev)
1020 {
1021         uint16_t rx_count, enqueue_count;
1022         struct rte_mbuf *pkts[MAX_PKT_BURST];
1023
1024         rx_count = rte_eth_rx_burst(ports[0], vdev->vmdq_rx_q,
1025                                     pkts, MAX_PKT_BURST);
1026         if (!rx_count)
1027                 return;
1028
1029         /*
1030          * When "enable_retry" is set, here we wait and retry when there
1031          * is no enough free slots in the queue to hold @rx_count packets,
1032          * to diminish packet loss.
1033          */
1034         if (enable_retry &&
1035             unlikely(rx_count > rte_vhost_avail_entries(vdev->vid,
1036                         VIRTIO_RXQ))) {
1037                 uint32_t retry;
1038
1039                 for (retry = 0; retry < burst_rx_retry_num; retry++) {
1040                         rte_delay_us(burst_rx_delay_time);
1041                         if (rx_count <= rte_vhost_avail_entries(vdev->vid,
1042                                         VIRTIO_RXQ))
1043                                 break;
1044                 }
1045         }
1046
1047         if (builtin_net_driver) {
1048                 enqueue_count = vs_enqueue_pkts(vdev, VIRTIO_RXQ,
1049                                                 pkts, rx_count);
1050         } else {
1051                 enqueue_count = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ,
1052                                                 pkts, rx_count);
1053         }
1054         if (enable_stats) {
1055                 rte_atomic64_add(&vdev->stats.rx_total_atomic, rx_count);
1056                 rte_atomic64_add(&vdev->stats.rx_atomic, enqueue_count);
1057         }
1058
1059         free_pkts(pkts, rx_count);
1060 }
1061
1062 static __rte_always_inline void
1063 drain_virtio_tx(struct vhost_dev *vdev)
1064 {
1065         struct rte_mbuf *pkts[MAX_PKT_BURST];
1066         uint16_t count;
1067         uint16_t i;
1068
1069         if (builtin_net_driver) {
1070                 count = vs_dequeue_pkts(vdev, VIRTIO_TXQ, mbuf_pool,
1071                                         pkts, MAX_PKT_BURST);
1072         } else {
1073                 count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ,
1074                                         mbuf_pool, pkts, MAX_PKT_BURST);
1075         }
1076
1077         /* setup VMDq for the first packet */
1078         if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count) {
1079                 if (vdev->remove || link_vmdq(vdev, pkts[0]) == -1)
1080                         free_pkts(pkts, count);
1081         }
1082
1083         for (i = 0; i < count; ++i)
1084                 virtio_tx_route(vdev, pkts[i], vlan_tags[vdev->vid]);
1085 }
1086
1087 /*
1088  * Main function of vhost-switch. It basically does:
1089  *
1090  * for each vhost device {
1091  *    - drain_eth_rx()
1092  *
1093  *      Which drains the host eth Rx queue linked to the vhost device,
1094  *      and deliver all of them to guest virito Rx ring associated with
1095  *      this vhost device.
1096  *
1097  *    - drain_virtio_tx()
1098  *
1099  *      Which drains the guest virtio Tx queue and deliver all of them
1100  *      to the target, which could be another vhost device, or the
1101  *      physical eth dev. The route is done in function "virtio_tx_route".
1102  * }
1103  */
1104 static int
1105 switch_worker(void *arg __rte_unused)
1106 {
1107         unsigned i;
1108         unsigned lcore_id = rte_lcore_id();
1109         struct vhost_dev *vdev;
1110         struct mbuf_table *tx_q;
1111
1112         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
1113
1114         tx_q = &lcore_tx_queue[lcore_id];
1115         for (i = 0; i < rte_lcore_count(); i++) {
1116                 if (lcore_ids[i] == lcore_id) {
1117                         tx_q->txq_id = i;
1118                         break;
1119                 }
1120         }
1121
1122         while(1) {
1123                 drain_mbuf_table(tx_q);
1124
1125                 /*
1126                  * Inform the configuration core that we have exited the
1127                  * linked list and that no devices are in use if requested.
1128                  */
1129                 if (lcore_info[lcore_id].dev_removal_flag == REQUEST_DEV_REMOVAL)
1130                         lcore_info[lcore_id].dev_removal_flag = ACK_DEV_REMOVAL;
1131
1132                 /*
1133                  * Process vhost devices
1134                  */
1135                 TAILQ_FOREACH(vdev, &lcore_info[lcore_id].vdev_list,
1136                               lcore_vdev_entry) {
1137                         if (unlikely(vdev->remove)) {
1138                                 unlink_vmdq(vdev);
1139                                 vdev->ready = DEVICE_SAFE_REMOVE;
1140                                 continue;
1141                         }
1142
1143                         if (likely(vdev->ready == DEVICE_RX))
1144                                 drain_eth_rx(vdev);
1145
1146                         if (likely(!vdev->remove))
1147                                 drain_virtio_tx(vdev);
1148                 }
1149         }
1150
1151         return 0;
1152 }
1153
1154 /*
1155  * Remove a device from the specific data core linked list and from the
1156  * main linked list. Synchonization  occurs through the use of the
1157  * lcore dev_removal_flag. Device is made volatile here to avoid re-ordering
1158  * of dev->remove=1 which can cause an infinite loop in the rte_pause loop.
1159  */
1160 static void
1161 destroy_device(int vid)
1162 {
1163         struct vhost_dev *vdev = NULL;
1164         int lcore;
1165
1166         TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
1167                 if (vdev->vid == vid)
1168                         break;
1169         }
1170         if (!vdev)
1171                 return;
1172         /*set the remove flag. */
1173         vdev->remove = 1;
1174         while(vdev->ready != DEVICE_SAFE_REMOVE) {
1175                 rte_pause();
1176         }
1177
1178         if (builtin_net_driver)
1179                 vs_vhost_net_remove(vdev);
1180
1181         TAILQ_REMOVE(&lcore_info[vdev->coreid].vdev_list, vdev,
1182                      lcore_vdev_entry);
1183         TAILQ_REMOVE(&vhost_dev_list, vdev, global_vdev_entry);
1184
1185
1186         /* Set the dev_removal_flag on each lcore. */
1187         RTE_LCORE_FOREACH_SLAVE(lcore)
1188                 lcore_info[lcore].dev_removal_flag = REQUEST_DEV_REMOVAL;
1189
1190         /*
1191          * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL
1192          * we can be sure that they can no longer access the device removed
1193          * from the linked lists and that the devices are no longer in use.
1194          */
1195         RTE_LCORE_FOREACH_SLAVE(lcore) {
1196                 while (lcore_info[lcore].dev_removal_flag != ACK_DEV_REMOVAL)
1197                         rte_pause();
1198         }
1199
1200         lcore_info[vdev->coreid].device_num--;
1201
1202         RTE_LOG(INFO, VHOST_DATA,
1203                 "(%d) device has been removed from data core\n",
1204                 vdev->vid);
1205
1206         rte_free(vdev);
1207 }
1208
1209 /*
1210  * A new device is added to a data core. First the device is added to the main linked list
1211  * and then allocated to a specific data core.
1212  */
1213 static int
1214 new_device(int vid)
1215 {
1216         int lcore, core_add = 0;
1217         uint32_t device_num_min = num_devices;
1218         struct vhost_dev *vdev;
1219
1220         vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
1221         if (vdev == NULL) {
1222                 RTE_LOG(INFO, VHOST_DATA,
1223                         "(%d) couldn't allocate memory for vhost dev\n",
1224                         vid);
1225                 return -1;
1226         }
1227         vdev->vid = vid;
1228
1229         if (builtin_net_driver)
1230                 vs_vhost_net_setup(vdev);
1231
1232         TAILQ_INSERT_TAIL(&vhost_dev_list, vdev, global_vdev_entry);
1233         vdev->vmdq_rx_q = vid * queues_per_pool + vmdq_queue_base;
1234
1235         /*reset ready flag*/
1236         vdev->ready = DEVICE_MAC_LEARNING;
1237         vdev->remove = 0;
1238
1239         /* Find a suitable lcore to add the device. */
1240         RTE_LCORE_FOREACH_SLAVE(lcore) {
1241                 if (lcore_info[lcore].device_num < device_num_min) {
1242                         device_num_min = lcore_info[lcore].device_num;
1243                         core_add = lcore;
1244                 }
1245         }
1246         vdev->coreid = core_add;
1247
1248         TAILQ_INSERT_TAIL(&lcore_info[vdev->coreid].vdev_list, vdev,
1249                           lcore_vdev_entry);
1250         lcore_info[vdev->coreid].device_num++;
1251
1252         /* Disable notifications. */
1253         rte_vhost_enable_guest_notification(vid, VIRTIO_RXQ, 0);
1254         rte_vhost_enable_guest_notification(vid, VIRTIO_TXQ, 0);
1255
1256         RTE_LOG(INFO, VHOST_DATA,
1257                 "(%d) device has been added to data core %d\n",
1258                 vid, vdev->coreid);
1259
1260         return 0;
1261 }
1262
1263 /*
1264  * These callback allow devices to be added to the data core when configuration
1265  * has been fully complete.
1266  */
1267 static const struct vhost_device_ops virtio_net_device_ops =
1268 {
1269         .new_device =  new_device,
1270         .destroy_device = destroy_device,
1271 };
1272
1273 /*
1274  * This is a thread will wake up after a period to print stats if the user has
1275  * enabled them.
1276  */
1277 static void *
1278 print_stats(__rte_unused void *arg)
1279 {
1280         struct vhost_dev *vdev;
1281         uint64_t tx_dropped, rx_dropped;
1282         uint64_t tx, tx_total, rx, rx_total;
1283         const char clr[] = { 27, '[', '2', 'J', '\0' };
1284         const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' };
1285
1286         while(1) {
1287                 sleep(enable_stats);
1288
1289                 /* Clear screen and move to top left */
1290                 printf("%s%s\n", clr, top_left);
1291                 printf("Device statistics =================================\n");
1292
1293                 TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {
1294                         tx_total   = vdev->stats.tx_total;
1295                         tx         = vdev->stats.tx;
1296                         tx_dropped = tx_total - tx;
1297
1298                         rx_total   = rte_atomic64_read(&vdev->stats.rx_total_atomic);
1299                         rx         = rte_atomic64_read(&vdev->stats.rx_atomic);
1300                         rx_dropped = rx_total - rx;
1301
1302                         printf("Statistics for device %d\n"
1303                                 "-----------------------\n"
1304                                 "TX total:              %" PRIu64 "\n"
1305                                 "TX dropped:            %" PRIu64 "\n"
1306                                 "TX successful:         %" PRIu64 "\n"
1307                                 "RX total:              %" PRIu64 "\n"
1308                                 "RX dropped:            %" PRIu64 "\n"
1309                                 "RX successful:         %" PRIu64 "\n",
1310                                 vdev->vid,
1311                                 tx_total, tx_dropped, tx,
1312                                 rx_total, rx_dropped, rx);
1313                 }
1314
1315                 printf("===================================================\n");
1316         }
1317
1318         return NULL;
1319 }
1320
1321 static void
1322 unregister_drivers(int socket_num)
1323 {
1324         int i, ret;
1325
1326         for (i = 0; i < socket_num; i++) {
1327                 ret = rte_vhost_driver_unregister(socket_files + i * PATH_MAX);
1328                 if (ret != 0)
1329                         RTE_LOG(ERR, VHOST_CONFIG,
1330                                 "Fail to unregister vhost driver for %s.\n",
1331                                 socket_files + i * PATH_MAX);
1332         }
1333 }
1334
1335 /* When we receive a INT signal, unregister vhost driver */
1336 static void
1337 sigint_handler(__rte_unused int signum)
1338 {
1339         /* Unregister vhost driver. */
1340         unregister_drivers(nb_sockets);
1341
1342         exit(0);
1343 }
1344
1345 /*
1346  * While creating an mbuf pool, one key thing is to figure out how
1347  * many mbuf entries is enough for our use. FYI, here are some
1348  * guidelines:
1349  *
1350  * - Each rx queue would reserve @nr_rx_desc mbufs at queue setup stage
1351  *
1352  * - For each switch core (A CPU core does the packet switch), we need
1353  *   also make some reservation for receiving the packets from virtio
1354  *   Tx queue. How many is enough depends on the usage. It's normally
1355  *   a simple calculation like following:
1356  *
1357  *       MAX_PKT_BURST * max packet size / mbuf size
1358  *
1359  *   So, we definitely need allocate more mbufs when TSO is enabled.
1360  *
1361  * - Similarly, for each switching core, we should serve @nr_rx_desc
1362  *   mbufs for receiving the packets from physical NIC device.
1363  *
1364  * - We also need make sure, for each switch core, we have allocated
1365  *   enough mbufs to fill up the mbuf cache.
1366  */
1367 static void
1368 create_mbuf_pool(uint16_t nr_port, uint32_t nr_switch_core, uint32_t mbuf_size,
1369         uint32_t nr_queues, uint32_t nr_rx_desc, uint32_t nr_mbuf_cache)
1370 {
1371         uint32_t nr_mbufs;
1372         uint32_t nr_mbufs_per_core;
1373         uint32_t mtu = 1500;
1374
1375         if (mergeable)
1376                 mtu = 9000;
1377         if (enable_tso)
1378                 mtu = 64 * 1024;
1379
1380         nr_mbufs_per_core  = (mtu + mbuf_size) * MAX_PKT_BURST /
1381                         (mbuf_size - RTE_PKTMBUF_HEADROOM);
1382         nr_mbufs_per_core += nr_rx_desc;
1383         nr_mbufs_per_core  = RTE_MAX(nr_mbufs_per_core, nr_mbuf_cache);
1384
1385         nr_mbufs  = nr_queues * nr_rx_desc;
1386         nr_mbufs += nr_mbufs_per_core * nr_switch_core;
1387         nr_mbufs *= nr_port;
1388
1389         mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", nr_mbufs,
1390                                             nr_mbuf_cache, 0, mbuf_size,
1391                                             rte_socket_id());
1392         if (mbuf_pool == NULL)
1393                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
1394 }
1395
1396 /*
1397  * Main function, does initialisation and calls the per-lcore functions.
1398  */
1399 int
1400 main(int argc, char *argv[])
1401 {
1402         unsigned lcore_id, core_id = 0;
1403         unsigned nb_ports, valid_num_ports;
1404         int ret, i;
1405         uint16_t portid;
1406         static pthread_t tid;
1407         uint64_t flags = 0;
1408
1409         signal(SIGINT, sigint_handler);
1410
1411         /* init EAL */
1412         ret = rte_eal_init(argc, argv);
1413         if (ret < 0)
1414                 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
1415         argc -= ret;
1416         argv += ret;
1417
1418         /* parse app arguments */
1419         ret = us_vhost_parse_args(argc, argv);
1420         if (ret < 0)
1421                 rte_exit(EXIT_FAILURE, "Invalid argument\n");
1422
1423         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1424                 TAILQ_INIT(&lcore_info[lcore_id].vdev_list);
1425
1426                 if (rte_lcore_is_enabled(lcore_id))
1427                         lcore_ids[core_id++] = lcore_id;
1428         }
1429
1430         if (rte_lcore_count() > RTE_MAX_LCORE)
1431                 rte_exit(EXIT_FAILURE,"Not enough cores\n");
1432
1433         /* Get the number of physical ports. */
1434         nb_ports = rte_eth_dev_count_avail();
1435
1436         /*
1437          * Update the global var NUM_PORTS and global array PORTS
1438          * and get value of var VALID_NUM_PORTS according to system ports number
1439          */
1440         valid_num_ports = check_ports_num(nb_ports);
1441
1442         if ((valid_num_ports ==  0) || (valid_num_ports > MAX_SUP_PORTS)) {
1443                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
1444                         "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
1445                 return -1;
1446         }
1447
1448         /*
1449          * FIXME: here we are trying to allocate mbufs big enough for
1450          * @MAX_QUEUES, but the truth is we're never going to use that
1451          * many queues here. We probably should only do allocation for
1452          * those queues we are going to use.
1453          */
1454         create_mbuf_pool(valid_num_ports, rte_lcore_count() - 1, MBUF_DATA_SIZE,
1455                          MAX_QUEUES, RTE_TEST_RX_DESC_DEFAULT, MBUF_CACHE_SIZE);
1456
1457         if (vm2vm_mode == VM2VM_HARDWARE) {
1458                 /* Enable VT loop back to let L2 switch to do it. */
1459                 vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1;
1460                 RTE_LOG(DEBUG, VHOST_CONFIG,
1461                         "Enable loop back for L2 switch in vmdq.\n");
1462         }
1463
1464         /* initialize all ports */
1465         RTE_ETH_FOREACH_DEV(portid) {
1466                 /* skip ports that are not enabled */
1467                 if ((enabled_port_mask & (1 << portid)) == 0) {
1468                         RTE_LOG(INFO, VHOST_PORT,
1469                                 "Skipping disabled port %d\n", portid);
1470                         continue;
1471                 }
1472                 if (port_init(portid) != 0)
1473                         rte_exit(EXIT_FAILURE,
1474                                 "Cannot initialize network ports\n");
1475         }
1476
1477         /* Enable stats if the user option is set. */
1478         if (enable_stats) {
1479                 ret = rte_ctrl_thread_create(&tid, "print-stats", NULL,
1480                                         print_stats, NULL);
1481                 if (ret < 0)
1482                         rte_exit(EXIT_FAILURE,
1483                                 "Cannot create print-stats thread\n");
1484         }
1485
1486         /* Launch all data cores. */
1487         RTE_LCORE_FOREACH_SLAVE(lcore_id)
1488                 rte_eal_remote_launch(switch_worker, NULL, lcore_id);
1489
1490         if (client_mode)
1491                 flags |= RTE_VHOST_USER_CLIENT;
1492
1493         if (dequeue_zero_copy)
1494                 flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
1495
1496         /* Register vhost user driver to handle vhost messages. */
1497         for (i = 0; i < nb_sockets; i++) {
1498                 char *file = socket_files + i * PATH_MAX;
1499                 ret = rte_vhost_driver_register(file, flags);
1500                 if (ret != 0) {
1501                         unregister_drivers(i);
1502                         rte_exit(EXIT_FAILURE,
1503                                 "vhost driver register failure.\n");
1504                 }
1505
1506                 if (builtin_net_driver)
1507                         rte_vhost_driver_set_features(file, VIRTIO_NET_FEATURES);
1508
1509                 if (mergeable == 0) {
1510                         rte_vhost_driver_disable_features(file,
1511                                 1ULL << VIRTIO_NET_F_MRG_RXBUF);
1512                 }
1513
1514                 if (enable_tx_csum == 0) {
1515                         rte_vhost_driver_disable_features(file,
1516                                 1ULL << VIRTIO_NET_F_CSUM);
1517                 }
1518
1519                 if (enable_tso == 0) {
1520                         rte_vhost_driver_disable_features(file,
1521                                 1ULL << VIRTIO_NET_F_HOST_TSO4);
1522                         rte_vhost_driver_disable_features(file,
1523                                 1ULL << VIRTIO_NET_F_HOST_TSO6);
1524                         rte_vhost_driver_disable_features(file,
1525                                 1ULL << VIRTIO_NET_F_GUEST_TSO4);
1526                         rte_vhost_driver_disable_features(file,
1527                                 1ULL << VIRTIO_NET_F_GUEST_TSO6);
1528                 }
1529
1530                 if (promiscuous) {
1531                         rte_vhost_driver_enable_features(file,
1532                                 1ULL << VIRTIO_NET_F_CTRL_RX);
1533                 }
1534
1535                 ret = rte_vhost_driver_callback_register(file,
1536                         &virtio_net_device_ops);
1537                 if (ret != 0) {
1538                         rte_exit(EXIT_FAILURE,
1539                                 "failed to register vhost driver callbacks.\n");
1540                 }
1541
1542                 if (rte_vhost_driver_start(file) < 0) {
1543                         rte_exit(EXIT_FAILURE,
1544                                 "failed to start vhost driver.\n");
1545                 }
1546         }
1547
1548         RTE_LCORE_FOREACH_SLAVE(lcore_id)
1549                 rte_eal_wait_lcore(lcore_id);
1550
1551         return 0;
1552
1553 }