1d6d463563767a99c5c6afea50232b1c5dce086e
[dpdk.git] / examples / tep_termination / main.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <arpa/inet.h>
35 #include <getopt.h>
36 #include <linux/if_ether.h>
37 #include <linux/if_vlan.h>
38 #include <linux/virtio_net.h>
39 #include <linux/virtio_ring.h>
40 #include <signal.h>
41 #include <stdint.h>
42 #include <sys/eventfd.h>
43 #include <sys/param.h>
44 #include <unistd.h>
45
46 #include <rte_atomic.h>
47 #include <rte_cycles.h>
48 #include <rte_ethdev.h>
49 #include <rte_log.h>
50 #include <rte_string_fns.h>
51 #include <rte_malloc.h>
52 #include <rte_virtio_net.h>
53
54 #include "main.h"
55 #include "vxlan.h"
56 #include "vxlan_setup.h"
57
58 /* the maximum number of external ports supported */
59 #define MAX_SUP_PORTS 1
60
61 /**
62  * Calculate the number of buffers needed per port
63  */
64 #define NUM_MBUFS_PER_PORT ((MAX_QUEUES * RTE_TEST_RX_DESC_DEFAULT) +\
65                                 (nb_switching_cores * MAX_PKT_BURST) +\
66                                 (nb_switching_cores * \
67                                 RTE_TEST_TX_DESC_DEFAULT) +\
68                                 (nb_switching_cores * MBUF_CACHE_SIZE))
69
70 #define MBUF_CACHE_SIZE 128
71 #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
72
73 #define MAX_PKT_BURST 32        /* Max burst size for RX/TX */
74 #define BURST_TX_DRAIN_US 100   /* TX drain every ~100us */
75
76 /* Defines how long we wait between retries on RX */
77 #define BURST_RX_WAIT_US 15
78
79 #define BURST_RX_RETRIES 4      /* Number of retries on RX. */
80
81 #define JUMBO_FRAME_MAX_SIZE    0x2600
82
83 /* State of virtio device. */
84 #define DEVICE_MAC_LEARNING 0
85 #define DEVICE_RX           1
86 #define DEVICE_SAFE_REMOVE  2
87
88 /* Config_core_flag status definitions. */
89 #define REQUEST_DEV_REMOVAL 1
90 #define ACK_DEV_REMOVAL     0
91
92 /* Configurable number of RX/TX ring descriptors */
93 #define RTE_TEST_RX_DESC_DEFAULT 1024
94 #define RTE_TEST_TX_DESC_DEFAULT 512
95
96 /* Get first 4 bytes in mbuf headroom. */
97 #define MBUF_HEADROOM_UINT32(mbuf) (*(uint32_t *)((uint8_t *)(mbuf) \
98                 + sizeof(struct rte_mbuf)))
99
100 #define INVALID_PORT_ID 0xFF
101
102 /* Size of buffers used for snprintfs. */
103 #define MAX_PRINT_BUFF 6072
104
105 /* Maximum character device basename size. */
106 #define MAX_BASENAME_SZ 20
107
108 /* Maximum long option length for option parsing. */
109 #define MAX_LONG_OPT_SZ 64
110
111 /* Used to compare MAC addresses. */
112 #define MAC_ADDR_CMP 0xFFFFFFFFFFFFULL
113
114 #define CMD_LINE_OPT_NB_DEVICES "nb-devices"
115 #define CMD_LINE_OPT_UDP_PORT "udp-port"
116 #define CMD_LINE_OPT_TX_CHECKSUM "tx-checksum"
117 #define CMD_LINE_OPT_TSO_SEGSZ "tso-segsz"
118 #define CMD_LINE_OPT_FILTER_TYPE "filter-type"
119 #define CMD_LINE_OPT_ENCAP "encap"
120 #define CMD_LINE_OPT_DECAP "decap"
121 #define CMD_LINE_OPT_RX_RETRY "rx-retry"
122 #define CMD_LINE_OPT_RX_RETRY_DELAY "rx-retry-delay"
123 #define CMD_LINE_OPT_RX_RETRY_NUM "rx-retry-num"
124 #define CMD_LINE_OPT_STATS "stats"
125 #define CMD_LINE_OPT_DEV_BASENAME "dev-basename"
126
127 /* mask of enabled ports */
128 static uint32_t enabled_port_mask;
129
130 /*Number of switching cores enabled*/
131 static uint32_t nb_switching_cores;
132
133 /* number of devices/queues to support*/
134 uint16_t nb_devices = 2;
135
136 /* max ring descriptor, ixgbe, i40e, e1000 all are 4096. */
137 #define MAX_RING_DESC 4096
138
139 struct vpool {
140         struct rte_mempool *pool;
141         struct rte_ring *ring;
142         uint32_t buf_size;
143 } vpool_array[MAX_QUEUES+MAX_QUEUES];
144
145 /* UDP tunneling port */
146 uint16_t udp_port = 4789;
147
148 /* enable/disable inner TX checksum */
149 uint8_t tx_checksum = 0;
150
151 /* TCP segment size */
152 uint16_t tso_segsz = 0;
153
154 /* enable/disable decapsulation */
155 uint8_t rx_decap = 1;
156
157 /* enable/disable encapsulation */
158 uint8_t tx_encap = 1;
159
160 /* RX filter type for tunneling packet */
161 uint8_t filter_idx = 1;
162
163 /* overlay packet operation */
164 struct ol_switch_ops overlay_options = {
165         .port_configure = vxlan_port_init,
166         .tunnel_setup = vxlan_link,
167         .tunnel_destroy = vxlan_unlink,
168         .tx_handle = vxlan_tx_pkts,
169         .rx_handle = vxlan_rx_pkts,
170         .param_handle = NULL,
171 };
172
173 /* Enable stats. */
174 uint32_t enable_stats = 0;
175 /* Enable retries on RX. */
176 static uint32_t enable_retry = 1;
177 /* Specify timeout (in useconds) between retries on RX. */
178 static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US;
179 /* Specify the number of retries on RX. */
180 static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;
181
182 /* Character device basename. Can be set by user. */
183 static char dev_basename[MAX_BASENAME_SZ] = "vhost-net";
184
185 static unsigned lcore_ids[RTE_MAX_LCORE];
186 uint8_t ports[RTE_MAX_ETHPORTS];
187
188 static unsigned nb_ports; /**< The number of ports specified in command line */
189
190 /* ethernet addresses of ports */
191 struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
192
193 /* heads for the main used and free linked lists for the data path. */
194 static struct virtio_net_data_ll *ll_root_used;
195 static struct virtio_net_data_ll *ll_root_free;
196
197 /**
198  * Array of data core structures containing information on
199  * individual core linked lists.
200  */
201 static struct lcore_info lcore_info[RTE_MAX_LCORE];
202
203 /* Used for queueing bursts of TX packets. */
204 struct mbuf_table {
205         unsigned len;
206         unsigned txq_id;
207         struct rte_mbuf *m_table[MAX_PKT_BURST];
208 };
209
210 /* TX queue for each data core. */
211 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE];
212
213 struct device_statistics dev_statistics[MAX_DEVICES];
214
215 /**
216  * Set character device basename.
217  */
218 static int
219 us_vhost_parse_basename(const char *q_arg)
220 {
221         /* parse number string */
222         if (strlen(q_arg) >= MAX_BASENAME_SZ)
223                 return -1;
224         else
225                 snprintf((char *)&dev_basename, MAX_BASENAME_SZ, "%s", q_arg);
226
227         return 0;
228 }
229
230 /**
231  * Parse the portmask provided at run time.
232  */
233 static int
234 parse_portmask(const char *portmask)
235 {
236         char *end = NULL;
237         unsigned long pm;
238
239         /* parse hexadecimal string */
240         pm = strtoul(portmask, &end, 16);
241         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
242                 return -1;
243
244         if (pm == 0)
245                 return -1;
246
247         return pm;
248 }
249
250 /**
251  * Parse num options at run time.
252  */
253 static int
254 parse_num_opt(const char *q_arg, uint32_t max_valid_value)
255 {
256         char *end = NULL;
257         unsigned long num;
258
259         /* parse unsigned int string */
260         num = strtoul(q_arg, &end, 10);
261         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
262                 return -1;
263
264         if (num > max_valid_value)
265                 return -1;
266
267         return num;
268 }
269
270 /**
271  * Display usage
272  */
273 static void
274 tep_termination_usage(const char *prgname)
275 {
276         RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n"
277         "               --udp-port: UDP destination port for VXLAN packet\n"
278         "               --nb-devices[1-64]: The number of virtIO device\n"
279         "               --tx-checksum [0|1]: inner Tx checksum offload\n"
280         "               --tso-segsz [0-N]: TCP segment size\n"
281         "               --decap [0|1]: tunneling packet decapsulation\n"
282         "               --encap [0|1]: tunneling packet encapsulation\n"
283         "               --filter-type[1-3]: filter type for tunneling packet\n"
284         "                   1: Inner MAC and tenent ID\n"
285         "                   2: Inner MAC and VLAN, and tenent ID\n"
286         "                   3: Outer MAC, Inner MAC and tenent ID\n"
287         "               -p PORTMASK: Set mask for ports to be used by application\n"
288         "               --rx-retry [0|1]: disable/enable(default) retries on rx."
289         "                Enable retry if destintation queue is full\n"
290         "               --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX."
291         "                This makes effect only if retries on rx enabled\n"
292         "               --rx-retry-num [0-N]: the number of retries on rx."
293         "                This makes effect only if retries on rx enabled\n"
294         "               --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
295         "               --dev-basename: The basename to be used for the character device.\n",
296                prgname);
297 }
298
299 /**
300  * Parse the arguments given in the command line of the application.
301  */
302 static int
303 tep_termination_parse_args(int argc, char **argv)
304 {
305         int opt, ret;
306         int option_index;
307         unsigned i;
308         const char *prgname = argv[0];
309         static struct option long_option[] = {
310                 {CMD_LINE_OPT_NB_DEVICES, required_argument, NULL, 0},
311                 {CMD_LINE_OPT_UDP_PORT, required_argument, NULL, 0},
312                 {CMD_LINE_OPT_TX_CHECKSUM, required_argument, NULL, 0},
313                 {CMD_LINE_OPT_TSO_SEGSZ, required_argument, NULL, 0},
314                 {CMD_LINE_OPT_DECAP, required_argument, NULL, 0},
315                 {CMD_LINE_OPT_ENCAP, required_argument, NULL, 0},
316                 {CMD_LINE_OPT_FILTER_TYPE, required_argument, NULL, 0},
317                 {CMD_LINE_OPT_RX_RETRY, required_argument, NULL, 0},
318                 {CMD_LINE_OPT_RX_RETRY_DELAY, required_argument, NULL, 0},
319                 {CMD_LINE_OPT_RX_RETRY_NUM, required_argument, NULL, 0},
320                 {CMD_LINE_OPT_STATS, required_argument, NULL, 0},
321                 {CMD_LINE_OPT_DEV_BASENAME, required_argument, NULL, 0},
322                 {NULL, 0, 0, 0},
323         };
324
325         /* Parse command line */
326         while ((opt = getopt_long(argc, argv, "p:",
327                         long_option, &option_index)) != EOF) {
328                 switch (opt) {
329                 /* Portmask */
330                 case 'p':
331                         enabled_port_mask = parse_portmask(optarg);
332                         if (enabled_port_mask == 0) {
333                                 RTE_LOG(INFO, VHOST_CONFIG,
334                                         "Invalid portmask\n");
335                                 tep_termination_usage(prgname);
336                                 return -1;
337                         }
338                         break;
339                 case 0:
340                         if (!strncmp(long_option[option_index].name,
341                                 CMD_LINE_OPT_NB_DEVICES,
342                                 sizeof(CMD_LINE_OPT_NB_DEVICES))) {
343                                 ret = parse_num_opt(optarg, MAX_DEVICES);
344                                 if (ret == -1) {
345                                         RTE_LOG(INFO, VHOST_CONFIG,
346                                         "Invalid argument for nb-devices [0-%d]\n",
347                                         MAX_DEVICES);
348                                         tep_termination_usage(prgname);
349                                         return -1;
350                                 } else
351                                         nb_devices = ret;
352                         }
353
354                         /* Enable/disable retries on RX. */
355                         if (!strncmp(long_option[option_index].name,
356                                 CMD_LINE_OPT_RX_RETRY,
357                                 sizeof(CMD_LINE_OPT_RX_RETRY))) {
358                                 ret = parse_num_opt(optarg, 1);
359                                 if (ret == -1) {
360                                         RTE_LOG(INFO, VHOST_CONFIG,
361                                                 "Invalid argument for rx-retry [0|1]\n");
362                                         tep_termination_usage(prgname);
363                                         return -1;
364                                 } else
365                                         enable_retry = ret;
366                         }
367
368                         if (!strncmp(long_option[option_index].name,
369                                 CMD_LINE_OPT_TSO_SEGSZ,
370                                 sizeof(CMD_LINE_OPT_TSO_SEGSZ))) {
371                                 ret = parse_num_opt(optarg, INT16_MAX);
372                                 if (ret == -1) {
373                                         RTE_LOG(INFO, VHOST_CONFIG,
374                                                 "Invalid argument for TCP segment size [0-N]\n");
375                                         tep_termination_usage(prgname);
376                                         return -1;
377                                 } else
378                                         tso_segsz = ret;
379                         }
380
381                         if (!strncmp(long_option[option_index].name,
382                                         CMD_LINE_OPT_UDP_PORT,
383                                         sizeof(CMD_LINE_OPT_UDP_PORT))) {
384                                 ret = parse_num_opt(optarg, INT16_MAX);
385                                 if (ret == -1) {
386                                         RTE_LOG(INFO, VHOST_CONFIG,
387                                                 "Invalid argument for UDP port [0-N]\n");
388                                         tep_termination_usage(prgname);
389                                         return -1;
390                                 } else
391                                         udp_port = ret;
392                         }
393
394                         /* Specify the retries delay time (in useconds) on RX.*/
395                         if (!strncmp(long_option[option_index].name,
396                                 CMD_LINE_OPT_RX_RETRY_DELAY,
397                                 sizeof(CMD_LINE_OPT_RX_RETRY_DELAY))) {
398                                 ret = parse_num_opt(optarg, INT32_MAX);
399                                 if (ret == -1) {
400                                         RTE_LOG(INFO, VHOST_CONFIG,
401                                                 "Invalid argument for rx-retry-delay [0-N]\n");
402                                         tep_termination_usage(prgname);
403                                         return -1;
404                                 } else
405                                         burst_rx_delay_time = ret;
406                         }
407
408                         /* Specify the retries number on RX. */
409                         if (!strncmp(long_option[option_index].name,
410                                 CMD_LINE_OPT_RX_RETRY_NUM,
411                                 sizeof(CMD_LINE_OPT_RX_RETRY_NUM))) {
412                                 ret = parse_num_opt(optarg, INT32_MAX);
413                                 if (ret == -1) {
414                                         RTE_LOG(INFO, VHOST_CONFIG,
415                                                 "Invalid argument for rx-retry-num [0-N]\n");
416                                         tep_termination_usage(prgname);
417                                         return -1;
418                                 } else
419                                         burst_rx_retry_num = ret;
420                         }
421
422                         if (!strncmp(long_option[option_index].name,
423                                 CMD_LINE_OPT_TX_CHECKSUM,
424                                 sizeof(CMD_LINE_OPT_TX_CHECKSUM))) {
425                                 ret = parse_num_opt(optarg, 1);
426                                 if (ret == -1) {
427                                         RTE_LOG(INFO, VHOST_CONFIG,
428                                                 "Invalid argument for tx-checksum [0|1]\n");
429                                         tep_termination_usage(prgname);
430                                         return -1;
431                                 } else
432                                         tx_checksum = ret;
433                         }
434
435                         if (!strncmp(long_option[option_index].name,
436                                         CMD_LINE_OPT_FILTER_TYPE,
437                                         sizeof(CMD_LINE_OPT_FILTER_TYPE))) {
438                                 ret = parse_num_opt(optarg, 3);
439                                 if ((ret == -1) || (ret == 0)) {
440                                         RTE_LOG(INFO, VHOST_CONFIG,
441                                                 "Invalid argument for filter type [1-3]\n");
442                                         tep_termination_usage(prgname);
443                                         return -1;
444                                 } else
445                                         filter_idx = ret - 1;
446                         }
447
448                         /* Enable/disable encapsulation on RX. */
449                         if (!strncmp(long_option[option_index].name,
450                                 CMD_LINE_OPT_DECAP,
451                                 sizeof(CMD_LINE_OPT_DECAP))) {
452                                 ret = parse_num_opt(optarg, 1);
453                                 if (ret == -1) {
454                                         RTE_LOG(INFO, VHOST_CONFIG,
455                                                 "Invalid argument for decap [0|1]\n");
456                                         tep_termination_usage(prgname);
457                                         return -1;
458                                 } else
459                                         rx_decap = ret;
460                         }
461
462                         /* Enable/disable encapsulation on TX. */
463                         if (!strncmp(long_option[option_index].name,
464                                 CMD_LINE_OPT_ENCAP,
465                                 sizeof(CMD_LINE_OPT_ENCAP))) {
466                                 ret = parse_num_opt(optarg, 1);
467                                 if (ret == -1) {
468                                         RTE_LOG(INFO, VHOST_CONFIG,
469                                                 "Invalid argument for encap [0|1]\n");
470                                         tep_termination_usage(prgname);
471                                         return -1;
472                                 } else
473                                         tx_encap = ret;
474                         }
475
476                         /* Enable/disable stats. */
477                         if (!strncmp(long_option[option_index].name,
478                                 CMD_LINE_OPT_STATS,
479                                 sizeof(CMD_LINE_OPT_STATS))) {
480                                 ret = parse_num_opt(optarg, INT32_MAX);
481                                 if (ret == -1) {
482                                         RTE_LOG(INFO, VHOST_CONFIG,
483                                                         "Invalid argument for stats [0..N]\n");
484                                         tep_termination_usage(prgname);
485                                         return -1;
486                                 } else
487                                         enable_stats = ret;
488                         }
489
490                         /* Set character device basename. */
491                         if (!strncmp(long_option[option_index].name,
492                                 CMD_LINE_OPT_DEV_BASENAME,
493                                 sizeof(CMD_LINE_OPT_DEV_BASENAME))) {
494                                 if (us_vhost_parse_basename(optarg) == -1) {
495                                         RTE_LOG(INFO, VHOST_CONFIG,
496                                                 "Invalid argument for character "
497                                                 "device basename (Max %d characters)\n",
498                                                 MAX_BASENAME_SZ);
499                                         tep_termination_usage(prgname);
500                                         return -1;
501                                 }
502                         }
503
504                         break;
505
506                         /* Invalid option - print options. */
507                 default:
508                         tep_termination_usage(prgname);
509                         return -1;
510                 }
511         }
512
513         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
514                 if (enabled_port_mask & (1 << i))
515                         ports[nb_ports++] = (uint8_t)i;
516         }
517
518         if ((nb_ports ==  0) || (nb_ports > MAX_SUP_PORTS)) {
519                 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
520                         "but only %u port can be enabled\n", nb_ports,
521                         MAX_SUP_PORTS);
522                 return -1;
523         }
524
525         return 0;
526 }
527
528 /**
529  * Update the global var NB_PORTS and array PORTS
530  * according to system ports number and return valid ports number
531  */
532 static unsigned
533 check_ports_num(unsigned max_nb_ports)
534 {
535         unsigned valid_nb_ports = nb_ports;
536         unsigned portid;
537
538         if (nb_ports > max_nb_ports) {
539                 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) "
540                         " exceeds total system port number(%u)\n",
541                         nb_ports, max_nb_ports);
542                 nb_ports = max_nb_ports;
543         }
544
545         for (portid = 0; portid < nb_ports; portid++) {
546                 if (ports[portid] >= max_nb_ports) {
547                         RTE_LOG(INFO, VHOST_PORT,
548                                 "\nSpecified port ID(%u) exceeds max "
549                                 " system port ID(%u)\n",
550                                 ports[portid], (max_nb_ports - 1));
551                         ports[portid] = INVALID_PORT_ID;
552                         valid_nb_ports--;
553                 }
554         }
555         return valid_nb_ports;
556 }
557
558 /**
559  * This function routes the TX packet to the correct interface. This may be a local device
560  * or the physical port.
561  */
562 static inline void __attribute__((always_inline))
563 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m)
564 {
565         struct mbuf_table *tx_q;
566         struct rte_mbuf **m_table;
567         unsigned len, ret = 0;
568         const uint16_t lcore_id = rte_lcore_id();
569
570         RTE_LOG(DEBUG, VHOST_DATA, "(%d) TX: MAC address is external\n",
571                 vdev->vid);
572
573         /* Add packet to the port tx queue */
574         tx_q = &lcore_tx_queue[lcore_id];
575         len = tx_q->len;
576
577         tx_q->m_table[len] = m;
578         len++;
579         if (enable_stats) {
580                 dev_statistics[vdev->vid].tx_total++;
581                 dev_statistics[vdev->vid].tx++;
582         }
583
584         if (unlikely(len == MAX_PKT_BURST)) {
585                 m_table = (struct rte_mbuf **)tx_q->m_table;
586                 ret = overlay_options.tx_handle(ports[0],
587                         (uint16_t)tx_q->txq_id, m_table,
588                         (uint16_t)tx_q->len);
589
590                 /* Free any buffers not handled by TX and update
591                  * the port stats.
592                  */
593                 if (unlikely(ret < len)) {
594                         do {
595                                 rte_pktmbuf_free(m_table[ret]);
596                         } while (++ret < len);
597                 }
598
599                 len = 0;
600         }
601
602         tx_q->len = len;
603         return;
604 }
605
606 /**
607  * This function is called by each data core. It handles all
608  * RX/TX registered with the core. For TX the specific lcore
609  * linked list is used. For RX, MAC addresses are compared
610  * with all devices in the main linked list.
611  */
612 static int
613 switch_worker(__rte_unused void *arg)
614 {
615         struct rte_mempool *mbuf_pool = arg;
616         struct vhost_dev *vdev = NULL;
617         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
618         struct virtio_net_data_ll *dev_ll;
619         struct mbuf_table *tx_q;
620         volatile struct lcore_ll_info *lcore_ll;
621         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
622                                         / US_PER_S * BURST_TX_DRAIN_US;
623         uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0;
624         unsigned i, ret = 0;
625         const uint16_t lcore_id = rte_lcore_id();
626         const uint16_t num_cores = (uint16_t)rte_lcore_count();
627         uint16_t rx_count = 0;
628         uint16_t tx_count;
629         uint32_t retry = 0;
630
631         RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
632         lcore_ll = lcore_info[lcore_id].lcore_ll;
633         prev_tsc = 0;
634
635         tx_q = &lcore_tx_queue[lcore_id];
636         for (i = 0; i < num_cores; i++) {
637                 if (lcore_ids[i] == lcore_id) {
638                         tx_q->txq_id = i;
639                         break;
640                 }
641         }
642
643         while (1) {
644                 cur_tsc = rte_rdtsc();
645                 /*
646                  * TX burst queue drain
647                  */
648                 diff_tsc = cur_tsc - prev_tsc;
649                 if (unlikely(diff_tsc > drain_tsc)) {
650
651                         if (tx_q->len) {
652                                 RTE_LOG(DEBUG, VHOST_DATA, "TX queue drained after "
653                                         "timeout with burst size %u\n",
654                                         tx_q->len);
655                                 ret = overlay_options.tx_handle(ports[0],
656                                         (uint16_t)tx_q->txq_id,
657                                         (struct rte_mbuf **)tx_q->m_table,
658                                         (uint16_t)tx_q->len);
659                                 if (unlikely(ret < tx_q->len)) {
660                                         do {
661                                                 rte_pktmbuf_free(tx_q->m_table[ret]);
662                                         } while (++ret < tx_q->len);
663                                 }
664
665                                 tx_q->len = 0;
666                         }
667
668                         prev_tsc = cur_tsc;
669
670                 }
671
672                 rte_prefetch0(lcore_ll->ll_root_used);
673
674                 /**
675                  * Inform the configuration core that we have exited
676                  * the linked list and that no devices are
677                  * in use if requested.
678                  */
679                 if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL)
680                         lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
681
682                 /*
683                  * Process devices
684                  */
685                 dev_ll = lcore_ll->ll_root_used;
686
687                 while (dev_ll != NULL) {
688                         vdev = dev_ll->vdev;
689
690                         if (unlikely(vdev->remove)) {
691                                 dev_ll = dev_ll->next;
692                                 overlay_options.tunnel_destroy(vdev);
693                                 vdev->ready = DEVICE_SAFE_REMOVE;
694                                 continue;
695                         }
696                         if (likely(vdev->ready == DEVICE_RX)) {
697                                 /* Handle guest RX */
698                                 rx_count = rte_eth_rx_burst(ports[0],
699                                         vdev->rx_q, pkts_burst, MAX_PKT_BURST);
700
701                                 if (rx_count) {
702                                         /*
703                                         * Retry is enabled and the queue is
704                                         * full then we wait and retry to
705                                         * avoid packet loss. Here MAX_PKT_BURST
706                                         * must be less than virtio queue size
707                                         */
708                                         if (enable_retry && unlikely(rx_count >
709                                                 rte_vhost_avail_entries(vdev->vid, VIRTIO_RXQ))) {
710                                                 for (retry = 0; retry < burst_rx_retry_num;
711                                                         retry++) {
712                                                         rte_delay_us(burst_rx_delay_time);
713                                                         if (rx_count <= rte_vhost_avail_entries(vdev->vid, VIRTIO_RXQ))
714                                                                 break;
715                                                 }
716                                         }
717
718                                         ret_count = overlay_options.rx_handle(vdev->vid, pkts_burst, rx_count);
719                                         if (enable_stats) {
720                                                 rte_atomic64_add(
721                                                 &dev_statistics[vdev->vid].rx_total_atomic,
722                                                 rx_count);
723                                                 rte_atomic64_add(
724                                                 &dev_statistics[vdev->vid].rx_atomic, ret_count);
725                                         }
726                                         while (likely(rx_count)) {
727                                                 rx_count--;
728                                                 rte_pktmbuf_free(pkts_burst[rx_count]);
729                                         }
730
731                                 }
732                         }
733
734                         if (likely(!vdev->remove)) {
735                                 /* Handle guest TX*/
736                                 tx_count = rte_vhost_dequeue_burst(vdev->vid,
737                                                 VIRTIO_TXQ, mbuf_pool,
738                                                 pkts_burst, MAX_PKT_BURST);
739                                 /* If this is the first received packet we need to learn the MAC */
740                                 if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) {
741                                         if (vdev->remove ||
742                                                 (overlay_options.tunnel_setup(vdev, pkts_burst[0]) == -1)) {
743                                                 while (tx_count)
744                                                         rte_pktmbuf_free(pkts_burst[--tx_count]);
745                                         }
746                                 }
747                                 while (tx_count)
748                                         virtio_tx_route(vdev, pkts_burst[--tx_count]);
749                         }
750
751                         /* move to the next device in the list */
752                         dev_ll = dev_ll->next;
753                 }
754         }
755
756         return 0;
757 }
758
759 /**
760  * Add an entry to a used linked list. A free entry must first be found
761  * in the free linked list using get_data_ll_free_entry();
762  */
763 static void
764 add_data_ll_entry(struct virtio_net_data_ll **ll_root_addr,
765         struct virtio_net_data_ll *ll_dev)
766 {
767         struct virtio_net_data_ll *ll = *ll_root_addr;
768
769         /* Set next as NULL and use a compiler barrier to avoid reordering. */
770         ll_dev->next = NULL;
771         rte_compiler_barrier();
772
773         /* If ll == NULL then this is the first device. */
774         if (ll) {
775                 /* Increment to the tail of the linked list. */
776                 while (ll->next != NULL)
777                         ll = ll->next;
778
779                 ll->next = ll_dev;
780         } else {
781                 *ll_root_addr = ll_dev;
782         }
783 }
784
785 /**
786  * Remove an entry from a used linked list. The entry must then be added to
787  * the free linked list using put_data_ll_free_entry().
788  */
789 static void
790 rm_data_ll_entry(struct virtio_net_data_ll **ll_root_addr,
791         struct virtio_net_data_ll *ll_dev,
792         struct virtio_net_data_ll *ll_dev_last)
793 {
794         struct virtio_net_data_ll *ll = *ll_root_addr;
795
796         if (unlikely((ll == NULL) || (ll_dev == NULL)))
797                 return;
798
799         if (ll_dev == ll)
800                 *ll_root_addr = ll_dev->next;
801         else
802                 if (likely(ll_dev_last != NULL))
803                         ll_dev_last->next = ll_dev->next;
804                 else
805                         RTE_LOG(ERR, VHOST_CONFIG,
806                                 "Remove entry form ll failed.\n");
807 }
808
809 /**
810  * Find and return an entry from the free linked list.
811  */
812 static struct virtio_net_data_ll *
813 get_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr)
814 {
815         struct virtio_net_data_ll *ll_free = *ll_root_addr;
816         struct virtio_net_data_ll *ll_dev;
817
818         if (ll_free == NULL)
819                 return NULL;
820
821         ll_dev = ll_free;
822         *ll_root_addr = ll_free->next;
823
824         return ll_dev;
825 }
826
827 /**
828  * Place an entry back on to the free linked list.
829  */
830 static void
831 put_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr,
832         struct virtio_net_data_ll *ll_dev)
833 {
834         struct virtio_net_data_ll *ll_free = *ll_root_addr;
835
836         if (ll_dev == NULL)
837                 return;
838
839         ll_dev->next = ll_free;
840         *ll_root_addr = ll_dev;
841 }
842
843 /**
844  * Creates a linked list of a given size.
845  */
846 static struct virtio_net_data_ll *
847 alloc_data_ll(uint32_t size)
848 {
849         struct virtio_net_data_ll *ll_new;
850         uint32_t i;
851
852         /* Malloc and then chain the linked list. */
853         ll_new = malloc(size * sizeof(struct virtio_net_data_ll));
854         if (ll_new == NULL) {
855                 RTE_LOG(ERR, VHOST_CONFIG,
856                         "Failed to allocate memory for ll_new.\n");
857                 return NULL;
858         }
859
860         for (i = 0; i < size - 1; i++) {
861                 ll_new[i].vdev = NULL;
862                 ll_new[i].next = &ll_new[i+1];
863         }
864         ll_new[i].next = NULL;
865
866         return ll_new;
867 }
868
869 /**
870  * Create the main linked list along with each individual cores
871  * linked list. A used and a free list are created to manage entries.
872  */
873 static int
874 init_data_ll(void)
875 {
876         int lcore;
877
878         RTE_LCORE_FOREACH_SLAVE(lcore) {
879                 lcore_info[lcore].lcore_ll =
880                         malloc(sizeof(struct lcore_ll_info));
881                 if (lcore_info[lcore].lcore_ll == NULL) {
882                         RTE_LOG(ERR, VHOST_CONFIG,
883                                 "Failed to allocate memory for lcore_ll.\n");
884                         return -1;
885                 }
886
887                 lcore_info[lcore].lcore_ll->device_num = 0;
888                 lcore_info[lcore].lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
889                 lcore_info[lcore].lcore_ll->ll_root_used = NULL;
890                 if (nb_devices % nb_switching_cores)
891                         lcore_info[lcore].lcore_ll->ll_root_free =
892                                 alloc_data_ll((nb_devices / nb_switching_cores)
893                                                 + 1);
894                 else
895                         lcore_info[lcore].lcore_ll->ll_root_free =
896                                 alloc_data_ll(nb_devices / nb_switching_cores);
897         }
898
899         /* Allocate devices up to a maximum of MAX_DEVICES. */
900         ll_root_free = alloc_data_ll(MIN((nb_devices), MAX_DEVICES));
901
902         return 0;
903 }
904
905 /**
906  * Remove a device from the specific data core linked list and
907  * from the main linked list. Synchonization occurs through the use
908  * of the lcore dev_removal_flag.
909  */
910 static void
911 destroy_device(int vid)
912 {
913         struct virtio_net_data_ll *ll_lcore_dev_cur;
914         struct virtio_net_data_ll *ll_main_dev_cur;
915         struct virtio_net_data_ll *ll_lcore_dev_last = NULL;
916         struct virtio_net_data_ll *ll_main_dev_last = NULL;
917         struct vhost_dev *vdev = NULL;
918         int lcore;
919
920         ll_main_dev_cur = ll_root_used;
921         while (ll_main_dev_cur != NULL) {
922                 if (ll_main_dev_cur->vdev->vid == vid) {
923                         vdev = ll_main_dev_cur->vdev;
924                         break;
925                 }
926         }
927         if (!vdev)
928                 return;
929
930         /* set the remove flag. */
931         vdev->remove = 1;
932         while (vdev->ready != DEVICE_SAFE_REMOVE)
933                 rte_pause();
934
935         /* Search for entry to be removed from lcore ll */
936         ll_lcore_dev_cur = lcore_info[vdev->coreid].lcore_ll->ll_root_used;
937         while (ll_lcore_dev_cur != NULL) {
938                 if (ll_lcore_dev_cur->vdev == vdev) {
939                         break;
940                 } else {
941                         ll_lcore_dev_last = ll_lcore_dev_cur;
942                         ll_lcore_dev_cur = ll_lcore_dev_cur->next;
943                 }
944         }
945
946         if (ll_lcore_dev_cur == NULL) {
947                 RTE_LOG(ERR, VHOST_CONFIG,
948                         "(%d) Failed to find the dev to be destroy.\n", vid);
949                 return;
950         }
951
952         /* Search for entry to be removed from main ll */
953         ll_main_dev_cur = ll_root_used;
954         ll_main_dev_last = NULL;
955         while (ll_main_dev_cur != NULL) {
956                 if (ll_main_dev_cur->vdev == vdev) {
957                         break;
958                 } else {
959                         ll_main_dev_last = ll_main_dev_cur;
960                         ll_main_dev_cur = ll_main_dev_cur->next;
961                 }
962         }
963
964         /* Remove entries from the lcore and main ll. */
965         rm_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used,
966                         ll_lcore_dev_cur, ll_lcore_dev_last);
967         rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
968
969         /* Set the dev_removal_flag on each lcore. */
970         RTE_LCORE_FOREACH_SLAVE(lcore) {
971                 lcore_info[lcore].lcore_ll->dev_removal_flag =
972                         REQUEST_DEV_REMOVAL;
973         }
974
975         /*
976          * Once each core has set the dev_removal_flag to
977          * ACK_DEV_REMOVAL we can be sure that they can no longer access
978          * the device removed from the linked lists and that the devices
979          * are no longer in use.
980          */
981         RTE_LCORE_FOREACH_SLAVE(lcore) {
982                 while (lcore_info[lcore].lcore_ll->dev_removal_flag
983                         != ACK_DEV_REMOVAL)
984                         rte_pause();
985         }
986
987         /* Add the entries back to the lcore and main free ll.*/
988         put_data_ll_free_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_free,
989                                 ll_lcore_dev_cur);
990         put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
991
992         /* Decrement number of device on the lcore. */
993         lcore_info[vdev->coreid].lcore_ll->device_num--;
994
995         RTE_LOG(INFO, VHOST_DATA, "(%d) Device has been removed "
996                 "from data core\n", vid);
997
998         rte_free(vdev);
999
1000 }
1001
1002 /**
1003  * A new device is added to a data core. First the device is added
1004  * to the main linked list and the allocated to a specific data core.
1005  */
1006 static int
1007 new_device(int vid)
1008 {
1009         struct virtio_net_data_ll *ll_dev;
1010         int lcore, core_add = 0;
1011         uint32_t device_num_min = nb_devices;
1012         struct vhost_dev *vdev;
1013
1014         vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
1015         if (vdev == NULL) {
1016                 RTE_LOG(INFO, VHOST_DATA,
1017                         "(%d) Couldn't allocate memory for vhost dev\n", vid);
1018                 return -1;
1019         }
1020         vdev->vid = vid;
1021         /* Add device to main ll */
1022         ll_dev = get_data_ll_free_entry(&ll_root_free);
1023         if (ll_dev == NULL) {
1024                 RTE_LOG(INFO, VHOST_DATA, "(%d) No free entry found in"
1025                         " linked list Device limit of %d devices per core"
1026                         " has been reached\n", vid, nb_devices);
1027                 if (vdev->regions_hpa)
1028                         rte_free(vdev->regions_hpa);
1029                 rte_free(vdev);
1030                 return -1;
1031         }
1032         ll_dev->vdev = vdev;
1033         add_data_ll_entry(&ll_root_used, ll_dev);
1034         vdev->rx_q = vid;
1035
1036         /* reset ready flag */
1037         vdev->ready = DEVICE_MAC_LEARNING;
1038         vdev->remove = 0;
1039
1040         /* Find a suitable lcore to add the device. */
1041         RTE_LCORE_FOREACH_SLAVE(lcore) {
1042                 if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
1043                         device_num_min = lcore_info[lcore].lcore_ll->device_num;
1044                         core_add = lcore;
1045                 }
1046         }
1047         /* Add device to lcore ll */
1048         ll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);
1049         if (ll_dev == NULL) {
1050                 RTE_LOG(INFO, VHOST_DATA,
1051                         "(%d) Failed to add device to data core\n",
1052                         vid);
1053                 vdev->ready = DEVICE_SAFE_REMOVE;
1054                 destroy_device(vid);
1055                 rte_free(vdev->regions_hpa);
1056                 rte_free(vdev);
1057                 return -1;
1058         }
1059         ll_dev->vdev = vdev;
1060         vdev->coreid = core_add;
1061
1062         add_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used,
1063                         ll_dev);
1064
1065         /* Initialize device stats */
1066         memset(&dev_statistics[vid], 0,
1067                 sizeof(struct device_statistics));
1068
1069         /* Disable notifications. */
1070         rte_vhost_enable_guest_notification(vid, VIRTIO_RXQ, 0);
1071         rte_vhost_enable_guest_notification(vid, VIRTIO_TXQ, 0);
1072         lcore_info[vdev->coreid].lcore_ll->device_num++;
1073
1074         RTE_LOG(INFO, VHOST_DATA, "(%d) Device has been added to data core %d\n",
1075                 vid, vdev->coreid);
1076
1077         return 0;
1078 }
1079
1080 /**
1081  * These callback allow devices to be added to the data core when configuration
1082  * has been fully complete.
1083  */
1084 static const struct virtio_net_device_ops virtio_net_device_ops = {
1085         .new_device =  new_device,
1086         .destroy_device = destroy_device,
1087 };
1088
1089 /**
1090  * This is a thread will wake up after a period to print stats if the user has
1091  * enabled them.
1092  */
1093 static void
1094 print_stats(void)
1095 {
1096         struct virtio_net_data_ll *dev_ll;
1097         uint64_t tx_dropped, rx_dropped;
1098         uint64_t tx, tx_total, rx, rx_total, rx_ip_csum, rx_l4_csum;
1099         int vid;
1100         const char clr[] = { 27, '[', '2', 'J', '\0' };
1101         const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
1102
1103         while (1) {
1104                 sleep(enable_stats);
1105
1106                 /* Clear screen and move to top left */
1107                 printf("%s%s", clr, top_left);
1108
1109                 printf("\nDevice statistics ================================");
1110
1111                 dev_ll = ll_root_used;
1112                 while (dev_ll != NULL) {
1113                         vid = dev_ll->vdev->vid;
1114                         tx_total = dev_statistics[vid].tx_total;
1115                         tx = dev_statistics[vid].tx;
1116                         tx_dropped = tx_total - tx;
1117
1118                         rx_total = rte_atomic64_read(
1119                                 &dev_statistics[vid].rx_total_atomic);
1120                         rx = rte_atomic64_read(
1121                                 &dev_statistics[vid].rx_atomic);
1122                         rx_dropped = rx_total - rx;
1123                         rx_ip_csum = rte_atomic64_read(
1124                                 &dev_statistics[vid].rx_bad_ip_csum);
1125                         rx_l4_csum = rte_atomic64_read(
1126                                 &dev_statistics[vid].rx_bad_l4_csum);
1127
1128                         printf("\nStatistics for device %d ----------"
1129                                         "\nTX total:            %"PRIu64""
1130                                         "\nTX dropped:          %"PRIu64""
1131                                         "\nTX successful:               %"PRIu64""
1132                                         "\nRX total:            %"PRIu64""
1133                                         "\nRX bad IP csum:      %"PRIu64""
1134                                         "\nRX bad L4 csum:      %"PRIu64""
1135                                         "\nRX dropped:          %"PRIu64""
1136                                         "\nRX successful:               %"PRIu64"",
1137                                         vid,
1138                                         tx_total,
1139                                         tx_dropped,
1140                                         tx,
1141                                         rx_total,
1142                                         rx_ip_csum,
1143                                         rx_l4_csum,
1144                                         rx_dropped,
1145                                         rx);
1146
1147                         dev_ll = dev_ll->next;
1148                 }
1149                 printf("\n================================================\n");
1150         }
1151 }
1152
1153 /**
1154  * Main function, does initialisation and calls the per-lcore functions.
1155  */
1156 int
1157 main(int argc, char *argv[])
1158 {
1159         struct rte_mempool *mbuf_pool = NULL;
1160         unsigned lcore_id, core_id = 0;
1161         unsigned nb_ports, valid_nb_ports;
1162         int ret;
1163         uint8_t portid;
1164         uint16_t queue_id;
1165         static pthread_t tid;
1166         char thread_name[RTE_MAX_THREAD_NAME_LEN];
1167
1168         /* init EAL */
1169         ret = rte_eal_init(argc, argv);
1170         if (ret < 0)
1171                 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
1172         argc -= ret;
1173         argv += ret;
1174
1175         /* parse app arguments */
1176         ret = tep_termination_parse_args(argc, argv);
1177         if (ret < 0)
1178                 rte_exit(EXIT_FAILURE, "Invalid argument\n");
1179
1180         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++)
1181                 if (rte_lcore_is_enabled(lcore_id))
1182                         lcore_ids[core_id++] = lcore_id;
1183
1184         /* set the number of swithcing cores available */
1185         nb_switching_cores = rte_lcore_count()-1;
1186
1187         /* Get the number of physical ports. */
1188         nb_ports = rte_eth_dev_count();
1189
1190         /*
1191          * Update the global var NB_PORTS and global array PORTS
1192          * and get value of var VALID_NB_PORTS according to system ports number
1193          */
1194         valid_nb_ports = check_ports_num(nb_ports);
1195
1196         if ((valid_nb_ports == 0) || (valid_nb_ports > MAX_SUP_PORTS)) {
1197                 rte_exit(EXIT_FAILURE, "Current enabled port number is %u,"
1198                         "but only %u port can be enabled\n", nb_ports,
1199                         MAX_SUP_PORTS);
1200         }
1201         /* Create the mbuf pool. */
1202         mbuf_pool = rte_mempool_create(
1203                         "MBUF_POOL",
1204                         NUM_MBUFS_PER_PORT
1205                         * valid_nb_ports,
1206                         MBUF_SIZE, MBUF_CACHE_SIZE,
1207                         sizeof(struct rte_pktmbuf_pool_private),
1208                         rte_pktmbuf_pool_init, NULL,
1209                         rte_pktmbuf_init, NULL,
1210                         rte_socket_id(), 0);
1211         if (mbuf_pool == NULL)
1212                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
1213
1214         for (queue_id = 0; queue_id < MAX_QUEUES + 1; queue_id++)
1215                 vpool_array[queue_id].pool = mbuf_pool;
1216
1217         /* initialize all ports */
1218         for (portid = 0; portid < nb_ports; portid++) {
1219                 /* skip ports that are not enabled */
1220                 if ((enabled_port_mask & (1 << portid)) == 0) {
1221                         RTE_LOG(INFO, VHOST_PORT,
1222                                 "Skipping disabled port %d\n", portid);
1223                         continue;
1224                 }
1225                 if (overlay_options.port_configure(portid, mbuf_pool) != 0)
1226                         rte_exit(EXIT_FAILURE,
1227                                 "Cannot initialize network ports\n");
1228         }
1229
1230         /* Initialise all linked lists. */
1231         if (init_data_ll() == -1)
1232                 rte_exit(EXIT_FAILURE, "Failed to initialize linked list\n");
1233
1234         /* Initialize device stats */
1235         memset(&dev_statistics, 0, sizeof(dev_statistics));
1236
1237         /* Enable stats if the user option is set. */
1238         if (enable_stats) {
1239                 ret = pthread_create(&tid, NULL, (void *)print_stats, NULL);
1240                 if (ret != 0)
1241                         rte_exit(EXIT_FAILURE, "Cannot create print-stats thread\n");
1242                 snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, "print-stats");
1243                 ret = rte_thread_setname(tid, thread_name);
1244                 if (ret != 0)
1245                         RTE_LOG(DEBUG, VHOST_CONFIG, "Cannot set print-stats name\n");
1246         }
1247
1248         /* Launch all data cores. */
1249         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1250                 rte_eal_remote_launch(switch_worker,
1251                         mbuf_pool, lcore_id);
1252         }
1253         rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_MRG_RXBUF);
1254
1255         ret = rte_vhost_driver_register((char *)&dev_basename, 0);
1256         if (ret != 0)
1257                 rte_exit(EXIT_FAILURE, "failed to register vhost driver.\n");
1258
1259         rte_vhost_driver_callback_register(&virtio_net_device_ops);
1260
1261         rte_vhost_driver_session_start();
1262
1263         return 0;
1264 }