4 * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <arpa/inet.h>
36 #include <linux/if_ether.h>
37 #include <linux/if_vlan.h>
38 #include <linux/virtio_net.h>
39 #include <linux/virtio_ring.h>
42 #include <sys/eventfd.h>
43 #include <sys/param.h>
46 #include <rte_atomic.h>
47 #include <rte_cycles.h>
48 #include <rte_ethdev.h>
50 #include <rte_string_fns.h>
51 #include <rte_pause.h>
54 #include "virtio-net.h"
55 #include "xen_vhost.h"
57 #define MAX_QUEUES 128
59 /* the maximum number of external ports supported */
60 #define MAX_SUP_PORTS 1
63 * Calculate the number of buffers needed per port
65 #define NUM_MBUFS_PER_PORT ((MAX_QUEUES*RTE_TEST_RX_DESC_DEFAULT) + \
66 (num_switching_cores*MAX_PKT_BURST) + \
67 (num_switching_cores*RTE_TEST_TX_DESC_DEFAULT) +\
68 (num_switching_cores*MBUF_CACHE_SIZE))
70 #define MBUF_CACHE_SIZE 64
73 * RX and TX Prefetch, Host, and Write-back threshold values should be
74 * carefully set for optimal performance. Consult the network
75 * controller's datasheet and supporting DPDK documentation for guidance
76 * on how these parameters should be set.
78 #define RX_PTHRESH 8 /* Default values of RX prefetch threshold reg. */
79 #define RX_HTHRESH 8 /* Default values of RX host threshold reg. */
80 #define RX_WTHRESH 4 /* Default values of RX write-back threshold reg. */
83 * These default values are optimized for use with the Intel(R) 82599 10 GbE
84 * Controller and the DPDK ixgbe PMD. Consider using other values for other
85 * network controllers and/or network drivers.
87 #define TX_PTHRESH 36 /* Default values of TX prefetch threshold reg. */
88 #define TX_HTHRESH 0 /* Default values of TX host threshold reg. */
89 #define TX_WTHRESH 0 /* Default values of TX write-back threshold reg. */
91 #define MAX_PKT_BURST 32 /* Max burst size for RX/TX */
92 #define MAX_MRG_PKT_BURST 16 /* Max burst for merge buffers. Set to 1 due to performance issue. */
93 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
95 /* State of virtio device. */
96 #define DEVICE_NOT_READY 0
97 #define DEVICE_READY 1
98 #define DEVICE_SAFE_REMOVE 2
100 /* Config_core_flag status definitions. */
101 #define REQUEST_DEV_REMOVAL 1
102 #define ACK_DEV_REMOVAL 0
104 /* Configurable number of RX/TX ring descriptors */
105 #define RTE_TEST_RX_DESC_DEFAULT 128
106 #define RTE_TEST_TX_DESC_DEFAULT 512
108 #define INVALID_PORT_ID 0xFF
110 /* Max number of devices. Limited by vmdq. */
111 #define MAX_DEVICES 64
113 /* Size of buffers used for snprintfs. */
114 #define MAX_PRINT_BUFF 6072
117 /* Maximum long option length for option parsing. */
118 #define MAX_LONG_OPT_SZ 64
120 /* Used to compare MAC addresses. */
121 #define MAC_ADDR_CMP 0xFFFFFFFFFFFF
123 /* mask of enabled ports */
124 static uint32_t enabled_port_mask = 0;
126 /*Number of switching cores enabled*/
127 static uint32_t num_switching_cores = 0;
129 /* number of devices/queues to support*/
130 static uint32_t num_queues = 0;
131 uint32_t num_devices = 0;
133 /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */
134 static uint32_t enable_vm2vm = 1;
136 static uint32_t enable_stats = 0;
138 /* empty vmdq configuration structure. Filled in programatically */
139 static const struct rte_eth_conf vmdq_conf_default = {
141 .mq_mode = ETH_MQ_RX_VMDQ_ONLY,
143 .header_split = 0, /**< Header Split disabled */
144 .hw_ip_checksum = 0, /**< IP checksum offload disabled */
145 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
147 * It is necessary for 1G NIC such as I350,
148 * this fixes bug of ipv4 forwarding in guest can't
149 * forward pakets from one virtio dev to another virtio dev.
151 .hw_vlan_strip = 1, /**< VLAN strip enabled. */
152 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */
153 .hw_strip_crc = 1, /**< CRC stripped by hardware */
157 .mq_mode = ETH_MQ_TX_NONE,
161 * should be overridden separately in code with
165 .nb_queue_pools = ETH_8_POOLS,
166 .enable_default_pool = 0,
169 .pool_map = {{0, 0},},
174 static unsigned lcore_ids[RTE_MAX_LCORE];
175 static uint8_t ports[RTE_MAX_ETHPORTS];
176 static unsigned num_ports = 0; /**< The number of ports specified in command line */
178 const uint16_t vlan_tags[] = {
179 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007,
180 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015,
181 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023,
182 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031,
183 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039,
184 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047,
185 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055,
186 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063,
189 /* ethernet addresses of ports */
190 static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS];
192 /* heads for the main used and free linked lists for the data path. */
193 static struct virtio_net_data_ll *ll_root_used = NULL;
194 static struct virtio_net_data_ll *ll_root_free = NULL;
196 /* Array of data core structures containing information on individual core linked lists. */
197 static struct lcore_info lcore_info[RTE_MAX_LCORE];
199 /* Used for queueing bursts of TX packets. */
203 struct rte_mbuf *m_table[MAX_PKT_BURST];
206 /* TX queue for each data core. */
207 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE];
209 /* Vlan header struct used to insert vlan tags on TX. */
211 unsigned char h_dest[ETH_ALEN];
212 unsigned char h_source[ETH_ALEN];
215 __be16 h_vlan_encapsulated_proto;
218 /* Header lengths. */
220 #define VLAN_ETH_HLEN 18
222 /* Per-device statistics struct */
223 struct device_statistics {
225 rte_atomic64_t rx_total;
228 } __rte_cache_aligned;
229 struct device_statistics dev_statistics[MAX_DEVICES];
232 * Builds up the correct configuration for VMDQ VLAN pool map
233 * according to the pool & queue limits.
236 get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices)
238 struct rte_eth_vmdq_rx_conf conf;
241 memset(&conf, 0, sizeof(conf));
242 conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices;
243 conf.nb_pool_maps = num_devices;
245 for (i = 0; i < conf.nb_pool_maps; i++) {
246 conf.pool_map[i].vlan_id = vlan_tags[ i ];
247 conf.pool_map[i].pools = (1UL << i);
250 (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf)));
251 (void)(rte_memcpy(ð_conf->rx_adv_conf.vmdq_rx_conf, &conf,
252 sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf)));
257 * Validate the device number according to the max pool number gotten form dev_info
258 * If the device number is invalid, give the error message and return -1.
259 * Each device must have its own pool.
262 validate_num_devices(uint32_t max_nb_devices)
264 if (num_devices > max_nb_devices) {
265 RTE_LOG(ERR, VHOST_PORT, "invalid number of devices\n");
272 * Initialises a given port using global settings and with the rx buffers
273 * coming from the mbuf_pool passed as parameter
276 port_init(uint8_t port, struct rte_mempool *mbuf_pool)
278 struct rte_eth_dev_info dev_info;
279 struct rte_eth_rxconf *rxconf;
280 struct rte_eth_conf port_conf;
281 uint16_t rx_rings, tx_rings = (uint16_t)rte_lcore_count();
282 const uint16_t rx_ring_size = RTE_TEST_RX_DESC_DEFAULT, tx_ring_size = RTE_TEST_TX_DESC_DEFAULT;
286 /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
287 rte_eth_dev_info_get (port, &dev_info);
289 /*configure the number of supported virtio devices based on VMDQ limits */
290 num_devices = dev_info.max_vmdq_pools;
291 num_queues = dev_info.max_rx_queues;
293 retval = validate_num_devices(MAX_DEVICES);
297 /* Get port configuration. */
298 retval = get_eth_conf(&port_conf, num_devices);
302 if (port >= rte_eth_dev_count()) return -1;
304 rx_rings = (uint16_t)num_queues,
305 /* Configure ethernet device. */
306 retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
310 rte_eth_dev_info_get(port, &dev_info);
311 rxconf = &dev_info.default_rxconf;
312 rxconf->rx_drop_en = 1;
313 /* Setup the queues. */
314 for (q = 0; q < rx_rings; q ++) {
315 retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
316 rte_eth_dev_socket_id(port), rxconf,
321 for (q = 0; q < tx_rings; q ++) {
322 retval = rte_eth_tx_queue_setup(port, q, tx_ring_size,
323 rte_eth_dev_socket_id(port),
329 /* Start the device. */
330 retval = rte_eth_dev_start(port);
334 rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]);
335 RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices);
336 RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
337 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
339 vmdq_ports_eth_addr[port].addr_bytes[0],
340 vmdq_ports_eth_addr[port].addr_bytes[1],
341 vmdq_ports_eth_addr[port].addr_bytes[2],
342 vmdq_ports_eth_addr[port].addr_bytes[3],
343 vmdq_ports_eth_addr[port].addr_bytes[4],
344 vmdq_ports_eth_addr[port].addr_bytes[5]);
350 * Parse the portmask provided at run time.
353 parse_portmask(const char *portmask)
360 /* parse hexadecimal string */
361 pm = strtoul(portmask, &end, 16);
362 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
373 * Parse num options at run time.
376 parse_num_opt(const char *q_arg, uint32_t max_valid_value)
383 /* parse unsigned int string */
384 num = strtoul(q_arg, &end, 10);
385 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0))
388 if (num > max_valid_value)
399 us_vhost_usage(const char *prgname)
401 RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK --vm2vm [0|1] --stats [0-N] --nb-devices ND\n"
402 " -p PORTMASK: Set mask for ports to be used by application\n"
403 " --vm2vm [0|1]: disable/enable(default) vm2vm comms\n"
404 " --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n",
409 * Parse the arguments given in the command line of the application.
412 us_vhost_parse_args(int argc, char **argv)
417 const char *prgname = argv[0];
418 static struct option long_option[] = {
419 {"vm2vm", required_argument, NULL, 0},
420 {"stats", required_argument, NULL, 0},
424 /* Parse command line */
425 while ((opt = getopt_long(argc, argv, "p:",long_option, &option_index)) != EOF) {
429 enabled_port_mask = parse_portmask(optarg);
430 if (enabled_port_mask == 0) {
431 RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n");
432 us_vhost_usage(prgname);
438 /* Enable/disable vm2vm comms. */
439 if (!strncmp(long_option[option_index].name, "vm2vm", MAX_LONG_OPT_SZ)) {
440 ret = parse_num_opt(optarg, 1);
442 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for vm2vm [0|1]\n");
443 us_vhost_usage(prgname);
450 /* Enable/disable stats. */
451 if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) {
452 ret = parse_num_opt(optarg, INT32_MAX);
454 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for stats [0..N]\n");
455 us_vhost_usage(prgname);
463 /* Invalid option - print options. */
465 us_vhost_usage(prgname);
470 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
471 if (enabled_port_mask & (1 << i))
472 ports[num_ports++] = (uint8_t)i;
475 if ((num_ports == 0) || (num_ports > MAX_SUP_PORTS)) {
476 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
477 "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
485 * Update the global var NUM_PORTS and array PORTS according to system ports number
486 * and return valid ports number
488 static unsigned check_ports_num(unsigned nb_ports)
490 unsigned valid_num_ports = num_ports;
493 if (num_ports > nb_ports) {
494 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n",
495 num_ports, nb_ports);
496 num_ports = nb_ports;
499 for (portid = 0; portid < num_ports; portid ++) {
500 if (ports[portid] >= nb_ports) {
501 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port ID(%u) exceeds max system port ID(%u)\n",
502 ports[portid], (nb_ports - 1));
503 ports[portid] = INVALID_PORT_ID;
507 return valid_num_ports;
511 * Function to convert guest physical addresses to vhost virtual addresses. This
512 * is used to convert virtio buffer addresses.
514 static __rte_always_inline uint64_t
515 gpa_to_vva(struct virtio_net *dev, uint64_t guest_pa)
517 struct virtio_memory_regions *region;
519 uint64_t vhost_va = 0;
521 for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
522 region = &dev->mem->regions[regionidx];
523 if ((guest_pa >= region->guest_phys_address) &&
524 (guest_pa <= region->guest_phys_address_end)) {
525 vhost_va = region->address_offset + guest_pa;
529 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") GPA %p| VVA %p\n",
530 dev->device_fh, (void*)(uintptr_t)guest_pa, (void*)(uintptr_t)vhost_va);
536 * This function adds buffers to the virtio devices RX virtqueue. Buffers can
537 * be received from the physical port or from another virtio device. A packet
538 * count is returned to indicate the number of packets that were successfully
539 * added to the RX queue.
541 static __rte_always_inline uint32_t
542 virtio_dev_rx(struct virtio_net *dev, struct rte_mbuf **pkts, uint32_t count)
544 struct vhost_virtqueue *vq;
545 struct vring_desc *desc;
546 struct rte_mbuf *buff;
547 /* The virtio_hdr is initialised to 0. */
548 struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0,0,0,0,0,0},0};
549 uint64_t buff_addr = 0;
550 uint64_t buff_hdr_addr = 0;
551 uint32_t head[MAX_PKT_BURST], packet_len = 0;
552 uint32_t head_idx, packet_success = 0;
553 uint16_t avail_idx, res_cur_idx;
554 uint16_t res_base_idx, res_end_idx;
555 uint16_t free_entries;
559 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") virtio_dev_rx()\n", dev->device_fh);
560 vq = dev->virtqueue_rx;
561 count = (count > MAX_PKT_BURST) ? MAX_PKT_BURST : count;
562 /* As many data cores may want access to available buffers, they need to be reserved. */
565 res_base_idx = vq->last_used_idx_res;
567 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
569 free_entries = (avail_idx - res_base_idx);
571 /*check that we have enough buffers*/
572 if (unlikely(count > free_entries))
573 count = free_entries;
578 res_end_idx = res_base_idx + count;
579 /* vq->last_used_idx_res is atomically updated. */
580 success = rte_atomic16_cmpset(&vq->last_used_idx_res, res_base_idx,
582 } while (unlikely(success == 0));
583 res_cur_idx = res_base_idx;
584 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") Current Index %d| End Index %d\n",
585 dev->device_fh, res_cur_idx, res_end_idx);
587 /* Prefetch available ring to retrieve indexes. */
588 rte_prefetch0(&vq->avail->ring[res_cur_idx & (vq->size - 1)]);
590 /* Retrieve all of the head indexes first to avoid caching issues. */
591 for (head_idx = 0; head_idx < count; head_idx++)
592 head[head_idx] = vq->avail->ring[(res_cur_idx + head_idx) & (vq->size - 1)];
594 /*Prefetch descriptor index. */
595 rte_prefetch0(&vq->desc[head[packet_success]]);
597 while (res_cur_idx != res_end_idx) {
598 /* Get descriptor from available ring */
599 desc = &vq->desc[head[packet_success]];
600 /* Prefetch descriptor address. */
603 buff = pkts[packet_success];
605 /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
606 buff_addr = gpa_to_vva(dev, desc->addr);
607 /* Prefetch buffer address. */
608 rte_prefetch0((void*)(uintptr_t)buff_addr);
611 /* Copy virtio_hdr to packet and increment buffer address */
612 buff_hdr_addr = buff_addr;
613 packet_len = rte_pktmbuf_data_len(buff) + vq->vhost_hlen;
616 * If the descriptors are chained the header and data are placed in
619 if (desc->flags & VRING_DESC_F_NEXT) {
620 desc->len = vq->vhost_hlen;
621 desc = &vq->desc[desc->next];
622 /* Buffer address translation. */
623 buff_addr = gpa_to_vva(dev, desc->addr);
624 desc->len = rte_pktmbuf_data_len(buff);
626 buff_addr += vq->vhost_hlen;
627 desc->len = packet_len;
631 /* Update used ring with desc information */
632 vq->used->ring[res_cur_idx & (vq->size - 1)].id = head[packet_success];
633 vq->used->ring[res_cur_idx & (vq->size - 1)].len = packet_len;
635 /* Copy mbuf data to buffer */
636 userdata = rte_pktmbuf_mtod(buff, void *);
637 rte_memcpy((void *)(uintptr_t)buff_addr, userdata, rte_pktmbuf_data_len(buff));
642 /* mergeable is disabled then a header is required per buffer. */
643 rte_memcpy((void *)(uintptr_t)buff_hdr_addr, (const void *)&virtio_hdr, vq->vhost_hlen);
644 if (res_cur_idx < res_end_idx) {
645 /* Prefetch descriptor index. */
646 rte_prefetch0(&vq->desc[head[packet_success]]);
650 rte_compiler_barrier();
652 /* Wait until it's our turn to add our buffer to the used ring. */
653 while (unlikely(vq->last_used_idx != res_base_idx))
656 *(volatile uint16_t *)&vq->used->idx += count;
658 vq->last_used_idx = res_end_idx;
664 * Compares a packet destination MAC address to a device MAC address.
666 static __rte_always_inline int
667 ether_addr_cmp(struct ether_addr *ea, struct ether_addr *eb)
669 return ((*(uint64_t *)ea ^ *(uint64_t *)eb) & MAC_ADDR_CMP) == 0;
673 * This function registers mac along with a
674 * vlan tag to a VMDQ.
677 link_vmdq(struct virtio_net *dev)
680 struct virtio_net_data_ll *dev_ll;
682 dev_ll = ll_root_used;
684 while (dev_ll != NULL) {
685 if ((dev != dev_ll->dev) && ether_addr_cmp(&dev->mac_address, &dev_ll->dev->mac_address)) {
686 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") WARNING: This device is using an existing MAC address and has not been registered.\n", dev->device_fh);
689 dev_ll = dev_ll->next;
692 /* vlan_tag currently uses the device_id. */
693 dev->vlan_tag = vlan_tags[dev->device_fh];
694 dev->vmdq_rx_q = dev->device_fh * (num_queues/num_devices);
696 /* Print out VMDQ registration info. */
697 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") MAC_ADDRESS %02x:%02x:%02x:%02x:%02x:%02x and VLAN_TAG %d registered\n",
699 dev->mac_address.addr_bytes[0], dev->mac_address.addr_bytes[1],
700 dev->mac_address.addr_bytes[2], dev->mac_address.addr_bytes[3],
701 dev->mac_address.addr_bytes[4], dev->mac_address.addr_bytes[5],
704 /* Register the MAC address. */
705 ret = rte_eth_dev_mac_addr_add(ports[0], &dev->mac_address, (uint32_t)dev->device_fh);
707 RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Failed to add device MAC address to VMDQ\n",
712 /* Enable stripping of the vlan tag as we handle routing. */
713 rte_eth_dev_set_vlan_strip_on_queue(ports[0], dev->vmdq_rx_q, 1);
715 rte_compiler_barrier();
716 /* Set device as ready for RX. */
717 dev->ready = DEVICE_READY;
723 * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX
724 * queue before disabling RX on the device.
727 unlink_vmdq(struct virtio_net *dev)
731 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
733 if (dev->ready == DEVICE_READY) {
734 /*clear MAC and VLAN settings*/
735 rte_eth_dev_mac_addr_remove(ports[0], &dev->mac_address);
736 for (i = 0; i < 6; i++)
737 dev->mac_address.addr_bytes[i] = 0;
741 /*Clear out the receive buffers*/
742 rx_count = rte_eth_rx_burst(ports[0],
743 (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
746 for (i = 0; i < rx_count; i++)
747 rte_pktmbuf_free(pkts_burst[i]);
749 rx_count = rte_eth_rx_burst(ports[0],
750 (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
753 dev->ready = DEVICE_NOT_READY;
758 * Check if the packet destination MAC address is for a local device. If so then put
759 * the packet on that devices RX queue. If not then return.
761 static __rte_always_inline unsigned
762 virtio_tx_local(struct virtio_net *dev, struct rte_mbuf *m)
764 struct virtio_net_data_ll *dev_ll;
765 struct ether_hdr *pkt_hdr;
768 pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
770 /*get the used devices list*/
771 dev_ll = ll_root_used;
773 while (dev_ll != NULL) {
774 if (likely(dev_ll->dev->ready == DEVICE_READY) && ether_addr_cmp(&(pkt_hdr->d_addr),
775 &dev_ll->dev->mac_address)) {
777 /* Drop the packet if the TX packet is destined for the TX device. */
778 if (dev_ll->dev->device_fh == dev->device_fh) {
779 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") TX: "
780 "Source and destination MAC addresses are the same. "
781 "Dropping packet.\n",
782 dev_ll->dev->device_fh);
787 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") TX: "
788 "MAC address is local\n", dev_ll->dev->device_fh);
790 if (dev_ll->dev->remove) {
791 /*drop the packet if the device is marked for removal*/
792 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") "
793 "Device is marked for removal\n",
794 dev_ll->dev->device_fh);
796 /*send the packet to the local virtio device*/
797 ret = virtio_dev_rx(dev_ll->dev, &m, 1);
799 rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx_total, 1);
800 rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx, ret);
801 dev_statistics[dev->device_fh].tx_total++;
802 dev_statistics[dev->device_fh].tx += ret;
808 dev_ll = dev_ll->next;
815 * This function routes the TX packet to the correct interface. This may be a local device
816 * or the physical port.
818 static __rte_always_inline void
819 virtio_tx_route(struct virtio_net* dev, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, uint16_t vlan_tag)
821 struct mbuf_table *tx_q;
822 struct vlan_ethhdr *vlan_hdr;
823 struct rte_mbuf **m_table;
824 struct rte_mbuf *mbuf;
826 const uint16_t lcore_id = rte_lcore_id();
828 /*check if destination is local VM*/
829 if (enable_vm2vm && (virtio_tx_local(dev, m) == 0)) {
833 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") TX: "
834 "MAC address is external\n", dev->device_fh);
836 /*Add packet to the port tx queue*/
837 tx_q = &lcore_tx_queue[lcore_id];
840 /* Allocate an mbuf and populate the structure. */
841 mbuf = rte_pktmbuf_alloc(mbuf_pool);
845 mbuf->data_len = m->data_len + VLAN_HLEN;
846 mbuf->pkt_len = mbuf->data_len;
848 /* Copy ethernet header to mbuf. */
849 rte_memcpy(rte_pktmbuf_mtod(mbuf, void*),
850 rte_pktmbuf_mtod(m, const void*), ETH_HLEN);
853 /* Setup vlan header. Bytes need to be re-ordered for network with htons()*/
854 vlan_hdr = rte_pktmbuf_mtod(mbuf, struct vlan_ethhdr *);
855 vlan_hdr->h_vlan_encapsulated_proto = vlan_hdr->h_vlan_proto;
856 vlan_hdr->h_vlan_proto = htons(ETH_P_8021Q);
857 vlan_hdr->h_vlan_TCI = htons(vlan_tag);
859 /* Copy the remaining packet contents to the mbuf. */
860 rte_memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, VLAN_ETH_HLEN),
861 rte_pktmbuf_mtod_offset(m, const void *, ETH_HLEN),
862 (m->data_len - ETH_HLEN));
863 tx_q->m_table[len] = mbuf;
866 dev_statistics[dev->device_fh].tx_total++;
867 dev_statistics[dev->device_fh].tx++;
870 if (unlikely(len == MAX_PKT_BURST)) {
871 m_table = (struct rte_mbuf **)tx_q->m_table;
872 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id, m_table, (uint16_t) len);
873 /* Free any buffers not handled by TX and update the port stats. */
874 if (unlikely(ret < len)) {
876 rte_pktmbuf_free(m_table[ret]);
877 } while (++ret < len);
887 static __rte_always_inline void
888 virtio_dev_tx(struct virtio_net* dev, struct rte_mempool *mbuf_pool)
891 struct vhost_virtqueue *vq;
892 struct vring_desc *desc;
893 uint64_t buff_addr = 0;
894 uint32_t head[MAX_PKT_BURST];
897 uint16_t free_entries, packet_success = 0;
900 vq = dev->virtqueue_tx;
901 avail_idx = *((volatile uint16_t *)&vq->avail->idx);
903 /* If there are no available buffers then return. */
904 if (vq->last_used_idx == avail_idx)
907 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") virtio_dev_tx()\n",
910 /* Prefetch available ring to retrieve head indexes. */
911 rte_prefetch0(&vq->avail->ring[vq->last_used_idx & (vq->size - 1)]);
913 /*get the number of free entries in the ring*/
914 free_entries = avail_idx - vq->last_used_idx;
915 free_entries = unlikely(free_entries < MAX_PKT_BURST) ? free_entries : MAX_PKT_BURST;
917 RTE_LOG_DP(DEBUG, VHOST_DATA, "(%" PRIu64 ") Buffers available %d\n",
918 dev->device_fh, free_entries);
919 /* Retrieve all of the head indexes first to avoid caching issues. */
920 for (i = 0; i < free_entries; i++)
921 head[i] = vq->avail->ring[(vq->last_used_idx + i) & (vq->size - 1)];
923 /* Prefetch descriptor index. */
924 rte_prefetch0(&vq->desc[head[packet_success]]);
926 while (packet_success < free_entries) {
927 desc = &vq->desc[head[packet_success]];
928 /* Prefetch descriptor address. */
931 if (packet_success < (free_entries - 1)) {
932 /* Prefetch descriptor index. */
933 rte_prefetch0(&vq->desc[head[packet_success+1]]);
936 /* Update used index buffer information. */
937 used_idx = vq->last_used_idx & (vq->size - 1);
938 vq->used->ring[used_idx].id = head[packet_success];
939 vq->used->ring[used_idx].len = 0;
941 /* Discard first buffer as it is the virtio header */
942 desc = &vq->desc[desc->next];
944 /* Buffer address translation. */
945 buff_addr = gpa_to_vva(dev, desc->addr);
946 /* Prefetch buffer address. */
947 rte_prefetch0((void*)(uintptr_t)buff_addr);
949 /* Setup dummy mbuf. This is copied to a real mbuf if transmitted out the physical port. */
950 m.data_len = desc->len;
954 virtio_tx_route(dev, &m, mbuf_pool, 0);
960 rte_compiler_barrier();
961 vq->used->idx += packet_success;
962 /* Kick guest if required. */
966 * This function is called by each data core. It handles all RX/TX registered with the
967 * core. For TX the specific lcore linked list is used. For RX, MAC addresses are compared
968 * with all devices in the main linked list.
971 switch_worker(__attribute__((unused)) void *arg)
973 struct rte_mempool *mbuf_pool = arg;
974 struct virtio_net *dev = NULL;
975 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
976 struct virtio_net_data_ll *dev_ll;
977 struct mbuf_table *tx_q;
978 volatile struct lcore_ll_info *lcore_ll;
979 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US;
980 uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0;
982 const uint16_t lcore_id = rte_lcore_id();
983 const uint16_t num_cores = (uint16_t)rte_lcore_count();
984 uint16_t rx_count = 0;
986 RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started \n", lcore_id);
987 lcore_ll = lcore_info[lcore_id].lcore_ll;
990 tx_q = &lcore_tx_queue[lcore_id];
991 for (i = 0; i < num_cores; i ++) {
992 if (lcore_ids[i] == lcore_id) {
999 cur_tsc = rte_rdtsc();
1001 * TX burst queue drain
1003 diff_tsc = cur_tsc - prev_tsc;
1004 if (unlikely(diff_tsc > drain_tsc)) {
1007 RTE_LOG_DP(DEBUG, VHOST_DATA,
1008 "TX queue drained after timeout with burst size %u\n",
1011 /*Tx any packets in the queue*/
1012 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id,
1013 (struct rte_mbuf **)tx_q->m_table,
1014 (uint16_t)tx_q->len);
1015 if (unlikely(ret < tx_q->len)) {
1017 rte_pktmbuf_free(tx_q->m_table[ret]);
1018 } while (++ret < tx_q->len);
1029 * Inform the configuration core that we have exited the linked list and that no devices are
1030 * in use if requested.
1032 if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL)
1033 lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
1038 dev_ll = lcore_ll->ll_root_used;
1040 while (dev_ll != NULL) {
1041 /*get virtio device ID*/
1044 if (unlikely(dev->remove)) {
1045 dev_ll = dev_ll->next;
1047 dev->ready = DEVICE_SAFE_REMOVE;
1050 if (likely(dev->ready == DEVICE_READY)) {
1052 rx_count = rte_eth_rx_burst(ports[0],
1053 (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
1056 ret_count = virtio_dev_rx(dev, pkts_burst, rx_count);
1058 rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx_total, rx_count);
1059 rte_atomic64_add(&dev_statistics[dev_ll->dev->device_fh].rx, ret_count);
1061 while (likely(rx_count)) {
1063 rte_pktmbuf_free_seg(pkts_burst[rx_count]);
1069 if (likely(!dev->remove))
1071 virtio_dev_tx(dev, mbuf_pool);
1073 /*move to the next device in the list*/
1074 dev_ll = dev_ll->next;
1082 * Add an entry to a used linked list. A free entry must first be found in the free linked list
1083 * using get_data_ll_free_entry();
1086 add_data_ll_entry(struct virtio_net_data_ll **ll_root_addr, struct virtio_net_data_ll *ll_dev)
1088 struct virtio_net_data_ll *ll = *ll_root_addr;
1090 /* Set next as NULL and use a compiler barrier to avoid reordering. */
1091 ll_dev->next = NULL;
1092 rte_compiler_barrier();
1094 /* If ll == NULL then this is the first device. */
1096 /* Increment to the tail of the linked list. */
1097 while ((ll->next != NULL) )
1102 *ll_root_addr = ll_dev;
1107 * Remove an entry from a used linked list. The entry must then be added to the free linked list
1108 * using put_data_ll_free_entry().
1111 rm_data_ll_entry(struct virtio_net_data_ll **ll_root_addr, struct virtio_net_data_ll *ll_dev, struct virtio_net_data_ll *ll_dev_last)
1113 struct virtio_net_data_ll *ll = *ll_root_addr;
1116 *ll_root_addr = ll_dev->next;
1118 ll_dev_last->next = ll_dev->next;
1122 * Find and return an entry from the free linked list.
1124 static struct virtio_net_data_ll *
1125 get_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr)
1127 struct virtio_net_data_ll *ll_free = *ll_root_addr;
1128 struct virtio_net_data_ll *ll_dev;
1130 if (ll_free == NULL)
1134 *ll_root_addr = ll_free->next;
1140 * Place an entry back on to the free linked list.
1143 put_data_ll_free_entry(struct virtio_net_data_ll **ll_root_addr, struct virtio_net_data_ll *ll_dev)
1145 struct virtio_net_data_ll *ll_free = *ll_root_addr;
1147 ll_dev->next = ll_free;
1148 *ll_root_addr = ll_dev;
1152 * Creates a linked list of a given size.
1154 static struct virtio_net_data_ll *
1155 alloc_data_ll(uint32_t size)
1157 struct virtio_net_data_ll *ll_new;
1160 /* Malloc and then chain the linked list. */
1161 ll_new = malloc(size * sizeof(struct virtio_net_data_ll));
1162 if (ll_new == NULL) {
1163 RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for ll_new.\n");
1167 for (i = 0; i < size - 1; i++) {
1168 ll_new[i].dev = NULL;
1169 ll_new[i].next = &ll_new[i+1];
1171 ll_new[i].next = NULL;
1177 * Create the main linked list along with each individual cores linked list. A used and a free list
1178 * are created to manage entries.
1185 RTE_LCORE_FOREACH_SLAVE(lcore) {
1186 lcore_info[lcore].lcore_ll = malloc(sizeof(struct lcore_ll_info));
1187 if (lcore_info[lcore].lcore_ll == NULL) {
1188 RTE_LOG(ERR, VHOST_CONFIG, "Failed to allocate memory for lcore_ll.\n");
1192 lcore_info[lcore].lcore_ll->device_num = 0;
1193 lcore_info[lcore].lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
1194 lcore_info[lcore].lcore_ll->ll_root_used = NULL;
1195 if (num_devices % num_switching_cores)
1196 lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll((num_devices / num_switching_cores) + 1);
1198 lcore_info[lcore].lcore_ll->ll_root_free = alloc_data_ll(num_devices / num_switching_cores);
1201 /* Allocate devices up to a maximum of MAX_DEVICES. */
1202 ll_root_free = alloc_data_ll(MIN((num_devices), MAX_DEVICES));
1207 * Remove a device from the specific data core linked list and from the main linked list. The
1208 * rx/tx thread must be set the flag to indicate that it is safe to remove the device.
1212 destroy_device (volatile struct virtio_net *dev)
1214 struct virtio_net_data_ll *ll_lcore_dev_cur;
1215 struct virtio_net_data_ll *ll_main_dev_cur;
1216 struct virtio_net_data_ll *ll_lcore_dev_last = NULL;
1217 struct virtio_net_data_ll *ll_main_dev_last = NULL;
1220 dev->flags &= ~VIRTIO_DEV_RUNNING;
1222 /*set the remove flag. */
1225 while(dev->ready != DEVICE_SAFE_REMOVE) {
1229 /* Search for entry to be removed from lcore ll */
1230 ll_lcore_dev_cur = lcore_info[dev->coreid].lcore_ll->ll_root_used;
1231 while (ll_lcore_dev_cur != NULL) {
1232 if (ll_lcore_dev_cur->dev == dev) {
1235 ll_lcore_dev_last = ll_lcore_dev_cur;
1236 ll_lcore_dev_cur = ll_lcore_dev_cur->next;
1240 /* Search for entry to be removed from main ll */
1241 ll_main_dev_cur = ll_root_used;
1242 ll_main_dev_last = NULL;
1243 while (ll_main_dev_cur != NULL) {
1244 if (ll_main_dev_cur->dev == dev) {
1247 ll_main_dev_last = ll_main_dev_cur;
1248 ll_main_dev_cur = ll_main_dev_cur->next;
1252 if (ll_lcore_dev_cur == NULL || ll_main_dev_cur == NULL) {
1253 RTE_LOG(ERR, XENHOST, "%s: could find device in per_cpu list or main_list\n", __func__);
1257 /* Remove entries from the lcore and main ll. */
1258 rm_data_ll_entry(&lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
1259 rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
1261 /* Set the dev_removal_flag on each lcore. */
1262 RTE_LCORE_FOREACH_SLAVE(lcore) {
1263 lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;
1267 * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that
1268 * they can no longer access the device removed from the linked lists and that the devices
1269 * are no longer in use.
1271 RTE_LCORE_FOREACH_SLAVE(lcore) {
1272 while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL) {
1277 /* Add the entries back to the lcore and main free ll.*/
1278 put_data_ll_free_entry(&lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);
1279 put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
1281 /* Decrement number of device on the lcore. */
1282 lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->device_num--;
1284 RTE_LOG(INFO, VHOST_DATA, " #####(%"PRIu64") Device has been removed from data core\n", dev->device_fh);
1288 * A new device is added to a data core. First the device is added to the main linked list
1289 * and the allocated to a specific data core.
1292 new_device (struct virtio_net *dev)
1294 struct virtio_net_data_ll *ll_dev;
1295 int lcore, core_add = 0;
1296 uint32_t device_num_min = num_devices;
1298 /* Add device to main ll */
1299 ll_dev = get_data_ll_free_entry(&ll_root_free);
1300 if (ll_dev == NULL) {
1301 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") No free entry found in linked list. Device limit "
1302 "of %d devices per core has been reached\n",
1303 dev->device_fh, num_devices);
1307 add_data_ll_entry(&ll_root_used, ll_dev);
1309 /*reset ready flag*/
1310 dev->ready = DEVICE_NOT_READY;
1313 /* Find a suitable lcore to add the device. */
1314 RTE_LCORE_FOREACH_SLAVE(lcore) {
1315 if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
1316 device_num_min = lcore_info[lcore].lcore_ll->device_num;
1320 /* Add device to lcore ll */
1321 ll_dev->dev->coreid = core_add;
1322 ll_dev = get_data_ll_free_entry(&lcore_info[ll_dev->dev->coreid].lcore_ll->ll_root_free);
1323 if (ll_dev == NULL) {
1324 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
1325 destroy_device(dev);
1329 add_data_ll_entry(&lcore_info[ll_dev->dev->coreid].lcore_ll->ll_root_used, ll_dev);
1331 /* Initialize device stats */
1332 memset(&dev_statistics[dev->device_fh], 0, sizeof(struct device_statistics));
1334 lcore_info[ll_dev->dev->coreid].lcore_ll->device_num++;
1335 dev->flags |= VIRTIO_DEV_RUNNING;
1337 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, dev->coreid);
1345 * These callback allow devices to be added to the data core when configuration
1346 * has been fully complete.
1348 static const struct virtio_net_device_ops virtio_net_device_ops =
1350 .new_device = new_device,
1351 .destroy_device = destroy_device,
1355 * This is a thread will wake up after a period to print stats if the user has
1361 struct virtio_net_data_ll *dev_ll;
1362 uint64_t tx_dropped, rx_dropped;
1363 uint64_t tx, tx_total, rx, rx_total;
1365 const char clr[] = { 27, '[', '2', 'J', '\0' };
1366 const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' };
1369 sleep(enable_stats);
1371 /* Clear screen and move to top left */
1372 printf("%s%s", clr, top_left);
1374 printf("\nDevice statistics ====================================");
1376 dev_ll = ll_root_used;
1377 while (dev_ll != NULL) {
1378 device_fh = (uint32_t)dev_ll->dev->device_fh;
1379 tx_total = dev_statistics[device_fh].tx_total;
1380 tx = dev_statistics[device_fh].tx;
1381 tx_dropped = tx_total - tx;
1382 rx_total = rte_atomic64_read(&dev_statistics[device_fh].rx_total);
1383 rx = rte_atomic64_read(&dev_statistics[device_fh].rx);
1384 rx_dropped = rx_total - rx;
1386 printf("\nStatistics for device %"PRIu32" ------------------------------"
1387 "\nTX total: %"PRIu64""
1388 "\nTX dropped: %"PRIu64""
1389 "\nTX successful: %"PRIu64""
1390 "\nRX total: %"PRIu64""
1391 "\nRX dropped: %"PRIu64""
1392 "\nRX successful: %"PRIu64"",
1401 dev_ll = dev_ll->next;
1403 printf("\n======================================================\n");
1408 int init_virtio_net(struct virtio_net_device_ops const * const ops);
1411 * Main function, does initialisation and calls the per-lcore functions.
1414 main(int argc, char *argv[])
1416 struct rte_mempool *mbuf_pool;
1417 unsigned lcore_id, core_id = 0;
1418 unsigned nb_ports, valid_num_ports;
1421 static pthread_t tid;
1422 char thread_name[RTE_MAX_THREAD_NAME_LEN];
1425 ret = rte_eal_init(argc, argv);
1427 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
1431 /* parse app arguments */
1432 ret = us_vhost_parse_args(argc, argv);
1434 rte_exit(EXIT_FAILURE, "Invalid argument\n");
1436 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++)
1437 if (rte_lcore_is_enabled(lcore_id))
1438 lcore_ids[core_id ++] = lcore_id;
1440 if (rte_lcore_count() > RTE_MAX_LCORE)
1441 rte_exit(EXIT_FAILURE,"Not enough cores\n");
1443 /*set the number of swithcing cores available*/
1444 num_switching_cores = rte_lcore_count()-1;
1446 /* Get the number of physical ports. */
1447 nb_ports = rte_eth_dev_count();
1450 * Update the global var NUM_PORTS and global array PORTS
1451 * and get value of var VALID_NUM_PORTS according to system ports number
1453 valid_num_ports = check_ports_num(nb_ports);
1455 if ((valid_num_ports == 0) || (valid_num_ports > MAX_SUP_PORTS)) {
1456 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u,"
1457 "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS);
1461 /* Create the mbuf pool. */
1462 mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL",
1463 NUM_MBUFS_PER_PORT * valid_num_ports, MBUF_CACHE_SIZE, 0,
1464 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1465 if (mbuf_pool == NULL)
1466 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
1468 /* initialize all ports */
1469 for (portid = 0; portid < nb_ports; portid++) {
1470 /* skip ports that are not enabled */
1471 if ((enabled_port_mask & (1 << portid)) == 0) {
1472 RTE_LOG(INFO, VHOST_PORT, "Skipping disabled port %d\n", portid);
1475 if (port_init(portid, mbuf_pool) != 0)
1476 rte_exit(EXIT_FAILURE, "Cannot initialize network ports\n");
1479 /* Initialise all linked lists. */
1480 if (init_data_ll() == -1)
1481 rte_exit(EXIT_FAILURE, "Failed to initialize linked list\n");
1483 /* Initialize device stats */
1484 memset(&dev_statistics, 0, sizeof(dev_statistics));
1486 /* Enable stats if the user option is set. */
1488 ret = pthread_create(&tid, NULL, (void *)print_stats, NULL);
1490 rte_exit(EXIT_FAILURE,
1491 "Cannot create print-stats thread\n");
1493 /* Set thread_name for aid in debugging. */
1494 snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, "print-xen-stats");
1495 ret = rte_thread_setname(tid, thread_name);
1497 RTE_LOG(DEBUG, VHOST_CONFIG,
1498 "Cannot set print-stats name\n");
1501 /* Launch all data cores. */
1502 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1503 rte_eal_remote_launch(switch_worker, mbuf_pool, lcore_id);
1506 init_virtio_xen(&virtio_net_device_ops);
1508 virtio_monitor_loop();