1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
11 #include <sys/queue.h>
16 #include <rte_common.h>
17 #include <rte_byteorder.h>
19 #include <rte_memory.h>
20 #include <rte_memcpy.h>
22 #include <rte_launch.h>
23 #include <rte_atomic.h>
24 #include <rte_cycles.h>
25 #include <rte_prefetch.h>
26 #include <rte_lcore.h>
27 #include <rte_per_lcore.h>
28 #include <rte_branch_prediction.h>
29 #include <rte_interrupts.h>
30 #include <rte_random.h>
31 #include <rte_debug.h>
32 #include <rte_ether.h>
33 #include <rte_ethdev.h>
35 #include <rte_mempool.h>
37 #include <rte_string_fns.h>
44 static struct rte_eth_conf port_conf = {
46 .mq_mode = ETH_MQ_RX_RSS,
48 .offloads = (DEV_RX_OFFLOAD_CHECKSUM |
49 DEV_RX_OFFLOAD_CRC_STRIP),
58 .mq_mode = ETH_MQ_TX_NONE,
63 app_assign_worker_ids(void)
65 uint32_t lcore, worker_id;
67 /* Assign ID for each worker */
69 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
70 struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore].worker;
72 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
76 lp_worker->worker_id = worker_id;
82 app_init_mbuf_pools(void)
84 unsigned socket, lcore;
86 /* Init the buffer pools */
87 for (socket = 0; socket < APP_MAX_SOCKETS; socket ++) {
89 if (app_is_socket_used(socket) == 0) {
93 snprintf(name, sizeof(name), "mbuf_pool_%u", socket);
94 printf("Creating the mbuf pool for socket %u ...\n", socket);
95 app.pools[socket] = rte_pktmbuf_pool_create(
96 name, APP_DEFAULT_MEMPOOL_BUFFERS,
97 APP_DEFAULT_MEMPOOL_CACHE_SIZE,
98 0, APP_DEFAULT_MBUF_DATA_SIZE, socket);
99 if (app.pools[socket] == NULL) {
100 rte_panic("Cannot create mbuf pool on socket %u\n", socket);
104 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
105 if (app.lcore_params[lcore].type == e_APP_LCORE_DISABLED) {
109 socket = rte_lcore_to_socket_id(lcore);
110 app.lcore_params[lcore].pool = app.pools[socket];
115 app_init_lpm_tables(void)
117 unsigned socket, lcore;
119 /* Init the LPM tables */
120 for (socket = 0; socket < APP_MAX_SOCKETS; socket ++) {
124 if (app_is_socket_used(socket) == 0) {
128 struct rte_lpm_config lpm_config;
130 lpm_config.max_rules = APP_MAX_LPM_RULES;
131 lpm_config.number_tbl8s = 256;
132 lpm_config.flags = 0;
133 snprintf(name, sizeof(name), "lpm_table_%u", socket);
134 printf("Creating the LPM table for socket %u ...\n", socket);
135 app.lpm_tables[socket] = rte_lpm_create(
139 if (app.lpm_tables[socket] == NULL) {
140 rte_panic("Unable to create LPM table on socket %u\n", socket);
143 for (rule = 0; rule < app.n_lpm_rules; rule ++) {
146 ret = rte_lpm_add(app.lpm_tables[socket],
147 app.lpm_rules[rule].ip,
148 app.lpm_rules[rule].depth,
149 app.lpm_rules[rule].if_out);
152 rte_panic("Unable to add entry %u (%x/%u => %u) to the LPM table on socket %u (%d)\n",
154 (unsigned) app.lpm_rules[rule].ip,
155 (unsigned) app.lpm_rules[rule].depth,
156 (unsigned) app.lpm_rules[rule].if_out,
164 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
165 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
169 socket = rte_lcore_to_socket_id(lcore);
170 app.lcore_params[lcore].worker.lpm_table = app.lpm_tables[socket];
175 app_init_rings_rx(void)
179 /* Initialize the rings for the RX side */
180 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
181 struct app_lcore_params_io *lp_io = &app.lcore_params[lcore].io;
182 unsigned socket_io, lcore_worker;
184 if ((app.lcore_params[lcore].type != e_APP_LCORE_IO) ||
185 (lp_io->rx.n_nic_queues == 0)) {
189 socket_io = rte_lcore_to_socket_id(lcore);
191 for (lcore_worker = 0; lcore_worker < APP_MAX_LCORES; lcore_worker ++) {
193 struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore_worker].worker;
194 struct rte_ring *ring = NULL;
196 if (app.lcore_params[lcore_worker].type != e_APP_LCORE_WORKER) {
200 printf("Creating ring to connect I/O lcore %u (socket %u) with worker lcore %u ...\n",
204 snprintf(name, sizeof(name), "app_ring_rx_s%u_io%u_w%u",
208 ring = rte_ring_create(
212 RING_F_SP_ENQ | RING_F_SC_DEQ);
214 rte_panic("Cannot create ring to connect I/O core %u with worker core %u\n",
219 lp_io->rx.rings[lp_io->rx.n_rings] = ring;
220 lp_io->rx.n_rings ++;
222 lp_worker->rings_in[lp_worker->n_rings_in] = ring;
223 lp_worker->n_rings_in ++;
227 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
228 struct app_lcore_params_io *lp_io = &app.lcore_params[lcore].io;
230 if ((app.lcore_params[lcore].type != e_APP_LCORE_IO) ||
231 (lp_io->rx.n_nic_queues == 0)) {
235 if (lp_io->rx.n_rings != app_get_lcores_worker()) {
236 rte_panic("Algorithmic error (I/O RX rings)\n");
240 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
241 struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore].worker;
243 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
247 if (lp_worker->n_rings_in != app_get_lcores_io_rx()) {
248 rte_panic("Algorithmic error (worker input rings)\n");
254 app_init_rings_tx(void)
258 /* Initialize the rings for the TX side */
259 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
260 struct app_lcore_params_worker *lp_worker = &app.lcore_params[lcore].worker;
263 if (app.lcore_params[lcore].type != e_APP_LCORE_WORKER) {
267 for (port = 0; port < APP_MAX_NIC_PORTS; port ++) {
269 struct app_lcore_params_io *lp_io = NULL;
270 struct rte_ring *ring;
271 uint32_t socket_io, lcore_io;
273 if (app.nic_tx_port_mask[port] == 0) {
277 if (app_get_lcore_for_nic_tx(port, &lcore_io) < 0) {
278 rte_panic("Algorithmic error (no I/O core to handle TX of port %u)\n",
282 lp_io = &app.lcore_params[lcore_io].io;
283 socket_io = rte_lcore_to_socket_id(lcore_io);
285 printf("Creating ring to connect worker lcore %u with TX port %u (through I/O lcore %u) (socket %u) ...\n",
286 lcore, port, (unsigned)lcore_io, (unsigned)socket_io);
287 snprintf(name, sizeof(name), "app_ring_tx_s%u_w%u_p%u", socket_io, lcore, port);
288 ring = rte_ring_create(
292 RING_F_SP_ENQ | RING_F_SC_DEQ);
294 rte_panic("Cannot create ring to connect worker core %u with TX port %u\n",
299 lp_worker->rings_out[port] = ring;
300 lp_io->tx.rings[port][lp_worker->worker_id] = ring;
304 for (lcore = 0; lcore < APP_MAX_LCORES; lcore ++) {
305 struct app_lcore_params_io *lp_io = &app.lcore_params[lcore].io;
308 if ((app.lcore_params[lcore].type != e_APP_LCORE_IO) ||
309 (lp_io->tx.n_nic_ports == 0)) {
313 for (i = 0; i < lp_io->tx.n_nic_ports; i ++){
316 port = lp_io->tx.nic_ports[i];
317 for (j = 0; j < app_get_lcores_worker(); j ++) {
318 if (lp_io->tx.rings[port][j] == NULL) {
319 rte_panic("Algorithmic error (I/O TX rings)\n");
326 /* Check the link status of all ports in up to 9s, and print them finally */
328 check_all_ports_link_status(uint16_t port_num, uint32_t port_mask)
330 #define CHECK_INTERVAL 100 /* 100ms */
331 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
333 uint8_t count, all_ports_up, print_flag = 0;
334 struct rte_eth_link link;
335 uint32_t n_rx_queues, n_tx_queues;
337 printf("\nChecking link status");
339 for (count = 0; count <= MAX_CHECK_TIME; count++) {
341 for (portid = 0; portid < port_num; portid++) {
342 if ((port_mask & (1 << portid)) == 0)
344 n_rx_queues = app_get_nic_rx_queues_per_port(portid);
345 n_tx_queues = app.nic_tx_port_mask[portid];
346 if ((n_rx_queues == 0) && (n_tx_queues == 0))
348 memset(&link, 0, sizeof(link));
349 rte_eth_link_get_nowait(portid, &link);
350 /* print link status if flag set */
351 if (print_flag == 1) {
352 if (link.link_status)
354 "Port%d Link Up - speed %uMbps - %s\n",
355 portid, link.link_speed,
356 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
357 ("full-duplex") : ("half-duplex\n"));
359 printf("Port %d Link Down\n", portid);
362 /* clear all_ports_up flag if any link down */
363 if (link.link_status == ETH_LINK_DOWN) {
368 /* after finally printing all link status, get out */
372 if (all_ports_up == 0) {
375 rte_delay_ms(CHECK_INTERVAL);
378 /* set the print_flag if all ports up or timeout */
379 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
394 uint32_t n_rx_queues, n_tx_queues;
396 /* Init NIC ports and queues, then start the ports */
397 for (port = 0; port < APP_MAX_NIC_PORTS; port ++) {
398 struct rte_mempool *pool;
399 uint16_t nic_rx_ring_size;
400 uint16_t nic_tx_ring_size;
401 struct rte_eth_rxconf rxq_conf;
402 struct rte_eth_txconf txq_conf;
403 struct rte_eth_dev_info dev_info;
404 struct rte_eth_conf local_port_conf = port_conf;
406 n_rx_queues = app_get_nic_rx_queues_per_port(port);
407 n_tx_queues = app.nic_tx_port_mask[port];
409 if ((n_rx_queues == 0) && (n_tx_queues == 0)) {
414 printf("Initializing NIC port %u ...\n", port);
415 rte_eth_dev_info_get(port, &dev_info);
416 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
417 local_port_conf.txmode.offloads |=
418 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
420 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
421 dev_info.flow_type_rss_offloads;
422 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
423 port_conf.rx_adv_conf.rss_conf.rss_hf) {
424 printf("Port %u modified RSS hash function based on hardware support,"
425 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
427 port_conf.rx_adv_conf.rss_conf.rss_hf,
428 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
431 ret = rte_eth_dev_configure(
433 (uint8_t) n_rx_queues,
434 (uint8_t) n_tx_queues,
437 rte_panic("Cannot init NIC port %u (%d)\n", port, ret);
439 rte_eth_promiscuous_enable(port);
441 nic_rx_ring_size = app.nic_rx_ring_size;
442 nic_tx_ring_size = app.nic_tx_ring_size;
443 ret = rte_eth_dev_adjust_nb_rx_tx_desc(
444 port, &nic_rx_ring_size, &nic_tx_ring_size);
446 rte_panic("Cannot adjust number of descriptors for port %u (%d)\n",
449 app.nic_rx_ring_size = nic_rx_ring_size;
450 app.nic_tx_ring_size = nic_tx_ring_size;
452 rxq_conf = dev_info.default_rxconf;
453 rxq_conf.offloads = local_port_conf.rxmode.offloads;
455 for (queue = 0; queue < APP_MAX_RX_QUEUES_PER_NIC_PORT; queue ++) {
456 if (app.nic_rx_queue_mask[port][queue] == 0) {
460 app_get_lcore_for_nic_rx(port, queue, &lcore);
461 socket = rte_lcore_to_socket_id(lcore);
462 pool = app.lcore_params[lcore].pool;
464 printf("Initializing NIC port %u RX queue %u ...\n",
466 ret = rte_eth_rx_queue_setup(
469 (uint16_t) app.nic_rx_ring_size,
474 rte_panic("Cannot init RX queue %u for port %u (%d)\n",
479 txq_conf = dev_info.default_txconf;
480 txq_conf.offloads = local_port_conf.txmode.offloads;
482 if (app.nic_tx_port_mask[port] == 1) {
483 app_get_lcore_for_nic_tx(port, &lcore);
484 socket = rte_lcore_to_socket_id(lcore);
485 printf("Initializing NIC port %u TX queue 0 ...\n",
487 ret = rte_eth_tx_queue_setup(
490 (uint16_t) app.nic_tx_ring_size,
494 rte_panic("Cannot init TX queue 0 for port %d (%d)\n",
501 ret = rte_eth_dev_start(port);
503 rte_panic("Cannot start port %d (%d)\n", port, ret);
507 check_all_ports_link_status(APP_MAX_NIC_PORTS, (~0x0));
513 app_assign_worker_ids();
514 app_init_mbuf_pools();
515 app_init_lpm_tables();
520 printf("Initialization completed.\n");