test/crypto: skip validation of head/tailroom used by PMD
[dpdk.git] / examples / multi_process / l2fwd_fork / main.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2016 Intel Corporation
3  */
4 #define _GNU_SOURCE
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <unistd.h>
8 #include <string.h>
9 #include <stdint.h>
10 #include <sched.h>
11 #include <inttypes.h>
12 #include <sys/types.h>
13 #include <sys/queue.h>
14 #include <netinet/in.h>
15 #include <setjmp.h>
16 #include <stdarg.h>
17 #include <ctype.h>
18 #include <errno.h>
19 #include <getopt.h>
20
21 #include <rte_common.h>
22 #include <rte_log.h>
23 #include <rte_memory.h>
24 #include <rte_memcpy.h>
25 #include <rte_eal.h>
26 #include <rte_launch.h>
27 #include <rte_atomic.h>
28 #include <rte_spinlock.h>
29 #include <rte_cycles.h>
30 #include <rte_prefetch.h>
31 #include <rte_lcore.h>
32 #include <rte_per_lcore.h>
33 #include <rte_branch_prediction.h>
34 #include <rte_interrupts.h>
35 #include <rte_random.h>
36 #include <rte_debug.h>
37 #include <rte_ether.h>
38 #include <rte_ethdev.h>
39 #include <rte_ring.h>
40 #include <rte_mempool.h>
41 #include <rte_mbuf.h>
42 #include <rte_malloc.h>
43
44 #include "flib.h"
45
46 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
47 #define MBUF_NAME       "mbuf_pool_%d"
48 #define MBUF_DATA_SIZE  RTE_MBUF_DEFAULT_BUF_SIZE
49 #define NB_MBUF   8192
50 #define RING_MASTER_NAME        "l2fwd_ring_m2s_"
51 #define RING_SLAVE_NAME         "l2fwd_ring_s2m_"
52 #define MAX_NAME_LEN    32
53 /* RECREATE flag indicate needs initialize resource and launch slave_core again */
54 #define SLAVE_RECREATE_FLAG 0x1
55 /* RESTART flag indicate needs restart port and send START command again */
56 #define SLAVE_RESTART_FLAG 0x2
57 #define INVALID_MAPPING_ID      ((unsigned)LCORE_ID_ANY)
58 /* Maximum message buffer per slave */
59 #define NB_CORE_MSGBUF  32
60 enum l2fwd_cmd{
61         CMD_START,
62         CMD_STOP,
63 };
64
65 #define MAX_PKT_BURST 32
66 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
67
68 /*
69  * Configurable number of RX/TX ring descriptors
70  */
71 #define RTE_TEST_RX_DESC_DEFAULT 1024
72 #define RTE_TEST_TX_DESC_DEFAULT 1024
73 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
74 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
75
76 /* ethernet addresses of ports */
77 static struct ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
78
79 /* mask of enabled ports */
80 static uint32_t l2fwd_enabled_port_mask = 0;
81
82 /* list of enabled ports */
83 static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS];
84
85 static unsigned int l2fwd_rx_queue_per_lcore = 1;
86
87 struct mbuf_table {
88         unsigned len;
89         struct rte_mbuf *m_table[MAX_PKT_BURST];
90 };
91
92 #define MAX_RX_QUEUE_PER_LCORE 16
93 #define MAX_TX_QUEUE_PER_PORT 16
94 struct lcore_queue_conf {
95         unsigned n_rx_port;
96         unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE];
97 } __rte_cache_aligned;
98 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
99
100 struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
101
102 struct lcore_resource_struct {
103         int enabled;    /* Only set in case this lcore involved into packet forwarding */
104         int flags;          /* Set only slave need to restart or recreate */
105         unsigned lcore_id;  /*  lcore ID */
106         unsigned pair_id;       /* dependency lcore ID on port */
107         char ring_name[2][MAX_NAME_LEN];
108         /* ring[0] for master send cmd, slave read */
109         /* ring[1] for slave send ack, master read */
110         struct rte_ring *ring[2];
111         int port_num;                                   /* Total port numbers */
112         /* Port id for that lcore to receive packets */
113         uint16_t port[RTE_MAX_ETHPORTS];
114 }__attribute__((packed)) __rte_cache_aligned;
115
116 static struct lcore_resource_struct lcore_resource[RTE_MAX_LCORE];
117 static struct rte_mempool *message_pool;
118 static rte_spinlock_t res_lock = RTE_SPINLOCK_INITIALIZER;
119 /* use floating processes */
120 static int float_proc = 0;
121 /* Save original cpu affinity */
122 struct cpu_aff_arg{
123         cpu_set_t set;
124         size_t size;
125 }cpu_aff;
126
127 static const struct rte_eth_conf port_conf = {
128         .rxmode = {
129                 .split_hdr_size = 0,
130                 .offloads = DEV_RX_OFFLOAD_CRC_STRIP,
131         },
132         .txmode = {
133                 .mq_mode = ETH_MQ_TX_NONE,
134         },
135 };
136
137 static struct rte_mempool * l2fwd_pktmbuf_pool[RTE_MAX_ETHPORTS];
138
139 /* Per-port statistics struct */
140 struct l2fwd_port_statistics {
141         uint64_t tx;
142         uint64_t rx;
143         uint64_t dropped;
144 } __rte_cache_aligned;
145 struct l2fwd_port_statistics *port_statistics;
146 /**
147  * pointer to lcore ID mapping array, used to return lcore id in case slave
148  * process exited unexpectedly, use only floating process option applied
149  **/
150 unsigned *mapping_id;
151
152 /* A tsc-based timer responsible for triggering statistics printout */
153 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
154 #define MAX_TIMER_PERIOD 86400 /* 1 day max */
155 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; /* default period is 10 seconds */
156
157 static int l2fwd_launch_one_lcore(void *dummy);
158
159 /* Print out statistics on packets dropped */
160 static void
161 print_stats(void)
162 {
163         uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
164         unsigned portid;
165
166         total_packets_dropped = 0;
167         total_packets_tx = 0;
168         total_packets_rx = 0;
169
170         const char clr[] = { 27, '[', '2', 'J', '\0' };
171         const char topLeft[] = { 27, '[', '1', ';', '1', 'H','\0' };
172
173                 /* Clear screen and move to top left */
174         printf("%s%s", clr, topLeft);
175
176         printf("\nPort statistics ====================================");
177
178         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
179                 /* skip disabled ports */
180                 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
181                         continue;
182                 printf("\nStatistics for port %u ------------------------------"
183                            "\nPackets sent: %24"PRIu64
184                            "\nPackets received: %20"PRIu64
185                            "\nPackets dropped: %21"PRIu64,
186                            portid,
187                            port_statistics[portid].tx,
188                            port_statistics[portid].rx,
189                            port_statistics[portid].dropped);
190
191                 total_packets_dropped += port_statistics[portid].dropped;
192                 total_packets_tx += port_statistics[portid].tx;
193                 total_packets_rx += port_statistics[portid].rx;
194         }
195         printf("\nAggregate statistics ==============================="
196                    "\nTotal packets sent: %18"PRIu64
197                    "\nTotal packets received: %14"PRIu64
198                    "\nTotal packets dropped: %15"PRIu64,
199                    total_packets_tx,
200                    total_packets_rx,
201                    total_packets_dropped);
202         printf("\n====================================================\n");
203 }
204
205 static int
206 clear_cpu_affinity(void)
207 {
208         int s;
209
210         s = sched_setaffinity(0, cpu_aff.size, &cpu_aff.set);
211         if (s != 0) {
212                 printf("sched_setaffinity failed:%s\n", strerror(errno));
213                 return -1;
214         }
215
216         return 0;
217 }
218
219 static int
220 get_cpu_affinity(void)
221 {
222         int s;
223
224         cpu_aff.size = sizeof(cpu_set_t);
225         CPU_ZERO(&cpu_aff.set);
226
227         s = sched_getaffinity(0, cpu_aff.size, &cpu_aff.set);
228         if (s != 0) {
229                 printf("sched_getaffinity failed:%s\n", strerror(errno));
230                 return -1;
231         }
232
233         return 0;
234 }
235
236 /**
237  * This fnciton demonstrates the approach to create ring in first instance
238  * or re-attach an existed ring in later instance.
239  **/
240 static struct rte_ring *
241 create_ring(const char *name, unsigned count,
242                                         int socket_id,unsigned flags)
243 {
244         struct rte_ring *ring;
245
246         if (name == NULL)
247                 return NULL;
248
249         /* If already create, just attached it */
250         if (likely((ring = rte_ring_lookup(name)) != NULL))
251                 return ring;
252
253         /* First call it, create one */
254         return rte_ring_create(name, count, socket_id, flags);
255 }
256
257 /* Malloc with rte_malloc on structures that shared by master and slave */
258 static int
259 l2fwd_malloc_shared_struct(void)
260 {
261         port_statistics = rte_zmalloc("port_stat",
262                                                 sizeof(struct l2fwd_port_statistics) * RTE_MAX_ETHPORTS,
263                                                 0);
264         if (port_statistics == NULL)
265                 return -1;
266
267         /* allocate  mapping_id array */
268         if (float_proc) {
269                 int i;
270                 mapping_id = rte_malloc("mapping_id", sizeof(unsigned) * RTE_MAX_LCORE,
271                                                                 0);
272
273                 if (mapping_id == NULL)
274                         return -1;
275
276                 for (i = 0 ;i < RTE_MAX_LCORE; i++)
277                         mapping_id[i] = INVALID_MAPPING_ID;
278         }
279         return 0;
280 }
281
282 /* Create ring which used for communicate among master and slave */
283 static int
284 create_ms_ring(unsigned slaveid)
285 {
286         unsigned flag = RING_F_SP_ENQ | RING_F_SC_DEQ;
287         struct lcore_resource_struct *res = &lcore_resource[slaveid];
288         unsigned socketid = rte_socket_id();
289
290         /* Always assume create ring on master socket_id */
291         /* Default only create a ring size 32 */
292         snprintf(res->ring_name[0], MAX_NAME_LEN, "%s%u",
293                         RING_MASTER_NAME, slaveid);
294         if ((res->ring[0] = create_ring(res->ring_name[0], NB_CORE_MSGBUF,
295                                 socketid, flag)) == NULL) {
296                 printf("Create m2s ring %s failed\n", res->ring_name[0]);
297                 return -1;
298         }
299
300         snprintf(res->ring_name[1], MAX_NAME_LEN, "%s%u",
301                         RING_SLAVE_NAME, slaveid);
302         if ((res->ring[1] = create_ring(res->ring_name[1], NB_CORE_MSGBUF,
303                 socketid, flag)) == NULL) {
304                 printf("Create s2m ring %s failed\n", res->ring_name[1]);
305                 return -1;
306         }
307
308         return 0;
309 }
310
311 /* send command to pair in paired master and slave ring */
312 static inline int
313 sendcmd(unsigned slaveid, enum l2fwd_cmd cmd, int is_master)
314 {
315         struct lcore_resource_struct *res = &lcore_resource[slaveid];
316         void *msg;
317         int fd = !is_master;
318
319         /* Only check master, it must be enabled and running if it is slave */
320         if (is_master && !res->enabled)
321                 return -1;
322
323         if (res->ring[fd] == NULL)
324                 return -1;
325
326         if (rte_mempool_get(message_pool, &msg) < 0) {
327                 printf("Error to get message buffer\n");
328                 return -1;
329         }
330
331         *(enum l2fwd_cmd *)msg = cmd;
332
333         if (rte_ring_enqueue(res->ring[fd], msg) != 0) {
334                 printf("Enqueue error\n");
335                 rte_mempool_put(message_pool, msg);
336                 return -1;
337         }
338
339         return 0;
340 }
341
342 /* Get command from pair in paired master and slave ring */
343 static inline int
344 getcmd(unsigned slaveid, enum l2fwd_cmd *cmd, int is_master)
345 {
346         struct lcore_resource_struct *res = &lcore_resource[slaveid];
347         void *msg;
348         int fd = !!is_master;
349         int ret;
350         /* Only check master, it must be enabled and running if it is slave */
351         if (is_master && (!res->enabled))
352                 return -1;
353
354         if (res->ring[fd] == NULL)
355                 return -1;
356
357         ret = rte_ring_dequeue(res->ring[fd], &msg);
358
359         if (ret == 0) {
360                 *cmd = *(enum l2fwd_cmd *)msg;
361                 rte_mempool_put(message_pool, msg);
362         }
363         return ret;
364 }
365
366 /* Master send command to slave and wait until ack received or error met */
367 static int
368 master_sendcmd_with_ack(unsigned slaveid, enum l2fwd_cmd cmd)
369 {
370         enum l2fwd_cmd ack_cmd;
371         int ret = -1;
372
373         if (sendcmd(slaveid, cmd, 1) != 0)
374                 rte_exit(EXIT_FAILURE, "Failed to send message\n");
375
376         /* Get ack */
377         while (1) {
378                 ret = getcmd(slaveid, &ack_cmd, 1);
379                 if (ret == 0 && cmd == ack_cmd)
380                         break;
381
382                 /* If slave not running yet, return an error */
383                 if (flib_query_slave_status(slaveid) != ST_RUN) {
384                         ret = -ENOENT;
385                         break;
386                 }
387         }
388
389         return ret;
390 }
391
392 /* restart all port that assigned to that slave lcore */
393 static int
394 reset_slave_all_ports(unsigned slaveid)
395 {
396         struct lcore_resource_struct *slave = &lcore_resource[slaveid];
397         int i, ret = 0;
398
399         /* stop/start port */
400         for (i = 0; i < slave->port_num; i++) {
401                 char buf_name[RTE_MEMPOOL_NAMESIZE];
402                 struct rte_mempool *pool;
403                 printf("Stop port :%d\n", slave->port[i]);
404                 rte_eth_dev_stop(slave->port[i]);
405                 snprintf(buf_name, RTE_MEMPOOL_NAMESIZE, MBUF_NAME, slave->port[i]);
406                 pool = rte_mempool_lookup(buf_name);
407                 if (pool)
408                         printf("Port %d mempool free object is %u(%u)\n", slave->port[i],
409                                 rte_mempool_avail_count(pool),
410                                 (unsigned int)NB_MBUF);
411                 else
412                         printf("Can't find mempool %s\n", buf_name);
413
414                 printf("Start port :%d\n", slave->port[i]);
415                 ret = rte_eth_dev_start(slave->port[i]);
416                 if (ret != 0)
417                         break;
418         }
419         return ret;
420 }
421
422 static int
423 reset_shared_structures(unsigned slaveid)
424 {
425         int ret;
426         /* Only port are shared resource here */
427         ret = reset_slave_all_ports(slaveid);
428
429         return ret;
430 }
431
432 /**
433  * Call this function to re-create resource that needed for slave process that
434  * exited in last instance
435  **/
436 static int
437 init_slave_res(unsigned slaveid)
438 {
439         struct lcore_resource_struct *slave = &lcore_resource[slaveid];
440         enum l2fwd_cmd cmd;
441
442         if (!slave->enabled) {
443                 printf("Something wrong with lcore=%u enabled=%d\n",slaveid,
444                         slave->enabled);
445                 return -1;
446         }
447
448         /* Initialize ring */
449         if (create_ms_ring(slaveid) != 0)
450                 rte_exit(EXIT_FAILURE, "failed to create ring for slave %u\n",
451                                 slaveid);
452
453         /* drain un-read buffer if have */
454         while (getcmd(slaveid, &cmd, 1) == 0);
455         while (getcmd(slaveid, &cmd, 0) == 0);
456
457         return 0;
458 }
459
460 static int
461 recreate_one_slave(unsigned slaveid)
462 {
463         int ret = 0;
464         /* Re-initialize resource for stalled slave */
465         if ((ret = init_slave_res(slaveid)) != 0) {
466                 printf("Init slave=%u failed\n", slaveid);
467                 return ret;
468         }
469
470         if ((ret = flib_remote_launch(l2fwd_launch_one_lcore, NULL, slaveid))
471                 != 0)
472                 printf("Launch slave %u failed\n", slaveid);
473
474         return ret;
475 }
476
477 /**
478  * remapping resource belong to slave_id to new lcore that gets from flib_assign_lcore_id(),
479  * used only floating process option applied.
480  *
481  * @param slaveid
482  *   original lcore_id that apply for remapping
483  */
484 static void
485 remapping_slave_resource(unsigned slaveid, unsigned map_id)
486 {
487
488         /* remapping lcore_resource */
489         memcpy(&lcore_resource[map_id], &lcore_resource[slaveid],
490                         sizeof(struct lcore_resource_struct));
491
492         /* remapping lcore_queue_conf */
493         memcpy(&lcore_queue_conf[map_id], &lcore_queue_conf[slaveid],
494                         sizeof(struct lcore_queue_conf));
495 }
496
497 static int
498 reset_pair(unsigned slaveid, unsigned pairid)
499 {
500         int ret;
501         if ((ret = reset_shared_structures(slaveid)) != 0)
502                 goto back;
503
504         if((ret = reset_shared_structures(pairid)) != 0)
505                 goto back;
506
507         if (float_proc) {
508                 unsigned map_id = mapping_id[slaveid];
509
510                 if (map_id != INVALID_MAPPING_ID) {
511                         printf("%u return mapping id %u\n", slaveid, map_id);
512                         flib_free_lcore_id(map_id);
513                         mapping_id[slaveid] = INVALID_MAPPING_ID;
514                 }
515
516                 map_id = mapping_id[pairid];
517                 if (map_id != INVALID_MAPPING_ID) {
518                         printf("%u return mapping id %u\n", pairid, map_id);
519                         flib_free_lcore_id(map_id);
520                         mapping_id[pairid] = INVALID_MAPPING_ID;
521                 }
522         }
523
524         if((ret = recreate_one_slave(slaveid)) != 0)
525                 goto back;
526
527         ret = recreate_one_slave(pairid);
528
529 back:
530         return ret;
531 }
532
533 static void
534 slave_exit_cb(unsigned slaveid, __attribute__((unused))int stat)
535 {
536         struct lcore_resource_struct *slave = &lcore_resource[slaveid];
537
538         printf("Get slave %u leave info\n", slaveid);
539         if (!slave->enabled) {
540                 printf("Lcore=%u not registered for it's exit\n", slaveid);
541                 return;
542         }
543         rte_spinlock_lock(&res_lock);
544
545         /* Change the state and wait master to start them */
546         slave->flags = SLAVE_RECREATE_FLAG;
547
548         rte_spinlock_unlock(&res_lock);
549 }
550
551 static void
552 l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid)
553 {
554         struct ether_hdr *eth;
555         void *tmp;
556         unsigned dst_port;
557         int sent;
558         struct rte_eth_dev_tx_buffer *buffer;
559
560         dst_port = l2fwd_dst_ports[portid];
561         eth = rte_pktmbuf_mtod(m, struct ether_hdr *);
562
563         /* 02:00:00:00:00:xx */
564         tmp = &eth->d_addr.addr_bytes[0];
565         *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40);
566
567         /* src addr */
568         ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], &eth->s_addr);
569
570         buffer = tx_buffer[dst_port];
571         sent = rte_eth_tx_buffer(dst_port, 0, buffer, m);
572         if (sent)
573                 port_statistics[dst_port].tx += sent;
574 }
575
576 /* main processing loop */
577 static void
578 l2fwd_main_loop(void)
579 {
580         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
581         struct rte_mbuf *m;
582         int sent;
583         unsigned lcore_id;
584         uint64_t prev_tsc, diff_tsc, cur_tsc;
585         unsigned i, j, portid, nb_rx;
586         struct lcore_queue_conf *qconf;
587         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S *
588                         BURST_TX_DRAIN_US;
589         struct rte_eth_dev_tx_buffer *buffer;
590
591         prev_tsc = 0;
592
593         lcore_id = rte_lcore_id();
594
595         qconf = &lcore_queue_conf[lcore_id];
596
597         if (qconf->n_rx_port == 0) {
598                 RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id);
599                 return;
600         }
601
602         RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id);
603
604         for (i = 0; i < qconf->n_rx_port; i++) {
605                 portid = qconf->rx_port_list[i];
606                 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id,
607                         portid);
608         }
609
610         while (1) {
611                 enum l2fwd_cmd cmd;
612                 cur_tsc = rte_rdtsc();
613
614                 if (unlikely(getcmd(lcore_id, &cmd, 0) == 0)) {
615                         sendcmd(lcore_id, cmd, 0);
616
617                         /* If get stop command, stop forwarding and exit */
618                         if (cmd == CMD_STOP) {
619                                 return;
620                         }
621                 }
622
623                 /*
624                  * TX burst queue drain
625                  */
626                 diff_tsc = cur_tsc - prev_tsc;
627                 if (unlikely(diff_tsc > drain_tsc)) {
628
629                         for (i = 0; i < qconf->n_rx_port; i++) {
630
631                                 portid = l2fwd_dst_ports[qconf->rx_port_list[i]];
632                                 buffer = tx_buffer[portid];
633
634                                 sent = rte_eth_tx_buffer_flush(portid, 0, buffer);
635                                 if (sent)
636                                         port_statistics[portid].tx += sent;
637
638                         }
639
640                         prev_tsc = cur_tsc;
641                 }
642
643                 /*
644                  * Read packet from RX queues
645                  */
646                 for (i = 0; i < qconf->n_rx_port; i++) {
647
648                         portid = qconf->rx_port_list[i];
649                         nb_rx = rte_eth_rx_burst((uint8_t) portid, 0,
650                                                  pkts_burst, MAX_PKT_BURST);
651
652                         port_statistics[portid].rx += nb_rx;
653
654                         for (j = 0; j < nb_rx; j++) {
655                                 m = pkts_burst[j];
656                                 rte_prefetch0(rte_pktmbuf_mtod(m, void *));
657                                 l2fwd_simple_forward(m, portid);
658                         }
659                 }
660         }
661 }
662
663 static int
664 l2fwd_launch_one_lcore(__attribute__((unused)) void *dummy)
665 {
666         unsigned lcore_id = rte_lcore_id();
667
668         if (float_proc) {
669                 unsigned flcore_id;
670
671                 /* Change it to floating process, also change it's lcore_id */
672                 clear_cpu_affinity();
673                 RTE_PER_LCORE(_lcore_id) = 0;
674                 /* Get a lcore_id */
675                 if (flib_assign_lcore_id() < 0 ) {
676                         printf("flib_assign_lcore_id failed\n");
677                         return -1;
678                 }
679                 flcore_id = rte_lcore_id();
680                 /* Set mapping id, so master can return it after slave exited */
681                 mapping_id[lcore_id] = flcore_id;
682                 printf("Org lcore_id = %u, cur lcore_id = %u\n",
683                                 lcore_id, flcore_id);
684                 remapping_slave_resource(lcore_id, flcore_id);
685         }
686
687         l2fwd_main_loop();
688
689         /* return lcore_id before return */
690         if (float_proc) {
691                 flib_free_lcore_id(rte_lcore_id());
692                 mapping_id[lcore_id] = INVALID_MAPPING_ID;
693         }
694         return 0;
695 }
696
697 /* display usage */
698 static void
699 l2fwd_usage(const char *prgname)
700 {
701         printf("%s [EAL options] -- -p PORTMASK -s COREMASK [-q NQ] -f\n"
702                "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
703                "  -q NQ: number of queue (=ports) per lcore (default is 1)\n"
704                "  -f use floating process which won't bind to any core to run\n"
705                    "  -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n",
706                prgname);
707 }
708
709 static int
710 l2fwd_parse_portmask(const char *portmask)
711 {
712         char *end = NULL;
713         unsigned long pm;
714
715         /* parse hexadecimal string */
716         pm = strtoul(portmask, &end, 16);
717         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
718                 return -1;
719
720         if (pm == 0)
721                 return -1;
722
723         return pm;
724 }
725
726 static unsigned int
727 l2fwd_parse_nqueue(const char *q_arg)
728 {
729         char *end = NULL;
730         unsigned long n;
731
732         /* parse hexadecimal string */
733         n = strtoul(q_arg, &end, 10);
734         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
735                 return 0;
736         if (n == 0)
737                 return 0;
738         if (n >= MAX_RX_QUEUE_PER_LCORE)
739                 return 0;
740
741         return n;
742 }
743
744 static int
745 l2fwd_parse_timer_period(const char *q_arg)
746 {
747         char *end = NULL;
748         int n;
749
750         /* parse number string */
751         n = strtol(q_arg, &end, 10);
752         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
753                 return -1;
754         if (n >= MAX_TIMER_PERIOD)
755                 return -1;
756
757         return n;
758 }
759
760 /* Parse the argument given in the command line of the application */
761 static int
762 l2fwd_parse_args(int argc, char **argv)
763 {
764         int opt, ret;
765         char **argvopt;
766         int option_index;
767         char *prgname = argv[0];
768         static struct option lgopts[] = {
769                 {NULL, 0, 0, 0}
770         };
771         int has_pmask = 0;
772
773         argvopt = argv;
774
775         while ((opt = getopt_long(argc, argvopt, "p:q:T:f",
776                                   lgopts, &option_index)) != EOF) {
777
778                 switch (opt) {
779                 /* portmask */
780                 case 'p':
781                         l2fwd_enabled_port_mask = l2fwd_parse_portmask(optarg);
782                         if (l2fwd_enabled_port_mask == 0) {
783                                 printf("invalid portmask\n");
784                                 l2fwd_usage(prgname);
785                                 return -1;
786                         }
787                         has_pmask = 1;
788                         break;
789
790                 /* nqueue */
791                 case 'q':
792                         l2fwd_rx_queue_per_lcore = l2fwd_parse_nqueue(optarg);
793                         if (l2fwd_rx_queue_per_lcore == 0) {
794                                 printf("invalid queue number\n");
795                                 l2fwd_usage(prgname);
796                                 return -1;
797                         }
798                         break;
799
800                 /* timer period */
801                 case 'T':
802                         timer_period = l2fwd_parse_timer_period(optarg) * 1000 * TIMER_MILLISECOND;
803                         if (timer_period < 0) {
804                                 printf("invalid timer period\n");
805                                 l2fwd_usage(prgname);
806                                 return -1;
807                         }
808                         break;
809
810                 /* use floating process */
811                 case 'f':
812                         float_proc = 1;
813                         break;
814
815                 /* long options */
816                 case 0:
817                         l2fwd_usage(prgname);
818                         return -1;
819
820                 default:
821                         l2fwd_usage(prgname);
822                         return -1;
823                 }
824         }
825
826         if (optind >= 0)
827                 argv[optind-1] = prgname;
828
829         if (!has_pmask) {
830                 l2fwd_usage(prgname);
831                 return -1;
832         }
833         ret = optind-1;
834         optind = 1; /* reset getopt lib */
835         return ret;
836 }
837
838 /* Check the link status of all ports in up to 9s, and print them finally */
839 static void
840 check_all_ports_link_status(uint32_t port_mask)
841 {
842 #define CHECK_INTERVAL 100 /* 100ms */
843 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
844         uint16_t portid;
845         uint8_t count, all_ports_up, print_flag = 0;
846         struct rte_eth_link link;
847
848         printf("\nChecking link status");
849         fflush(stdout);
850         for (count = 0; count <= MAX_CHECK_TIME; count++) {
851                 all_ports_up = 1;
852                 RTE_ETH_FOREACH_DEV(portid) {
853                         if ((port_mask & (1 << portid)) == 0)
854                                 continue;
855                         memset(&link, 0, sizeof(link));
856                         rte_eth_link_get_nowait(portid, &link);
857                         /* print link status if flag set */
858                         if (print_flag == 1) {
859                                 if (link.link_status)
860                                         printf(
861                                         "Port%d Link Up- speed %u Mbps- %s\n",
862                                         portid, link.link_speed,
863                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
864                                         ("full-duplex") : ("half-duplex\n"));
865                                 else
866                                         printf("Port %d Link Down\n", portid);
867                                 continue;
868                         }
869                         /* clear all_ports_up flag if any link down */
870                         if (link.link_status == ETH_LINK_DOWN) {
871                                 all_ports_up = 0;
872                                 break;
873                         }
874                 }
875                 /* after finally printing all link status, get out */
876                 if (print_flag == 1)
877                         break;
878
879                 if (all_ports_up == 0) {
880                         printf(".");
881                         fflush(stdout);
882                         rte_delay_ms(CHECK_INTERVAL);
883                 }
884
885                 /* set the print_flag if all ports up or timeout */
886                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
887                         print_flag = 1;
888                         printf("done\n");
889                 }
890         }
891 }
892
893 int
894 main(int argc, char **argv)
895 {
896         struct lcore_queue_conf *qconf;
897         int ret;
898         uint16_t nb_ports;
899         uint16_t nb_ports_available = 0;
900         uint16_t portid, last_port;
901         unsigned rx_lcore_id;
902         unsigned nb_ports_in_mask = 0;
903         unsigned i;
904         uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc;
905
906         /* Save cpu_affinity first, restore it in case it's floating process option */
907         if (get_cpu_affinity() != 0)
908                 rte_exit(EXIT_FAILURE, "get_cpu_affinity error\n");
909
910         /* Also tries to set cpu affinity to detect whether  it will fail in child process */
911         if(clear_cpu_affinity() != 0)
912                 rte_exit(EXIT_FAILURE, "clear_cpu_affinity error\n");
913
914         /* init EAL */
915         ret = rte_eal_init(argc, argv);
916         if (ret < 0)
917                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
918         argc -= ret;
919         argv += ret;
920
921         /* parse application arguments (after the EAL ones) */
922         ret = l2fwd_parse_args(argc, argv);
923         if (ret < 0)
924                 rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n");
925
926         /*flib init */
927         if (flib_init() != 0)
928                 rte_exit(EXIT_FAILURE, "flib init error");
929
930         /**
931           * Allocated structures that slave lcore would change. For those that slaves are
932           * read only, needn't use malloc to share and global or static variables is ok since
933           * slave inherit all the knowledge that master initialized.
934           **/
935         if (l2fwd_malloc_shared_struct() != 0)
936                 rte_exit(EXIT_FAILURE, "malloc mem failed\n");
937
938         /* Initialize lcore_resource structures */
939         memset(lcore_resource, 0, sizeof(lcore_resource));
940         for (i = 0; i < RTE_MAX_LCORE; i++)
941                 lcore_resource[i].lcore_id = i;
942
943         nb_ports = rte_eth_dev_count_avail();
944         if (nb_ports == 0)
945                 rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
946
947         /* create the mbuf pool */
948         RTE_ETH_FOREACH_DEV(portid) {
949                 /* skip ports that are not enabled */
950                 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
951                         continue;
952                 char buf_name[RTE_MEMPOOL_NAMESIZE];
953                 snprintf(buf_name, RTE_MEMPOOL_NAMESIZE, MBUF_NAME, portid);
954                 l2fwd_pktmbuf_pool[portid] =
955                         rte_pktmbuf_pool_create(buf_name, NB_MBUF, 32,
956                                 0, MBUF_DATA_SIZE, rte_socket_id());
957                 if (l2fwd_pktmbuf_pool[portid] == NULL)
958                         rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
959
960                 printf("Create mbuf %s\n", buf_name);
961         }
962
963         /* reset l2fwd_dst_ports */
964         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++)
965                 l2fwd_dst_ports[portid] = 0;
966         last_port = 0;
967
968         /*
969          * Each logical core is assigned a dedicated TX queue on each port.
970          */
971         RTE_ETH_FOREACH_DEV(portid) {
972                 /* skip ports that are not enabled */
973                 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
974                         continue;
975
976                 if (nb_ports_in_mask % 2) {
977                         l2fwd_dst_ports[portid] = last_port;
978                         l2fwd_dst_ports[last_port] = portid;
979                 }
980                 else
981                         last_port = portid;
982
983                 nb_ports_in_mask++;
984         }
985         if (nb_ports_in_mask % 2) {
986                 printf("Notice: odd number of ports in portmask.\n");
987                 l2fwd_dst_ports[last_port] = last_port;
988         }
989
990         rx_lcore_id = 0;
991         qconf = NULL;
992
993         /* Initialize the port/queue configuration of each logical core */
994         RTE_ETH_FOREACH_DEV(portid) {
995                 struct lcore_resource_struct *res;
996                 /* skip ports that are not enabled */
997                 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
998                         continue;
999
1000                 /* get the lcore_id for this port */
1001                 /* skip master lcore */
1002                 while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
1003                            rte_get_master_lcore() == rx_lcore_id ||
1004                        lcore_queue_conf[rx_lcore_id].n_rx_port ==
1005                        l2fwd_rx_queue_per_lcore) {
1006
1007                         rx_lcore_id++;
1008                         if (rx_lcore_id >= RTE_MAX_LCORE)
1009                                 rte_exit(EXIT_FAILURE, "Not enough cores\n");
1010                 }
1011
1012                 if (qconf != &lcore_queue_conf[rx_lcore_id])
1013                         /* Assigned a new logical core in the loop above. */
1014                         qconf = &lcore_queue_conf[rx_lcore_id];
1015
1016                 qconf->rx_port_list[qconf->n_rx_port] = portid;
1017                 qconf->n_rx_port++;
1018
1019                 /* Save the port resource info into lcore_resource strucutres */
1020                 res = &lcore_resource[rx_lcore_id];
1021                 res->enabled = 1;
1022                 res->port[res->port_num++] = portid;
1023
1024                 printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned) portid);
1025         }
1026
1027         /* Initialise each port */
1028         RTE_ETH_FOREACH_DEV(portid) {
1029                 struct rte_eth_rxconf rxq_conf;
1030                 struct rte_eth_txconf txq_conf;
1031                 struct rte_eth_conf local_port_conf = port_conf;
1032
1033                 /* skip ports that are not enabled */
1034                 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) {
1035                         printf("Skipping disabled port %u\n", (unsigned) portid);
1036                         continue;
1037                 }
1038                 nb_ports_available++;
1039                 /* init port */
1040                 printf("Initializing port %u... ", (unsigned) portid);
1041                 fflush(stdout);
1042                 rte_eth_dev_info_get(portid, &dev_info);
1043                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
1044                         local_port_conf.txmode.offloads |=
1045                                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1046                 ret = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
1047                 if (ret < 0)
1048                         rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
1049                                   ret, (unsigned) portid);
1050
1051                 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
1052                                                        &nb_txd);
1053                 if (ret < 0)
1054                         rte_exit(EXIT_FAILURE,
1055                                  "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%u\n",
1056                                  ret, (unsigned) portid);
1057
1058                 rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]);
1059
1060                 /* init one RX queue */
1061                 fflush(stdout);
1062                 rxq_conf = dev_info.default_rxconf;
1063                 rxq_conf.offloads = local_port_conf.rxmode.offloads;
1064                 ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
1065                                              rte_eth_dev_socket_id(portid),
1066                                              &rxq_conf,
1067                                              l2fwd_pktmbuf_pool[portid]);
1068                 if (ret < 0)
1069                         rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n",
1070                                   ret, (unsigned) portid);
1071
1072                 /* init one TX queue on each port */
1073                 fflush(stdout);
1074                 txq_conf = dev_info.default_txconf;
1075                 txq_conf.tx_offloads = local_port_conf.txmode.offloads;
1076                 ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
1077                                 rte_eth_dev_socket_id(portid),
1078                                 &txq_conf);
1079                 if (ret < 0)
1080                         rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n",
1081                                 ret, (unsigned) portid);
1082
1083                 /* Initialize TX buffers */
1084                 tx_buffer[portid] = rte_zmalloc_socket("tx_buffer",
1085                                 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
1086                                 rte_eth_dev_socket_id(portid));
1087                 if (tx_buffer[portid] == NULL)
1088                         rte_exit(EXIT_FAILURE, "Cannot allocate buffer for tx on port %u\n",
1089                                         (unsigned) portid);
1090
1091                 rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST);
1092
1093                 ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid],
1094                                 rte_eth_tx_buffer_count_callback,
1095                                 &port_statistics[portid].dropped);
1096                 if (ret < 0)
1097                                 rte_exit(EXIT_FAILURE, "Cannot set error callback for "
1098                                                 "tx buffer on port %u\n", (unsigned) portid);
1099
1100                 /* Start device */
1101                 ret = rte_eth_dev_start(portid);
1102                 if (ret < 0)
1103                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1104                                   ret, (unsigned) portid);
1105
1106                 printf("done: \n");
1107
1108                 rte_eth_promiscuous_enable(portid);
1109
1110                 printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
1111                                 (unsigned) portid,
1112                                 l2fwd_ports_eth_addr[portid].addr_bytes[0],
1113                                 l2fwd_ports_eth_addr[portid].addr_bytes[1],
1114                                 l2fwd_ports_eth_addr[portid].addr_bytes[2],
1115                                 l2fwd_ports_eth_addr[portid].addr_bytes[3],
1116                                 l2fwd_ports_eth_addr[portid].addr_bytes[4],
1117                                 l2fwd_ports_eth_addr[portid].addr_bytes[5]);
1118
1119                 /* initialize port stats */
1120                 //memset(&port_statistics, 0, sizeof(port_statistics));
1121         }
1122
1123         if (!nb_ports_available) {
1124                 rte_exit(EXIT_FAILURE,
1125                         "All available ports are disabled. Please set portmask.\n");
1126         }
1127
1128         check_all_ports_link_status(l2fwd_enabled_port_mask);
1129
1130         /* Record pair lcore */
1131         /**
1132          * Since l2fwd example would create pair between different neighbour port, that's
1133          * port 0 receive and forward to port 1, the same to port 1, these 2 ports will have
1134          * dependency. If one port stopped working (killed, for example), the port need to
1135          * be stopped/started again. During the time, another port need to wait until stop/start
1136          * procedure completed. So, record the pair relationship for those lcores working
1137          * on ports.
1138          **/
1139         RTE_ETH_FOREACH_DEV(portid) {
1140                 uint32_t pair_port;
1141                 unsigned lcore = 0, pair_lcore = 0;
1142                 unsigned j, find_lcore, find_pair_lcore;
1143                 /* skip ports that are not enabled */
1144                 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
1145                         continue;
1146
1147                 /* Find pair ports' lcores */
1148                 find_lcore = find_pair_lcore = 0;
1149                 pair_port = l2fwd_dst_ports[portid];
1150                 for (i = 0; i < RTE_MAX_LCORE; i++) {
1151                         if (!rte_lcore_is_enabled(i))
1152                                 continue;
1153                         for (j = 0; j < lcore_queue_conf[i].n_rx_port;j++) {
1154                                 if (lcore_queue_conf[i].rx_port_list[j] == portid) {
1155                                         lcore = i;
1156                                         find_lcore = 1;
1157                                         break;
1158                                 }
1159                                 if (lcore_queue_conf[i].rx_port_list[j] == pair_port) {
1160                                         pair_lcore = i;
1161                                         find_pair_lcore = 1;
1162                                         break;
1163                                 }
1164                         }
1165                         if (find_lcore && find_pair_lcore)
1166                                 break;
1167                 }
1168                 if (!find_lcore || !find_pair_lcore)
1169                         rte_exit(EXIT_FAILURE, "Not find port=%d pair\n", portid);
1170
1171                 printf("lcore %u and %u paired\n", lcore, pair_lcore);
1172                 lcore_resource[lcore].pair_id = pair_lcore;
1173                 lcore_resource[pair_lcore].pair_id = lcore;
1174         }
1175
1176         /* Create message buffer for all master and slave */
1177         message_pool = rte_mempool_create("ms_msg_pool",
1178                            NB_CORE_MSGBUF * RTE_MAX_LCORE,
1179                            sizeof(enum l2fwd_cmd), NB_CORE_MSGBUF / 2,
1180                            0, NULL, NULL, NULL, NULL, rte_socket_id(), 0);
1181
1182         if (message_pool == NULL)
1183                 rte_exit(EXIT_FAILURE, "Create msg mempool failed\n");
1184
1185         /* Create ring for each master and slave pair, also register cb when slave leaves */
1186         for (i = 0; i < RTE_MAX_LCORE; i++) {
1187                 /**
1188                  * Only create ring and register slave_exit cb in case that core involved into
1189                  * packet forwarding
1190                  **/
1191                 if (lcore_resource[i].enabled) {
1192                         /* Create ring for master and slave communication */
1193                         ret = create_ms_ring(i);
1194                         if (ret != 0)
1195                                 rte_exit(EXIT_FAILURE, "Create ring for lcore=%u failed",
1196                                 i);
1197
1198                         if (flib_register_slave_exit_notify(i,
1199                                 slave_exit_cb) != 0)
1200                                 rte_exit(EXIT_FAILURE,
1201                                                 "Register master_trace_slave_exit failed");
1202                 }
1203         }
1204
1205         /* launch per-lcore init on every lcore except master */
1206         flib_mp_remote_launch(l2fwd_launch_one_lcore, NULL, SKIP_MASTER);
1207
1208         /* print statistics 10 second */
1209         prev_tsc = cur_tsc = rte_rdtsc();
1210         timer_tsc = 0;
1211         while (1) {
1212                 sleep(1);
1213                 cur_tsc = rte_rdtsc();
1214                 diff_tsc = cur_tsc - prev_tsc;
1215                 /* if timer is enabled */
1216                 if (timer_period > 0) {
1217
1218                         /* advance the timer */
1219                         timer_tsc += diff_tsc;
1220
1221                         /* if timer has reached its timeout */
1222                         if (unlikely(timer_tsc >= (uint64_t) timer_period)) {
1223
1224                                 print_stats();
1225                                 /* reset the timer */
1226                                 timer_tsc = 0;
1227                         }
1228                 }
1229
1230                 prev_tsc = cur_tsc;
1231
1232                 /* Check any slave need restart or recreate */
1233                 rte_spinlock_lock(&res_lock);
1234                 for (i = 0; i < RTE_MAX_LCORE; i++) {
1235                         struct lcore_resource_struct *res  = &lcore_resource[i];
1236                         struct lcore_resource_struct *pair = &lcore_resource[res->pair_id];
1237
1238                         /* If find slave exited, try to reset pair */
1239                         if (res->enabled && res->flags && pair->enabled) {
1240                                 if (!pair->flags) {
1241                                         master_sendcmd_with_ack(pair->lcore_id, CMD_STOP);
1242                                         rte_spinlock_unlock(&res_lock);
1243                                         sleep(1);
1244                                         rte_spinlock_lock(&res_lock);
1245                                         if (pair->flags)
1246                                                 continue;
1247                                 }
1248                                 if (reset_pair(res->lcore_id, pair->lcore_id) != 0)
1249                                         rte_exit(EXIT_FAILURE, "failed to reset slave");
1250                                 res->flags  = 0;
1251                                 pair->flags = 0;
1252                         }
1253                 }
1254                 rte_spinlock_unlock(&res_lock);
1255         }
1256
1257 }