29b64014d772dfd99f1939a43ebaa2736baa8aed
[dpdk.git] / app / test-eventdev / test_pipeline_common.c
1 /*
2  * SPDX-License-Identifier: BSD-3-Clause
3  * Copyright 2017 Cavium, Inc.
4  */
5
6 #include "test_pipeline_common.h"
7
8 int
9 pipeline_test_result(struct evt_test *test, struct evt_options *opt)
10 {
11         RTE_SET_USED(opt);
12         int i;
13         uint64_t total = 0;
14         struct test_pipeline *t = evt_test_priv(test);
15
16         evt_info("Packet distribution across worker cores :");
17         for (i = 0; i < t->nb_workers; i++)
18                 total += t->worker[i].processed_pkts;
19         for (i = 0; i < t->nb_workers; i++)
20                 evt_info("Worker %d packets: "CLGRN"%"PRIx64""CLNRM" percentage:"
21                                 CLGRN" %3.2f"CLNRM, i,
22                                 t->worker[i].processed_pkts,
23                                 (((double)t->worker[i].processed_pkts)/total)
24                                 * 100);
25         return t->result;
26 }
27
28 void
29 pipeline_opt_dump(struct evt_options *opt, uint8_t nb_queues)
30 {
31         evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores));
32         evt_dump_worker_lcores(opt);
33         evt_dump_nb_stages(opt);
34         evt_dump("nb_evdev_ports", "%d", pipeline_nb_event_ports(opt));
35         evt_dump("nb_evdev_queues", "%d", nb_queues);
36         evt_dump_queue_priority(opt);
37         evt_dump_sched_type_list(opt);
38         evt_dump_producer_type(opt);
39         evt_dump("nb_eth_rx_queues", "%d", opt->eth_queues);
40         evt_dump("event_vector", "%d", opt->ena_vector);
41         if (opt->ena_vector) {
42                 evt_dump("vector_size", "%d", opt->vector_size);
43                 evt_dump("vector_tmo_ns", "%" PRIu64 "", opt->vector_tmo_nsec);
44         }
45 }
46
47 static inline uint64_t
48 processed_pkts(struct test_pipeline *t)
49 {
50         uint8_t i;
51         uint64_t total = 0;
52
53         for (i = 0; i < t->nb_workers; i++)
54                 total += t->worker[i].processed_pkts;
55
56         return total;
57 }
58
59 int
60 pipeline_launch_lcores(struct evt_test *test, struct evt_options *opt,
61                 int (*worker)(void *))
62 {
63         int ret, lcore_id;
64         struct test_pipeline *t = evt_test_priv(test);
65
66         int port_idx = 0;
67         /* launch workers */
68         RTE_LCORE_FOREACH_WORKER(lcore_id) {
69                 if (!(opt->wlcores[lcore_id]))
70                         continue;
71
72                 ret = rte_eal_remote_launch(worker,
73                                  &t->worker[port_idx], lcore_id);
74                 if (ret) {
75                         evt_err("failed to launch worker %d", lcore_id);
76                         return ret;
77                 }
78                 port_idx++;
79         }
80
81         uint64_t perf_cycles = rte_get_timer_cycles();
82         const uint64_t perf_sample = rte_get_timer_hz();
83
84         static float total_mpps;
85         static uint64_t samples;
86
87         uint64_t prev_pkts = 0;
88
89         while (t->done == false) {
90                 const uint64_t new_cycles = rte_get_timer_cycles();
91
92                 if ((new_cycles - perf_cycles) > perf_sample) {
93                         const uint64_t curr_pkts = processed_pkts(t);
94
95                         float mpps = (float)(curr_pkts - prev_pkts)/1000000;
96
97                         prev_pkts = curr_pkts;
98                         perf_cycles = new_cycles;
99                         total_mpps += mpps;
100                         ++samples;
101                         printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM,
102                                         mpps, total_mpps/samples);
103                         fflush(stdout);
104                 }
105         }
106         printf("\n");
107         return 0;
108 }
109
110 int
111 pipeline_opt_check(struct evt_options *opt, uint64_t nb_queues)
112 {
113         unsigned int lcores;
114
115         /* N worker + main */
116         lcores = 2;
117
118         if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR) {
119                 evt_err("Invalid producer type '%s' valid producer '%s'",
120                         evt_prod_id_to_name(opt->prod_type),
121                         evt_prod_id_to_name(EVT_PROD_TYPE_ETH_RX_ADPTR));
122                 return -1;
123         }
124
125         if (!rte_eth_dev_count_avail()) {
126                 evt_err("test needs minimum 1 ethernet dev");
127                 return -1;
128         }
129
130         if (rte_lcore_count() < lcores) {
131                 evt_err("test need minimum %d lcores", lcores);
132                 return -1;
133         }
134
135         /* Validate worker lcores */
136         if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) {
137                 evt_err("worker lcores overlaps with main lcore");
138                 return -1;
139         }
140         if (evt_has_disabled_lcore(opt->wlcores)) {
141                 evt_err("one or more workers lcores are not enabled");
142                 return -1;
143         }
144         if (!evt_has_active_lcore(opt->wlcores)) {
145                 evt_err("minimum one worker is required");
146                 return -1;
147         }
148
149         if (nb_queues > EVT_MAX_QUEUES) {
150                 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES);
151                 return -1;
152         }
153         if (pipeline_nb_event_ports(opt) > EVT_MAX_PORTS) {
154                 evt_err("number of ports exceeds %d", EVT_MAX_PORTS);
155                 return -1;
156         }
157
158         if (evt_has_invalid_stage(opt))
159                 return -1;
160
161         if (evt_has_invalid_sched_type(opt))
162                 return -1;
163
164         return 0;
165 }
166
167 #define NB_RX_DESC                      128
168 #define NB_TX_DESC                      512
169 int
170 pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt)
171 {
172         uint16_t i, j;
173         int ret;
174         uint8_t nb_queues = 1;
175         struct test_pipeline *t = evt_test_priv(test);
176         struct rte_eth_rxconf rx_conf;
177         struct rte_eth_conf port_conf = {
178                 .rxmode = {
179                         .mq_mode = RTE_ETH_MQ_RX_RSS,
180                 },
181                 .rx_adv_conf = {
182                         .rss_conf = {
183                                 .rss_key = NULL,
184                                 .rss_hf = RTE_ETH_RSS_IP,
185                         },
186                 },
187         };
188
189         if (!rte_eth_dev_count_avail()) {
190                 evt_err("No ethernet ports found.");
191                 return -ENODEV;
192         }
193
194         if (opt->max_pkt_sz < RTE_ETHER_MIN_LEN) {
195                 evt_err("max_pkt_sz can not be less than %d",
196                         RTE_ETHER_MIN_LEN);
197                 return -EINVAL;
198         }
199
200         port_conf.rxmode.mtu = opt->max_pkt_sz - RTE_ETHER_HDR_LEN -
201                 RTE_ETHER_CRC_LEN;
202
203         t->internal_port = 1;
204         RTE_ETH_FOREACH_DEV(i) {
205                 struct rte_eth_dev_info dev_info;
206                 struct rte_eth_conf local_port_conf = port_conf;
207                 uint32_t caps = 0;
208
209                 ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id, i, &caps);
210                 if (ret != 0) {
211                         evt_err("failed to get event tx adapter[%d] caps", i);
212                         return ret;
213                 }
214
215                 if (!(caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT))
216                         t->internal_port = 0;
217
218                 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, i, &caps);
219                 if (ret != 0) {
220                         evt_err("failed to get event tx adapter[%d] caps", i);
221                         return ret;
222                 }
223
224                 if (!(caps & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT))
225                         local_port_conf.rxmode.offloads |=
226                                 RTE_ETH_RX_OFFLOAD_RSS_HASH;
227
228                 ret = rte_eth_dev_info_get(i, &dev_info);
229                 if (ret != 0) {
230                         evt_err("Error during getting device (port %u) info: %s\n",
231                                 i, strerror(-ret));
232                         return ret;
233                 }
234
235                 /* Enable mbuf fast free if PMD has the capability. */
236                 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
237                         local_port_conf.txmode.offloads |=
238                                 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
239
240                 rx_conf = dev_info.default_rxconf;
241                 rx_conf.offloads = port_conf.rxmode.offloads;
242
243                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
244                         dev_info.flow_type_rss_offloads;
245                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
246                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
247                         evt_info("Port %u modified RSS hash function based on hardware support,"
248                                 "requested:%#"PRIx64" configured:%#"PRIx64"",
249                                 i,
250                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
251                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
252                 }
253
254                 if (rte_eth_dev_configure(i, opt->eth_queues, nb_queues,
255                                           &local_port_conf) < 0) {
256                         evt_err("Failed to configure eth port [%d]", i);
257                         return -EINVAL;
258                 }
259
260                 for (j = 0; j < opt->eth_queues; j++) {
261                         if (rte_eth_rx_queue_setup(
262                                     i, j, NB_RX_DESC, rte_socket_id(), &rx_conf,
263                                     opt->per_port_pool ? t->pool[i] :
264                                                               t->pool[0]) < 0) {
265                                 evt_err("Failed to setup eth port [%d] rx_queue: %d.",
266                                         i, 0);
267                                 return -EINVAL;
268                         }
269                 }
270
271                 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC,
272                                         rte_socket_id(), NULL) < 0) {
273                         evt_err("Failed to setup eth port [%d] tx_queue: %d.",
274                                         i, 0);
275                         return -EINVAL;
276                 }
277
278                 ret = rte_eth_promiscuous_enable(i);
279                 if (ret != 0) {
280                         evt_err("Failed to enable promiscuous mode for eth port [%d]: %s",
281                                 i, rte_strerror(-ret));
282                         return ret;
283                 }
284         }
285
286         return 0;
287 }
288
289 int
290 pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt,
291                 uint8_t *queue_arr, uint8_t nb_queues,
292                 const struct rte_event_port_conf p_conf)
293 {
294         int ret;
295         uint8_t port;
296         struct test_pipeline *t = evt_test_priv(test);
297
298
299         /* setup one port per worker, linking to all queues */
300         for (port = 0; port < evt_nr_active_lcores(opt->wlcores); port++) {
301                 struct worker_data *w = &t->worker[port];
302
303                 w->dev_id = opt->dev_id;
304                 w->port_id = port;
305                 w->t = t;
306                 w->processed_pkts = 0;
307
308                 ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
309                 if (ret) {
310                         evt_err("failed to setup port %d", port);
311                         return ret;
312                 }
313
314                 if (rte_event_port_link(opt->dev_id, port, queue_arr, NULL,
315                                         nb_queues) != nb_queues)
316                         goto link_fail;
317         }
318
319         return 0;
320
321 link_fail:
322         evt_err("failed to link queues to port %d", port);
323         return -EINVAL;
324 }
325
326 int
327 pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride,
328                 struct rte_event_port_conf prod_conf)
329 {
330         int ret = 0;
331         uint16_t prod;
332         struct rte_mempool *vector_pool = NULL;
333         struct rte_event_eth_rx_adapter_queue_conf queue_conf;
334
335         memset(&queue_conf, 0,
336                         sizeof(struct rte_event_eth_rx_adapter_queue_conf));
337         queue_conf.ev.sched_type = opt->sched_type_list[0];
338         if (opt->ena_vector) {
339                 unsigned int nb_elem = (opt->pool_sz / opt->vector_size) << 1;
340
341                 nb_elem = nb_elem ? nb_elem : 1;
342                 vector_pool = rte_event_vector_pool_create(
343                         "vector_pool", nb_elem, 0, opt->vector_size,
344                         opt->socket_id);
345                 if (vector_pool == NULL) {
346                         evt_err("failed to create event vector pool");
347                         return -ENOMEM;
348                 }
349         }
350         RTE_ETH_FOREACH_DEV(prod) {
351                 struct rte_event_eth_rx_adapter_vector_limits limits;
352                 uint32_t cap;
353
354                 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id,
355                                 prod, &cap);
356                 if (ret) {
357                         evt_err("failed to get event rx adapter[%d]"
358                                         " capabilities",
359                                         opt->dev_id);
360                         return ret;
361                 }
362
363                 if (opt->ena_vector) {
364                         memset(&limits, 0, sizeof(limits));
365                         ret = rte_event_eth_rx_adapter_vector_limits_get(
366                                 opt->dev_id, prod, &limits);
367                         if (ret) {
368                                 evt_err("failed to get vector limits");
369                                 return ret;
370                         }
371
372                         if (opt->vector_size < limits.min_sz ||
373                             opt->vector_size > limits.max_sz) {
374                                 evt_err("Vector size [%d] not within limits max[%d] min[%d]",
375                                         opt->vector_size, limits.min_sz,
376                                         limits.max_sz);
377                                 return -EINVAL;
378                         }
379
380                         if (limits.log2_sz &&
381                             !rte_is_power_of_2(opt->vector_size)) {
382                                 evt_err("Vector size [%d] not power of 2",
383                                         opt->vector_size);
384                                 return -EINVAL;
385                         }
386
387                         if (opt->vector_tmo_nsec > limits.max_timeout_ns ||
388                             opt->vector_tmo_nsec < limits.min_timeout_ns) {
389                                 evt_err("Vector timeout [%" PRIu64
390                                         "] not within limits max[%" PRIu64
391                                         "] min[%" PRIu64 "]",
392                                         opt->vector_tmo_nsec,
393                                         limits.max_timeout_ns,
394                                         limits.min_timeout_ns);
395                                 return -EINVAL;
396                         }
397
398                         if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) {
399                                 queue_conf.vector_sz = opt->vector_size;
400                                 queue_conf.vector_timeout_ns =
401                                         opt->vector_tmo_nsec;
402                                 queue_conf.rx_queue_flags |=
403                                 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
404                                 queue_conf.vector_mp = vector_pool;
405                         } else {
406                                 evt_err("Rx adapter doesn't support event vector");
407                                 return -EINVAL;
408                         }
409                 }
410                 queue_conf.ev.queue_id = prod * stride;
411                 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id,
412                                 &prod_conf);
413                 if (ret) {
414                         evt_err("failed to create rx adapter[%d]", prod);
415                         return ret;
416                 }
417                 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1,
418                                 &queue_conf);
419                 if (ret) {
420                         evt_err("failed to add rx queues to adapter[%d]", prod);
421                         return ret;
422                 }
423
424                 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
425                         uint32_t service_id = -1U;
426
427                         rte_event_eth_rx_adapter_service_id_get(prod,
428                                         &service_id);
429                         ret = evt_service_setup(service_id);
430                         if (ret) {
431                                 evt_err("Failed to setup service core"
432                                                 " for Rx adapter");
433                                 return ret;
434                         }
435                 }
436
437                 evt_info("Port[%d] using Rx adapter[%d] configured", prod,
438                                 prod);
439         }
440
441         return ret;
442 }
443
444 int
445 pipeline_event_tx_adapter_setup(struct evt_options *opt,
446                 struct rte_event_port_conf port_conf)
447 {
448         int ret = 0;
449         uint16_t consm;
450
451         RTE_ETH_FOREACH_DEV(consm) {
452                 uint32_t cap;
453
454                 ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id,
455                                 consm, &cap);
456                 if (ret) {
457                         evt_err("failed to get event tx adapter[%d] caps",
458                                         consm);
459                         return ret;
460                 }
461
462                 if (opt->ena_vector) {
463                         if (!(cap &
464                               RTE_EVENT_ETH_TX_ADAPTER_CAP_EVENT_VECTOR)) {
465                                 evt_err("Tx adapter doesn't support event vector");
466                                 return -EINVAL;
467                         }
468                 }
469
470                 ret = rte_event_eth_tx_adapter_create(consm, opt->dev_id,
471                                 &port_conf);
472                 if (ret) {
473                         evt_err("failed to create tx adapter[%d]", consm);
474                         return ret;
475                 }
476
477                 ret = rte_event_eth_tx_adapter_queue_add(consm, consm, -1);
478                 if (ret) {
479                         evt_err("failed to add tx queues to adapter[%d]",
480                                         consm);
481                         return ret;
482                 }
483
484                 if (!(cap & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT)) {
485                         uint32_t service_id = -1U;
486
487                         ret = rte_event_eth_tx_adapter_service_id_get(consm,
488                                                                    &service_id);
489                         if (ret != -ESRCH && ret != 0) {
490                                 evt_err("Failed to get Tx adptr service ID");
491                                 return ret;
492                         }
493                         ret = evt_service_setup(service_id);
494                         if (ret) {
495                                 evt_err("Failed to setup service core"
496                                                 " for Tx adapter");
497                                 return ret;
498                         }
499                 }
500
501                 evt_info("Port[%d] using Tx adapter[%d] Configured", consm,
502                                 consm);
503         }
504
505         return ret;
506 }
507
508 void
509 pipeline_ethdev_rx_stop(struct evt_test *test, struct evt_options *opt)
510 {
511         uint16_t i, j;
512         RTE_SET_USED(test);
513
514         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
515                 RTE_ETH_FOREACH_DEV(i) {
516                         rte_event_eth_rx_adapter_stop(i);
517                         rte_event_eth_rx_adapter_queue_del(i, i, -1);
518                         for (j = 0; j < opt->eth_queues; j++)
519                                 rte_eth_dev_rx_queue_stop(i, j);
520                 }
521         }
522 }
523
524 void
525 pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
526 {
527         uint16_t i;
528         RTE_SET_USED(test);
529         RTE_SET_USED(opt);
530
531         RTE_ETH_FOREACH_DEV(i) {
532                 rte_event_eth_tx_adapter_stop(i);
533                 rte_event_eth_tx_adapter_queue_del(i, i, -1);
534                 rte_eth_dev_tx_queue_stop(i, 0);
535                 rte_eth_dev_stop(i);
536         }
537 }
538
539 void
540 pipeline_eventdev_destroy(struct evt_test *test, struct evt_options *opt)
541 {
542         RTE_SET_USED(test);
543
544         rte_event_dev_stop(opt->dev_id);
545         rte_event_dev_close(opt->dev_id);
546 }
547
548 int
549 pipeline_mempool_setup(struct evt_test *test, struct evt_options *opt)
550 {
551         struct test_pipeline *t = evt_test_priv(test);
552         int i, ret;
553
554         if (!opt->mbuf_sz)
555                 opt->mbuf_sz = RTE_MBUF_DEFAULT_BUF_SIZE;
556
557         if (!opt->max_pkt_sz)
558                 opt->max_pkt_sz = RTE_ETHER_MAX_LEN;
559
560         RTE_ETH_FOREACH_DEV(i) {
561                 struct rte_eth_dev_info dev_info;
562                 uint16_t data_size = 0;
563
564                 memset(&dev_info, 0, sizeof(dev_info));
565                 ret = rte_eth_dev_info_get(i, &dev_info);
566                 if (ret != 0) {
567                         evt_err("Error during getting device (port %u) info: %s\n",
568                                 i, strerror(-ret));
569                         return ret;
570                 }
571
572                 if (dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
573                                 dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
574                         data_size = opt->max_pkt_sz /
575                                 dev_info.rx_desc_lim.nb_mtu_seg_max;
576                         data_size += RTE_PKTMBUF_HEADROOM;
577
578                         if (data_size  > opt->mbuf_sz)
579                                 opt->mbuf_sz = data_size;
580                 }
581                 if (opt->per_port_pool) {
582                         char name[RTE_MEMPOOL_NAMESIZE];
583
584                         snprintf(name, RTE_MEMPOOL_NAMESIZE, "%s-%d",
585                                  test->name, i);
586                         t->pool[i] = rte_pktmbuf_pool_create(
587                                 name,         /* mempool name */
588                                 opt->pool_sz, /* number of elements*/
589                                 0,            /* cache size*/
590                                 0, opt->mbuf_sz, opt->socket_id); /* flags */
591
592                         if (t->pool[i] == NULL) {
593                                 evt_err("failed to create mempool %s", name);
594                                 return -ENOMEM;
595                         }
596                 }
597         }
598
599         if (!opt->per_port_pool) {
600                 t->pool[0] = rte_pktmbuf_pool_create(
601                         test->name,   /* mempool name */
602                         opt->pool_sz, /* number of elements*/
603                         0,            /* cache size*/
604                         0, opt->mbuf_sz, opt->socket_id); /* flags */
605
606                 if (t->pool[0] == NULL) {
607                         evt_err("failed to create mempool");
608                         return -ENOMEM;
609                 }
610         }
611
612         return 0;
613 }
614
615 void
616 pipeline_mempool_destroy(struct evt_test *test, struct evt_options *opt)
617 {
618         struct test_pipeline *t = evt_test_priv(test);
619         int i;
620
621         RTE_SET_USED(opt);
622         if (opt->per_port_pool) {
623                 RTE_ETH_FOREACH_DEV(i)
624                         rte_mempool_free(t->pool[i]);
625         } else {
626                 rte_mempool_free(t->pool[0]);
627         }
628 }
629
630 int
631 pipeline_test_setup(struct evt_test *test, struct evt_options *opt)
632 {
633         void *test_pipeline;
634
635         test_pipeline = rte_zmalloc_socket(test->name,
636                         sizeof(struct test_pipeline), RTE_CACHE_LINE_SIZE,
637                         opt->socket_id);
638         if (test_pipeline  == NULL) {
639                 evt_err("failed to allocate test_pipeline memory");
640                 goto nomem;
641         }
642         test->test_priv = test_pipeline;
643
644         struct test_pipeline *t = evt_test_priv(test);
645
646         t->nb_workers = evt_nr_active_lcores(opt->wlcores);
647         t->outstand_pkts = opt->nb_pkts * evt_nr_active_lcores(opt->wlcores);
648         t->done = false;
649         t->nb_flows = opt->nb_flows;
650         t->result = EVT_TEST_FAILED;
651         t->opt = opt;
652         opt->prod_type = EVT_PROD_TYPE_ETH_RX_ADPTR;
653         memcpy(t->sched_type_list, opt->sched_type_list,
654                         sizeof(opt->sched_type_list));
655         return 0;
656 nomem:
657         return -ENOMEM;
658 }
659
660 void
661 pipeline_test_destroy(struct evt_test *test, struct evt_options *opt)
662 {
663         RTE_SET_USED(opt);
664
665         rte_free(test->test_priv);
666 }