2 * SPDX-License-Identifier: BSD-3-Clause
3 * Copyright 2017 Cavium, Inc.
6 #include "test_pipeline_common.h"
9 pipeline_test_result(struct evt_test *test, struct evt_options *opt)
14 struct test_pipeline *t = evt_test_priv(test);
16 evt_info("Packet distribution across worker cores :");
17 for (i = 0; i < t->nb_workers; i++)
18 total += t->worker[i].processed_pkts;
19 for (i = 0; i < t->nb_workers; i++)
20 evt_info("Worker %d packets: "CLGRN"%"PRIx64""CLNRM" percentage:"
21 CLGRN" %3.2f"CLNRM, i,
22 t->worker[i].processed_pkts,
23 (((double)t->worker[i].processed_pkts)/total)
29 pipeline_opt_dump(struct evt_options *opt, uint8_t nb_queues)
31 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores));
32 evt_dump_worker_lcores(opt);
33 evt_dump_nb_stages(opt);
34 evt_dump("nb_evdev_ports", "%d", pipeline_nb_event_ports(opt));
35 evt_dump("nb_evdev_queues", "%d", nb_queues);
36 evt_dump_queue_priority(opt);
37 evt_dump_sched_type_list(opt);
38 evt_dump_producer_type(opt);
39 evt_dump("nb_eth_rx_queues", "%d", opt->eth_queues);
40 evt_dump("event_vector", "%d", opt->ena_vector);
41 if (opt->ena_vector) {
42 evt_dump("vector_size", "%d", opt->vector_size);
43 evt_dump("vector_tmo_ns", "%" PRIu64 "", opt->vector_tmo_nsec);
47 static inline uint64_t
48 processed_pkts(struct test_pipeline *t)
53 for (i = 0; i < t->nb_workers; i++)
54 total += t->worker[i].processed_pkts;
60 pipeline_launch_lcores(struct evt_test *test, struct evt_options *opt,
61 int (*worker)(void *))
64 struct test_pipeline *t = evt_test_priv(test);
68 RTE_LCORE_FOREACH_WORKER(lcore_id) {
69 if (!(opt->wlcores[lcore_id]))
72 ret = rte_eal_remote_launch(worker,
73 &t->worker[port_idx], lcore_id);
75 evt_err("failed to launch worker %d", lcore_id);
81 uint64_t perf_cycles = rte_get_timer_cycles();
82 const uint64_t perf_sample = rte_get_timer_hz();
84 static float total_mpps;
85 static uint64_t samples;
87 uint64_t prev_pkts = 0;
89 while (t->done == false) {
90 const uint64_t new_cycles = rte_get_timer_cycles();
92 if ((new_cycles - perf_cycles) > perf_sample) {
93 const uint64_t curr_pkts = processed_pkts(t);
95 float mpps = (float)(curr_pkts - prev_pkts)/1000000;
97 prev_pkts = curr_pkts;
98 perf_cycles = new_cycles;
101 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM,
102 mpps, total_mpps/samples);
111 pipeline_opt_check(struct evt_options *opt, uint64_t nb_queues)
115 /* N worker + main */
118 if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR) {
119 evt_err("Invalid producer type '%s' valid producer '%s'",
120 evt_prod_id_to_name(opt->prod_type),
121 evt_prod_id_to_name(EVT_PROD_TYPE_ETH_RX_ADPTR));
125 if (!rte_eth_dev_count_avail()) {
126 evt_err("test needs minimum 1 ethernet dev");
130 if (rte_lcore_count() < lcores) {
131 evt_err("test need minimum %d lcores", lcores);
135 /* Validate worker lcores */
136 if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) {
137 evt_err("worker lcores overlaps with main lcore");
140 if (evt_has_disabled_lcore(opt->wlcores)) {
141 evt_err("one or more workers lcores are not enabled");
144 if (!evt_has_active_lcore(opt->wlcores)) {
145 evt_err("minimum one worker is required");
149 if (nb_queues > EVT_MAX_QUEUES) {
150 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES);
153 if (pipeline_nb_event_ports(opt) > EVT_MAX_PORTS) {
154 evt_err("number of ports exceeds %d", EVT_MAX_PORTS);
158 if (evt_has_invalid_stage(opt))
161 if (evt_has_invalid_sched_type(opt))
167 #define NB_RX_DESC 128
168 #define NB_TX_DESC 512
170 pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt)
174 uint8_t nb_queues = 1;
175 struct test_pipeline *t = evt_test_priv(test);
176 struct rte_eth_rxconf rx_conf;
177 struct rte_eth_conf port_conf = {
179 .mq_mode = RTE_ETH_MQ_RX_RSS,
184 .rss_hf = RTE_ETH_RSS_IP,
189 if (!rte_eth_dev_count_avail()) {
190 evt_err("No ethernet ports found.");
194 if (opt->max_pkt_sz < RTE_ETHER_MIN_LEN) {
195 evt_err("max_pkt_sz can not be less than %d",
200 port_conf.rxmode.mtu = opt->max_pkt_sz - RTE_ETHER_HDR_LEN -
203 t->internal_port = 1;
204 RTE_ETH_FOREACH_DEV(i) {
205 struct rte_eth_dev_info dev_info;
206 struct rte_eth_conf local_port_conf = port_conf;
209 ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id, i, &caps);
211 evt_err("failed to get event tx adapter[%d] caps", i);
215 if (!(caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT))
216 t->internal_port = 0;
218 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, i, &caps);
220 evt_err("failed to get event tx adapter[%d] caps", i);
224 if (!(caps & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT))
225 local_port_conf.rxmode.offloads |=
226 RTE_ETH_RX_OFFLOAD_RSS_HASH;
228 ret = rte_eth_dev_info_get(i, &dev_info);
230 evt_err("Error during getting device (port %u) info: %s\n",
235 /* Enable mbuf fast free if PMD has the capability. */
236 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
237 local_port_conf.txmode.offloads |=
238 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
240 rx_conf = dev_info.default_rxconf;
241 rx_conf.offloads = port_conf.rxmode.offloads;
243 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
244 dev_info.flow_type_rss_offloads;
245 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
246 port_conf.rx_adv_conf.rss_conf.rss_hf) {
247 evt_info("Port %u modified RSS hash function based on hardware support,"
248 "requested:%#"PRIx64" configured:%#"PRIx64"",
250 port_conf.rx_adv_conf.rss_conf.rss_hf,
251 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
254 if (rte_eth_dev_configure(i, opt->eth_queues, nb_queues,
255 &local_port_conf) < 0) {
256 evt_err("Failed to configure eth port [%d]", i);
260 for (j = 0; j < opt->eth_queues; j++) {
261 if (rte_eth_rx_queue_setup(
262 i, j, NB_RX_DESC, rte_socket_id(), &rx_conf,
263 opt->per_port_pool ? t->pool[i] :
265 evt_err("Failed to setup eth port [%d] rx_queue: %d.",
271 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC,
272 rte_socket_id(), NULL) < 0) {
273 evt_err("Failed to setup eth port [%d] tx_queue: %d.",
278 ret = rte_eth_promiscuous_enable(i);
280 evt_err("Failed to enable promiscuous mode for eth port [%d]: %s",
281 i, rte_strerror(-ret));
290 pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt,
291 uint8_t *queue_arr, uint8_t nb_queues,
292 const struct rte_event_port_conf p_conf)
296 struct test_pipeline *t = evt_test_priv(test);
299 /* setup one port per worker, linking to all queues */
300 for (port = 0; port < evt_nr_active_lcores(opt->wlcores); port++) {
301 struct worker_data *w = &t->worker[port];
303 w->dev_id = opt->dev_id;
306 w->processed_pkts = 0;
308 ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
310 evt_err("failed to setup port %d", port);
314 if (rte_event_port_link(opt->dev_id, port, queue_arr, NULL,
315 nb_queues) != nb_queues)
322 evt_err("failed to link queues to port %d", port);
327 pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride,
328 struct rte_event_port_conf prod_conf)
332 struct rte_mempool *vector_pool = NULL;
333 struct rte_event_eth_rx_adapter_queue_conf queue_conf;
335 memset(&queue_conf, 0,
336 sizeof(struct rte_event_eth_rx_adapter_queue_conf));
337 queue_conf.ev.sched_type = opt->sched_type_list[0];
338 if (opt->ena_vector) {
339 unsigned int nb_elem = (opt->pool_sz / opt->vector_size) << 1;
341 nb_elem = nb_elem ? nb_elem : 1;
342 vector_pool = rte_event_vector_pool_create(
343 "vector_pool", nb_elem, 0, opt->vector_size,
345 if (vector_pool == NULL) {
346 evt_err("failed to create event vector pool");
350 RTE_ETH_FOREACH_DEV(prod) {
351 struct rte_event_eth_rx_adapter_vector_limits limits;
354 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id,
357 evt_err("failed to get event rx adapter[%d]"
363 if (opt->ena_vector) {
364 memset(&limits, 0, sizeof(limits));
365 ret = rte_event_eth_rx_adapter_vector_limits_get(
366 opt->dev_id, prod, &limits);
368 evt_err("failed to get vector limits");
372 if (opt->vector_size < limits.min_sz ||
373 opt->vector_size > limits.max_sz) {
374 evt_err("Vector size [%d] not within limits max[%d] min[%d]",
375 opt->vector_size, limits.min_sz,
380 if (limits.log2_sz &&
381 !rte_is_power_of_2(opt->vector_size)) {
382 evt_err("Vector size [%d] not power of 2",
387 if (opt->vector_tmo_nsec > limits.max_timeout_ns ||
388 opt->vector_tmo_nsec < limits.min_timeout_ns) {
389 evt_err("Vector timeout [%" PRIu64
390 "] not within limits max[%" PRIu64
391 "] min[%" PRIu64 "]",
392 opt->vector_tmo_nsec,
393 limits.max_timeout_ns,
394 limits.min_timeout_ns);
398 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) {
399 queue_conf.vector_sz = opt->vector_size;
400 queue_conf.vector_timeout_ns =
401 opt->vector_tmo_nsec;
402 queue_conf.rx_queue_flags |=
403 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
404 queue_conf.vector_mp = vector_pool;
406 evt_err("Rx adapter doesn't support event vector");
410 queue_conf.ev.queue_id = prod * stride;
411 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id,
414 evt_err("failed to create rx adapter[%d]", prod);
417 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1,
420 evt_err("failed to add rx queues to adapter[%d]", prod);
424 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
425 uint32_t service_id = -1U;
427 rte_event_eth_rx_adapter_service_id_get(prod,
429 ret = evt_service_setup(service_id);
431 evt_err("Failed to setup service core"
437 evt_info("Port[%d] using Rx adapter[%d] configured", prod,
445 pipeline_event_tx_adapter_setup(struct evt_options *opt,
446 struct rte_event_port_conf port_conf)
451 RTE_ETH_FOREACH_DEV(consm) {
454 ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id,
457 evt_err("failed to get event tx adapter[%d] caps",
462 if (opt->ena_vector) {
464 RTE_EVENT_ETH_TX_ADAPTER_CAP_EVENT_VECTOR)) {
465 evt_err("Tx adapter doesn't support event vector");
470 ret = rte_event_eth_tx_adapter_create(consm, opt->dev_id,
473 evt_err("failed to create tx adapter[%d]", consm);
477 ret = rte_event_eth_tx_adapter_queue_add(consm, consm, -1);
479 evt_err("failed to add tx queues to adapter[%d]",
484 if (!(cap & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT)) {
485 uint32_t service_id = -1U;
487 ret = rte_event_eth_tx_adapter_service_id_get(consm,
489 if (ret != -ESRCH && ret != 0) {
490 evt_err("Failed to get Tx adptr service ID");
493 ret = evt_service_setup(service_id);
495 evt_err("Failed to setup service core"
501 evt_info("Port[%d] using Tx adapter[%d] Configured", consm,
509 pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
515 RTE_ETH_FOREACH_DEV(i) {
516 rte_event_eth_rx_adapter_stop(i);
517 rte_event_eth_tx_adapter_stop(i);
523 pipeline_eventdev_destroy(struct evt_test *test, struct evt_options *opt)
527 rte_event_dev_stop(opt->dev_id);
528 rte_event_dev_close(opt->dev_id);
532 pipeline_mempool_setup(struct evt_test *test, struct evt_options *opt)
534 struct test_pipeline *t = evt_test_priv(test);
538 opt->mbuf_sz = RTE_MBUF_DEFAULT_BUF_SIZE;
540 if (!opt->max_pkt_sz)
541 opt->max_pkt_sz = RTE_ETHER_MAX_LEN;
543 RTE_ETH_FOREACH_DEV(i) {
544 struct rte_eth_dev_info dev_info;
545 uint16_t data_size = 0;
547 memset(&dev_info, 0, sizeof(dev_info));
548 ret = rte_eth_dev_info_get(i, &dev_info);
550 evt_err("Error during getting device (port %u) info: %s\n",
555 if (dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
556 dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
557 data_size = opt->max_pkt_sz /
558 dev_info.rx_desc_lim.nb_mtu_seg_max;
559 data_size += RTE_PKTMBUF_HEADROOM;
561 if (data_size > opt->mbuf_sz)
562 opt->mbuf_sz = data_size;
564 if (opt->per_port_pool) {
565 char name[RTE_MEMPOOL_NAMESIZE];
567 snprintf(name, RTE_MEMPOOL_NAMESIZE, "%s-%d",
569 t->pool[i] = rte_pktmbuf_pool_create(
570 name, /* mempool name */
571 opt->pool_sz, /* number of elements*/
573 0, opt->mbuf_sz, opt->socket_id); /* flags */
575 if (t->pool[i] == NULL) {
576 evt_err("failed to create mempool %s", name);
582 if (!opt->per_port_pool) {
583 t->pool[0] = rte_pktmbuf_pool_create(
584 test->name, /* mempool name */
585 opt->pool_sz, /* number of elements*/
587 0, opt->mbuf_sz, opt->socket_id); /* flags */
589 if (t->pool[0] == NULL) {
590 evt_err("failed to create mempool");
599 pipeline_mempool_destroy(struct evt_test *test, struct evt_options *opt)
601 struct test_pipeline *t = evt_test_priv(test);
605 if (opt->per_port_pool) {
606 RTE_ETH_FOREACH_DEV(i)
607 rte_mempool_free(t->pool[i]);
609 rte_mempool_free(t->pool[0]);
614 pipeline_test_setup(struct evt_test *test, struct evt_options *opt)
618 test_pipeline = rte_zmalloc_socket(test->name,
619 sizeof(struct test_pipeline), RTE_CACHE_LINE_SIZE,
621 if (test_pipeline == NULL) {
622 evt_err("failed to allocate test_pipeline memory");
625 test->test_priv = test_pipeline;
627 struct test_pipeline *t = evt_test_priv(test);
629 t->nb_workers = evt_nr_active_lcores(opt->wlcores);
630 t->outstand_pkts = opt->nb_pkts * evt_nr_active_lcores(opt->wlcores);
632 t->nb_flows = opt->nb_flows;
633 t->result = EVT_TEST_FAILED;
635 opt->prod_type = EVT_PROD_TYPE_ETH_RX_ADPTR;
636 memcpy(t->sched_type_list, opt->sched_type_list,
637 sizeof(opt->sched_type_list));
644 pipeline_test_destroy(struct evt_test *test, struct evt_options *opt)
648 rte_free(test->test_priv);