app/test-eventdev/test_perf_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include "test_perf_common.h"
   6
   7 int
   8 perf_test_result(struct evt_test *test, struct evt_options *opt)
   9 {
  10         RTE_SET_USED(opt);
  11         int i;
  12         uint64_t total = 0;
  13         struct test_perf *t = evt_test_priv(test);
  14
  15         printf("Packet distribution across worker cores :\n");
  16         for (i = 0; i < t->nb_workers; i++)
  17                 total += t->worker[i].processed_pkts;
  18         for (i = 0; i < t->nb_workers; i++)
  19                 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:"
  20                                 CLGRN" %3.2f\n"CLNRM, i,
  21                                 t->worker[i].processed_pkts,
  22                                 (((double)t->worker[i].processed_pkts)/total)
  23                                 * 100);
  24
  25         return t->result;
  26 }
  27
  28 static inline int
  29 perf_producer(void *arg)
  30 {
  31         int i;
  32         struct prod_data *p  = arg;
  33         struct test_perf *t = p->t;
  34         struct evt_options *opt = t->opt;
  35         const uint8_t dev_id = p->dev_id;
  36         const uint8_t port = p->port_id;
  37         struct rte_mempool *pool = t->pool;
  38         const uint64_t nb_pkts = t->nb_pkts;
  39         const uint32_t nb_flows = t->nb_flows;
  40         uint32_t flow_counter = 0;
  41         uint64_t count = 0;
  42         struct perf_elt *m[BURST_SIZE + 1] = {NULL};
  43         struct rte_event ev;
  44
  45         if (opt->verbose_level > 1)
  46                 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__,
  47                                 rte_lcore_id(), dev_id, port, p->queue_id);
  48
  49         ev.event = 0;
  50         ev.op = RTE_EVENT_OP_NEW;
  51         ev.queue_id = p->queue_id;
  52         ev.sched_type = t->opt->sched_type_list[0];
  53         ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
  54         ev.event_type =  RTE_EVENT_TYPE_CPU;
  55         ev.sub_event_type = 0; /* stage 0 */
  56
  57         while (count < nb_pkts && t->done == false) {
  58                 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0)
  59                         continue;
  60                 for (i = 0; i < BURST_SIZE; i++) {
  61                         ev.flow_id = flow_counter++ % nb_flows;
  62                         ev.event_ptr = m[i];
  63                         m[i]->timestamp = rte_get_timer_cycles();
  64                         while (rte_event_enqueue_burst(dev_id,
  65                                                        port, &ev, 1) != 1) {
  66                                 if (t->done)
  67                                         break;
  68                                 rte_pause();
  69                                 m[i]->timestamp = rte_get_timer_cycles();
  70                         }
  71                 }
  72                 count += BURST_SIZE;
  73         }
  74
  75         return 0;
  76 }
  77
  78 static inline int
  79 perf_event_timer_producer(void *arg)
  80 {
  81         int i;
  82         struct prod_data *p  = arg;
  83         struct test_perf *t = p->t;
  84         struct evt_options *opt = t->opt;
  85         uint32_t flow_counter = 0;
  86         uint64_t count = 0;
  87         uint64_t arm_latency = 0;
  88         const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs;
  89         const uint32_t nb_flows = t->nb_flows;
  90         const uint64_t nb_timers = opt->nb_timers;
  91         struct rte_mempool *pool = t->pool;
  92         struct perf_elt *m[BURST_SIZE + 1] = {NULL};
  93         struct rte_event_timer_adapter **adptr = t->timer_adptr;
  94         struct rte_event_timer tim;
  95         uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec;
  96
  97         memset(&tim, 0, sizeof(struct rte_event_timer));
  98         timeout_ticks = opt->optm_timer_tick_nsec ?
  99                         (timeout_ticks * opt->timer_tick_nsec)
 100                         / opt->optm_timer_tick_nsec : timeout_ticks;
 101         timeout_ticks += timeout_ticks ? 0 : 1;
 102         tim.ev.event_type =  RTE_EVENT_TYPE_TIMER;
 103         tim.ev.op = RTE_EVENT_OP_NEW;
 104         tim.ev.sched_type = t->opt->sched_type_list[0];
 105         tim.ev.queue_id = p->queue_id;
 106         tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 107         tim.state = RTE_EVENT_TIMER_NOT_ARMED;
 108         tim.timeout_ticks = timeout_ticks;
 109
 110         if (opt->verbose_level > 1)
 111                 printf("%s(): lcore %d\n", __func__, rte_lcore_id());
 112
 113         while (count < nb_timers && t->done == false) {
 114                 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0)
 115                         continue;
 116                 for (i = 0; i < BURST_SIZE; i++) {
 117                         rte_prefetch0(m[i + 1]);
 118                         m[i]->tim = tim;
 119                         m[i]->tim.ev.flow_id = flow_counter++ % nb_flows;
 120                         m[i]->tim.ev.event_ptr = m[i];
 121                         m[i]->timestamp = rte_get_timer_cycles();
 122                         while (rte_event_timer_arm_burst(
 123                                adptr[flow_counter % nb_timer_adptrs],
 124                                (struct rte_event_timer **)&m[i], 1) != 1) {
 125                                 if (t->done)
 126                                         break;
 127                                 m[i]->timestamp = rte_get_timer_cycles();
 128                         }
 129                         arm_latency += rte_get_timer_cycles() - m[i]->timestamp;
 130                 }
 131                 count += BURST_SIZE;
 132         }
 133         fflush(stdout);
 134         rte_delay_ms(1000);
 135         printf("%s(): lcore %d Average event timer arm latency = %.3f us\n",
 136                         __func__, rte_lcore_id(),
 137                         count ? (float)(arm_latency / count) /
 138                         (rte_get_timer_hz() / 1000000) : 0);
 139         return 0;
 140 }
 141
 142 static inline int
 143 perf_event_timer_producer_burst(void *arg)
 144 {
 145         int i;
 146         struct prod_data *p  = arg;
 147         struct test_perf *t = p->t;
 148         struct evt_options *opt = t->opt;
 149         uint32_t flow_counter = 0;
 150         uint64_t count = 0;
 151         uint64_t arm_latency = 0;
 152         const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs;
 153         const uint32_t nb_flows = t->nb_flows;
 154         const uint64_t nb_timers = opt->nb_timers;
 155         struct rte_mempool *pool = t->pool;
 156         struct perf_elt *m[BURST_SIZE + 1] = {NULL};
 157         struct rte_event_timer_adapter **adptr = t->timer_adptr;
 158         struct rte_event_timer tim;
 159         uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec;
 160
 161         memset(&tim, 0, sizeof(struct rte_event_timer));
 162         timeout_ticks = opt->optm_timer_tick_nsec ?
 163                         (timeout_ticks * opt->timer_tick_nsec)
 164                         / opt->optm_timer_tick_nsec : timeout_ticks;
 165         timeout_ticks += timeout_ticks ? 0 : 1;
 166         tim.ev.event_type =  RTE_EVENT_TYPE_TIMER;
 167         tim.ev.op = RTE_EVENT_OP_NEW;
 168         tim.ev.sched_type = t->opt->sched_type_list[0];
 169         tim.ev.queue_id = p->queue_id;
 170         tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 171         tim.state = RTE_EVENT_TIMER_NOT_ARMED;
 172         tim.timeout_ticks = timeout_ticks;
 173
 174         if (opt->verbose_level > 1)
 175                 printf("%s(): lcore %d\n", __func__, rte_lcore_id());
 176
 177         while (count < nb_timers && t->done == false) {
 178                 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0)
 179                         continue;
 180                 for (i = 0; i < BURST_SIZE; i++) {
 181                         rte_prefetch0(m[i + 1]);
 182                         m[i]->tim = tim;
 183                         m[i]->tim.ev.flow_id = flow_counter++ % nb_flows;
 184                         m[i]->tim.ev.event_ptr = m[i];
 185                         m[i]->timestamp = rte_get_timer_cycles();
 186                 }
 187                 rte_event_timer_arm_tmo_tick_burst(
 188                                 adptr[flow_counter % nb_timer_adptrs],
 189                                 (struct rte_event_timer **)m,
 190                                 tim.timeout_ticks,
 191                                 BURST_SIZE);
 192                 arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp;
 193                 count += BURST_SIZE;
 194         }
 195         fflush(stdout);
 196         rte_delay_ms(1000);
 197         printf("%s(): lcore %d Average event timer arm latency = %.3f us\n",
 198                         __func__, rte_lcore_id(),
 199                         count ? (float)(arm_latency / count) /
 200                         (rte_get_timer_hz() / 1000000) : 0);
 201         return 0;
 202 }
 203
 204 static int
 205 perf_producer_wrapper(void *arg)
 206 {
 207         struct prod_data *p  = arg;
 208         struct test_perf *t = p->t;
 209         /* Launch the producer function only in case of synthetic producer. */
 210         if (t->opt->prod_type == EVT_PROD_TYPE_SYNT)
 211                 return perf_producer(arg);
 212         else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR &&
 213                         !t->opt->timdev_use_burst)
 214                 return perf_event_timer_producer(arg);
 215         else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR &&
 216                         t->opt->timdev_use_burst)
 217                 return perf_event_timer_producer_burst(arg);
 218         return 0;
 219 }
 220
 221 static inline uint64_t
 222 processed_pkts(struct test_perf *t)
 223 {
 224         uint8_t i;
 225         uint64_t total = 0;
 226
 227         rte_smp_rmb();
 228         for (i = 0; i < t->nb_workers; i++)
 229                 total += t->worker[i].processed_pkts;
 230
 231         return total;
 232 }
 233
 234 static inline uint64_t
 235 total_latency(struct test_perf *t)
 236 {
 237         uint8_t i;
 238         uint64_t total = 0;
 239
 240         rte_smp_rmb();
 241         for (i = 0; i < t->nb_workers; i++)
 242                 total += t->worker[i].latency;
 243
 244         return total;
 245 }
 246
 247
 248 int
 249 perf_launch_lcores(struct evt_test *test, struct evt_options *opt,
 250                 int (*worker)(void *))
 251 {
 252         int ret, lcore_id;
 253         struct test_perf *t = evt_test_priv(test);
 254
 255         int port_idx = 0;
 256         /* launch workers */
 257         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 258                 if (!(opt->wlcores[lcore_id]))
 259                         continue;
 260
 261                 ret = rte_eal_remote_launch(worker,
 262                                  &t->worker[port_idx], lcore_id);
 263                 if (ret) {
 264                         evt_err("failed to launch worker %d", lcore_id);
 265                         return ret;
 266                 }
 267                 port_idx++;
 268         }
 269
 270         /* launch producers */
 271         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 272                 if (!(opt->plcores[lcore_id]))
 273                         continue;
 274
 275                 ret = rte_eal_remote_launch(perf_producer_wrapper,
 276                                 &t->prod[port_idx], lcore_id);
 277                 if (ret) {
 278                         evt_err("failed to launch perf_producer %d", lcore_id);
 279                         return ret;
 280                 }
 281                 port_idx++;
 282         }
 283
 284         const uint64_t total_pkts = t->outstand_pkts;
 285
 286         uint64_t dead_lock_cycles = rte_get_timer_cycles();
 287         int64_t dead_lock_remaining  =  total_pkts;
 288         const uint64_t dead_lock_sample = rte_get_timer_hz() * 5;
 289
 290         uint64_t perf_cycles = rte_get_timer_cycles();
 291         int64_t perf_remaining  = total_pkts;
 292         const uint64_t perf_sample = rte_get_timer_hz();
 293
 294         static float total_mpps;
 295         static uint64_t samples;
 296
 297         const uint64_t freq_mhz = rte_get_timer_hz() / 1000000;
 298         int64_t remaining = t->outstand_pkts - processed_pkts(t);
 299
 300         while (t->done == false) {
 301                 const uint64_t new_cycles = rte_get_timer_cycles();
 302
 303                 if ((new_cycles - perf_cycles) > perf_sample) {
 304                         const uint64_t latency = total_latency(t);
 305                         const uint64_t pkts = processed_pkts(t);
 306
 307                         remaining = t->outstand_pkts - pkts;
 308                         float mpps = (float)(perf_remaining-remaining)/1000000;
 309
 310                         perf_remaining = remaining;
 311                         perf_cycles = new_cycles;
 312                         total_mpps += mpps;
 313                         ++samples;
 314                         if (opt->fwd_latency && pkts > 0) {
 315                                 printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM,
 316                                         mpps, total_mpps/samples,
 317                                         (float)(latency/pkts)/freq_mhz);
 318                         } else {
 319                                 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM,
 320                                         mpps, total_mpps/samples);
 321                         }
 322                         fflush(stdout);
 323
 324                         if (remaining <= 0) {
 325                                 t->result = EVT_TEST_SUCCESS;
 326                                 if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 327                                         opt->prod_type ==
 328                                         EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 329                                         t->done = true;
 330                                         rte_smp_wmb();
 331                                         break;
 332                                 }
 333                         }
 334                 }
 335
 336                 if (new_cycles - dead_lock_cycles > dead_lock_sample &&
 337                     (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 338                      opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR)) {
 339                         remaining = t->outstand_pkts - processed_pkts(t);
 340                         if (dead_lock_remaining == remaining) {
 341                                 rte_event_dev_dump(opt->dev_id, stdout);
 342                                 evt_err("No schedules for seconds, deadlock");
 343                                 t->done = true;
 344                                 rte_smp_wmb();
 345                                 break;
 346                         }
 347                         dead_lock_remaining = remaining;
 348                         dead_lock_cycles = new_cycles;
 349                 }
 350         }
 351         printf("\n");
 352         return 0;
 353 }
 354
 355 static int
 356 perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride,
 357                 struct rte_event_port_conf prod_conf)
 358 {
 359         int ret = 0;
 360         uint16_t prod;
 361         struct rte_event_eth_rx_adapter_queue_conf queue_conf;
 362
 363         memset(&queue_conf, 0,
 364                         sizeof(struct rte_event_eth_rx_adapter_queue_conf));
 365         queue_conf.ev.sched_type = opt->sched_type_list[0];
 366         RTE_ETH_FOREACH_DEV(prod) {
 367                 uint32_t cap;
 368
 369                 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id,
 370                                 prod, &cap);
 371                 if (ret) {
 372                         evt_err("failed to get event rx adapter[%d]"
 373                                         " capabilities",
 374                                         opt->dev_id);
 375                         return ret;
 376                 }
 377                 queue_conf.ev.queue_id = prod * stride;
 378                 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id,
 379                                 &prod_conf);
 380                 if (ret) {
 381                         evt_err("failed to create rx adapter[%d]", prod);
 382                         return ret;
 383                 }
 384                 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1,
 385                                 &queue_conf);
 386                 if (ret) {
 387                         evt_err("failed to add rx queues to adapter[%d]", prod);
 388                         return ret;
 389                 }
 390
 391                 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
 392                         uint32_t service_id;
 393
 394                         rte_event_eth_rx_adapter_service_id_get(prod,
 395                                         &service_id);
 396                         ret = evt_service_setup(service_id);
 397                         if (ret) {
 398                                 evt_err("Failed to setup service core"
 399                                                 " for Rx adapter\n");
 400                                 return ret;
 401                         }
 402                 }
 403         }
 404
 405         return ret;
 406 }
 407
 408 static int
 409 perf_event_timer_adapter_setup(struct test_perf *t)
 410 {
 411         int i;
 412         int ret;
 413         struct rte_event_timer_adapter_info adapter_info;
 414         struct rte_event_timer_adapter *wl;
 415         uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores);
 416         uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES;
 417
 418         if (nb_producers == 1)
 419                 flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT;
 420
 421         for (i = 0; i < t->opt->nb_timer_adptrs; i++) {
 422                 struct rte_event_timer_adapter_conf config = {
 423                         .event_dev_id = t->opt->dev_id,
 424                         .timer_adapter_id = i,
 425                         .timer_tick_ns = t->opt->timer_tick_nsec,
 426                         .max_tmo_ns = t->opt->max_tmo_nsec,
 427                         .nb_timers = t->opt->pool_sz,
 428                         .flags = flags,
 429                 };
 430
 431                 wl = rte_event_timer_adapter_create(&config);
 432                 if (wl == NULL) {
 433                         evt_err("failed to create event timer ring %d", i);
 434                         return rte_errno;
 435                 }
 436
 437                 memset(&adapter_info, 0,
 438                                 sizeof(struct rte_event_timer_adapter_info));
 439                 rte_event_timer_adapter_get_info(wl, &adapter_info);
 440                 t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns;
 441
 442                 if (!(adapter_info.caps &
 443                                 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) {
 444                         uint32_t service_id = -1U;
 445
 446                         rte_event_timer_adapter_service_id_get(wl,
 447                                         &service_id);
 448                         ret = evt_service_setup(service_id);
 449                         if (ret) {
 450                                 evt_err("Failed to setup service core"
 451                                                 " for timer adapter\n");
 452                                 return ret;
 453                         }
 454                         rte_service_runstate_set(service_id, 1);
 455                 }
 456                 t->timer_adptr[i] = wl;
 457         }
 458         return 0;
 459 }
 460
 461 int
 462 perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
 463                                 uint8_t stride, uint8_t nb_queues,
 464                                 const struct rte_event_port_conf *port_conf)
 465 {
 466         struct test_perf *t = evt_test_priv(test);
 467         uint16_t port, prod;
 468         int ret = -1;
 469
 470         /* setup one port per worker, linking to all queues */
 471         for (port = 0; port < evt_nr_active_lcores(opt->wlcores);
 472                                 port++) {
 473                 struct worker_data *w = &t->worker[port];
 474
 475                 w->dev_id = opt->dev_id;
 476                 w->port_id = port;
 477                 w->t = t;
 478                 w->processed_pkts = 0;
 479                 w->latency = 0;
 480
 481                 ret = rte_event_port_setup(opt->dev_id, port, port_conf);
 482                 if (ret) {
 483                         evt_err("failed to setup port %d", port);
 484                         return ret;
 485                 }
 486
 487                 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0);
 488                 if (ret != nb_queues) {
 489                         evt_err("failed to link all queues to port %d", port);
 490                         return -EINVAL;
 491                 }
 492         }
 493
 494         /* port for producers, no links */
 495         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
 496                 for ( ; port < perf_nb_event_ports(opt); port++) {
 497                         struct prod_data *p = &t->prod[port];
 498                         p->t = t;
 499                 }
 500
 501                 ret = perf_event_rx_adapter_setup(opt, stride, *port_conf);
 502                 if (ret)
 503                         return ret;
 504         } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 505                 prod = 0;
 506                 for ( ; port < perf_nb_event_ports(opt); port++) {
 507                         struct prod_data *p = &t->prod[port];
 508                         p->queue_id = prod * stride;
 509                         p->t = t;
 510                         prod++;
 511                 }
 512
 513                 ret = perf_event_timer_adapter_setup(t);
 514                 if (ret)
 515                         return ret;
 516         } else {
 517                 prod = 0;
 518                 for ( ; port < perf_nb_event_ports(opt); port++) {
 519                         struct prod_data *p = &t->prod[port];
 520
 521                         p->dev_id = opt->dev_id;
 522                         p->port_id = port;
 523                         p->queue_id = prod * stride;
 524                         p->t = t;
 525
 526                         ret = rte_event_port_setup(opt->dev_id, port,
 527                                         port_conf);
 528                         if (ret) {
 529                                 evt_err("failed to setup port %d", port);
 530                                 return ret;
 531                         }
 532                         prod++;
 533                 }
 534         }
 535
 536         return ret;
 537 }
 538
 539 int
 540 perf_opt_check(struct evt_options *opt, uint64_t nb_queues)
 541 {
 542         unsigned int lcores;
 543
 544         /* N producer + N worker + 1 master when producer cores are used
 545          * Else N worker + 1 master when Rx adapter is used
 546          */
 547         lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2;
 548
 549         if (rte_lcore_count() < lcores) {
 550                 evt_err("test need minimum %d lcores", lcores);
 551                 return -1;
 552         }
 553
 554         /* Validate worker lcores */
 555         if (evt_lcores_has_overlap(opt->wlcores, rte_get_master_lcore())) {
 556                 evt_err("worker lcores overlaps with master lcore");
 557                 return -1;
 558         }
 559         if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) {
 560                 evt_err("worker lcores overlaps producer lcores");
 561                 return -1;
 562         }
 563         if (evt_has_disabled_lcore(opt->wlcores)) {
 564                 evt_err("one or more workers lcores are not enabled");
 565                 return -1;
 566         }
 567         if (!evt_has_active_lcore(opt->wlcores)) {
 568                 evt_err("minimum one worker is required");
 569                 return -1;
 570         }
 571
 572         if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 573                         opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 574                 /* Validate producer lcores */
 575                 if (evt_lcores_has_overlap(opt->plcores,
 576                                         rte_get_master_lcore())) {
 577                         evt_err("producer lcores overlaps with master lcore");
 578                         return -1;
 579                 }
 580                 if (evt_has_disabled_lcore(opt->plcores)) {
 581                         evt_err("one or more producer lcores are not enabled");
 582                         return -1;
 583                 }
 584                 if (!evt_has_active_lcore(opt->plcores)) {
 585                         evt_err("minimum one producer is required");
 586                         return -1;
 587                 }
 588         }
 589
 590         if (evt_has_invalid_stage(opt))
 591                 return -1;
 592
 593         if (evt_has_invalid_sched_type(opt))
 594                 return -1;
 595
 596         if (nb_queues > EVT_MAX_QUEUES) {
 597                 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES);
 598                 return -1;
 599         }
 600         if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) {
 601                 evt_err("number of ports exceeds %d", EVT_MAX_PORTS);
 602                 return -1;
 603         }
 604
 605         /* Fixups */
 606         if ((opt->nb_stages == 1 &&
 607                         opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) &&
 608                         opt->fwd_latency) {
 609                 evt_info("fwd_latency is valid when nb_stages > 1, disabling");
 610                 opt->fwd_latency = 0;
 611         }
 612
 613         if (opt->fwd_latency && !opt->q_priority) {
 614                 evt_info("enabled queue priority for latency measurement");
 615                 opt->q_priority = 1;
 616         }
 617         if (opt->nb_pkts == 0)
 618                 opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores);
 619
 620         return 0;
 621 }
 622
 623 void
 624 perf_opt_dump(struct evt_options *opt, uint8_t nb_queues)
 625 {
 626         evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores));
 627         evt_dump_producer_lcores(opt);
 628         evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores));
 629         evt_dump_worker_lcores(opt);
 630         evt_dump_nb_stages(opt);
 631         evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt));
 632         evt_dump("nb_evdev_queues", "%d", nb_queues);
 633         evt_dump_queue_priority(opt);
 634         evt_dump_sched_type_list(opt);
 635         evt_dump_producer_type(opt);
 636 }
 637
 638 void
 639 perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt)
 640 {
 641         int i;
 642         struct test_perf *t = evt_test_priv(test);
 643
 644         if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 645                 for (i = 0; i < opt->nb_timer_adptrs; i++)
 646                         rte_event_timer_adapter_stop(t->timer_adptr[i]);
 647         }
 648         rte_event_dev_stop(opt->dev_id);
 649         rte_event_dev_close(opt->dev_id);
 650 }
 651
 652 static inline void
 653 perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused,
 654             void *obj, unsigned i __rte_unused)
 655 {
 656         memset(obj, 0, mp->elt_size);
 657 }
 658
 659 #define NB_RX_DESC                      128
 660 #define NB_TX_DESC                      512
 661 int
 662 perf_ethdev_setup(struct evt_test *test, struct evt_options *opt)
 663 {
 664         uint16_t i;
 665         int ret;
 666         struct test_perf *t = evt_test_priv(test);
 667         struct rte_eth_conf port_conf = {
 668                 .rxmode = {
 669                         .mq_mode = ETH_MQ_RX_RSS,
 670                         .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 671                         .split_hdr_size = 0,
 672                 },
 673                 .rx_adv_conf = {
 674                         .rss_conf = {
 675                                 .rss_key = NULL,
 676                                 .rss_hf = ETH_RSS_IP,
 677                         },
 678                 },
 679         };
 680
 681         if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 682                         opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR)
 683                 return 0;
 684
 685         if (!rte_eth_dev_count_avail()) {
 686                 evt_err("No ethernet ports found.");
 687                 return -ENODEV;
 688         }
 689
 690         RTE_ETH_FOREACH_DEV(i) {
 691                 struct rte_eth_dev_info dev_info;
 692                 struct rte_eth_conf local_port_conf = port_conf;
 693
 694                 ret = rte_eth_dev_info_get(i, &dev_info);
 695                 if (ret != 0) {
 696                         evt_err("Error during getting device (port %u) info: %s\n",
 697                                         i, strerror(-ret));
 698                         return ret;
 699                 }
 700
 701                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
 702                         dev_info.flow_type_rss_offloads;
 703                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
 704                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
 705                         evt_info("Port %u modified RSS hash function based on hardware support,"
 706                                 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
 707                                 i,
 708                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
 709                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
 710                 }
 711
 712                 if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) {
 713                         evt_err("Failed to configure eth port [%d]", i);
 714                         return -EINVAL;
 715                 }
 716
 717                 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC,
 718                                 rte_socket_id(), NULL, t->pool) < 0) {
 719                         evt_err("Failed to setup eth port [%d] rx_queue: %d.",
 720                                         i, 0);
 721                         return -EINVAL;
 722                 }
 723
 724                 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC,
 725                                         rte_socket_id(), NULL) < 0) {
 726                         evt_err("Failed to setup eth port [%d] tx_queue: %d.",
 727                                         i, 0);
 728                         return -EINVAL;
 729                 }
 730
 731                 ret = rte_eth_promiscuous_enable(i);
 732                 if (ret != 0) {
 733                         evt_err("Failed to enable promiscuous mode for eth port [%d]: %s",
 734                                 i, rte_strerror(-ret));
 735                         return ret;
 736                 }
 737         }
 738
 739         return 0;
 740 }
 741
 742 void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
 743 {
 744         uint16_t i;
 745         RTE_SET_USED(test);
 746
 747         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
 748                 RTE_ETH_FOREACH_DEV(i) {
 749                         rte_event_eth_rx_adapter_stop(i);
 750                         rte_eth_dev_stop(i);
 751                 }
 752         }
 753 }
 754
 755 int
 756 perf_mempool_setup(struct evt_test *test, struct evt_options *opt)
 757 {
 758         struct test_perf *t = evt_test_priv(test);
 759
 760         if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 761                         opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 762                 t->pool = rte_mempool_create(test->name, /* mempool name */
 763                                 opt->pool_sz, /* number of elements*/
 764                                 sizeof(struct perf_elt), /* element size*/
 765                                 512, /* cache size*/
 766                                 0, NULL, NULL,
 767                                 perf_elt_init, /* obj constructor */
 768                                 NULL, opt->socket_id, 0); /* flags */
 769         } else {
 770                 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */
 771                                 opt->pool_sz, /* number of elements*/
 772                                 512, /* cache size*/
 773                                 0,
 774                                 RTE_MBUF_DEFAULT_BUF_SIZE,
 775                                 opt->socket_id); /* flags */
 776
 777         }
 778
 779         if (t->pool == NULL) {
 780                 evt_err("failed to create mempool");
 781                 return -ENOMEM;
 782         }
 783
 784         return 0;
 785 }
 786
 787 void
 788 perf_mempool_destroy(struct evt_test *test, struct evt_options *opt)
 789 {
 790         RTE_SET_USED(opt);
 791         struct test_perf *t = evt_test_priv(test);
 792
 793         rte_mempool_free(t->pool);
 794 }
 795
 796 int
 797 perf_test_setup(struct evt_test *test, struct evt_options *opt)
 798 {
 799         void *test_perf;
 800
 801         test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf),
 802                                 RTE_CACHE_LINE_SIZE, opt->socket_id);
 803         if (test_perf  == NULL) {
 804                 evt_err("failed to allocate test_perf memory");
 805                 goto nomem;
 806         }
 807         test->test_priv = test_perf;
 808
 809         struct test_perf *t = evt_test_priv(test);
 810
 811         if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 812                 t->outstand_pkts = opt->nb_timers *
 813                         evt_nr_active_lcores(opt->plcores);
 814                 t->nb_pkts = opt->nb_timers;
 815         } else {
 816                 t->outstand_pkts = opt->nb_pkts *
 817                         evt_nr_active_lcores(opt->plcores);
 818                 t->nb_pkts = opt->nb_pkts;
 819         }
 820
 821         t->nb_workers = evt_nr_active_lcores(opt->wlcores);
 822         t->done = false;
 823         t->nb_flows = opt->nb_flows;
 824         t->result = EVT_TEST_FAILED;
 825         t->opt = opt;
 826         memcpy(t->sched_type_list, opt->sched_type_list,
 827                         sizeof(opt->sched_type_list));
 828         return 0;
 829 nomem:
 830         return -ENOMEM;
 831 }
 832
 833 void
 834 perf_test_destroy(struct evt_test *test, struct evt_options *opt)
 835 {
 836         RTE_SET_USED(opt);
 837
 838         rte_free(test->test_priv);
 839 }