app/test-eventdev/test_perf_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include <math.h>
   6
   7 #include "test_perf_common.h"
   8
   9 #define NB_CRYPTODEV_DESCRIPTORS 128
  10
  11 int
  12 perf_test_result(struct evt_test *test, struct evt_options *opt)
  13 {
  14         RTE_SET_USED(opt);
  15         int i;
  16         uint64_t total = 0;
  17         struct test_perf *t = evt_test_priv(test);
  18
  19         printf("Packet distribution across worker cores :\n");
  20         for (i = 0; i < t->nb_workers; i++)
  21                 total += t->worker[i].processed_pkts;
  22         for (i = 0; i < t->nb_workers; i++)
  23                 printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:"
  24                                 CLGRN" %3.2f"CLNRM"\n", i,
  25                                 t->worker[i].processed_pkts,
  26                                 (((double)t->worker[i].processed_pkts)/total)
  27                                 * 100);
  28
  29         return t->result;
  30 }
  31
  32 static inline int
  33 perf_producer(void *arg)
  34 {
  35         int i;
  36         struct prod_data *p  = arg;
  37         struct test_perf *t = p->t;
  38         struct evt_options *opt = t->opt;
  39         const uint8_t dev_id = p->dev_id;
  40         const uint8_t port = p->port_id;
  41         struct rte_mempool *pool = t->pool;
  42         const uint64_t nb_pkts = t->nb_pkts;
  43         const uint32_t nb_flows = t->nb_flows;
  44         uint32_t flow_counter = 0;
  45         uint64_t count = 0;
  46         struct perf_elt *m[BURST_SIZE + 1] = {NULL};
  47         struct rte_event ev;
  48
  49         if (opt->verbose_level > 1)
  50                 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__,
  51                                 rte_lcore_id(), dev_id, port, p->queue_id);
  52
  53         ev.event = 0;
  54         ev.op = RTE_EVENT_OP_NEW;
  55         ev.queue_id = p->queue_id;
  56         ev.sched_type = t->opt->sched_type_list[0];
  57         ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
  58         ev.event_type =  RTE_EVENT_TYPE_CPU;
  59         ev.sub_event_type = 0; /* stage 0 */
  60
  61         while (count < nb_pkts && t->done == false) {
  62                 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0)
  63                         continue;
  64                 for (i = 0; i < BURST_SIZE; i++) {
  65                         ev.flow_id = flow_counter++ % nb_flows;
  66                         ev.event_ptr = m[i];
  67                         m[i]->timestamp = rte_get_timer_cycles();
  68                         while (rte_event_enqueue_burst(dev_id,
  69                                                        port, &ev, 1) != 1) {
  70                                 if (t->done)
  71                                         break;
  72                                 rte_pause();
  73                                 m[i]->timestamp = rte_get_timer_cycles();
  74                         }
  75                 }
  76                 count += BURST_SIZE;
  77         }
  78
  79         return 0;
  80 }
  81
  82 static inline int
  83 perf_producer_burst(void *arg)
  84 {
  85         uint32_t i;
  86         uint64_t timestamp;
  87         struct rte_event_dev_info dev_info;
  88         struct prod_data *p  = arg;
  89         struct test_perf *t = p->t;
  90         struct evt_options *opt = t->opt;
  91         const uint8_t dev_id = p->dev_id;
  92         const uint8_t port = p->port_id;
  93         struct rte_mempool *pool = t->pool;
  94         const uint64_t nb_pkts = t->nb_pkts;
  95         const uint32_t nb_flows = t->nb_flows;
  96         uint32_t flow_counter = 0;
  97         uint16_t enq = 0;
  98         uint64_t count = 0;
  99         struct perf_elt *m[MAX_PROD_ENQ_BURST_SIZE + 1];
 100         struct rte_event ev[MAX_PROD_ENQ_BURST_SIZE + 1];
 101         uint32_t burst_size = opt->prod_enq_burst_sz;
 102
 103         memset(m, 0, sizeof(*m) * (MAX_PROD_ENQ_BURST_SIZE + 1));
 104         rte_event_dev_info_get(dev_id, &dev_info);
 105         if (dev_info.max_event_port_enqueue_depth < burst_size)
 106                 burst_size = dev_info.max_event_port_enqueue_depth;
 107
 108         if (opt->verbose_level > 1)
 109                 printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__,
 110                                 rte_lcore_id(), dev_id, port, p->queue_id);
 111
 112         for (i = 0; i < burst_size; i++) {
 113                 ev[i].op = RTE_EVENT_OP_NEW;
 114                 ev[i].queue_id = p->queue_id;
 115                 ev[i].sched_type = t->opt->sched_type_list[0];
 116                 ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 117                 ev[i].event_type =  RTE_EVENT_TYPE_CPU;
 118                 ev[i].sub_event_type = 0; /* stage 0 */
 119         }
 120
 121         while (count < nb_pkts && t->done == false) {
 122                 if (rte_mempool_get_bulk(pool, (void **)m, burst_size) < 0)
 123                         continue;
 124                 timestamp = rte_get_timer_cycles();
 125                 for (i = 0; i < burst_size; i++) {
 126                         ev[i].flow_id = flow_counter++ % nb_flows;
 127                         ev[i].event_ptr = m[i];
 128                         m[i]->timestamp = timestamp;
 129                 }
 130                 enq = rte_event_enqueue_burst(dev_id, port, ev, burst_size);
 131                 while (enq < burst_size) {
 132                         enq += rte_event_enqueue_burst(dev_id, port,
 133                                                         ev + enq,
 134                                                         burst_size - enq);
 135                         if (t->done)
 136                                 break;
 137                         rte_pause();
 138                         timestamp = rte_get_timer_cycles();
 139                         for (i = enq; i < burst_size; i++)
 140                                 m[i]->timestamp = timestamp;
 141                 }
 142                 count += burst_size;
 143         }
 144         return 0;
 145 }
 146
 147 static inline int
 148 perf_event_timer_producer(void *arg)
 149 {
 150         int i;
 151         struct prod_data *p  = arg;
 152         struct test_perf *t = p->t;
 153         struct evt_options *opt = t->opt;
 154         uint32_t flow_counter = 0;
 155         uint64_t count = 0;
 156         uint64_t arm_latency = 0;
 157         const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs;
 158         const uint32_t nb_flows = t->nb_flows;
 159         const uint64_t nb_timers = opt->nb_timers;
 160         struct rte_mempool *pool = t->pool;
 161         struct perf_elt *m[BURST_SIZE + 1] = {NULL};
 162         struct rte_event_timer_adapter **adptr = t->timer_adptr;
 163         struct rte_event_timer tim;
 164         uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec;
 165
 166         memset(&tim, 0, sizeof(struct rte_event_timer));
 167         timeout_ticks =
 168                 opt->optm_timer_tick_nsec
 169                         ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) /
 170                                opt->optm_timer_tick_nsec)
 171                         : timeout_ticks;
 172         timeout_ticks += timeout_ticks ? 0 : 1;
 173         tim.ev.event_type = RTE_EVENT_TYPE_TIMER;
 174         tim.ev.op = RTE_EVENT_OP_NEW;
 175         tim.ev.sched_type = t->opt->sched_type_list[0];
 176         tim.ev.queue_id = p->queue_id;
 177         tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 178         tim.state = RTE_EVENT_TIMER_NOT_ARMED;
 179         tim.timeout_ticks = timeout_ticks;
 180
 181         if (opt->verbose_level > 1)
 182                 printf("%s(): lcore %d\n", __func__, rte_lcore_id());
 183
 184         while (count < nb_timers && t->done == false) {
 185                 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0)
 186                         continue;
 187                 for (i = 0; i < BURST_SIZE; i++) {
 188                         rte_prefetch0(m[i + 1]);
 189                         m[i]->tim = tim;
 190                         m[i]->tim.ev.flow_id = flow_counter++ % nb_flows;
 191                         m[i]->tim.ev.event_ptr = m[i];
 192                         m[i]->timestamp = rte_get_timer_cycles();
 193                         while (rte_event_timer_arm_burst(
 194                                adptr[flow_counter % nb_timer_adptrs],
 195                                (struct rte_event_timer **)&m[i], 1) != 1) {
 196                                 if (t->done)
 197                                         break;
 198                                 m[i]->timestamp = rte_get_timer_cycles();
 199                         }
 200                         arm_latency += rte_get_timer_cycles() - m[i]->timestamp;
 201                 }
 202                 count += BURST_SIZE;
 203         }
 204         fflush(stdout);
 205         rte_delay_ms(1000);
 206         printf("%s(): lcore %d Average event timer arm latency = %.3f us\n",
 207                         __func__, rte_lcore_id(),
 208                         count ? (float)(arm_latency / count) /
 209                         (rte_get_timer_hz() / 1000000) : 0);
 210         return 0;
 211 }
 212
 213 static inline int
 214 perf_event_timer_producer_burst(void *arg)
 215 {
 216         int i;
 217         struct prod_data *p  = arg;
 218         struct test_perf *t = p->t;
 219         struct evt_options *opt = t->opt;
 220         uint32_t flow_counter = 0;
 221         uint64_t count = 0;
 222         uint64_t arm_latency = 0;
 223         const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs;
 224         const uint32_t nb_flows = t->nb_flows;
 225         const uint64_t nb_timers = opt->nb_timers;
 226         struct rte_mempool *pool = t->pool;
 227         struct perf_elt *m[BURST_SIZE + 1] = {NULL};
 228         struct rte_event_timer_adapter **adptr = t->timer_adptr;
 229         struct rte_event_timer tim;
 230         uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec;
 231
 232         memset(&tim, 0, sizeof(struct rte_event_timer));
 233         timeout_ticks =
 234                 opt->optm_timer_tick_nsec
 235                         ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) /
 236                                opt->optm_timer_tick_nsec)
 237                         : timeout_ticks;
 238         timeout_ticks += timeout_ticks ? 0 : 1;
 239         tim.ev.event_type = RTE_EVENT_TYPE_TIMER;
 240         tim.ev.op = RTE_EVENT_OP_NEW;
 241         tim.ev.sched_type = t->opt->sched_type_list[0];
 242         tim.ev.queue_id = p->queue_id;
 243         tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
 244         tim.state = RTE_EVENT_TIMER_NOT_ARMED;
 245         tim.timeout_ticks = timeout_ticks;
 246
 247         if (opt->verbose_level > 1)
 248                 printf("%s(): lcore %d\n", __func__, rte_lcore_id());
 249
 250         while (count < nb_timers && t->done == false) {
 251                 if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0)
 252                         continue;
 253                 for (i = 0; i < BURST_SIZE; i++) {
 254                         rte_prefetch0(m[i + 1]);
 255                         m[i]->tim = tim;
 256                         m[i]->tim.ev.flow_id = flow_counter++ % nb_flows;
 257                         m[i]->tim.ev.event_ptr = m[i];
 258                         m[i]->timestamp = rte_get_timer_cycles();
 259                 }
 260                 rte_event_timer_arm_tmo_tick_burst(
 261                                 adptr[flow_counter % nb_timer_adptrs],
 262                                 (struct rte_event_timer **)m,
 263                                 tim.timeout_ticks,
 264                                 BURST_SIZE);
 265                 arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp;
 266                 count += BURST_SIZE;
 267         }
 268         fflush(stdout);
 269         rte_delay_ms(1000);
 270         printf("%s(): lcore %d Average event timer arm latency = %.3f us\n",
 271                         __func__, rte_lcore_id(),
 272                         count ? (float)(arm_latency / count) /
 273                         (rte_get_timer_hz() / 1000000) : 0);
 274         return 0;
 275 }
 276
 277 static inline void
 278 crypto_adapter_enq_op_new(struct prod_data *p)
 279 {
 280         struct rte_cryptodev_sym_session **crypto_sess = p->ca.crypto_sess;
 281         struct test_perf *t = p->t;
 282         const uint32_t nb_flows = t->nb_flows;
 283         const uint64_t nb_pkts = t->nb_pkts;
 284         struct rte_mempool *pool = t->pool;
 285         struct rte_crypto_sym_op *sym_op;
 286         struct evt_options *opt = t->opt;
 287         uint16_t qp_id = p->ca.cdev_qp_id;
 288         uint8_t cdev_id = p->ca.cdev_id;
 289         uint32_t flow_counter = 0;
 290         struct rte_crypto_op *op;
 291         struct rte_mbuf *m;
 292         uint64_t count = 0;
 293         uint16_t len;
 294
 295         if (opt->verbose_level > 1)
 296                 printf("%s(): lcore %d queue %d cdev_id %u cdev_qp_id %u\n",
 297                        __func__, rte_lcore_id(), p->queue_id, p->ca.cdev_id,
 298                        p->ca.cdev_qp_id);
 299
 300         len = opt->mbuf_sz ? opt->mbuf_sz : RTE_ETHER_MIN_LEN;
 301
 302         while (count < nb_pkts && t->done == false) {
 303                 m = rte_pktmbuf_alloc(pool);
 304                 if (m == NULL)
 305                         continue;
 306
 307                 rte_pktmbuf_append(m, len);
 308                 op = rte_crypto_op_alloc(t->ca_op_pool,
 309                                          RTE_CRYPTO_OP_TYPE_SYMMETRIC);
 310                 sym_op = op->sym;
 311                 sym_op->m_src = m;
 312                 sym_op->cipher.data.offset = 0;
 313                 sym_op->cipher.data.length = len;
 314                 rte_crypto_op_attach_sym_session(
 315                         op, crypto_sess[flow_counter++ % nb_flows]);
 316
 317                 while (rte_cryptodev_enqueue_burst(cdev_id, qp_id, &op, 1) != 1 &&
 318                        t->done == false)
 319                         rte_pause();
 320
 321                 count++;
 322         }
 323 }
 324
 325 static inline void
 326 crypto_adapter_enq_op_fwd(struct prod_data *p)
 327 {
 328         struct rte_cryptodev_sym_session **crypto_sess = p->ca.crypto_sess;
 329         const uint8_t dev_id = p->dev_id;
 330         const uint8_t port = p->port_id;
 331         struct test_perf *t = p->t;
 332         const uint32_t nb_flows = t->nb_flows;
 333         const uint64_t nb_pkts = t->nb_pkts;
 334         struct rte_mempool *pool = t->pool;
 335         struct evt_options *opt = t->opt;
 336         struct rte_crypto_sym_op *sym_op;
 337         uint32_t flow_counter = 0;
 338         struct rte_crypto_op *op;
 339         struct rte_event ev;
 340         struct rte_mbuf *m;
 341         uint64_t count = 0;
 342         uint16_t len;
 343
 344         if (opt->verbose_level > 1)
 345                 printf("%s(): lcore %d port %d queue %d cdev_id %u cdev_qp_id %u\n",
 346                        __func__, rte_lcore_id(), port, p->queue_id,
 347                        p->ca.cdev_id, p->ca.cdev_qp_id);
 348
 349         ev.event = 0;
 350         ev.op = RTE_EVENT_OP_NEW;
 351         ev.queue_id = p->queue_id;
 352         ev.sched_type = RTE_SCHED_TYPE_ATOMIC;
 353         ev.event_type = RTE_EVENT_TYPE_CPU;
 354         len = opt->mbuf_sz ? opt->mbuf_sz : RTE_ETHER_MIN_LEN;
 355
 356         while (count < nb_pkts && t->done == false) {
 357                 m = rte_pktmbuf_alloc(pool);
 358                 if (m == NULL)
 359                         continue;
 360
 361                 rte_pktmbuf_append(m, len);
 362                 op = rte_crypto_op_alloc(t->ca_op_pool,
 363                                          RTE_CRYPTO_OP_TYPE_SYMMETRIC);
 364                 sym_op = op->sym;
 365                 sym_op->m_src = m;
 366                 sym_op->cipher.data.offset = 0;
 367                 sym_op->cipher.data.length = len;
 368                 rte_crypto_op_attach_sym_session(
 369                         op, crypto_sess[flow_counter++ % nb_flows]);
 370                 ev.event_ptr = op;
 371
 372                 while (rte_event_crypto_adapter_enqueue(dev_id, port, &ev, 1) != 1 &&
 373                        t->done == false)
 374                         rte_pause();
 375
 376                 count++;
 377         }
 378 }
 379
 380 static inline int
 381 perf_event_crypto_producer(void *arg)
 382 {
 383         struct prod_data *p = arg;
 384         struct evt_options *opt = p->t->opt;
 385
 386         if (opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)
 387                 crypto_adapter_enq_op_new(p);
 388         else
 389                 crypto_adapter_enq_op_fwd(p);
 390
 391         return 0;
 392 }
 393
 394 static int
 395 perf_producer_wrapper(void *arg)
 396 {
 397         struct prod_data *p  = arg;
 398         struct test_perf *t = p->t;
 399         bool burst = evt_has_burst_mode(p->dev_id);
 400
 401         /* In case of synthetic producer, launch perf_producer or
 402          * perf_producer_burst depending on producer enqueue burst size
 403          */
 404         if (t->opt->prod_type == EVT_PROD_TYPE_SYNT &&
 405                         t->opt->prod_enq_burst_sz == 1)
 406                 return perf_producer(arg);
 407         else if (t->opt->prod_type == EVT_PROD_TYPE_SYNT &&
 408                         t->opt->prod_enq_burst_sz > 1) {
 409                 if (!burst)
 410                         evt_err("This event device does not support burst mode");
 411                 else
 412                         return perf_producer_burst(arg);
 413         }
 414         else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR &&
 415                         !t->opt->timdev_use_burst)
 416                 return perf_event_timer_producer(arg);
 417         else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR &&
 418                         t->opt->timdev_use_burst)
 419                 return perf_event_timer_producer_burst(arg);
 420         else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
 421                 return perf_event_crypto_producer(arg);
 422         return 0;
 423 }
 424
 425 static inline uint64_t
 426 processed_pkts(struct test_perf *t)
 427 {
 428         uint8_t i;
 429         uint64_t total = 0;
 430
 431         for (i = 0; i < t->nb_workers; i++)
 432                 total += t->worker[i].processed_pkts;
 433
 434         return total;
 435 }
 436
 437 static inline uint64_t
 438 total_latency(struct test_perf *t)
 439 {
 440         uint8_t i;
 441         uint64_t total = 0;
 442
 443         for (i = 0; i < t->nb_workers; i++)
 444                 total += t->worker[i].latency;
 445
 446         return total;
 447 }
 448
 449
 450 int
 451 perf_launch_lcores(struct evt_test *test, struct evt_options *opt,
 452                 int (*worker)(void *))
 453 {
 454         int ret, lcore_id;
 455         struct test_perf *t = evt_test_priv(test);
 456
 457         int port_idx = 0;
 458         /* launch workers */
 459         RTE_LCORE_FOREACH_WORKER(lcore_id) {
 460                 if (!(opt->wlcores[lcore_id]))
 461                         continue;
 462
 463                 ret = rte_eal_remote_launch(worker,
 464                                  &t->worker[port_idx], lcore_id);
 465                 if (ret) {
 466                         evt_err("failed to launch worker %d", lcore_id);
 467                         return ret;
 468                 }
 469                 port_idx++;
 470         }
 471
 472         /* launch producers */
 473         RTE_LCORE_FOREACH_WORKER(lcore_id) {
 474                 if (!(opt->plcores[lcore_id]))
 475                         continue;
 476
 477                 ret = rte_eal_remote_launch(perf_producer_wrapper,
 478                                 &t->prod[port_idx], lcore_id);
 479                 if (ret) {
 480                         evt_err("failed to launch perf_producer %d", lcore_id);
 481                         return ret;
 482                 }
 483                 port_idx++;
 484         }
 485
 486         const uint64_t total_pkts = t->outstand_pkts;
 487
 488         uint64_t dead_lock_cycles = rte_get_timer_cycles();
 489         int64_t dead_lock_remaining  =  total_pkts;
 490         const uint64_t dead_lock_sample = rte_get_timer_hz() * 5;
 491
 492         uint64_t perf_cycles = rte_get_timer_cycles();
 493         int64_t perf_remaining  = total_pkts;
 494         const uint64_t perf_sample = rte_get_timer_hz();
 495
 496         static float total_mpps;
 497         static uint64_t samples;
 498
 499         const uint64_t freq_mhz = rte_get_timer_hz() / 1000000;
 500         int64_t remaining = t->outstand_pkts - processed_pkts(t);
 501
 502         while (t->done == false) {
 503                 const uint64_t new_cycles = rte_get_timer_cycles();
 504
 505                 if ((new_cycles - perf_cycles) > perf_sample) {
 506                         const uint64_t latency = total_latency(t);
 507                         const uint64_t pkts = processed_pkts(t);
 508
 509                         remaining = t->outstand_pkts - pkts;
 510                         float mpps = (float)(perf_remaining-remaining)/1000000;
 511
 512                         perf_remaining = remaining;
 513                         perf_cycles = new_cycles;
 514                         total_mpps += mpps;
 515                         ++samples;
 516                         if (opt->fwd_latency && pkts > 0) {
 517                                 printf(CLGRN"\r%.3f mpps avg %.3f mpps [avg fwd latency %.3f us] "CLNRM,
 518                                         mpps, total_mpps/samples,
 519                                         (float)(latency/pkts)/freq_mhz);
 520                         } else {
 521                                 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM,
 522                                         mpps, total_mpps/samples);
 523                         }
 524                         fflush(stdout);
 525
 526                         if (remaining <= 0) {
 527                                 t->result = EVT_TEST_SUCCESS;
 528                                 if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 529                                     opt->prod_type ==
 530                                             EVT_PROD_TYPE_EVENT_TIMER_ADPTR ||
 531                                     opt->prod_type ==
 532                                             EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) {
 533                                         t->done = true;
 534                                         break;
 535                                 }
 536                         }
 537                 }
 538
 539                 if (new_cycles - dead_lock_cycles > dead_lock_sample &&
 540                     (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 541                      opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR ||
 542                      opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)) {
 543                         remaining = t->outstand_pkts - processed_pkts(t);
 544                         if (dead_lock_remaining == remaining) {
 545                                 rte_event_dev_dump(opt->dev_id, stdout);
 546                                 evt_err("No schedules for seconds, deadlock");
 547                                 t->done = true;
 548                                 break;
 549                         }
 550                         dead_lock_remaining = remaining;
 551                         dead_lock_cycles = new_cycles;
 552                 }
 553         }
 554         printf("\n");
 555         return 0;
 556 }
 557
 558 static int
 559 perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride,
 560                 struct rte_event_port_conf prod_conf)
 561 {
 562         int ret = 0;
 563         uint16_t prod;
 564         struct rte_event_eth_rx_adapter_queue_conf queue_conf;
 565
 566         memset(&queue_conf, 0,
 567                         sizeof(struct rte_event_eth_rx_adapter_queue_conf));
 568         queue_conf.ev.sched_type = opt->sched_type_list[0];
 569         RTE_ETH_FOREACH_DEV(prod) {
 570                 uint32_t cap;
 571
 572                 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id,
 573                                 prod, &cap);
 574                 if (ret) {
 575                         evt_err("failed to get event rx adapter[%d]"
 576                                         " capabilities",
 577                                         opt->dev_id);
 578                         return ret;
 579                 }
 580                 queue_conf.ev.queue_id = prod * stride;
 581                 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id,
 582                                 &prod_conf);
 583                 if (ret) {
 584                         evt_err("failed to create rx adapter[%d]", prod);
 585                         return ret;
 586                 }
 587                 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1,
 588                                 &queue_conf);
 589                 if (ret) {
 590                         evt_err("failed to add rx queues to adapter[%d]", prod);
 591                         return ret;
 592                 }
 593
 594                 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
 595                         uint32_t service_id;
 596
 597                         rte_event_eth_rx_adapter_service_id_get(prod,
 598                                         &service_id);
 599                         ret = evt_service_setup(service_id);
 600                         if (ret) {
 601                                 evt_err("Failed to setup service core"
 602                                                 " for Rx adapter\n");
 603                                 return ret;
 604                         }
 605                 }
 606         }
 607
 608         return ret;
 609 }
 610
 611 static int
 612 perf_event_timer_adapter_setup(struct test_perf *t)
 613 {
 614         int i;
 615         int ret;
 616         struct rte_event_timer_adapter_info adapter_info;
 617         struct rte_event_timer_adapter *wl;
 618         uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores);
 619         uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES;
 620
 621         if (nb_producers == 1)
 622                 flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT;
 623
 624         for (i = 0; i < t->opt->nb_timer_adptrs; i++) {
 625                 struct rte_event_timer_adapter_conf config = {
 626                         .event_dev_id = t->opt->dev_id,
 627                         .timer_adapter_id = i,
 628                         .timer_tick_ns = t->opt->timer_tick_nsec,
 629                         .max_tmo_ns = t->opt->max_tmo_nsec,
 630                         .nb_timers = t->opt->pool_sz,
 631                         .flags = flags,
 632                 };
 633
 634                 wl = rte_event_timer_adapter_create(&config);
 635                 if (wl == NULL) {
 636                         evt_err("failed to create event timer ring %d", i);
 637                         return rte_errno;
 638                 }
 639
 640                 memset(&adapter_info, 0,
 641                                 sizeof(struct rte_event_timer_adapter_info));
 642                 rte_event_timer_adapter_get_info(wl, &adapter_info);
 643                 t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns;
 644
 645                 if (!(adapter_info.caps &
 646                                 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) {
 647                         uint32_t service_id = -1U;
 648
 649                         rte_event_timer_adapter_service_id_get(wl,
 650                                         &service_id);
 651                         ret = evt_service_setup(service_id);
 652                         if (ret) {
 653                                 evt_err("Failed to setup service core"
 654                                                 " for timer adapter\n");
 655                                 return ret;
 656                         }
 657                         rte_service_runstate_set(service_id, 1);
 658                 }
 659                 t->timer_adptr[i] = wl;
 660         }
 661         return 0;
 662 }
 663
 664 static int
 665 perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p)
 666 {
 667         struct evt_options *opt = t->opt;
 668         uint32_t cap;
 669         int ret;
 670
 671         ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id, &cap);
 672         if (ret) {
 673                 evt_err("Failed to get crypto adapter capabilities");
 674                 return ret;
 675         }
 676
 677         if (((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) &&
 678              !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW)) ||
 679             ((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) &&
 680              !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD))) {
 681                 evt_err("crypto adapter %s mode unsupported\n",
 682                         opt->crypto_adptr_mode ? "OP_FORWARD" : "OP_NEW");
 683                 return -ENOTSUP;
 684         } else if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA)) {
 685                 evt_err("Storing crypto session not supported");
 686                 return -ENOTSUP;
 687         }
 688
 689         if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) {
 690                 struct rte_event response_info;
 691
 692                 response_info.event = 0;
 693                 response_info.sched_type = RTE_SCHED_TYPE_ATOMIC;
 694                 response_info.queue_id = p->queue_id;
 695                 ret = rte_event_crypto_adapter_queue_pair_add(
 696                         TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id,
 697                         &response_info);
 698         } else {
 699                 ret = rte_event_crypto_adapter_queue_pair_add(
 700                         TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, NULL);
 701         }
 702
 703         return ret;
 704 }
 705
 706 static struct rte_cryptodev_sym_session *
 707 cryptodev_sym_sess_create(struct prod_data *p, struct test_perf *t)
 708 {
 709         struct rte_crypto_sym_xform cipher_xform;
 710         struct rte_cryptodev_sym_session *sess;
 711
 712         cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
 713         cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 714         cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 715         cipher_xform.next = NULL;
 716
 717         sess = rte_cryptodev_sym_session_create(t->ca_sess_pool);
 718         if (sess == NULL) {
 719                 evt_err("Failed to create sym session");
 720                 return NULL;
 721         }
 722
 723         if (rte_cryptodev_sym_session_init(p->ca.cdev_id, sess, &cipher_xform,
 724                                            t->ca_sess_priv_pool)) {
 725                 evt_err("Failed to init session");
 726                 return NULL;
 727         }
 728
 729         return sess;
 730 }
 731
 732 int
 733 perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
 734                                 uint8_t stride, uint8_t nb_queues,
 735                                 const struct rte_event_port_conf *port_conf)
 736 {
 737         struct test_perf *t = evt_test_priv(test);
 738         uint16_t port, prod;
 739         int ret = -1;
 740
 741         /* setup one port per worker, linking to all queues */
 742         for (port = 0; port < evt_nr_active_lcores(opt->wlcores);
 743                                 port++) {
 744                 struct worker_data *w = &t->worker[port];
 745
 746                 w->dev_id = opt->dev_id;
 747                 w->port_id = port;
 748                 w->t = t;
 749                 w->processed_pkts = 0;
 750                 w->latency = 0;
 751
 752                 struct rte_event_port_conf conf = *port_conf;
 753                 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_WORKER;
 754
 755                 ret = rte_event_port_setup(opt->dev_id, port, &conf);
 756                 if (ret) {
 757                         evt_err("failed to setup port %d", port);
 758                         return ret;
 759                 }
 760
 761                 ret = rte_event_port_link(opt->dev_id, port, NULL, NULL, 0);
 762                 if (ret != nb_queues) {
 763                         evt_err("failed to link all queues to port %d", port);
 764                         return -EINVAL;
 765                 }
 766         }
 767
 768         /* port for producers, no links */
 769         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
 770                 for ( ; port < perf_nb_event_ports(opt); port++) {
 771                         struct prod_data *p = &t->prod[port];
 772                         p->t = t;
 773                 }
 774
 775                 struct rte_event_port_conf conf = *port_conf;
 776                 conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_PRODUCER;
 777
 778                 ret = perf_event_rx_adapter_setup(opt, stride, conf);
 779                 if (ret)
 780                         return ret;
 781         } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
 782                 prod = 0;
 783                 for ( ; port < perf_nb_event_ports(opt); port++) {
 784                         struct prod_data *p = &t->prod[port];
 785                         p->queue_id = prod * stride;
 786                         p->t = t;
 787                         prod++;
 788                 }
 789
 790                 ret = perf_event_timer_adapter_setup(t);
 791                 if (ret)
 792                         return ret;
 793         } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) {
 794                 struct rte_event_port_conf conf = *port_conf;
 795                 uint8_t cdev_id = 0;
 796                 uint16_t qp_id = 0;
 797
 798                 ret = rte_event_crypto_adapter_create(TEST_PERF_CA_ID,
 799                                                       opt->dev_id, &conf, 0);
 800                 if (ret) {
 801                         evt_err("Failed to create crypto adapter");
 802                         return ret;
 803                 }
 804
 805                 prod = 0;
 806                 for (; port < perf_nb_event_ports(opt); port++) {
 807                         struct rte_cryptodev_sym_session *crypto_sess;
 808                         union rte_event_crypto_metadata m_data;
 809                         struct prod_data *p = &t->prod[port];
 810                         uint32_t flow_id;
 811
 812                         if (qp_id == rte_cryptodev_queue_pair_count(cdev_id)) {
 813                                 cdev_id++;
 814                                 qp_id = 0;
 815                         }
 816
 817                         p->dev_id = opt->dev_id;
 818                         p->port_id = port;
 819                         p->queue_id = prod * stride;
 820                         p->ca.cdev_id = cdev_id;
 821                         p->ca.cdev_qp_id = qp_id;
 822                         p->ca.crypto_sess = rte_zmalloc_socket(
 823                                 NULL, sizeof(crypto_sess) * t->nb_flows,
 824                                 RTE_CACHE_LINE_SIZE, opt->socket_id);
 825                         p->t = t;
 826
 827                         m_data.request_info.cdev_id = p->ca.cdev_id;
 828                         m_data.request_info.queue_pair_id = p->ca.cdev_qp_id;
 829                         m_data.response_info.sched_type = RTE_SCHED_TYPE_ATOMIC;
 830                         m_data.response_info.queue_id = p->queue_id;
 831
 832                         for (flow_id = 0; flow_id < t->nb_flows; flow_id++) {
 833                                 crypto_sess = cryptodev_sym_sess_create(p, t);
 834                                 if (crypto_sess == NULL)
 835                                         return -ENOMEM;
 836
 837                                 m_data.response_info.flow_id = flow_id;
 838                                 rte_cryptodev_session_event_mdata_set(cdev_id,
 839                                                 crypto_sess,
 840                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 841                                                 RTE_CRYPTO_OP_WITH_SESSION,
 842                                                 &m_data, sizeof(m_data));
 843
 844                                 p->ca.crypto_sess[flow_id] = crypto_sess;
 845                         }
 846
 847                         conf.event_port_cfg |=
 848                                 RTE_EVENT_PORT_CFG_HINT_PRODUCER |
 849                                 RTE_EVENT_PORT_CFG_HINT_CONSUMER;
 850
 851                         ret = rte_event_port_setup(opt->dev_id, port, &conf);
 852                         if (ret) {
 853                                 evt_err("failed to setup port %d", port);
 854                                 return ret;
 855                         }
 856
 857                         ret = perf_event_crypto_adapter_setup(t, p);
 858                         if (ret)
 859                                 return ret;
 860
 861                         qp_id++;
 862                         prod++;
 863                 }
 864         } else {
 865                 prod = 0;
 866                 for ( ; port < perf_nb_event_ports(opt); port++) {
 867                         struct prod_data *p = &t->prod[port];
 868
 869                         p->dev_id = opt->dev_id;
 870                         p->port_id = port;
 871                         p->queue_id = prod * stride;
 872                         p->t = t;
 873
 874                         struct rte_event_port_conf conf = *port_conf;
 875                         conf.event_port_cfg |=
 876                                 RTE_EVENT_PORT_CFG_HINT_PRODUCER |
 877                                 RTE_EVENT_PORT_CFG_HINT_CONSUMER;
 878
 879                         ret = rte_event_port_setup(opt->dev_id, port, &conf);
 880                         if (ret) {
 881                                 evt_err("failed to setup port %d", port);
 882                                 return ret;
 883                         }
 884                         prod++;
 885                 }
 886         }
 887
 888         return ret;
 889 }
 890
 891 int
 892 perf_opt_check(struct evt_options *opt, uint64_t nb_queues)
 893 {
 894         unsigned int lcores;
 895
 896         /* N producer + N worker + main when producer cores are used
 897          * Else N worker + main when Rx adapter is used
 898          */
 899         lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2;
 900
 901         if (rte_lcore_count() < lcores) {
 902                 evt_err("test need minimum %d lcores", lcores);
 903                 return -1;
 904         }
 905
 906         /* Validate worker lcores */
 907         if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) {
 908                 evt_err("worker lcores overlaps with main lcore");
 909                 return -1;
 910         }
 911         if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) {
 912                 evt_err("worker lcores overlaps producer lcores");
 913                 return -1;
 914         }
 915         if (evt_has_disabled_lcore(opt->wlcores)) {
 916                 evt_err("one or more workers lcores are not enabled");
 917                 return -1;
 918         }
 919         if (!evt_has_active_lcore(opt->wlcores)) {
 920                 evt_err("minimum one worker is required");
 921                 return -1;
 922         }
 923
 924         if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
 925             opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR ||
 926             opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) {
 927                 /* Validate producer lcores */
 928                 if (evt_lcores_has_overlap(opt->plcores,
 929                                         rte_get_main_lcore())) {
 930                         evt_err("producer lcores overlaps with main lcore");
 931                         return -1;
 932                 }
 933                 if (evt_has_disabled_lcore(opt->plcores)) {
 934                         evt_err("one or more producer lcores are not enabled");
 935                         return -1;
 936                 }
 937                 if (!evt_has_active_lcore(opt->plcores)) {
 938                         evt_err("minimum one producer is required");
 939                         return -1;
 940                 }
 941         }
 942
 943         if (evt_has_invalid_stage(opt))
 944                 return -1;
 945
 946         if (evt_has_invalid_sched_type(opt))
 947                 return -1;
 948
 949         if (nb_queues > EVT_MAX_QUEUES) {
 950                 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES);
 951                 return -1;
 952         }
 953         if (perf_nb_event_ports(opt) > EVT_MAX_PORTS) {
 954                 evt_err("number of ports exceeds %d", EVT_MAX_PORTS);
 955                 return -1;
 956         }
 957
 958         /* Fixups */
 959         if ((opt->nb_stages == 1 &&
 960                         opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) &&
 961                         opt->fwd_latency) {
 962                 evt_info("fwd_latency is valid when nb_stages > 1, disabling");
 963                 opt->fwd_latency = 0;
 964         }
 965
 966         if (opt->fwd_latency && !opt->q_priority) {
 967                 evt_info("enabled queue priority for latency measurement");
 968                 opt->q_priority = 1;
 969         }
 970         if (opt->nb_pkts == 0)
 971                 opt->nb_pkts = INT64_MAX/evt_nr_active_lcores(opt->plcores);
 972
 973         return 0;
 974 }
 975
 976 void
 977 perf_opt_dump(struct evt_options *opt, uint8_t nb_queues)
 978 {
 979         evt_dump("nb_prod_lcores", "%d", evt_nr_active_lcores(opt->plcores));
 980         evt_dump_producer_lcores(opt);
 981         evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores));
 982         evt_dump_worker_lcores(opt);
 983         evt_dump_nb_stages(opt);
 984         evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt));
 985         evt_dump("nb_evdev_queues", "%d", nb_queues);
 986         evt_dump_queue_priority(opt);
 987         evt_dump_sched_type_list(opt);
 988         evt_dump_producer_type(opt);
 989         evt_dump("prod_enq_burst_sz", "%d", opt->prod_enq_burst_sz);
 990 }
 991
 992 static void
 993 perf_event_port_flush(uint8_t dev_id __rte_unused, struct rte_event ev,
 994                       void *args)
 995 {
 996         rte_mempool_put(args, ev.event_ptr);
 997 }
 998
 999 void
1000 perf_worker_cleanup(struct rte_mempool *const pool, uint8_t dev_id,
1001                     uint8_t port_id, struct rte_event events[], uint16_t nb_enq,
1002                     uint16_t nb_deq)
1003 {
1004         int i;
1005
1006         if (nb_deq) {
1007                 for (i = nb_enq; i < nb_deq; i++)
1008                         rte_mempool_put(pool, events[i].event_ptr);
1009
1010                 for (i = 0; i < nb_deq; i++)
1011                         events[i].op = RTE_EVENT_OP_RELEASE;
1012                 rte_event_enqueue_burst(dev_id, port_id, events, nb_deq);
1013         }
1014         rte_event_port_quiesce(dev_id, port_id, perf_event_port_flush, pool);
1015 }
1016
1017 void
1018 perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt)
1019 {
1020         int i;
1021         struct test_perf *t = evt_test_priv(test);
1022
1023         if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
1024                 for (i = 0; i < opt->nb_timer_adptrs; i++)
1025                         rte_event_timer_adapter_stop(t->timer_adptr[i]);
1026         }
1027         rte_event_dev_stop(opt->dev_id);
1028         rte_event_dev_close(opt->dev_id);
1029 }
1030
1031 static inline void
1032 perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused,
1033             void *obj, unsigned i __rte_unused)
1034 {
1035         memset(obj, 0, mp->elt_size);
1036 }
1037
1038 #define NB_RX_DESC                      128
1039 #define NB_TX_DESC                      512
1040 int
1041 perf_ethdev_setup(struct evt_test *test, struct evt_options *opt)
1042 {
1043         uint16_t i;
1044         int ret;
1045         struct test_perf *t = evt_test_priv(test);
1046         struct rte_eth_conf port_conf = {
1047                 .rxmode = {
1048                         .mq_mode = RTE_ETH_MQ_RX_RSS,
1049                         .split_hdr_size = 0,
1050                 },
1051                 .rx_adv_conf = {
1052                         .rss_conf = {
1053                                 .rss_key = NULL,
1054                                 .rss_hf = RTE_ETH_RSS_IP,
1055                         },
1056                 },
1057         };
1058
1059         if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR)
1060                 return 0;
1061
1062         if (!rte_eth_dev_count_avail()) {
1063                 evt_err("No ethernet ports found.");
1064                 return -ENODEV;
1065         }
1066
1067         RTE_ETH_FOREACH_DEV(i) {
1068                 struct rte_eth_dev_info dev_info;
1069                 struct rte_eth_conf local_port_conf = port_conf;
1070
1071                 ret = rte_eth_dev_info_get(i, &dev_info);
1072                 if (ret != 0) {
1073                         evt_err("Error during getting device (port %u) info: %s\n",
1074                                         i, strerror(-ret));
1075                         return ret;
1076                 }
1077
1078                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
1079                         dev_info.flow_type_rss_offloads;
1080                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
1081                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
1082                         evt_info("Port %u modified RSS hash function based on hardware support,"
1083                                 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
1084                                 i,
1085                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
1086                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
1087                 }
1088
1089                 if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) {
1090                         evt_err("Failed to configure eth port [%d]", i);
1091                         return -EINVAL;
1092                 }
1093
1094                 if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC,
1095                                 rte_socket_id(), NULL, t->pool) < 0) {
1096                         evt_err("Failed to setup eth port [%d] rx_queue: %d.",
1097                                         i, 0);
1098                         return -EINVAL;
1099                 }
1100
1101                 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC,
1102                                         rte_socket_id(), NULL) < 0) {
1103                         evt_err("Failed to setup eth port [%d] tx_queue: %d.",
1104                                         i, 0);
1105                         return -EINVAL;
1106                 }
1107
1108                 ret = rte_eth_promiscuous_enable(i);
1109                 if (ret != 0) {
1110                         evt_err("Failed to enable promiscuous mode for eth port [%d]: %s",
1111                                 i, rte_strerror(-ret));
1112                         return ret;
1113                 }
1114         }
1115
1116         return 0;
1117 }
1118
1119 void
1120 perf_ethdev_rx_stop(struct evt_test *test, struct evt_options *opt)
1121 {
1122         uint16_t i;
1123         RTE_SET_USED(test);
1124
1125         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
1126                 RTE_ETH_FOREACH_DEV(i) {
1127                         rte_event_eth_rx_adapter_stop(i);
1128                         rte_event_eth_rx_adapter_queue_del(i, i, -1);
1129                         rte_eth_dev_rx_queue_stop(i, 0);
1130                 }
1131         }
1132 }
1133
1134 void
1135 perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
1136 {
1137         uint16_t i;
1138         RTE_SET_USED(test);
1139
1140         if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
1141                 RTE_ETH_FOREACH_DEV(i) {
1142                         rte_event_eth_tx_adapter_stop(i);
1143                         rte_event_eth_tx_adapter_queue_del(i, i, -1);
1144                         rte_eth_dev_tx_queue_stop(i, 0);
1145                         rte_eth_dev_stop(i);
1146                 }
1147         }
1148 }
1149
1150 int
1151 perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt)
1152 {
1153         uint8_t cdev_count, cdev_id, nb_plcores, nb_qps;
1154         struct test_perf *t = evt_test_priv(test);
1155         unsigned int max_session_size;
1156         uint32_t nb_sessions;
1157         int ret;
1158
1159         if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
1160                 return 0;
1161
1162         cdev_count = rte_cryptodev_count();
1163         if (cdev_count == 0) {
1164                 evt_err("No crypto devices available\n");
1165                 return -ENODEV;
1166         }
1167
1168         t->ca_op_pool = rte_crypto_op_pool_create(
1169                 "crypto_op_pool", RTE_CRYPTO_OP_TYPE_SYMMETRIC, opt->pool_sz,
1170                 128, 0, rte_socket_id());
1171         if (t->ca_op_pool == NULL) {
1172                 evt_err("Failed to create crypto op pool");
1173                 return -ENOMEM;
1174         }
1175
1176         nb_sessions = evt_nr_active_lcores(opt->plcores) * t->nb_flows;
1177         t->ca_sess_pool = rte_cryptodev_sym_session_pool_create(
1178                 "ca_sess_pool", nb_sessions, 0, 0,
1179                 sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY);
1180         if (t->ca_sess_pool == NULL) {
1181                 evt_err("Failed to create sym session pool");
1182                 ret = -ENOMEM;
1183                 goto err;
1184         }
1185
1186         max_session_size = 0;
1187         for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) {
1188                 unsigned int session_size;
1189
1190                 session_size =
1191                         rte_cryptodev_sym_get_private_session_size(cdev_id);
1192                 if (session_size > max_session_size)
1193                         max_session_size = session_size;
1194         }
1195
1196         max_session_size += sizeof(union rte_event_crypto_metadata);
1197         t->ca_sess_priv_pool = rte_mempool_create(
1198                 "ca_sess_priv_pool", nb_sessions, max_session_size, 0, 0, NULL,
1199                 NULL, NULL, NULL, SOCKET_ID_ANY, 0);
1200         if (t->ca_sess_priv_pool == NULL) {
1201                 evt_err("failed to create sym session private pool");
1202                 ret = -ENOMEM;
1203                 goto err;
1204         }
1205
1206         /*
1207          * Calculate number of needed queue pairs, based on the amount of
1208          * available number of logical cores and crypto devices. For instance,
1209          * if there are 4 cores and 2 crypto devices, 2 queue pairs will be set
1210          * up per device.
1211          */
1212         nb_plcores = evt_nr_active_lcores(opt->plcores);
1213         nb_qps = (nb_plcores % cdev_count) ? (nb_plcores / cdev_count) + 1 :
1214                                              nb_plcores / cdev_count;
1215         for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) {
1216                 struct rte_cryptodev_qp_conf qp_conf;
1217                 struct rte_cryptodev_config conf;
1218                 struct rte_cryptodev_info info;
1219                 int qp_id;
1220
1221                 rte_cryptodev_info_get(cdev_id, &info);
1222                 if (nb_qps > info.max_nb_queue_pairs) {
1223                         evt_err("Not enough queue pairs per cryptodev (%u)",
1224                                 nb_qps);
1225                         ret = -EINVAL;
1226                         goto err;
1227                 }
1228
1229                 conf.nb_queue_pairs = nb_qps;
1230                 conf.socket_id = SOCKET_ID_ANY;
1231                 conf.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
1232
1233                 ret = rte_cryptodev_configure(cdev_id, &conf);
1234                 if (ret) {
1235                         evt_err("Failed to configure cryptodev (%u)", cdev_id);
1236                         goto err;
1237                 }
1238
1239                 qp_conf.nb_descriptors = NB_CRYPTODEV_DESCRIPTORS;
1240                 qp_conf.mp_session = t->ca_sess_pool;
1241                 qp_conf.mp_session_private = t->ca_sess_priv_pool;
1242
1243                 for (qp_id = 0; qp_id < conf.nb_queue_pairs; qp_id++) {
1244                         ret = rte_cryptodev_queue_pair_setup(
1245                                 cdev_id, qp_id, &qp_conf,
1246                                 rte_cryptodev_socket_id(cdev_id));
1247                         if (ret) {
1248                                 evt_err("Failed to setup queue pairs on cryptodev %u\n",
1249                                         cdev_id);
1250                                 goto err;
1251                         }
1252                 }
1253         }
1254
1255         return 0;
1256 err:
1257         for (cdev_id = 0; cdev_id < cdev_count; cdev_id++)
1258                 rte_cryptodev_close(cdev_id);
1259
1260         rte_mempool_free(t->ca_op_pool);
1261         rte_mempool_free(t->ca_sess_pool);
1262         rte_mempool_free(t->ca_sess_priv_pool);
1263
1264         return ret;
1265 }
1266
1267 void
1268 perf_cryptodev_destroy(struct evt_test *test, struct evt_options *opt)
1269 {
1270         uint8_t cdev_id, cdev_count = rte_cryptodev_count();
1271         struct test_perf *t = evt_test_priv(test);
1272         uint16_t port;
1273
1274         if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)
1275                 return;
1276
1277         for (port = t->nb_workers; port < perf_nb_event_ports(opt); port++) {
1278                 struct rte_cryptodev_sym_session *sess;
1279                 struct prod_data *p = &t->prod[port];
1280                 uint32_t flow_id;
1281                 uint8_t cdev_id;
1282
1283                 for (flow_id = 0; flow_id < t->nb_flows; flow_id++) {
1284                         sess = p->ca.crypto_sess[flow_id];
1285                         cdev_id = p->ca.cdev_id;
1286                         rte_cryptodev_sym_session_clear(cdev_id, sess);
1287                         rte_cryptodev_sym_session_free(sess);
1288                 }
1289
1290                 rte_event_crypto_adapter_queue_pair_del(
1291                         TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id);
1292         }
1293
1294         rte_event_crypto_adapter_free(TEST_PERF_CA_ID);
1295
1296         for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) {
1297                 rte_cryptodev_stop(cdev_id);
1298                 rte_cryptodev_close(cdev_id);
1299         }
1300
1301         rte_mempool_free(t->ca_op_pool);
1302         rte_mempool_free(t->ca_sess_pool);
1303         rte_mempool_free(t->ca_sess_priv_pool);
1304 }
1305
1306 int
1307 perf_mempool_setup(struct evt_test *test, struct evt_options *opt)
1308 {
1309         struct test_perf *t = evt_test_priv(test);
1310
1311         if (opt->prod_type == EVT_PROD_TYPE_SYNT ||
1312                         opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
1313                 t->pool = rte_mempool_create(test->name, /* mempool name */
1314                                 opt->pool_sz, /* number of elements*/
1315                                 sizeof(struct perf_elt), /* element size*/
1316                                 512, /* cache size*/
1317                                 0, NULL, NULL,
1318                                 perf_elt_init, /* obj constructor */
1319                                 NULL, opt->socket_id, 0); /* flags */
1320         } else {
1321                 t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */
1322                                 opt->pool_sz, /* number of elements*/
1323                                 512, /* cache size*/
1324                                 0,
1325                                 RTE_MBUF_DEFAULT_BUF_SIZE,
1326                                 opt->socket_id); /* flags */
1327
1328         }
1329
1330         if (t->pool == NULL) {
1331                 evt_err("failed to create mempool");
1332                 return -ENOMEM;
1333         }
1334
1335         return 0;
1336 }
1337
1338 void
1339 perf_mempool_destroy(struct evt_test *test, struct evt_options *opt)
1340 {
1341         RTE_SET_USED(opt);
1342         struct test_perf *t = evt_test_priv(test);
1343
1344         rte_mempool_free(t->pool);
1345 }
1346
1347 int
1348 perf_test_setup(struct evt_test *test, struct evt_options *opt)
1349 {
1350         void *test_perf;
1351
1352         test_perf = rte_zmalloc_socket(test->name, sizeof(struct test_perf),
1353                                 RTE_CACHE_LINE_SIZE, opt->socket_id);
1354         if (test_perf  == NULL) {
1355                 evt_err("failed to allocate test_perf memory");
1356                 goto nomem;
1357         }
1358         test->test_priv = test_perf;
1359
1360         struct test_perf *t = evt_test_priv(test);
1361
1362         if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
1363                 t->outstand_pkts = opt->nb_timers *
1364                         evt_nr_active_lcores(opt->plcores);
1365                 t->nb_pkts = opt->nb_timers;
1366         } else {
1367                 t->outstand_pkts = opt->nb_pkts *
1368                         evt_nr_active_lcores(opt->plcores);
1369                 t->nb_pkts = opt->nb_pkts;
1370         }
1371
1372         t->nb_workers = evt_nr_active_lcores(opt->wlcores);
1373         t->done = false;
1374         t->nb_flows = opt->nb_flows;
1375         t->result = EVT_TEST_FAILED;
1376         t->opt = opt;
1377         memcpy(t->sched_type_list, opt->sched_type_list,
1378                         sizeof(opt->sched_type_list));
1379         return 0;
1380 nomem:
1381         return -ENOMEM;
1382 }
1383
1384 void
1385 perf_test_destroy(struct evt_test *test, struct evt_options *opt)
1386 {
1387         RTE_SET_USED(opt);
1388
1389         rte_free(test->test_priv);
1390 }