app/test-crypto-perf/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <unistd.h>
   7
   8 #include <rte_malloc.h>
   9 #include <rte_random.h>
  10 #include <rte_eal.h>
  11 #include <rte_cryptodev.h>
  12 #ifdef RTE_CRYPTO_SCHEDULER
  13 #include <rte_cryptodev_scheduler.h>
  14 #endif
  15
  16 #include "cperf.h"
  17 #include "cperf_options.h"
  18 #include "cperf_test_vector_parsing.h"
  19 #include "cperf_test_throughput.h"
  20 #include "cperf_test_latency.h"
  21 #include "cperf_test_verify.h"
  22 #include "cperf_test_pmd_cyclecount.h"
  23
  24 static struct {
  25         struct rte_mempool *sess_mp;
  26         struct rte_mempool *priv_mp;
  27 } session_pool_socket[RTE_MAX_NUMA_NODES];
  28
  29 const char *cperf_test_type_strs[] = {
  30         [CPERF_TEST_TYPE_THROUGHPUT] = "throughput",
  31         [CPERF_TEST_TYPE_LATENCY] = "latency",
  32         [CPERF_TEST_TYPE_VERIFY] = "verify",
  33         [CPERF_TEST_TYPE_PMDCC] = "pmd-cyclecount"
  34 };
  35
  36 const char *cperf_op_type_strs[] = {
  37         [CPERF_CIPHER_ONLY] = "cipher-only",
  38         [CPERF_AUTH_ONLY] = "auth-only",
  39         [CPERF_CIPHER_THEN_AUTH] = "cipher-then-auth",
  40         [CPERF_AUTH_THEN_CIPHER] = "auth-then-cipher",
  41         [CPERF_AEAD] = "aead",
  42         [CPERF_PDCP] = "pdcp",
  43         [CPERF_DOCSIS] = "docsis",
  44         [CPERF_IPSEC] = "ipsec",
  45         [CPERF_ASYM_MODEX] = "modex"
  46 };
  47
  48 const struct cperf_test cperf_testmap[] = {
  49                 [CPERF_TEST_TYPE_THROUGHPUT] = {
  50                                 cperf_throughput_test_constructor,
  51                                 cperf_throughput_test_runner,
  52                                 cperf_throughput_test_destructor
  53                 },
  54                 [CPERF_TEST_TYPE_LATENCY] = {
  55                                 cperf_latency_test_constructor,
  56                                 cperf_latency_test_runner,
  57                                 cperf_latency_test_destructor
  58                 },
  59                 [CPERF_TEST_TYPE_VERIFY] = {
  60                                 cperf_verify_test_constructor,
  61                                 cperf_verify_test_runner,
  62                                 cperf_verify_test_destructor
  63                 },
  64                 [CPERF_TEST_TYPE_PMDCC] = {
  65                                 cperf_pmd_cyclecount_test_constructor,
  66                                 cperf_pmd_cyclecount_test_runner,
  67                                 cperf_pmd_cyclecount_test_destructor
  68                 }
  69 };
  70
  71 static int
  72 create_asym_op_pool_socket(uint8_t dev_id, int32_t socket_id,
  73                            uint32_t nb_sessions)
  74 {
  75         char mp_name[RTE_MEMPOOL_NAMESIZE];
  76         struct rte_mempool *mpool = NULL;
  77         unsigned int session_size =
  78                 RTE_MAX(rte_cryptodev_asym_get_private_session_size(dev_id),
  79                         rte_cryptodev_asym_get_header_session_size());
  80
  81         if (session_pool_socket[socket_id].priv_mp == NULL) {
  82                 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, "perf_asym_priv_pool%u",
  83                          socket_id);
  84
  85                 mpool = rte_mempool_create(mp_name, nb_sessions, session_size,
  86                                            0, 0, NULL, NULL, NULL, NULL,
  87                                            socket_id, 0);
  88                 if (mpool == NULL) {
  89                         printf("Cannot create pool \"%s\" on socket %d\n",
  90                                mp_name, socket_id);
  91                         return -ENOMEM;
  92                 }
  93                 printf("Allocated pool \"%s\" on socket %d\n", mp_name,
  94                        socket_id);
  95                 session_pool_socket[socket_id].priv_mp = mpool;
  96         }
  97
  98         if (session_pool_socket[socket_id].sess_mp == NULL) {
  99
 100                 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, "perf_asym_sess_pool%u",
 101                          socket_id);
 102                 mpool = rte_mempool_create(mp_name, nb_sessions,
 103                                            session_size, 0, 0, NULL, NULL, NULL,
 104                                            NULL, socket_id, 0);
 105                 if (mpool == NULL) {
 106                         printf("Cannot create pool \"%s\" on socket %d\n",
 107                                mp_name, socket_id);
 108                         return -ENOMEM;
 109                 }
 110                 session_pool_socket[socket_id].sess_mp = mpool;
 111         }
 112         return 0;
 113 }
 114
 115 static int
 116 fill_session_pool_socket(int32_t socket_id, uint32_t session_priv_size,
 117                 uint32_t nb_sessions)
 118 {
 119         char mp_name[RTE_MEMPOOL_NAMESIZE];
 120         struct rte_mempool *sess_mp;
 121
 122         if (session_pool_socket[socket_id].priv_mp == NULL) {
 123                 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 124                         "priv_sess_mp_%u", socket_id);
 125
 126                 sess_mp = rte_mempool_create(mp_name,
 127                                         nb_sessions,
 128                                         session_priv_size,
 129                                         0, 0, NULL, NULL, NULL,
 130                                         NULL, socket_id,
 131                                         0);
 132
 133                 if (sess_mp == NULL) {
 134                         printf("Cannot create pool \"%s\" on socket %d\n",
 135                                 mp_name, socket_id);
 136                         return -ENOMEM;
 137                 }
 138
 139                 printf("Allocated pool \"%s\" on socket %d\n",
 140                         mp_name, socket_id);
 141                 session_pool_socket[socket_id].priv_mp = sess_mp;
 142         }
 143
 144         if (session_pool_socket[socket_id].sess_mp == NULL) {
 145
 146                 snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 147                         "sess_mp_%u", socket_id);
 148
 149                 sess_mp = rte_cryptodev_sym_session_pool_create(mp_name,
 150                                         nb_sessions, 0, 0, 0, socket_id);
 151
 152                 if (sess_mp == NULL) {
 153                         printf("Cannot create pool \"%s\" on socket %d\n",
 154                                 mp_name, socket_id);
 155                         return -ENOMEM;
 156                 }
 157
 158                 printf("Allocated pool \"%s\" on socket %d\n",
 159                         mp_name, socket_id);
 160                 session_pool_socket[socket_id].sess_mp = sess_mp;
 161         }
 162
 163         return 0;
 164 }
 165
 166 static int
 167 cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs)
 168 {
 169         uint8_t enabled_cdev_count = 0, nb_lcores, cdev_id;
 170         uint32_t sessions_needed = 0;
 171         unsigned int i, j;
 172         int ret;
 173
 174         enabled_cdev_count = rte_cryptodev_devices_get(opts->device_type,
 175                         enabled_cdevs, RTE_CRYPTO_MAX_DEVS);
 176         if (enabled_cdev_count == 0) {
 177                 printf("No crypto devices type %s available\n",
 178                                 opts->device_type);
 179                 return -EINVAL;
 180         }
 181
 182         nb_lcores = rte_lcore_count() - 1;
 183
 184         if (nb_lcores < 1) {
 185                 RTE_LOG(ERR, USER1,
 186                         "Number of enabled cores need to be higher than 1\n");
 187                 return -EINVAL;
 188         }
 189
 190         /*
 191          * Use less number of devices,
 192          * if there are more available than cores.
 193          */
 194         if (enabled_cdev_count > nb_lcores)
 195                 enabled_cdev_count = nb_lcores;
 196
 197         /* Create a mempool shared by all the devices */
 198         uint32_t max_sess_size = 0, sess_size;
 199
 200         for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 201                 sess_size = rte_cryptodev_sym_get_private_session_size(cdev_id);
 202                 if (sess_size > max_sess_size)
 203                         max_sess_size = sess_size;
 204         }
 205 #ifdef RTE_LIB_SECURITY
 206         for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 207                 sess_size = rte_security_session_get_size(
 208                                 rte_cryptodev_get_sec_ctx(cdev_id));
 209                 if (sess_size > max_sess_size)
 210                         max_sess_size = sess_size;
 211         }
 212 #endif
 213         /*
 214          * Calculate number of needed queue pairs, based on the amount
 215          * of available number of logical cores and crypto devices.
 216          * For instance, if there are 4 cores and 2 crypto devices,
 217          * 2 queue pairs will be set up per device.
 218          */
 219         opts->nb_qps = (nb_lcores % enabled_cdev_count) ?
 220                                 (nb_lcores / enabled_cdev_count) + 1 :
 221                                 nb_lcores / enabled_cdev_count;
 222
 223         for (i = 0; i < enabled_cdev_count &&
 224                         i < RTE_CRYPTO_MAX_DEVS; i++) {
 225                 cdev_id = enabled_cdevs[i];
 226 #ifdef RTE_CRYPTO_SCHEDULER
 227                 /*
 228                  * If multi-core scheduler is used, limit the number
 229                  * of queue pairs to 1, as there is no way to know
 230                  * how many cores are being used by the PMD, and
 231                  * how many will be available for the application.
 232                  */
 233                 if (!strcmp((const char *)opts->device_type, "crypto_scheduler") &&
 234                                 rte_cryptodev_scheduler_mode_get(cdev_id) ==
 235                                 CDEV_SCHED_MODE_MULTICORE)
 236                         opts->nb_qps = 1;
 237 #endif
 238
 239                 struct rte_cryptodev_info cdev_info;
 240                 uint8_t socket_id = rte_cryptodev_socket_id(cdev_id);
 241                 /* range check the socket_id - negative values become big
 242                  * positive ones due to use of unsigned value
 243                  */
 244                 if (socket_id >= RTE_MAX_NUMA_NODES)
 245                         socket_id = 0;
 246
 247                 rte_cryptodev_info_get(cdev_id, &cdev_info);
 248
 249                 if (opts->op_type == CPERF_ASYM_MODEX) {
 250                         if ((cdev_info.feature_flags &
 251                              RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) == 0)
 252                                 continue;
 253                 }
 254
 255                 if (opts->nb_qps > cdev_info.max_nb_queue_pairs) {
 256                         printf("Number of needed queue pairs is higher "
 257                                 "than the maximum number of queue pairs "
 258                                 "per device.\n");
 259                         printf("Lower the number of cores or increase "
 260                                 "the number of crypto devices\n");
 261                         return -EINVAL;
 262                 }
 263                 struct rte_cryptodev_config conf = {
 264                         .nb_queue_pairs = opts->nb_qps,
 265                         .socket_id = socket_id,
 266                 };
 267
 268                 switch (opts->op_type) {
 269                 case CPERF_ASYM_MODEX:
 270                         conf.ff_disable |= (RTE_CRYPTODEV_FF_SECURITY |
 271                                             RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO);
 272                         break;
 273                 case CPERF_CIPHER_ONLY:
 274                 case CPERF_AUTH_ONLY:
 275                 case CPERF_CIPHER_THEN_AUTH:
 276                 case CPERF_AUTH_THEN_CIPHER:
 277                 case CPERF_AEAD:
 278                         conf.ff_disable |= RTE_CRYPTODEV_FF_SECURITY;
 279                         /* Fall through */
 280                 case CPERF_PDCP:
 281                 case CPERF_DOCSIS:
 282                 case CPERF_IPSEC:
 283                         /* Fall through */
 284                 default:
 285                         conf.ff_disable |= RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
 286                 }
 287
 288                 struct rte_cryptodev_qp_conf qp_conf = {
 289                         .nb_descriptors = opts->nb_descriptors
 290                 };
 291
 292                 /**
 293                  * Device info specifies the min headroom and tailroom
 294                  * requirement for the crypto PMD. This need to be honoured
 295                  * by the application, while creating mbuf.
 296                  */
 297                 if (opts->headroom_sz < cdev_info.min_mbuf_headroom_req) {
 298                         /* Update headroom */
 299                         opts->headroom_sz = cdev_info.min_mbuf_headroom_req;
 300                 }
 301                 if (opts->tailroom_sz < cdev_info.min_mbuf_tailroom_req) {
 302                         /* Update tailroom */
 303                         opts->tailroom_sz = cdev_info.min_mbuf_tailroom_req;
 304                 }
 305
 306                 /* Update segment size to include headroom & tailroom */
 307                 opts->segment_sz += (opts->headroom_sz + opts->tailroom_sz);
 308
 309                 uint32_t dev_max_nb_sess = cdev_info.sym.max_nb_sessions;
 310                 /*
 311                  * Two sessions objects are required for each session
 312                  * (one for the header, one for the private data)
 313                  */
 314                 if (!strcmp((const char *)opts->device_type,
 315                                         "crypto_scheduler")) {
 316 #ifdef RTE_CRYPTO_SCHEDULER
 317                         uint32_t nb_slaves =
 318                                 rte_cryptodev_scheduler_workers_get(cdev_id,
 319                                                                 NULL);
 320
 321                         sessions_needed = enabled_cdev_count *
 322                                 opts->nb_qps * nb_slaves;
 323 #endif
 324                 } else
 325                         sessions_needed = enabled_cdev_count * opts->nb_qps;
 326
 327                 /*
 328                  * A single session is required per queue pair
 329                  * in each device
 330                  */
 331                 if (dev_max_nb_sess != 0 && dev_max_nb_sess < opts->nb_qps) {
 332                         RTE_LOG(ERR, USER1,
 333                                 "Device does not support at least "
 334                                 "%u sessions\n", opts->nb_qps);
 335                         return -ENOTSUP;
 336                 }
 337
 338                 if (opts->op_type == CPERF_ASYM_MODEX)
 339                         ret = create_asym_op_pool_socket(cdev_id, socket_id,
 340                                                          sessions_needed);
 341                 else
 342                         ret = fill_session_pool_socket(socket_id, max_sess_size,
 343                                                        sessions_needed);
 344                 if (ret < 0)
 345                         return ret;
 346
 347                 qp_conf.mp_session = session_pool_socket[socket_id].sess_mp;
 348                 qp_conf.mp_session_private =
 349                                 session_pool_socket[socket_id].priv_mp;
 350
 351                 if (opts->op_type == CPERF_ASYM_MODEX) {
 352                         qp_conf.mp_session = NULL;
 353                         qp_conf.mp_session_private = NULL;
 354                 }
 355
 356                 ret = rte_cryptodev_configure(cdev_id, &conf);
 357                 if (ret < 0) {
 358                         printf("Failed to configure cryptodev %u", cdev_id);
 359                         return -EINVAL;
 360                 }
 361
 362                 for (j = 0; j < opts->nb_qps; j++) {
 363                         ret = rte_cryptodev_queue_pair_setup(cdev_id, j,
 364                                 &qp_conf, socket_id);
 365                         if (ret < 0) {
 366                                 printf("Failed to setup queue pair %u on "
 367                                         "cryptodev %u", j, cdev_id);
 368                                 return -EINVAL;
 369                         }
 370                 }
 371
 372                 ret = rte_cryptodev_start(cdev_id);
 373                 if (ret < 0) {
 374                         printf("Failed to start device %u: error %d\n",
 375                                         cdev_id, ret);
 376                         return -EPERM;
 377                 }
 378         }
 379
 380         return enabled_cdev_count;
 381 }
 382
 383 static int
 384 cperf_verify_devices_capabilities(struct cperf_options *opts,
 385                 uint8_t *enabled_cdevs, uint8_t nb_cryptodevs)
 386 {
 387         struct rte_cryptodev_sym_capability_idx cap_idx;
 388         const struct rte_cryptodev_symmetric_capability *capability;
 389         struct rte_cryptodev_asym_capability_idx asym_cap_idx;
 390         const struct rte_cryptodev_asymmetric_xform_capability *asym_capability;
 391
 392
 393         uint8_t i, cdev_id;
 394         int ret;
 395
 396         for (i = 0; i < nb_cryptodevs; i++) {
 397
 398                 cdev_id = enabled_cdevs[i];
 399
 400                 if (opts->op_type == CPERF_ASYM_MODEX) {
 401                         asym_cap_idx.type = RTE_CRYPTO_ASYM_XFORM_MODEX;
 402                         asym_capability = rte_cryptodev_asym_capability_get(
 403                                 cdev_id, &asym_cap_idx);
 404                         if (asym_capability == NULL)
 405                                 return -1;
 406
 407                         ret = rte_cryptodev_asym_xform_capability_check_modlen(
 408                                 asym_capability, sizeof(perf_mod_p));
 409                         if (ret != 0)
 410                                 return ret;
 411
 412                 }
 413
 414                 if (opts->op_type == CPERF_AUTH_ONLY ||
 415                                 opts->op_type == CPERF_CIPHER_THEN_AUTH ||
 416                                 opts->op_type == CPERF_AUTH_THEN_CIPHER) {
 417
 418                         cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
 419                         cap_idx.algo.auth = opts->auth_algo;
 420
 421                         capability = rte_cryptodev_sym_capability_get(cdev_id,
 422                                         &cap_idx);
 423                         if (capability == NULL)
 424                                 return -1;
 425
 426                         ret = rte_cryptodev_sym_capability_check_auth(
 427                                         capability,
 428                                         opts->auth_key_sz,
 429                                         opts->digest_sz,
 430                                         opts->auth_iv_sz);
 431                         if (ret != 0)
 432                                 return ret;
 433                 }
 434
 435                 if (opts->op_type == CPERF_CIPHER_ONLY ||
 436                                 opts->op_type == CPERF_CIPHER_THEN_AUTH ||
 437                                 opts->op_type == CPERF_AUTH_THEN_CIPHER) {
 438
 439                         cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
 440                         cap_idx.algo.cipher = opts->cipher_algo;
 441
 442                         capability = rte_cryptodev_sym_capability_get(cdev_id,
 443                                         &cap_idx);
 444                         if (capability == NULL)
 445                                 return -1;
 446
 447                         ret = rte_cryptodev_sym_capability_check_cipher(
 448                                         capability,
 449                                         opts->cipher_key_sz,
 450                                         opts->cipher_iv_sz);
 451                         if (ret != 0)
 452                                 return ret;
 453                 }
 454
 455                 if (opts->op_type == CPERF_AEAD) {
 456
 457                         cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
 458                         cap_idx.algo.aead = opts->aead_algo;
 459
 460                         capability = rte_cryptodev_sym_capability_get(cdev_id,
 461                                         &cap_idx);
 462                         if (capability == NULL)
 463                                 return -1;
 464
 465                         ret = rte_cryptodev_sym_capability_check_aead(
 466                                         capability,
 467                                         opts->aead_key_sz,
 468                                         opts->digest_sz,
 469                                         opts->aead_aad_sz,
 470                                         opts->aead_iv_sz);
 471                         if (ret != 0)
 472                                 return ret;
 473                 }
 474         }
 475
 476         return 0;
 477 }
 478
 479 static int
 480 cperf_check_test_vector(struct cperf_options *opts,
 481                 struct cperf_test_vector *test_vec)
 482 {
 483         if (opts->op_type == CPERF_CIPHER_ONLY) {
 484                 if (opts->cipher_algo == RTE_CRYPTO_CIPHER_NULL) {
 485                         if (test_vec->plaintext.data == NULL)
 486                                 return -1;
 487                 } else {
 488                         if (test_vec->plaintext.data == NULL)
 489                                 return -1;
 490                         if (test_vec->plaintext.length < opts->max_buffer_size)
 491                                 return -1;
 492                         if (test_vec->ciphertext.data == NULL)
 493                                 return -1;
 494                         if (test_vec->ciphertext.length < opts->max_buffer_size)
 495                                 return -1;
 496                         /* Cipher IV is only required for some algorithms */
 497                         if (opts->cipher_iv_sz &&
 498                                         test_vec->cipher_iv.data == NULL)
 499                                 return -1;
 500                         if (test_vec->cipher_iv.length != opts->cipher_iv_sz)
 501                                 return -1;
 502                         if (test_vec->cipher_key.data == NULL)
 503                                 return -1;
 504                         if (test_vec->cipher_key.length != opts->cipher_key_sz)
 505                                 return -1;
 506                 }
 507         } else if (opts->op_type == CPERF_AUTH_ONLY) {
 508                 if (opts->auth_algo != RTE_CRYPTO_AUTH_NULL) {
 509                         if (test_vec->plaintext.data == NULL)
 510                                 return -1;
 511                         if (test_vec->plaintext.length < opts->max_buffer_size)
 512                                 return -1;
 513                         /* Auth key is only required for some algorithms */
 514                         if (opts->auth_key_sz &&
 515                                         test_vec->auth_key.data == NULL)
 516                                 return -1;
 517                         if (test_vec->auth_key.length != opts->auth_key_sz)
 518                                 return -1;
 519                         if (test_vec->auth_iv.length != opts->auth_iv_sz)
 520                                 return -1;
 521                         /* Auth IV is only required for some algorithms */
 522                         if (opts->auth_iv_sz && test_vec->auth_iv.data == NULL)
 523                                 return -1;
 524                         if (test_vec->digest.data == NULL)
 525                                 return -1;
 526                         if (test_vec->digest.length < opts->digest_sz)
 527                                 return -1;
 528                 }
 529
 530         } else if (opts->op_type == CPERF_CIPHER_THEN_AUTH ||
 531                         opts->op_type == CPERF_AUTH_THEN_CIPHER) {
 532                 if (opts->cipher_algo == RTE_CRYPTO_CIPHER_NULL) {
 533                         if (test_vec->plaintext.data == NULL)
 534                                 return -1;
 535                         if (test_vec->plaintext.length < opts->max_buffer_size)
 536                                 return -1;
 537                 } else {
 538                         if (test_vec->plaintext.data == NULL)
 539                                 return -1;
 540                         if (test_vec->plaintext.length < opts->max_buffer_size)
 541                                 return -1;
 542                         if (test_vec->ciphertext.data == NULL)
 543                                 return -1;
 544                         if (test_vec->ciphertext.length < opts->max_buffer_size)
 545                                 return -1;
 546                         if (test_vec->cipher_iv.data == NULL)
 547                                 return -1;
 548                         if (test_vec->cipher_iv.length != opts->cipher_iv_sz)
 549                                 return -1;
 550                         if (test_vec->cipher_key.data == NULL)
 551                                 return -1;
 552                         if (test_vec->cipher_key.length != opts->cipher_key_sz)
 553                                 return -1;
 554                 }
 555                 if (opts->auth_algo != RTE_CRYPTO_AUTH_NULL) {
 556                         if (test_vec->auth_key.data == NULL)
 557                                 return -1;
 558                         if (test_vec->auth_key.length != opts->auth_key_sz)
 559                                 return -1;
 560                         if (test_vec->auth_iv.length != opts->auth_iv_sz)
 561                                 return -1;
 562                         /* Auth IV is only required for some algorithms */
 563                         if (opts->auth_iv_sz && test_vec->auth_iv.data == NULL)
 564                                 return -1;
 565                         if (test_vec->digest.data == NULL)
 566                                 return -1;
 567                         if (test_vec->digest.length < opts->digest_sz)
 568                                 return -1;
 569                 }
 570         } else if (opts->op_type == CPERF_AEAD) {
 571                 if (test_vec->plaintext.data == NULL)
 572                         return -1;
 573                 if (test_vec->plaintext.length < opts->max_buffer_size)
 574                         return -1;
 575                 if (test_vec->ciphertext.data == NULL)
 576                         return -1;
 577                 if (test_vec->ciphertext.length < opts->max_buffer_size)
 578                         return -1;
 579                 if (test_vec->aead_key.data == NULL)
 580                         return -1;
 581                 if (test_vec->aead_key.length != opts->aead_key_sz)
 582                         return -1;
 583                 if (test_vec->aead_iv.data == NULL)
 584                         return -1;
 585                 if (test_vec->aead_iv.length != opts->aead_iv_sz)
 586                         return -1;
 587                 if (test_vec->aad.data == NULL)
 588                         return -1;
 589                 if (test_vec->aad.length != opts->aead_aad_sz)
 590                         return -1;
 591                 if (test_vec->digest.data == NULL)
 592                         return -1;
 593                 if (test_vec->digest.length < opts->digest_sz)
 594                         return -1;
 595         }
 596         return 0;
 597 }
 598
 599 int
 600 main(int argc, char **argv)
 601 {
 602         struct cperf_options opts = {0};
 603         struct cperf_test_vector *t_vec = NULL;
 604         struct cperf_op_fns op_fns;
 605         void *ctx[RTE_MAX_LCORE] = { };
 606         int nb_cryptodevs = 0;
 607         uint16_t total_nb_qps = 0;
 608         uint8_t cdev_id, i;
 609         uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = { 0 };
 610
 611         uint8_t buffer_size_idx = 0;
 612
 613         int ret;
 614         uint32_t lcore_id;
 615
 616         /* Initialise DPDK EAL */
 617         ret = rte_eal_init(argc, argv);
 618         if (ret < 0)
 619                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments!\n");
 620         argc -= ret;
 621         argv += ret;
 622
 623         cperf_options_default(&opts);
 624
 625         ret = cperf_options_parse(&opts, argc, argv);
 626         if (ret) {
 627                 RTE_LOG(ERR, USER1, "Parsing one or more user options failed\n");
 628                 goto err;
 629         }
 630
 631         ret = cperf_options_check(&opts);
 632         if (ret) {
 633                 RTE_LOG(ERR, USER1,
 634                                 "Checking one or more user options failed\n");
 635                 goto err;
 636         }
 637
 638         nb_cryptodevs = cperf_initialize_cryptodev(&opts, enabled_cdevs);
 639
 640         if (!opts.silent)
 641                 cperf_options_dump(&opts);
 642
 643         if (nb_cryptodevs < 1) {
 644                 RTE_LOG(ERR, USER1, "Failed to initialise requested crypto "
 645                                 "device type\n");
 646                 nb_cryptodevs = 0;
 647                 goto err;
 648         }
 649
 650         ret = cperf_verify_devices_capabilities(&opts, enabled_cdevs,
 651                         nb_cryptodevs);
 652         if (ret) {
 653                 RTE_LOG(ERR, USER1, "Crypto device type does not support "
 654                                 "capabilities requested\n");
 655                 goto err;
 656         }
 657
 658         if (opts.test_file != NULL) {
 659                 t_vec = cperf_test_vector_get_from_file(&opts);
 660                 if (t_vec == NULL) {
 661                         RTE_LOG(ERR, USER1,
 662                                         "Failed to create test vector for"
 663                                         " specified file\n");
 664                         goto err;
 665                 }
 666
 667                 if (cperf_check_test_vector(&opts, t_vec)) {
 668                         RTE_LOG(ERR, USER1, "Incomplete necessary test vectors"
 669                                         "\n");
 670                         goto err;
 671                 }
 672         } else {
 673                 t_vec = cperf_test_vector_get_dummy(&opts);
 674                 if (t_vec == NULL) {
 675                         RTE_LOG(ERR, USER1,
 676                                         "Failed to create test vector for"
 677                                         " specified algorithms\n");
 678                         goto err;
 679                 }
 680         }
 681
 682         ret = cperf_get_op_functions(&opts, &op_fns);
 683         if (ret) {
 684                 RTE_LOG(ERR, USER1, "Failed to find function ops set for "
 685                                 "specified algorithms combination\n");
 686                 goto err;
 687         }
 688
 689         if (!opts.silent && opts.test != CPERF_TEST_TYPE_THROUGHPUT &&
 690                         opts.test != CPERF_TEST_TYPE_LATENCY)
 691                 show_test_vector(t_vec);
 692
 693         total_nb_qps = nb_cryptodevs * opts.nb_qps;
 694
 695         i = 0;
 696         uint8_t qp_id = 0, cdev_index = 0;
 697         RTE_LCORE_FOREACH_WORKER(lcore_id) {
 698
 699                 if (i == total_nb_qps)
 700                         break;
 701
 702                 cdev_id = enabled_cdevs[cdev_index];
 703
 704                 uint8_t socket_id = rte_cryptodev_socket_id(cdev_id);
 705
 706                 ctx[i] = cperf_testmap[opts.test].constructor(
 707                                 session_pool_socket[socket_id].sess_mp,
 708                                 session_pool_socket[socket_id].priv_mp,
 709                                 cdev_id, qp_id,
 710                                 &opts, t_vec, &op_fns);
 711                 if (ctx[i] == NULL) {
 712                         RTE_LOG(ERR, USER1, "Test run constructor failed\n");
 713                         goto err;
 714                 }
 715                 qp_id = (qp_id + 1) % opts.nb_qps;
 716                 if (qp_id == 0)
 717                         cdev_index++;
 718                 i++;
 719         }
 720
 721         if (opts.imix_distribution_count != 0) {
 722                 uint8_t buffer_size_count = opts.buffer_size_count;
 723                 uint16_t distribution_total[buffer_size_count];
 724                 uint32_t op_idx;
 725                 uint32_t test_average_size = 0;
 726                 const uint32_t *buffer_size_list = opts.buffer_size_list;
 727                 const uint32_t *imix_distribution_list = opts.imix_distribution_list;
 728
 729                 opts.imix_buffer_sizes = rte_malloc(NULL,
 730                                         sizeof(uint32_t) * opts.pool_sz,
 731                                         0);
 732                 /*
 733                  * Calculate accumulated distribution of
 734                  * probabilities per packet size
 735                  */
 736                 distribution_total[0] = imix_distribution_list[0];
 737                 for (i = 1; i < buffer_size_count; i++)
 738                         distribution_total[i] = imix_distribution_list[i] +
 739                                 distribution_total[i-1];
 740
 741                 /* Calculate a random sequence of packet sizes, based on distribution */
 742                 for (op_idx = 0; op_idx < opts.pool_sz; op_idx++) {
 743                         uint16_t random_number = rte_rand() %
 744                                 distribution_total[buffer_size_count - 1];
 745                         for (i = 0; i < buffer_size_count; i++)
 746                                 if (random_number < distribution_total[i])
 747                                         break;
 748
 749                         opts.imix_buffer_sizes[op_idx] = buffer_size_list[i];
 750                 }
 751
 752                 /* Calculate average buffer size for the IMIX distribution */
 753                 for (i = 0; i < buffer_size_count; i++)
 754                         test_average_size += buffer_size_list[i] *
 755                                 imix_distribution_list[i];
 756
 757                 opts.test_buffer_size = test_average_size /
 758                                 distribution_total[buffer_size_count - 1];
 759
 760                 i = 0;
 761                 RTE_LCORE_FOREACH_WORKER(lcore_id) {
 762
 763                         if (i == total_nb_qps)
 764                                 break;
 765
 766                         rte_eal_remote_launch(cperf_testmap[opts.test].runner,
 767                                 ctx[i], lcore_id);
 768                         i++;
 769                 }
 770                 i = 0;
 771                 RTE_LCORE_FOREACH_WORKER(lcore_id) {
 772
 773                         if (i == total_nb_qps)
 774                                 break;
 775                         ret |= rte_eal_wait_lcore(lcore_id);
 776                         i++;
 777                 }
 778
 779                 if (ret != EXIT_SUCCESS)
 780                         goto err;
 781         } else {
 782
 783                 /* Get next size from range or list */
 784                 if (opts.inc_buffer_size != 0)
 785                         opts.test_buffer_size = opts.min_buffer_size;
 786                 else
 787                         opts.test_buffer_size = opts.buffer_size_list[0];
 788
 789                 while (opts.test_buffer_size <= opts.max_buffer_size) {
 790                         i = 0;
 791                         RTE_LCORE_FOREACH_WORKER(lcore_id) {
 792
 793                                 if (i == total_nb_qps)
 794                                         break;
 795
 796                                 rte_eal_remote_launch(cperf_testmap[opts.test].runner,
 797                                         ctx[i], lcore_id);
 798                                 i++;
 799                         }
 800                         i = 0;
 801                         RTE_LCORE_FOREACH_WORKER(lcore_id) {
 802
 803                                 if (i == total_nb_qps)
 804                                         break;
 805                                 ret |= rte_eal_wait_lcore(lcore_id);
 806                                 i++;
 807                         }
 808
 809                         if (ret != EXIT_SUCCESS)
 810                                 goto err;
 811
 812                         /* Get next size from range or list */
 813                         if (opts.inc_buffer_size != 0)
 814                                 opts.test_buffer_size += opts.inc_buffer_size;
 815                         else {
 816                                 if (++buffer_size_idx == opts.buffer_size_count)
 817                                         break;
 818                                 opts.test_buffer_size =
 819                                         opts.buffer_size_list[buffer_size_idx];
 820                         }
 821                 }
 822         }
 823
 824         i = 0;
 825         RTE_LCORE_FOREACH_WORKER(lcore_id) {
 826
 827                 if (i == total_nb_qps)
 828                         break;
 829
 830                 cperf_testmap[opts.test].destructor(ctx[i]);
 831                 i++;
 832         }
 833
 834         for (i = 0; i < nb_cryptodevs &&
 835                         i < RTE_CRYPTO_MAX_DEVS; i++) {
 836                 rte_cryptodev_stop(enabled_cdevs[i]);
 837                 ret = rte_cryptodev_close(enabled_cdevs[i]);
 838                 if (ret)
 839                         RTE_LOG(ERR, USER1,
 840                                         "Crypto device close error %d\n", ret);
 841         }
 842
 843         free_test_vector(t_vec, &opts);
 844
 845         printf("\n");
 846         return EXIT_SUCCESS;
 847
 848 err:
 849         i = 0;
 850         RTE_LCORE_FOREACH_WORKER(lcore_id) {
 851                 if (i == total_nb_qps)
 852                         break;
 853
 854                 if (ctx[i] && cperf_testmap[opts.test].destructor)
 855                         cperf_testmap[opts.test].destructor(ctx[i]);
 856                 i++;
 857         }
 858
 859         for (i = 0; i < nb_cryptodevs &&
 860                         i < RTE_CRYPTO_MAX_DEVS; i++) {
 861                 rte_cryptodev_stop(enabled_cdevs[i]);
 862                 ret = rte_cryptodev_close(enabled_cdevs[i]);
 863                 if (ret)
 864                         RTE_LOG(ERR, USER1,
 865                                         "Crypto device close error %d\n", ret);
 866
 867         }
 868         rte_free(opts.imix_buffer_sizes);
 869         free_test_vector(t_vec, &opts);
 870
 871         printf("\n");
 872         return EXIT_FAILURE;
 873 }