app/test/test_cryptodev_blockcipher.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2017 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_hexdump.h>
   7 #include <rte_mbuf.h>
   8 #include <rte_malloc.h>
   9 #include <rte_memcpy.h>
  10 #include <rte_pause.h>
  11
  12 #include <rte_crypto.h>
  13 #include <rte_cryptodev.h>
  14 #include <rte_cryptodev_pmd.h>
  15
  16 #include "test.h"
  17 #include "test_cryptodev.h"
  18 #include "test_cryptodev_blockcipher.h"
  19 #include "test_cryptodev_aes_test_vectors.h"
  20 #include "test_cryptodev_des_test_vectors.h"
  21 #include "test_cryptodev_hash_test_vectors.h"
  22
  23 static int
  24 verify_algo_support(const struct blockcipher_test_case *t,
  25                 const uint8_t dev_id, const uint32_t digest_len)
  26 {
  27         int ret = 0;
  28         const struct blockcipher_test_data *tdata = t->test_data;
  29         struct rte_cryptodev_sym_capability_idx cap_idx;
  30         const struct rte_cryptodev_symmetric_capability *capability;
  31
  32         if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) {
  33                 cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
  34                 cap_idx.algo.cipher = tdata->crypto_algo;
  35                 capability = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
  36                 if (capability == NULL)
  37                         return -1;
  38
  39                 if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL)
  40                         ret = rte_cryptodev_sym_capability_check_cipher(capability,
  41                                                         tdata->cipher_key.len,
  42                                                         tdata->iv.len);
  43                 if (ret != 0)
  44                         return -1;
  45         }
  46
  47         if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) {
  48                 cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
  49                 cap_idx.algo.auth = tdata->auth_algo;
  50                 capability = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
  51                 if (capability == NULL)
  52                         return -1;
  53
  54                 if (cap_idx.algo.auth != RTE_CRYPTO_AUTH_NULL)
  55                         ret = rte_cryptodev_sym_capability_check_auth(capability,
  56                                                         tdata->auth_key.len,
  57                                                         digest_len,
  58                                                         0);
  59                 if (ret != 0)
  60                         return -1;
  61         }
  62
  63         return 0;
  64 }
  65
  66 static int
  67 test_blockcipher_one_case(const struct blockcipher_test_case *t,
  68         struct rte_mempool *mbuf_pool,
  69         struct rte_mempool *op_mpool,
  70         struct rte_mempool *sess_mpool,
  71         struct rte_mempool *sess_priv_mpool,
  72         uint8_t dev_id,
  73         int driver_id,
  74         char *test_msg)
  75 {
  76         struct rte_mbuf *ibuf = NULL;
  77         struct rte_mbuf *obuf = NULL;
  78         struct rte_mbuf *iobuf;
  79         struct rte_crypto_sym_xform *cipher_xform = NULL;
  80         struct rte_crypto_sym_xform *auth_xform = NULL;
  81         struct rte_crypto_sym_xform *init_xform = NULL;
  82         struct rte_crypto_sym_op *sym_op = NULL;
  83         struct rte_crypto_op *op = NULL;
  84         struct rte_cryptodev_info dev_info;
  85         struct rte_cryptodev_sym_session *sess = NULL;
  86
  87         int status = TEST_SUCCESS;
  88         const struct blockcipher_test_data *tdata = t->test_data;
  89         uint8_t cipher_key[tdata->cipher_key.len];
  90         uint8_t auth_key[tdata->auth_key.len];
  91         uint32_t buf_len = tdata->ciphertext.len;
  92         uint32_t digest_len = 0;
  93         char *buf_p = NULL;
  94         uint8_t src_pattern = 0xa5;
  95         uint8_t dst_pattern = 0xb6;
  96         uint8_t tmp_src_buf[MBUF_SIZE];
  97         uint8_t tmp_dst_buf[MBUF_SIZE];
  98
  99         int openssl_pmd = rte_cryptodev_driver_id_get(
 100                         RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD));
 101         int ccp_pmd = rte_cryptodev_driver_id_get(
 102                         RTE_STR(CRYPTODEV_NAME_CCP_PMD));
 103         int scheduler_pmd = rte_cryptodev_driver_id_get(
 104                         RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD));
 105         int armv8_pmd = rte_cryptodev_driver_id_get(
 106                         RTE_STR(CRYPTODEV_NAME_ARMV8_PMD));
 107         int aesni_mb_pmd = rte_cryptodev_driver_id_get(
 108                         RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD));
 109         int qat_pmd = rte_cryptodev_driver_id_get(
 110                         RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD));
 111         int dpaa2_sec_pmd = rte_cryptodev_driver_id_get(
 112                         RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD));
 113         int dpaa_sec_pmd = rte_cryptodev_driver_id_get(
 114                         RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD));
 115         int caam_jr_pmd = rte_cryptodev_driver_id_get(
 116                         RTE_STR(CRYPTODEV_NAME_CAAM_JR_PMD));
 117         int mrvl_pmd = rte_cryptodev_driver_id_get(
 118                         RTE_STR(CRYPTODEV_NAME_MVSAM_PMD));
 119         int virtio_pmd = rte_cryptodev_driver_id_get(
 120                         RTE_STR(CRYPTODEV_NAME_VIRTIO_PMD));
 121         int octeontx_pmd = rte_cryptodev_driver_id_get(
 122                         RTE_STR(CRYPTODEV_NAME_OCTEONTX_SYM_PMD));
 123         int octeontx2_pmd = rte_cryptodev_driver_id_get(
 124                         RTE_STR(CRYPTODEV_NAME_OCTEONTX2_PMD));
 125         int null_pmd = rte_cryptodev_driver_id_get(
 126                                 RTE_STR(CRYPTODEV_NAME_NULL_PMD));
 127         int nitrox_pmd = rte_cryptodev_driver_id_get(
 128                         RTE_STR(CRYPTODEV_NAME_NITROX_PMD));
 129
 130         int nb_segs = 1;
 131         uint32_t nb_iterates = 0;
 132
 133         rte_cryptodev_info_get(dev_id, &dev_info);
 134         uint64_t feat_flags = dev_info.feature_flags;
 135
 136         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) {
 137                 if (!(feat_flags & RTE_CRYPTODEV_FF_SYM_SESSIONLESS)) {
 138                         printf("Device doesn't support sesionless operations "
 139                                 "Test Skipped.\n");
 140                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 141                                 "SKIPPED");
 142                         return 0;
 143                 }
 144         }
 145         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SG) {
 146                 uint64_t oop_flag = RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT;
 147
 148                 if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) {
 149                         if (!(feat_flags & oop_flag)) {
 150                                 printf("Device doesn't support out-of-place "
 151                                         "scatter-gather in input mbuf. "
 152                                         "Test Skipped.\n");
 153                                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 154                                         "SKIPPED");
 155                                 return 0;
 156                         }
 157                 } else {
 158                         if (!(feat_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL)) {
 159                                 printf("Device doesn't support in-place "
 160                                         "scatter-gather mbufs. "
 161                                         "Test Skipped.\n");
 162                                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 163                                         "SKIPPED");
 164                                 return 0;
 165                         }
 166                 }
 167
 168                 nb_segs = 3;
 169         }
 170
 171         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) {
 172                 uint64_t oop_flags = RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
 173                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
 174                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 175                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT;
 176                 if (!(feat_flags & oop_flags)) {
 177                         printf("Device doesn't support out-of-place operations."
 178                                 "Test Skipped.\n");
 179                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 180                                 "SKIPPED");
 181                         return 0;
 182                 }
 183         }
 184
 185         if (tdata->cipher_key.len)
 186                 memcpy(cipher_key, tdata->cipher_key.data,
 187                         tdata->cipher_key.len);
 188         if (tdata->auth_key.len)
 189                 memcpy(auth_key, tdata->auth_key.data,
 190                         tdata->auth_key.len);
 191
 192         if (driver_id == dpaa2_sec_pmd ||
 193                         driver_id == dpaa_sec_pmd ||
 194                         driver_id == caam_jr_pmd ||
 195                         driver_id == qat_pmd ||
 196                         driver_id == openssl_pmd ||
 197                         driver_id == armv8_pmd ||
 198                         driver_id == mrvl_pmd ||
 199                         driver_id == ccp_pmd ||
 200                         driver_id == virtio_pmd ||
 201                         driver_id == octeontx_pmd ||
 202                         driver_id == octeontx2_pmd ||
 203                         driver_id == null_pmd ||
 204                         driver_id == nitrox_pmd) { /* Fall through */
 205                 digest_len = tdata->digest.len;
 206         } else if (driver_id == aesni_mb_pmd ||
 207                         driver_id == scheduler_pmd) {
 208                 digest_len = tdata->digest.truncated_len;
 209         } else {
 210                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 211                         "line %u FAILED: %s",
 212                         __LINE__, "Unsupported PMD type");
 213                 status = TEST_FAILED;
 214                 goto error_exit;
 215         }
 216
 217         /* Check if PMD is capable of performing that test */
 218         if (verify_algo_support(t, dev_id, digest_len) < 0) {
 219                 RTE_LOG(DEBUG, USER1,
 220                         "Device does not support this algorithm."
 221                         "Test Skipped.\n");
 222                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "SKIPPED");
 223                 return 0;
 224         }
 225
 226         /* preparing data */
 227         if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH)
 228                 buf_len += digest_len;
 229
 230         /* for contiguous mbuf, nb_segs is 1 */
 231         ibuf = create_segmented_mbuf(mbuf_pool,
 232                         tdata->ciphertext.len, nb_segs, src_pattern);
 233         if (ibuf == NULL) {
 234                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 235                         "line %u FAILED: %s",
 236                         __LINE__, "Cannot create source mbuf");
 237                 status = TEST_FAILED;
 238                 goto error_exit;
 239         }
 240
 241         /* only encryption requires plaintext.data input,
 242          * decryption/(digest gen)/(digest verify) use ciphertext.data
 243          * to be computed
 244          */
 245         if (t->op_mask & BLOCKCIPHER_TEST_OP_ENCRYPT)
 246                 pktmbuf_write(ibuf, 0, tdata->plaintext.len,
 247                                 tdata->plaintext.data);
 248         else
 249                 pktmbuf_write(ibuf, 0, tdata->ciphertext.len,
 250                                 tdata->ciphertext.data);
 251
 252         buf_p = rte_pktmbuf_append(ibuf, digest_len);
 253         if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_VERIFY)
 254                 rte_memcpy(buf_p, tdata->digest.data, digest_len);
 255         else
 256                 memset(buf_p, 0, digest_len);
 257
 258         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) {
 259                 obuf = rte_pktmbuf_alloc(mbuf_pool);
 260                 if (!obuf) {
 261                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 262                                 "FAILED: %s", __LINE__,
 263                                 "Allocation of rte_mbuf failed");
 264                         status = TEST_FAILED;
 265                         goto error_exit;
 266                 }
 267                 memset(obuf->buf_addr, dst_pattern, obuf->buf_len);
 268
 269                 buf_p = rte_pktmbuf_append(obuf, buf_len);
 270                 if (!buf_p) {
 271                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 272                                 "FAILED: %s", __LINE__,
 273                                 "No room to append mbuf");
 274                         status = TEST_FAILED;
 275                         goto error_exit;
 276                 }
 277                 memset(buf_p, 0, buf_len);
 278         }
 279
 280         /* Generate Crypto op data structure */
 281         op = rte_crypto_op_alloc(op_mpool, RTE_CRYPTO_OP_TYPE_SYMMETRIC);
 282         if (!op) {
 283                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 284                         "line %u FAILED: %s",
 285                         __LINE__, "Failed to allocate symmetric crypto "
 286                         "operation struct");
 287                 status = TEST_FAILED;
 288                 goto error_exit;
 289         }
 290
 291         sym_op = op->sym;
 292
 293 iterate:
 294         if (nb_iterates) {
 295                 struct rte_mbuf *tmp_buf = ibuf;
 296
 297                 ibuf = obuf;
 298                 obuf = tmp_buf;
 299
 300                 rte_pktmbuf_reset(ibuf);
 301                 rte_pktmbuf_reset(obuf);
 302
 303                 rte_pktmbuf_append(ibuf, tdata->ciphertext.len);
 304
 305                 /* only encryption requires plaintext.data input,
 306                  * decryption/(digest gen)/(digest verify) use ciphertext.data
 307                  * to be computed
 308                  */
 309                 if (t->op_mask & BLOCKCIPHER_TEST_OP_ENCRYPT)
 310                         pktmbuf_write(ibuf, 0, tdata->plaintext.len,
 311                                         tdata->plaintext.data);
 312                 else
 313                         pktmbuf_write(ibuf, 0, tdata->ciphertext.len,
 314                                         tdata->ciphertext.data);
 315
 316                 buf_p = rte_pktmbuf_append(ibuf, digest_len);
 317                 if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_VERIFY)
 318                         rte_memcpy(buf_p, tdata->digest.data, digest_len);
 319                 else
 320                         memset(buf_p, 0, digest_len);
 321
 322                 memset(obuf->buf_addr, dst_pattern, obuf->buf_len);
 323
 324                 buf_p = rte_pktmbuf_append(obuf, buf_len);
 325                 if (!buf_p) {
 326                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 327                                 "FAILED: %s", __LINE__,
 328                                 "No room to append mbuf");
 329                         status = TEST_FAILED;
 330                         goto error_exit;
 331                 }
 332                 memset(buf_p, 0, buf_len);
 333         }
 334
 335         sym_op->m_src = ibuf;
 336
 337         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) {
 338                 sym_op->m_dst = obuf;
 339                 iobuf = obuf;
 340         } else {
 341                 sym_op->m_dst = NULL;
 342                 iobuf = ibuf;
 343         }
 344
 345         /* sessionless op requires allocate xform using
 346          * rte_crypto_op_sym_xforms_alloc(), otherwise rte_zmalloc()
 347          * is used
 348          */
 349         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) {
 350                 uint32_t n_xforms = 0;
 351
 352                 if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER)
 353                         n_xforms++;
 354                 if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH)
 355                         n_xforms++;
 356
 357                 if (rte_crypto_op_sym_xforms_alloc(op, n_xforms)
 358                         == NULL) {
 359                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 360                                 "FAILED: %s", __LINE__, "Failed to "
 361                                 "allocate space for crypto transforms");
 362                         status = TEST_FAILED;
 363                         goto error_exit;
 364                 }
 365         } else {
 366                 cipher_xform = rte_zmalloc(NULL,
 367                         sizeof(struct rte_crypto_sym_xform), 0);
 368
 369                 auth_xform = rte_zmalloc(NULL,
 370                         sizeof(struct rte_crypto_sym_xform), 0);
 371
 372                 if (!cipher_xform || !auth_xform) {
 373                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 374                                 "FAILED: %s", __LINE__, "Failed to "
 375                                 "allocate memory for crypto transforms");
 376                         status = TEST_FAILED;
 377                         goto error_exit;
 378                 }
 379         }
 380
 381         /* preparing xform, for sessioned op, init_xform is initialized
 382          * here and later as param in rte_cryptodev_sym_session_create() call
 383          */
 384         if (t->op_mask == BLOCKCIPHER_TEST_OP_ENC_AUTH_GEN) {
 385                 if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) {
 386                         cipher_xform = op->sym->xform;
 387                         auth_xform = cipher_xform->next;
 388                         auth_xform->next = NULL;
 389                 } else {
 390                         cipher_xform->next = auth_xform;
 391                         auth_xform->next = NULL;
 392                         init_xform = cipher_xform;
 393                 }
 394         } else if (t->op_mask == BLOCKCIPHER_TEST_OP_AUTH_VERIFY_DEC) {
 395                 if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) {
 396                         auth_xform = op->sym->xform;
 397                         cipher_xform = auth_xform->next;
 398                         cipher_xform->next = NULL;
 399                 } else {
 400                         auth_xform->next = cipher_xform;
 401                         cipher_xform->next = NULL;
 402                         init_xform = auth_xform;
 403                 }
 404         } else if ((t->op_mask == BLOCKCIPHER_TEST_OP_ENCRYPT) ||
 405                         (t->op_mask == BLOCKCIPHER_TEST_OP_DECRYPT)) {
 406                 if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)
 407                         cipher_xform = op->sym->xform;
 408                 else
 409                         init_xform = cipher_xform;
 410                 cipher_xform->next = NULL;
 411         } else if ((t->op_mask == BLOCKCIPHER_TEST_OP_AUTH_GEN) ||
 412                         (t->op_mask == BLOCKCIPHER_TEST_OP_AUTH_VERIFY)) {
 413                 if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)
 414                         auth_xform = op->sym->xform;
 415                 else
 416                         init_xform = auth_xform;
 417                 auth_xform->next = NULL;
 418         } else {
 419                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 420                         "line %u FAILED: %s",
 421                         __LINE__, "Unrecognized operation");
 422                 status = TEST_FAILED;
 423                 goto error_exit;
 424         }
 425
 426         /*configure xforms & sym_op cipher and auth data*/
 427         if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) {
 428                 cipher_xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
 429                 cipher_xform->cipher.algo = tdata->crypto_algo;
 430                 if (t->op_mask & BLOCKCIPHER_TEST_OP_ENCRYPT)
 431                         cipher_xform->cipher.op =
 432                                 RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 433                 else
 434                         cipher_xform->cipher.op =
 435                                 RTE_CRYPTO_CIPHER_OP_DECRYPT;
 436                 cipher_xform->cipher.key.data = cipher_key;
 437                 cipher_xform->cipher.key.length = tdata->cipher_key.len;
 438                 cipher_xform->cipher.iv.offset = IV_OFFSET;
 439                 cipher_xform->cipher.iv.length = tdata->iv.len;
 440
 441                 sym_op->cipher.data.offset = tdata->cipher_offset;
 442                 sym_op->cipher.data.length = tdata->ciphertext.len -
 443                                 tdata->cipher_offset;
 444                 rte_memcpy(rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET),
 445                                 tdata->iv.data,
 446                                 tdata->iv.len);
 447         }
 448
 449         if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) {
 450                 uint32_t digest_offset = tdata->ciphertext.len;
 451
 452                 auth_xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
 453                 auth_xform->auth.algo = tdata->auth_algo;
 454                 auth_xform->auth.key.length = tdata->auth_key.len;
 455                 auth_xform->auth.key.data = auth_key;
 456                 auth_xform->auth.digest_length = digest_len;
 457
 458                 if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN) {
 459                         auth_xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
 460                         sym_op->auth.digest.data = pktmbuf_mtod_offset
 461                                 (iobuf, digest_offset);
 462                         sym_op->auth.digest.phys_addr =
 463                                 pktmbuf_iova_offset(iobuf,
 464                                         digest_offset);
 465                 } else {
 466                         auth_xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
 467                         sym_op->auth.digest.data = pktmbuf_mtod_offset
 468                                 (sym_op->m_src, digest_offset);
 469                         sym_op->auth.digest.phys_addr =
 470                                 pktmbuf_iova_offset(sym_op->m_src,
 471                                         digest_offset);
 472                 }
 473
 474                 sym_op->auth.data.offset = tdata->auth_offset;
 475                 sym_op->auth.data.length = tdata->ciphertext.len -
 476                                 tdata->auth_offset;
 477         }
 478
 479         /**
 480          * Create session for sessioned op. For mbuf iteration test,
 481          * skip the session creation for the second iteration.
 482          */
 483         if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS) &&
 484                         nb_iterates == 0) {
 485                 sess = rte_cryptodev_sym_session_create(sess_mpool);
 486
 487                 rte_cryptodev_sym_session_init(dev_id, sess, init_xform,
 488                                 sess_priv_mpool);
 489                 if (!sess) {
 490                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 491                                 "FAILED: %s", __LINE__,
 492                                 "Session creation failed");
 493                         status = TEST_FAILED;
 494                         goto error_exit;
 495                 }
 496
 497                 /* attach symmetric crypto session to crypto operations */
 498                 rte_crypto_op_attach_sym_session(op, sess);
 499         }
 500
 501         debug_hexdump(stdout, "m_src(before):",
 502                         sym_op->m_src->buf_addr, sym_op->m_src->buf_len);
 503         rte_memcpy(tmp_src_buf, sym_op->m_src->buf_addr,
 504                                                 sym_op->m_src->buf_len);
 505         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) {
 506                 debug_hexdump(stdout, "m_dst(before):",
 507                         sym_op->m_dst->buf_addr, sym_op->m_dst->buf_len);
 508                 rte_memcpy(tmp_dst_buf, sym_op->m_dst->buf_addr,
 509                                                 sym_op->m_dst->buf_len);
 510         }
 511
 512         /* Process crypto operation */
 513         if (rte_cryptodev_enqueue_burst(dev_id, 0, &op, 1) != 1) {
 514                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 515                         "line %u FAILED: %s",
 516                         __LINE__, "Error sending packet for encryption");
 517                 status = TEST_FAILED;
 518                 goto error_exit;
 519         }
 520
 521         op = NULL;
 522
 523         while (rte_cryptodev_dequeue_burst(dev_id, 0, &op, 1) == 0)
 524                 rte_pause();
 525
 526         if (!op) {
 527                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 528                         "line %u FAILED: %s",
 529                         __LINE__, "Failed to process sym crypto op");
 530                 status = TEST_FAILED;
 531                 goto error_exit;
 532         }
 533
 534         debug_hexdump(stdout, "m_src(after):",
 535                         sym_op->m_src->buf_addr, sym_op->m_src->buf_len);
 536         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP)
 537                 debug_hexdump(stdout, "m_dst(after):",
 538                         sym_op->m_dst->buf_addr, sym_op->m_dst->buf_len);
 539
 540         /* Verify results */
 541         if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS) {
 542                 if ((t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_VERIFY) &&
 543                         (op->status == RTE_CRYPTO_OP_STATUS_AUTH_FAILED))
 544                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 545                                 "FAILED: Digest verification failed "
 546                                 "(0x%X)", __LINE__, op->status);
 547                 else
 548                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 549                                 "FAILED: Operation failed "
 550                                 "(0x%X)", __LINE__, op->status);
 551                 status = TEST_FAILED;
 552                 goto error_exit;
 553         }
 554
 555         if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) {
 556                 uint8_t buffer[2048];
 557                 const uint8_t *compare_ref;
 558                 uint32_t compare_len;
 559
 560                 if (t->op_mask & BLOCKCIPHER_TEST_OP_ENCRYPT) {
 561                         compare_ref = tdata->ciphertext.data +
 562                                         tdata->cipher_offset;
 563                         compare_len = tdata->ciphertext.len -
 564                                         tdata->cipher_offset;
 565                 } else {
 566                         compare_ref = tdata->plaintext.data +
 567                                         tdata->cipher_offset;
 568                         compare_len = tdata->plaintext.len -
 569                                         tdata->cipher_offset;
 570                 }
 571
 572                 if (memcmp(rte_pktmbuf_read(iobuf, tdata->cipher_offset,
 573                                 compare_len, buffer), compare_ref,
 574                                 compare_len)) {
 575                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 576                                 "FAILED: %s", __LINE__,
 577                                 "Crypto data not as expected");
 578                         status = TEST_FAILED;
 579                         goto error_exit;
 580                 }
 581         }
 582
 583         if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN) {
 584                 uint8_t *auth_res = pktmbuf_mtod_offset(iobuf,
 585                                         tdata->ciphertext.len);
 586
 587                 if (memcmp(auth_res, tdata->digest.data, digest_len)) {
 588                         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u "
 589                                 "FAILED: %s", __LINE__, "Generated "
 590                                 "digest data not as expected");
 591                         status = TEST_FAILED;
 592                         goto error_exit;
 593                 }
 594         }
 595
 596         /* The only parts that should have changed in the buffer are
 597          * plaintext/ciphertext and digest.
 598          * In OOP only the dest buffer should change.
 599          */
 600         if (t->feature_mask & BLOCKCIPHER_TEST_FEATURE_OOP) {
 601                 struct rte_mbuf *mbuf;
 602                 uint8_t value;
 603                 uint32_t head_unchanged_len, changed_len = 0;
 604                 uint32_t i;
 605                 uint32_t hdroom_used = 0, tlroom_used = 0;
 606                 uint32_t hdroom = 0;
 607
 608                 mbuf = sym_op->m_src;
 609                 /*
 610                  * Crypto PMDs specify the headroom & tailroom it would use
 611                  * when processing the crypto operation. PMD is free to modify
 612                  * this space, and so the verification check should skip that
 613                  * block.
 614                  */
 615                 hdroom_used = dev_info.min_mbuf_headroom_req;
 616                 tlroom_used = dev_info.min_mbuf_tailroom_req;
 617
 618                 /* Get headroom */
 619                 hdroom = rte_pktmbuf_headroom(mbuf);
 620
 621                 head_unchanged_len = mbuf->buf_len;
 622
 623                 for (i = 0; i < mbuf->buf_len; i++) {
 624
 625                         /* Skip headroom used by PMD */
 626                         if (i == hdroom - hdroom_used)
 627                                 i += hdroom_used;
 628
 629                         /* Skip tailroom used by PMD */
 630                         if (i == (hdroom + mbuf->data_len))
 631                                 i += tlroom_used;
 632
 633                         value = *((uint8_t *)(mbuf->buf_addr)+i);
 634                         if (value != tmp_src_buf[i]) {
 635                                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 636         "line %u FAILED: OOP src outer mbuf data (0x%x) not as expected (0x%x)",
 637                                         __LINE__, value, tmp_src_buf[i]);
 638                                 status = TEST_FAILED;
 639                                 goto error_exit;
 640                         }
 641                 }
 642
 643                 mbuf = sym_op->m_dst;
 644                 if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH) {
 645                         head_unchanged_len = hdroom + sym_op->auth.data.offset;
 646                         changed_len = sym_op->auth.data.length;
 647                         if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN)
 648                                 changed_len += digest_len;
 649                 } else {
 650                         /* cipher-only */
 651                         head_unchanged_len = hdroom +
 652                                         sym_op->cipher.data.offset;
 653                         changed_len = sym_op->cipher.data.length;
 654                 }
 655
 656                 for (i = 0; i < mbuf->buf_len; i++) {
 657                         if (i == head_unchanged_len)
 658                                 i += changed_len;
 659                         value = *((uint8_t *)(mbuf->buf_addr)+i);
 660                         if (value != tmp_dst_buf[i]) {
 661                                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 662                                 "line %u FAILED: OOP dst outer mbuf data "
 663                                 "(0x%x) not as expected (0x%x)",
 664                                 __LINE__, value, tmp_dst_buf[i]);
 665                                 status = TEST_FAILED;
 666                                 goto error_exit;
 667                         }
 668                 }
 669
 670                 if (!nb_iterates) {
 671                         nb_iterates++;
 672                         goto iterate;
 673                 }
 674         } else {
 675                 /* In-place operation */
 676                 struct rte_mbuf *mbuf;
 677                 uint8_t value;
 678                 uint32_t head_unchanged_len = 0, changed_len = 0;
 679                 uint32_t i;
 680                 uint32_t hdroom_used = 0, tlroom_used = 0;
 681                 uint32_t hdroom = 0;
 682
 683                 /*
 684                  * Crypto PMDs specify the headroom & tailroom it would use
 685                  * when processing the crypto operation. PMD is free to modify
 686                  * this space, and so the verification check should skip that
 687                  * block.
 688                  */
 689                 hdroom_used = dev_info.min_mbuf_headroom_req;
 690                 tlroom_used = dev_info.min_mbuf_tailroom_req;
 691
 692                 mbuf = sym_op->m_src;
 693
 694                 /* Get headroom */
 695                 hdroom = rte_pktmbuf_headroom(mbuf);
 696
 697                 if (t->op_mask & BLOCKCIPHER_TEST_OP_CIPHER) {
 698                         head_unchanged_len = hdroom +
 699                                         sym_op->cipher.data.offset;
 700                         changed_len = sym_op->cipher.data.length;
 701                 } else {
 702                         /* auth-only */
 703                         head_unchanged_len = hdroom +
 704                                         sym_op->auth.data.offset +
 705                                         sym_op->auth.data.length;
 706                         changed_len = 0;
 707                 }
 708
 709                 if (t->op_mask & BLOCKCIPHER_TEST_OP_AUTH_GEN)
 710                         changed_len += digest_len;
 711
 712                 for (i = 0; i < mbuf->buf_len; i++) {
 713
 714                         /* Skip headroom used by PMD */
 715                         if (i == hdroom - hdroom_used)
 716                                 i += hdroom_used;
 717
 718                         if (i == head_unchanged_len)
 719                                 i += changed_len;
 720
 721                         /* Skip tailroom used by PMD */
 722                         if (i == (hdroom + mbuf->data_len))
 723                                 i += tlroom_used;
 724
 725                         value = *((uint8_t *)(mbuf->buf_addr)+i);
 726                         if (value != tmp_src_buf[i]) {
 727                                 snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
 728                                 "line %u FAILED: outer mbuf data (0x%x) "
 729                                 "not as expected (0x%x)",
 730                                 __LINE__, value, tmp_src_buf[i]);
 731                                 status = TEST_FAILED;
 732                                 goto error_exit;
 733                         }
 734                 }
 735         }
 736
 737         snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "PASS");
 738
 739 error_exit:
 740         if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)) {
 741                 if (sess) {
 742                         rte_cryptodev_sym_session_clear(dev_id, sess);
 743                         rte_cryptodev_sym_session_free(sess);
 744                 }
 745                 if (cipher_xform)
 746                         rte_free(cipher_xform);
 747                 if (auth_xform)
 748                         rte_free(auth_xform);
 749         }
 750
 751         if (op)
 752                 rte_crypto_op_free(op);
 753
 754         if (obuf)
 755                 rte_pktmbuf_free(obuf);
 756
 757         if (ibuf)
 758                 rte_pktmbuf_free(ibuf);
 759
 760         return status;
 761 }
 762
 763 int
 764 test_blockcipher_all_tests(struct rte_mempool *mbuf_pool,
 765         struct rte_mempool *op_mpool,
 766         struct rte_mempool *sess_mpool,
 767         struct rte_mempool *sess_priv_mpool,
 768         uint8_t dev_id,
 769         int driver_id,
 770         enum blockcipher_test_type test_type)
 771 {
 772         int status, overall_status = TEST_SUCCESS;
 773         uint32_t i, test_index = 0;
 774         char test_msg[BLOCKCIPHER_TEST_MSG_LEN + 1];
 775         uint32_t n_test_cases = 0;
 776         uint32_t target_pmd_mask = 0;
 777         const struct blockcipher_test_case *tcs = NULL;
 778
 779         int openssl_pmd = rte_cryptodev_driver_id_get(
 780                         RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD));
 781         int ccp_pmd = rte_cryptodev_driver_id_get(
 782                         RTE_STR(CRYPTODEV_NAME_CCP_PMD));
 783         int dpaa2_sec_pmd = rte_cryptodev_driver_id_get(
 784                         RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD));
 785         int dpaa_sec_pmd = rte_cryptodev_driver_id_get(
 786                         RTE_STR(CRYPTODEV_NAME_DPAA_SEC_PMD));
 787         int caam_jr_pmd = rte_cryptodev_driver_id_get(
 788                         RTE_STR(CRYPTODEV_NAME_CAAM_JR_PMD));
 789         int scheduler_pmd = rte_cryptodev_driver_id_get(
 790                         RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD));
 791         int armv8_pmd = rte_cryptodev_driver_id_get(
 792                         RTE_STR(CRYPTODEV_NAME_ARMV8_PMD));
 793         int aesni_mb_pmd = rte_cryptodev_driver_id_get(
 794                         RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD));
 795         int qat_pmd = rte_cryptodev_driver_id_get(
 796                         RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD));
 797         int mrvl_pmd = rte_cryptodev_driver_id_get(
 798                         RTE_STR(CRYPTODEV_NAME_MVSAM_PMD));
 799         int virtio_pmd = rte_cryptodev_driver_id_get(
 800                         RTE_STR(CRYPTODEV_NAME_VIRTIO_PMD));
 801         int octeontx_pmd = rte_cryptodev_driver_id_get(
 802                         RTE_STR(CRYPTODEV_NAME_OCTEONTX_SYM_PMD));
 803         int octeontx2_pmd = rte_cryptodev_driver_id_get(
 804                         RTE_STR(CRYPTODEV_NAME_OCTEONTX2_PMD));
 805         int null_pmd = rte_cryptodev_driver_id_get(
 806                                 RTE_STR(CRYPTODEV_NAME_NULL_PMD));
 807         int nitrox_pmd = rte_cryptodev_driver_id_get(
 808                         RTE_STR(CRYPTODEV_NAME_NITROX_PMD));
 809
 810         switch (test_type) {
 811         case BLKCIPHER_AES_CHAIN_TYPE:
 812                 n_test_cases = sizeof(aes_chain_test_cases) /
 813                 sizeof(aes_chain_test_cases[0]);
 814                 tcs = aes_chain_test_cases;
 815                 break;
 816         case BLKCIPHER_AES_CIPHERONLY_TYPE:
 817                 n_test_cases = sizeof(aes_cipheronly_test_cases) /
 818                 sizeof(aes_cipheronly_test_cases[0]);
 819                 tcs = aes_cipheronly_test_cases;
 820                 break;
 821         case BLKCIPHER_AES_DOCSIS_TYPE:
 822                 n_test_cases = sizeof(aes_docsis_test_cases) /
 823                 sizeof(aes_docsis_test_cases[0]);
 824                 tcs = aes_docsis_test_cases;
 825                 break;
 826         case BLKCIPHER_3DES_CHAIN_TYPE:
 827                 n_test_cases = sizeof(triple_des_chain_test_cases) /
 828                 sizeof(triple_des_chain_test_cases[0]);
 829                 tcs = triple_des_chain_test_cases;
 830                 break;
 831         case BLKCIPHER_3DES_CIPHERONLY_TYPE:
 832                 n_test_cases = sizeof(triple_des_cipheronly_test_cases) /
 833                 sizeof(triple_des_cipheronly_test_cases[0]);
 834                 tcs = triple_des_cipheronly_test_cases;
 835                 break;
 836         case BLKCIPHER_DES_CIPHERONLY_TYPE:
 837                 n_test_cases = sizeof(des_cipheronly_test_cases) /
 838                 sizeof(des_cipheronly_test_cases[0]);
 839                 tcs = des_cipheronly_test_cases;
 840                 break;
 841         case BLKCIPHER_DES_DOCSIS_TYPE:
 842                 n_test_cases = sizeof(des_docsis_test_cases) /
 843                 sizeof(des_docsis_test_cases[0]);
 844                 tcs = des_docsis_test_cases;
 845                 break;
 846         case BLKCIPHER_AUTHONLY_TYPE:
 847                 n_test_cases = sizeof(hash_test_cases) /
 848                 sizeof(hash_test_cases[0]);
 849                 tcs = hash_test_cases;
 850                 break;
 851         default:
 852                 break;
 853         }
 854
 855         if (driver_id == aesni_mb_pmd)
 856                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MB;
 857         else if (driver_id == qat_pmd)
 858                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_QAT;
 859         else if (driver_id == openssl_pmd)
 860                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OPENSSL;
 861         else if (driver_id == armv8_pmd)
 862                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_ARMV8;
 863         else if (driver_id == scheduler_pmd)
 864                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_SCHEDULER;
 865         else if (driver_id == dpaa2_sec_pmd)
 866                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_DPAA2_SEC;
 867         else if (driver_id == ccp_pmd)
 868                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_CCP;
 869         else if (driver_id == dpaa_sec_pmd)
 870                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_DPAA_SEC;
 871         else if (driver_id == caam_jr_pmd)
 872                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_CAAM_JR;
 873         else if (driver_id == mrvl_pmd)
 874                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_MVSAM;
 875         else if (driver_id == virtio_pmd)
 876                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_VIRTIO;
 877         else if (driver_id == octeontx_pmd)
 878                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OCTEONTX;
 879         else if (driver_id == octeontx2_pmd)
 880                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_OCTEONTX2;
 881         else if (driver_id == null_pmd)
 882                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_NULL;
 883         else if (driver_id == nitrox_pmd)
 884                 target_pmd_mask = BLOCKCIPHER_TEST_TARGET_PMD_NITROX;
 885         else
 886                 return -ENOTSUP; /* Unrecognized cryptodev type */
 887
 888         for (i = 0; i < n_test_cases; i++) {
 889                 const struct blockcipher_test_case *tc = &tcs[i];
 890
 891                 if (!(tc->pmd_mask & target_pmd_mask))
 892                         continue;
 893
 894                 status = test_blockcipher_one_case(tc, mbuf_pool, op_mpool,
 895                         sess_mpool, sess_priv_mpool, dev_id, driver_id,
 896                         test_msg);
 897
 898                 printf("  %u) TestCase %s %s\n", test_index ++,
 899                         tc->test_descr, test_msg);
 900
 901                 if (status != TEST_SUCCESS) {
 902                         if (overall_status == TEST_SUCCESS)
 903                                 overall_status = status;
 904
 905                         if (tc->feature_mask & BLOCKCIPHER_TEST_FEATURE_STOPPER)
 906                                 break;
 907                 }
 908         }
 909
 910         return overall_status;
 911 }