app/test-crypto-perf/cperf_test_verify.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_cycles.h>
   7 #include <rte_crypto.h>
   8 #include <rte_cryptodev.h>
   9
  10 #include "cperf_test_verify.h"
  11 #include "cperf_ops.h"
  12 #include "cperf_test_common.h"
  13
  14 struct cperf_verify_ctx {
  15         uint8_t dev_id;
  16         uint16_t qp_id;
  17         uint8_t lcore_id;
  18
  19         struct rte_mempool *pool;
  20
  21         struct rte_cryptodev_sym_session *sess;
  22
  23         cperf_populate_ops_t populate_ops;
  24
  25         uint32_t src_buf_offset;
  26         uint32_t dst_buf_offset;
  27
  28         const struct cperf_options *options;
  29         const struct cperf_test_vector *test_vector;
  30 };
  31
  32 struct cperf_op_result {
  33         enum rte_crypto_op_status status;
  34 };
  35
  36 static void
  37 cperf_verify_test_free(struct cperf_verify_ctx *ctx)
  38 {
  39         if (ctx) {
  40                 if (ctx->sess) {
  41                         rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess);
  42                         rte_cryptodev_sym_session_free(ctx->sess);
  43                 }
  44
  45                 rte_mempool_free(ctx->pool);
  46
  47                 rte_free(ctx);
  48         }
  49 }
  50
  51 void *
  52 cperf_verify_test_constructor(struct rte_mempool *sess_mp,
  53                 struct rte_mempool *sess_priv_mp,
  54                 uint8_t dev_id, uint16_t qp_id,
  55                 const struct cperf_options *options,
  56                 const struct cperf_test_vector *test_vector,
  57                 const struct cperf_op_fns *op_fns)
  58 {
  59         struct cperf_verify_ctx *ctx = NULL;
  60
  61         ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0);
  62         if (ctx == NULL)
  63                 goto err;
  64
  65         ctx->dev_id = dev_id;
  66         ctx->qp_id = qp_id;
  67
  68         ctx->populate_ops = op_fns->populate_ops;
  69         ctx->options = options;
  70         ctx->test_vector = test_vector;
  71
  72         /* IV goes at the end of the crypto operation */
  73         uint16_t iv_offset = sizeof(struct rte_crypto_op) +
  74                 sizeof(struct rte_crypto_sym_op);
  75
  76         ctx->sess = op_fns->sess_create(sess_mp, sess_priv_mp, dev_id, options,
  77                         test_vector, iv_offset);
  78         if (ctx->sess == NULL)
  79                 goto err;
  80
  81         if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0,
  82                         &ctx->src_buf_offset, &ctx->dst_buf_offset,
  83                         &ctx->pool) < 0)
  84                 goto err;
  85
  86         return ctx;
  87 err:
  88         cperf_verify_test_free(ctx);
  89
  90         return NULL;
  91 }
  92
  93 static int
  94 cperf_verify_op(struct rte_crypto_op *op,
  95                 const struct cperf_options *options,
  96                 const struct cperf_test_vector *vector)
  97 {
  98         const struct rte_mbuf *m;
  99         uint32_t len;
 100         uint16_t nb_segs;
 101         uint8_t *data;
 102         uint32_t cipher_offset, auth_offset;
 103         uint8_t cipher, auth;
 104         int res = 0;
 105
 106         if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)
 107                 return 1;
 108
 109         if (op->sym->m_dst)
 110                 m = op->sym->m_dst;
 111         else
 112                 m = op->sym->m_src;
 113         nb_segs = m->nb_segs;
 114         len = 0;
 115         while (m && nb_segs != 0) {
 116                 len += m->data_len;
 117                 m = m->next;
 118                 nb_segs--;
 119         }
 120
 121         data = rte_malloc(NULL, len, 0);
 122         if (data == NULL)
 123                 return 1;
 124
 125         if (op->sym->m_dst)
 126                 m = op->sym->m_dst;
 127         else
 128                 m = op->sym->m_src;
 129         nb_segs = m->nb_segs;
 130         len = 0;
 131         while (m && nb_segs != 0) {
 132                 memcpy(data + len, rte_pktmbuf_mtod(m, uint8_t *),
 133                                 m->data_len);
 134                 len += m->data_len;
 135                 m = m->next;
 136                 nb_segs--;
 137         }
 138
 139         switch (options->op_type) {
 140         case CPERF_CIPHER_ONLY:
 141                 cipher = 1;
 142                 cipher_offset = 0;
 143                 auth = 0;
 144                 auth_offset = 0;
 145                 break;
 146         case CPERF_CIPHER_THEN_AUTH:
 147                 cipher = 1;
 148                 cipher_offset = 0;
 149                 auth = 1;
 150                 auth_offset = options->test_buffer_size;
 151                 break;
 152         case CPERF_AUTH_ONLY:
 153                 cipher = 0;
 154                 cipher_offset = 0;
 155                 auth = 1;
 156                 auth_offset = options->test_buffer_size;
 157                 break;
 158         case CPERF_AUTH_THEN_CIPHER:
 159                 cipher = 1;
 160                 cipher_offset = 0;
 161                 auth = 1;
 162                 auth_offset = options->test_buffer_size;
 163                 break;
 164         case CPERF_AEAD:
 165                 cipher = 1;
 166                 cipher_offset = 0;
 167                 auth = 1;
 168                 auth_offset = options->test_buffer_size;
 169                 break;
 170         default:
 171                 res = 1;
 172                 goto out;
 173         }
 174
 175         if (cipher == 1) {
 176                 if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 177                         res += memcmp(data + cipher_offset,
 178                                         vector->ciphertext.data,
 179                                         options->test_buffer_size);
 180                 else
 181                         res += memcmp(data + cipher_offset,
 182                                         vector->plaintext.data,
 183                                         options->test_buffer_size);
 184         }
 185
 186         if (auth == 1) {
 187                 if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE)
 188                         res += memcmp(data + auth_offset,
 189                                         vector->digest.data,
 190                                         options->digest_sz);
 191         }
 192
 193 out:
 194         rte_free(data);
 195         return !!res;
 196 }
 197
 198 int
 199 cperf_verify_test_runner(void *test_ctx)
 200 {
 201         struct cperf_verify_ctx *ctx = test_ctx;
 202
 203         uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0;
 204         uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0;
 205         uint64_t ops_failed = 0;
 206
 207         static uint16_t display_once;
 208
 209         uint64_t i;
 210         uint16_t ops_unused = 0;
 211         uint32_t imix_idx = 0;
 212
 213         struct rte_crypto_op *ops[ctx->options->max_burst_size];
 214         struct rte_crypto_op *ops_processed[ctx->options->max_burst_size];
 215
 216         uint32_t lcore = rte_lcore_id();
 217
 218 #ifdef CPERF_LINEARIZATION_ENABLE
 219         struct rte_cryptodev_info dev_info;
 220         int linearize = 0;
 221
 222         /* Check if source mbufs require coalescing */
 223         if (ctx->options->segment_sz < ctx->options->max_buffer_size) {
 224                 rte_cryptodev_info_get(ctx->dev_id, &dev_info);
 225                 if ((dev_info.feature_flags &
 226                                 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0)
 227                         linearize = 1;
 228         }
 229 #endif /* CPERF_LINEARIZATION_ENABLE */
 230
 231         ctx->lcore_id = lcore;
 232
 233         if (!ctx->options->csv)
 234                 printf("\n# Running verify test on device: %u, lcore: %u\n",
 235                         ctx->dev_id, lcore);
 236
 237         uint16_t iv_offset = sizeof(struct rte_crypto_op) +
 238                 sizeof(struct rte_crypto_sym_op);
 239
 240         while (ops_enqd_total < ctx->options->total_ops) {
 241
 242                 uint16_t burst_size = ((ops_enqd_total + ctx->options->max_burst_size)
 243                                 <= ctx->options->total_ops) ?
 244                                                 ctx->options->max_burst_size :
 245                                                 ctx->options->total_ops -
 246                                                 ops_enqd_total;
 247
 248                 uint16_t ops_needed = burst_size - ops_unused;
 249
 250                 /* Allocate objects containing crypto operations and mbufs */
 251                 if (rte_mempool_get_bulk(ctx->pool, (void **)ops,
 252                                         ops_needed) != 0) {
 253                         RTE_LOG(ERR, USER1,
 254                                 "Failed to allocate more crypto operations "
 255                                 "from the crypto operation pool.\n"
 256                                 "Consider increasing the pool size "
 257                                 "with --pool-sz\n");
 258                         return -1;
 259                 }
 260
 261                 /* Setup crypto op, attach mbuf etc */
 262                 (ctx->populate_ops)(ops, ctx->src_buf_offset,
 263                                 ctx->dst_buf_offset,
 264                                 ops_needed, ctx->sess, ctx->options,
 265                                 ctx->test_vector, iv_offset, &imix_idx, NULL);
 266
 267
 268                 /* Populate the mbuf with the test vector, for verification */
 269                 for (i = 0; i < ops_needed; i++)
 270                         cperf_mbuf_set(ops[i]->sym->m_src,
 271                                         ctx->options,
 272                                         ctx->test_vector);
 273
 274 #ifdef CPERF_LINEARIZATION_ENABLE
 275                 if (linearize) {
 276                         /* PMD doesn't support scatter-gather and source buffer
 277                          * is segmented.
 278                          * We need to linearize it before enqueuing.
 279                          */
 280                         for (i = 0; i < burst_size; i++)
 281                                 rte_pktmbuf_linearize(ops[i]->sym->m_src);
 282                 }
 283 #endif /* CPERF_LINEARIZATION_ENABLE */
 284
 285                 /* Enqueue burst of ops on crypto device */
 286                 ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id,
 287                                 ops, burst_size);
 288                 if (ops_enqd < burst_size)
 289                         ops_enqd_failed++;
 290
 291                 /**
 292                  * Calculate number of ops not enqueued (mainly for hw
 293                  * accelerators whose ingress queue can fill up).
 294                  */
 295                 ops_unused = burst_size - ops_enqd;
 296                 ops_enqd_total += ops_enqd;
 297
 298
 299                 /* Dequeue processed burst of ops from crypto device */
 300                 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id,
 301                                 ops_processed, ctx->options->max_burst_size);
 302
 303                 if (ops_deqd == 0) {
 304                         /**
 305                          * Count dequeue polls which didn't return any
 306                          * processed operations. This statistic is mainly
 307                          * relevant to hw accelerators.
 308                          */
 309                         ops_deqd_failed++;
 310                         continue;
 311                 }
 312
 313                 for (i = 0; i < ops_deqd; i++) {
 314                         if (cperf_verify_op(ops_processed[i], ctx->options,
 315                                                 ctx->test_vector))
 316                                 ops_failed++;
 317                 }
 318                 /* Free crypto ops so they can be reused. */
 319                 rte_mempool_put_bulk(ctx->pool,
 320                                         (void **)ops_processed, ops_deqd);
 321                 ops_deqd_total += ops_deqd;
 322         }
 323
 324         /* Dequeue any operations still in the crypto device */
 325
 326         while (ops_deqd_total < ctx->options->total_ops) {
 327                 /* Sending 0 length burst to flush sw crypto device */
 328                 rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0);
 329
 330                 /* dequeue burst */
 331                 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id,
 332                                 ops_processed, ctx->options->max_burst_size);
 333                 if (ops_deqd == 0) {
 334                         ops_deqd_failed++;
 335                         continue;
 336                 }
 337
 338                 for (i = 0; i < ops_deqd; i++) {
 339                         if (cperf_verify_op(ops_processed[i], ctx->options,
 340                                                 ctx->test_vector))
 341                                 ops_failed++;
 342                 }
 343                 /* Free crypto ops so they can be reused. */
 344                 rte_mempool_put_bulk(ctx->pool,
 345                                         (void **)ops_processed, ops_deqd);
 346                 ops_deqd_total += ops_deqd;
 347         }
 348
 349         uint16_t exp = 0;
 350         if (!ctx->options->csv) {
 351                 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0,
 352                                 __ATOMIC_RELAXED, __ATOMIC_RELAXED))
 353                         printf("%12s%12s%12s%12s%12s%12s%12s%12s\n\n",
 354                                 "lcore id", "Buf Size", "Burst size",
 355                                 "Enqueued", "Dequeued", "Failed Enq",
 356                                 "Failed Deq", "Failed Ops");
 357
 358                 printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64
 359                                 "%12"PRIu64"%12"PRIu64"\n",
 360                                 ctx->lcore_id,
 361                                 ctx->options->max_buffer_size,
 362                                 ctx->options->max_burst_size,
 363                                 ops_enqd_total,
 364                                 ops_deqd_total,
 365                                 ops_enqd_failed,
 366                                 ops_deqd_failed,
 367                                 ops_failed);
 368         } else {
 369                 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0,
 370                                 __ATOMIC_RELAXED, __ATOMIC_RELAXED))
 371                         printf("\n# lcore id, Buffer Size(B), "
 372                                 "Burst Size,Enqueued,Dequeued,Failed Enq,"
 373                                 "Failed Deq,Failed Ops\n");
 374
 375                 printf("%10u,%10u,%u,%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64","
 376                                 "%"PRIu64"\n",
 377                                 ctx->lcore_id,
 378                                 ctx->options->max_buffer_size,
 379                                 ctx->options->max_burst_size,
 380                                 ops_enqd_total,
 381                                 ops_deqd_total,
 382                                 ops_enqd_failed,
 383                                 ops_deqd_failed,
 384                                 ops_failed);
 385         }
 386
 387         return 0;
 388 }
 389
 390
 391
 392 void
 393 cperf_verify_test_destructor(void *arg)
 394 {
 395         struct cperf_verify_ctx *ctx = arg;
 396
 397         if (ctx == NULL)
 398                 return;
 399
 400         cperf_verify_test_free(ctx);
 401 }