drivers/crypto/ccp/ccp_crypto.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <dirent.h>
   6 #include <fcntl.h>
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <sys/mman.h>
  10 #include <sys/queue.h>
  11 #include <sys/types.h>
  12 #include <unistd.h>
  13 #include <openssl/sha.h>
  14 #include <openssl/cmac.h> /*sub key apis*/
  15 #include <openssl/evp.h> /*sub key apis*/
  16
  17 #include <rte_hexdump.h>
  18 #include <rte_memzone.h>
  19 #include <rte_malloc.h>
  20 #include <rte_memory.h>
  21 #include <rte_spinlock.h>
  22 #include <rte_string_fns.h>
  23 #include <cryptodev_pmd.h>
  24
  25 #include "ccp_dev.h"
  26 #include "ccp_crypto.h"
  27 #include "ccp_pci.h"
  28 #include "ccp_pmd_private.h"
  29
  30 #include <openssl/conf.h>
  31 #include <openssl/err.h>
  32 #include <openssl/hmac.h>
  33
  34 extern int iommu_mode;
  35 void *sha_ctx;
  36 /* SHA initial context values */
  37 uint32_t ccp_sha1_init[SHA_COMMON_DIGEST_SIZE / sizeof(uint32_t)] = {
  38         SHA1_H4, SHA1_H3,
  39         SHA1_H2, SHA1_H1,
  40         SHA1_H0, 0x0U,
  41         0x0U, 0x0U,
  42 };
  43
  44 uint32_t ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(uint32_t)] = {
  45         SHA224_H7, SHA224_H6,
  46         SHA224_H5, SHA224_H4,
  47         SHA224_H3, SHA224_H2,
  48         SHA224_H1, SHA224_H0,
  49 };
  50
  51 uint32_t ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(uint32_t)] = {
  52         SHA256_H7, SHA256_H6,
  53         SHA256_H5, SHA256_H4,
  54         SHA256_H3, SHA256_H2,
  55         SHA256_H1, SHA256_H0,
  56 };
  57
  58 uint64_t ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(uint64_t)] = {
  59         SHA384_H7, SHA384_H6,
  60         SHA384_H5, SHA384_H4,
  61         SHA384_H3, SHA384_H2,
  62         SHA384_H1, SHA384_H0,
  63 };
  64
  65 uint64_t ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(uint64_t)] = {
  66         SHA512_H7, SHA512_H6,
  67         SHA512_H5, SHA512_H4,
  68         SHA512_H3, SHA512_H2,
  69         SHA512_H1, SHA512_H0,
  70 };
  71
  72 #if defined(_MSC_VER)
  73 #define SHA3_CONST(x) x
  74 #else
  75 #define SHA3_CONST(x) x##L
  76 #endif
  77
  78 /** 'Words' here refers to uint64_t */
  79 #define SHA3_KECCAK_SPONGE_WORDS \
  80         (((1600) / 8) / sizeof(uint64_t))
  81 typedef struct sha3_context_ {
  82         uint64_t saved;
  83         /**
  84          * The portion of the input message that we
  85          * didn't consume yet
  86          */
  87         union {
  88                 uint64_t s[SHA3_KECCAK_SPONGE_WORDS];
  89                 /* Keccak's state */
  90                 uint8_t sb[SHA3_KECCAK_SPONGE_WORDS * 8];
  91                 /**total 200 ctx size**/
  92         };
  93         unsigned int byteIndex;
  94         /**
  95          * 0..7--the next byte after the set one
  96          * (starts from 0; 0--none are buffered)
  97          */
  98         unsigned int wordIndex;
  99         /**
 100          * 0..24--the next word to integrate input
 101          * (starts from 0)
 102          */
 103         unsigned int capacityWords;
 104         /**
 105          * the double size of the hash output in
 106          * words (e.g. 16 for Keccak 512)
 107          */
 108 } sha3_context;
 109
 110 #ifndef SHA3_ROTL64
 111 #define SHA3_ROTL64(x, y) \
 112         (((x) << (y)) | ((x) >> ((sizeof(uint64_t)*8) - (y))))
 113 #endif
 114
 115 static const uint64_t keccakf_rndc[24] = {
 116         SHA3_CONST(0x0000000000000001UL), SHA3_CONST(0x0000000000008082UL),
 117         SHA3_CONST(0x800000000000808aUL), SHA3_CONST(0x8000000080008000UL),
 118         SHA3_CONST(0x000000000000808bUL), SHA3_CONST(0x0000000080000001UL),
 119         SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008009UL),
 120         SHA3_CONST(0x000000000000008aUL), SHA3_CONST(0x0000000000000088UL),
 121         SHA3_CONST(0x0000000080008009UL), SHA3_CONST(0x000000008000000aUL),
 122         SHA3_CONST(0x000000008000808bUL), SHA3_CONST(0x800000000000008bUL),
 123         SHA3_CONST(0x8000000000008089UL), SHA3_CONST(0x8000000000008003UL),
 124         SHA3_CONST(0x8000000000008002UL), SHA3_CONST(0x8000000000000080UL),
 125         SHA3_CONST(0x000000000000800aUL), SHA3_CONST(0x800000008000000aUL),
 126         SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008080UL),
 127         SHA3_CONST(0x0000000080000001UL), SHA3_CONST(0x8000000080008008UL)
 128 };
 129
 130 static const unsigned int keccakf_rotc[24] = {
 131         1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62,
 132         18, 39, 61, 20, 44
 133 };
 134
 135 static const unsigned int keccakf_piln[24] = {
 136         10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20,
 137         14, 22, 9, 6, 1
 138 };
 139
 140 static enum ccp_cmd_order
 141 ccp_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 142 {
 143         enum ccp_cmd_order res = CCP_CMD_NOT_SUPPORTED;
 144
 145         if (xform == NULL)
 146                 return res;
 147         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 148                 if (xform->next == NULL)
 149                         return CCP_CMD_AUTH;
 150                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 151                         return CCP_CMD_HASH_CIPHER;
 152         }
 153         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 154                 if (xform->next == NULL)
 155                         return CCP_CMD_CIPHER;
 156                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 157                         return CCP_CMD_CIPHER_HASH;
 158         }
 159         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 160                 return CCP_CMD_COMBINED;
 161         return res;
 162 }
 163
 164 /* partial hash using openssl */
 165 static int partial_hash_sha1(uint8_t *data_in, uint8_t *data_out)
 166 {
 167         SHA_CTX ctx;
 168
 169         if (!SHA1_Init(&ctx))
 170                 return -EFAULT;
 171         SHA1_Transform(&ctx, data_in);
 172         rte_memcpy(data_out, &ctx, SHA_DIGEST_LENGTH);
 173         return 0;
 174 }
 175
 176 static int partial_hash_sha224(uint8_t *data_in, uint8_t *data_out)
 177 {
 178         SHA256_CTX ctx;
 179
 180         if (!SHA224_Init(&ctx))
 181                 return -EFAULT;
 182         SHA256_Transform(&ctx, data_in);
 183         rte_memcpy(data_out, &ctx,
 184                    SHA256_DIGEST_LENGTH);
 185         return 0;
 186 }
 187
 188 static int partial_hash_sha256(uint8_t *data_in, uint8_t *data_out)
 189 {
 190         SHA256_CTX ctx;
 191
 192         if (!SHA256_Init(&ctx))
 193                 return -EFAULT;
 194         SHA256_Transform(&ctx, data_in);
 195         rte_memcpy(data_out, &ctx,
 196                    SHA256_DIGEST_LENGTH);
 197         return 0;
 198 }
 199
 200 static int partial_hash_sha384(uint8_t *data_in, uint8_t *data_out)
 201 {
 202         SHA512_CTX ctx;
 203
 204         if (!SHA384_Init(&ctx))
 205                 return -EFAULT;
 206         SHA512_Transform(&ctx, data_in);
 207         rte_memcpy(data_out, &ctx,
 208                    SHA512_DIGEST_LENGTH);
 209         return 0;
 210 }
 211
 212 static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out)
 213 {
 214         SHA512_CTX ctx;
 215
 216         if (!SHA512_Init(&ctx))
 217                 return -EFAULT;
 218         SHA512_Transform(&ctx, data_in);
 219         rte_memcpy(data_out, &ctx,
 220                    SHA512_DIGEST_LENGTH);
 221         return 0;
 222 }
 223
 224 static void
 225 keccakf(uint64_t s[25])
 226 {
 227         int i, j, round;
 228         uint64_t t, bc[5];
 229 #define KECCAK_ROUNDS 24
 230
 231         for (round = 0; round < KECCAK_ROUNDS; round++) {
 232
 233                 /* Theta */
 234                 for (i = 0; i < 5; i++)
 235                         bc[i] = s[i] ^ s[i + 5] ^ s[i + 10] ^ s[i + 15] ^
 236                                 s[i + 20];
 237
 238                 for (i = 0; i < 5; i++) {
 239                         t = bc[(i + 4) % 5] ^ SHA3_ROTL64(bc[(i + 1) % 5], 1);
 240                         for (j = 0; j < 25; j += 5)
 241                                 s[j + i] ^= t;
 242                 }
 243
 244                 /* Rho Pi */
 245                 t = s[1];
 246                 for (i = 0; i < 24; i++) {
 247                         j = keccakf_piln[i];
 248                         bc[0] = s[j];
 249                         s[j] = SHA3_ROTL64(t, keccakf_rotc[i]);
 250                         t = bc[0];
 251                 }
 252
 253                 /* Chi */
 254                 for (j = 0; j < 25; j += 5) {
 255                         for (i = 0; i < 5; i++)
 256                                 bc[i] = s[j + i];
 257                         for (i = 0; i < 5; i++)
 258                                 s[j + i] ^= (~bc[(i + 1) % 5]) &
 259                                             bc[(i + 2) % 5];
 260                 }
 261
 262                 /* Iota */
 263                 s[0] ^= keccakf_rndc[round];
 264         }
 265 }
 266
 267 static void
 268 sha3_Init224(void *priv)
 269 {
 270         sha3_context *ctx = (sha3_context *) priv;
 271
 272         memset(ctx, 0, sizeof(*ctx));
 273         ctx->capacityWords = 2 * 224 / (8 * sizeof(uint64_t));
 274 }
 275
 276 static void
 277 sha3_Init256(void *priv)
 278 {
 279         sha3_context *ctx = (sha3_context *) priv;
 280
 281         memset(ctx, 0, sizeof(*ctx));
 282         ctx->capacityWords = 2 * 256 / (8 * sizeof(uint64_t));
 283 }
 284
 285 static void
 286 sha3_Init384(void *priv)
 287 {
 288         sha3_context *ctx = (sha3_context *) priv;
 289
 290         memset(ctx, 0, sizeof(*ctx));
 291         ctx->capacityWords = 2 * 384 / (8 * sizeof(uint64_t));
 292 }
 293
 294 static void
 295 sha3_Init512(void *priv)
 296 {
 297         sha3_context *ctx = (sha3_context *) priv;
 298
 299         memset(ctx, 0, sizeof(*ctx));
 300         ctx->capacityWords = 2 * 512 / (8 * sizeof(uint64_t));
 301 }
 302
 303
 304 /* This is simply the 'update' with the padding block.
 305  * The padding block is 0x01 || 0x00* || 0x80. First 0x01 and last 0x80
 306  * bytes are always present, but they can be the same byte.
 307  */
 308 static void
 309 sha3_Update(void *priv, void const *bufIn, size_t len)
 310 {
 311         sha3_context *ctx = (sha3_context *) priv;
 312         unsigned int old_tail = (8 - ctx->byteIndex) & 7;
 313         size_t words;
 314         unsigned int tail;
 315         size_t i;
 316         const uint8_t *buf = bufIn;
 317
 318         if (len < old_tail) {
 319                 while (len--)
 320                         ctx->saved |= (uint64_t) (*(buf++)) <<
 321                                       ((ctx->byteIndex++) * 8);
 322                 return;
 323         }
 324
 325         if (old_tail) {
 326                 len -= old_tail;
 327                 while (old_tail--)
 328                         ctx->saved |= (uint64_t) (*(buf++)) <<
 329                                       ((ctx->byteIndex++) * 8);
 330
 331                 ctx->s[ctx->wordIndex] ^= ctx->saved;
 332                 ctx->byteIndex = 0;
 333                 ctx->saved = 0;
 334                 if (++ctx->wordIndex ==
 335                    (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
 336                         keccakf(ctx->s);
 337                         ctx->wordIndex = 0;
 338                 }
 339         }
 340
 341         words = len / sizeof(uint64_t);
 342         tail = len - words * sizeof(uint64_t);
 343
 344         for (i = 0; i < words; i++, buf += sizeof(uint64_t)) {
 345                 const uint64_t t = (uint64_t) (buf[0]) |
 346                         ((uint64_t) (buf[1]) << 8 * 1) |
 347                         ((uint64_t) (buf[2]) << 8 * 2) |
 348                         ((uint64_t) (buf[3]) << 8 * 3) |
 349                         ((uint64_t) (buf[4]) << 8 * 4) |
 350                         ((uint64_t) (buf[5]) << 8 * 5) |
 351                         ((uint64_t) (buf[6]) << 8 * 6) |
 352                         ((uint64_t) (buf[7]) << 8 * 7);
 353                 ctx->s[ctx->wordIndex] ^= t;
 354                 if (++ctx->wordIndex ==
 355                    (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
 356                         keccakf(ctx->s);
 357                         ctx->wordIndex = 0;
 358                 }
 359         }
 360
 361         while (tail--)
 362                 ctx->saved |= (uint64_t) (*(buf++)) << ((ctx->byteIndex++) * 8);
 363 }
 364
 365 int partial_hash_sha3_224(uint8_t *data_in, uint8_t *data_out)
 366 {
 367         sha3_context *ctx;
 368         int i;
 369
 370         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 371         if (!ctx) {
 372                 CCP_LOG_ERR("sha3-ctx creation failed");
 373                 return -ENOMEM;
 374         }
 375         sha3_Init224(ctx);
 376         sha3_Update(ctx, data_in, SHA3_224_BLOCK_SIZE);
 377         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 378                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 379         rte_free(ctx);
 380
 381         return 0;
 382 }
 383
 384 int partial_hash_sha3_256(uint8_t *data_in, uint8_t *data_out)
 385 {
 386         sha3_context *ctx;
 387         int i;
 388
 389         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 390         if (!ctx) {
 391                 CCP_LOG_ERR("sha3-ctx creation failed");
 392                 return -ENOMEM;
 393         }
 394         sha3_Init256(ctx);
 395         sha3_Update(ctx, data_in, SHA3_256_BLOCK_SIZE);
 396         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 397                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 398         rte_free(ctx);
 399
 400         return 0;
 401 }
 402
 403 int partial_hash_sha3_384(uint8_t *data_in, uint8_t *data_out)
 404 {
 405         sha3_context *ctx;
 406         int i;
 407
 408         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 409         if (!ctx) {
 410                 CCP_LOG_ERR("sha3-ctx creation failed");
 411                 return -ENOMEM;
 412         }
 413         sha3_Init384(ctx);
 414         sha3_Update(ctx, data_in, SHA3_384_BLOCK_SIZE);
 415         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 416                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 417         rte_free(ctx);
 418
 419         return 0;
 420 }
 421
 422 int partial_hash_sha3_512(uint8_t *data_in, uint8_t *data_out)
 423 {
 424         sha3_context *ctx;
 425         int i;
 426
 427         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 428         if (!ctx) {
 429                 CCP_LOG_ERR("sha3-ctx creation failed");
 430                 return -ENOMEM;
 431         }
 432         sha3_Init512(ctx);
 433         sha3_Update(ctx, data_in, SHA3_512_BLOCK_SIZE);
 434         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 435                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 436         rte_free(ctx);
 437
 438         return 0;
 439 }
 440
 441 static int generate_partial_hash(struct ccp_session *sess)
 442 {
 443
 444         uint8_t ipad[sess->auth.block_size];
 445         uint8_t opad[sess->auth.block_size];
 446         uint8_t *ipad_t, *opad_t;
 447         uint32_t *hash_value_be32, hash_temp32[8];
 448         uint64_t *hash_value_be64, hash_temp64[8];
 449         int i, count;
 450         uint8_t *hash_value_sha3;
 451
 452         opad_t = ipad_t = (uint8_t *)sess->auth.key;
 453
 454         hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute);
 455         hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute);
 456
 457         /* considering key size is always equal to block size of algorithm */
 458         for (i = 0; i < sess->auth.block_size; i++) {
 459                 ipad[i] = (ipad_t[i] ^ HMAC_IPAD_VALUE);
 460                 opad[i] = (opad_t[i] ^ HMAC_OPAD_VALUE);
 461         }
 462
 463         switch (sess->auth.algo) {
 464         case CCP_AUTH_ALGO_SHA1_HMAC:
 465                 count = SHA1_DIGEST_SIZE >> 2;
 466
 467                 if (partial_hash_sha1(ipad, (uint8_t *)hash_temp32))
 468                         return -1;
 469                 for (i = 0; i < count; i++, hash_value_be32++)
 470                         *hash_value_be32 = hash_temp32[count - 1 - i];
 471
 472                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 473                                                + sess->auth.ctx_len);
 474                 if (partial_hash_sha1(opad, (uint8_t *)hash_temp32))
 475                         return -1;
 476                 for (i = 0; i < count; i++, hash_value_be32++)
 477                         *hash_value_be32 = hash_temp32[count - 1 - i];
 478                 return 0;
 479         case CCP_AUTH_ALGO_SHA224_HMAC:
 480                 count = SHA256_DIGEST_SIZE >> 2;
 481
 482                 if (partial_hash_sha224(ipad, (uint8_t *)hash_temp32))
 483                         return -1;
 484                 for (i = 0; i < count; i++, hash_value_be32++)
 485                         *hash_value_be32 = hash_temp32[count - 1 - i];
 486
 487                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 488                                                + sess->auth.ctx_len);
 489                 if (partial_hash_sha224(opad, (uint8_t *)hash_temp32))
 490                         return -1;
 491                 for (i = 0; i < count; i++, hash_value_be32++)
 492                         *hash_value_be32 = hash_temp32[count - 1 - i];
 493                 return 0;
 494         case CCP_AUTH_ALGO_SHA3_224_HMAC:
 495                 hash_value_sha3 = sess->auth.pre_compute;
 496                 if (partial_hash_sha3_224(ipad, hash_value_sha3))
 497                         return -1;
 498
 499                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 500                                                + sess->auth.ctx_len);
 501                 if (partial_hash_sha3_224(opad, hash_value_sha3))
 502                         return -1;
 503                 return 0;
 504         case CCP_AUTH_ALGO_SHA256_HMAC:
 505                 count = SHA256_DIGEST_SIZE >> 2;
 506
 507                 if (partial_hash_sha256(ipad, (uint8_t *)hash_temp32))
 508                         return -1;
 509                 for (i = 0; i < count; i++, hash_value_be32++)
 510                         *hash_value_be32 = hash_temp32[count - 1 - i];
 511
 512                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 513                                                + sess->auth.ctx_len);
 514                 if (partial_hash_sha256(opad, (uint8_t *)hash_temp32))
 515                         return -1;
 516                 for (i = 0; i < count; i++, hash_value_be32++)
 517                         *hash_value_be32 = hash_temp32[count - 1 - i];
 518                 return 0;
 519         case CCP_AUTH_ALGO_SHA3_256_HMAC:
 520                 hash_value_sha3 = sess->auth.pre_compute;
 521                 if (partial_hash_sha3_256(ipad, hash_value_sha3))
 522                         return -1;
 523
 524                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 525                                               + sess->auth.ctx_len);
 526                 if (partial_hash_sha3_256(opad, hash_value_sha3))
 527                         return -1;
 528                 return 0;
 529         case CCP_AUTH_ALGO_SHA384_HMAC:
 530                 count = SHA512_DIGEST_SIZE >> 3;
 531
 532                 if (partial_hash_sha384(ipad, (uint8_t *)hash_temp64))
 533                         return -1;
 534                 for (i = 0; i < count; i++, hash_value_be64++)
 535                         *hash_value_be64 = hash_temp64[count - 1 - i];
 536
 537                 hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute
 538                                                + sess->auth.ctx_len);
 539                 if (partial_hash_sha384(opad, (uint8_t *)hash_temp64))
 540                         return -1;
 541                 for (i = 0; i < count; i++, hash_value_be64++)
 542                         *hash_value_be64 = hash_temp64[count - 1 - i];
 543                 return 0;
 544         case CCP_AUTH_ALGO_SHA3_384_HMAC:
 545                 hash_value_sha3 = sess->auth.pre_compute;
 546                 if (partial_hash_sha3_384(ipad, hash_value_sha3))
 547                         return -1;
 548
 549                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 550                                               + sess->auth.ctx_len);
 551                 if (partial_hash_sha3_384(opad, hash_value_sha3))
 552                         return -1;
 553                 return 0;
 554         case CCP_AUTH_ALGO_SHA512_HMAC:
 555                 count = SHA512_DIGEST_SIZE >> 3;
 556
 557                 if (partial_hash_sha512(ipad, (uint8_t *)hash_temp64))
 558                         return -1;
 559                 for (i = 0; i < count; i++, hash_value_be64++)
 560                         *hash_value_be64 = hash_temp64[count - 1 - i];
 561
 562                 hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute
 563                                                + sess->auth.ctx_len);
 564                 if (partial_hash_sha512(opad, (uint8_t *)hash_temp64))
 565                         return -1;
 566                 for (i = 0; i < count; i++, hash_value_be64++)
 567                         *hash_value_be64 = hash_temp64[count - 1 - i];
 568                 return 0;
 569         case CCP_AUTH_ALGO_SHA3_512_HMAC:
 570                 hash_value_sha3 = sess->auth.pre_compute;
 571                 if (partial_hash_sha3_512(ipad, hash_value_sha3))
 572                         return -1;
 573
 574                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 575                                               + sess->auth.ctx_len);
 576                 if (partial_hash_sha3_512(opad, hash_value_sha3))
 577                         return -1;
 578                 return 0;
 579         default:
 580                 CCP_LOG_ERR("Invalid auth algo");
 581                 return -1;
 582         }
 583 }
 584
 585 /* prepare temporary keys K1 and K2 */
 586 static void prepare_key(unsigned char *k, unsigned char *l, int bl)
 587 {
 588         int i;
 589         /* Shift block to left, including carry */
 590         for (i = 0; i < bl; i++) {
 591                 k[i] = l[i] << 1;
 592                 if (i < bl - 1 && l[i + 1] & 0x80)
 593                         k[i] |= 1;
 594         }
 595         /* If MSB set fixup with R */
 596         if (l[0] & 0x80)
 597                 k[bl - 1] ^= bl == 16 ? 0x87 : 0x1b;
 598 }
 599
 600 /* subkeys K1 and K2 generation for CMAC */
 601 static int
 602 generate_cmac_subkeys(struct ccp_session *sess)
 603 {
 604         const EVP_CIPHER *algo;
 605         EVP_CIPHER_CTX *ctx;
 606         unsigned char *ccp_ctx;
 607         size_t i;
 608         int dstlen, totlen;
 609         unsigned char zero_iv[AES_BLOCK_SIZE] = {0};
 610         unsigned char dst[2 * AES_BLOCK_SIZE] = {0};
 611         unsigned char k1[AES_BLOCK_SIZE] = {0};
 612         unsigned char k2[AES_BLOCK_SIZE] = {0};
 613
 614         if (sess->auth.ut.aes_type == CCP_AES_TYPE_128)
 615                 algo =  EVP_aes_128_cbc();
 616         else if (sess->auth.ut.aes_type == CCP_AES_TYPE_192)
 617                 algo =  EVP_aes_192_cbc();
 618         else if (sess->auth.ut.aes_type == CCP_AES_TYPE_256)
 619                 algo =  EVP_aes_256_cbc();
 620         else {
 621                 CCP_LOG_ERR("Invalid CMAC type length");
 622                 return -1;
 623         }
 624
 625         ctx = EVP_CIPHER_CTX_new();
 626         if (!ctx) {
 627                 CCP_LOG_ERR("ctx creation failed");
 628                 return -1;
 629         }
 630         if (EVP_EncryptInit(ctx, algo, (unsigned char *)sess->auth.key,
 631                             (unsigned char *)zero_iv) <= 0)
 632                 goto key_generate_err;
 633         if (EVP_CIPHER_CTX_set_padding(ctx, 0) <= 0)
 634                 goto key_generate_err;
 635         if (EVP_EncryptUpdate(ctx, dst, &dstlen, zero_iv,
 636                               AES_BLOCK_SIZE) <= 0)
 637                 goto key_generate_err;
 638         if (EVP_EncryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0)
 639                 goto key_generate_err;
 640
 641         memset(sess->auth.pre_compute, 0, CCP_SB_BYTES * 2);
 642
 643         ccp_ctx = (unsigned char *)(sess->auth.pre_compute + CCP_SB_BYTES - 1);
 644         prepare_key(k1, dst, AES_BLOCK_SIZE);
 645         for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
 646                 *ccp_ctx = k1[i];
 647
 648         ccp_ctx = (unsigned char *)(sess->auth.pre_compute +
 649                                    (2 * CCP_SB_BYTES) - 1);
 650         prepare_key(k2, k1, AES_BLOCK_SIZE);
 651         for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
 652                 *ccp_ctx = k2[i];
 653
 654         EVP_CIPHER_CTX_free(ctx);
 655
 656         return 0;
 657
 658 key_generate_err:
 659         CCP_LOG_ERR("CMAC Init failed");
 660                 return -1;
 661 }
 662
 663 /* configure session */
 664 static int
 665 ccp_configure_session_cipher(struct ccp_session *sess,
 666                              const struct rte_crypto_sym_xform *xform)
 667 {
 668         const struct rte_crypto_cipher_xform *cipher_xform = NULL;
 669         size_t i, j, x;
 670
 671         cipher_xform = &xform->cipher;
 672
 673         /* set cipher direction */
 674         if (cipher_xform->op ==  RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 675                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
 676         else
 677                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
 678
 679         /* set cipher key */
 680         sess->cipher.key_length = cipher_xform->key.length;
 681         rte_memcpy(sess->cipher.key, cipher_xform->key.data,
 682                    cipher_xform->key.length);
 683
 684         /* set iv parameters */
 685         sess->iv.offset = cipher_xform->iv.offset;
 686         sess->iv.length = cipher_xform->iv.length;
 687
 688         switch (cipher_xform->algo) {
 689         case RTE_CRYPTO_CIPHER_AES_CTR:
 690                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CTR;
 691                 sess->cipher.um.aes_mode = CCP_AES_MODE_CTR;
 692                 sess->cipher.engine = CCP_ENGINE_AES;
 693                 break;
 694         case RTE_CRYPTO_CIPHER_AES_ECB:
 695                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
 696                 sess->cipher.um.aes_mode = CCP_AES_MODE_ECB;
 697                 sess->cipher.engine = CCP_ENGINE_AES;
 698                 break;
 699         case RTE_CRYPTO_CIPHER_AES_CBC:
 700                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
 701                 sess->cipher.um.aes_mode = CCP_AES_MODE_CBC;
 702                 sess->cipher.engine = CCP_ENGINE_AES;
 703                 break;
 704         case RTE_CRYPTO_CIPHER_3DES_CBC:
 705                 sess->cipher.algo = CCP_CIPHER_ALGO_3DES_CBC;
 706                 sess->cipher.um.des_mode = CCP_DES_MODE_CBC;
 707                 sess->cipher.engine = CCP_ENGINE_3DES;
 708                 break;
 709         default:
 710                 CCP_LOG_ERR("Unsupported cipher algo");
 711                 return -1;
 712         }
 713
 714
 715         switch (sess->cipher.engine) {
 716         case CCP_ENGINE_AES:
 717                 if (sess->cipher.key_length == 16)
 718                         sess->cipher.ut.aes_type = CCP_AES_TYPE_128;
 719                 else if (sess->cipher.key_length == 24)
 720                         sess->cipher.ut.aes_type = CCP_AES_TYPE_192;
 721                 else if (sess->cipher.key_length == 32)
 722                         sess->cipher.ut.aes_type = CCP_AES_TYPE_256;
 723                 else {
 724                         CCP_LOG_ERR("Invalid cipher key length");
 725                         return -1;
 726                 }
 727                 for (i = 0; i < sess->cipher.key_length ; i++)
 728                         sess->cipher.key_ccp[sess->cipher.key_length - i - 1] =
 729                                 sess->cipher.key[i];
 730                 break;
 731         case CCP_ENGINE_3DES:
 732                 if (sess->cipher.key_length == 16)
 733                         sess->cipher.ut.des_type = CCP_DES_TYPE_128;
 734                 else if (sess->cipher.key_length == 24)
 735                         sess->cipher.ut.des_type = CCP_DES_TYPE_192;
 736                 else {
 737                         CCP_LOG_ERR("Invalid cipher key length");
 738                         return -1;
 739                 }
 740                 for (j = 0, x = 0; j < sess->cipher.key_length/8; j++, x += 8)
 741                         for (i = 0; i < 8; i++)
 742                                 sess->cipher.key_ccp[(8 + x) - i - 1] =
 743                                         sess->cipher.key[i + x];
 744                 break;
 745         default:
 746                 CCP_LOG_ERR("Invalid CCP Engine");
 747                 return -ENOTSUP;
 748         }
 749         if (iommu_mode == 2) {
 750                 sess->cipher.nonce_phys = rte_mem_virt2iova(sess->cipher.nonce);
 751                 sess->cipher.key_phys = rte_mem_virt2iova(sess->cipher.key_ccp);
 752         } else {
 753                 sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce);
 754                 sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp);
 755         }
 756         return 0;
 757 }
 758
 759 static int
 760 ccp_configure_session_auth(struct ccp_session *sess,
 761                            const struct rte_crypto_sym_xform *xform)
 762 {
 763         const struct rte_crypto_auth_xform *auth_xform = NULL;
 764         size_t i;
 765
 766         auth_xform = &xform->auth;
 767
 768         sess->auth.digest_length = auth_xform->digest_length;
 769         if (auth_xform->op ==  RTE_CRYPTO_AUTH_OP_GENERATE)
 770                 sess->auth.op = CCP_AUTH_OP_GENERATE;
 771         else
 772                 sess->auth.op = CCP_AUTH_OP_VERIFY;
 773         switch (auth_xform->algo) {
 774         case RTE_CRYPTO_AUTH_MD5_HMAC:
 775                 if (sess->auth_opt) {
 776                         sess->auth.algo = CCP_AUTH_ALGO_MD5_HMAC;
 777                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
 778                                              MD5_DIGEST_SIZE);
 779                         sess->auth.key_length = auth_xform->key.length;
 780                         sess->auth.block_size = MD5_BLOCK_SIZE;
 781                         memset(sess->auth.key, 0, sess->auth.block_size);
 782                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 783                                    auth_xform->key.length);
 784                 } else
 785                         return -1; /* HMAC MD5 not supported on CCP */
 786                 break;
 787         case RTE_CRYPTO_AUTH_SHA1:
 788                 sess->auth.engine = CCP_ENGINE_SHA;
 789                 sess->auth.algo = CCP_AUTH_ALGO_SHA1;
 790                 sess->auth.ut.sha_type = CCP_SHA_TYPE_1;
 791                 sess->auth.ctx = (void *)ccp_sha1_init;
 792                 sess->auth.ctx_len = CCP_SB_BYTES;
 793                 sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 794                 rte_memcpy(sha_ctx, sess->auth.ctx, SHA_COMMON_DIGEST_SIZE);
 795                 break;
 796         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 797                 if (sess->auth_opt) {
 798                         if (auth_xform->key.length > SHA1_BLOCK_SIZE)
 799                                 return -1;
 800                         sess->auth.algo = CCP_AUTH_ALGO_SHA1_HMAC;
 801                         sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 802                         sess->auth.block_size = SHA1_BLOCK_SIZE;
 803                         sess->auth.key_length = auth_xform->key.length;
 804                         memset(sess->auth.key, 0, sess->auth.block_size);
 805                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 806                                    auth_xform->key.length);
 807                 } else {
 808                         if (auth_xform->key.length > SHA1_BLOCK_SIZE)
 809                                 return -1;
 810                         sess->auth.engine = CCP_ENGINE_SHA;
 811                         sess->auth.algo = CCP_AUTH_ALGO_SHA1_HMAC;
 812                         sess->auth.ut.sha_type = CCP_SHA_TYPE_1;
 813                         sess->auth.ctx_len = CCP_SB_BYTES;
 814                         sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 815                         sess->auth.block_size = SHA1_BLOCK_SIZE;
 816                         sess->auth.key_length = auth_xform->key.length;
 817                         memset(sess->auth.key, 0, sess->auth.block_size);
 818                         memset(sess->auth.pre_compute, 0,
 819                                sess->auth.ctx_len << 1);
 820                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 821                                    auth_xform->key.length);
 822                         if (generate_partial_hash(sess))
 823                                 return -1;
 824                 }
 825                 break;
 826         case RTE_CRYPTO_AUTH_SHA224:
 827                 sess->auth.algo = CCP_AUTH_ALGO_SHA224;
 828                 sess->auth.engine = CCP_ENGINE_SHA;
 829                 sess->auth.ut.sha_type = CCP_SHA_TYPE_224;
 830                 sess->auth.ctx = (void *)ccp_sha224_init;
 831                 sess->auth.ctx_len = CCP_SB_BYTES;
 832                 sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 833                 rte_memcpy(sha_ctx, sess->auth.ctx, SHA256_DIGEST_SIZE);
 834                 break;
 835         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 836                 if (sess->auth_opt) {
 837                         if (auth_xform->key.length > SHA224_BLOCK_SIZE)
 838                                 return -1;
 839                         sess->auth.algo = CCP_AUTH_ALGO_SHA224_HMAC;
 840                         sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 841                         sess->auth.block_size = SHA224_BLOCK_SIZE;
 842                         sess->auth.key_length = auth_xform->key.length;
 843                         memset(sess->auth.key, 0, sess->auth.block_size);
 844                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 845                                    auth_xform->key.length);
 846                 } else {
 847                         if (auth_xform->key.length > SHA224_BLOCK_SIZE)
 848                                 return -1;
 849                         sess->auth.algo = CCP_AUTH_ALGO_SHA224_HMAC;
 850                         sess->auth.engine = CCP_ENGINE_SHA;
 851                         sess->auth.ut.sha_type = CCP_SHA_TYPE_224;
 852                         sess->auth.ctx_len = CCP_SB_BYTES;
 853                         sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 854                         sess->auth.block_size = SHA224_BLOCK_SIZE;
 855                         sess->auth.key_length = auth_xform->key.length;
 856                         memset(sess->auth.key, 0, sess->auth.block_size);
 857                         memset(sess->auth.pre_compute, 0,
 858                                sess->auth.ctx_len << 1);
 859                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 860                                    auth_xform->key.length);
 861                         if (generate_partial_hash(sess))
 862                                 return -1;
 863                 }
 864                 break;
 865         case RTE_CRYPTO_AUTH_SHA3_224:
 866                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_224;
 867                 sess->auth.engine = CCP_ENGINE_SHA;
 868                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
 869                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 870                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
 871                 break;
 872         case RTE_CRYPTO_AUTH_SHA3_224_HMAC:
 873                 if (auth_xform->key.length > SHA3_224_BLOCK_SIZE)
 874                         return -1;
 875                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_224_HMAC;
 876                 sess->auth.engine = CCP_ENGINE_SHA;
 877                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
 878                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 879                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
 880                 sess->auth.block_size = SHA3_224_BLOCK_SIZE;
 881                 sess->auth.key_length = auth_xform->key.length;
 882                 memset(sess->auth.key, 0, sess->auth.block_size);
 883                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
 884                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 885                            auth_xform->key.length);
 886                 if (generate_partial_hash(sess))
 887                         return -1;
 888                 break;
 889         case RTE_CRYPTO_AUTH_SHA256:
 890                 sess->auth.algo = CCP_AUTH_ALGO_SHA256;
 891                 sess->auth.engine = CCP_ENGINE_SHA;
 892                 sess->auth.ut.sha_type = CCP_SHA_TYPE_256;
 893                 sess->auth.ctx = (void *)ccp_sha256_init;
 894                 sess->auth.ctx_len = CCP_SB_BYTES;
 895                 sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 896                 rte_memcpy(sha_ctx, sess->auth.ctx, SHA256_DIGEST_SIZE);
 897                 break;
 898         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 899                 if (sess->auth_opt) {
 900                         if (auth_xform->key.length > SHA256_BLOCK_SIZE)
 901                                 return -1;
 902                         sess->auth.algo = CCP_AUTH_ALGO_SHA256_HMAC;
 903                         sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 904                         sess->auth.block_size = SHA256_BLOCK_SIZE;
 905                         sess->auth.key_length = auth_xform->key.length;
 906                         memset(sess->auth.key, 0, sess->auth.block_size);
 907                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 908                                    auth_xform->key.length);
 909                 } else {
 910                         if (auth_xform->key.length > SHA256_BLOCK_SIZE)
 911                                 return -1;
 912                         sess->auth.algo = CCP_AUTH_ALGO_SHA256_HMAC;
 913                         sess->auth.engine = CCP_ENGINE_SHA;
 914                         sess->auth.ut.sha_type = CCP_SHA_TYPE_256;
 915                         sess->auth.ctx_len = CCP_SB_BYTES;
 916                         sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 917                         sess->auth.block_size = SHA256_BLOCK_SIZE;
 918                         sess->auth.key_length = auth_xform->key.length;
 919                         memset(sess->auth.key, 0, sess->auth.block_size);
 920                         memset(sess->auth.pre_compute, 0,
 921                                sess->auth.ctx_len << 1);
 922                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 923                                    auth_xform->key.length);
 924                         if (generate_partial_hash(sess))
 925                                 return -1;
 926                 }
 927                 break;
 928         case RTE_CRYPTO_AUTH_SHA3_256:
 929                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_256;
 930                 sess->auth.engine = CCP_ENGINE_SHA;
 931                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
 932                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 933                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
 934                 break;
 935         case RTE_CRYPTO_AUTH_SHA3_256_HMAC:
 936                 if (auth_xform->key.length > SHA3_256_BLOCK_SIZE)
 937                         return -1;
 938                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_256_HMAC;
 939                 sess->auth.engine = CCP_ENGINE_SHA;
 940                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
 941                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 942                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
 943                 sess->auth.block_size = SHA3_256_BLOCK_SIZE;
 944                 sess->auth.key_length = auth_xform->key.length;
 945                 memset(sess->auth.key, 0, sess->auth.block_size);
 946                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
 947                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 948                            auth_xform->key.length);
 949                 if (generate_partial_hash(sess))
 950                         return -1;
 951                 break;
 952         case RTE_CRYPTO_AUTH_SHA384:
 953                 sess->auth.algo = CCP_AUTH_ALGO_SHA384;
 954                 sess->auth.engine = CCP_ENGINE_SHA;
 955                 sess->auth.ut.sha_type = CCP_SHA_TYPE_384;
 956                 sess->auth.ctx = (void *)ccp_sha384_init;
 957                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
 958                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA384_DIGEST_SIZE;
 959                 rte_memcpy(sha_ctx, sess->auth.ctx, SHA512_DIGEST_SIZE);
 960                 break;
 961         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 962                 if (sess->auth_opt) {
 963                         if (auth_xform->key.length > SHA384_BLOCK_SIZE)
 964                                 return -1;
 965                         sess->auth.algo = CCP_AUTH_ALGO_SHA384_HMAC;
 966                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
 967                                              SHA384_DIGEST_SIZE);
 968                         sess->auth.block_size = SHA384_BLOCK_SIZE;
 969                         sess->auth.key_length = auth_xform->key.length;
 970                         memset(sess->auth.key, 0, sess->auth.block_size);
 971                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 972                                    auth_xform->key.length);
 973                 } else {
 974                         if (auth_xform->key.length > SHA384_BLOCK_SIZE)
 975                                 return -1;
 976                         sess->auth.algo = CCP_AUTH_ALGO_SHA384_HMAC;
 977                         sess->auth.engine = CCP_ENGINE_SHA;
 978                         sess->auth.ut.sha_type = CCP_SHA_TYPE_384;
 979                         sess->auth.ctx_len = CCP_SB_BYTES << 1;
 980                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
 981                                              SHA384_DIGEST_SIZE);
 982                         sess->auth.block_size = SHA384_BLOCK_SIZE;
 983                         sess->auth.key_length = auth_xform->key.length;
 984                         memset(sess->auth.key, 0, sess->auth.block_size);
 985                         memset(sess->auth.pre_compute, 0,
 986                                sess->auth.ctx_len << 1);
 987                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 988                                    auth_xform->key.length);
 989                         if (generate_partial_hash(sess))
 990                                 return -1;
 991                 }
 992                 break;
 993         case RTE_CRYPTO_AUTH_SHA3_384:
 994                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_384;
 995                 sess->auth.engine = CCP_ENGINE_SHA;
 996                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
 997                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 998                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
 999                 break;
1000         case RTE_CRYPTO_AUTH_SHA3_384_HMAC:
1001                 if (auth_xform->key.length > SHA3_384_BLOCK_SIZE)
1002                         return -1;
1003                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_384_HMAC;
1004                 sess->auth.engine = CCP_ENGINE_SHA;
1005                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
1006                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
1007                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
1008                 sess->auth.block_size = SHA3_384_BLOCK_SIZE;
1009                 sess->auth.key_length = auth_xform->key.length;
1010                 memset(sess->auth.key, 0, sess->auth.block_size);
1011                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
1012                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1013                            auth_xform->key.length);
1014                 if (generate_partial_hash(sess))
1015                         return -1;
1016                 break;
1017         case RTE_CRYPTO_AUTH_SHA512:
1018                 sess->auth.algo = CCP_AUTH_ALGO_SHA512;
1019                 sess->auth.engine = CCP_ENGINE_SHA;
1020                 sess->auth.ut.sha_type = CCP_SHA_TYPE_512;
1021                 sess->auth.ctx = (void *)ccp_sha512_init;
1022                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
1023                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA512_DIGEST_SIZE;
1024                 rte_memcpy(sha_ctx, sess->auth.ctx, SHA512_DIGEST_SIZE);
1025                 break;
1026         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1027                 if (sess->auth_opt) {
1028                         if (auth_xform->key.length > SHA512_BLOCK_SIZE)
1029                                 return -1;
1030                         sess->auth.algo = CCP_AUTH_ALGO_SHA512_HMAC;
1031                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
1032                                              SHA512_DIGEST_SIZE);
1033                         sess->auth.block_size = SHA512_BLOCK_SIZE;
1034                         sess->auth.key_length = auth_xform->key.length;
1035                         memset(sess->auth.key, 0, sess->auth.block_size);
1036                         rte_memcpy(sess->auth.key, auth_xform->key.data,
1037                                    auth_xform->key.length);
1038                 } else {
1039                         if (auth_xform->key.length > SHA512_BLOCK_SIZE)
1040                                 return -1;
1041                         sess->auth.algo = CCP_AUTH_ALGO_SHA512_HMAC;
1042                         sess->auth.engine = CCP_ENGINE_SHA;
1043                         sess->auth.ut.sha_type = CCP_SHA_TYPE_512;
1044                         sess->auth.ctx_len = CCP_SB_BYTES << 1;
1045                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
1046                                              SHA512_DIGEST_SIZE);
1047                         sess->auth.block_size = SHA512_BLOCK_SIZE;
1048                         sess->auth.key_length = auth_xform->key.length;
1049                         memset(sess->auth.key, 0, sess->auth.block_size);
1050                         memset(sess->auth.pre_compute, 0,
1051                                sess->auth.ctx_len << 1);
1052                         rte_memcpy(sess->auth.key, auth_xform->key.data,
1053                                    auth_xform->key.length);
1054                         if (generate_partial_hash(sess))
1055                                 return -1;
1056                 }
1057                 break;
1058         case RTE_CRYPTO_AUTH_SHA3_512:
1059                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_512;
1060                 sess->auth.engine = CCP_ENGINE_SHA;
1061                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
1062                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
1063                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
1064                 break;
1065         case RTE_CRYPTO_AUTH_SHA3_512_HMAC:
1066                 if (auth_xform->key.length > SHA3_512_BLOCK_SIZE)
1067                         return -1;
1068                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_512_HMAC;
1069                 sess->auth.engine = CCP_ENGINE_SHA;
1070                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
1071                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
1072                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
1073                 sess->auth.block_size = SHA3_512_BLOCK_SIZE;
1074                 sess->auth.key_length = auth_xform->key.length;
1075                 memset(sess->auth.key, 0, sess->auth.block_size);
1076                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
1077                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1078                            auth_xform->key.length);
1079                 if (generate_partial_hash(sess))
1080                         return -1;
1081                 break;
1082         case RTE_CRYPTO_AUTH_AES_CMAC:
1083                 sess->auth.algo = CCP_AUTH_ALGO_AES_CMAC;
1084                 sess->auth.engine = CCP_ENGINE_AES;
1085                 sess->auth.um.aes_mode = CCP_AES_MODE_CMAC;
1086                 sess->auth.key_length = auth_xform->key.length;
1087                 /* padding and hash result */
1088                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
1089                 sess->auth.offset = AES_BLOCK_SIZE;
1090                 sess->auth.block_size = AES_BLOCK_SIZE;
1091                 if (sess->auth.key_length == 16)
1092                         sess->auth.ut.aes_type = CCP_AES_TYPE_128;
1093                 else if (sess->auth.key_length == 24)
1094                         sess->auth.ut.aes_type = CCP_AES_TYPE_192;
1095                 else if (sess->auth.key_length == 32)
1096                         sess->auth.ut.aes_type = CCP_AES_TYPE_256;
1097                 else {
1098                         CCP_LOG_ERR("Invalid CMAC key length");
1099                         return -1;
1100                 }
1101                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1102                            sess->auth.key_length);
1103                 for (i = 0; i < sess->auth.key_length; i++)
1104                         sess->auth.key_ccp[sess->auth.key_length - i - 1] =
1105                                 sess->auth.key[i];
1106                 if (generate_cmac_subkeys(sess))
1107                         return -1;
1108                 break;
1109         default:
1110                 CCP_LOG_ERR("Unsupported hash algo");
1111                 return -ENOTSUP;
1112         }
1113         return 0;
1114 }
1115
1116 static int
1117 ccp_configure_session_aead(struct ccp_session *sess,
1118                            const struct rte_crypto_sym_xform *xform)
1119 {
1120         const struct rte_crypto_aead_xform *aead_xform = NULL;
1121         size_t i;
1122
1123         aead_xform = &xform->aead;
1124
1125         sess->cipher.key_length = aead_xform->key.length;
1126         rte_memcpy(sess->cipher.key, aead_xform->key.data,
1127                    aead_xform->key.length);
1128
1129         if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
1130                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
1131                 sess->auth.op = CCP_AUTH_OP_GENERATE;
1132         } else {
1133                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
1134                 sess->auth.op = CCP_AUTH_OP_VERIFY;
1135         }
1136         sess->aead_algo = aead_xform->algo;
1137         sess->auth.aad_length = aead_xform->aad_length;
1138         sess->auth.digest_length = aead_xform->digest_length;
1139
1140         /* set iv parameters */
1141         sess->iv.offset = aead_xform->iv.offset;
1142         sess->iv.length = aead_xform->iv.length;
1143
1144         switch (aead_xform->algo) {
1145         case RTE_CRYPTO_AEAD_AES_GCM:
1146                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_GCM;
1147                 sess->cipher.um.aes_mode = CCP_AES_MODE_GCTR;
1148                 sess->cipher.engine = CCP_ENGINE_AES;
1149                 if (sess->cipher.key_length == 16)
1150                         sess->cipher.ut.aes_type = CCP_AES_TYPE_128;
1151                 else if (sess->cipher.key_length == 24)
1152                         sess->cipher.ut.aes_type = CCP_AES_TYPE_192;
1153                 else if (sess->cipher.key_length == 32)
1154                         sess->cipher.ut.aes_type = CCP_AES_TYPE_256;
1155                 else {
1156                         CCP_LOG_ERR("Invalid aead key length");
1157                         return -1;
1158                 }
1159                 for (i = 0; i < sess->cipher.key_length; i++)
1160                         sess->cipher.key_ccp[sess->cipher.key_length - i - 1] =
1161                                 sess->cipher.key[i];
1162                 sess->auth.algo = CCP_AUTH_ALGO_AES_GCM;
1163                 sess->auth.engine = CCP_ENGINE_AES;
1164                 sess->auth.um.aes_mode = CCP_AES_MODE_GHASH;
1165                 sess->auth.ctx_len = CCP_SB_BYTES;
1166                 sess->auth.offset = 0;
1167                 sess->auth.block_size = AES_BLOCK_SIZE;
1168                 sess->cmd_id = CCP_CMD_COMBINED;
1169                 break;
1170         default:
1171                 CCP_LOG_ERR("Unsupported aead algo");
1172                 return -ENOTSUP;
1173         }
1174         if (iommu_mode == 2) {
1175                 sess->cipher.nonce_phys = rte_mem_virt2iova(sess->cipher.nonce);
1176                 sess->cipher.key_phys = rte_mem_virt2iova(sess->cipher.key_ccp);
1177         } else {
1178                 sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce);
1179                 sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp);
1180         }
1181         return 0;
1182 }
1183
1184 int
1185 ccp_set_session_parameters(struct ccp_session *sess,
1186                            const struct rte_crypto_sym_xform *xform,
1187                            struct ccp_private *internals)
1188 {
1189         const struct rte_crypto_sym_xform *cipher_xform = NULL;
1190         const struct rte_crypto_sym_xform *auth_xform = NULL;
1191         const struct rte_crypto_sym_xform *aead_xform = NULL;
1192         int ret = 0;
1193
1194         sess->auth_opt = internals->auth_opt;
1195         sess->cmd_id = ccp_get_cmd_id(xform);
1196
1197         switch (sess->cmd_id) {
1198         case CCP_CMD_CIPHER:
1199                 cipher_xform = xform;
1200                 break;
1201         case CCP_CMD_AUTH:
1202                 auth_xform = xform;
1203                 break;
1204         case CCP_CMD_CIPHER_HASH:
1205                 cipher_xform = xform;
1206                 auth_xform = xform->next;
1207                 break;
1208         case CCP_CMD_HASH_CIPHER:
1209                 auth_xform = xform;
1210                 cipher_xform = xform->next;
1211                 break;
1212         case CCP_CMD_COMBINED:
1213                 aead_xform = xform;
1214                 break;
1215         default:
1216                 CCP_LOG_ERR("Unsupported cmd_id");
1217                 return -1;
1218         }
1219
1220         /* Default IV length = 0 */
1221         sess->iv.length = 0;
1222         if (cipher_xform) {
1223                 ret = ccp_configure_session_cipher(sess, cipher_xform);
1224                 if (ret != 0) {
1225                         CCP_LOG_ERR("Invalid/unsupported cipher parameters");
1226                         return ret;
1227                 }
1228         }
1229         if (auth_xform) {
1230                 ret = ccp_configure_session_auth(sess, auth_xform);
1231                 if (ret != 0) {
1232                         CCP_LOG_ERR("Invalid/unsupported auth parameters");
1233                         return ret;
1234                 }
1235         }
1236         if (aead_xform) {
1237                 ret = ccp_configure_session_aead(sess, aead_xform);
1238                 if (ret != 0) {
1239                         CCP_LOG_ERR("Invalid/unsupported aead parameters");
1240                         return ret;
1241                 }
1242         }
1243         return ret;
1244 }
1245
1246 /* calculate CCP descriptors requirement */
1247 static inline int
1248 ccp_cipher_slot(struct ccp_session *session)
1249 {
1250         int count = 0;
1251
1252         switch (session->cipher.algo) {
1253         case CCP_CIPHER_ALGO_AES_CBC:
1254                 count = 2;
1255                 /**< op + passthrough for iv */
1256                 break;
1257         case CCP_CIPHER_ALGO_AES_ECB:
1258                 count = 1;
1259                 /**<only op*/
1260                 break;
1261         case CCP_CIPHER_ALGO_AES_CTR:
1262                 count = 2;
1263                 /**< op + passthrough for iv */
1264                 break;
1265         case CCP_CIPHER_ALGO_3DES_CBC:
1266                 count = 2;
1267                 /**< op + passthrough for iv */
1268                 break;
1269         default:
1270                 CCP_LOG_ERR("Unsupported cipher algo %d",
1271                             session->cipher.algo);
1272         }
1273         return count;
1274 }
1275
1276 static inline int
1277 ccp_auth_slot(struct ccp_session *session)
1278 {
1279         int count = 0;
1280
1281         switch (session->auth.algo) {
1282         case CCP_AUTH_ALGO_SHA1:
1283         case CCP_AUTH_ALGO_SHA224:
1284         case CCP_AUTH_ALGO_SHA256:
1285         case CCP_AUTH_ALGO_SHA384:
1286         case CCP_AUTH_ALGO_SHA512:
1287                 count = 3;
1288                 /**< op + lsb passthrough cpy to/from*/
1289                 break;
1290         case CCP_AUTH_ALGO_MD5_HMAC:
1291                 break;
1292         case CCP_AUTH_ALGO_SHA1_HMAC:
1293         case CCP_AUTH_ALGO_SHA224_HMAC:
1294         case CCP_AUTH_ALGO_SHA256_HMAC:
1295                 if (session->auth_opt == 0)
1296                         count = 6;
1297                 break;
1298         case CCP_AUTH_ALGO_SHA384_HMAC:
1299         case CCP_AUTH_ALGO_SHA512_HMAC:
1300                 /**
1301                  * 1. Load PHash1 = H(k ^ ipad); to LSB
1302                  * 2. generate IHash = H(hash on meassage with PHash1
1303                  * as init values);
1304                  * 3. Retrieve IHash 2 slots for 384/512
1305                  * 4. Load Phash2 = H(k ^ opad); to LSB
1306                  * 5. generate FHash = H(hash on Ihash with Phash2
1307                  * as init value);
1308                  * 6. Retrieve HMAC output from LSB to host memory
1309                  */
1310                 if (session->auth_opt == 0)
1311                         count = 7;
1312                 break;
1313         case CCP_AUTH_ALGO_SHA3_224:
1314         case CCP_AUTH_ALGO_SHA3_256:
1315         case CCP_AUTH_ALGO_SHA3_384:
1316         case CCP_AUTH_ALGO_SHA3_512:
1317                 count = 1;
1318                 /**< only op ctx and dst in host memory*/
1319                 break;
1320         case CCP_AUTH_ALGO_SHA3_224_HMAC:
1321         case CCP_AUTH_ALGO_SHA3_256_HMAC:
1322                 count = 3;
1323                 break;
1324         case CCP_AUTH_ALGO_SHA3_384_HMAC:
1325         case CCP_AUTH_ALGO_SHA3_512_HMAC:
1326                 count = 4;
1327                 /**
1328                  * 1. Op to Perform Ihash
1329                  * 2. Retrieve result from LSB to host memory
1330                  * 3. Perform final hash
1331                  */
1332                 break;
1333         case CCP_AUTH_ALGO_AES_CMAC:
1334                 count = 4;
1335                 /**
1336                  * op
1337                  * extra descriptor in padding case
1338                  * (k1/k2(255:128) with iv(127:0))
1339                  * Retrieve result
1340                  */
1341                 break;
1342         default:
1343                 CCP_LOG_ERR("Unsupported auth algo %d",
1344                             session->auth.algo);
1345         }
1346
1347         return count;
1348 }
1349
1350 static int
1351 ccp_aead_slot(struct ccp_session *session)
1352 {
1353         int count = 0;
1354
1355         switch (session->aead_algo) {
1356         case RTE_CRYPTO_AEAD_AES_GCM:
1357                 break;
1358         default:
1359                 CCP_LOG_ERR("Unsupported aead algo %d",
1360                             session->aead_algo);
1361         }
1362         switch (session->auth.algo) {
1363         case CCP_AUTH_ALGO_AES_GCM:
1364                 count = 5;
1365                 /**
1366                  * 1. Passthru iv
1367                  * 2. Hash AAD
1368                  * 3. GCTR
1369                  * 4. Reload passthru
1370                  * 5. Hash Final tag
1371                  */
1372                 break;
1373         default:
1374                 CCP_LOG_ERR("Unsupported combined auth ALGO %d",
1375                             session->auth.algo);
1376         }
1377         return count;
1378 }
1379
1380 int
1381 ccp_compute_slot_count(struct ccp_session *session)
1382 {
1383         int count = 0;
1384
1385         switch (session->cmd_id) {
1386         case CCP_CMD_CIPHER:
1387                 count = ccp_cipher_slot(session);
1388                 break;
1389         case CCP_CMD_AUTH:
1390                 count = ccp_auth_slot(session);
1391                 break;
1392         case CCP_CMD_CIPHER_HASH:
1393         case CCP_CMD_HASH_CIPHER:
1394                 count = ccp_cipher_slot(session);
1395                 count += ccp_auth_slot(session);
1396                 break;
1397         case CCP_CMD_COMBINED:
1398                 count = ccp_aead_slot(session);
1399                 break;
1400         default:
1401                 CCP_LOG_ERR("Unsupported cmd_id");
1402
1403         }
1404
1405         return count;
1406 }
1407
1408 static uint8_t
1409 algo_select(int sessalgo,
1410             const EVP_MD **algo)
1411 {
1412         int res = 0;
1413
1414         switch (sessalgo) {
1415         case CCP_AUTH_ALGO_MD5_HMAC:
1416                 *algo = EVP_md5();
1417                 break;
1418         case CCP_AUTH_ALGO_SHA1_HMAC:
1419                 *algo = EVP_sha1();
1420                 break;
1421         case CCP_AUTH_ALGO_SHA224_HMAC:
1422                 *algo = EVP_sha224();
1423                 break;
1424         case CCP_AUTH_ALGO_SHA256_HMAC:
1425                 *algo = EVP_sha256();
1426                 break;
1427         case CCP_AUTH_ALGO_SHA384_HMAC:
1428                 *algo = EVP_sha384();
1429                 break;
1430         case CCP_AUTH_ALGO_SHA512_HMAC:
1431                 *algo = EVP_sha512();
1432                 break;
1433         default:
1434                 res = -EINVAL;
1435                 break;
1436         }
1437         return res;
1438 }
1439
1440 static int
1441 process_cpu_auth_hmac(uint8_t *src, uint8_t *dst,
1442                       __rte_unused uint8_t *iv,
1443                       EVP_PKEY *pkey,
1444                       int srclen,
1445                       EVP_MD_CTX *ctx,
1446                       const EVP_MD *algo,
1447                       uint16_t d_len)
1448 {
1449         size_t dstlen;
1450         unsigned char temp_dst[64];
1451
1452         if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0)
1453                 goto process_auth_err;
1454
1455         if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0)
1456                 goto process_auth_err;
1457
1458         if (EVP_DigestSignFinal(ctx, temp_dst, &dstlen) <= 0)
1459                 goto process_auth_err;
1460
1461         memcpy(dst, temp_dst, d_len);
1462         return 0;
1463 process_auth_err:
1464         CCP_LOG_ERR("Process cpu auth failed");
1465         return -EINVAL;
1466 }
1467
1468 static int cpu_crypto_auth(struct ccp_qp *qp,
1469                            struct rte_crypto_op *op,
1470                            struct ccp_session *sess,
1471                            EVP_MD_CTX *ctx)
1472 {
1473         uint8_t *src, *dst;
1474         int srclen, status;
1475         struct rte_mbuf *mbuf_src, *mbuf_dst;
1476         const EVP_MD *algo = NULL;
1477         EVP_PKEY *pkey;
1478
1479         algo_select(sess->auth.algo, &algo);
1480         pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, sess->auth.key,
1481                                     sess->auth.key_length);
1482         mbuf_src = op->sym->m_src;
1483         mbuf_dst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1484         srclen = op->sym->auth.data.length;
1485         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1486                                       op->sym->auth.data.offset);
1487
1488         if (sess->auth.op == CCP_AUTH_OP_VERIFY) {
1489                 dst = qp->temp_digest;
1490         } else {
1491                 dst = op->sym->auth.digest.data;
1492                 if (dst == NULL) {
1493                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1494                                                      op->sym->auth.data.offset +
1495                                                      sess->auth.digest_length);
1496                 }
1497         }
1498         status = process_cpu_auth_hmac(src, dst, NULL,
1499                                        pkey, srclen,
1500                                        ctx,
1501                                        algo,
1502                                        sess->auth.digest_length);
1503         if (status) {
1504                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1505                 return status;
1506         }
1507
1508         if (sess->auth.op == CCP_AUTH_OP_VERIFY) {
1509                 if (memcmp(dst, op->sym->auth.digest.data,
1510                            sess->auth.digest_length) != 0) {
1511                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1512                 } else {
1513                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1514                 }
1515         } else {
1516                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1517         }
1518         EVP_PKEY_free(pkey);
1519         return 0;
1520 }
1521
1522 static void
1523 ccp_perform_passthru(struct ccp_passthru *pst,
1524                      struct ccp_queue *cmd_q)
1525 {
1526         struct ccp_desc *desc;
1527         union ccp_function function;
1528
1529         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1530
1531         CCP_CMD_ENGINE(desc) = CCP_ENGINE_PASSTHRU;
1532
1533         CCP_CMD_SOC(desc) = 0;
1534         CCP_CMD_IOC(desc) = 0;
1535         CCP_CMD_INIT(desc) = 0;
1536         CCP_CMD_EOM(desc) = 0;
1537         CCP_CMD_PROT(desc) = 0;
1538
1539         function.raw = 0;
1540         CCP_PT_BYTESWAP(&function) = pst->byte_swap;
1541         CCP_PT_BITWISE(&function) = pst->bit_mod;
1542         CCP_CMD_FUNCTION(desc) = function.raw;
1543
1544         CCP_CMD_LEN(desc) = pst->len;
1545
1546         if (pst->dir) {
1547                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
1548                 CCP_CMD_SRC_HI(desc) = high32_value(pst->src_addr);
1549                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1550
1551                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
1552                 CCP_CMD_DST_HI(desc) = 0;
1553                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
1554
1555                 if (pst->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1556                         CCP_CMD_LSB_ID(desc) = cmd_q->sb_key;
1557         } else {
1558
1559                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
1560                 CCP_CMD_SRC_HI(desc) = 0;
1561                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SB;
1562
1563                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
1564                 CCP_CMD_DST_HI(desc) = high32_value(pst->dest_addr);
1565                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1566         }
1567
1568         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1569 }
1570
1571 static int
1572 ccp_perform_hmac(struct rte_crypto_op *op,
1573                  struct ccp_queue *cmd_q)
1574 {
1575
1576         struct ccp_session *session;
1577         union ccp_function function;
1578         struct ccp_desc *desc;
1579         uint32_t tail;
1580         phys_addr_t src_addr, dest_addr, dest_addr_t;
1581         struct ccp_passthru pst;
1582         uint64_t auth_msg_bits;
1583         void *append_ptr;
1584         uint8_t *addr;
1585
1586         session = (struct ccp_session *)get_sym_session_private_data(
1587                                          op->sym->session,
1588                                          ccp_cryptodev_driver_id);
1589         addr = session->auth.pre_compute;
1590
1591         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
1592                                               op->sym->auth.data.offset);
1593         append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src,
1594                                                 session->auth.ctx_len);
1595         if (iommu_mode == 2) {
1596                 dest_addr = (phys_addr_t)rte_mem_virt2iova(append_ptr);
1597                 pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)addr);
1598         } else {
1599                 dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr);
1600                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr);
1601         }
1602         dest_addr_t = dest_addr;
1603
1604         /** Load PHash1 to LSB*/
1605         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1606         pst.len = session->auth.ctx_len;
1607         pst.dir = 1;
1608         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1609         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1610         ccp_perform_passthru(&pst, cmd_q);
1611
1612         /**sha engine command descriptor for IntermediateHash*/
1613
1614         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1615         memset(desc, 0, Q_DESC_SIZE);
1616
1617         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1618
1619         CCP_CMD_SOC(desc) = 0;
1620         CCP_CMD_IOC(desc) = 0;
1621         CCP_CMD_INIT(desc) = 1;
1622         CCP_CMD_EOM(desc) = 1;
1623         CCP_CMD_PROT(desc) = 0;
1624
1625         function.raw = 0;
1626         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1627         CCP_CMD_FUNCTION(desc) = function.raw;
1628
1629         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1630         auth_msg_bits = (op->sym->auth.data.length +
1631                          session->auth.block_size)  * 8;
1632
1633         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1634         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1635         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1636
1637         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1638         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1639         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1640
1641         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1642
1643         rte_wmb();
1644
1645         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1646         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1647         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1648                       cmd_q->qcontrol | CMD_Q_RUN);
1649
1650         /* Intermediate Hash value retrieve */
1651         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1652             (session->auth.ut.sha_type == CCP_SHA_TYPE_512)) {
1653
1654                 pst.src_addr =
1655                         (phys_addr_t)((cmd_q->sb_sha + 1) * CCP_SB_BYTES);
1656                 pst.dest_addr = dest_addr_t;
1657                 pst.len = CCP_SB_BYTES;
1658                 pst.dir = 0;
1659                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1660                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1661                 ccp_perform_passthru(&pst, cmd_q);
1662
1663                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1664                 pst.dest_addr = dest_addr_t + CCP_SB_BYTES;
1665                 pst.len = CCP_SB_BYTES;
1666                 pst.dir = 0;
1667                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1668                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1669                 ccp_perform_passthru(&pst, cmd_q);
1670
1671         } else {
1672                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1673                 pst.dest_addr = dest_addr_t;
1674                 pst.len = session->auth.ctx_len;
1675                 pst.dir = 0;
1676                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1677                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1678                 ccp_perform_passthru(&pst, cmd_q);
1679
1680         }
1681
1682         /** Load PHash2 to LSB*/
1683         addr += session->auth.ctx_len;
1684         if (iommu_mode == 2)
1685                 pst.src_addr = (phys_addr_t)rte_mem_virt2iova((void *)addr);
1686         else
1687                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr);
1688         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1689         pst.len = session->auth.ctx_len;
1690         pst.dir = 1;
1691         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1692         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1693         ccp_perform_passthru(&pst, cmd_q);
1694
1695         /**sha engine command descriptor for FinalHash*/
1696         dest_addr_t += session->auth.offset;
1697
1698         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1699         memset(desc, 0, Q_DESC_SIZE);
1700
1701         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1702
1703         CCP_CMD_SOC(desc) = 0;
1704         CCP_CMD_IOC(desc) = 0;
1705         CCP_CMD_INIT(desc) = 1;
1706         CCP_CMD_EOM(desc) = 1;
1707         CCP_CMD_PROT(desc) = 0;
1708
1709         function.raw = 0;
1710         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1711         CCP_CMD_FUNCTION(desc) = function.raw;
1712
1713         CCP_CMD_LEN(desc) = (session->auth.ctx_len -
1714                              session->auth.offset);
1715         auth_msg_bits = (session->auth.block_size +
1716                          session->auth.ctx_len -
1717                          session->auth.offset) * 8;
1718
1719         CCP_CMD_SRC_LO(desc) = (uint32_t)(dest_addr_t);
1720         CCP_CMD_SRC_HI(desc) = high32_value(dest_addr_t);
1721         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1722
1723         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1724         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1725         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1726
1727         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1728
1729         rte_wmb();
1730
1731         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1732         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1733         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1734                       cmd_q->qcontrol | CMD_Q_RUN);
1735
1736         /* Retrieve hmac output */
1737         pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1738         pst.dest_addr = dest_addr;
1739         pst.len = session->auth.ctx_len;
1740         pst.dir = 0;
1741         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1742         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1743             (session->auth.ut.sha_type == CCP_SHA_TYPE_512))
1744                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1745         else
1746                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1747         ccp_perform_passthru(&pst, cmd_q);
1748
1749         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1750         return 0;
1751
1752 }
1753
1754 static int
1755 ccp_perform_sha(struct rte_crypto_op *op,
1756                 struct ccp_queue *cmd_q)
1757 {
1758         struct ccp_session *session;
1759         union ccp_function function;
1760         struct ccp_desc *desc;
1761         uint32_t tail;
1762         phys_addr_t src_addr, dest_addr;
1763         struct ccp_passthru pst;
1764         void *append_ptr;
1765         uint64_t auth_msg_bits;
1766
1767         session = (struct ccp_session *)get_sym_session_private_data(
1768                                          op->sym->session,
1769                                         ccp_cryptodev_driver_id);
1770
1771         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
1772                                               op->sym->auth.data.offset);
1773         append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src,
1774                                                 session->auth.ctx_len);
1775         if (iommu_mode == 2) {
1776                 dest_addr = (phys_addr_t)rte_mem_virt2iova(append_ptr);
1777                 pst.src_addr = (phys_addr_t)sha_ctx;
1778         } else {
1779                 dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr);
1780                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)
1781                                                      session->auth.ctx);
1782         }
1783
1784         /** Passthru sha context*/
1785
1786         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1787         pst.len = session->auth.ctx_len;
1788         pst.dir = 1;
1789         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1790         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1791         ccp_perform_passthru(&pst, cmd_q);
1792
1793         /**prepare sha command descriptor*/
1794
1795         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1796         memset(desc, 0, Q_DESC_SIZE);
1797
1798         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1799
1800         CCP_CMD_SOC(desc) = 0;
1801         CCP_CMD_IOC(desc) = 0;
1802         CCP_CMD_INIT(desc) = 1;
1803         CCP_CMD_EOM(desc) = 1;
1804         CCP_CMD_PROT(desc) = 0;
1805
1806         function.raw = 0;
1807         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1808         CCP_CMD_FUNCTION(desc) = function.raw;
1809
1810         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1811         auth_msg_bits = op->sym->auth.data.length * 8;
1812
1813         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1814         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1815         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1816
1817         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1818         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1819         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1820
1821         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1822
1823         rte_wmb();
1824
1825         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1826         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1827         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1828                       cmd_q->qcontrol | CMD_Q_RUN);
1829
1830         /* Hash value retrieve */
1831         pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1832         pst.dest_addr = dest_addr;
1833         pst.len = session->auth.ctx_len;
1834         pst.dir = 0;
1835         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1836         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1837             (session->auth.ut.sha_type == CCP_SHA_TYPE_512))
1838                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1839         else
1840                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1841         ccp_perform_passthru(&pst, cmd_q);
1842
1843         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1844         return 0;
1845
1846 }
1847
1848 static int
1849 ccp_perform_sha3_hmac(struct rte_crypto_op *op,
1850                       struct ccp_queue *cmd_q)
1851 {
1852         struct ccp_session *session;
1853         struct ccp_passthru pst;
1854         union ccp_function function;
1855         struct ccp_desc *desc;
1856         uint8_t *append_ptr;
1857         uint32_t tail;
1858         phys_addr_t src_addr, dest_addr, ctx_paddr, dest_addr_t;
1859
1860         session = (struct ccp_session *)get_sym_session_private_data(
1861                                          op->sym->session,
1862                                         ccp_cryptodev_driver_id);
1863
1864         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
1865                                               op->sym->auth.data.offset);
1866         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
1867                                                 session->auth.ctx_len);
1868         if (!append_ptr) {
1869                 CCP_LOG_ERR("CCP MBUF append failed\n");
1870                 return -1;
1871         }
1872         if (iommu_mode == 2) {
1873                 dest_addr = (phys_addr_t)rte_mem_virt2iova((void *)append_ptr);
1874                 ctx_paddr = (phys_addr_t)rte_mem_virt2iova(
1875                                         session->auth.pre_compute);
1876         } else {
1877                 dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
1878                 ctx_paddr = (phys_addr_t)rte_mem_virt2phy(
1879                                         session->auth.pre_compute);
1880         }
1881         dest_addr_t = dest_addr + (session->auth.ctx_len / 2);
1882         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1883         memset(desc, 0, Q_DESC_SIZE);
1884
1885         /*desc1 for SHA3-Ihash operation */
1886         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1887         CCP_CMD_INIT(desc) = 1;
1888         CCP_CMD_EOM(desc) = 1;
1889
1890         function.raw = 0;
1891         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1892         CCP_CMD_FUNCTION(desc) = function.raw;
1893         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1894
1895         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1896         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1897         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1898
1899         CCP_CMD_DST_LO(desc) = (cmd_q->sb_sha * CCP_SB_BYTES);
1900         CCP_CMD_DST_HI(desc) = 0;
1901         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
1902
1903         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
1904         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
1905         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1906
1907         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1908
1909         rte_wmb();
1910         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1911         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1912         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1913                       cmd_q->qcontrol | CMD_Q_RUN);
1914
1915         /* Intermediate Hash value retrieve */
1916         if ((session->auth.ut.sha_type == CCP_SHA3_TYPE_384) ||
1917             (session->auth.ut.sha_type == CCP_SHA3_TYPE_512)) {
1918
1919                 pst.src_addr =
1920                         (phys_addr_t)((cmd_q->sb_sha + 1) * CCP_SB_BYTES);
1921                 pst.dest_addr = dest_addr_t;
1922                 pst.len = CCP_SB_BYTES;
1923                 pst.dir = 0;
1924                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1925                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1926                 ccp_perform_passthru(&pst, cmd_q);
1927
1928                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1929                 pst.dest_addr = dest_addr_t + CCP_SB_BYTES;
1930                 pst.len = CCP_SB_BYTES;
1931                 pst.dir = 0;
1932                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1933                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1934                 ccp_perform_passthru(&pst, cmd_q);
1935
1936         } else {
1937                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1938                 pst.dest_addr = dest_addr_t;
1939                 pst.len = CCP_SB_BYTES;
1940                 pst.dir = 0;
1941                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1942                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1943                 ccp_perform_passthru(&pst, cmd_q);
1944         }
1945
1946         /**sha engine command descriptor for FinalHash*/
1947         ctx_paddr += CCP_SHA3_CTX_SIZE;
1948         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1949         memset(desc, 0, Q_DESC_SIZE);
1950
1951         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1952         CCP_CMD_INIT(desc) = 1;
1953         CCP_CMD_EOM(desc) = 1;
1954
1955         function.raw = 0;
1956         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1957         CCP_CMD_FUNCTION(desc) = function.raw;
1958
1959         if (session->auth.ut.sha_type == CCP_SHA3_TYPE_224) {
1960                 dest_addr_t += (CCP_SB_BYTES - SHA224_DIGEST_SIZE);
1961                 CCP_CMD_LEN(desc) = SHA224_DIGEST_SIZE;
1962         } else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_256) {
1963                 CCP_CMD_LEN(desc) = SHA256_DIGEST_SIZE;
1964         } else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_384) {
1965                 dest_addr_t += (2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE);
1966                 CCP_CMD_LEN(desc) = SHA384_DIGEST_SIZE;
1967         } else {
1968                 CCP_CMD_LEN(desc) = SHA512_DIGEST_SIZE;
1969         }
1970
1971         CCP_CMD_SRC_LO(desc) = ((uint32_t)dest_addr_t);
1972         CCP_CMD_SRC_HI(desc) = high32_value(dest_addr_t);
1973         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1974
1975         CCP_CMD_DST_LO(desc) = (uint32_t)dest_addr;
1976         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
1977         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1978
1979         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
1980         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
1981         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1982
1983         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1984
1985         rte_wmb();
1986         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1987         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1988         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1989                       cmd_q->qcontrol | CMD_Q_RUN);
1990
1991         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1992         return 0;
1993 }
1994
1995 static int
1996 ccp_perform_sha3(struct rte_crypto_op *op,
1997                  struct ccp_queue *cmd_q)
1998 {
1999         struct ccp_session *session;
2000         union ccp_function function;
2001         struct ccp_desc *desc;
2002         uint8_t *ctx_addr = NULL, *append_ptr = NULL;
2003         uint32_t tail;
2004         phys_addr_t src_addr, dest_addr, ctx_paddr;
2005
2006         session = (struct ccp_session *)get_sym_session_private_data(
2007                                          op->sym->session,
2008                                         ccp_cryptodev_driver_id);
2009
2010         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
2011                                               op->sym->auth.data.offset);
2012         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
2013                                                 session->auth.ctx_len);
2014         if (!append_ptr) {
2015                 CCP_LOG_ERR("CCP MBUF append failed\n");
2016                 return -1;
2017         }
2018         if (iommu_mode == 2) {
2019                 dest_addr = (phys_addr_t)rte_mem_virt2iova((void *)append_ptr);
2020                 ctx_paddr = (phys_addr_t)rte_mem_virt2iova((void *)ctx_addr);
2021         } else {
2022                 dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
2023                 ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
2024         }
2025
2026         ctx_addr = session->auth.sha3_ctx;
2027
2028         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2029         memset(desc, 0, Q_DESC_SIZE);
2030
2031         /* prepare desc for SHA3 operation */
2032         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
2033         CCP_CMD_INIT(desc) = 1;
2034         CCP_CMD_EOM(desc) = 1;
2035
2036         function.raw = 0;
2037         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
2038         CCP_CMD_FUNCTION(desc) = function.raw;
2039
2040         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
2041
2042         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2043         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2044         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2045
2046         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2047         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2048         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2049
2050         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
2051         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
2052         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2053
2054         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2055
2056         rte_wmb();
2057
2058         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2059         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2060         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2061                       cmd_q->qcontrol | CMD_Q_RUN);
2062
2063         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2064         return 0;
2065 }
2066
2067 static int
2068 ccp_perform_aes_cmac(struct rte_crypto_op *op,
2069                      struct ccp_queue *cmd_q)
2070 {
2071         struct ccp_session *session;
2072         union ccp_function function;
2073         struct ccp_passthru pst;
2074         struct ccp_desc *desc;
2075         uint32_t tail;
2076         uint8_t *src_tb, *append_ptr, *ctx_addr;
2077         phys_addr_t src_addr, dest_addr, key_addr;
2078         int length, non_align_len;
2079
2080         session = (struct ccp_session *)get_sym_session_private_data(
2081                                          op->sym->session,
2082                                         ccp_cryptodev_driver_id);
2083         key_addr = rte_mem_virt2phy(session->auth.key_ccp);
2084
2085         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
2086                                               op->sym->auth.data.offset);
2087         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
2088                                                 session->auth.ctx_len);
2089         dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
2090
2091         function.raw = 0;
2092         CCP_AES_ENCRYPT(&function) = CCP_CIPHER_DIR_ENCRYPT;
2093         CCP_AES_MODE(&function) = session->auth.um.aes_mode;
2094         CCP_AES_TYPE(&function) = session->auth.ut.aes_type;
2095
2096         if (op->sym->auth.data.length % session->auth.block_size == 0) {
2097
2098                 ctx_addr = session->auth.pre_compute;
2099                 memset(ctx_addr, 0, AES_BLOCK_SIZE);
2100                 if (iommu_mode == 2)
2101                         pst.src_addr = (phys_addr_t)rte_mem_virt2iova(
2102                                                         (void *)ctx_addr);
2103                 else
2104                         pst.src_addr = (phys_addr_t)rte_mem_virt2phy(
2105                                                         (void *)ctx_addr);
2106
2107                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2108                 pst.len = CCP_SB_BYTES;
2109                 pst.dir = 1;
2110                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2111                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2112                 ccp_perform_passthru(&pst, cmd_q);
2113
2114                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2115                 memset(desc, 0, Q_DESC_SIZE);
2116
2117                 /* prepare desc for aes-cmac command */
2118                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2119                 CCP_CMD_EOM(desc) = 1;
2120                 CCP_CMD_FUNCTION(desc) = function.raw;
2121
2122                 CCP_CMD_LEN(desc) = op->sym->auth.data.length;
2123                 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2124                 CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2125                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2126
2127                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2128                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2129                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2130                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2131
2132                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2133
2134                 rte_wmb();
2135
2136                 tail =
2137                 (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2138                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2139                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2140                               cmd_q->qcontrol | CMD_Q_RUN);
2141         } else {
2142                 ctx_addr = session->auth.pre_compute + CCP_SB_BYTES;
2143                 memset(ctx_addr, 0, AES_BLOCK_SIZE);
2144                 if (iommu_mode == 2)
2145                         pst.src_addr = (phys_addr_t)rte_mem_virt2iova(
2146                                                         (void *)ctx_addr);
2147                 else
2148                         pst.src_addr = (phys_addr_t)rte_mem_virt2phy(
2149                                                         (void *)ctx_addr);
2150                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2151                 pst.len = CCP_SB_BYTES;
2152                 pst.dir = 1;
2153                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2154                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2155                 ccp_perform_passthru(&pst, cmd_q);
2156
2157                 length = (op->sym->auth.data.length / AES_BLOCK_SIZE);
2158                 length *= AES_BLOCK_SIZE;
2159                 non_align_len = op->sym->auth.data.length - length;
2160                 /* prepare desc for aes-cmac command */
2161                 /*Command 1*/
2162                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2163                 memset(desc, 0, Q_DESC_SIZE);
2164
2165                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2166                 CCP_CMD_INIT(desc) = 1;
2167                 CCP_CMD_FUNCTION(desc) = function.raw;
2168
2169                 CCP_CMD_LEN(desc) = length;
2170                 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2171                 CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2172                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2173
2174                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2175                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2176                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2177                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2178
2179                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2180
2181                 /*Command 2*/
2182                 append_ptr = append_ptr + CCP_SB_BYTES;
2183                 memset(append_ptr, 0, AES_BLOCK_SIZE);
2184                 src_tb = rte_pktmbuf_mtod_offset(op->sym->m_src,
2185                                                  uint8_t *,
2186                                                  op->sym->auth.data.offset +
2187                                                  length);
2188                 rte_memcpy(append_ptr, src_tb, non_align_len);
2189                 append_ptr[non_align_len] = CMAC_PAD_VALUE;
2190
2191                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2192                 memset(desc, 0, Q_DESC_SIZE);
2193
2194                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2195                 CCP_CMD_EOM(desc) = 1;
2196                 CCP_CMD_FUNCTION(desc) = function.raw;
2197                 CCP_CMD_LEN(desc) = AES_BLOCK_SIZE;
2198
2199                 CCP_CMD_SRC_LO(desc) = ((uint32_t)(dest_addr + CCP_SB_BYTES));
2200                 CCP_CMD_SRC_HI(desc) = high32_value(dest_addr + CCP_SB_BYTES);
2201                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2202
2203                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2204                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2205                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2206                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2207
2208                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2209
2210                 rte_wmb();
2211                 tail =
2212                 (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2213                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2214                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2215                               cmd_q->qcontrol | CMD_Q_RUN);
2216         }
2217         /* Retrieve result */
2218         pst.dest_addr = dest_addr;
2219         pst.src_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2220         pst.len = CCP_SB_BYTES;
2221         pst.dir = 0;
2222         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2223         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2224         ccp_perform_passthru(&pst, cmd_q);
2225
2226         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2227         return 0;
2228 }
2229
2230 static int
2231 ccp_perform_aes(struct rte_crypto_op *op,
2232                 struct ccp_queue *cmd_q,
2233                 struct ccp_batch_info *b_info)
2234 {
2235         struct ccp_session *session;
2236         union ccp_function function;
2237         uint8_t *lsb_buf;
2238         struct ccp_passthru pst = {0};
2239         struct ccp_desc *desc;
2240         phys_addr_t src_addr, dest_addr, key_addr;
2241         uint8_t *iv;
2242
2243         session = (struct ccp_session *)get_sym_session_private_data(
2244                                          op->sym->session,
2245                                         ccp_cryptodev_driver_id);
2246         function.raw = 0;
2247
2248         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2249         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB) {
2250                 if (session->cipher.um.aes_mode == CCP_AES_MODE_CTR) {
2251                         rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE,
2252                                    iv, session->iv.length);
2253                         pst.src_addr = (phys_addr_t)session->cipher.nonce_phys;
2254                         CCP_AES_SIZE(&function) = 0x1F;
2255                 } else {
2256                         lsb_buf =
2257                         &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]);
2258                         rte_memcpy(lsb_buf +
2259                                    (CCP_SB_BYTES - session->iv.length),
2260                                    iv, session->iv.length);
2261                         pst.src_addr = b_info->lsb_buf_phys +
2262                                 (b_info->lsb_buf_idx * CCP_SB_BYTES);
2263                         b_info->lsb_buf_idx++;
2264                 }
2265
2266                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2267                 pst.len = CCP_SB_BYTES;
2268                 pst.dir = 1;
2269                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2270                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2271                 ccp_perform_passthru(&pst, cmd_q);
2272         }
2273
2274         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2275
2276         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
2277                                               op->sym->cipher.data.offset);
2278         if (likely(op->sym->m_dst != NULL))
2279                 dest_addr = rte_pktmbuf_iova_offset(op->sym->m_dst,
2280                                                 op->sym->cipher.data.offset);
2281         else
2282                 dest_addr = src_addr;
2283         key_addr = session->cipher.key_phys;
2284
2285         /* prepare desc for aes command */
2286         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2287         CCP_CMD_INIT(desc) = 1;
2288         CCP_CMD_EOM(desc) = 1;
2289
2290         CCP_AES_ENCRYPT(&function) = session->cipher.dir;
2291         CCP_AES_MODE(&function) = session->cipher.um.aes_mode;
2292         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2293         CCP_CMD_FUNCTION(desc) = function.raw;
2294
2295         CCP_CMD_LEN(desc) = op->sym->cipher.data.length;
2296
2297         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2298         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2299         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2300
2301         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2302         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2303         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2304
2305         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2306         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2307         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2308
2309         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB)
2310                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2311
2312         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2313         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2314         return 0;
2315 }
2316
2317 static int
2318 ccp_perform_3des(struct rte_crypto_op *op,
2319                 struct ccp_queue *cmd_q,
2320                 struct ccp_batch_info *b_info)
2321 {
2322         struct ccp_session *session;
2323         union ccp_function function;
2324         unsigned char *lsb_buf;
2325         struct ccp_passthru pst;
2326         struct ccp_desc *desc;
2327         uint32_t tail;
2328         uint8_t *iv;
2329         phys_addr_t src_addr, dest_addr, key_addr;
2330
2331         session = (struct ccp_session *)get_sym_session_private_data(
2332                                          op->sym->session,
2333                                         ccp_cryptodev_driver_id);
2334
2335         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2336         switch (session->cipher.um.des_mode) {
2337         case CCP_DES_MODE_CBC:
2338                 lsb_buf = &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]);
2339                 b_info->lsb_buf_idx++;
2340
2341                 rte_memcpy(lsb_buf + (CCP_SB_BYTES - session->iv.length),
2342                            iv, session->iv.length);
2343                 if (iommu_mode == 2)
2344                         pst.src_addr = (phys_addr_t)rte_mem_virt2iova(
2345                                                         (void *) lsb_buf);
2346                 else
2347                         pst.src_addr = (phys_addr_t)rte_mem_virt2phy(
2348                                                         (void *) lsb_buf);
2349                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2350                 pst.len = CCP_SB_BYTES;
2351                 pst.dir = 1;
2352                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2353                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2354                 ccp_perform_passthru(&pst, cmd_q);
2355                 break;
2356         case CCP_DES_MODE_CFB:
2357         case CCP_DES_MODE_ECB:
2358                 CCP_LOG_ERR("Unsupported DES cipher mode");
2359                 return -ENOTSUP;
2360         }
2361
2362         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
2363                                               op->sym->cipher.data.offset);
2364         if (unlikely(op->sym->m_dst != NULL))
2365                 dest_addr =
2366                         rte_pktmbuf_iova_offset(op->sym->m_dst,
2367                                                    op->sym->cipher.data.offset);
2368         else
2369                 dest_addr = src_addr;
2370
2371         if (iommu_mode == 2)
2372                 key_addr = rte_mem_virt2iova(session->cipher.key_ccp);
2373         else
2374                 key_addr = rte_mem_virt2phy(session->cipher.key_ccp);
2375
2376         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2377
2378         memset(desc, 0, Q_DESC_SIZE);
2379
2380         /* prepare desc for des command */
2381         CCP_CMD_ENGINE(desc) = CCP_ENGINE_3DES;
2382
2383         CCP_CMD_SOC(desc) = 0;
2384         CCP_CMD_IOC(desc) = 0;
2385         CCP_CMD_INIT(desc) = 1;
2386         CCP_CMD_EOM(desc) = 1;
2387         CCP_CMD_PROT(desc) = 0;
2388
2389         function.raw = 0;
2390         CCP_DES_ENCRYPT(&function) = session->cipher.dir;
2391         CCP_DES_MODE(&function) = session->cipher.um.des_mode;
2392         CCP_DES_TYPE(&function) = session->cipher.ut.des_type;
2393         CCP_CMD_FUNCTION(desc) = function.raw;
2394
2395         CCP_CMD_LEN(desc) = op->sym->cipher.data.length;
2396
2397         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2398         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2399         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2400
2401         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2402         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2403         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2404
2405         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2406         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2407         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2408
2409         if (session->cipher.um.des_mode)
2410                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2411
2412         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2413
2414         rte_wmb();
2415
2416         /* Write the new tail address back to the queue register */
2417         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2418         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2419         /* Turn the queue back on using our cached control register */
2420         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2421                       cmd_q->qcontrol | CMD_Q_RUN);
2422
2423         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2424         return 0;
2425 }
2426
2427 static int
2428 ccp_perform_aes_gcm(struct rte_crypto_op *op, struct ccp_queue *cmd_q)
2429 {
2430         struct ccp_session *session;
2431         union ccp_function function;
2432         uint8_t *iv;
2433         struct ccp_passthru pst;
2434         struct ccp_desc *desc;
2435         uint32_t tail;
2436         uint64_t *temp;
2437         phys_addr_t src_addr, dest_addr, key_addr, aad_addr;
2438         phys_addr_t digest_dest_addr;
2439         int length, non_align_len;
2440
2441         session = (struct ccp_session *)get_sym_session_private_data(
2442                                          op->sym->session,
2443                                          ccp_cryptodev_driver_id);
2444         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2445         key_addr = session->cipher.key_phys;
2446
2447         src_addr = rte_pktmbuf_iova_offset(op->sym->m_src,
2448                                               op->sym->aead.data.offset);
2449         if (unlikely(op->sym->m_dst != NULL))
2450                 dest_addr = rte_pktmbuf_iova_offset(op->sym->m_dst,
2451                                                 op->sym->aead.data.offset);
2452         else
2453                 dest_addr = src_addr;
2454         rte_pktmbuf_append(op->sym->m_src, session->auth.ctx_len);
2455         digest_dest_addr = op->sym->aead.digest.phys_addr;
2456         temp = (uint64_t *)(op->sym->aead.digest.data + AES_BLOCK_SIZE);
2457         *temp++ = rte_bswap64(session->auth.aad_length << 3);
2458         *temp = rte_bswap64(op->sym->aead.data.length << 3);
2459
2460         non_align_len = op->sym->aead.data.length % AES_BLOCK_SIZE;
2461         length = CCP_ALIGN(op->sym->aead.data.length, AES_BLOCK_SIZE);
2462
2463         aad_addr = op->sym->aead.aad.phys_addr;
2464
2465         /* CMD1 IV Passthru */
2466         rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE, iv,
2467                    session->iv.length);
2468         pst.src_addr = session->cipher.nonce_phys;
2469         pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2470         pst.len = CCP_SB_BYTES;
2471         pst.dir = 1;
2472         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2473         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2474         ccp_perform_passthru(&pst, cmd_q);
2475
2476         /* CMD2 GHASH-AAD */
2477         function.raw = 0;
2478         CCP_AES_ENCRYPT(&function) = CCP_AES_MODE_GHASH_AAD;
2479         CCP_AES_MODE(&function) = CCP_AES_MODE_GHASH;
2480         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2481
2482         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2483         memset(desc, 0, Q_DESC_SIZE);
2484
2485         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2486         CCP_CMD_INIT(desc) = 1;
2487         CCP_CMD_FUNCTION(desc) = function.raw;
2488
2489         CCP_CMD_LEN(desc) = session->auth.aad_length;
2490
2491         CCP_CMD_SRC_LO(desc) = ((uint32_t)aad_addr);
2492         CCP_CMD_SRC_HI(desc) = high32_value(aad_addr);
2493         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2494
2495         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2496         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2497         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2498
2499         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2500
2501         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2502         rte_wmb();
2503
2504         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2505         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2506         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2507                       cmd_q->qcontrol | CMD_Q_RUN);
2508
2509         /* CMD3 : GCTR Plain text */
2510         function.raw = 0;
2511         CCP_AES_ENCRYPT(&function) = session->cipher.dir;
2512         CCP_AES_MODE(&function) = CCP_AES_MODE_GCTR;
2513         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2514         if (non_align_len == 0)
2515                 CCP_AES_SIZE(&function) = (AES_BLOCK_SIZE << 3) - 1;
2516         else
2517                 CCP_AES_SIZE(&function) = (non_align_len << 3) - 1;
2518
2519
2520         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2521         memset(desc, 0, Q_DESC_SIZE);
2522
2523         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2524         CCP_CMD_EOM(desc) = 1;
2525         CCP_CMD_FUNCTION(desc) = function.raw;
2526
2527         CCP_CMD_LEN(desc) = length;
2528
2529         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2530         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2531         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2532
2533         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2534         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2535         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2536
2537         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2538         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2539         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2540
2541         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2542
2543         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2544         rte_wmb();
2545
2546         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2547         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2548         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2549                       cmd_q->qcontrol | CMD_Q_RUN);
2550
2551         /* CMD4 : PT to copy IV */
2552         pst.src_addr = session->cipher.nonce_phys;
2553         pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2554         pst.len = AES_BLOCK_SIZE;
2555         pst.dir = 1;
2556         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2557         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2558         ccp_perform_passthru(&pst, cmd_q);
2559
2560         /* CMD5 : GHASH-Final */
2561         function.raw = 0;
2562         CCP_AES_ENCRYPT(&function) = CCP_AES_MODE_GHASH_FINAL;
2563         CCP_AES_MODE(&function) = CCP_AES_MODE_GHASH;
2564         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2565
2566         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2567         memset(desc, 0, Q_DESC_SIZE);
2568
2569         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2570         CCP_CMD_FUNCTION(desc) = function.raw;
2571         /* Last block (AAD_len || PT_len)*/
2572         CCP_CMD_LEN(desc) = AES_BLOCK_SIZE;
2573
2574         CCP_CMD_SRC_LO(desc) = ((uint32_t)digest_dest_addr + AES_BLOCK_SIZE);
2575         CCP_CMD_SRC_HI(desc) = high32_value(digest_dest_addr + AES_BLOCK_SIZE);
2576         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2577
2578         CCP_CMD_DST_LO(desc) = ((uint32_t)digest_dest_addr);
2579         CCP_CMD_DST_HI(desc) = high32_value(digest_dest_addr);
2580         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2581
2582         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2583         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2584         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2585
2586         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2587
2588         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2589         rte_wmb();
2590
2591         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2592         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2593         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2594                       cmd_q->qcontrol | CMD_Q_RUN);
2595
2596         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2597         return 0;
2598 }
2599
2600 static inline int
2601 ccp_crypto_cipher(struct rte_crypto_op *op,
2602                   struct ccp_queue *cmd_q,
2603                   struct ccp_batch_info *b_info)
2604 {
2605         int result = 0;
2606         struct ccp_session *session;
2607
2608         session = (struct ccp_session *)get_sym_session_private_data(
2609                                          op->sym->session,
2610                                          ccp_cryptodev_driver_id);
2611
2612         switch (session->cipher.algo) {
2613         case CCP_CIPHER_ALGO_AES_CBC:
2614                 result = ccp_perform_aes(op, cmd_q, b_info);
2615                 b_info->desccnt += 2;
2616                 break;
2617         case CCP_CIPHER_ALGO_AES_CTR:
2618                 result = ccp_perform_aes(op, cmd_q, b_info);
2619                 b_info->desccnt += 2;
2620                 break;
2621         case CCP_CIPHER_ALGO_AES_ECB:
2622                 result = ccp_perform_aes(op, cmd_q, b_info);
2623                 b_info->desccnt += 1;
2624                 break;
2625         case CCP_CIPHER_ALGO_3DES_CBC:
2626                 result = ccp_perform_3des(op, cmd_q, b_info);
2627                 b_info->desccnt += 2;
2628                 break;
2629         default:
2630                 CCP_LOG_ERR("Unsupported cipher algo %d",
2631                             session->cipher.algo);
2632                 return -ENOTSUP;
2633         }
2634         return result;
2635 }
2636
2637 static inline int
2638 ccp_crypto_auth(struct rte_crypto_op *op,
2639                 struct ccp_queue *cmd_q,
2640                 struct ccp_batch_info *b_info)
2641 {
2642
2643         int result = 0;
2644         struct ccp_session *session;
2645
2646         session = (struct ccp_session *)get_sym_session_private_data(
2647                                          op->sym->session,
2648                                         ccp_cryptodev_driver_id);
2649
2650         switch (session->auth.algo) {
2651         case CCP_AUTH_ALGO_SHA1:
2652         case CCP_AUTH_ALGO_SHA224:
2653         case CCP_AUTH_ALGO_SHA256:
2654         case CCP_AUTH_ALGO_SHA384:
2655         case CCP_AUTH_ALGO_SHA512:
2656                 result = ccp_perform_sha(op, cmd_q);
2657                 b_info->desccnt += 3;
2658                 break;
2659         case CCP_AUTH_ALGO_MD5_HMAC:
2660                 if (session->auth_opt == 0)
2661                         result = -1;
2662                 break;
2663         case CCP_AUTH_ALGO_SHA1_HMAC:
2664         case CCP_AUTH_ALGO_SHA224_HMAC:
2665         case CCP_AUTH_ALGO_SHA256_HMAC:
2666                 if (session->auth_opt == 0) {
2667                         result = ccp_perform_hmac(op, cmd_q);
2668                         b_info->desccnt += 6;
2669                 }
2670                 break;
2671         case CCP_AUTH_ALGO_SHA384_HMAC:
2672         case CCP_AUTH_ALGO_SHA512_HMAC:
2673                 if (session->auth_opt == 0) {
2674                         result = ccp_perform_hmac(op, cmd_q);
2675                         b_info->desccnt += 7;
2676                 }
2677                 break;
2678         case CCP_AUTH_ALGO_SHA3_224:
2679         case CCP_AUTH_ALGO_SHA3_256:
2680         case CCP_AUTH_ALGO_SHA3_384:
2681         case CCP_AUTH_ALGO_SHA3_512:
2682                 result = ccp_perform_sha3(op, cmd_q);
2683                 b_info->desccnt += 1;
2684                 break;
2685         case CCP_AUTH_ALGO_SHA3_224_HMAC:
2686         case CCP_AUTH_ALGO_SHA3_256_HMAC:
2687                 result = ccp_perform_sha3_hmac(op, cmd_q);
2688                 b_info->desccnt += 3;
2689                 break;
2690         case CCP_AUTH_ALGO_SHA3_384_HMAC:
2691         case CCP_AUTH_ALGO_SHA3_512_HMAC:
2692                 result = ccp_perform_sha3_hmac(op, cmd_q);
2693                 b_info->desccnt += 4;
2694                 break;
2695         case CCP_AUTH_ALGO_AES_CMAC:
2696                 result = ccp_perform_aes_cmac(op, cmd_q);
2697                 b_info->desccnt += 4;
2698                 break;
2699         default:
2700                 CCP_LOG_ERR("Unsupported auth algo %d",
2701                             session->auth.algo);
2702                 return -ENOTSUP;
2703         }
2704
2705         return result;
2706 }
2707
2708 static inline int
2709 ccp_crypto_aead(struct rte_crypto_op *op,
2710                 struct ccp_queue *cmd_q,
2711                 struct ccp_batch_info *b_info)
2712 {
2713         int result = 0;
2714         struct ccp_session *session;
2715
2716         session = (struct ccp_session *)get_sym_session_private_data(
2717                                         op->sym->session,
2718                                         ccp_cryptodev_driver_id);
2719
2720         switch (session->auth.algo) {
2721         case CCP_AUTH_ALGO_AES_GCM:
2722                 if (session->cipher.algo != CCP_CIPHER_ALGO_AES_GCM) {
2723                         CCP_LOG_ERR("Incorrect chain order");
2724                         return -1;
2725                 }
2726                 result = ccp_perform_aes_gcm(op, cmd_q);
2727                 b_info->desccnt += 5;
2728                 break;
2729         default:
2730                 CCP_LOG_ERR("Unsupported aead algo %d",
2731                             session->aead_algo);
2732                 return -ENOTSUP;
2733         }
2734         return result;
2735 }
2736
2737 int
2738 process_ops_to_enqueue(struct ccp_qp *qp,
2739                        struct rte_crypto_op **op,
2740                        struct ccp_queue *cmd_q,
2741                        uint16_t nb_ops,
2742                        uint16_t total_nb_ops,
2743                        int slots_req,
2744                        uint16_t b_idx)
2745 {
2746         int i, result = 0;
2747         struct ccp_batch_info *b_info;
2748         struct ccp_session *session;
2749         EVP_MD_CTX *auth_ctx = NULL;
2750
2751         if (rte_mempool_get(qp->batch_mp, (void **)&b_info)) {
2752                 CCP_LOG_ERR("batch info allocation failed");
2753                 return 0;
2754         }
2755
2756         auth_ctx = EVP_MD_CTX_create();
2757         if (unlikely(!auth_ctx)) {
2758                 CCP_LOG_ERR("Unable to create auth ctx");
2759                 return 0;
2760         }
2761         b_info->auth_ctr = 0;
2762
2763         /* populate batch info necessary for dequeue */
2764         b_info->op_idx = 0;
2765         b_info->b_idx = 0;
2766         b_info->lsb_buf_idx = 0;
2767         b_info->desccnt = 0;
2768         b_info->cmd_q = cmd_q;
2769         if (iommu_mode == 2)
2770                 b_info->lsb_buf_phys =
2771                         (phys_addr_t)rte_mem_virt2iova((void *)b_info->lsb_buf);
2772         else
2773                 b_info->lsb_buf_phys =
2774                         (phys_addr_t)rte_mem_virt2phy((void *)b_info->lsb_buf);
2775
2776         rte_atomic64_sub(&b_info->cmd_q->free_slots, slots_req);
2777
2778         b_info->head_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
2779                                          Q_DESC_SIZE);
2780         for (i = b_idx; i < (nb_ops+b_idx); i++) {
2781                 session = (struct ccp_session *)get_sym_session_private_data(
2782                                                  op[i]->sym->session,
2783                                                  ccp_cryptodev_driver_id);
2784                 switch (session->cmd_id) {
2785                 case CCP_CMD_CIPHER:
2786                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2787                         break;
2788                 case CCP_CMD_AUTH:
2789                         if (session->auth_opt) {
2790                                 b_info->auth_ctr++;
2791                                 result = cpu_crypto_auth(qp, op[i],
2792                                                          session, auth_ctx);
2793                         } else
2794                                 result = ccp_crypto_auth(op[i], cmd_q, b_info);
2795                         break;
2796                 case CCP_CMD_CIPHER_HASH:
2797                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2798                         if (result)
2799                                 break;
2800                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
2801                         break;
2802                 case CCP_CMD_HASH_CIPHER:
2803                         if (session->auth_opt) {
2804                                 result = cpu_crypto_auth(qp, op[i],
2805                                                          session, auth_ctx);
2806                                 if (op[i]->status !=
2807                                     RTE_CRYPTO_OP_STATUS_SUCCESS)
2808                                         CCP_LOG_ERR("RTE_CRYPTO_OP_STATUS_AUTH_FAILED");
2809                         } else
2810                                 result = ccp_crypto_auth(op[i], cmd_q, b_info);
2811
2812                         if (result)
2813                                 break;
2814                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2815                         break;
2816                 case CCP_CMD_COMBINED:
2817                         result = ccp_crypto_aead(op[i], cmd_q, b_info);
2818                         break;
2819                 default:
2820                         CCP_LOG_ERR("Unsupported cmd_id");
2821                         result = -1;
2822                 }
2823                 if (unlikely(result < 0)) {
2824                         rte_atomic64_add(&b_info->cmd_q->free_slots,
2825                                          (slots_req - b_info->desccnt));
2826                         break;
2827                 }
2828                 b_info->op[i] = op[i];
2829         }
2830
2831         b_info->opcnt = i;
2832         b_info->b_idx = b_idx;
2833         b_info->total_nb_ops = total_nb_ops;
2834         b_info->tail_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
2835                                          Q_DESC_SIZE);
2836
2837         rte_wmb();
2838         /* Write the new tail address back to the queue register */
2839         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
2840                               b_info->tail_offset);
2841         /* Turn the queue back on using our cached control register */
2842         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2843                               cmd_q->qcontrol | CMD_Q_RUN);
2844
2845         rte_ring_enqueue(qp->processed_pkts, (void *)b_info);
2846
2847         EVP_MD_CTX_destroy(auth_ctx);
2848         return i-b_idx;
2849 }
2850
2851 static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op)
2852 {
2853         struct ccp_session *session;
2854         uint8_t *digest_data, *addr;
2855         struct rte_mbuf *m_last;
2856         int offset, digest_offset;
2857         uint8_t digest_le[64];
2858
2859         session = (struct ccp_session *)get_sym_session_private_data(
2860                                          op->sym->session,
2861                                         ccp_cryptodev_driver_id);
2862
2863         if (session->cmd_id == CCP_CMD_COMBINED) {
2864                 digest_data = op->sym->aead.digest.data;
2865                 digest_offset = op->sym->aead.data.offset +
2866                                         op->sym->aead.data.length;
2867         } else {
2868                 digest_data = op->sym->auth.digest.data;
2869                 digest_offset = op->sym->auth.data.offset +
2870                                         op->sym->auth.data.length;
2871         }
2872         m_last = rte_pktmbuf_lastseg(op->sym->m_src);
2873         addr = (uint8_t *)((char *)m_last->buf_addr + m_last->data_off +
2874                            m_last->data_len - session->auth.ctx_len);
2875
2876         rte_mb();
2877         offset = session->auth.offset;
2878
2879         if (session->auth.engine == CCP_ENGINE_SHA)
2880                 if ((session->auth.ut.sha_type != CCP_SHA_TYPE_1) &&
2881                     (session->auth.ut.sha_type != CCP_SHA_TYPE_224) &&
2882                     (session->auth.ut.sha_type != CCP_SHA_TYPE_256)) {
2883                         /* All other algorithms require byte
2884                          * swap done by host
2885                          */
2886                         unsigned int i;
2887
2888                         offset = session->auth.ctx_len -
2889                                 session->auth.offset - 1;
2890                         for (i = 0; i < session->auth.digest_length; i++)
2891                                 digest_le[i] = addr[offset - i];
2892                         offset = 0;
2893                         addr = digest_le;
2894                 }
2895
2896         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2897         if (session->auth.op == CCP_AUTH_OP_VERIFY) {
2898                 if (memcmp(addr + offset, digest_data,
2899                            session->auth.digest_length) != 0)
2900                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
2901
2902         } else {
2903                 if (unlikely(digest_data == 0))
2904                         digest_data = rte_pktmbuf_mtod_offset(
2905                                         op->sym->m_dst, uint8_t *,
2906                                         digest_offset);
2907                 rte_memcpy(digest_data, addr + offset,
2908                            session->auth.digest_length);
2909         }
2910         /* Trim area used for digest from mbuf. */
2911         rte_pktmbuf_trim(op->sym->m_src,
2912                          session->auth.ctx_len);
2913 }
2914
2915 static int
2916 ccp_prepare_ops(struct ccp_qp *qp,
2917                 struct rte_crypto_op **op_d,
2918                 struct ccp_batch_info *b_info,
2919                 uint16_t nb_ops)
2920 {
2921         int i, min_ops;
2922         struct ccp_session *session;
2923
2924         EVP_MD_CTX *auth_ctx = NULL;
2925
2926         auth_ctx = EVP_MD_CTX_create();
2927         if (unlikely(!auth_ctx)) {
2928                 CCP_LOG_ERR("Unable to create auth ctx");
2929                 return 0;
2930         }
2931         min_ops = RTE_MIN(nb_ops, b_info->opcnt);
2932
2933         for (i =  b_info->b_idx; i < min_ops; i++) {
2934                 op_d[i] = b_info->op[b_info->b_idx + b_info->op_idx++];
2935                 session = (struct ccp_session *)get_sym_session_private_data(
2936                                                  op_d[i]->sym->session,
2937                                                 ccp_cryptodev_driver_id);
2938                 switch (session->cmd_id) {
2939                 case CCP_CMD_CIPHER:
2940                         op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2941                         break;
2942                 case CCP_CMD_AUTH:
2943                         if (session->auth_opt == 0)
2944                                 ccp_auth_dq_prepare(op_d[i]);
2945                         break;
2946                 case CCP_CMD_CIPHER_HASH:
2947                         if (session->auth_opt)
2948                                 cpu_crypto_auth(qp, op_d[i],
2949                                                 session, auth_ctx);
2950                         else
2951                                 ccp_auth_dq_prepare(op_d[i]);
2952                         break;
2953                 case CCP_CMD_HASH_CIPHER:
2954                         if (session->auth_opt)
2955                                 op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2956                         else
2957                                 ccp_auth_dq_prepare(op_d[i]);
2958                         break;
2959                 case CCP_CMD_COMBINED:
2960                         ccp_auth_dq_prepare(op_d[i]);
2961                         break;
2962                 default:
2963                         CCP_LOG_ERR("Unsupported cmd_id");
2964                 }
2965         }
2966
2967         EVP_MD_CTX_destroy(auth_ctx);
2968         b_info->opcnt -= min_ops;
2969         return min_ops;
2970 }
2971
2972 int
2973 process_ops_to_dequeue(struct ccp_qp *qp,
2974                        struct rte_crypto_op **op,
2975                        uint16_t nb_ops,
2976                        uint16_t *total_nb_ops)
2977 {
2978         struct ccp_batch_info *b_info;
2979         uint32_t cur_head_offset;
2980
2981         if (qp->b_info != NULL) {
2982                 b_info = qp->b_info;
2983                 if (unlikely(b_info->op_idx > 0))
2984                         goto success;
2985         } else if (rte_ring_dequeue(qp->processed_pkts,
2986                                     (void **)&b_info))
2987                 return 0;
2988
2989         if (b_info->auth_ctr == b_info->opcnt)
2990                 goto success;
2991         *total_nb_ops = b_info->total_nb_ops;
2992         cur_head_offset = CCP_READ_REG(b_info->cmd_q->reg_base,
2993                                        CMD_Q_HEAD_LO_BASE);
2994
2995         if (b_info->head_offset < b_info->tail_offset) {
2996                 if ((cur_head_offset >= b_info->head_offset) &&
2997                     (cur_head_offset < b_info->tail_offset)) {
2998                         qp->b_info = b_info;
2999                         return 0;
3000                 }
3001         } else if (b_info->tail_offset != b_info->head_offset) {
3002                 if ((cur_head_offset >= b_info->head_offset) ||
3003                     (cur_head_offset < b_info->tail_offset)) {
3004                         qp->b_info = b_info;
3005                         return 0;
3006                 }
3007         }
3008
3009
3010 success:
3011         *total_nb_ops = b_info->total_nb_ops;
3012         nb_ops = ccp_prepare_ops(qp, op, b_info, nb_ops);
3013         rte_atomic64_add(&b_info->cmd_q->free_slots, b_info->desccnt);
3014         b_info->desccnt = 0;
3015         if (b_info->opcnt > 0) {
3016                 qp->b_info = b_info;
3017         } else {
3018                 rte_mempool_put(qp->batch_mp, (void *)b_info);
3019                 qp->b_info = NULL;
3020         }
3021
3022         return nb_ops;
3023 }