drivers/crypto/ccp/ccp_crypto.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <dirent.h>
   6 #include <fcntl.h>
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <sys/mman.h>
  10 #include <sys/queue.h>
  11 #include <sys/types.h>
  12 #include <unistd.h>
  13 #include <openssl/sha.h>
  14 #include <openssl/cmac.h> /*sub key apis*/
  15 #include <openssl/evp.h> /*sub key apis*/
  16
  17 #include <rte_hexdump.h>
  18 #include <rte_memzone.h>
  19 #include <rte_malloc.h>
  20 #include <rte_memory.h>
  21 #include <rte_spinlock.h>
  22 #include <rte_string_fns.h>
  23 #include <rte_cryptodev_pmd.h>
  24
  25 #include "ccp_dev.h"
  26 #include "ccp_crypto.h"
  27 #include "ccp_pci.h"
  28 #include "ccp_pmd_private.h"
  29
  30 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
  31 #include <openssl/conf.h>
  32 #include <openssl/err.h>
  33 #include <openssl/hmac.h>
  34 #endif
  35
  36 /* SHA initial context values */
  37 static uint32_t ccp_sha1_init[SHA_COMMON_DIGEST_SIZE / sizeof(uint32_t)] = {
  38         SHA1_H4, SHA1_H3,
  39         SHA1_H2, SHA1_H1,
  40         SHA1_H0, 0x0U,
  41         0x0U, 0x0U,
  42 };
  43
  44 uint32_t ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(uint32_t)] = {
  45         SHA224_H7, SHA224_H6,
  46         SHA224_H5, SHA224_H4,
  47         SHA224_H3, SHA224_H2,
  48         SHA224_H1, SHA224_H0,
  49 };
  50
  51 uint32_t ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(uint32_t)] = {
  52         SHA256_H7, SHA256_H6,
  53         SHA256_H5, SHA256_H4,
  54         SHA256_H3, SHA256_H2,
  55         SHA256_H1, SHA256_H0,
  56 };
  57
  58 uint64_t ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(uint64_t)] = {
  59         SHA384_H7, SHA384_H6,
  60         SHA384_H5, SHA384_H4,
  61         SHA384_H3, SHA384_H2,
  62         SHA384_H1, SHA384_H0,
  63 };
  64
  65 uint64_t ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(uint64_t)] = {
  66         SHA512_H7, SHA512_H6,
  67         SHA512_H5, SHA512_H4,
  68         SHA512_H3, SHA512_H2,
  69         SHA512_H1, SHA512_H0,
  70 };
  71
  72 #if defined(_MSC_VER)
  73 #define SHA3_CONST(x) x
  74 #else
  75 #define SHA3_CONST(x) x##L
  76 #endif
  77
  78 /** 'Words' here refers to uint64_t */
  79 #define SHA3_KECCAK_SPONGE_WORDS \
  80         (((1600) / 8) / sizeof(uint64_t))
  81 typedef struct sha3_context_ {
  82         uint64_t saved;
  83         /**
  84          * The portion of the input message that we
  85          * didn't consume yet
  86          */
  87         union {
  88                 uint64_t s[SHA3_KECCAK_SPONGE_WORDS];
  89                 /* Keccak's state */
  90                 uint8_t sb[SHA3_KECCAK_SPONGE_WORDS * 8];
  91                 /**total 200 ctx size**/
  92         };
  93         unsigned int byteIndex;
  94         /**
  95          * 0..7--the next byte after the set one
  96          * (starts from 0; 0--none are buffered)
  97          */
  98         unsigned int wordIndex;
  99         /**
 100          * 0..24--the next word to integrate input
 101          * (starts from 0)
 102          */
 103         unsigned int capacityWords;
 104         /**
 105          * the double size of the hash output in
 106          * words (e.g. 16 for Keccak 512)
 107          */
 108 } sha3_context;
 109
 110 #ifndef SHA3_ROTL64
 111 #define SHA3_ROTL64(x, y) \
 112         (((x) << (y)) | ((x) >> ((sizeof(uint64_t)*8) - (y))))
 113 #endif
 114
 115 static const uint64_t keccakf_rndc[24] = {
 116         SHA3_CONST(0x0000000000000001UL), SHA3_CONST(0x0000000000008082UL),
 117         SHA3_CONST(0x800000000000808aUL), SHA3_CONST(0x8000000080008000UL),
 118         SHA3_CONST(0x000000000000808bUL), SHA3_CONST(0x0000000080000001UL),
 119         SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008009UL),
 120         SHA3_CONST(0x000000000000008aUL), SHA3_CONST(0x0000000000000088UL),
 121         SHA3_CONST(0x0000000080008009UL), SHA3_CONST(0x000000008000000aUL),
 122         SHA3_CONST(0x000000008000808bUL), SHA3_CONST(0x800000000000008bUL),
 123         SHA3_CONST(0x8000000000008089UL), SHA3_CONST(0x8000000000008003UL),
 124         SHA3_CONST(0x8000000000008002UL), SHA3_CONST(0x8000000000000080UL),
 125         SHA3_CONST(0x000000000000800aUL), SHA3_CONST(0x800000008000000aUL),
 126         SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008080UL),
 127         SHA3_CONST(0x0000000080000001UL), SHA3_CONST(0x8000000080008008UL)
 128 };
 129
 130 static const unsigned int keccakf_rotc[24] = {
 131         1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62,
 132         18, 39, 61, 20, 44
 133 };
 134
 135 static const unsigned int keccakf_piln[24] = {
 136         10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20,
 137         14, 22, 9, 6, 1
 138 };
 139
 140 static enum ccp_cmd_order
 141 ccp_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 142 {
 143         enum ccp_cmd_order res = CCP_CMD_NOT_SUPPORTED;
 144
 145         if (xform == NULL)
 146                 return res;
 147         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 148                 if (xform->next == NULL)
 149                         return CCP_CMD_AUTH;
 150                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 151                         return CCP_CMD_HASH_CIPHER;
 152         }
 153         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 154                 if (xform->next == NULL)
 155                         return CCP_CMD_CIPHER;
 156                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 157                         return CCP_CMD_CIPHER_HASH;
 158         }
 159         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 160                 return CCP_CMD_COMBINED;
 161         return res;
 162 }
 163
 164 /* partial hash using openssl */
 165 static int partial_hash_sha1(uint8_t *data_in, uint8_t *data_out)
 166 {
 167         SHA_CTX ctx;
 168
 169         if (!SHA1_Init(&ctx))
 170                 return -EFAULT;
 171         SHA1_Transform(&ctx, data_in);
 172         rte_memcpy(data_out, &ctx, SHA_DIGEST_LENGTH);
 173         return 0;
 174 }
 175
 176 static int partial_hash_sha224(uint8_t *data_in, uint8_t *data_out)
 177 {
 178         SHA256_CTX ctx;
 179
 180         if (!SHA224_Init(&ctx))
 181                 return -EFAULT;
 182         SHA256_Transform(&ctx, data_in);
 183         rte_memcpy(data_out, &ctx,
 184                    SHA256_DIGEST_LENGTH);
 185         return 0;
 186 }
 187
 188 static int partial_hash_sha256(uint8_t *data_in, uint8_t *data_out)
 189 {
 190         SHA256_CTX ctx;
 191
 192         if (!SHA256_Init(&ctx))
 193                 return -EFAULT;
 194         SHA256_Transform(&ctx, data_in);
 195         rte_memcpy(data_out, &ctx,
 196                    SHA256_DIGEST_LENGTH);
 197         return 0;
 198 }
 199
 200 static int partial_hash_sha384(uint8_t *data_in, uint8_t *data_out)
 201 {
 202         SHA512_CTX ctx;
 203
 204         if (!SHA384_Init(&ctx))
 205                 return -EFAULT;
 206         SHA512_Transform(&ctx, data_in);
 207         rte_memcpy(data_out, &ctx,
 208                    SHA512_DIGEST_LENGTH);
 209         return 0;
 210 }
 211
 212 static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out)
 213 {
 214         SHA512_CTX ctx;
 215
 216         if (!SHA512_Init(&ctx))
 217                 return -EFAULT;
 218         SHA512_Transform(&ctx, data_in);
 219         rte_memcpy(data_out, &ctx,
 220                    SHA512_DIGEST_LENGTH);
 221         return 0;
 222 }
 223
 224 static void
 225 keccakf(uint64_t s[25])
 226 {
 227         int i, j, round;
 228         uint64_t t, bc[5];
 229 #define KECCAK_ROUNDS 24
 230
 231         for (round = 0; round < KECCAK_ROUNDS; round++) {
 232
 233                 /* Theta */
 234                 for (i = 0; i < 5; i++)
 235                         bc[i] = s[i] ^ s[i + 5] ^ s[i + 10] ^ s[i + 15] ^
 236                                 s[i + 20];
 237
 238                 for (i = 0; i < 5; i++) {
 239                         t = bc[(i + 4) % 5] ^ SHA3_ROTL64(bc[(i + 1) % 5], 1);
 240                         for (j = 0; j < 25; j += 5)
 241                                 s[j + i] ^= t;
 242                 }
 243
 244                 /* Rho Pi */
 245                 t = s[1];
 246                 for (i = 0; i < 24; i++) {
 247                         j = keccakf_piln[i];
 248                         bc[0] = s[j];
 249                         s[j] = SHA3_ROTL64(t, keccakf_rotc[i]);
 250                         t = bc[0];
 251                 }
 252
 253                 /* Chi */
 254                 for (j = 0; j < 25; j += 5) {
 255                         for (i = 0; i < 5; i++)
 256                                 bc[i] = s[j + i];
 257                         for (i = 0; i < 5; i++)
 258                                 s[j + i] ^= (~bc[(i + 1) % 5]) &
 259                                             bc[(i + 2) % 5];
 260                 }
 261
 262                 /* Iota */
 263                 s[0] ^= keccakf_rndc[round];
 264         }
 265 }
 266
 267 static void
 268 sha3_Init224(void *priv)
 269 {
 270         sha3_context *ctx = (sha3_context *) priv;
 271
 272         memset(ctx, 0, sizeof(*ctx));
 273         ctx->capacityWords = 2 * 224 / (8 * sizeof(uint64_t));
 274 }
 275
 276 static void
 277 sha3_Init256(void *priv)
 278 {
 279         sha3_context *ctx = (sha3_context *) priv;
 280
 281         memset(ctx, 0, sizeof(*ctx));
 282         ctx->capacityWords = 2 * 256 / (8 * sizeof(uint64_t));
 283 }
 284
 285 static void
 286 sha3_Init384(void *priv)
 287 {
 288         sha3_context *ctx = (sha3_context *) priv;
 289
 290         memset(ctx, 0, sizeof(*ctx));
 291         ctx->capacityWords = 2 * 384 / (8 * sizeof(uint64_t));
 292 }
 293
 294 static void
 295 sha3_Init512(void *priv)
 296 {
 297         sha3_context *ctx = (sha3_context *) priv;
 298
 299         memset(ctx, 0, sizeof(*ctx));
 300         ctx->capacityWords = 2 * 512 / (8 * sizeof(uint64_t));
 301 }
 302
 303
 304 /* This is simply the 'update' with the padding block.
 305  * The padding block is 0x01 || 0x00* || 0x80. First 0x01 and last 0x80
 306  * bytes are always present, but they can be the same byte.
 307  */
 308 static void
 309 sha3_Update(void *priv, void const *bufIn, size_t len)
 310 {
 311         sha3_context *ctx = (sha3_context *) priv;
 312         unsigned int old_tail = (8 - ctx->byteIndex) & 7;
 313         size_t words;
 314         unsigned int tail;
 315         size_t i;
 316         const uint8_t *buf = bufIn;
 317
 318         if (len < old_tail) {
 319                 while (len--)
 320                         ctx->saved |= (uint64_t) (*(buf++)) <<
 321                                       ((ctx->byteIndex++) * 8);
 322                 return;
 323         }
 324
 325         if (old_tail) {
 326                 len -= old_tail;
 327                 while (old_tail--)
 328                         ctx->saved |= (uint64_t) (*(buf++)) <<
 329                                       ((ctx->byteIndex++) * 8);
 330
 331                 ctx->s[ctx->wordIndex] ^= ctx->saved;
 332                 ctx->byteIndex = 0;
 333                 ctx->saved = 0;
 334                 if (++ctx->wordIndex ==
 335                    (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
 336                         keccakf(ctx->s);
 337                         ctx->wordIndex = 0;
 338                 }
 339         }
 340
 341         words = len / sizeof(uint64_t);
 342         tail = len - words * sizeof(uint64_t);
 343
 344         for (i = 0; i < words; i++, buf += sizeof(uint64_t)) {
 345                 const uint64_t t = (uint64_t) (buf[0]) |
 346                         ((uint64_t) (buf[1]) << 8 * 1) |
 347                         ((uint64_t) (buf[2]) << 8 * 2) |
 348                         ((uint64_t) (buf[3]) << 8 * 3) |
 349                         ((uint64_t) (buf[4]) << 8 * 4) |
 350                         ((uint64_t) (buf[5]) << 8 * 5) |
 351                         ((uint64_t) (buf[6]) << 8 * 6) |
 352                         ((uint64_t) (buf[7]) << 8 * 7);
 353                 ctx->s[ctx->wordIndex] ^= t;
 354                 if (++ctx->wordIndex ==
 355                    (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
 356                         keccakf(ctx->s);
 357                         ctx->wordIndex = 0;
 358                 }
 359         }
 360
 361         while (tail--)
 362                 ctx->saved |= (uint64_t) (*(buf++)) << ((ctx->byteIndex++) * 8);
 363 }
 364
 365 int partial_hash_sha3_224(uint8_t *data_in, uint8_t *data_out)
 366 {
 367         sha3_context *ctx;
 368         int i;
 369
 370         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 371         if (!ctx) {
 372                 CCP_LOG_ERR("sha3-ctx creation failed");
 373                 return -ENOMEM;
 374         }
 375         sha3_Init224(ctx);
 376         sha3_Update(ctx, data_in, SHA3_224_BLOCK_SIZE);
 377         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 378                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 379         rte_free(ctx);
 380
 381         return 0;
 382 }
 383
 384 int partial_hash_sha3_256(uint8_t *data_in, uint8_t *data_out)
 385 {
 386         sha3_context *ctx;
 387         int i;
 388
 389         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 390         if (!ctx) {
 391                 CCP_LOG_ERR("sha3-ctx creation failed");
 392                 return -ENOMEM;
 393         }
 394         sha3_Init256(ctx);
 395         sha3_Update(ctx, data_in, SHA3_256_BLOCK_SIZE);
 396         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 397                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 398         rte_free(ctx);
 399
 400         return 0;
 401 }
 402
 403 int partial_hash_sha3_384(uint8_t *data_in, uint8_t *data_out)
 404 {
 405         sha3_context *ctx;
 406         int i;
 407
 408         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 409         if (!ctx) {
 410                 CCP_LOG_ERR("sha3-ctx creation failed");
 411                 return -ENOMEM;
 412         }
 413         sha3_Init384(ctx);
 414         sha3_Update(ctx, data_in, SHA3_384_BLOCK_SIZE);
 415         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 416                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 417         rte_free(ctx);
 418
 419         return 0;
 420 }
 421
 422 int partial_hash_sha3_512(uint8_t *data_in, uint8_t *data_out)
 423 {
 424         sha3_context *ctx;
 425         int i;
 426
 427         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 428         if (!ctx) {
 429                 CCP_LOG_ERR("sha3-ctx creation failed");
 430                 return -ENOMEM;
 431         }
 432         sha3_Init512(ctx);
 433         sha3_Update(ctx, data_in, SHA3_512_BLOCK_SIZE);
 434         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 435                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 436         rte_free(ctx);
 437
 438         return 0;
 439 }
 440
 441 static int generate_partial_hash(struct ccp_session *sess)
 442 {
 443
 444         uint8_t ipad[sess->auth.block_size];
 445         uint8_t opad[sess->auth.block_size];
 446         uint8_t *ipad_t, *opad_t;
 447         uint32_t *hash_value_be32, hash_temp32[8];
 448         uint64_t *hash_value_be64, hash_temp64[8];
 449         int i, count;
 450         uint8_t *hash_value_sha3;
 451
 452         opad_t = ipad_t = (uint8_t *)sess->auth.key;
 453
 454         hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute);
 455         hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute);
 456
 457         /* considering key size is always equal to block size of algorithm */
 458         for (i = 0; i < sess->auth.block_size; i++) {
 459                 ipad[i] = (ipad_t[i] ^ HMAC_IPAD_VALUE);
 460                 opad[i] = (opad_t[i] ^ HMAC_OPAD_VALUE);
 461         }
 462
 463         switch (sess->auth.algo) {
 464         case CCP_AUTH_ALGO_SHA1_HMAC:
 465                 count = SHA1_DIGEST_SIZE >> 2;
 466
 467                 if (partial_hash_sha1(ipad, (uint8_t *)hash_temp32))
 468                         return -1;
 469                 for (i = 0; i < count; i++, hash_value_be32++)
 470                         *hash_value_be32 = hash_temp32[count - 1 - i];
 471
 472                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 473                                                + sess->auth.ctx_len);
 474                 if (partial_hash_sha1(opad, (uint8_t *)hash_temp32))
 475                         return -1;
 476                 for (i = 0; i < count; i++, hash_value_be32++)
 477                         *hash_value_be32 = hash_temp32[count - 1 - i];
 478                 return 0;
 479         case CCP_AUTH_ALGO_SHA224_HMAC:
 480                 count = SHA256_DIGEST_SIZE >> 2;
 481
 482                 if (partial_hash_sha224(ipad, (uint8_t *)hash_temp32))
 483                         return -1;
 484                 for (i = 0; i < count; i++, hash_value_be32++)
 485                         *hash_value_be32 = hash_temp32[count - 1 - i];
 486
 487                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 488                                                + sess->auth.ctx_len);
 489                 if (partial_hash_sha224(opad, (uint8_t *)hash_temp32))
 490                         return -1;
 491                 for (i = 0; i < count; i++, hash_value_be32++)
 492                         *hash_value_be32 = hash_temp32[count - 1 - i];
 493                 return 0;
 494         case CCP_AUTH_ALGO_SHA3_224_HMAC:
 495                 hash_value_sha3 = sess->auth.pre_compute;
 496                 if (partial_hash_sha3_224(ipad, hash_value_sha3))
 497                         return -1;
 498
 499                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 500                                                + sess->auth.ctx_len);
 501                 if (partial_hash_sha3_224(opad, hash_value_sha3))
 502                         return -1;
 503                 return 0;
 504         case CCP_AUTH_ALGO_SHA256_HMAC:
 505                 count = SHA256_DIGEST_SIZE >> 2;
 506
 507                 if (partial_hash_sha256(ipad, (uint8_t *)hash_temp32))
 508                         return -1;
 509                 for (i = 0; i < count; i++, hash_value_be32++)
 510                         *hash_value_be32 = hash_temp32[count - 1 - i];
 511
 512                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 513                                                + sess->auth.ctx_len);
 514                 if (partial_hash_sha256(opad, (uint8_t *)hash_temp32))
 515                         return -1;
 516                 for (i = 0; i < count; i++, hash_value_be32++)
 517                         *hash_value_be32 = hash_temp32[count - 1 - i];
 518                 return 0;
 519         case CCP_AUTH_ALGO_SHA3_256_HMAC:
 520                 hash_value_sha3 = sess->auth.pre_compute;
 521                 if (partial_hash_sha3_256(ipad, hash_value_sha3))
 522                         return -1;
 523
 524                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 525                                               + sess->auth.ctx_len);
 526                 if (partial_hash_sha3_256(opad, hash_value_sha3))
 527                         return -1;
 528                 return 0;
 529         case CCP_AUTH_ALGO_SHA384_HMAC:
 530                 count = SHA512_DIGEST_SIZE >> 3;
 531
 532                 if (partial_hash_sha384(ipad, (uint8_t *)hash_temp64))
 533                         return -1;
 534                 for (i = 0; i < count; i++, hash_value_be64++)
 535                         *hash_value_be64 = hash_temp64[count - 1 - i];
 536
 537                 hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute
 538                                                + sess->auth.ctx_len);
 539                 if (partial_hash_sha384(opad, (uint8_t *)hash_temp64))
 540                         return -1;
 541                 for (i = 0; i < count; i++, hash_value_be64++)
 542                         *hash_value_be64 = hash_temp64[count - 1 - i];
 543                 return 0;
 544         case CCP_AUTH_ALGO_SHA3_384_HMAC:
 545                 hash_value_sha3 = sess->auth.pre_compute;
 546                 if (partial_hash_sha3_384(ipad, hash_value_sha3))
 547                         return -1;
 548
 549                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 550                                               + sess->auth.ctx_len);
 551                 if (partial_hash_sha3_384(opad, hash_value_sha3))
 552                         return -1;
 553                 return 0;
 554         case CCP_AUTH_ALGO_SHA512_HMAC:
 555                 count = SHA512_DIGEST_SIZE >> 3;
 556
 557                 if (partial_hash_sha512(ipad, (uint8_t *)hash_temp64))
 558                         return -1;
 559                 for (i = 0; i < count; i++, hash_value_be64++)
 560                         *hash_value_be64 = hash_temp64[count - 1 - i];
 561
 562                 hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute
 563                                                + sess->auth.ctx_len);
 564                 if (partial_hash_sha512(opad, (uint8_t *)hash_temp64))
 565                         return -1;
 566                 for (i = 0; i < count; i++, hash_value_be64++)
 567                         *hash_value_be64 = hash_temp64[count - 1 - i];
 568                 return 0;
 569         case CCP_AUTH_ALGO_SHA3_512_HMAC:
 570                 hash_value_sha3 = sess->auth.pre_compute;
 571                 if (partial_hash_sha3_512(ipad, hash_value_sha3))
 572                         return -1;
 573
 574                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 575                                               + sess->auth.ctx_len);
 576                 if (partial_hash_sha3_512(opad, hash_value_sha3))
 577                         return -1;
 578                 return 0;
 579         default:
 580                 CCP_LOG_ERR("Invalid auth algo");
 581                 return -1;
 582         }
 583 }
 584
 585 /* prepare temporary keys K1 and K2 */
 586 static void prepare_key(unsigned char *k, unsigned char *l, int bl)
 587 {
 588         int i;
 589         /* Shift block to left, including carry */
 590         for (i = 0; i < bl; i++) {
 591                 k[i] = l[i] << 1;
 592                 if (i < bl - 1 && l[i + 1] & 0x80)
 593                         k[i] |= 1;
 594         }
 595         /* If MSB set fixup with R */
 596         if (l[0] & 0x80)
 597                 k[bl - 1] ^= bl == 16 ? 0x87 : 0x1b;
 598 }
 599
 600 /* subkeys K1 and K2 generation for CMAC */
 601 static int
 602 generate_cmac_subkeys(struct ccp_session *sess)
 603 {
 604         const EVP_CIPHER *algo;
 605         EVP_CIPHER_CTX *ctx;
 606         unsigned char *ccp_ctx;
 607         size_t i;
 608         int dstlen, totlen;
 609         unsigned char zero_iv[AES_BLOCK_SIZE] = {0};
 610         unsigned char dst[2 * AES_BLOCK_SIZE] = {0};
 611         unsigned char k1[AES_BLOCK_SIZE] = {0};
 612         unsigned char k2[AES_BLOCK_SIZE] = {0};
 613
 614         if (sess->auth.ut.aes_type == CCP_AES_TYPE_128)
 615                 algo =  EVP_aes_128_cbc();
 616         else if (sess->auth.ut.aes_type == CCP_AES_TYPE_192)
 617                 algo =  EVP_aes_192_cbc();
 618         else if (sess->auth.ut.aes_type == CCP_AES_TYPE_256)
 619                 algo =  EVP_aes_256_cbc();
 620         else {
 621                 CCP_LOG_ERR("Invalid CMAC type length");
 622                 return -1;
 623         }
 624
 625         ctx = EVP_CIPHER_CTX_new();
 626         if (!ctx) {
 627                 CCP_LOG_ERR("ctx creation failed");
 628                 return -1;
 629         }
 630         if (EVP_EncryptInit(ctx, algo, (unsigned char *)sess->auth.key,
 631                             (unsigned char *)zero_iv) <= 0)
 632                 goto key_generate_err;
 633         if (EVP_CIPHER_CTX_set_padding(ctx, 0) <= 0)
 634                 goto key_generate_err;
 635         if (EVP_EncryptUpdate(ctx, dst, &dstlen, zero_iv,
 636                               AES_BLOCK_SIZE) <= 0)
 637                 goto key_generate_err;
 638         if (EVP_EncryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0)
 639                 goto key_generate_err;
 640
 641         memset(sess->auth.pre_compute, 0, CCP_SB_BYTES * 2);
 642
 643         ccp_ctx = (unsigned char *)(sess->auth.pre_compute + CCP_SB_BYTES - 1);
 644         prepare_key(k1, dst, AES_BLOCK_SIZE);
 645         for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
 646                 *ccp_ctx = k1[i];
 647
 648         ccp_ctx = (unsigned char *)(sess->auth.pre_compute +
 649                                    (2 * CCP_SB_BYTES) - 1);
 650         prepare_key(k2, k1, AES_BLOCK_SIZE);
 651         for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
 652                 *ccp_ctx = k2[i];
 653
 654         EVP_CIPHER_CTX_free(ctx);
 655
 656         return 0;
 657
 658 key_generate_err:
 659         CCP_LOG_ERR("CMAC Init failed");
 660                 return -1;
 661 }
 662
 663 /* configure session */
 664 static int
 665 ccp_configure_session_cipher(struct ccp_session *sess,
 666                              const struct rte_crypto_sym_xform *xform)
 667 {
 668         const struct rte_crypto_cipher_xform *cipher_xform = NULL;
 669         size_t i, j, x;
 670
 671         cipher_xform = &xform->cipher;
 672
 673         /* set cipher direction */
 674         if (cipher_xform->op ==  RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 675                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
 676         else
 677                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
 678
 679         /* set cipher key */
 680         sess->cipher.key_length = cipher_xform->key.length;
 681         rte_memcpy(sess->cipher.key, cipher_xform->key.data,
 682                    cipher_xform->key.length);
 683
 684         /* set iv parameters */
 685         sess->iv.offset = cipher_xform->iv.offset;
 686         sess->iv.length = cipher_xform->iv.length;
 687
 688         switch (cipher_xform->algo) {
 689         case RTE_CRYPTO_CIPHER_AES_CTR:
 690                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CTR;
 691                 sess->cipher.um.aes_mode = CCP_AES_MODE_CTR;
 692                 sess->cipher.engine = CCP_ENGINE_AES;
 693                 break;
 694         case RTE_CRYPTO_CIPHER_AES_ECB:
 695                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
 696                 sess->cipher.um.aes_mode = CCP_AES_MODE_ECB;
 697                 sess->cipher.engine = CCP_ENGINE_AES;
 698                 break;
 699         case RTE_CRYPTO_CIPHER_AES_CBC:
 700                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
 701                 sess->cipher.um.aes_mode = CCP_AES_MODE_CBC;
 702                 sess->cipher.engine = CCP_ENGINE_AES;
 703                 break;
 704         case RTE_CRYPTO_CIPHER_3DES_CBC:
 705                 sess->cipher.algo = CCP_CIPHER_ALGO_3DES_CBC;
 706                 sess->cipher.um.des_mode = CCP_DES_MODE_CBC;
 707                 sess->cipher.engine = CCP_ENGINE_3DES;
 708                 break;
 709         default:
 710                 CCP_LOG_ERR("Unsupported cipher algo");
 711                 return -1;
 712         }
 713
 714
 715         switch (sess->cipher.engine) {
 716         case CCP_ENGINE_AES:
 717                 if (sess->cipher.key_length == 16)
 718                         sess->cipher.ut.aes_type = CCP_AES_TYPE_128;
 719                 else if (sess->cipher.key_length == 24)
 720                         sess->cipher.ut.aes_type = CCP_AES_TYPE_192;
 721                 else if (sess->cipher.key_length == 32)
 722                         sess->cipher.ut.aes_type = CCP_AES_TYPE_256;
 723                 else {
 724                         CCP_LOG_ERR("Invalid cipher key length");
 725                         return -1;
 726                 }
 727                 for (i = 0; i < sess->cipher.key_length ; i++)
 728                         sess->cipher.key_ccp[sess->cipher.key_length - i - 1] =
 729                                 sess->cipher.key[i];
 730                 break;
 731         case CCP_ENGINE_3DES:
 732                 if (sess->cipher.key_length == 16)
 733                         sess->cipher.ut.des_type = CCP_DES_TYPE_128;
 734                 else if (sess->cipher.key_length == 24)
 735                         sess->cipher.ut.des_type = CCP_DES_TYPE_192;
 736                 else {
 737                         CCP_LOG_ERR("Invalid cipher key length");
 738                         return -1;
 739                 }
 740                 for (j = 0, x = 0; j < sess->cipher.key_length/8; j++, x += 8)
 741                         for (i = 0; i < 8; i++)
 742                                 sess->cipher.key_ccp[(8 + x) - i - 1] =
 743                                         sess->cipher.key[i + x];
 744                 break;
 745         default:
 746                 CCP_LOG_ERR("Invalid CCP Engine");
 747                 return -ENOTSUP;
 748         }
 749         sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce);
 750         sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp);
 751         return 0;
 752 }
 753
 754 static int
 755 ccp_configure_session_auth(struct ccp_session *sess,
 756                            const struct rte_crypto_sym_xform *xform)
 757 {
 758         const struct rte_crypto_auth_xform *auth_xform = NULL;
 759         size_t i;
 760
 761         auth_xform = &xform->auth;
 762
 763         sess->auth.digest_length = auth_xform->digest_length;
 764         if (auth_xform->op ==  RTE_CRYPTO_AUTH_OP_GENERATE)
 765                 sess->auth.op = CCP_AUTH_OP_GENERATE;
 766         else
 767                 sess->auth.op = CCP_AUTH_OP_VERIFY;
 768         switch (auth_xform->algo) {
 769 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
 770         case RTE_CRYPTO_AUTH_MD5_HMAC:
 771                 sess->auth.algo = CCP_AUTH_ALGO_MD5_HMAC;
 772                 sess->auth.offset = (CCP_SB_BYTES << 1) - MD5_DIGEST_SIZE;
 773                 sess->auth.key_length = auth_xform->key.length;
 774                 sess->auth.block_size = MD5_BLOCK_SIZE;
 775                 memset(sess->auth.key, 0, sess->auth.block_size);
 776                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 777                            auth_xform->key.length);
 778                 break;
 779 #endif
 780         case RTE_CRYPTO_AUTH_SHA1:
 781                 sess->auth.engine = CCP_ENGINE_SHA;
 782                 sess->auth.algo = CCP_AUTH_ALGO_SHA1;
 783                 sess->auth.ut.sha_type = CCP_SHA_TYPE_1;
 784                 sess->auth.ctx = (void *)ccp_sha1_init;
 785                 sess->auth.ctx_len = CCP_SB_BYTES;
 786                 sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 787                 break;
 788         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 789 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
 790                 if (auth_xform->key.length > SHA1_BLOCK_SIZE)
 791                         return -1;
 792                 sess->auth.algo = CCP_AUTH_ALGO_SHA1_HMAC;
 793                 sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 794                 sess->auth.block_size = SHA1_BLOCK_SIZE;
 795                 sess->auth.key_length = auth_xform->key.length;
 796                 memset(sess->auth.key, 0, sess->auth.block_size);
 797                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 798                            auth_xform->key.length);
 799 #else
 800                 if (auth_xform->key.length > SHA1_BLOCK_SIZE)
 801                         return -1;
 802                 sess->auth.engine = CCP_ENGINE_SHA;
 803                 sess->auth.algo = CCP_AUTH_ALGO_SHA1_HMAC;
 804                 sess->auth.ut.sha_type = CCP_SHA_TYPE_1;
 805                 sess->auth.ctx_len = CCP_SB_BYTES;
 806                 sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 807                 sess->auth.block_size = SHA1_BLOCK_SIZE;
 808                 sess->auth.key_length = auth_xform->key.length;
 809                 memset(sess->auth.key, 0, sess->auth.block_size);
 810                 memset(sess->auth.pre_compute, 0, sess->auth.ctx_len << 1);
 811                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 812                            auth_xform->key.length);
 813                 if (generate_partial_hash(sess))
 814                         return -1;
 815 #endif
 816                 break;
 817         case RTE_CRYPTO_AUTH_SHA224:
 818                 sess->auth.algo = CCP_AUTH_ALGO_SHA224;
 819                 sess->auth.engine = CCP_ENGINE_SHA;
 820                 sess->auth.ut.sha_type = CCP_SHA_TYPE_224;
 821                 sess->auth.ctx = (void *)ccp_sha224_init;
 822                 sess->auth.ctx_len = CCP_SB_BYTES;
 823                 sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 824                 break;
 825         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 826 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
 827                 if (auth_xform->key.length > SHA224_BLOCK_SIZE)
 828                         return -1;
 829                 sess->auth.algo = CCP_AUTH_ALGO_SHA224_HMAC;
 830                 sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 831                 sess->auth.block_size = SHA224_BLOCK_SIZE;
 832                 sess->auth.key_length = auth_xform->key.length;
 833                 memset(sess->auth.key, 0, sess->auth.block_size);
 834                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 835                            auth_xform->key.length);
 836 #else
 837                 if (auth_xform->key.length > SHA224_BLOCK_SIZE)
 838                         return -1;
 839                 sess->auth.algo = CCP_AUTH_ALGO_SHA224_HMAC;
 840                 sess->auth.engine = CCP_ENGINE_SHA;
 841                 sess->auth.ut.sha_type = CCP_SHA_TYPE_224;
 842                 sess->auth.ctx_len = CCP_SB_BYTES;
 843                 sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 844                 sess->auth.block_size = SHA224_BLOCK_SIZE;
 845                 sess->auth.key_length = auth_xform->key.length;
 846                 memset(sess->auth.key, 0, sess->auth.block_size);
 847                 memset(sess->auth.pre_compute, 0, sess->auth.ctx_len << 1);
 848                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 849                            auth_xform->key.length);
 850                 if (generate_partial_hash(sess))
 851                         return -1;
 852 #endif
 853                 break;
 854         case RTE_CRYPTO_AUTH_SHA3_224:
 855                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_224;
 856                 sess->auth.engine = CCP_ENGINE_SHA;
 857                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
 858                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 859                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
 860                 break;
 861         case RTE_CRYPTO_AUTH_SHA3_224_HMAC:
 862                 if (auth_xform->key.length > SHA3_224_BLOCK_SIZE)
 863                         return -1;
 864                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_224_HMAC;
 865                 sess->auth.engine = CCP_ENGINE_SHA;
 866                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
 867                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 868                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
 869                 sess->auth.block_size = SHA3_224_BLOCK_SIZE;
 870                 sess->auth.key_length = auth_xform->key.length;
 871                 memset(sess->auth.key, 0, sess->auth.block_size);
 872                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
 873                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 874                            auth_xform->key.length);
 875                 if (generate_partial_hash(sess))
 876                         return -1;
 877                 break;
 878         case RTE_CRYPTO_AUTH_SHA256:
 879                 sess->auth.algo = CCP_AUTH_ALGO_SHA256;
 880                 sess->auth.engine = CCP_ENGINE_SHA;
 881                 sess->auth.ut.sha_type = CCP_SHA_TYPE_256;
 882                 sess->auth.ctx = (void *)ccp_sha256_init;
 883                 sess->auth.ctx_len = CCP_SB_BYTES;
 884                 sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 885                 break;
 886         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 887 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
 888                 if (auth_xform->key.length > SHA256_BLOCK_SIZE)
 889                         return -1;
 890                 sess->auth.algo = CCP_AUTH_ALGO_SHA256_HMAC;
 891                 sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 892                 sess->auth.block_size = SHA256_BLOCK_SIZE;
 893                 sess->auth.key_length = auth_xform->key.length;
 894                 memset(sess->auth.key, 0, sess->auth.block_size);
 895                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 896                            auth_xform->key.length);
 897 #else
 898                 if (auth_xform->key.length > SHA256_BLOCK_SIZE)
 899                         return -1;
 900                 sess->auth.algo = CCP_AUTH_ALGO_SHA256_HMAC;
 901                 sess->auth.engine = CCP_ENGINE_SHA;
 902                 sess->auth.ut.sha_type = CCP_SHA_TYPE_256;
 903                 sess->auth.ctx_len = CCP_SB_BYTES;
 904                 sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 905                 sess->auth.block_size = SHA256_BLOCK_SIZE;
 906                 sess->auth.key_length = auth_xform->key.length;
 907                 memset(sess->auth.key, 0, sess->auth.block_size);
 908                 memset(sess->auth.pre_compute, 0, sess->auth.ctx_len << 1);
 909                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 910                            auth_xform->key.length);
 911                 if (generate_partial_hash(sess))
 912                         return -1;
 913 #endif
 914                 break;
 915         case RTE_CRYPTO_AUTH_SHA3_256:
 916                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_256;
 917                 sess->auth.engine = CCP_ENGINE_SHA;
 918                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
 919                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 920                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
 921                 break;
 922         case RTE_CRYPTO_AUTH_SHA3_256_HMAC:
 923                 if (auth_xform->key.length > SHA3_256_BLOCK_SIZE)
 924                         return -1;
 925                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_256_HMAC;
 926                 sess->auth.engine = CCP_ENGINE_SHA;
 927                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
 928                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 929                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
 930                 sess->auth.block_size = SHA3_256_BLOCK_SIZE;
 931                 sess->auth.key_length = auth_xform->key.length;
 932                 memset(sess->auth.key, 0, sess->auth.block_size);
 933                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
 934                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 935                            auth_xform->key.length);
 936                 if (generate_partial_hash(sess))
 937                         return -1;
 938                 break;
 939         case RTE_CRYPTO_AUTH_SHA384:
 940                 sess->auth.algo = CCP_AUTH_ALGO_SHA384;
 941                 sess->auth.engine = CCP_ENGINE_SHA;
 942                 sess->auth.ut.sha_type = CCP_SHA_TYPE_384;
 943                 sess->auth.ctx = (void *)ccp_sha384_init;
 944                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
 945                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA384_DIGEST_SIZE;
 946                 break;
 947         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 948 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
 949                 if (auth_xform->key.length > SHA384_BLOCK_SIZE)
 950                         return -1;
 951                 sess->auth.algo = CCP_AUTH_ALGO_SHA384_HMAC;
 952                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA384_DIGEST_SIZE;
 953                 sess->auth.block_size = SHA384_BLOCK_SIZE;
 954                 sess->auth.key_length = auth_xform->key.length;
 955                 memset(sess->auth.key, 0, sess->auth.block_size);
 956                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 957                            auth_xform->key.length);
 958 #else
 959                 if (auth_xform->key.length > SHA384_BLOCK_SIZE)
 960                         return -1;
 961                 sess->auth.algo = CCP_AUTH_ALGO_SHA384_HMAC;
 962                 sess->auth.engine = CCP_ENGINE_SHA;
 963                 sess->auth.ut.sha_type = CCP_SHA_TYPE_384;
 964                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
 965                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA384_DIGEST_SIZE;
 966                 sess->auth.block_size = SHA384_BLOCK_SIZE;
 967                 sess->auth.key_length = auth_xform->key.length;
 968                 memset(sess->auth.key, 0, sess->auth.block_size);
 969                 memset(sess->auth.pre_compute, 0, sess->auth.ctx_len << 1);
 970                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 971                            auth_xform->key.length);
 972                 if (generate_partial_hash(sess))
 973                         return -1;
 974 #endif
 975                 break;
 976         case RTE_CRYPTO_AUTH_SHA3_384:
 977                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_384;
 978                 sess->auth.engine = CCP_ENGINE_SHA;
 979                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
 980                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 981                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
 982                 break;
 983         case RTE_CRYPTO_AUTH_SHA3_384_HMAC:
 984                 if (auth_xform->key.length > SHA3_384_BLOCK_SIZE)
 985                         return -1;
 986                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_384_HMAC;
 987                 sess->auth.engine = CCP_ENGINE_SHA;
 988                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
 989                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 990                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
 991                 sess->auth.block_size = SHA3_384_BLOCK_SIZE;
 992                 sess->auth.key_length = auth_xform->key.length;
 993                 memset(sess->auth.key, 0, sess->auth.block_size);
 994                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
 995                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 996                            auth_xform->key.length);
 997                 if (generate_partial_hash(sess))
 998                         return -1;
 999                 break;
1000         case RTE_CRYPTO_AUTH_SHA512:
1001                 sess->auth.algo = CCP_AUTH_ALGO_SHA512;
1002                 sess->auth.engine = CCP_ENGINE_SHA;
1003                 sess->auth.ut.sha_type = CCP_SHA_TYPE_512;
1004                 sess->auth.ctx = (void *)ccp_sha512_init;
1005                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
1006                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA512_DIGEST_SIZE;
1007                 break;
1008         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1009 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
1010                 if (auth_xform->key.length > SHA512_BLOCK_SIZE)
1011                         return -1;
1012                 sess->auth.algo = CCP_AUTH_ALGO_SHA512_HMAC;
1013                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA512_DIGEST_SIZE;
1014                 sess->auth.block_size = SHA512_BLOCK_SIZE;
1015                 sess->auth.key_length = auth_xform->key.length;
1016                 memset(sess->auth.key, 0, sess->auth.block_size);
1017                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1018                            auth_xform->key.length);
1019 #else
1020                 if (auth_xform->key.length > SHA512_BLOCK_SIZE)
1021                         return -1;
1022                 sess->auth.algo = CCP_AUTH_ALGO_SHA512_HMAC;
1023                 sess->auth.engine = CCP_ENGINE_SHA;
1024                 sess->auth.ut.sha_type = CCP_SHA_TYPE_512;
1025                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
1026                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA512_DIGEST_SIZE;
1027                 sess->auth.block_size = SHA512_BLOCK_SIZE;
1028                 sess->auth.key_length = auth_xform->key.length;
1029                 memset(sess->auth.key, 0, sess->auth.block_size);
1030                 memset(sess->auth.pre_compute, 0, sess->auth.ctx_len << 1);
1031                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1032                            auth_xform->key.length);
1033                 if (generate_partial_hash(sess))
1034                         return -1;
1035 #endif
1036                 break;
1037         case RTE_CRYPTO_AUTH_SHA3_512:
1038                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_512;
1039                 sess->auth.engine = CCP_ENGINE_SHA;
1040                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
1041                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
1042                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
1043                 break;
1044         case RTE_CRYPTO_AUTH_SHA3_512_HMAC:
1045                 if (auth_xform->key.length > SHA3_512_BLOCK_SIZE)
1046                         return -1;
1047                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_512_HMAC;
1048                 sess->auth.engine = CCP_ENGINE_SHA;
1049                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
1050                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
1051                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
1052                 sess->auth.block_size = SHA3_512_BLOCK_SIZE;
1053                 sess->auth.key_length = auth_xform->key.length;
1054                 memset(sess->auth.key, 0, sess->auth.block_size);
1055                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
1056                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1057                            auth_xform->key.length);
1058                 if (generate_partial_hash(sess))
1059                         return -1;
1060                 break;
1061         case RTE_CRYPTO_AUTH_AES_CMAC:
1062                 sess->auth.algo = CCP_AUTH_ALGO_AES_CMAC;
1063                 sess->auth.engine = CCP_ENGINE_AES;
1064                 sess->auth.um.aes_mode = CCP_AES_MODE_CMAC;
1065                 sess->auth.key_length = auth_xform->key.length;
1066                 /* padding and hash result */
1067                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
1068                 sess->auth.offset = AES_BLOCK_SIZE;
1069                 sess->auth.block_size = AES_BLOCK_SIZE;
1070                 if (sess->auth.key_length == 16)
1071                         sess->auth.ut.aes_type = CCP_AES_TYPE_128;
1072                 else if (sess->auth.key_length == 24)
1073                         sess->auth.ut.aes_type = CCP_AES_TYPE_192;
1074                 else if (sess->auth.key_length == 32)
1075                         sess->auth.ut.aes_type = CCP_AES_TYPE_256;
1076                 else {
1077                         CCP_LOG_ERR("Invalid CMAC key length");
1078                         return -1;
1079                 }
1080                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1081                            sess->auth.key_length);
1082                 for (i = 0; i < sess->auth.key_length; i++)
1083                         sess->auth.key_ccp[sess->auth.key_length - i - 1] =
1084                                 sess->auth.key[i];
1085                 if (generate_cmac_subkeys(sess))
1086                         return -1;
1087                 break;
1088         default:
1089                 CCP_LOG_ERR("Unsupported hash algo");
1090                 return -ENOTSUP;
1091         }
1092         return 0;
1093 }
1094
1095 static int
1096 ccp_configure_session_aead(struct ccp_session *sess,
1097                            const struct rte_crypto_sym_xform *xform)
1098 {
1099         const struct rte_crypto_aead_xform *aead_xform = NULL;
1100         size_t i;
1101
1102         aead_xform = &xform->aead;
1103
1104         sess->cipher.key_length = aead_xform->key.length;
1105         rte_memcpy(sess->cipher.key, aead_xform->key.data,
1106                    aead_xform->key.length);
1107
1108         if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
1109                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
1110                 sess->auth.op = CCP_AUTH_OP_GENERATE;
1111         } else {
1112                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
1113                 sess->auth.op = CCP_AUTH_OP_VERIFY;
1114         }
1115         sess->aead_algo = aead_xform->algo;
1116         sess->auth.aad_length = aead_xform->aad_length;
1117         sess->auth.digest_length = aead_xform->digest_length;
1118
1119         /* set iv parameters */
1120         sess->iv.offset = aead_xform->iv.offset;
1121         sess->iv.length = aead_xform->iv.length;
1122
1123         switch (aead_xform->algo) {
1124         case RTE_CRYPTO_AEAD_AES_GCM:
1125                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_GCM;
1126                 sess->cipher.um.aes_mode = CCP_AES_MODE_GCTR;
1127                 sess->cipher.engine = CCP_ENGINE_AES;
1128                 if (sess->cipher.key_length == 16)
1129                         sess->cipher.ut.aes_type = CCP_AES_TYPE_128;
1130                 else if (sess->cipher.key_length == 24)
1131                         sess->cipher.ut.aes_type = CCP_AES_TYPE_192;
1132                 else if (sess->cipher.key_length == 32)
1133                         sess->cipher.ut.aes_type = CCP_AES_TYPE_256;
1134                 else {
1135                         CCP_LOG_ERR("Invalid aead key length");
1136                         return -1;
1137                 }
1138                 for (i = 0; i < sess->cipher.key_length; i++)
1139                         sess->cipher.key_ccp[sess->cipher.key_length - i - 1] =
1140                                 sess->cipher.key[i];
1141                 sess->auth.algo = CCP_AUTH_ALGO_AES_GCM;
1142                 sess->auth.engine = CCP_ENGINE_AES;
1143                 sess->auth.um.aes_mode = CCP_AES_MODE_GHASH;
1144                 sess->auth.ctx_len = CCP_SB_BYTES;
1145                 sess->auth.offset = 0;
1146                 sess->auth.block_size = AES_BLOCK_SIZE;
1147                 sess->cmd_id = CCP_CMD_COMBINED;
1148                 break;
1149         default:
1150                 CCP_LOG_ERR("Unsupported aead algo");
1151                 return -ENOTSUP;
1152         }
1153         sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce);
1154         sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp);
1155         return 0;
1156 }
1157
1158 int
1159 ccp_set_session_parameters(struct ccp_session *sess,
1160                            const struct rte_crypto_sym_xform *xform)
1161 {
1162         const struct rte_crypto_sym_xform *cipher_xform = NULL;
1163         const struct rte_crypto_sym_xform *auth_xform = NULL;
1164         const struct rte_crypto_sym_xform *aead_xform = NULL;
1165         int ret = 0;
1166
1167         sess->cmd_id = ccp_get_cmd_id(xform);
1168
1169         switch (sess->cmd_id) {
1170         case CCP_CMD_CIPHER:
1171                 cipher_xform = xform;
1172                 break;
1173         case CCP_CMD_AUTH:
1174                 auth_xform = xform;
1175                 break;
1176         case CCP_CMD_CIPHER_HASH:
1177                 cipher_xform = xform;
1178                 auth_xform = xform->next;
1179                 break;
1180         case CCP_CMD_HASH_CIPHER:
1181                 auth_xform = xform;
1182                 cipher_xform = xform->next;
1183                 break;
1184         case CCP_CMD_COMBINED:
1185                 aead_xform = xform;
1186                 break;
1187         default:
1188                 CCP_LOG_ERR("Unsupported cmd_id");
1189                 return -1;
1190         }
1191
1192         /* Default IV length = 0 */
1193         sess->iv.length = 0;
1194         if (cipher_xform) {
1195                 ret = ccp_configure_session_cipher(sess, cipher_xform);
1196                 if (ret != 0) {
1197                         CCP_LOG_ERR("Invalid/unsupported cipher parameters");
1198                         return ret;
1199                 }
1200         }
1201         if (auth_xform) {
1202                 ret = ccp_configure_session_auth(sess, auth_xform);
1203                 if (ret != 0) {
1204                         CCP_LOG_ERR("Invalid/unsupported auth parameters");
1205                         return ret;
1206                 }
1207         }
1208         if (aead_xform) {
1209                 ret = ccp_configure_session_aead(sess, aead_xform);
1210                 if (ret != 0) {
1211                         CCP_LOG_ERR("Invalid/unsupported aead parameters");
1212                         return ret;
1213                 }
1214         }
1215         return ret;
1216 }
1217
1218 /* calculate CCP descriptors requirement */
1219 static inline int
1220 ccp_cipher_slot(struct ccp_session *session)
1221 {
1222         int count = 0;
1223
1224         switch (session->cipher.algo) {
1225         case CCP_CIPHER_ALGO_AES_CBC:
1226                 count = 2;
1227                 /**< op + passthrough for iv */
1228                 break;
1229         case CCP_CIPHER_ALGO_AES_ECB:
1230                 count = 1;
1231                 /**<only op*/
1232                 break;
1233         case CCP_CIPHER_ALGO_AES_CTR:
1234                 count = 2;
1235                 /**< op + passthrough for iv */
1236                 break;
1237         case CCP_CIPHER_ALGO_3DES_CBC:
1238                 count = 2;
1239                 /**< op + passthrough for iv */
1240                 break;
1241         default:
1242                 CCP_LOG_ERR("Unsupported cipher algo %d",
1243                             session->cipher.algo);
1244         }
1245         return count;
1246 }
1247
1248 static inline int
1249 ccp_auth_slot(struct ccp_session *session)
1250 {
1251         int count = 0;
1252
1253         switch (session->auth.algo) {
1254         case CCP_AUTH_ALGO_SHA1:
1255         case CCP_AUTH_ALGO_SHA224:
1256         case CCP_AUTH_ALGO_SHA256:
1257         case CCP_AUTH_ALGO_SHA384:
1258         case CCP_AUTH_ALGO_SHA512:
1259                 count = 3;
1260                 /**< op + lsb passthrough cpy to/from*/
1261                 break;
1262 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
1263         case CCP_AUTH_ALGO_MD5_HMAC:
1264                 break;
1265 #endif
1266         case CCP_AUTH_ALGO_SHA1_HMAC:
1267         case CCP_AUTH_ALGO_SHA224_HMAC:
1268         case CCP_AUTH_ALGO_SHA256_HMAC:
1269 #ifndef RTE_LIBRTE_PMD_CCP_CPU_AUTH
1270                 count = 6;
1271 #endif
1272                 break;
1273         case CCP_AUTH_ALGO_SHA384_HMAC:
1274         case CCP_AUTH_ALGO_SHA512_HMAC:
1275 #ifndef RTE_LIBRTE_PMD_CCP_CPU_AUTH
1276                 count = 7;
1277 #endif
1278                 /**
1279                  * 1. Load PHash1 = H(k ^ ipad); to LSB
1280                  * 2. generate IHash = H(hash on meassage with PHash1
1281                  * as init values);
1282                  * 3. Retrieve IHash 2 slots for 384/512
1283                  * 4. Load Phash2 = H(k ^ opad); to LSB
1284                  * 5. generate FHash = H(hash on Ihash with Phash2
1285                  * as init value);
1286                  * 6. Retrieve HMAC output from LSB to host memory
1287                  */
1288                 break;
1289         case CCP_AUTH_ALGO_SHA3_224:
1290         case CCP_AUTH_ALGO_SHA3_256:
1291         case CCP_AUTH_ALGO_SHA3_384:
1292         case CCP_AUTH_ALGO_SHA3_512:
1293                 count = 1;
1294                 /**< only op ctx and dst in host memory*/
1295                 break;
1296         case CCP_AUTH_ALGO_SHA3_224_HMAC:
1297         case CCP_AUTH_ALGO_SHA3_256_HMAC:
1298                 count = 3;
1299                 break;
1300         case CCP_AUTH_ALGO_SHA3_384_HMAC:
1301         case CCP_AUTH_ALGO_SHA3_512_HMAC:
1302                 count = 4;
1303                 /**
1304                  * 1. Op to Perform Ihash
1305                  * 2. Retrieve result from LSB to host memory
1306                  * 3. Perform final hash
1307                  */
1308                 break;
1309         case CCP_AUTH_ALGO_AES_CMAC:
1310                 count = 4;
1311                 /**
1312                  * op
1313                  * extra descriptor in padding case
1314                  * (k1/k2(255:128) with iv(127:0))
1315                  * Retrieve result
1316                  */
1317                 break;
1318         default:
1319                 CCP_LOG_ERR("Unsupported auth algo %d",
1320                             session->auth.algo);
1321         }
1322
1323         return count;
1324 }
1325
1326 static int
1327 ccp_aead_slot(struct ccp_session *session)
1328 {
1329         int count = 0;
1330
1331         switch (session->aead_algo) {
1332         case RTE_CRYPTO_AEAD_AES_GCM:
1333                 break;
1334         default:
1335                 CCP_LOG_ERR("Unsupported aead algo %d",
1336                             session->aead_algo);
1337         }
1338         switch (session->auth.algo) {
1339         case CCP_AUTH_ALGO_AES_GCM:
1340                 count = 5;
1341                 /**
1342                  * 1. Passthru iv
1343                  * 2. Hash AAD
1344                  * 3. GCTR
1345                  * 4. Reload passthru
1346                  * 5. Hash Final tag
1347                  */
1348                 break;
1349         default:
1350                 CCP_LOG_ERR("Unsupported combined auth ALGO %d",
1351                             session->auth.algo);
1352         }
1353         return count;
1354 }
1355
1356 int
1357 ccp_compute_slot_count(struct ccp_session *session)
1358 {
1359         int count = 0;
1360
1361         switch (session->cmd_id) {
1362         case CCP_CMD_CIPHER:
1363                 count = ccp_cipher_slot(session);
1364                 break;
1365         case CCP_CMD_AUTH:
1366                 count = ccp_auth_slot(session);
1367                 break;
1368         case CCP_CMD_CIPHER_HASH:
1369         case CCP_CMD_HASH_CIPHER:
1370                 count = ccp_cipher_slot(session);
1371                 count += ccp_auth_slot(session);
1372                 break;
1373         case CCP_CMD_COMBINED:
1374                 count = ccp_aead_slot(session);
1375                 break;
1376         default:
1377                 CCP_LOG_ERR("Unsupported cmd_id");
1378
1379         }
1380
1381         return count;
1382 }
1383
1384 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
1385 static uint8_t
1386 algo_select(int sessalgo,
1387             const EVP_MD **algo)
1388 {
1389         int res = 0;
1390
1391         switch (sessalgo) {
1392         case CCP_AUTH_ALGO_MD5_HMAC:
1393                 *algo = EVP_md5();
1394                 break;
1395         case CCP_AUTH_ALGO_SHA1_HMAC:
1396                 *algo = EVP_sha1();
1397                 break;
1398         case CCP_AUTH_ALGO_SHA224_HMAC:
1399                 *algo = EVP_sha224();
1400                 break;
1401         case CCP_AUTH_ALGO_SHA256_HMAC:
1402                 *algo = EVP_sha256();
1403                 break;
1404         case CCP_AUTH_ALGO_SHA384_HMAC:
1405                 *algo = EVP_sha384();
1406                 break;
1407         case CCP_AUTH_ALGO_SHA512_HMAC:
1408                 *algo = EVP_sha512();
1409                 break;
1410         default:
1411                 res = -EINVAL;
1412                 break;
1413         }
1414         return res;
1415 }
1416
1417 static int
1418 process_cpu_auth_hmac(uint8_t *src, uint8_t *dst,
1419                       __rte_unused uint8_t *iv,
1420                       EVP_PKEY *pkey,
1421                       int srclen,
1422                       EVP_MD_CTX *ctx,
1423                       const EVP_MD *algo,
1424                       uint16_t d_len)
1425 {
1426         size_t dstlen;
1427         unsigned char temp_dst[64];
1428
1429         if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0)
1430                 goto process_auth_err;
1431
1432         if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0)
1433                 goto process_auth_err;
1434
1435         if (EVP_DigestSignFinal(ctx, temp_dst, &dstlen) <= 0)
1436                 goto process_auth_err;
1437
1438         memcpy(dst, temp_dst, d_len);
1439         return 0;
1440 process_auth_err:
1441         CCP_LOG_ERR("Process cpu auth failed");
1442         return -EINVAL;
1443 }
1444
1445 static int cpu_crypto_auth(struct ccp_qp *qp,
1446                            struct rte_crypto_op *op,
1447                            struct ccp_session *sess,
1448                            EVP_MD_CTX *ctx)
1449 {
1450         uint8_t *src, *dst;
1451         int srclen, status;
1452         struct rte_mbuf *mbuf_src, *mbuf_dst;
1453         const EVP_MD *algo = NULL;
1454         EVP_PKEY *pkey;
1455
1456         algo_select(sess->auth.algo, &algo);
1457         pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, sess->auth.key,
1458                                     sess->auth.key_length);
1459         mbuf_src = op->sym->m_src;
1460         mbuf_dst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1461         srclen = op->sym->auth.data.length;
1462         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1463                                       op->sym->auth.data.offset);
1464
1465         if (sess->auth.op == CCP_AUTH_OP_VERIFY) {
1466                 dst = qp->temp_digest;
1467         } else {
1468                 dst = op->sym->auth.digest.data;
1469                 if (dst == NULL) {
1470                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1471                                                      op->sym->auth.data.offset +
1472                                                      sess->auth.digest_length);
1473                 }
1474         }
1475         status = process_cpu_auth_hmac(src, dst, NULL,
1476                                        pkey, srclen,
1477                                        ctx,
1478                                        algo,
1479                                        sess->auth.digest_length);
1480         if (status) {
1481                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1482                 return status;
1483         }
1484
1485         if (sess->auth.op == CCP_AUTH_OP_VERIFY) {
1486                 if (memcmp(dst, op->sym->auth.digest.data,
1487                            sess->auth.digest_length) != 0) {
1488                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1489                 } else {
1490                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1491                 }
1492         } else {
1493                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1494         }
1495         EVP_PKEY_free(pkey);
1496         return 0;
1497 }
1498 #endif
1499
1500 static void
1501 ccp_perform_passthru(struct ccp_passthru *pst,
1502                      struct ccp_queue *cmd_q)
1503 {
1504         struct ccp_desc *desc;
1505         union ccp_function function;
1506
1507         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1508
1509         CCP_CMD_ENGINE(desc) = CCP_ENGINE_PASSTHRU;
1510
1511         CCP_CMD_SOC(desc) = 0;
1512         CCP_CMD_IOC(desc) = 0;
1513         CCP_CMD_INIT(desc) = 0;
1514         CCP_CMD_EOM(desc) = 0;
1515         CCP_CMD_PROT(desc) = 0;
1516
1517         function.raw = 0;
1518         CCP_PT_BYTESWAP(&function) = pst->byte_swap;
1519         CCP_PT_BITWISE(&function) = pst->bit_mod;
1520         CCP_CMD_FUNCTION(desc) = function.raw;
1521
1522         CCP_CMD_LEN(desc) = pst->len;
1523
1524         if (pst->dir) {
1525                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
1526                 CCP_CMD_SRC_HI(desc) = high32_value(pst->src_addr);
1527                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1528
1529                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
1530                 CCP_CMD_DST_HI(desc) = 0;
1531                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
1532
1533                 if (pst->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1534                         CCP_CMD_LSB_ID(desc) = cmd_q->sb_key;
1535         } else {
1536
1537                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
1538                 CCP_CMD_SRC_HI(desc) = 0;
1539                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SB;
1540
1541                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
1542                 CCP_CMD_DST_HI(desc) = high32_value(pst->dest_addr);
1543                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1544         }
1545
1546         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1547 }
1548
1549 static int
1550 ccp_perform_hmac(struct rte_crypto_op *op,
1551                  struct ccp_queue *cmd_q)
1552 {
1553
1554         struct ccp_session *session;
1555         union ccp_function function;
1556         struct ccp_desc *desc;
1557         uint32_t tail;
1558         phys_addr_t src_addr, dest_addr, dest_addr_t;
1559         struct ccp_passthru pst;
1560         uint64_t auth_msg_bits;
1561         void *append_ptr;
1562         uint8_t *addr;
1563
1564         session = (struct ccp_session *)get_session_private_data(
1565                                          op->sym->session,
1566                                          ccp_cryptodev_driver_id);
1567         addr = session->auth.pre_compute;
1568
1569         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1570                                               op->sym->auth.data.offset);
1571         append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src,
1572                                                 session->auth.ctx_len);
1573         dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr);
1574         dest_addr_t = dest_addr;
1575
1576         /** Load PHash1 to LSB*/
1577         pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr);
1578         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1579         pst.len = session->auth.ctx_len;
1580         pst.dir = 1;
1581         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1582         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1583         ccp_perform_passthru(&pst, cmd_q);
1584
1585         /**sha engine command descriptor for IntermediateHash*/
1586
1587         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1588         memset(desc, 0, Q_DESC_SIZE);
1589
1590         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1591
1592         CCP_CMD_SOC(desc) = 0;
1593         CCP_CMD_IOC(desc) = 0;
1594         CCP_CMD_INIT(desc) = 1;
1595         CCP_CMD_EOM(desc) = 1;
1596         CCP_CMD_PROT(desc) = 0;
1597
1598         function.raw = 0;
1599         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1600         CCP_CMD_FUNCTION(desc) = function.raw;
1601
1602         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1603         auth_msg_bits = (op->sym->auth.data.length +
1604                          session->auth.block_size)  * 8;
1605
1606         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1607         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1608         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1609
1610         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1611         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1612         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1613
1614         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1615
1616         rte_wmb();
1617
1618         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1619         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1620         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1621                       cmd_q->qcontrol | CMD_Q_RUN);
1622
1623         /* Intermediate Hash value retrieve */
1624         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1625             (session->auth.ut.sha_type == CCP_SHA_TYPE_512)) {
1626
1627                 pst.src_addr =
1628                         (phys_addr_t)((cmd_q->sb_sha + 1) * CCP_SB_BYTES);
1629                 pst.dest_addr = dest_addr_t;
1630                 pst.len = CCP_SB_BYTES;
1631                 pst.dir = 0;
1632                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1633                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1634                 ccp_perform_passthru(&pst, cmd_q);
1635
1636                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1637                 pst.dest_addr = dest_addr_t + CCP_SB_BYTES;
1638                 pst.len = CCP_SB_BYTES;
1639                 pst.dir = 0;
1640                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1641                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1642                 ccp_perform_passthru(&pst, cmd_q);
1643
1644         } else {
1645                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1646                 pst.dest_addr = dest_addr_t;
1647                 pst.len = session->auth.ctx_len;
1648                 pst.dir = 0;
1649                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1650                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1651                 ccp_perform_passthru(&pst, cmd_q);
1652
1653         }
1654
1655         /** Load PHash2 to LSB*/
1656         addr += session->auth.ctx_len;
1657         pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr);
1658         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1659         pst.len = session->auth.ctx_len;
1660         pst.dir = 1;
1661         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1662         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1663         ccp_perform_passthru(&pst, cmd_q);
1664
1665         /**sha engine command descriptor for FinalHash*/
1666         dest_addr_t += session->auth.offset;
1667
1668         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1669         memset(desc, 0, Q_DESC_SIZE);
1670
1671         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1672
1673         CCP_CMD_SOC(desc) = 0;
1674         CCP_CMD_IOC(desc) = 0;
1675         CCP_CMD_INIT(desc) = 1;
1676         CCP_CMD_EOM(desc) = 1;
1677         CCP_CMD_PROT(desc) = 0;
1678
1679         function.raw = 0;
1680         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1681         CCP_CMD_FUNCTION(desc) = function.raw;
1682
1683         CCP_CMD_LEN(desc) = (session->auth.ctx_len -
1684                              session->auth.offset);
1685         auth_msg_bits = (session->auth.block_size +
1686                          session->auth.ctx_len -
1687                          session->auth.offset) * 8;
1688
1689         CCP_CMD_SRC_LO(desc) = (uint32_t)(dest_addr_t);
1690         CCP_CMD_SRC_HI(desc) = high32_value(dest_addr_t);
1691         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1692
1693         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1694         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1695         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1696
1697         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1698
1699         rte_wmb();
1700
1701         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1702         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1703         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1704                       cmd_q->qcontrol | CMD_Q_RUN);
1705
1706         /* Retrieve hmac output */
1707         pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1708         pst.dest_addr = dest_addr;
1709         pst.len = session->auth.ctx_len;
1710         pst.dir = 0;
1711         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1712         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1713             (session->auth.ut.sha_type == CCP_SHA_TYPE_512))
1714                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1715         else
1716                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1717         ccp_perform_passthru(&pst, cmd_q);
1718
1719         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1720         return 0;
1721
1722 }
1723
1724 static int
1725 ccp_perform_sha(struct rte_crypto_op *op,
1726                 struct ccp_queue *cmd_q)
1727 {
1728         struct ccp_session *session;
1729         union ccp_function function;
1730         struct ccp_desc *desc;
1731         uint32_t tail;
1732         phys_addr_t src_addr, dest_addr;
1733         struct ccp_passthru pst;
1734         void *append_ptr;
1735         uint64_t auth_msg_bits;
1736
1737         session = (struct ccp_session *)get_session_private_data(
1738                                          op->sym->session,
1739                                         ccp_cryptodev_driver_id);
1740
1741         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1742                                               op->sym->auth.data.offset);
1743
1744         append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src,
1745                                                 session->auth.ctx_len);
1746         dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr);
1747
1748         /** Passthru sha context*/
1749
1750         pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)
1751                                                      session->auth.ctx);
1752         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1753         pst.len = session->auth.ctx_len;
1754         pst.dir = 1;
1755         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1756         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1757         ccp_perform_passthru(&pst, cmd_q);
1758
1759         /**prepare sha command descriptor*/
1760
1761         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1762         memset(desc, 0, Q_DESC_SIZE);
1763
1764         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1765
1766         CCP_CMD_SOC(desc) = 0;
1767         CCP_CMD_IOC(desc) = 0;
1768         CCP_CMD_INIT(desc) = 1;
1769         CCP_CMD_EOM(desc) = 1;
1770         CCP_CMD_PROT(desc) = 0;
1771
1772         function.raw = 0;
1773         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1774         CCP_CMD_FUNCTION(desc) = function.raw;
1775
1776         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1777         auth_msg_bits = op->sym->auth.data.length * 8;
1778
1779         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1780         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1781         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1782
1783         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1784         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1785         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1786
1787         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1788
1789         rte_wmb();
1790
1791         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1792         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1793         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1794                       cmd_q->qcontrol | CMD_Q_RUN);
1795
1796         /* Hash value retrieve */
1797         pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1798         pst.dest_addr = dest_addr;
1799         pst.len = session->auth.ctx_len;
1800         pst.dir = 0;
1801         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1802         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1803             (session->auth.ut.sha_type == CCP_SHA_TYPE_512))
1804                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1805         else
1806                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1807         ccp_perform_passthru(&pst, cmd_q);
1808
1809         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1810         return 0;
1811
1812 }
1813
1814 static int
1815 ccp_perform_sha3_hmac(struct rte_crypto_op *op,
1816                       struct ccp_queue *cmd_q)
1817 {
1818         struct ccp_session *session;
1819         struct ccp_passthru pst;
1820         union ccp_function function;
1821         struct ccp_desc *desc;
1822         uint8_t *append_ptr;
1823         uint32_t tail;
1824         phys_addr_t src_addr, dest_addr, ctx_paddr, dest_addr_t;
1825
1826         session = (struct ccp_session *)get_session_private_data(
1827                                          op->sym->session,
1828                                         ccp_cryptodev_driver_id);
1829
1830         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1831                                               op->sym->auth.data.offset);
1832         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
1833                                                 session->auth.ctx_len);
1834         if (!append_ptr) {
1835                 CCP_LOG_ERR("CCP MBUF append failed\n");
1836                 return -1;
1837         }
1838         dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
1839         dest_addr_t = dest_addr + (session->auth.ctx_len / 2);
1840         ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void
1841                                                    *)session->auth.pre_compute);
1842         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1843         memset(desc, 0, Q_DESC_SIZE);
1844
1845         /*desc1 for SHA3-Ihash operation */
1846         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1847         CCP_CMD_INIT(desc) = 1;
1848         CCP_CMD_EOM(desc) = 1;
1849
1850         function.raw = 0;
1851         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1852         CCP_CMD_FUNCTION(desc) = function.raw;
1853         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1854
1855         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1856         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1857         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1858
1859         CCP_CMD_DST_LO(desc) = (cmd_q->sb_sha * CCP_SB_BYTES);
1860         CCP_CMD_DST_HI(desc) = 0;
1861         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
1862
1863         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
1864         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
1865         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1866
1867         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1868
1869         rte_wmb();
1870         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1871         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1872         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1873                       cmd_q->qcontrol | CMD_Q_RUN);
1874
1875         /* Intermediate Hash value retrieve */
1876         if ((session->auth.ut.sha_type == CCP_SHA3_TYPE_384) ||
1877             (session->auth.ut.sha_type == CCP_SHA3_TYPE_512)) {
1878
1879                 pst.src_addr =
1880                         (phys_addr_t)((cmd_q->sb_sha + 1) * CCP_SB_BYTES);
1881                 pst.dest_addr = dest_addr_t;
1882                 pst.len = CCP_SB_BYTES;
1883                 pst.dir = 0;
1884                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1885                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1886                 ccp_perform_passthru(&pst, cmd_q);
1887
1888                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1889                 pst.dest_addr = dest_addr_t + CCP_SB_BYTES;
1890                 pst.len = CCP_SB_BYTES;
1891                 pst.dir = 0;
1892                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1893                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1894                 ccp_perform_passthru(&pst, cmd_q);
1895
1896         } else {
1897                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1898                 pst.dest_addr = dest_addr_t;
1899                 pst.len = CCP_SB_BYTES;
1900                 pst.dir = 0;
1901                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1902                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1903                 ccp_perform_passthru(&pst, cmd_q);
1904         }
1905
1906         /**sha engine command descriptor for FinalHash*/
1907         ctx_paddr += CCP_SHA3_CTX_SIZE;
1908         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1909         memset(desc, 0, Q_DESC_SIZE);
1910
1911         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1912         CCP_CMD_INIT(desc) = 1;
1913         CCP_CMD_EOM(desc) = 1;
1914
1915         function.raw = 0;
1916         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1917         CCP_CMD_FUNCTION(desc) = function.raw;
1918
1919         if (session->auth.ut.sha_type == CCP_SHA3_TYPE_224) {
1920                 dest_addr_t += (CCP_SB_BYTES - SHA224_DIGEST_SIZE);
1921                 CCP_CMD_LEN(desc) = SHA224_DIGEST_SIZE;
1922         } else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_256) {
1923                 CCP_CMD_LEN(desc) = SHA256_DIGEST_SIZE;
1924         } else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_384) {
1925                 dest_addr_t += (2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE);
1926                 CCP_CMD_LEN(desc) = SHA384_DIGEST_SIZE;
1927         } else {
1928                 CCP_CMD_LEN(desc) = SHA512_DIGEST_SIZE;
1929         }
1930
1931         CCP_CMD_SRC_LO(desc) = ((uint32_t)dest_addr_t);
1932         CCP_CMD_SRC_HI(desc) = high32_value(dest_addr_t);
1933         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1934
1935         CCP_CMD_DST_LO(desc) = (uint32_t)dest_addr;
1936         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
1937         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1938
1939         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
1940         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
1941         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1942
1943         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1944
1945         rte_wmb();
1946         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1947         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1948         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1949                       cmd_q->qcontrol | CMD_Q_RUN);
1950
1951         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1952         return 0;
1953 }
1954
1955 static int
1956 ccp_perform_sha3(struct rte_crypto_op *op,
1957                  struct ccp_queue *cmd_q)
1958 {
1959         struct ccp_session *session;
1960         union ccp_function function;
1961         struct ccp_desc *desc;
1962         uint8_t *ctx_addr, *append_ptr;
1963         uint32_t tail;
1964         phys_addr_t src_addr, dest_addr, ctx_paddr;
1965
1966         session = (struct ccp_session *)get_session_private_data(
1967                                          op->sym->session,
1968                                         ccp_cryptodev_driver_id);
1969
1970         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1971                                               op->sym->auth.data.offset);
1972         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
1973                                                 session->auth.ctx_len);
1974         if (!append_ptr) {
1975                 CCP_LOG_ERR("CCP MBUF append failed\n");
1976                 return -1;
1977         }
1978         dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
1979         ctx_addr = session->auth.sha3_ctx;
1980         ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
1981
1982         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1983         memset(desc, 0, Q_DESC_SIZE);
1984
1985         /* prepare desc for SHA3 operation */
1986         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1987         CCP_CMD_INIT(desc) = 1;
1988         CCP_CMD_EOM(desc) = 1;
1989
1990         function.raw = 0;
1991         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1992         CCP_CMD_FUNCTION(desc) = function.raw;
1993
1994         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1995
1996         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1997         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1998         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1999
2000         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2001         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2002         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2003
2004         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
2005         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
2006         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2007
2008         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2009
2010         rte_wmb();
2011
2012         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2013         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2014         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2015                       cmd_q->qcontrol | CMD_Q_RUN);
2016
2017         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2018         return 0;
2019 }
2020
2021 static int
2022 ccp_perform_aes_cmac(struct rte_crypto_op *op,
2023                      struct ccp_queue *cmd_q)
2024 {
2025         struct ccp_session *session;
2026         union ccp_function function;
2027         struct ccp_passthru pst;
2028         struct ccp_desc *desc;
2029         uint32_t tail;
2030         uint8_t *src_tb, *append_ptr, *ctx_addr;
2031         phys_addr_t src_addr, dest_addr, key_addr;
2032         int length, non_align_len;
2033
2034         session = (struct ccp_session *)get_session_private_data(
2035                                          op->sym->session,
2036                                         ccp_cryptodev_driver_id);
2037         key_addr = rte_mem_virt2phy(session->auth.key_ccp);
2038
2039         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2040                                               op->sym->auth.data.offset);
2041         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
2042                                                 session->auth.ctx_len);
2043         dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
2044
2045         function.raw = 0;
2046         CCP_AES_ENCRYPT(&function) = CCP_CIPHER_DIR_ENCRYPT;
2047         CCP_AES_MODE(&function) = session->auth.um.aes_mode;
2048         CCP_AES_TYPE(&function) = session->auth.ut.aes_type;
2049
2050         if (op->sym->auth.data.length % session->auth.block_size == 0) {
2051
2052                 ctx_addr = session->auth.pre_compute;
2053                 memset(ctx_addr, 0, AES_BLOCK_SIZE);
2054                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
2055                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2056                 pst.len = CCP_SB_BYTES;
2057                 pst.dir = 1;
2058                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2059                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2060                 ccp_perform_passthru(&pst, cmd_q);
2061
2062                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2063                 memset(desc, 0, Q_DESC_SIZE);
2064
2065                 /* prepare desc for aes-cmac command */
2066                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2067                 CCP_CMD_EOM(desc) = 1;
2068                 CCP_CMD_FUNCTION(desc) = function.raw;
2069
2070                 CCP_CMD_LEN(desc) = op->sym->auth.data.length;
2071                 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2072                 CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2073                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2074
2075                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2076                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2077                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2078                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2079
2080                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2081
2082                 rte_wmb();
2083
2084                 tail =
2085                 (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2086                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2087                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2088                               cmd_q->qcontrol | CMD_Q_RUN);
2089         } else {
2090                 ctx_addr = session->auth.pre_compute + CCP_SB_BYTES;
2091                 memset(ctx_addr, 0, AES_BLOCK_SIZE);
2092                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
2093                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2094                 pst.len = CCP_SB_BYTES;
2095                 pst.dir = 1;
2096                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2097                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2098                 ccp_perform_passthru(&pst, cmd_q);
2099
2100                 length = (op->sym->auth.data.length / AES_BLOCK_SIZE);
2101                 length *= AES_BLOCK_SIZE;
2102                 non_align_len = op->sym->auth.data.length - length;
2103                 /* prepare desc for aes-cmac command */
2104                 /*Command 1*/
2105                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2106                 memset(desc, 0, Q_DESC_SIZE);
2107
2108                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2109                 CCP_CMD_INIT(desc) = 1;
2110                 CCP_CMD_FUNCTION(desc) = function.raw;
2111
2112                 CCP_CMD_LEN(desc) = length;
2113                 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2114                 CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2115                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2116
2117                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2118                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2119                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2120                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2121
2122                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2123
2124                 /*Command 2*/
2125                 append_ptr = append_ptr + CCP_SB_BYTES;
2126                 memset(append_ptr, 0, AES_BLOCK_SIZE);
2127                 src_tb = rte_pktmbuf_mtod_offset(op->sym->m_src,
2128                                                  uint8_t *,
2129                                                  op->sym->auth.data.offset +
2130                                                  length);
2131                 rte_memcpy(append_ptr, src_tb, non_align_len);
2132                 append_ptr[non_align_len] = CMAC_PAD_VALUE;
2133
2134                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2135                 memset(desc, 0, Q_DESC_SIZE);
2136
2137                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2138                 CCP_CMD_EOM(desc) = 1;
2139                 CCP_CMD_FUNCTION(desc) = function.raw;
2140                 CCP_CMD_LEN(desc) = AES_BLOCK_SIZE;
2141
2142                 CCP_CMD_SRC_LO(desc) = ((uint32_t)(dest_addr + CCP_SB_BYTES));
2143                 CCP_CMD_SRC_HI(desc) = high32_value(dest_addr + CCP_SB_BYTES);
2144                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2145
2146                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2147                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2148                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2149                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2150
2151                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2152
2153                 rte_wmb();
2154                 tail =
2155                 (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2156                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2157                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2158                               cmd_q->qcontrol | CMD_Q_RUN);
2159         }
2160         /* Retrieve result */
2161         pst.dest_addr = dest_addr;
2162         pst.src_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2163         pst.len = CCP_SB_BYTES;
2164         pst.dir = 0;
2165         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2166         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2167         ccp_perform_passthru(&pst, cmd_q);
2168
2169         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2170         return 0;
2171 }
2172
2173 static int
2174 ccp_perform_aes(struct rte_crypto_op *op,
2175                 struct ccp_queue *cmd_q,
2176                 struct ccp_batch_info *b_info)
2177 {
2178         struct ccp_session *session;
2179         union ccp_function function;
2180         uint8_t *lsb_buf;
2181         struct ccp_passthru pst = {0};
2182         struct ccp_desc *desc;
2183         phys_addr_t src_addr, dest_addr, key_addr;
2184         uint8_t *iv;
2185
2186         session = (struct ccp_session *)get_session_private_data(
2187                                          op->sym->session,
2188                                         ccp_cryptodev_driver_id);
2189         function.raw = 0;
2190
2191         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2192         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB) {
2193                 if (session->cipher.um.aes_mode == CCP_AES_MODE_CTR) {
2194                         rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE,
2195                                    iv, session->iv.length);
2196                         pst.src_addr = (phys_addr_t)session->cipher.nonce_phys;
2197                         CCP_AES_SIZE(&function) = 0x1F;
2198                 } else {
2199                         lsb_buf =
2200                         &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]);
2201                         rte_memcpy(lsb_buf +
2202                                    (CCP_SB_BYTES - session->iv.length),
2203                                    iv, session->iv.length);
2204                         pst.src_addr = b_info->lsb_buf_phys +
2205                                 (b_info->lsb_buf_idx * CCP_SB_BYTES);
2206                         b_info->lsb_buf_idx++;
2207                 }
2208
2209                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2210                 pst.len = CCP_SB_BYTES;
2211                 pst.dir = 1;
2212                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2213                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2214                 ccp_perform_passthru(&pst, cmd_q);
2215         }
2216
2217         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2218
2219         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2220                                               op->sym->cipher.data.offset);
2221         if (likely(op->sym->m_dst != NULL))
2222                 dest_addr = rte_pktmbuf_mtophys_offset(op->sym->m_dst,
2223                                                 op->sym->cipher.data.offset);
2224         else
2225                 dest_addr = src_addr;
2226         key_addr = session->cipher.key_phys;
2227
2228         /* prepare desc for aes command */
2229         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2230         CCP_CMD_INIT(desc) = 1;
2231         CCP_CMD_EOM(desc) = 1;
2232
2233         CCP_AES_ENCRYPT(&function) = session->cipher.dir;
2234         CCP_AES_MODE(&function) = session->cipher.um.aes_mode;
2235         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2236         CCP_CMD_FUNCTION(desc) = function.raw;
2237
2238         CCP_CMD_LEN(desc) = op->sym->cipher.data.length;
2239
2240         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2241         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2242         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2243
2244         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2245         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2246         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2247
2248         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2249         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2250         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2251
2252         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB)
2253                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2254
2255         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2256         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2257         return 0;
2258 }
2259
2260 static int
2261 ccp_perform_3des(struct rte_crypto_op *op,
2262                 struct ccp_queue *cmd_q,
2263                 struct ccp_batch_info *b_info)
2264 {
2265         struct ccp_session *session;
2266         union ccp_function function;
2267         unsigned char *lsb_buf;
2268         struct ccp_passthru pst;
2269         struct ccp_desc *desc;
2270         uint32_t tail;
2271         uint8_t *iv;
2272         phys_addr_t src_addr, dest_addr, key_addr;
2273
2274         session = (struct ccp_session *)get_session_private_data(
2275                                          op->sym->session,
2276                                         ccp_cryptodev_driver_id);
2277
2278         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2279         switch (session->cipher.um.des_mode) {
2280         case CCP_DES_MODE_CBC:
2281                 lsb_buf = &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]);
2282                 b_info->lsb_buf_idx++;
2283
2284                 rte_memcpy(lsb_buf + (CCP_SB_BYTES - session->iv.length),
2285                            iv, session->iv.length);
2286
2287                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *) lsb_buf);
2288                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2289                 pst.len = CCP_SB_BYTES;
2290                 pst.dir = 1;
2291                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2292                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2293                 ccp_perform_passthru(&pst, cmd_q);
2294                 break;
2295         case CCP_DES_MODE_CFB:
2296         case CCP_DES_MODE_ECB:
2297                 CCP_LOG_ERR("Unsupported DES cipher mode");
2298                 return -ENOTSUP;
2299         }
2300
2301         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2302                                               op->sym->cipher.data.offset);
2303         if (unlikely(op->sym->m_dst != NULL))
2304                 dest_addr =
2305                         rte_pktmbuf_mtophys_offset(op->sym->m_dst,
2306                                                    op->sym->cipher.data.offset);
2307         else
2308                 dest_addr = src_addr;
2309
2310         key_addr = rte_mem_virt2phy(session->cipher.key_ccp);
2311
2312         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2313
2314         memset(desc, 0, Q_DESC_SIZE);
2315
2316         /* prepare desc for des command */
2317         CCP_CMD_ENGINE(desc) = CCP_ENGINE_3DES;
2318
2319         CCP_CMD_SOC(desc) = 0;
2320         CCP_CMD_IOC(desc) = 0;
2321         CCP_CMD_INIT(desc) = 1;
2322         CCP_CMD_EOM(desc) = 1;
2323         CCP_CMD_PROT(desc) = 0;
2324
2325         function.raw = 0;
2326         CCP_DES_ENCRYPT(&function) = session->cipher.dir;
2327         CCP_DES_MODE(&function) = session->cipher.um.des_mode;
2328         CCP_DES_TYPE(&function) = session->cipher.ut.des_type;
2329         CCP_CMD_FUNCTION(desc) = function.raw;
2330
2331         CCP_CMD_LEN(desc) = op->sym->cipher.data.length;
2332
2333         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2334         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2335         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2336
2337         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2338         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2339         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2340
2341         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2342         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2343         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2344
2345         if (session->cipher.um.des_mode)
2346                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2347
2348         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2349
2350         rte_wmb();
2351
2352         /* Write the new tail address back to the queue register */
2353         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2354         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2355         /* Turn the queue back on using our cached control register */
2356         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2357                       cmd_q->qcontrol | CMD_Q_RUN);
2358
2359         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2360         return 0;
2361 }
2362
2363 static int
2364 ccp_perform_aes_gcm(struct rte_crypto_op *op, struct ccp_queue *cmd_q)
2365 {
2366         struct ccp_session *session;
2367         union ccp_function function;
2368         uint8_t *iv;
2369         struct ccp_passthru pst;
2370         struct ccp_desc *desc;
2371         uint32_t tail;
2372         uint64_t *temp;
2373         phys_addr_t src_addr, dest_addr, key_addr, aad_addr;
2374         phys_addr_t digest_dest_addr;
2375         int length, non_align_len;
2376
2377         session = (struct ccp_session *)get_session_private_data(
2378                                          op->sym->session,
2379                                          ccp_cryptodev_driver_id);
2380         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2381         key_addr = session->cipher.key_phys;
2382
2383         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2384                                               op->sym->aead.data.offset);
2385         if (unlikely(op->sym->m_dst != NULL))
2386                 dest_addr = rte_pktmbuf_mtophys_offset(op->sym->m_dst,
2387                                                 op->sym->aead.data.offset);
2388         else
2389                 dest_addr = src_addr;
2390         rte_pktmbuf_append(op->sym->m_src, session->auth.ctx_len);
2391         digest_dest_addr = op->sym->aead.digest.phys_addr;
2392         temp = (uint64_t *)(op->sym->aead.digest.data + AES_BLOCK_SIZE);
2393         *temp++ = rte_bswap64(session->auth.aad_length << 3);
2394         *temp = rte_bswap64(op->sym->aead.data.length << 3);
2395
2396         non_align_len = op->sym->aead.data.length % AES_BLOCK_SIZE;
2397         length = CCP_ALIGN(op->sym->aead.data.length, AES_BLOCK_SIZE);
2398
2399         aad_addr = op->sym->aead.aad.phys_addr;
2400
2401         /* CMD1 IV Passthru */
2402         rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE, iv,
2403                    session->iv.length);
2404         pst.src_addr = session->cipher.nonce_phys;
2405         pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2406         pst.len = CCP_SB_BYTES;
2407         pst.dir = 1;
2408         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2409         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2410         ccp_perform_passthru(&pst, cmd_q);
2411
2412         /* CMD2 GHASH-AAD */
2413         function.raw = 0;
2414         CCP_AES_ENCRYPT(&function) = CCP_AES_MODE_GHASH_AAD;
2415         CCP_AES_MODE(&function) = CCP_AES_MODE_GHASH;
2416         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2417
2418         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2419         memset(desc, 0, Q_DESC_SIZE);
2420
2421         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2422         CCP_CMD_INIT(desc) = 1;
2423         CCP_CMD_FUNCTION(desc) = function.raw;
2424
2425         CCP_CMD_LEN(desc) = session->auth.aad_length;
2426
2427         CCP_CMD_SRC_LO(desc) = ((uint32_t)aad_addr);
2428         CCP_CMD_SRC_HI(desc) = high32_value(aad_addr);
2429         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2430
2431         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2432         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2433         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2434
2435         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2436
2437         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2438         rte_wmb();
2439
2440         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2441         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2442         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2443                       cmd_q->qcontrol | CMD_Q_RUN);
2444
2445         /* CMD3 : GCTR Plain text */
2446         function.raw = 0;
2447         CCP_AES_ENCRYPT(&function) = session->cipher.dir;
2448         CCP_AES_MODE(&function) = CCP_AES_MODE_GCTR;
2449         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2450         if (non_align_len == 0)
2451                 CCP_AES_SIZE(&function) = (AES_BLOCK_SIZE << 3) - 1;
2452         else
2453                 CCP_AES_SIZE(&function) = (non_align_len << 3) - 1;
2454
2455
2456         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2457         memset(desc, 0, Q_DESC_SIZE);
2458
2459         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2460         CCP_CMD_EOM(desc) = 1;
2461         CCP_CMD_FUNCTION(desc) = function.raw;
2462
2463         CCP_CMD_LEN(desc) = length;
2464
2465         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2466         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2467         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2468
2469         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2470         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2471         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2472
2473         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2474         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2475         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2476
2477         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2478
2479         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2480         rte_wmb();
2481
2482         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2483         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2484         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2485                       cmd_q->qcontrol | CMD_Q_RUN);
2486
2487         /* CMD4 : PT to copy IV */
2488         pst.src_addr = session->cipher.nonce_phys;
2489         pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2490         pst.len = AES_BLOCK_SIZE;
2491         pst.dir = 1;
2492         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2493         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2494         ccp_perform_passthru(&pst, cmd_q);
2495
2496         /* CMD5 : GHASH-Final */
2497         function.raw = 0;
2498         CCP_AES_ENCRYPT(&function) = CCP_AES_MODE_GHASH_FINAL;
2499         CCP_AES_MODE(&function) = CCP_AES_MODE_GHASH;
2500         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2501
2502         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2503         memset(desc, 0, Q_DESC_SIZE);
2504
2505         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2506         CCP_CMD_FUNCTION(desc) = function.raw;
2507         /* Last block (AAD_len || PT_len)*/
2508         CCP_CMD_LEN(desc) = AES_BLOCK_SIZE;
2509
2510         CCP_CMD_SRC_LO(desc) = ((uint32_t)digest_dest_addr + AES_BLOCK_SIZE);
2511         CCP_CMD_SRC_HI(desc) = high32_value(digest_dest_addr + AES_BLOCK_SIZE);
2512         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2513
2514         CCP_CMD_DST_LO(desc) = ((uint32_t)digest_dest_addr);
2515         CCP_CMD_DST_HI(desc) = high32_value(digest_dest_addr);
2516         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2517
2518         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2519         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2520         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2521
2522         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2523
2524         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2525         rte_wmb();
2526
2527         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2528         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2529         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2530                       cmd_q->qcontrol | CMD_Q_RUN);
2531
2532         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2533         return 0;
2534 }
2535
2536 static inline int
2537 ccp_crypto_cipher(struct rte_crypto_op *op,
2538                   struct ccp_queue *cmd_q,
2539                   struct ccp_batch_info *b_info)
2540 {
2541         int result = 0;
2542         struct ccp_session *session;
2543
2544         session = (struct ccp_session *)get_session_private_data(
2545                                          op->sym->session,
2546                                          ccp_cryptodev_driver_id);
2547
2548         switch (session->cipher.algo) {
2549         case CCP_CIPHER_ALGO_AES_CBC:
2550                 result = ccp_perform_aes(op, cmd_q, b_info);
2551                 b_info->desccnt += 2;
2552                 break;
2553         case CCP_CIPHER_ALGO_AES_CTR:
2554                 result = ccp_perform_aes(op, cmd_q, b_info);
2555                 b_info->desccnt += 2;
2556                 break;
2557         case CCP_CIPHER_ALGO_AES_ECB:
2558                 result = ccp_perform_aes(op, cmd_q, b_info);
2559                 b_info->desccnt += 1;
2560                 break;
2561         case CCP_CIPHER_ALGO_3DES_CBC:
2562                 result = ccp_perform_3des(op, cmd_q, b_info);
2563                 b_info->desccnt += 2;
2564                 break;
2565         default:
2566                 CCP_LOG_ERR("Unsupported cipher algo %d",
2567                             session->cipher.algo);
2568                 return -ENOTSUP;
2569         }
2570         return result;
2571 }
2572
2573 static inline int
2574 ccp_crypto_auth(struct rte_crypto_op *op,
2575                 struct ccp_queue *cmd_q,
2576                 struct ccp_batch_info *b_info)
2577 {
2578
2579         int result = 0;
2580         struct ccp_session *session;
2581
2582         session = (struct ccp_session *)get_session_private_data(
2583                                          op->sym->session,
2584                                         ccp_cryptodev_driver_id);
2585
2586         switch (session->auth.algo) {
2587         case CCP_AUTH_ALGO_SHA1:
2588         case CCP_AUTH_ALGO_SHA224:
2589         case CCP_AUTH_ALGO_SHA256:
2590         case CCP_AUTH_ALGO_SHA384:
2591         case CCP_AUTH_ALGO_SHA512:
2592                 result = ccp_perform_sha(op, cmd_q);
2593                 b_info->desccnt += 3;
2594                 break;
2595 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2596         case CCP_AUTH_ALGO_MD5_HMAC:
2597                 break;
2598 #endif
2599         case CCP_AUTH_ALGO_SHA1_HMAC:
2600         case CCP_AUTH_ALGO_SHA224_HMAC:
2601         case CCP_AUTH_ALGO_SHA256_HMAC:
2602 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2603                 break;
2604 #endif
2605                 result = ccp_perform_hmac(op, cmd_q);
2606                 b_info->desccnt += 6;
2607                 break;
2608         case CCP_AUTH_ALGO_SHA384_HMAC:
2609         case CCP_AUTH_ALGO_SHA512_HMAC:
2610 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2611                 break;
2612 #endif
2613                 result = ccp_perform_hmac(op, cmd_q);
2614                 b_info->desccnt += 7;
2615                 break;
2616         case CCP_AUTH_ALGO_SHA3_224:
2617         case CCP_AUTH_ALGO_SHA3_256:
2618         case CCP_AUTH_ALGO_SHA3_384:
2619         case CCP_AUTH_ALGO_SHA3_512:
2620                 result = ccp_perform_sha3(op, cmd_q);
2621                 b_info->desccnt += 1;
2622                 break;
2623         case CCP_AUTH_ALGO_SHA3_224_HMAC:
2624         case CCP_AUTH_ALGO_SHA3_256_HMAC:
2625                 result = ccp_perform_sha3_hmac(op, cmd_q);
2626                 b_info->desccnt += 3;
2627                 break;
2628         case CCP_AUTH_ALGO_SHA3_384_HMAC:
2629         case CCP_AUTH_ALGO_SHA3_512_HMAC:
2630                 result = ccp_perform_sha3_hmac(op, cmd_q);
2631                 b_info->desccnt += 4;
2632                 break;
2633         case CCP_AUTH_ALGO_AES_CMAC:
2634                 result = ccp_perform_aes_cmac(op, cmd_q);
2635                 b_info->desccnt += 4;
2636                 break;
2637         default:
2638                 CCP_LOG_ERR("Unsupported auth algo %d",
2639                             session->auth.algo);
2640                 return -ENOTSUP;
2641         }
2642
2643         return result;
2644 }
2645
2646 static inline int
2647 ccp_crypto_aead(struct rte_crypto_op *op,
2648                 struct ccp_queue *cmd_q,
2649                 struct ccp_batch_info *b_info)
2650 {
2651         int result = 0;
2652         struct ccp_session *session;
2653
2654         session = (struct ccp_session *)get_session_private_data(
2655                                         op->sym->session,
2656                                         ccp_cryptodev_driver_id);
2657
2658         switch (session->auth.algo) {
2659         case CCP_AUTH_ALGO_AES_GCM:
2660                 if (session->cipher.algo != CCP_CIPHER_ALGO_AES_GCM) {
2661                         CCP_LOG_ERR("Incorrect chain order");
2662                         return -1;
2663                 }
2664                 result = ccp_perform_aes_gcm(op, cmd_q);
2665                 b_info->desccnt += 5;
2666                 break;
2667         default:
2668                 CCP_LOG_ERR("Unsupported aead algo %d",
2669                             session->aead_algo);
2670                 return -ENOTSUP;
2671         }
2672         return result;
2673 }
2674
2675 int
2676 process_ops_to_enqueue(struct ccp_qp *qp,
2677                        struct rte_crypto_op **op,
2678                        struct ccp_queue *cmd_q,
2679                        uint16_t nb_ops,
2680                        int slots_req)
2681 {
2682         int i, result = 0;
2683         struct ccp_batch_info *b_info;
2684         struct ccp_session *session;
2685 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2686         EVP_MD_CTX *auth_ctx = NULL;
2687 #endif
2688
2689         if (rte_mempool_get(qp->batch_mp, (void **)&b_info)) {
2690                 CCP_LOG_ERR("batch info allocation failed");
2691                 return 0;
2692         }
2693 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2694         auth_ctx = EVP_MD_CTX_create();
2695         if (unlikely(!auth_ctx)) {
2696                 CCP_LOG_ERR("Unable to create auth ctx");
2697                 return 0;
2698         }
2699         b_info->auth_ctr = 0;
2700 #endif
2701         /* populate batch info necessary for dequeue */
2702         b_info->op_idx = 0;
2703         b_info->lsb_buf_idx = 0;
2704         b_info->desccnt = 0;
2705         b_info->cmd_q = cmd_q;
2706         b_info->lsb_buf_phys =
2707                 (phys_addr_t)rte_mem_virt2phy((void *)b_info->lsb_buf);
2708         rte_atomic64_sub(&b_info->cmd_q->free_slots, slots_req);
2709
2710         b_info->head_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
2711                                          Q_DESC_SIZE);
2712         for (i = 0; i < nb_ops; i++) {
2713                 session = (struct ccp_session *)get_session_private_data(
2714                                                  op[i]->sym->session,
2715                                                  ccp_cryptodev_driver_id);
2716                 switch (session->cmd_id) {
2717                 case CCP_CMD_CIPHER:
2718                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2719                         break;
2720                 case CCP_CMD_AUTH:
2721                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
2722 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2723                         b_info->auth_ctr++;
2724                         result = cpu_crypto_auth(qp, op[i],
2725                                                  session, auth_ctx);
2726 #endif
2727                         break;
2728                 case CCP_CMD_CIPHER_HASH:
2729                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2730                         if (result)
2731                                 break;
2732                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
2733                         break;
2734                 case CCP_CMD_HASH_CIPHER:
2735                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
2736 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2737                         result = cpu_crypto_auth(qp, op[i],
2738                                                  session, auth_ctx);
2739                         if (op[i]->status != RTE_CRYPTO_OP_STATUS_SUCCESS)
2740                                 continue;
2741 #endif
2742                         if (result)
2743                                 break;
2744                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2745                         break;
2746                 case CCP_CMD_COMBINED:
2747                         result = ccp_crypto_aead(op[i], cmd_q, b_info);
2748                         break;
2749                 default:
2750                         CCP_LOG_ERR("Unsupported cmd_id");
2751                         result = -1;
2752                 }
2753                 if (unlikely(result < 0)) {
2754                         rte_atomic64_add(&b_info->cmd_q->free_slots,
2755                                          (slots_req - b_info->desccnt));
2756                         break;
2757                 }
2758                 b_info->op[i] = op[i];
2759         }
2760
2761         b_info->opcnt = i;
2762         b_info->tail_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
2763                                          Q_DESC_SIZE);
2764
2765         rte_wmb();
2766         /* Write the new tail address back to the queue register */
2767         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
2768                               b_info->tail_offset);
2769         /* Turn the queue back on using our cached control register */
2770         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2771                               cmd_q->qcontrol | CMD_Q_RUN);
2772
2773         rte_ring_enqueue(qp->processed_pkts, (void *)b_info);
2774
2775 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2776         EVP_MD_CTX_destroy(auth_ctx);
2777 #endif
2778         return i;
2779 }
2780
2781 static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op)
2782 {
2783         struct ccp_session *session;
2784         uint8_t *digest_data, *addr;
2785         struct rte_mbuf *m_last;
2786         int offset, digest_offset;
2787         uint8_t digest_le[64];
2788
2789         session = (struct ccp_session *)get_session_private_data(
2790                                          op->sym->session,
2791                                         ccp_cryptodev_driver_id);
2792
2793         if (session->cmd_id == CCP_CMD_COMBINED) {
2794                 digest_data = op->sym->aead.digest.data;
2795                 digest_offset = op->sym->aead.data.offset +
2796                                         op->sym->aead.data.length;
2797         } else {
2798                 digest_data = op->sym->auth.digest.data;
2799                 digest_offset = op->sym->auth.data.offset +
2800                                         op->sym->auth.data.length;
2801         }
2802         m_last = rte_pktmbuf_lastseg(op->sym->m_src);
2803         addr = (uint8_t *)((char *)m_last->buf_addr + m_last->data_off +
2804                            m_last->data_len - session->auth.ctx_len);
2805
2806         rte_mb();
2807         offset = session->auth.offset;
2808
2809         if (session->auth.engine == CCP_ENGINE_SHA)
2810                 if ((session->auth.ut.sha_type != CCP_SHA_TYPE_1) &&
2811                     (session->auth.ut.sha_type != CCP_SHA_TYPE_224) &&
2812                     (session->auth.ut.sha_type != CCP_SHA_TYPE_256)) {
2813                         /* All other algorithms require byte
2814                          * swap done by host
2815                          */
2816                         unsigned int i;
2817
2818                         offset = session->auth.ctx_len -
2819                                 session->auth.offset - 1;
2820                         for (i = 0; i < session->auth.digest_length; i++)
2821                                 digest_le[i] = addr[offset - i];
2822                         offset = 0;
2823                         addr = digest_le;
2824                 }
2825
2826         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2827         if (session->auth.op == CCP_AUTH_OP_VERIFY) {
2828                 if (memcmp(addr + offset, digest_data,
2829                            session->auth.digest_length) != 0)
2830                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
2831
2832         } else {
2833                 if (unlikely(digest_data == 0))
2834                         digest_data = rte_pktmbuf_mtod_offset(
2835                                         op->sym->m_dst, uint8_t *,
2836                                         digest_offset);
2837                 rte_memcpy(digest_data, addr + offset,
2838                            session->auth.digest_length);
2839         }
2840         /* Trim area used for digest from mbuf. */
2841         rte_pktmbuf_trim(op->sym->m_src,
2842                          session->auth.ctx_len);
2843 }
2844
2845 static int
2846 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2847 ccp_prepare_ops(struct ccp_qp *qp,
2848 #else
2849 ccp_prepare_ops(struct ccp_qp *qp __rte_unused,
2850 #endif
2851                 struct rte_crypto_op **op_d,
2852                 struct ccp_batch_info *b_info,
2853                 uint16_t nb_ops)
2854 {
2855         int i, min_ops;
2856         struct ccp_session *session;
2857
2858 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2859         EVP_MD_CTX *auth_ctx = NULL;
2860
2861         auth_ctx = EVP_MD_CTX_create();
2862         if (unlikely(!auth_ctx)) {
2863                 CCP_LOG_ERR("Unable to create auth ctx");
2864                 return 0;
2865         }
2866 #endif
2867         min_ops = RTE_MIN(nb_ops, b_info->opcnt);
2868
2869         for (i = 0; i < min_ops; i++) {
2870                 op_d[i] = b_info->op[b_info->op_idx++];
2871                 session = (struct ccp_session *)get_session_private_data(
2872                                                  op_d[i]->sym->session,
2873                                                 ccp_cryptodev_driver_id);
2874                 switch (session->cmd_id) {
2875                 case CCP_CMD_CIPHER:
2876                         op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2877                         break;
2878                 case CCP_CMD_AUTH:
2879 #ifndef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2880                         ccp_auth_dq_prepare(op_d[i]);
2881 #endif
2882                         break;
2883                 case CCP_CMD_CIPHER_HASH:
2884 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2885                         cpu_crypto_auth(qp, op_d[i],
2886                                         session, auth_ctx);
2887 #else
2888                         ccp_auth_dq_prepare(op_d[i]);
2889 #endif
2890                         break;
2891                 case CCP_CMD_HASH_CIPHER:
2892 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2893                         op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2894 #else
2895                         ccp_auth_dq_prepare(op_d[i]);
2896 #endif
2897                         break;
2898                 case CCP_CMD_COMBINED:
2899                         ccp_auth_dq_prepare(op_d[i]);
2900                         break;
2901                 default:
2902                         CCP_LOG_ERR("Unsupported cmd_id");
2903                 }
2904         }
2905
2906 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2907         EVP_MD_CTX_destroy(auth_ctx);
2908 #endif
2909         b_info->opcnt -= min_ops;
2910         return min_ops;
2911 }
2912
2913 int
2914 process_ops_to_dequeue(struct ccp_qp *qp,
2915                        struct rte_crypto_op **op,
2916                        uint16_t nb_ops)
2917 {
2918         struct ccp_batch_info *b_info;
2919         uint32_t cur_head_offset;
2920
2921         if (qp->b_info != NULL) {
2922                 b_info = qp->b_info;
2923                 if (unlikely(b_info->op_idx > 0))
2924                         goto success;
2925         } else if (rte_ring_dequeue(qp->processed_pkts,
2926                                     (void **)&b_info))
2927                 return 0;
2928 #ifdef RTE_LIBRTE_PMD_CCP_CPU_AUTH
2929         if (b_info->auth_ctr == b_info->opcnt)
2930                 goto success;
2931 #endif
2932         cur_head_offset = CCP_READ_REG(b_info->cmd_q->reg_base,
2933                                        CMD_Q_HEAD_LO_BASE);
2934
2935         if (b_info->head_offset < b_info->tail_offset) {
2936                 if ((cur_head_offset >= b_info->head_offset) &&
2937                     (cur_head_offset < b_info->tail_offset)) {
2938                         qp->b_info = b_info;
2939                         return 0;
2940                 }
2941         } else {
2942                 if ((cur_head_offset >= b_info->head_offset) ||
2943                     (cur_head_offset < b_info->tail_offset)) {
2944                         qp->b_info = b_info;
2945                         return 0;
2946                 }
2947         }
2948
2949
2950 success:
2951         nb_ops = ccp_prepare_ops(qp, op, b_info, nb_ops);
2952         rte_atomic64_add(&b_info->cmd_q->free_slots, b_info->desccnt);
2953         b_info->desccnt = 0;
2954         if (b_info->opcnt > 0) {
2955                 qp->b_info = b_info;
2956         } else {
2957                 rte_mempool_put(qp->batch_mp, (void *)b_info);
2958                 qp->b_info = NULL;
2959         }
2960
2961         return nb_ops;
2962 }