drivers/crypto/ccp/ccp_crypto.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <dirent.h>
   6 #include <fcntl.h>
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <sys/mman.h>
  10 #include <sys/queue.h>
  11 #include <sys/types.h>
  12 #include <unistd.h>
  13 #include <openssl/sha.h>
  14 #include <openssl/cmac.h> /*sub key apis*/
  15 #include <openssl/evp.h> /*sub key apis*/
  16
  17 #include <rte_hexdump.h>
  18 #include <rte_memzone.h>
  19 #include <rte_malloc.h>
  20 #include <rte_memory.h>
  21 #include <rte_spinlock.h>
  22 #include <rte_string_fns.h>
  23 #include <rte_cryptodev_pmd.h>
  24
  25 #include "ccp_dev.h"
  26 #include "ccp_crypto.h"
  27 #include "ccp_pci.h"
  28 #include "ccp_pmd_private.h"
  29
  30 #include <openssl/conf.h>
  31 #include <openssl/err.h>
  32 #include <openssl/hmac.h>
  33
  34 /* SHA initial context values */
  35 static uint32_t ccp_sha1_init[SHA_COMMON_DIGEST_SIZE / sizeof(uint32_t)] = {
  36         SHA1_H4, SHA1_H3,
  37         SHA1_H2, SHA1_H1,
  38         SHA1_H0, 0x0U,
  39         0x0U, 0x0U,
  40 };
  41
  42 uint32_t ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(uint32_t)] = {
  43         SHA224_H7, SHA224_H6,
  44         SHA224_H5, SHA224_H4,
  45         SHA224_H3, SHA224_H2,
  46         SHA224_H1, SHA224_H0,
  47 };
  48
  49 uint32_t ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(uint32_t)] = {
  50         SHA256_H7, SHA256_H6,
  51         SHA256_H5, SHA256_H4,
  52         SHA256_H3, SHA256_H2,
  53         SHA256_H1, SHA256_H0,
  54 };
  55
  56 uint64_t ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(uint64_t)] = {
  57         SHA384_H7, SHA384_H6,
  58         SHA384_H5, SHA384_H4,
  59         SHA384_H3, SHA384_H2,
  60         SHA384_H1, SHA384_H0,
  61 };
  62
  63 uint64_t ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(uint64_t)] = {
  64         SHA512_H7, SHA512_H6,
  65         SHA512_H5, SHA512_H4,
  66         SHA512_H3, SHA512_H2,
  67         SHA512_H1, SHA512_H0,
  68 };
  69
  70 #if defined(_MSC_VER)
  71 #define SHA3_CONST(x) x
  72 #else
  73 #define SHA3_CONST(x) x##L
  74 #endif
  75
  76 /** 'Words' here refers to uint64_t */
  77 #define SHA3_KECCAK_SPONGE_WORDS \
  78         (((1600) / 8) / sizeof(uint64_t))
  79 typedef struct sha3_context_ {
  80         uint64_t saved;
  81         /**
  82          * The portion of the input message that we
  83          * didn't consume yet
  84          */
  85         union {
  86                 uint64_t s[SHA3_KECCAK_SPONGE_WORDS];
  87                 /* Keccak's state */
  88                 uint8_t sb[SHA3_KECCAK_SPONGE_WORDS * 8];
  89                 /**total 200 ctx size**/
  90         };
  91         unsigned int byteIndex;
  92         /**
  93          * 0..7--the next byte after the set one
  94          * (starts from 0; 0--none are buffered)
  95          */
  96         unsigned int wordIndex;
  97         /**
  98          * 0..24--the next word to integrate input
  99          * (starts from 0)
 100          */
 101         unsigned int capacityWords;
 102         /**
 103          * the double size of the hash output in
 104          * words (e.g. 16 for Keccak 512)
 105          */
 106 } sha3_context;
 107
 108 #ifndef SHA3_ROTL64
 109 #define SHA3_ROTL64(x, y) \
 110         (((x) << (y)) | ((x) >> ((sizeof(uint64_t)*8) - (y))))
 111 #endif
 112
 113 static const uint64_t keccakf_rndc[24] = {
 114         SHA3_CONST(0x0000000000000001UL), SHA3_CONST(0x0000000000008082UL),
 115         SHA3_CONST(0x800000000000808aUL), SHA3_CONST(0x8000000080008000UL),
 116         SHA3_CONST(0x000000000000808bUL), SHA3_CONST(0x0000000080000001UL),
 117         SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008009UL),
 118         SHA3_CONST(0x000000000000008aUL), SHA3_CONST(0x0000000000000088UL),
 119         SHA3_CONST(0x0000000080008009UL), SHA3_CONST(0x000000008000000aUL),
 120         SHA3_CONST(0x000000008000808bUL), SHA3_CONST(0x800000000000008bUL),
 121         SHA3_CONST(0x8000000000008089UL), SHA3_CONST(0x8000000000008003UL),
 122         SHA3_CONST(0x8000000000008002UL), SHA3_CONST(0x8000000000000080UL),
 123         SHA3_CONST(0x000000000000800aUL), SHA3_CONST(0x800000008000000aUL),
 124         SHA3_CONST(0x8000000080008081UL), SHA3_CONST(0x8000000000008080UL),
 125         SHA3_CONST(0x0000000080000001UL), SHA3_CONST(0x8000000080008008UL)
 126 };
 127
 128 static const unsigned int keccakf_rotc[24] = {
 129         1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, 27, 41, 56, 8, 25, 43, 62,
 130         18, 39, 61, 20, 44
 131 };
 132
 133 static const unsigned int keccakf_piln[24] = {
 134         10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, 15, 23, 19, 13, 12, 2, 20,
 135         14, 22, 9, 6, 1
 136 };
 137
 138 static enum ccp_cmd_order
 139 ccp_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 140 {
 141         enum ccp_cmd_order res = CCP_CMD_NOT_SUPPORTED;
 142
 143         if (xform == NULL)
 144                 return res;
 145         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 146                 if (xform->next == NULL)
 147                         return CCP_CMD_AUTH;
 148                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 149                         return CCP_CMD_HASH_CIPHER;
 150         }
 151         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 152                 if (xform->next == NULL)
 153                         return CCP_CMD_CIPHER;
 154                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 155                         return CCP_CMD_CIPHER_HASH;
 156         }
 157         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 158                 return CCP_CMD_COMBINED;
 159         return res;
 160 }
 161
 162 /* partial hash using openssl */
 163 static int partial_hash_sha1(uint8_t *data_in, uint8_t *data_out)
 164 {
 165         SHA_CTX ctx;
 166
 167         if (!SHA1_Init(&ctx))
 168                 return -EFAULT;
 169         SHA1_Transform(&ctx, data_in);
 170         rte_memcpy(data_out, &ctx, SHA_DIGEST_LENGTH);
 171         return 0;
 172 }
 173
 174 static int partial_hash_sha224(uint8_t *data_in, uint8_t *data_out)
 175 {
 176         SHA256_CTX ctx;
 177
 178         if (!SHA224_Init(&ctx))
 179                 return -EFAULT;
 180         SHA256_Transform(&ctx, data_in);
 181         rte_memcpy(data_out, &ctx,
 182                    SHA256_DIGEST_LENGTH);
 183         return 0;
 184 }
 185
 186 static int partial_hash_sha256(uint8_t *data_in, uint8_t *data_out)
 187 {
 188         SHA256_CTX ctx;
 189
 190         if (!SHA256_Init(&ctx))
 191                 return -EFAULT;
 192         SHA256_Transform(&ctx, data_in);
 193         rte_memcpy(data_out, &ctx,
 194                    SHA256_DIGEST_LENGTH);
 195         return 0;
 196 }
 197
 198 static int partial_hash_sha384(uint8_t *data_in, uint8_t *data_out)
 199 {
 200         SHA512_CTX ctx;
 201
 202         if (!SHA384_Init(&ctx))
 203                 return -EFAULT;
 204         SHA512_Transform(&ctx, data_in);
 205         rte_memcpy(data_out, &ctx,
 206                    SHA512_DIGEST_LENGTH);
 207         return 0;
 208 }
 209
 210 static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out)
 211 {
 212         SHA512_CTX ctx;
 213
 214         if (!SHA512_Init(&ctx))
 215                 return -EFAULT;
 216         SHA512_Transform(&ctx, data_in);
 217         rte_memcpy(data_out, &ctx,
 218                    SHA512_DIGEST_LENGTH);
 219         return 0;
 220 }
 221
 222 static void
 223 keccakf(uint64_t s[25])
 224 {
 225         int i, j, round;
 226         uint64_t t, bc[5];
 227 #define KECCAK_ROUNDS 24
 228
 229         for (round = 0; round < KECCAK_ROUNDS; round++) {
 230
 231                 /* Theta */
 232                 for (i = 0; i < 5; i++)
 233                         bc[i] = s[i] ^ s[i + 5] ^ s[i + 10] ^ s[i + 15] ^
 234                                 s[i + 20];
 235
 236                 for (i = 0; i < 5; i++) {
 237                         t = bc[(i + 4) % 5] ^ SHA3_ROTL64(bc[(i + 1) % 5], 1);
 238                         for (j = 0; j < 25; j += 5)
 239                                 s[j + i] ^= t;
 240                 }
 241
 242                 /* Rho Pi */
 243                 t = s[1];
 244                 for (i = 0; i < 24; i++) {
 245                         j = keccakf_piln[i];
 246                         bc[0] = s[j];
 247                         s[j] = SHA3_ROTL64(t, keccakf_rotc[i]);
 248                         t = bc[0];
 249                 }
 250
 251                 /* Chi */
 252                 for (j = 0; j < 25; j += 5) {
 253                         for (i = 0; i < 5; i++)
 254                                 bc[i] = s[j + i];
 255                         for (i = 0; i < 5; i++)
 256                                 s[j + i] ^= (~bc[(i + 1) % 5]) &
 257                                             bc[(i + 2) % 5];
 258                 }
 259
 260                 /* Iota */
 261                 s[0] ^= keccakf_rndc[round];
 262         }
 263 }
 264
 265 static void
 266 sha3_Init224(void *priv)
 267 {
 268         sha3_context *ctx = (sha3_context *) priv;
 269
 270         memset(ctx, 0, sizeof(*ctx));
 271         ctx->capacityWords = 2 * 224 / (8 * sizeof(uint64_t));
 272 }
 273
 274 static void
 275 sha3_Init256(void *priv)
 276 {
 277         sha3_context *ctx = (sha3_context *) priv;
 278
 279         memset(ctx, 0, sizeof(*ctx));
 280         ctx->capacityWords = 2 * 256 / (8 * sizeof(uint64_t));
 281 }
 282
 283 static void
 284 sha3_Init384(void *priv)
 285 {
 286         sha3_context *ctx = (sha3_context *) priv;
 287
 288         memset(ctx, 0, sizeof(*ctx));
 289         ctx->capacityWords = 2 * 384 / (8 * sizeof(uint64_t));
 290 }
 291
 292 static void
 293 sha3_Init512(void *priv)
 294 {
 295         sha3_context *ctx = (sha3_context *) priv;
 296
 297         memset(ctx, 0, sizeof(*ctx));
 298         ctx->capacityWords = 2 * 512 / (8 * sizeof(uint64_t));
 299 }
 300
 301
 302 /* This is simply the 'update' with the padding block.
 303  * The padding block is 0x01 || 0x00* || 0x80. First 0x01 and last 0x80
 304  * bytes are always present, but they can be the same byte.
 305  */
 306 static void
 307 sha3_Update(void *priv, void const *bufIn, size_t len)
 308 {
 309         sha3_context *ctx = (sha3_context *) priv;
 310         unsigned int old_tail = (8 - ctx->byteIndex) & 7;
 311         size_t words;
 312         unsigned int tail;
 313         size_t i;
 314         const uint8_t *buf = bufIn;
 315
 316         if (len < old_tail) {
 317                 while (len--)
 318                         ctx->saved |= (uint64_t) (*(buf++)) <<
 319                                       ((ctx->byteIndex++) * 8);
 320                 return;
 321         }
 322
 323         if (old_tail) {
 324                 len -= old_tail;
 325                 while (old_tail--)
 326                         ctx->saved |= (uint64_t) (*(buf++)) <<
 327                                       ((ctx->byteIndex++) * 8);
 328
 329                 ctx->s[ctx->wordIndex] ^= ctx->saved;
 330                 ctx->byteIndex = 0;
 331                 ctx->saved = 0;
 332                 if (++ctx->wordIndex ==
 333                    (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
 334                         keccakf(ctx->s);
 335                         ctx->wordIndex = 0;
 336                 }
 337         }
 338
 339         words = len / sizeof(uint64_t);
 340         tail = len - words * sizeof(uint64_t);
 341
 342         for (i = 0; i < words; i++, buf += sizeof(uint64_t)) {
 343                 const uint64_t t = (uint64_t) (buf[0]) |
 344                         ((uint64_t) (buf[1]) << 8 * 1) |
 345                         ((uint64_t) (buf[2]) << 8 * 2) |
 346                         ((uint64_t) (buf[3]) << 8 * 3) |
 347                         ((uint64_t) (buf[4]) << 8 * 4) |
 348                         ((uint64_t) (buf[5]) << 8 * 5) |
 349                         ((uint64_t) (buf[6]) << 8 * 6) |
 350                         ((uint64_t) (buf[7]) << 8 * 7);
 351                 ctx->s[ctx->wordIndex] ^= t;
 352                 if (++ctx->wordIndex ==
 353                    (SHA3_KECCAK_SPONGE_WORDS - ctx->capacityWords)) {
 354                         keccakf(ctx->s);
 355                         ctx->wordIndex = 0;
 356                 }
 357         }
 358
 359         while (tail--)
 360                 ctx->saved |= (uint64_t) (*(buf++)) << ((ctx->byteIndex++) * 8);
 361 }
 362
 363 int partial_hash_sha3_224(uint8_t *data_in, uint8_t *data_out)
 364 {
 365         sha3_context *ctx;
 366         int i;
 367
 368         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 369         if (!ctx) {
 370                 CCP_LOG_ERR("sha3-ctx creation failed");
 371                 return -ENOMEM;
 372         }
 373         sha3_Init224(ctx);
 374         sha3_Update(ctx, data_in, SHA3_224_BLOCK_SIZE);
 375         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 376                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 377         rte_free(ctx);
 378
 379         return 0;
 380 }
 381
 382 int partial_hash_sha3_256(uint8_t *data_in, uint8_t *data_out)
 383 {
 384         sha3_context *ctx;
 385         int i;
 386
 387         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 388         if (!ctx) {
 389                 CCP_LOG_ERR("sha3-ctx creation failed");
 390                 return -ENOMEM;
 391         }
 392         sha3_Init256(ctx);
 393         sha3_Update(ctx, data_in, SHA3_256_BLOCK_SIZE);
 394         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 395                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 396         rte_free(ctx);
 397
 398         return 0;
 399 }
 400
 401 int partial_hash_sha3_384(uint8_t *data_in, uint8_t *data_out)
 402 {
 403         sha3_context *ctx;
 404         int i;
 405
 406         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 407         if (!ctx) {
 408                 CCP_LOG_ERR("sha3-ctx creation failed");
 409                 return -ENOMEM;
 410         }
 411         sha3_Init384(ctx);
 412         sha3_Update(ctx, data_in, SHA3_384_BLOCK_SIZE);
 413         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 414                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 415         rte_free(ctx);
 416
 417         return 0;
 418 }
 419
 420 int partial_hash_sha3_512(uint8_t *data_in, uint8_t *data_out)
 421 {
 422         sha3_context *ctx;
 423         int i;
 424
 425         ctx = rte_zmalloc("sha3-ctx", sizeof(sha3_context), 0);
 426         if (!ctx) {
 427                 CCP_LOG_ERR("sha3-ctx creation failed");
 428                 return -ENOMEM;
 429         }
 430         sha3_Init512(ctx);
 431         sha3_Update(ctx, data_in, SHA3_512_BLOCK_SIZE);
 432         for (i = 0; i < CCP_SHA3_CTX_SIZE; i++, data_out++)
 433                 *data_out = ctx->sb[CCP_SHA3_CTX_SIZE - i - 1];
 434         rte_free(ctx);
 435
 436         return 0;
 437 }
 438
 439 static int generate_partial_hash(struct ccp_session *sess)
 440 {
 441
 442         uint8_t ipad[sess->auth.block_size];
 443         uint8_t opad[sess->auth.block_size];
 444         uint8_t *ipad_t, *opad_t;
 445         uint32_t *hash_value_be32, hash_temp32[8];
 446         uint64_t *hash_value_be64, hash_temp64[8];
 447         int i, count;
 448         uint8_t *hash_value_sha3;
 449
 450         opad_t = ipad_t = (uint8_t *)sess->auth.key;
 451
 452         hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute);
 453         hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute);
 454
 455         /* considering key size is always equal to block size of algorithm */
 456         for (i = 0; i < sess->auth.block_size; i++) {
 457                 ipad[i] = (ipad_t[i] ^ HMAC_IPAD_VALUE);
 458                 opad[i] = (opad_t[i] ^ HMAC_OPAD_VALUE);
 459         }
 460
 461         switch (sess->auth.algo) {
 462         case CCP_AUTH_ALGO_SHA1_HMAC:
 463                 count = SHA1_DIGEST_SIZE >> 2;
 464
 465                 if (partial_hash_sha1(ipad, (uint8_t *)hash_temp32))
 466                         return -1;
 467                 for (i = 0; i < count; i++, hash_value_be32++)
 468                         *hash_value_be32 = hash_temp32[count - 1 - i];
 469
 470                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 471                                                + sess->auth.ctx_len);
 472                 if (partial_hash_sha1(opad, (uint8_t *)hash_temp32))
 473                         return -1;
 474                 for (i = 0; i < count; i++, hash_value_be32++)
 475                         *hash_value_be32 = hash_temp32[count - 1 - i];
 476                 return 0;
 477         case CCP_AUTH_ALGO_SHA224_HMAC:
 478                 count = SHA256_DIGEST_SIZE >> 2;
 479
 480                 if (partial_hash_sha224(ipad, (uint8_t *)hash_temp32))
 481                         return -1;
 482                 for (i = 0; i < count; i++, hash_value_be32++)
 483                         *hash_value_be32 = hash_temp32[count - 1 - i];
 484
 485                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 486                                                + sess->auth.ctx_len);
 487                 if (partial_hash_sha224(opad, (uint8_t *)hash_temp32))
 488                         return -1;
 489                 for (i = 0; i < count; i++, hash_value_be32++)
 490                         *hash_value_be32 = hash_temp32[count - 1 - i];
 491                 return 0;
 492         case CCP_AUTH_ALGO_SHA3_224_HMAC:
 493                 hash_value_sha3 = sess->auth.pre_compute;
 494                 if (partial_hash_sha3_224(ipad, hash_value_sha3))
 495                         return -1;
 496
 497                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 498                                                + sess->auth.ctx_len);
 499                 if (partial_hash_sha3_224(opad, hash_value_sha3))
 500                         return -1;
 501                 return 0;
 502         case CCP_AUTH_ALGO_SHA256_HMAC:
 503                 count = SHA256_DIGEST_SIZE >> 2;
 504
 505                 if (partial_hash_sha256(ipad, (uint8_t *)hash_temp32))
 506                         return -1;
 507                 for (i = 0; i < count; i++, hash_value_be32++)
 508                         *hash_value_be32 = hash_temp32[count - 1 - i];
 509
 510                 hash_value_be32 = (uint32_t *)((uint8_t *)sess->auth.pre_compute
 511                                                + sess->auth.ctx_len);
 512                 if (partial_hash_sha256(opad, (uint8_t *)hash_temp32))
 513                         return -1;
 514                 for (i = 0; i < count; i++, hash_value_be32++)
 515                         *hash_value_be32 = hash_temp32[count - 1 - i];
 516                 return 0;
 517         case CCP_AUTH_ALGO_SHA3_256_HMAC:
 518                 hash_value_sha3 = sess->auth.pre_compute;
 519                 if (partial_hash_sha3_256(ipad, hash_value_sha3))
 520                         return -1;
 521
 522                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 523                                               + sess->auth.ctx_len);
 524                 if (partial_hash_sha3_256(opad, hash_value_sha3))
 525                         return -1;
 526                 return 0;
 527         case CCP_AUTH_ALGO_SHA384_HMAC:
 528                 count = SHA512_DIGEST_SIZE >> 3;
 529
 530                 if (partial_hash_sha384(ipad, (uint8_t *)hash_temp64))
 531                         return -1;
 532                 for (i = 0; i < count; i++, hash_value_be64++)
 533                         *hash_value_be64 = hash_temp64[count - 1 - i];
 534
 535                 hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute
 536                                                + sess->auth.ctx_len);
 537                 if (partial_hash_sha384(opad, (uint8_t *)hash_temp64))
 538                         return -1;
 539                 for (i = 0; i < count; i++, hash_value_be64++)
 540                         *hash_value_be64 = hash_temp64[count - 1 - i];
 541                 return 0;
 542         case CCP_AUTH_ALGO_SHA3_384_HMAC:
 543                 hash_value_sha3 = sess->auth.pre_compute;
 544                 if (partial_hash_sha3_384(ipad, hash_value_sha3))
 545                         return -1;
 546
 547                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 548                                               + sess->auth.ctx_len);
 549                 if (partial_hash_sha3_384(opad, hash_value_sha3))
 550                         return -1;
 551                 return 0;
 552         case CCP_AUTH_ALGO_SHA512_HMAC:
 553                 count = SHA512_DIGEST_SIZE >> 3;
 554
 555                 if (partial_hash_sha512(ipad, (uint8_t *)hash_temp64))
 556                         return -1;
 557                 for (i = 0; i < count; i++, hash_value_be64++)
 558                         *hash_value_be64 = hash_temp64[count - 1 - i];
 559
 560                 hash_value_be64 = (uint64_t *)((uint8_t *)sess->auth.pre_compute
 561                                                + sess->auth.ctx_len);
 562                 if (partial_hash_sha512(opad, (uint8_t *)hash_temp64))
 563                         return -1;
 564                 for (i = 0; i < count; i++, hash_value_be64++)
 565                         *hash_value_be64 = hash_temp64[count - 1 - i];
 566                 return 0;
 567         case CCP_AUTH_ALGO_SHA3_512_HMAC:
 568                 hash_value_sha3 = sess->auth.pre_compute;
 569                 if (partial_hash_sha3_512(ipad, hash_value_sha3))
 570                         return -1;
 571
 572                 hash_value_sha3 = (uint8_t *)(sess->auth.pre_compute
 573                                               + sess->auth.ctx_len);
 574                 if (partial_hash_sha3_512(opad, hash_value_sha3))
 575                         return -1;
 576                 return 0;
 577         default:
 578                 CCP_LOG_ERR("Invalid auth algo");
 579                 return -1;
 580         }
 581 }
 582
 583 /* prepare temporary keys K1 and K2 */
 584 static void prepare_key(unsigned char *k, unsigned char *l, int bl)
 585 {
 586         int i;
 587         /* Shift block to left, including carry */
 588         for (i = 0; i < bl; i++) {
 589                 k[i] = l[i] << 1;
 590                 if (i < bl - 1 && l[i + 1] & 0x80)
 591                         k[i] |= 1;
 592         }
 593         /* If MSB set fixup with R */
 594         if (l[0] & 0x80)
 595                 k[bl - 1] ^= bl == 16 ? 0x87 : 0x1b;
 596 }
 597
 598 /* subkeys K1 and K2 generation for CMAC */
 599 static int
 600 generate_cmac_subkeys(struct ccp_session *sess)
 601 {
 602         const EVP_CIPHER *algo;
 603         EVP_CIPHER_CTX *ctx;
 604         unsigned char *ccp_ctx;
 605         size_t i;
 606         int dstlen, totlen;
 607         unsigned char zero_iv[AES_BLOCK_SIZE] = {0};
 608         unsigned char dst[2 * AES_BLOCK_SIZE] = {0};
 609         unsigned char k1[AES_BLOCK_SIZE] = {0};
 610         unsigned char k2[AES_BLOCK_SIZE] = {0};
 611
 612         if (sess->auth.ut.aes_type == CCP_AES_TYPE_128)
 613                 algo =  EVP_aes_128_cbc();
 614         else if (sess->auth.ut.aes_type == CCP_AES_TYPE_192)
 615                 algo =  EVP_aes_192_cbc();
 616         else if (sess->auth.ut.aes_type == CCP_AES_TYPE_256)
 617                 algo =  EVP_aes_256_cbc();
 618         else {
 619                 CCP_LOG_ERR("Invalid CMAC type length");
 620                 return -1;
 621         }
 622
 623         ctx = EVP_CIPHER_CTX_new();
 624         if (!ctx) {
 625                 CCP_LOG_ERR("ctx creation failed");
 626                 return -1;
 627         }
 628         if (EVP_EncryptInit(ctx, algo, (unsigned char *)sess->auth.key,
 629                             (unsigned char *)zero_iv) <= 0)
 630                 goto key_generate_err;
 631         if (EVP_CIPHER_CTX_set_padding(ctx, 0) <= 0)
 632                 goto key_generate_err;
 633         if (EVP_EncryptUpdate(ctx, dst, &dstlen, zero_iv,
 634                               AES_BLOCK_SIZE) <= 0)
 635                 goto key_generate_err;
 636         if (EVP_EncryptFinal_ex(ctx, dst + dstlen, &totlen) <= 0)
 637                 goto key_generate_err;
 638
 639         memset(sess->auth.pre_compute, 0, CCP_SB_BYTES * 2);
 640
 641         ccp_ctx = (unsigned char *)(sess->auth.pre_compute + CCP_SB_BYTES - 1);
 642         prepare_key(k1, dst, AES_BLOCK_SIZE);
 643         for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
 644                 *ccp_ctx = k1[i];
 645
 646         ccp_ctx = (unsigned char *)(sess->auth.pre_compute +
 647                                    (2 * CCP_SB_BYTES) - 1);
 648         prepare_key(k2, k1, AES_BLOCK_SIZE);
 649         for (i = 0; i < AES_BLOCK_SIZE;  i++, ccp_ctx--)
 650                 *ccp_ctx = k2[i];
 651
 652         EVP_CIPHER_CTX_free(ctx);
 653
 654         return 0;
 655
 656 key_generate_err:
 657         CCP_LOG_ERR("CMAC Init failed");
 658                 return -1;
 659 }
 660
 661 /* configure session */
 662 static int
 663 ccp_configure_session_cipher(struct ccp_session *sess,
 664                              const struct rte_crypto_sym_xform *xform)
 665 {
 666         const struct rte_crypto_cipher_xform *cipher_xform = NULL;
 667         size_t i, j, x;
 668
 669         cipher_xform = &xform->cipher;
 670
 671         /* set cipher direction */
 672         if (cipher_xform->op ==  RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 673                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
 674         else
 675                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
 676
 677         /* set cipher key */
 678         sess->cipher.key_length = cipher_xform->key.length;
 679         rte_memcpy(sess->cipher.key, cipher_xform->key.data,
 680                    cipher_xform->key.length);
 681
 682         /* set iv parameters */
 683         sess->iv.offset = cipher_xform->iv.offset;
 684         sess->iv.length = cipher_xform->iv.length;
 685
 686         switch (cipher_xform->algo) {
 687         case RTE_CRYPTO_CIPHER_AES_CTR:
 688                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CTR;
 689                 sess->cipher.um.aes_mode = CCP_AES_MODE_CTR;
 690                 sess->cipher.engine = CCP_ENGINE_AES;
 691                 break;
 692         case RTE_CRYPTO_CIPHER_AES_ECB:
 693                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
 694                 sess->cipher.um.aes_mode = CCP_AES_MODE_ECB;
 695                 sess->cipher.engine = CCP_ENGINE_AES;
 696                 break;
 697         case RTE_CRYPTO_CIPHER_AES_CBC:
 698                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
 699                 sess->cipher.um.aes_mode = CCP_AES_MODE_CBC;
 700                 sess->cipher.engine = CCP_ENGINE_AES;
 701                 break;
 702         case RTE_CRYPTO_CIPHER_3DES_CBC:
 703                 sess->cipher.algo = CCP_CIPHER_ALGO_3DES_CBC;
 704                 sess->cipher.um.des_mode = CCP_DES_MODE_CBC;
 705                 sess->cipher.engine = CCP_ENGINE_3DES;
 706                 break;
 707         default:
 708                 CCP_LOG_ERR("Unsupported cipher algo");
 709                 return -1;
 710         }
 711
 712
 713         switch (sess->cipher.engine) {
 714         case CCP_ENGINE_AES:
 715                 if (sess->cipher.key_length == 16)
 716                         sess->cipher.ut.aes_type = CCP_AES_TYPE_128;
 717                 else if (sess->cipher.key_length == 24)
 718                         sess->cipher.ut.aes_type = CCP_AES_TYPE_192;
 719                 else if (sess->cipher.key_length == 32)
 720                         sess->cipher.ut.aes_type = CCP_AES_TYPE_256;
 721                 else {
 722                         CCP_LOG_ERR("Invalid cipher key length");
 723                         return -1;
 724                 }
 725                 for (i = 0; i < sess->cipher.key_length ; i++)
 726                         sess->cipher.key_ccp[sess->cipher.key_length - i - 1] =
 727                                 sess->cipher.key[i];
 728                 break;
 729         case CCP_ENGINE_3DES:
 730                 if (sess->cipher.key_length == 16)
 731                         sess->cipher.ut.des_type = CCP_DES_TYPE_128;
 732                 else if (sess->cipher.key_length == 24)
 733                         sess->cipher.ut.des_type = CCP_DES_TYPE_192;
 734                 else {
 735                         CCP_LOG_ERR("Invalid cipher key length");
 736                         return -1;
 737                 }
 738                 for (j = 0, x = 0; j < sess->cipher.key_length/8; j++, x += 8)
 739                         for (i = 0; i < 8; i++)
 740                                 sess->cipher.key_ccp[(8 + x) - i - 1] =
 741                                         sess->cipher.key[i + x];
 742                 break;
 743         default:
 744                 CCP_LOG_ERR("Invalid CCP Engine");
 745                 return -ENOTSUP;
 746         }
 747         sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce);
 748         sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp);
 749         return 0;
 750 }
 751
 752 static int
 753 ccp_configure_session_auth(struct ccp_session *sess,
 754                            const struct rte_crypto_sym_xform *xform)
 755 {
 756         const struct rte_crypto_auth_xform *auth_xform = NULL;
 757         size_t i;
 758
 759         auth_xform = &xform->auth;
 760
 761         sess->auth.digest_length = auth_xform->digest_length;
 762         if (auth_xform->op ==  RTE_CRYPTO_AUTH_OP_GENERATE)
 763                 sess->auth.op = CCP_AUTH_OP_GENERATE;
 764         else
 765                 sess->auth.op = CCP_AUTH_OP_VERIFY;
 766         switch (auth_xform->algo) {
 767         case RTE_CRYPTO_AUTH_MD5_HMAC:
 768                 if (sess->auth_opt) {
 769                         sess->auth.algo = CCP_AUTH_ALGO_MD5_HMAC;
 770                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
 771                                              MD5_DIGEST_SIZE);
 772                         sess->auth.key_length = auth_xform->key.length;
 773                         sess->auth.block_size = MD5_BLOCK_SIZE;
 774                         memset(sess->auth.key, 0, sess->auth.block_size);
 775                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 776                                    auth_xform->key.length);
 777                 } else
 778                         return -1; /* HMAC MD5 not supported on CCP */
 779                 break;
 780         case RTE_CRYPTO_AUTH_SHA1:
 781                 sess->auth.engine = CCP_ENGINE_SHA;
 782                 sess->auth.algo = CCP_AUTH_ALGO_SHA1;
 783                 sess->auth.ut.sha_type = CCP_SHA_TYPE_1;
 784                 sess->auth.ctx = (void *)ccp_sha1_init;
 785                 sess->auth.ctx_len = CCP_SB_BYTES;
 786                 sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 787                 break;
 788         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 789                 if (sess->auth_opt) {
 790                         if (auth_xform->key.length > SHA1_BLOCK_SIZE)
 791                                 return -1;
 792                         sess->auth.algo = CCP_AUTH_ALGO_SHA1_HMAC;
 793                         sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 794                         sess->auth.block_size = SHA1_BLOCK_SIZE;
 795                         sess->auth.key_length = auth_xform->key.length;
 796                         memset(sess->auth.key, 0, sess->auth.block_size);
 797                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 798                                    auth_xform->key.length);
 799                 } else {
 800                         if (auth_xform->key.length > SHA1_BLOCK_SIZE)
 801                                 return -1;
 802                         sess->auth.engine = CCP_ENGINE_SHA;
 803                         sess->auth.algo = CCP_AUTH_ALGO_SHA1_HMAC;
 804                         sess->auth.ut.sha_type = CCP_SHA_TYPE_1;
 805                         sess->auth.ctx_len = CCP_SB_BYTES;
 806                         sess->auth.offset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
 807                         sess->auth.block_size = SHA1_BLOCK_SIZE;
 808                         sess->auth.key_length = auth_xform->key.length;
 809                         memset(sess->auth.key, 0, sess->auth.block_size);
 810                         memset(sess->auth.pre_compute, 0,
 811                                sess->auth.ctx_len << 1);
 812                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 813                                    auth_xform->key.length);
 814                         if (generate_partial_hash(sess))
 815                                 return -1;
 816                 }
 817                 break;
 818         case RTE_CRYPTO_AUTH_SHA224:
 819                 sess->auth.algo = CCP_AUTH_ALGO_SHA224;
 820                 sess->auth.engine = CCP_ENGINE_SHA;
 821                 sess->auth.ut.sha_type = CCP_SHA_TYPE_224;
 822                 sess->auth.ctx = (void *)ccp_sha224_init;
 823                 sess->auth.ctx_len = CCP_SB_BYTES;
 824                 sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 825                 break;
 826         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 827                 if (sess->auth_opt) {
 828                         if (auth_xform->key.length > SHA224_BLOCK_SIZE)
 829                                 return -1;
 830                         sess->auth.algo = CCP_AUTH_ALGO_SHA224_HMAC;
 831                         sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 832                         sess->auth.block_size = SHA224_BLOCK_SIZE;
 833                         sess->auth.key_length = auth_xform->key.length;
 834                         memset(sess->auth.key, 0, sess->auth.block_size);
 835                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 836                                    auth_xform->key.length);
 837                 } else {
 838                         if (auth_xform->key.length > SHA224_BLOCK_SIZE)
 839                                 return -1;
 840                         sess->auth.algo = CCP_AUTH_ALGO_SHA224_HMAC;
 841                         sess->auth.engine = CCP_ENGINE_SHA;
 842                         sess->auth.ut.sha_type = CCP_SHA_TYPE_224;
 843                         sess->auth.ctx_len = CCP_SB_BYTES;
 844                         sess->auth.offset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
 845                         sess->auth.block_size = SHA224_BLOCK_SIZE;
 846                         sess->auth.key_length = auth_xform->key.length;
 847                         memset(sess->auth.key, 0, sess->auth.block_size);
 848                         memset(sess->auth.pre_compute, 0,
 849                                sess->auth.ctx_len << 1);
 850                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 851                                    auth_xform->key.length);
 852                         if (generate_partial_hash(sess))
 853                                 return -1;
 854                 }
 855                 break;
 856         case RTE_CRYPTO_AUTH_SHA3_224:
 857                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_224;
 858                 sess->auth.engine = CCP_ENGINE_SHA;
 859                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
 860                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 861                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
 862                 break;
 863         case RTE_CRYPTO_AUTH_SHA3_224_HMAC:
 864                 if (auth_xform->key.length > SHA3_224_BLOCK_SIZE)
 865                         return -1;
 866                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_224_HMAC;
 867                 sess->auth.engine = CCP_ENGINE_SHA;
 868                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_224;
 869                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 870                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA224_DIGEST_SIZE;
 871                 sess->auth.block_size = SHA3_224_BLOCK_SIZE;
 872                 sess->auth.key_length = auth_xform->key.length;
 873                 memset(sess->auth.key, 0, sess->auth.block_size);
 874                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
 875                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 876                            auth_xform->key.length);
 877                 if (generate_partial_hash(sess))
 878                         return -1;
 879                 break;
 880         case RTE_CRYPTO_AUTH_SHA256:
 881                 sess->auth.algo = CCP_AUTH_ALGO_SHA256;
 882                 sess->auth.engine = CCP_ENGINE_SHA;
 883                 sess->auth.ut.sha_type = CCP_SHA_TYPE_256;
 884                 sess->auth.ctx = (void *)ccp_sha256_init;
 885                 sess->auth.ctx_len = CCP_SB_BYTES;
 886                 sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 887                 break;
 888         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 889                 if (sess->auth_opt) {
 890                         if (auth_xform->key.length > SHA256_BLOCK_SIZE)
 891                                 return -1;
 892                         sess->auth.algo = CCP_AUTH_ALGO_SHA256_HMAC;
 893                         sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 894                         sess->auth.block_size = SHA256_BLOCK_SIZE;
 895                         sess->auth.key_length = auth_xform->key.length;
 896                         memset(sess->auth.key, 0, sess->auth.block_size);
 897                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 898                                    auth_xform->key.length);
 899                 } else {
 900                         if (auth_xform->key.length > SHA256_BLOCK_SIZE)
 901                                 return -1;
 902                         sess->auth.algo = CCP_AUTH_ALGO_SHA256_HMAC;
 903                         sess->auth.engine = CCP_ENGINE_SHA;
 904                         sess->auth.ut.sha_type = CCP_SHA_TYPE_256;
 905                         sess->auth.ctx_len = CCP_SB_BYTES;
 906                         sess->auth.offset = CCP_SB_BYTES - SHA256_DIGEST_SIZE;
 907                         sess->auth.block_size = SHA256_BLOCK_SIZE;
 908                         sess->auth.key_length = auth_xform->key.length;
 909                         memset(sess->auth.key, 0, sess->auth.block_size);
 910                         memset(sess->auth.pre_compute, 0,
 911                                sess->auth.ctx_len << 1);
 912                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 913                                    auth_xform->key.length);
 914                         if (generate_partial_hash(sess))
 915                                 return -1;
 916                 }
 917                 break;
 918         case RTE_CRYPTO_AUTH_SHA3_256:
 919                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_256;
 920                 sess->auth.engine = CCP_ENGINE_SHA;
 921                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
 922                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 923                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
 924                 break;
 925         case RTE_CRYPTO_AUTH_SHA3_256_HMAC:
 926                 if (auth_xform->key.length > SHA3_256_BLOCK_SIZE)
 927                         return -1;
 928                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_256_HMAC;
 929                 sess->auth.engine = CCP_ENGINE_SHA;
 930                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_256;
 931                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 932                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA256_DIGEST_SIZE;
 933                 sess->auth.block_size = SHA3_256_BLOCK_SIZE;
 934                 sess->auth.key_length = auth_xform->key.length;
 935                 memset(sess->auth.key, 0, sess->auth.block_size);
 936                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
 937                 rte_memcpy(sess->auth.key, auth_xform->key.data,
 938                            auth_xform->key.length);
 939                 if (generate_partial_hash(sess))
 940                         return -1;
 941                 break;
 942         case RTE_CRYPTO_AUTH_SHA384:
 943                 sess->auth.algo = CCP_AUTH_ALGO_SHA384;
 944                 sess->auth.engine = CCP_ENGINE_SHA;
 945                 sess->auth.ut.sha_type = CCP_SHA_TYPE_384;
 946                 sess->auth.ctx = (void *)ccp_sha384_init;
 947                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
 948                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA384_DIGEST_SIZE;
 949                 break;
 950         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 951                 if (sess->auth_opt) {
 952                         if (auth_xform->key.length > SHA384_BLOCK_SIZE)
 953                                 return -1;
 954                         sess->auth.algo = CCP_AUTH_ALGO_SHA384_HMAC;
 955                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
 956                                              SHA384_DIGEST_SIZE);
 957                         sess->auth.block_size = SHA384_BLOCK_SIZE;
 958                         sess->auth.key_length = auth_xform->key.length;
 959                         memset(sess->auth.key, 0, sess->auth.block_size);
 960                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 961                                    auth_xform->key.length);
 962                 } else {
 963                         if (auth_xform->key.length > SHA384_BLOCK_SIZE)
 964                                 return -1;
 965                         sess->auth.algo = CCP_AUTH_ALGO_SHA384_HMAC;
 966                         sess->auth.engine = CCP_ENGINE_SHA;
 967                         sess->auth.ut.sha_type = CCP_SHA_TYPE_384;
 968                         sess->auth.ctx_len = CCP_SB_BYTES << 1;
 969                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
 970                                              SHA384_DIGEST_SIZE);
 971                         sess->auth.block_size = SHA384_BLOCK_SIZE;
 972                         sess->auth.key_length = auth_xform->key.length;
 973                         memset(sess->auth.key, 0, sess->auth.block_size);
 974                         memset(sess->auth.pre_compute, 0,
 975                                sess->auth.ctx_len << 1);
 976                         rte_memcpy(sess->auth.key, auth_xform->key.data,
 977                                    auth_xform->key.length);
 978                         if (generate_partial_hash(sess))
 979                                 return -1;
 980                 }
 981                 break;
 982         case RTE_CRYPTO_AUTH_SHA3_384:
 983                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_384;
 984                 sess->auth.engine = CCP_ENGINE_SHA;
 985                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
 986                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 987                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
 988                 break;
 989         case RTE_CRYPTO_AUTH_SHA3_384_HMAC:
 990                 if (auth_xform->key.length > SHA3_384_BLOCK_SIZE)
 991                         return -1;
 992                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_384_HMAC;
 993                 sess->auth.engine = CCP_ENGINE_SHA;
 994                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_384;
 995                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
 996                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA384_DIGEST_SIZE;
 997                 sess->auth.block_size = SHA3_384_BLOCK_SIZE;
 998                 sess->auth.key_length = auth_xform->key.length;
 999                 memset(sess->auth.key, 0, sess->auth.block_size);
1000                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
1001                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1002                            auth_xform->key.length);
1003                 if (generate_partial_hash(sess))
1004                         return -1;
1005                 break;
1006         case RTE_CRYPTO_AUTH_SHA512:
1007                 sess->auth.algo = CCP_AUTH_ALGO_SHA512;
1008                 sess->auth.engine = CCP_ENGINE_SHA;
1009                 sess->auth.ut.sha_type = CCP_SHA_TYPE_512;
1010                 sess->auth.ctx = (void *)ccp_sha512_init;
1011                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
1012                 sess->auth.offset = (CCP_SB_BYTES << 1) - SHA512_DIGEST_SIZE;
1013                 break;
1014         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1015                 if (sess->auth_opt) {
1016                         if (auth_xform->key.length > SHA512_BLOCK_SIZE)
1017                                 return -1;
1018                         sess->auth.algo = CCP_AUTH_ALGO_SHA512_HMAC;
1019                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
1020                                              SHA512_DIGEST_SIZE);
1021                         sess->auth.block_size = SHA512_BLOCK_SIZE;
1022                         sess->auth.key_length = auth_xform->key.length;
1023                         memset(sess->auth.key, 0, sess->auth.block_size);
1024                         rte_memcpy(sess->auth.key, auth_xform->key.data,
1025                                    auth_xform->key.length);
1026                 } else {
1027                         if (auth_xform->key.length > SHA512_BLOCK_SIZE)
1028                                 return -1;
1029                         sess->auth.algo = CCP_AUTH_ALGO_SHA512_HMAC;
1030                         sess->auth.engine = CCP_ENGINE_SHA;
1031                         sess->auth.ut.sha_type = CCP_SHA_TYPE_512;
1032                         sess->auth.ctx_len = CCP_SB_BYTES << 1;
1033                         sess->auth.offset = ((CCP_SB_BYTES << 1) -
1034                                              SHA512_DIGEST_SIZE);
1035                         sess->auth.block_size = SHA512_BLOCK_SIZE;
1036                         sess->auth.key_length = auth_xform->key.length;
1037                         memset(sess->auth.key, 0, sess->auth.block_size);
1038                         memset(sess->auth.pre_compute, 0,
1039                                sess->auth.ctx_len << 1);
1040                         rte_memcpy(sess->auth.key, auth_xform->key.data,
1041                                    auth_xform->key.length);
1042                         if (generate_partial_hash(sess))
1043                                 return -1;
1044                 }
1045                 break;
1046         case RTE_CRYPTO_AUTH_SHA3_512:
1047                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_512;
1048                 sess->auth.engine = CCP_ENGINE_SHA;
1049                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
1050                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
1051                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
1052                 break;
1053         case RTE_CRYPTO_AUTH_SHA3_512_HMAC:
1054                 if (auth_xform->key.length > SHA3_512_BLOCK_SIZE)
1055                         return -1;
1056                 sess->auth.algo = CCP_AUTH_ALGO_SHA3_512_HMAC;
1057                 sess->auth.engine = CCP_ENGINE_SHA;
1058                 sess->auth.ut.sha_type = CCP_SHA3_TYPE_512;
1059                 sess->auth.ctx_len = CCP_SHA3_CTX_SIZE;
1060                 sess->auth.offset = CCP_SHA3_CTX_SIZE - SHA512_DIGEST_SIZE;
1061                 sess->auth.block_size = SHA3_512_BLOCK_SIZE;
1062                 sess->auth.key_length = auth_xform->key.length;
1063                 memset(sess->auth.key, 0, sess->auth.block_size);
1064                 memset(sess->auth.pre_compute, 0, 2 * sess->auth.ctx_len);
1065                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1066                            auth_xform->key.length);
1067                 if (generate_partial_hash(sess))
1068                         return -1;
1069                 break;
1070         case RTE_CRYPTO_AUTH_AES_CMAC:
1071                 sess->auth.algo = CCP_AUTH_ALGO_AES_CMAC;
1072                 sess->auth.engine = CCP_ENGINE_AES;
1073                 sess->auth.um.aes_mode = CCP_AES_MODE_CMAC;
1074                 sess->auth.key_length = auth_xform->key.length;
1075                 /* padding and hash result */
1076                 sess->auth.ctx_len = CCP_SB_BYTES << 1;
1077                 sess->auth.offset = AES_BLOCK_SIZE;
1078                 sess->auth.block_size = AES_BLOCK_SIZE;
1079                 if (sess->auth.key_length == 16)
1080                         sess->auth.ut.aes_type = CCP_AES_TYPE_128;
1081                 else if (sess->auth.key_length == 24)
1082                         sess->auth.ut.aes_type = CCP_AES_TYPE_192;
1083                 else if (sess->auth.key_length == 32)
1084                         sess->auth.ut.aes_type = CCP_AES_TYPE_256;
1085                 else {
1086                         CCP_LOG_ERR("Invalid CMAC key length");
1087                         return -1;
1088                 }
1089                 rte_memcpy(sess->auth.key, auth_xform->key.data,
1090                            sess->auth.key_length);
1091                 for (i = 0; i < sess->auth.key_length; i++)
1092                         sess->auth.key_ccp[sess->auth.key_length - i - 1] =
1093                                 sess->auth.key[i];
1094                 if (generate_cmac_subkeys(sess))
1095                         return -1;
1096                 break;
1097         default:
1098                 CCP_LOG_ERR("Unsupported hash algo");
1099                 return -ENOTSUP;
1100         }
1101         return 0;
1102 }
1103
1104 static int
1105 ccp_configure_session_aead(struct ccp_session *sess,
1106                            const struct rte_crypto_sym_xform *xform)
1107 {
1108         const struct rte_crypto_aead_xform *aead_xform = NULL;
1109         size_t i;
1110
1111         aead_xform = &xform->aead;
1112
1113         sess->cipher.key_length = aead_xform->key.length;
1114         rte_memcpy(sess->cipher.key, aead_xform->key.data,
1115                    aead_xform->key.length);
1116
1117         if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
1118                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
1119                 sess->auth.op = CCP_AUTH_OP_GENERATE;
1120         } else {
1121                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
1122                 sess->auth.op = CCP_AUTH_OP_VERIFY;
1123         }
1124         sess->aead_algo = aead_xform->algo;
1125         sess->auth.aad_length = aead_xform->aad_length;
1126         sess->auth.digest_length = aead_xform->digest_length;
1127
1128         /* set iv parameters */
1129         sess->iv.offset = aead_xform->iv.offset;
1130         sess->iv.length = aead_xform->iv.length;
1131
1132         switch (aead_xform->algo) {
1133         case RTE_CRYPTO_AEAD_AES_GCM:
1134                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_GCM;
1135                 sess->cipher.um.aes_mode = CCP_AES_MODE_GCTR;
1136                 sess->cipher.engine = CCP_ENGINE_AES;
1137                 if (sess->cipher.key_length == 16)
1138                         sess->cipher.ut.aes_type = CCP_AES_TYPE_128;
1139                 else if (sess->cipher.key_length == 24)
1140                         sess->cipher.ut.aes_type = CCP_AES_TYPE_192;
1141                 else if (sess->cipher.key_length == 32)
1142                         sess->cipher.ut.aes_type = CCP_AES_TYPE_256;
1143                 else {
1144                         CCP_LOG_ERR("Invalid aead key length");
1145                         return -1;
1146                 }
1147                 for (i = 0; i < sess->cipher.key_length; i++)
1148                         sess->cipher.key_ccp[sess->cipher.key_length - i - 1] =
1149                                 sess->cipher.key[i];
1150                 sess->auth.algo = CCP_AUTH_ALGO_AES_GCM;
1151                 sess->auth.engine = CCP_ENGINE_AES;
1152                 sess->auth.um.aes_mode = CCP_AES_MODE_GHASH;
1153                 sess->auth.ctx_len = CCP_SB_BYTES;
1154                 sess->auth.offset = 0;
1155                 sess->auth.block_size = AES_BLOCK_SIZE;
1156                 sess->cmd_id = CCP_CMD_COMBINED;
1157                 break;
1158         default:
1159                 CCP_LOG_ERR("Unsupported aead algo");
1160                 return -ENOTSUP;
1161         }
1162         sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce);
1163         sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp);
1164         return 0;
1165 }
1166
1167 int
1168 ccp_set_session_parameters(struct ccp_session *sess,
1169                            const struct rte_crypto_sym_xform *xform,
1170                            struct ccp_private *internals)
1171 {
1172         const struct rte_crypto_sym_xform *cipher_xform = NULL;
1173         const struct rte_crypto_sym_xform *auth_xform = NULL;
1174         const struct rte_crypto_sym_xform *aead_xform = NULL;
1175         int ret = 0;
1176
1177         sess->auth_opt = internals->auth_opt;
1178         sess->cmd_id = ccp_get_cmd_id(xform);
1179
1180         switch (sess->cmd_id) {
1181         case CCP_CMD_CIPHER:
1182                 cipher_xform = xform;
1183                 break;
1184         case CCP_CMD_AUTH:
1185                 auth_xform = xform;
1186                 break;
1187         case CCP_CMD_CIPHER_HASH:
1188                 cipher_xform = xform;
1189                 auth_xform = xform->next;
1190                 break;
1191         case CCP_CMD_HASH_CIPHER:
1192                 auth_xform = xform;
1193                 cipher_xform = xform->next;
1194                 break;
1195         case CCP_CMD_COMBINED:
1196                 aead_xform = xform;
1197                 break;
1198         default:
1199                 CCP_LOG_ERR("Unsupported cmd_id");
1200                 return -1;
1201         }
1202
1203         /* Default IV length = 0 */
1204         sess->iv.length = 0;
1205         if (cipher_xform) {
1206                 ret = ccp_configure_session_cipher(sess, cipher_xform);
1207                 if (ret != 0) {
1208                         CCP_LOG_ERR("Invalid/unsupported cipher parameters");
1209                         return ret;
1210                 }
1211         }
1212         if (auth_xform) {
1213                 ret = ccp_configure_session_auth(sess, auth_xform);
1214                 if (ret != 0) {
1215                         CCP_LOG_ERR("Invalid/unsupported auth parameters");
1216                         return ret;
1217                 }
1218         }
1219         if (aead_xform) {
1220                 ret = ccp_configure_session_aead(sess, aead_xform);
1221                 if (ret != 0) {
1222                         CCP_LOG_ERR("Invalid/unsupported aead parameters");
1223                         return ret;
1224                 }
1225         }
1226         return ret;
1227 }
1228
1229 /* calculate CCP descriptors requirement */
1230 static inline int
1231 ccp_cipher_slot(struct ccp_session *session)
1232 {
1233         int count = 0;
1234
1235         switch (session->cipher.algo) {
1236         case CCP_CIPHER_ALGO_AES_CBC:
1237                 count = 2;
1238                 /**< op + passthrough for iv */
1239                 break;
1240         case CCP_CIPHER_ALGO_AES_ECB:
1241                 count = 1;
1242                 /**<only op*/
1243                 break;
1244         case CCP_CIPHER_ALGO_AES_CTR:
1245                 count = 2;
1246                 /**< op + passthrough for iv */
1247                 break;
1248         case CCP_CIPHER_ALGO_3DES_CBC:
1249                 count = 2;
1250                 /**< op + passthrough for iv */
1251                 break;
1252         default:
1253                 CCP_LOG_ERR("Unsupported cipher algo %d",
1254                             session->cipher.algo);
1255         }
1256         return count;
1257 }
1258
1259 static inline int
1260 ccp_auth_slot(struct ccp_session *session)
1261 {
1262         int count = 0;
1263
1264         switch (session->auth.algo) {
1265         case CCP_AUTH_ALGO_SHA1:
1266         case CCP_AUTH_ALGO_SHA224:
1267         case CCP_AUTH_ALGO_SHA256:
1268         case CCP_AUTH_ALGO_SHA384:
1269         case CCP_AUTH_ALGO_SHA512:
1270                 count = 3;
1271                 /**< op + lsb passthrough cpy to/from*/
1272                 break;
1273         case CCP_AUTH_ALGO_MD5_HMAC:
1274                 break;
1275         case CCP_AUTH_ALGO_SHA1_HMAC:
1276         case CCP_AUTH_ALGO_SHA224_HMAC:
1277         case CCP_AUTH_ALGO_SHA256_HMAC:
1278                 if (session->auth_opt == 0)
1279                         count = 6;
1280                 break;
1281         case CCP_AUTH_ALGO_SHA384_HMAC:
1282         case CCP_AUTH_ALGO_SHA512_HMAC:
1283                 /**
1284                  * 1. Load PHash1 = H(k ^ ipad); to LSB
1285                  * 2. generate IHash = H(hash on meassage with PHash1
1286                  * as init values);
1287                  * 3. Retrieve IHash 2 slots for 384/512
1288                  * 4. Load Phash2 = H(k ^ opad); to LSB
1289                  * 5. generate FHash = H(hash on Ihash with Phash2
1290                  * as init value);
1291                  * 6. Retrieve HMAC output from LSB to host memory
1292                  */
1293                 if (session->auth_opt == 0)
1294                         count = 7;
1295                 break;
1296         case CCP_AUTH_ALGO_SHA3_224:
1297         case CCP_AUTH_ALGO_SHA3_256:
1298         case CCP_AUTH_ALGO_SHA3_384:
1299         case CCP_AUTH_ALGO_SHA3_512:
1300                 count = 1;
1301                 /**< only op ctx and dst in host memory*/
1302                 break;
1303         case CCP_AUTH_ALGO_SHA3_224_HMAC:
1304         case CCP_AUTH_ALGO_SHA3_256_HMAC:
1305                 count = 3;
1306                 break;
1307         case CCP_AUTH_ALGO_SHA3_384_HMAC:
1308         case CCP_AUTH_ALGO_SHA3_512_HMAC:
1309                 count = 4;
1310                 /**
1311                  * 1. Op to Perform Ihash
1312                  * 2. Retrieve result from LSB to host memory
1313                  * 3. Perform final hash
1314                  */
1315                 break;
1316         case CCP_AUTH_ALGO_AES_CMAC:
1317                 count = 4;
1318                 /**
1319                  * op
1320                  * extra descriptor in padding case
1321                  * (k1/k2(255:128) with iv(127:0))
1322                  * Retrieve result
1323                  */
1324                 break;
1325         default:
1326                 CCP_LOG_ERR("Unsupported auth algo %d",
1327                             session->auth.algo);
1328         }
1329
1330         return count;
1331 }
1332
1333 static int
1334 ccp_aead_slot(struct ccp_session *session)
1335 {
1336         int count = 0;
1337
1338         switch (session->aead_algo) {
1339         case RTE_CRYPTO_AEAD_AES_GCM:
1340                 break;
1341         default:
1342                 CCP_LOG_ERR("Unsupported aead algo %d",
1343                             session->aead_algo);
1344         }
1345         switch (session->auth.algo) {
1346         case CCP_AUTH_ALGO_AES_GCM:
1347                 count = 5;
1348                 /**
1349                  * 1. Passthru iv
1350                  * 2. Hash AAD
1351                  * 3. GCTR
1352                  * 4. Reload passthru
1353                  * 5. Hash Final tag
1354                  */
1355                 break;
1356         default:
1357                 CCP_LOG_ERR("Unsupported combined auth ALGO %d",
1358                             session->auth.algo);
1359         }
1360         return count;
1361 }
1362
1363 int
1364 ccp_compute_slot_count(struct ccp_session *session)
1365 {
1366         int count = 0;
1367
1368         switch (session->cmd_id) {
1369         case CCP_CMD_CIPHER:
1370                 count = ccp_cipher_slot(session);
1371                 break;
1372         case CCP_CMD_AUTH:
1373                 count = ccp_auth_slot(session);
1374                 break;
1375         case CCP_CMD_CIPHER_HASH:
1376         case CCP_CMD_HASH_CIPHER:
1377                 count = ccp_cipher_slot(session);
1378                 count += ccp_auth_slot(session);
1379                 break;
1380         case CCP_CMD_COMBINED:
1381                 count = ccp_aead_slot(session);
1382                 break;
1383         default:
1384                 CCP_LOG_ERR("Unsupported cmd_id");
1385
1386         }
1387
1388         return count;
1389 }
1390
1391 static uint8_t
1392 algo_select(int sessalgo,
1393             const EVP_MD **algo)
1394 {
1395         int res = 0;
1396
1397         switch (sessalgo) {
1398         case CCP_AUTH_ALGO_MD5_HMAC:
1399                 *algo = EVP_md5();
1400                 break;
1401         case CCP_AUTH_ALGO_SHA1_HMAC:
1402                 *algo = EVP_sha1();
1403                 break;
1404         case CCP_AUTH_ALGO_SHA224_HMAC:
1405                 *algo = EVP_sha224();
1406                 break;
1407         case CCP_AUTH_ALGO_SHA256_HMAC:
1408                 *algo = EVP_sha256();
1409                 break;
1410         case CCP_AUTH_ALGO_SHA384_HMAC:
1411                 *algo = EVP_sha384();
1412                 break;
1413         case CCP_AUTH_ALGO_SHA512_HMAC:
1414                 *algo = EVP_sha512();
1415                 break;
1416         default:
1417                 res = -EINVAL;
1418                 break;
1419         }
1420         return res;
1421 }
1422
1423 static int
1424 process_cpu_auth_hmac(uint8_t *src, uint8_t *dst,
1425                       __rte_unused uint8_t *iv,
1426                       EVP_PKEY *pkey,
1427                       int srclen,
1428                       EVP_MD_CTX *ctx,
1429                       const EVP_MD *algo,
1430                       uint16_t d_len)
1431 {
1432         size_t dstlen;
1433         unsigned char temp_dst[64];
1434
1435         if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0)
1436                 goto process_auth_err;
1437
1438         if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0)
1439                 goto process_auth_err;
1440
1441         if (EVP_DigestSignFinal(ctx, temp_dst, &dstlen) <= 0)
1442                 goto process_auth_err;
1443
1444         memcpy(dst, temp_dst, d_len);
1445         return 0;
1446 process_auth_err:
1447         CCP_LOG_ERR("Process cpu auth failed");
1448         return -EINVAL;
1449 }
1450
1451 static int cpu_crypto_auth(struct ccp_qp *qp,
1452                            struct rte_crypto_op *op,
1453                            struct ccp_session *sess,
1454                            EVP_MD_CTX *ctx)
1455 {
1456         uint8_t *src, *dst;
1457         int srclen, status;
1458         struct rte_mbuf *mbuf_src, *mbuf_dst;
1459         const EVP_MD *algo = NULL;
1460         EVP_PKEY *pkey;
1461
1462         algo_select(sess->auth.algo, &algo);
1463         pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, sess->auth.key,
1464                                     sess->auth.key_length);
1465         mbuf_src = op->sym->m_src;
1466         mbuf_dst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1467         srclen = op->sym->auth.data.length;
1468         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1469                                       op->sym->auth.data.offset);
1470
1471         if (sess->auth.op == CCP_AUTH_OP_VERIFY) {
1472                 dst = qp->temp_digest;
1473         } else {
1474                 dst = op->sym->auth.digest.data;
1475                 if (dst == NULL) {
1476                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1477                                                      op->sym->auth.data.offset +
1478                                                      sess->auth.digest_length);
1479                 }
1480         }
1481         status = process_cpu_auth_hmac(src, dst, NULL,
1482                                        pkey, srclen,
1483                                        ctx,
1484                                        algo,
1485                                        sess->auth.digest_length);
1486         if (status) {
1487                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1488                 return status;
1489         }
1490
1491         if (sess->auth.op == CCP_AUTH_OP_VERIFY) {
1492                 if (memcmp(dst, op->sym->auth.digest.data,
1493                            sess->auth.digest_length) != 0) {
1494                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1495                 } else {
1496                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1497                 }
1498         } else {
1499                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1500         }
1501         EVP_PKEY_free(pkey);
1502         return 0;
1503 }
1504
1505 static void
1506 ccp_perform_passthru(struct ccp_passthru *pst,
1507                      struct ccp_queue *cmd_q)
1508 {
1509         struct ccp_desc *desc;
1510         union ccp_function function;
1511
1512         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1513
1514         CCP_CMD_ENGINE(desc) = CCP_ENGINE_PASSTHRU;
1515
1516         CCP_CMD_SOC(desc) = 0;
1517         CCP_CMD_IOC(desc) = 0;
1518         CCP_CMD_INIT(desc) = 0;
1519         CCP_CMD_EOM(desc) = 0;
1520         CCP_CMD_PROT(desc) = 0;
1521
1522         function.raw = 0;
1523         CCP_PT_BYTESWAP(&function) = pst->byte_swap;
1524         CCP_PT_BITWISE(&function) = pst->bit_mod;
1525         CCP_CMD_FUNCTION(desc) = function.raw;
1526
1527         CCP_CMD_LEN(desc) = pst->len;
1528
1529         if (pst->dir) {
1530                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
1531                 CCP_CMD_SRC_HI(desc) = high32_value(pst->src_addr);
1532                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1533
1534                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
1535                 CCP_CMD_DST_HI(desc) = 0;
1536                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
1537
1538                 if (pst->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1539                         CCP_CMD_LSB_ID(desc) = cmd_q->sb_key;
1540         } else {
1541
1542                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
1543                 CCP_CMD_SRC_HI(desc) = 0;
1544                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SB;
1545
1546                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
1547                 CCP_CMD_DST_HI(desc) = high32_value(pst->dest_addr);
1548                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1549         }
1550
1551         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1552 }
1553
1554 static int
1555 ccp_perform_hmac(struct rte_crypto_op *op,
1556                  struct ccp_queue *cmd_q)
1557 {
1558
1559         struct ccp_session *session;
1560         union ccp_function function;
1561         struct ccp_desc *desc;
1562         uint32_t tail;
1563         phys_addr_t src_addr, dest_addr, dest_addr_t;
1564         struct ccp_passthru pst;
1565         uint64_t auth_msg_bits;
1566         void *append_ptr;
1567         uint8_t *addr;
1568
1569         session = (struct ccp_session *)get_sym_session_private_data(
1570                                          op->sym->session,
1571                                          ccp_cryptodev_driver_id);
1572         addr = session->auth.pre_compute;
1573
1574         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1575                                               op->sym->auth.data.offset);
1576         append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src,
1577                                                 session->auth.ctx_len);
1578         dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr);
1579         dest_addr_t = dest_addr;
1580
1581         /** Load PHash1 to LSB*/
1582         pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr);
1583         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1584         pst.len = session->auth.ctx_len;
1585         pst.dir = 1;
1586         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1587         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1588         ccp_perform_passthru(&pst, cmd_q);
1589
1590         /**sha engine command descriptor for IntermediateHash*/
1591
1592         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1593         memset(desc, 0, Q_DESC_SIZE);
1594
1595         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1596
1597         CCP_CMD_SOC(desc) = 0;
1598         CCP_CMD_IOC(desc) = 0;
1599         CCP_CMD_INIT(desc) = 1;
1600         CCP_CMD_EOM(desc) = 1;
1601         CCP_CMD_PROT(desc) = 0;
1602
1603         function.raw = 0;
1604         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1605         CCP_CMD_FUNCTION(desc) = function.raw;
1606
1607         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1608         auth_msg_bits = (op->sym->auth.data.length +
1609                          session->auth.block_size)  * 8;
1610
1611         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1612         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1613         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1614
1615         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1616         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1617         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1618
1619         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1620
1621         rte_wmb();
1622
1623         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1624         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1625         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1626                       cmd_q->qcontrol | CMD_Q_RUN);
1627
1628         /* Intermediate Hash value retrieve */
1629         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1630             (session->auth.ut.sha_type == CCP_SHA_TYPE_512)) {
1631
1632                 pst.src_addr =
1633                         (phys_addr_t)((cmd_q->sb_sha + 1) * CCP_SB_BYTES);
1634                 pst.dest_addr = dest_addr_t;
1635                 pst.len = CCP_SB_BYTES;
1636                 pst.dir = 0;
1637                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1638                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1639                 ccp_perform_passthru(&pst, cmd_q);
1640
1641                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1642                 pst.dest_addr = dest_addr_t + CCP_SB_BYTES;
1643                 pst.len = CCP_SB_BYTES;
1644                 pst.dir = 0;
1645                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1646                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1647                 ccp_perform_passthru(&pst, cmd_q);
1648
1649         } else {
1650                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1651                 pst.dest_addr = dest_addr_t;
1652                 pst.len = session->auth.ctx_len;
1653                 pst.dir = 0;
1654                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1655                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1656                 ccp_perform_passthru(&pst, cmd_q);
1657
1658         }
1659
1660         /** Load PHash2 to LSB*/
1661         addr += session->auth.ctx_len;
1662         pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)addr);
1663         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1664         pst.len = session->auth.ctx_len;
1665         pst.dir = 1;
1666         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1667         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1668         ccp_perform_passthru(&pst, cmd_q);
1669
1670         /**sha engine command descriptor for FinalHash*/
1671         dest_addr_t += session->auth.offset;
1672
1673         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1674         memset(desc, 0, Q_DESC_SIZE);
1675
1676         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1677
1678         CCP_CMD_SOC(desc) = 0;
1679         CCP_CMD_IOC(desc) = 0;
1680         CCP_CMD_INIT(desc) = 1;
1681         CCP_CMD_EOM(desc) = 1;
1682         CCP_CMD_PROT(desc) = 0;
1683
1684         function.raw = 0;
1685         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1686         CCP_CMD_FUNCTION(desc) = function.raw;
1687
1688         CCP_CMD_LEN(desc) = (session->auth.ctx_len -
1689                              session->auth.offset);
1690         auth_msg_bits = (session->auth.block_size +
1691                          session->auth.ctx_len -
1692                          session->auth.offset) * 8;
1693
1694         CCP_CMD_SRC_LO(desc) = (uint32_t)(dest_addr_t);
1695         CCP_CMD_SRC_HI(desc) = high32_value(dest_addr_t);
1696         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1697
1698         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1699         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1700         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1701
1702         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1703
1704         rte_wmb();
1705
1706         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1707         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1708         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1709                       cmd_q->qcontrol | CMD_Q_RUN);
1710
1711         /* Retrieve hmac output */
1712         pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1713         pst.dest_addr = dest_addr;
1714         pst.len = session->auth.ctx_len;
1715         pst.dir = 0;
1716         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1717         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1718             (session->auth.ut.sha_type == CCP_SHA_TYPE_512))
1719                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1720         else
1721                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1722         ccp_perform_passthru(&pst, cmd_q);
1723
1724         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1725         return 0;
1726
1727 }
1728
1729 static int
1730 ccp_perform_sha(struct rte_crypto_op *op,
1731                 struct ccp_queue *cmd_q)
1732 {
1733         struct ccp_session *session;
1734         union ccp_function function;
1735         struct ccp_desc *desc;
1736         uint32_t tail;
1737         phys_addr_t src_addr, dest_addr;
1738         struct ccp_passthru pst;
1739         void *append_ptr;
1740         uint64_t auth_msg_bits;
1741
1742         session = (struct ccp_session *)get_sym_session_private_data(
1743                                          op->sym->session,
1744                                         ccp_cryptodev_driver_id);
1745
1746         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1747                                               op->sym->auth.data.offset);
1748
1749         append_ptr = (void *)rte_pktmbuf_append(op->sym->m_src,
1750                                                 session->auth.ctx_len);
1751         dest_addr = (phys_addr_t)rte_mem_virt2phy(append_ptr);
1752
1753         /** Passthru sha context*/
1754
1755         pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)
1756                                                      session->auth.ctx);
1757         pst.dest_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1758         pst.len = session->auth.ctx_len;
1759         pst.dir = 1;
1760         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1761         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1762         ccp_perform_passthru(&pst, cmd_q);
1763
1764         /**prepare sha command descriptor*/
1765
1766         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1767         memset(desc, 0, Q_DESC_SIZE);
1768
1769         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1770
1771         CCP_CMD_SOC(desc) = 0;
1772         CCP_CMD_IOC(desc) = 0;
1773         CCP_CMD_INIT(desc) = 1;
1774         CCP_CMD_EOM(desc) = 1;
1775         CCP_CMD_PROT(desc) = 0;
1776
1777         function.raw = 0;
1778         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1779         CCP_CMD_FUNCTION(desc) = function.raw;
1780
1781         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1782         auth_msg_bits = op->sym->auth.data.length * 8;
1783
1784         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1785         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1786         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1787
1788         CCP_CMD_LSB_ID(desc) = cmd_q->sb_sha;
1789         CCP_CMD_SHA_LO(desc) = ((uint32_t)auth_msg_bits);
1790         CCP_CMD_SHA_HI(desc) = high32_value(auth_msg_bits);
1791
1792         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1793
1794         rte_wmb();
1795
1796         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1797         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1798         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1799                       cmd_q->qcontrol | CMD_Q_RUN);
1800
1801         /* Hash value retrieve */
1802         pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1803         pst.dest_addr = dest_addr;
1804         pst.len = session->auth.ctx_len;
1805         pst.dir = 0;
1806         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1807         if ((session->auth.ut.sha_type == CCP_SHA_TYPE_384) ||
1808             (session->auth.ut.sha_type == CCP_SHA_TYPE_512))
1809                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
1810         else
1811                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1812         ccp_perform_passthru(&pst, cmd_q);
1813
1814         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1815         return 0;
1816
1817 }
1818
1819 static int
1820 ccp_perform_sha3_hmac(struct rte_crypto_op *op,
1821                       struct ccp_queue *cmd_q)
1822 {
1823         struct ccp_session *session;
1824         struct ccp_passthru pst;
1825         union ccp_function function;
1826         struct ccp_desc *desc;
1827         uint8_t *append_ptr;
1828         uint32_t tail;
1829         phys_addr_t src_addr, dest_addr, ctx_paddr, dest_addr_t;
1830
1831         session = (struct ccp_session *)get_sym_session_private_data(
1832                                          op->sym->session,
1833                                         ccp_cryptodev_driver_id);
1834
1835         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1836                                               op->sym->auth.data.offset);
1837         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
1838                                                 session->auth.ctx_len);
1839         if (!append_ptr) {
1840                 CCP_LOG_ERR("CCP MBUF append failed\n");
1841                 return -1;
1842         }
1843         dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
1844         dest_addr_t = dest_addr + (session->auth.ctx_len / 2);
1845         ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void
1846                                                    *)session->auth.pre_compute);
1847         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1848         memset(desc, 0, Q_DESC_SIZE);
1849
1850         /*desc1 for SHA3-Ihash operation */
1851         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1852         CCP_CMD_INIT(desc) = 1;
1853         CCP_CMD_EOM(desc) = 1;
1854
1855         function.raw = 0;
1856         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1857         CCP_CMD_FUNCTION(desc) = function.raw;
1858         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
1859
1860         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
1861         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
1862         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1863
1864         CCP_CMD_DST_LO(desc) = (cmd_q->sb_sha * CCP_SB_BYTES);
1865         CCP_CMD_DST_HI(desc) = 0;
1866         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
1867
1868         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
1869         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
1870         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1871
1872         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1873
1874         rte_wmb();
1875         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1876         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1877         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1878                       cmd_q->qcontrol | CMD_Q_RUN);
1879
1880         /* Intermediate Hash value retrieve */
1881         if ((session->auth.ut.sha_type == CCP_SHA3_TYPE_384) ||
1882             (session->auth.ut.sha_type == CCP_SHA3_TYPE_512)) {
1883
1884                 pst.src_addr =
1885                         (phys_addr_t)((cmd_q->sb_sha + 1) * CCP_SB_BYTES);
1886                 pst.dest_addr = dest_addr_t;
1887                 pst.len = CCP_SB_BYTES;
1888                 pst.dir = 0;
1889                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1890                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1891                 ccp_perform_passthru(&pst, cmd_q);
1892
1893                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1894                 pst.dest_addr = dest_addr_t + CCP_SB_BYTES;
1895                 pst.len = CCP_SB_BYTES;
1896                 pst.dir = 0;
1897                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1898                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1899                 ccp_perform_passthru(&pst, cmd_q);
1900
1901         } else {
1902                 pst.src_addr = (phys_addr_t)(cmd_q->sb_sha * CCP_SB_BYTES);
1903                 pst.dest_addr = dest_addr_t;
1904                 pst.len = CCP_SB_BYTES;
1905                 pst.dir = 0;
1906                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
1907                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
1908                 ccp_perform_passthru(&pst, cmd_q);
1909         }
1910
1911         /**sha engine command descriptor for FinalHash*/
1912         ctx_paddr += CCP_SHA3_CTX_SIZE;
1913         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1914         memset(desc, 0, Q_DESC_SIZE);
1915
1916         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1917         CCP_CMD_INIT(desc) = 1;
1918         CCP_CMD_EOM(desc) = 1;
1919
1920         function.raw = 0;
1921         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1922         CCP_CMD_FUNCTION(desc) = function.raw;
1923
1924         if (session->auth.ut.sha_type == CCP_SHA3_TYPE_224) {
1925                 dest_addr_t += (CCP_SB_BYTES - SHA224_DIGEST_SIZE);
1926                 CCP_CMD_LEN(desc) = SHA224_DIGEST_SIZE;
1927         } else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_256) {
1928                 CCP_CMD_LEN(desc) = SHA256_DIGEST_SIZE;
1929         } else if (session->auth.ut.sha_type == CCP_SHA3_TYPE_384) {
1930                 dest_addr_t += (2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE);
1931                 CCP_CMD_LEN(desc) = SHA384_DIGEST_SIZE;
1932         } else {
1933                 CCP_CMD_LEN(desc) = SHA512_DIGEST_SIZE;
1934         }
1935
1936         CCP_CMD_SRC_LO(desc) = ((uint32_t)dest_addr_t);
1937         CCP_CMD_SRC_HI(desc) = high32_value(dest_addr_t);
1938         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1939
1940         CCP_CMD_DST_LO(desc) = (uint32_t)dest_addr;
1941         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
1942         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1943
1944         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
1945         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
1946         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
1947
1948         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
1949
1950         rte_wmb();
1951         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
1952         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
1953         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
1954                       cmd_q->qcontrol | CMD_Q_RUN);
1955
1956         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1957         return 0;
1958 }
1959
1960 static int
1961 ccp_perform_sha3(struct rte_crypto_op *op,
1962                  struct ccp_queue *cmd_q)
1963 {
1964         struct ccp_session *session;
1965         union ccp_function function;
1966         struct ccp_desc *desc;
1967         uint8_t *ctx_addr, *append_ptr;
1968         uint32_t tail;
1969         phys_addr_t src_addr, dest_addr, ctx_paddr;
1970
1971         session = (struct ccp_session *)get_sym_session_private_data(
1972                                          op->sym->session,
1973                                         ccp_cryptodev_driver_id);
1974
1975         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
1976                                               op->sym->auth.data.offset);
1977         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
1978                                                 session->auth.ctx_len);
1979         if (!append_ptr) {
1980                 CCP_LOG_ERR("CCP MBUF append failed\n");
1981                 return -1;
1982         }
1983         dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
1984         ctx_addr = session->auth.sha3_ctx;
1985         ctx_paddr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
1986
1987         desc = &cmd_q->qbase_desc[cmd_q->qidx];
1988         memset(desc, 0, Q_DESC_SIZE);
1989
1990         /* prepare desc for SHA3 operation */
1991         CCP_CMD_ENGINE(desc) = CCP_ENGINE_SHA;
1992         CCP_CMD_INIT(desc) = 1;
1993         CCP_CMD_EOM(desc) = 1;
1994
1995         function.raw = 0;
1996         CCP_SHA_TYPE(&function) = session->auth.ut.sha_type;
1997         CCP_CMD_FUNCTION(desc) = function.raw;
1998
1999         CCP_CMD_LEN(desc) = op->sym->auth.data.length;
2000
2001         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2002         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2003         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2004
2005         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2006         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2007         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2008
2009         CCP_CMD_KEY_LO(desc) = ((uint32_t)ctx_paddr);
2010         CCP_CMD_KEY_HI(desc) = high32_value(ctx_paddr);
2011         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2012
2013         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2014
2015         rte_wmb();
2016
2017         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2018         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2019         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2020                       cmd_q->qcontrol | CMD_Q_RUN);
2021
2022         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2023         return 0;
2024 }
2025
2026 static int
2027 ccp_perform_aes_cmac(struct rte_crypto_op *op,
2028                      struct ccp_queue *cmd_q)
2029 {
2030         struct ccp_session *session;
2031         union ccp_function function;
2032         struct ccp_passthru pst;
2033         struct ccp_desc *desc;
2034         uint32_t tail;
2035         uint8_t *src_tb, *append_ptr, *ctx_addr;
2036         phys_addr_t src_addr, dest_addr, key_addr;
2037         int length, non_align_len;
2038
2039         session = (struct ccp_session *)get_sym_session_private_data(
2040                                          op->sym->session,
2041                                         ccp_cryptodev_driver_id);
2042         key_addr = rte_mem_virt2phy(session->auth.key_ccp);
2043
2044         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2045                                               op->sym->auth.data.offset);
2046         append_ptr = (uint8_t *)rte_pktmbuf_append(op->sym->m_src,
2047                                                 session->auth.ctx_len);
2048         dest_addr = (phys_addr_t)rte_mem_virt2phy((void *)append_ptr);
2049
2050         function.raw = 0;
2051         CCP_AES_ENCRYPT(&function) = CCP_CIPHER_DIR_ENCRYPT;
2052         CCP_AES_MODE(&function) = session->auth.um.aes_mode;
2053         CCP_AES_TYPE(&function) = session->auth.ut.aes_type;
2054
2055         if (op->sym->auth.data.length % session->auth.block_size == 0) {
2056
2057                 ctx_addr = session->auth.pre_compute;
2058                 memset(ctx_addr, 0, AES_BLOCK_SIZE);
2059                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
2060                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2061                 pst.len = CCP_SB_BYTES;
2062                 pst.dir = 1;
2063                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2064                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2065                 ccp_perform_passthru(&pst, cmd_q);
2066
2067                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2068                 memset(desc, 0, Q_DESC_SIZE);
2069
2070                 /* prepare desc for aes-cmac command */
2071                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2072                 CCP_CMD_EOM(desc) = 1;
2073                 CCP_CMD_FUNCTION(desc) = function.raw;
2074
2075                 CCP_CMD_LEN(desc) = op->sym->auth.data.length;
2076                 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2077                 CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2078                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2079
2080                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2081                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2082                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2083                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2084
2085                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2086
2087                 rte_wmb();
2088
2089                 tail =
2090                 (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2091                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2092                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2093                               cmd_q->qcontrol | CMD_Q_RUN);
2094         } else {
2095                 ctx_addr = session->auth.pre_compute + CCP_SB_BYTES;
2096                 memset(ctx_addr, 0, AES_BLOCK_SIZE);
2097                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *)ctx_addr);
2098                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2099                 pst.len = CCP_SB_BYTES;
2100                 pst.dir = 1;
2101                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2102                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2103                 ccp_perform_passthru(&pst, cmd_q);
2104
2105                 length = (op->sym->auth.data.length / AES_BLOCK_SIZE);
2106                 length *= AES_BLOCK_SIZE;
2107                 non_align_len = op->sym->auth.data.length - length;
2108                 /* prepare desc for aes-cmac command */
2109                 /*Command 1*/
2110                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2111                 memset(desc, 0, Q_DESC_SIZE);
2112
2113                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2114                 CCP_CMD_INIT(desc) = 1;
2115                 CCP_CMD_FUNCTION(desc) = function.raw;
2116
2117                 CCP_CMD_LEN(desc) = length;
2118                 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2119                 CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2120                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2121
2122                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2123                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2124                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2125                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2126
2127                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2128
2129                 /*Command 2*/
2130                 append_ptr = append_ptr + CCP_SB_BYTES;
2131                 memset(append_ptr, 0, AES_BLOCK_SIZE);
2132                 src_tb = rte_pktmbuf_mtod_offset(op->sym->m_src,
2133                                                  uint8_t *,
2134                                                  op->sym->auth.data.offset +
2135                                                  length);
2136                 rte_memcpy(append_ptr, src_tb, non_align_len);
2137                 append_ptr[non_align_len] = CMAC_PAD_VALUE;
2138
2139                 desc = &cmd_q->qbase_desc[cmd_q->qidx];
2140                 memset(desc, 0, Q_DESC_SIZE);
2141
2142                 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2143                 CCP_CMD_EOM(desc) = 1;
2144                 CCP_CMD_FUNCTION(desc) = function.raw;
2145                 CCP_CMD_LEN(desc) = AES_BLOCK_SIZE;
2146
2147                 CCP_CMD_SRC_LO(desc) = ((uint32_t)(dest_addr + CCP_SB_BYTES));
2148                 CCP_CMD_SRC_HI(desc) = high32_value(dest_addr + CCP_SB_BYTES);
2149                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2150
2151                 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2152                 CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2153                 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2154                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2155
2156                 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2157
2158                 rte_wmb();
2159                 tail =
2160                 (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2161                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2162                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2163                               cmd_q->qcontrol | CMD_Q_RUN);
2164         }
2165         /* Retrieve result */
2166         pst.dest_addr = dest_addr;
2167         pst.src_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2168         pst.len = CCP_SB_BYTES;
2169         pst.dir = 0;
2170         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2171         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2172         ccp_perform_passthru(&pst, cmd_q);
2173
2174         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2175         return 0;
2176 }
2177
2178 static int
2179 ccp_perform_aes(struct rte_crypto_op *op,
2180                 struct ccp_queue *cmd_q,
2181                 struct ccp_batch_info *b_info)
2182 {
2183         struct ccp_session *session;
2184         union ccp_function function;
2185         uint8_t *lsb_buf;
2186         struct ccp_passthru pst = {0};
2187         struct ccp_desc *desc;
2188         phys_addr_t src_addr, dest_addr, key_addr;
2189         uint8_t *iv;
2190
2191         session = (struct ccp_session *)get_sym_session_private_data(
2192                                          op->sym->session,
2193                                         ccp_cryptodev_driver_id);
2194         function.raw = 0;
2195
2196         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2197         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB) {
2198                 if (session->cipher.um.aes_mode == CCP_AES_MODE_CTR) {
2199                         rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE,
2200                                    iv, session->iv.length);
2201                         pst.src_addr = (phys_addr_t)session->cipher.nonce_phys;
2202                         CCP_AES_SIZE(&function) = 0x1F;
2203                 } else {
2204                         lsb_buf =
2205                         &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]);
2206                         rte_memcpy(lsb_buf +
2207                                    (CCP_SB_BYTES - session->iv.length),
2208                                    iv, session->iv.length);
2209                         pst.src_addr = b_info->lsb_buf_phys +
2210                                 (b_info->lsb_buf_idx * CCP_SB_BYTES);
2211                         b_info->lsb_buf_idx++;
2212                 }
2213
2214                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2215                 pst.len = CCP_SB_BYTES;
2216                 pst.dir = 1;
2217                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2218                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2219                 ccp_perform_passthru(&pst, cmd_q);
2220         }
2221
2222         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2223
2224         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2225                                               op->sym->cipher.data.offset);
2226         if (likely(op->sym->m_dst != NULL))
2227                 dest_addr = rte_pktmbuf_mtophys_offset(op->sym->m_dst,
2228                                                 op->sym->cipher.data.offset);
2229         else
2230                 dest_addr = src_addr;
2231         key_addr = session->cipher.key_phys;
2232
2233         /* prepare desc for aes command */
2234         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2235         CCP_CMD_INIT(desc) = 1;
2236         CCP_CMD_EOM(desc) = 1;
2237
2238         CCP_AES_ENCRYPT(&function) = session->cipher.dir;
2239         CCP_AES_MODE(&function) = session->cipher.um.aes_mode;
2240         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2241         CCP_CMD_FUNCTION(desc) = function.raw;
2242
2243         CCP_CMD_LEN(desc) = op->sym->cipher.data.length;
2244
2245         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2246         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2247         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2248
2249         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2250         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2251         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2252
2253         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2254         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2255         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2256
2257         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB)
2258                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2259
2260         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2261         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2262         return 0;
2263 }
2264
2265 static int
2266 ccp_perform_3des(struct rte_crypto_op *op,
2267                 struct ccp_queue *cmd_q,
2268                 struct ccp_batch_info *b_info)
2269 {
2270         struct ccp_session *session;
2271         union ccp_function function;
2272         unsigned char *lsb_buf;
2273         struct ccp_passthru pst;
2274         struct ccp_desc *desc;
2275         uint32_t tail;
2276         uint8_t *iv;
2277         phys_addr_t src_addr, dest_addr, key_addr;
2278
2279         session = (struct ccp_session *)get_sym_session_private_data(
2280                                          op->sym->session,
2281                                         ccp_cryptodev_driver_id);
2282
2283         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2284         switch (session->cipher.um.des_mode) {
2285         case CCP_DES_MODE_CBC:
2286                 lsb_buf = &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]);
2287                 b_info->lsb_buf_idx++;
2288
2289                 rte_memcpy(lsb_buf + (CCP_SB_BYTES - session->iv.length),
2290                            iv, session->iv.length);
2291
2292                 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *) lsb_buf);
2293                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2294                 pst.len = CCP_SB_BYTES;
2295                 pst.dir = 1;
2296                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2297                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
2298                 ccp_perform_passthru(&pst, cmd_q);
2299                 break;
2300         case CCP_DES_MODE_CFB:
2301         case CCP_DES_MODE_ECB:
2302                 CCP_LOG_ERR("Unsupported DES cipher mode");
2303                 return -ENOTSUP;
2304         }
2305
2306         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2307                                               op->sym->cipher.data.offset);
2308         if (unlikely(op->sym->m_dst != NULL))
2309                 dest_addr =
2310                         rte_pktmbuf_mtophys_offset(op->sym->m_dst,
2311                                                    op->sym->cipher.data.offset);
2312         else
2313                 dest_addr = src_addr;
2314
2315         key_addr = rte_mem_virt2phy(session->cipher.key_ccp);
2316
2317         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2318
2319         memset(desc, 0, Q_DESC_SIZE);
2320
2321         /* prepare desc for des command */
2322         CCP_CMD_ENGINE(desc) = CCP_ENGINE_3DES;
2323
2324         CCP_CMD_SOC(desc) = 0;
2325         CCP_CMD_IOC(desc) = 0;
2326         CCP_CMD_INIT(desc) = 1;
2327         CCP_CMD_EOM(desc) = 1;
2328         CCP_CMD_PROT(desc) = 0;
2329
2330         function.raw = 0;
2331         CCP_DES_ENCRYPT(&function) = session->cipher.dir;
2332         CCP_DES_MODE(&function) = session->cipher.um.des_mode;
2333         CCP_DES_TYPE(&function) = session->cipher.ut.des_type;
2334         CCP_CMD_FUNCTION(desc) = function.raw;
2335
2336         CCP_CMD_LEN(desc) = op->sym->cipher.data.length;
2337
2338         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2339         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2340         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2341
2342         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2343         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2344         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2345
2346         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2347         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2348         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2349
2350         if (session->cipher.um.des_mode)
2351                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2352
2353         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2354
2355         rte_wmb();
2356
2357         /* Write the new tail address back to the queue register */
2358         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2359         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2360         /* Turn the queue back on using our cached control register */
2361         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2362                       cmd_q->qcontrol | CMD_Q_RUN);
2363
2364         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2365         return 0;
2366 }
2367
2368 static int
2369 ccp_perform_aes_gcm(struct rte_crypto_op *op, struct ccp_queue *cmd_q)
2370 {
2371         struct ccp_session *session;
2372         union ccp_function function;
2373         uint8_t *iv;
2374         struct ccp_passthru pst;
2375         struct ccp_desc *desc;
2376         uint32_t tail;
2377         uint64_t *temp;
2378         phys_addr_t src_addr, dest_addr, key_addr, aad_addr;
2379         phys_addr_t digest_dest_addr;
2380         int length, non_align_len;
2381
2382         session = (struct ccp_session *)get_sym_session_private_data(
2383                                          op->sym->session,
2384                                          ccp_cryptodev_driver_id);
2385         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
2386         key_addr = session->cipher.key_phys;
2387
2388         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
2389                                               op->sym->aead.data.offset);
2390         if (unlikely(op->sym->m_dst != NULL))
2391                 dest_addr = rte_pktmbuf_mtophys_offset(op->sym->m_dst,
2392                                                 op->sym->aead.data.offset);
2393         else
2394                 dest_addr = src_addr;
2395         rte_pktmbuf_append(op->sym->m_src, session->auth.ctx_len);
2396         digest_dest_addr = op->sym->aead.digest.phys_addr;
2397         temp = (uint64_t *)(op->sym->aead.digest.data + AES_BLOCK_SIZE);
2398         *temp++ = rte_bswap64(session->auth.aad_length << 3);
2399         *temp = rte_bswap64(op->sym->aead.data.length << 3);
2400
2401         non_align_len = op->sym->aead.data.length % AES_BLOCK_SIZE;
2402         length = CCP_ALIGN(op->sym->aead.data.length, AES_BLOCK_SIZE);
2403
2404         aad_addr = op->sym->aead.aad.phys_addr;
2405
2406         /* CMD1 IV Passthru */
2407         rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE, iv,
2408                    session->iv.length);
2409         pst.src_addr = session->cipher.nonce_phys;
2410         pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2411         pst.len = CCP_SB_BYTES;
2412         pst.dir = 1;
2413         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2414         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2415         ccp_perform_passthru(&pst, cmd_q);
2416
2417         /* CMD2 GHASH-AAD */
2418         function.raw = 0;
2419         CCP_AES_ENCRYPT(&function) = CCP_AES_MODE_GHASH_AAD;
2420         CCP_AES_MODE(&function) = CCP_AES_MODE_GHASH;
2421         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2422
2423         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2424         memset(desc, 0, Q_DESC_SIZE);
2425
2426         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2427         CCP_CMD_INIT(desc) = 1;
2428         CCP_CMD_FUNCTION(desc) = function.raw;
2429
2430         CCP_CMD_LEN(desc) = session->auth.aad_length;
2431
2432         CCP_CMD_SRC_LO(desc) = ((uint32_t)aad_addr);
2433         CCP_CMD_SRC_HI(desc) = high32_value(aad_addr);
2434         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2435
2436         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2437         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2438         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2439
2440         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2441
2442         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2443         rte_wmb();
2444
2445         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2446         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2447         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2448                       cmd_q->qcontrol | CMD_Q_RUN);
2449
2450         /* CMD3 : GCTR Plain text */
2451         function.raw = 0;
2452         CCP_AES_ENCRYPT(&function) = session->cipher.dir;
2453         CCP_AES_MODE(&function) = CCP_AES_MODE_GCTR;
2454         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2455         if (non_align_len == 0)
2456                 CCP_AES_SIZE(&function) = (AES_BLOCK_SIZE << 3) - 1;
2457         else
2458                 CCP_AES_SIZE(&function) = (non_align_len << 3) - 1;
2459
2460
2461         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2462         memset(desc, 0, Q_DESC_SIZE);
2463
2464         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2465         CCP_CMD_EOM(desc) = 1;
2466         CCP_CMD_FUNCTION(desc) = function.raw;
2467
2468         CCP_CMD_LEN(desc) = length;
2469
2470         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
2471         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
2472         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2473
2474         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
2475         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
2476         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2477
2478         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2479         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2480         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2481
2482         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2483
2484         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2485         rte_wmb();
2486
2487         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2488         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2489         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2490                       cmd_q->qcontrol | CMD_Q_RUN);
2491
2492         /* CMD4 : PT to copy IV */
2493         pst.src_addr = session->cipher.nonce_phys;
2494         pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
2495         pst.len = AES_BLOCK_SIZE;
2496         pst.dir = 1;
2497         pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
2498         pst.byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
2499         ccp_perform_passthru(&pst, cmd_q);
2500
2501         /* CMD5 : GHASH-Final */
2502         function.raw = 0;
2503         CCP_AES_ENCRYPT(&function) = CCP_AES_MODE_GHASH_FINAL;
2504         CCP_AES_MODE(&function) = CCP_AES_MODE_GHASH;
2505         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
2506
2507         desc = &cmd_q->qbase_desc[cmd_q->qidx];
2508         memset(desc, 0, Q_DESC_SIZE);
2509
2510         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
2511         CCP_CMD_FUNCTION(desc) = function.raw;
2512         /* Last block (AAD_len || PT_len)*/
2513         CCP_CMD_LEN(desc) = AES_BLOCK_SIZE;
2514
2515         CCP_CMD_SRC_LO(desc) = ((uint32_t)digest_dest_addr + AES_BLOCK_SIZE);
2516         CCP_CMD_SRC_HI(desc) = high32_value(digest_dest_addr + AES_BLOCK_SIZE);
2517         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2518
2519         CCP_CMD_DST_LO(desc) = ((uint32_t)digest_dest_addr);
2520         CCP_CMD_DST_HI(desc) = high32_value(digest_dest_addr);
2521         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2522
2523         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
2524         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
2525         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
2526
2527         CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
2528
2529         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
2530         rte_wmb();
2531
2532         tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE);
2533         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail);
2534         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2535                       cmd_q->qcontrol | CMD_Q_RUN);
2536
2537         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2538         return 0;
2539 }
2540
2541 static inline int
2542 ccp_crypto_cipher(struct rte_crypto_op *op,
2543                   struct ccp_queue *cmd_q,
2544                   struct ccp_batch_info *b_info)
2545 {
2546         int result = 0;
2547         struct ccp_session *session;
2548
2549         session = (struct ccp_session *)get_sym_session_private_data(
2550                                          op->sym->session,
2551                                          ccp_cryptodev_driver_id);
2552
2553         switch (session->cipher.algo) {
2554         case CCP_CIPHER_ALGO_AES_CBC:
2555                 result = ccp_perform_aes(op, cmd_q, b_info);
2556                 b_info->desccnt += 2;
2557                 break;
2558         case CCP_CIPHER_ALGO_AES_CTR:
2559                 result = ccp_perform_aes(op, cmd_q, b_info);
2560                 b_info->desccnt += 2;
2561                 break;
2562         case CCP_CIPHER_ALGO_AES_ECB:
2563                 result = ccp_perform_aes(op, cmd_q, b_info);
2564                 b_info->desccnt += 1;
2565                 break;
2566         case CCP_CIPHER_ALGO_3DES_CBC:
2567                 result = ccp_perform_3des(op, cmd_q, b_info);
2568                 b_info->desccnt += 2;
2569                 break;
2570         default:
2571                 CCP_LOG_ERR("Unsupported cipher algo %d",
2572                             session->cipher.algo);
2573                 return -ENOTSUP;
2574         }
2575         return result;
2576 }
2577
2578 static inline int
2579 ccp_crypto_auth(struct rte_crypto_op *op,
2580                 struct ccp_queue *cmd_q,
2581                 struct ccp_batch_info *b_info)
2582 {
2583
2584         int result = 0;
2585         struct ccp_session *session;
2586
2587         session = (struct ccp_session *)get_sym_session_private_data(
2588                                          op->sym->session,
2589                                         ccp_cryptodev_driver_id);
2590
2591         switch (session->auth.algo) {
2592         case CCP_AUTH_ALGO_SHA1:
2593         case CCP_AUTH_ALGO_SHA224:
2594         case CCP_AUTH_ALGO_SHA256:
2595         case CCP_AUTH_ALGO_SHA384:
2596         case CCP_AUTH_ALGO_SHA512:
2597                 result = ccp_perform_sha(op, cmd_q);
2598                 b_info->desccnt += 3;
2599                 break;
2600         case CCP_AUTH_ALGO_MD5_HMAC:
2601                 if (session->auth_opt == 0)
2602                         result = -1;
2603                 break;
2604         case CCP_AUTH_ALGO_SHA1_HMAC:
2605         case CCP_AUTH_ALGO_SHA224_HMAC:
2606         case CCP_AUTH_ALGO_SHA256_HMAC:
2607                 if (session->auth_opt == 0) {
2608                         result = ccp_perform_hmac(op, cmd_q);
2609                         b_info->desccnt += 6;
2610                 }
2611                 break;
2612         case CCP_AUTH_ALGO_SHA384_HMAC:
2613         case CCP_AUTH_ALGO_SHA512_HMAC:
2614                 if (session->auth_opt == 0) {
2615                         result = ccp_perform_hmac(op, cmd_q);
2616                         b_info->desccnt += 7;
2617                 }
2618                 break;
2619         case CCP_AUTH_ALGO_SHA3_224:
2620         case CCP_AUTH_ALGO_SHA3_256:
2621         case CCP_AUTH_ALGO_SHA3_384:
2622         case CCP_AUTH_ALGO_SHA3_512:
2623                 result = ccp_perform_sha3(op, cmd_q);
2624                 b_info->desccnt += 1;
2625                 break;
2626         case CCP_AUTH_ALGO_SHA3_224_HMAC:
2627         case CCP_AUTH_ALGO_SHA3_256_HMAC:
2628                 result = ccp_perform_sha3_hmac(op, cmd_q);
2629                 b_info->desccnt += 3;
2630                 break;
2631         case CCP_AUTH_ALGO_SHA3_384_HMAC:
2632         case CCP_AUTH_ALGO_SHA3_512_HMAC:
2633                 result = ccp_perform_sha3_hmac(op, cmd_q);
2634                 b_info->desccnt += 4;
2635                 break;
2636         case CCP_AUTH_ALGO_AES_CMAC:
2637                 result = ccp_perform_aes_cmac(op, cmd_q);
2638                 b_info->desccnt += 4;
2639                 break;
2640         default:
2641                 CCP_LOG_ERR("Unsupported auth algo %d",
2642                             session->auth.algo);
2643                 return -ENOTSUP;
2644         }
2645
2646         return result;
2647 }
2648
2649 static inline int
2650 ccp_crypto_aead(struct rte_crypto_op *op,
2651                 struct ccp_queue *cmd_q,
2652                 struct ccp_batch_info *b_info)
2653 {
2654         int result = 0;
2655         struct ccp_session *session;
2656
2657         session = (struct ccp_session *)get_sym_session_private_data(
2658                                         op->sym->session,
2659                                         ccp_cryptodev_driver_id);
2660
2661         switch (session->auth.algo) {
2662         case CCP_AUTH_ALGO_AES_GCM:
2663                 if (session->cipher.algo != CCP_CIPHER_ALGO_AES_GCM) {
2664                         CCP_LOG_ERR("Incorrect chain order");
2665                         return -1;
2666                 }
2667                 result = ccp_perform_aes_gcm(op, cmd_q);
2668                 b_info->desccnt += 5;
2669                 break;
2670         default:
2671                 CCP_LOG_ERR("Unsupported aead algo %d",
2672                             session->aead_algo);
2673                 return -ENOTSUP;
2674         }
2675         return result;
2676 }
2677
2678 int
2679 process_ops_to_enqueue(struct ccp_qp *qp,
2680                        struct rte_crypto_op **op,
2681                        struct ccp_queue *cmd_q,
2682                        uint16_t nb_ops,
2683                        int slots_req)
2684 {
2685         int i, result = 0;
2686         struct ccp_batch_info *b_info;
2687         struct ccp_session *session;
2688         EVP_MD_CTX *auth_ctx = NULL;
2689
2690         if (rte_mempool_get(qp->batch_mp, (void **)&b_info)) {
2691                 CCP_LOG_ERR("batch info allocation failed");
2692                 return 0;
2693         }
2694
2695         auth_ctx = EVP_MD_CTX_create();
2696         if (unlikely(!auth_ctx)) {
2697                 CCP_LOG_ERR("Unable to create auth ctx");
2698                 return 0;
2699         }
2700         b_info->auth_ctr = 0;
2701
2702         /* populate batch info necessary for dequeue */
2703         b_info->op_idx = 0;
2704         b_info->lsb_buf_idx = 0;
2705         b_info->desccnt = 0;
2706         b_info->cmd_q = cmd_q;
2707         b_info->lsb_buf_phys =
2708                 (phys_addr_t)rte_mem_virt2phy((void *)b_info->lsb_buf);
2709         rte_atomic64_sub(&b_info->cmd_q->free_slots, slots_req);
2710
2711         b_info->head_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
2712                                          Q_DESC_SIZE);
2713         for (i = 0; i < nb_ops; i++) {
2714                 session = (struct ccp_session *)get_sym_session_private_data(
2715                                                  op[i]->sym->session,
2716                                                  ccp_cryptodev_driver_id);
2717                 switch (session->cmd_id) {
2718                 case CCP_CMD_CIPHER:
2719                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2720                         break;
2721                 case CCP_CMD_AUTH:
2722                         if (session->auth_opt) {
2723                                 b_info->auth_ctr++;
2724                                 result = cpu_crypto_auth(qp, op[i],
2725                                                          session, auth_ctx);
2726                         } else
2727                                 result = ccp_crypto_auth(op[i], cmd_q, b_info);
2728                         break;
2729                 case CCP_CMD_CIPHER_HASH:
2730                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2731                         if (result)
2732                                 break;
2733                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
2734                         break;
2735                 case CCP_CMD_HASH_CIPHER:
2736                         if (session->auth_opt) {
2737                                 result = cpu_crypto_auth(qp, op[i],
2738                                                          session, auth_ctx);
2739                                 if (op[i]->status !=
2740                                     RTE_CRYPTO_OP_STATUS_SUCCESS)
2741                                         continue;
2742                         } else
2743                                 result = ccp_crypto_auth(op[i], cmd_q, b_info);
2744
2745                         if (result)
2746                                 break;
2747                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
2748                         break;
2749                 case CCP_CMD_COMBINED:
2750                         result = ccp_crypto_aead(op[i], cmd_q, b_info);
2751                         break;
2752                 default:
2753                         CCP_LOG_ERR("Unsupported cmd_id");
2754                         result = -1;
2755                 }
2756                 if (unlikely(result < 0)) {
2757                         rte_atomic64_add(&b_info->cmd_q->free_slots,
2758                                          (slots_req - b_info->desccnt));
2759                         break;
2760                 }
2761                 b_info->op[i] = op[i];
2762         }
2763
2764         b_info->opcnt = i;
2765         b_info->tail_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
2766                                          Q_DESC_SIZE);
2767
2768         rte_wmb();
2769         /* Write the new tail address back to the queue register */
2770         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
2771                               b_info->tail_offset);
2772         /* Turn the queue back on using our cached control register */
2773         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
2774                               cmd_q->qcontrol | CMD_Q_RUN);
2775
2776         rte_ring_enqueue(qp->processed_pkts, (void *)b_info);
2777
2778         EVP_MD_CTX_destroy(auth_ctx);
2779         return i;
2780 }
2781
2782 static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op)
2783 {
2784         struct ccp_session *session;
2785         uint8_t *digest_data, *addr;
2786         struct rte_mbuf *m_last;
2787         int offset, digest_offset;
2788         uint8_t digest_le[64];
2789
2790         session = (struct ccp_session *)get_sym_session_private_data(
2791                                          op->sym->session,
2792                                         ccp_cryptodev_driver_id);
2793
2794         if (session->cmd_id == CCP_CMD_COMBINED) {
2795                 digest_data = op->sym->aead.digest.data;
2796                 digest_offset = op->sym->aead.data.offset +
2797                                         op->sym->aead.data.length;
2798         } else {
2799                 digest_data = op->sym->auth.digest.data;
2800                 digest_offset = op->sym->auth.data.offset +
2801                                         op->sym->auth.data.length;
2802         }
2803         m_last = rte_pktmbuf_lastseg(op->sym->m_src);
2804         addr = (uint8_t *)((char *)m_last->buf_addr + m_last->data_off +
2805                            m_last->data_len - session->auth.ctx_len);
2806
2807         rte_mb();
2808         offset = session->auth.offset;
2809
2810         if (session->auth.engine == CCP_ENGINE_SHA)
2811                 if ((session->auth.ut.sha_type != CCP_SHA_TYPE_1) &&
2812                     (session->auth.ut.sha_type != CCP_SHA_TYPE_224) &&
2813                     (session->auth.ut.sha_type != CCP_SHA_TYPE_256)) {
2814                         /* All other algorithms require byte
2815                          * swap done by host
2816                          */
2817                         unsigned int i;
2818
2819                         offset = session->auth.ctx_len -
2820                                 session->auth.offset - 1;
2821                         for (i = 0; i < session->auth.digest_length; i++)
2822                                 digest_le[i] = addr[offset - i];
2823                         offset = 0;
2824                         addr = digest_le;
2825                 }
2826
2827         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2828         if (session->auth.op == CCP_AUTH_OP_VERIFY) {
2829                 if (memcmp(addr + offset, digest_data,
2830                            session->auth.digest_length) != 0)
2831                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
2832
2833         } else {
2834                 if (unlikely(digest_data == 0))
2835                         digest_data = rte_pktmbuf_mtod_offset(
2836                                         op->sym->m_dst, uint8_t *,
2837                                         digest_offset);
2838                 rte_memcpy(digest_data, addr + offset,
2839                            session->auth.digest_length);
2840         }
2841         /* Trim area used for digest from mbuf. */
2842         rte_pktmbuf_trim(op->sym->m_src,
2843                          session->auth.ctx_len);
2844 }
2845
2846 static int
2847 ccp_prepare_ops(struct ccp_qp *qp,
2848                 struct rte_crypto_op **op_d,
2849                 struct ccp_batch_info *b_info,
2850                 uint16_t nb_ops)
2851 {
2852         int i, min_ops;
2853         struct ccp_session *session;
2854
2855         EVP_MD_CTX *auth_ctx = NULL;
2856
2857         auth_ctx = EVP_MD_CTX_create();
2858         if (unlikely(!auth_ctx)) {
2859                 CCP_LOG_ERR("Unable to create auth ctx");
2860                 return 0;
2861         }
2862         min_ops = RTE_MIN(nb_ops, b_info->opcnt);
2863
2864         for (i = 0; i < min_ops; i++) {
2865                 op_d[i] = b_info->op[b_info->op_idx++];
2866                 session = (struct ccp_session *)get_sym_session_private_data(
2867                                                  op_d[i]->sym->session,
2868                                                 ccp_cryptodev_driver_id);
2869                 switch (session->cmd_id) {
2870                 case CCP_CMD_CIPHER:
2871                         op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2872                         break;
2873                 case CCP_CMD_AUTH:
2874                         if (session->auth_opt == 0)
2875                                 ccp_auth_dq_prepare(op_d[i]);
2876                         break;
2877                 case CCP_CMD_CIPHER_HASH:
2878                         if (session->auth_opt)
2879                                 cpu_crypto_auth(qp, op_d[i],
2880                                                 session, auth_ctx);
2881                         else
2882                                 ccp_auth_dq_prepare(op_d[i]);
2883                         break;
2884                 case CCP_CMD_HASH_CIPHER:
2885                         if (session->auth_opt)
2886                                 op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2887                         else
2888                                 ccp_auth_dq_prepare(op_d[i]);
2889                         break;
2890                 case CCP_CMD_COMBINED:
2891                         ccp_auth_dq_prepare(op_d[i]);
2892                         break;
2893                 default:
2894                         CCP_LOG_ERR("Unsupported cmd_id");
2895                 }
2896         }
2897
2898         EVP_MD_CTX_destroy(auth_ctx);
2899         b_info->opcnt -= min_ops;
2900         return min_ops;
2901 }
2902
2903 int
2904 process_ops_to_dequeue(struct ccp_qp *qp,
2905                        struct rte_crypto_op **op,
2906                        uint16_t nb_ops)
2907 {
2908         struct ccp_batch_info *b_info;
2909         uint32_t cur_head_offset;
2910
2911         if (qp->b_info != NULL) {
2912                 b_info = qp->b_info;
2913                 if (unlikely(b_info->op_idx > 0))
2914                         goto success;
2915         } else if (rte_ring_dequeue(qp->processed_pkts,
2916                                     (void **)&b_info))
2917                 return 0;
2918
2919         if (b_info->auth_ctr == b_info->opcnt)
2920                 goto success;
2921         cur_head_offset = CCP_READ_REG(b_info->cmd_q->reg_base,
2922                                        CMD_Q_HEAD_LO_BASE);
2923
2924         if (b_info->head_offset < b_info->tail_offset) {
2925                 if ((cur_head_offset >= b_info->head_offset) &&
2926                     (cur_head_offset < b_info->tail_offset)) {
2927                         qp->b_info = b_info;
2928                         return 0;
2929                 }
2930         } else {
2931                 if ((cur_head_offset >= b_info->head_offset) ||
2932                     (cur_head_offset < b_info->tail_offset)) {
2933                         qp->b_info = b_info;
2934                         return 0;
2935                 }
2936         }
2937
2938
2939 success:
2940         nb_ops = ccp_prepare_ops(qp, op, b_info, nb_ops);
2941         rte_atomic64_add(&b_info->cmd_q->free_slots, b_info->desccnt);
2942         b_info->desccnt = 0;
2943         if (b_info->opcnt > 0) {
2944                 qp->b_info = b_info;
2945         } else {
2946                 rte_mempool_put(qp->batch_mp, (void *)b_info);
2947                 qp->b_info = NULL;
2948         }
2949
2950         return nb_ops;
2951 }