drivers/crypto/qat/qat_sym_session.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4 #include <rte_memcpy.h>
   5 #include <rte_common.h>
   6 #include <rte_spinlock.h>
   7 #include <rte_byteorder.h>
   8 #include <rte_log.h>
   9 #include <rte_malloc.h>
  10 #include <rte_crypto_sym.h>
  11
  12 #include "qat_logs.h"
  13 #include "qat_device.h"
  14
  15 #include <openssl/sha.h>        /* Needed to calculate pre-compute values */
  16 #include <openssl/aes.h>        /* Needed to calculate pre-compute values */
  17 #include <openssl/md5.h>        /* Needed to calculate pre-compute values */
  18 #include <openssl/evp.h>
  19
  20 #include "qat_sym_session.h"
  21
  22 /** Frees a context previously created
  23  *  Depends on openssl libcrypto
  24  */
  25 static void
  26 bpi_cipher_ctx_free(void *bpi_ctx)
  27 {
  28         if (bpi_ctx != NULL)
  29                 EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)bpi_ctx);
  30 }
  31
  32 static int
  33 bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
  34                 enum rte_crypto_cipher_operation direction __rte_unused,
  35                 uint8_t *key, void **ctx)
  36 {
  37         const EVP_CIPHER *algo = NULL;
  38         int ret;
  39         *ctx = EVP_CIPHER_CTX_new();
  40
  41         if (*ctx == NULL) {
  42                 ret = -ENOMEM;
  43                 goto ctx_init_err;
  44         }
  45
  46         if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI)
  47                 algo = EVP_des_ecb();
  48         else
  49                 algo = EVP_aes_128_ecb();
  50
  51         /* IV will be ECB encrypted whether direction is encrypt or decrypt*/
  52         if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) {
  53                 ret = -EINVAL;
  54                 goto ctx_init_err;
  55         }
  56
  57         return 0;
  58
  59 ctx_init_err:
  60         if (*ctx != NULL)
  61                 EVP_CIPHER_CTX_free(*ctx);
  62         return ret;
  63 }
  64
  65 static int
  66 qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
  67                 struct qat_pmd_private *internals)
  68 {
  69         int i = 0;
  70         const struct rte_cryptodev_capabilities *capability;
  71
  72         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
  73                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
  74                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
  75                         continue;
  76
  77                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
  78                         continue;
  79
  80                 if (capability->sym.cipher.algo == algo)
  81                         return 1;
  82         }
  83         return 0;
  84 }
  85
  86 static int
  87 qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
  88                 struct qat_pmd_private *internals)
  89 {
  90         int i = 0;
  91         const struct rte_cryptodev_capabilities *capability;
  92
  93         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
  94                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
  95                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
  96                         continue;
  97
  98                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
  99                         continue;
 100
 101                 if (capability->sym.auth.algo == algo)
 102                         return 1;
 103         }
 104         return 0;
 105 }
 106
 107 void
 108 qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 109                 struct rte_cryptodev_sym_session *sess)
 110 {
 111         PMD_INIT_FUNC_TRACE();
 112         uint8_t index = dev->driver_id;
 113         void *sess_priv = get_session_private_data(sess, index);
 114         struct qat_session *s = (struct qat_session *)sess_priv;
 115
 116         if (sess_priv) {
 117                 if (s->bpi_ctx)
 118                         bpi_cipher_ctx_free(s->bpi_ctx);
 119                 memset(s, 0, qat_crypto_sym_get_session_private_size(dev));
 120                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 121
 122                 set_session_private_data(sess, index, NULL);
 123                 rte_mempool_put(sess_mp, sess_priv);
 124         }
 125 }
 126
 127 static int
 128 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 129 {
 130         /* Cipher Only */
 131         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 132                 return ICP_QAT_FW_LA_CMD_CIPHER;
 133
 134         /* Authentication Only */
 135         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 136                 return ICP_QAT_FW_LA_CMD_AUTH;
 137
 138         /* AEAD */
 139         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 140                 /* AES-GCM and AES-CCM works with different direction
 141                  * GCM first encrypts and generate hash where AES-CCM
 142                  * first generate hash and encrypts. Similar relation
 143                  * applies to decryption.
 144                  */
 145                 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 146                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 147                                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 148                         else
 149                                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 150                 else
 151                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 152                                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 153                         else
 154                                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 155         }
 156
 157         if (xform->next == NULL)
 158                 return -1;
 159
 160         /* Cipher then Authenticate */
 161         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 162                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 163                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 164
 165         /* Authenticate then Cipher */
 166         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 167                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 168                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 169
 170         return -1;
 171 }
 172
 173 static struct rte_crypto_auth_xform *
 174 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 175 {
 176         do {
 177                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 178                         return &xform->auth;
 179
 180                 xform = xform->next;
 181         } while (xform);
 182
 183         return NULL;
 184 }
 185
 186 static struct rte_crypto_cipher_xform *
 187 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 188 {
 189         do {
 190                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 191                         return &xform->cipher;
 192
 193                 xform = xform->next;
 194         } while (xform);
 195
 196         return NULL;
 197 }
 198
 199 int
 200 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 201                 struct rte_crypto_sym_xform *xform,
 202                 struct qat_session *session)
 203 {
 204         struct qat_pmd_private *internals = dev->data->dev_private;
 205         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 206         int ret;
 207
 208         /* Get cipher xform from crypto xform chain */
 209         cipher_xform = qat_get_cipher_xform(xform);
 210
 211         session->cipher_iv.offset = cipher_xform->iv.offset;
 212         session->cipher_iv.length = cipher_xform->iv.length;
 213
 214         switch (cipher_xform->algo) {
 215         case RTE_CRYPTO_CIPHER_AES_CBC:
 216                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 217                                 &session->qat_cipher_alg) != 0) {
 218                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 219                         ret = -EINVAL;
 220                         goto error_out;
 221                 }
 222                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 223                 break;
 224         case RTE_CRYPTO_CIPHER_AES_CTR:
 225                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 226                                 &session->qat_cipher_alg) != 0) {
 227                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 228                         ret = -EINVAL;
 229                         goto error_out;
 230                 }
 231                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 232                 break;
 233         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 234                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 235                                         &session->qat_cipher_alg) != 0) {
 236                         PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size");
 237                         ret = -EINVAL;
 238                         goto error_out;
 239                 }
 240                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 241                 break;
 242         case RTE_CRYPTO_CIPHER_NULL:
 243                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 244                 break;
 245         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 246                 if (qat_alg_validate_kasumi_key(cipher_xform->key.length,
 247                                         &session->qat_cipher_alg) != 0) {
 248                         PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size");
 249                         ret = -EINVAL;
 250                         goto error_out;
 251                 }
 252                 session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE;
 253                 break;
 254         case RTE_CRYPTO_CIPHER_3DES_CBC:
 255                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 256                                 &session->qat_cipher_alg) != 0) {
 257                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 258                         ret = -EINVAL;
 259                         goto error_out;
 260                 }
 261                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 262                 break;
 263         case RTE_CRYPTO_CIPHER_DES_CBC:
 264                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 265                                 &session->qat_cipher_alg) != 0) {
 266                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 267                         ret = -EINVAL;
 268                         goto error_out;
 269                 }
 270                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 271                 break;
 272         case RTE_CRYPTO_CIPHER_3DES_CTR:
 273                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 274                                 &session->qat_cipher_alg) != 0) {
 275                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 276                         ret = -EINVAL;
 277                         goto error_out;
 278                 }
 279                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 280                 break;
 281         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 282                 ret = bpi_cipher_ctx_init(
 283                                         cipher_xform->algo,
 284                                         cipher_xform->op,
 285                                         cipher_xform->key.data,
 286                                         &session->bpi_ctx);
 287                 if (ret != 0) {
 288                         PMD_DRV_LOG(ERR, "failed to create DES BPI ctx");
 289                         goto error_out;
 290                 }
 291                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 292                                 &session->qat_cipher_alg) != 0) {
 293                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 294                         ret = -EINVAL;
 295                         goto error_out;
 296                 }
 297                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 298                 break;
 299         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 300                 ret = bpi_cipher_ctx_init(
 301                                         cipher_xform->algo,
 302                                         cipher_xform->op,
 303                                         cipher_xform->key.data,
 304                                         &session->bpi_ctx);
 305                 if (ret != 0) {
 306                         PMD_DRV_LOG(ERR, "failed to create AES BPI ctx");
 307                         goto error_out;
 308                 }
 309                 if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length,
 310                                 &session->qat_cipher_alg) != 0) {
 311                         PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size");
 312                         ret = -EINVAL;
 313                         goto error_out;
 314                 }
 315                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 316                 break;
 317         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 318                 if (!qat_is_cipher_alg_supported(
 319                         cipher_xform->algo, internals)) {
 320                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 321                                 rte_crypto_cipher_algorithm_strings
 322                                         [cipher_xform->algo]);
 323                         ret = -ENOTSUP;
 324                         goto error_out;
 325                 }
 326                 if (qat_alg_validate_zuc_key(cipher_xform->key.length,
 327                                 &session->qat_cipher_alg) != 0) {
 328                         PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size");
 329                         ret = -EINVAL;
 330                         goto error_out;
 331                 }
 332                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 333                 break;
 334         case RTE_CRYPTO_CIPHER_3DES_ECB:
 335         case RTE_CRYPTO_CIPHER_AES_ECB:
 336         case RTE_CRYPTO_CIPHER_AES_F8:
 337         case RTE_CRYPTO_CIPHER_AES_XTS:
 338         case RTE_CRYPTO_CIPHER_ARC4:
 339                 PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u",
 340                                 cipher_xform->algo);
 341                 ret = -ENOTSUP;
 342                 goto error_out;
 343         default:
 344                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 345                                 cipher_xform->algo);
 346                 ret = -EINVAL;
 347                 goto error_out;
 348         }
 349
 350         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 351                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 352         else
 353                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 354
 355         if (qat_alg_aead_session_create_content_desc_cipher(session,
 356                                                 cipher_xform->key.data,
 357                                                 cipher_xform->key.length)) {
 358                 ret = -EINVAL;
 359                 goto error_out;
 360         }
 361
 362         return 0;
 363
 364 error_out:
 365         if (session->bpi_ctx) {
 366                 bpi_cipher_ctx_free(session->bpi_ctx);
 367                 session->bpi_ctx = NULL;
 368         }
 369         return ret;
 370 }
 371
 372 int
 373 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 374                 struct rte_crypto_sym_xform *xform,
 375                 struct rte_cryptodev_sym_session *sess,
 376                 struct rte_mempool *mempool)
 377 {
 378         void *sess_private_data;
 379         int ret;
 380
 381         if (rte_mempool_get(mempool, &sess_private_data)) {
 382                 CDEV_LOG_ERR(
 383                         "Couldn't get object from session mempool");
 384                 return -ENOMEM;
 385         }
 386
 387         ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data);
 388         if (ret != 0) {
 389                 PMD_DRV_LOG(ERR,
 390                     "Crypto QAT PMD: failed to configure session parameters");
 391
 392                 /* Return session to mempool */
 393                 rte_mempool_put(mempool, sess_private_data);
 394                 return ret;
 395         }
 396
 397         set_session_private_data(sess, dev->driver_id,
 398                 sess_private_data);
 399
 400         return 0;
 401 }
 402
 403 int
 404 qat_crypto_set_session_parameters(struct rte_cryptodev *dev,
 405                 struct rte_crypto_sym_xform *xform, void *session_private)
 406 {
 407         struct qat_session *session = session_private;
 408         int ret;
 409         int qat_cmd_id;
 410
 411         PMD_INIT_FUNC_TRACE();
 412
 413         /* Set context descriptor physical address */
 414         session->cd_paddr = rte_mempool_virt2iova(session) +
 415                         offsetof(struct qat_session, cd);
 416
 417         session->min_qat_dev_gen = QAT_GEN1;
 418
 419         /* Get requested QAT command id */
 420         qat_cmd_id = qat_get_cmd_id(xform);
 421         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 422                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 423                 return -ENOTSUP;
 424         }
 425         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 426         switch (session->qat_cmd) {
 427         case ICP_QAT_FW_LA_CMD_CIPHER:
 428                 ret = qat_crypto_sym_configure_session_cipher(dev,
 429                                                         xform, session);
 430                 if (ret < 0)
 431                         return ret;
 432                 break;
 433         case ICP_QAT_FW_LA_CMD_AUTH:
 434                 ret = qat_crypto_sym_configure_session_auth(dev,
 435                                                         xform, session);
 436                 if (ret < 0)
 437                         return ret;
 438                 break;
 439         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 440                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 441                         ret = qat_crypto_sym_configure_session_aead(xform,
 442                                         session);
 443                         if (ret < 0)
 444                                 return ret;
 445                 } else {
 446                         ret = qat_crypto_sym_configure_session_cipher(dev,
 447                                         xform, session);
 448                         if (ret < 0)
 449                                 return ret;
 450                         ret = qat_crypto_sym_configure_session_auth(dev,
 451                                         xform, session);
 452                         if (ret < 0)
 453                                 return ret;
 454                 }
 455                 break;
 456         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 457                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 458                         ret = qat_crypto_sym_configure_session_aead(xform,
 459                                         session);
 460                         if (ret < 0)
 461                                 return ret;
 462                 } else {
 463                         ret = qat_crypto_sym_configure_session_auth(dev,
 464                                         xform, session);
 465                         if (ret < 0)
 466                                 return ret;
 467                         ret = qat_crypto_sym_configure_session_cipher(dev,
 468                                         xform, session);
 469                         if (ret < 0)
 470                                 return ret;
 471                 }
 472                 break;
 473         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 474         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 475         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 476         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 477         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 478         case ICP_QAT_FW_LA_CMD_MGF1:
 479         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 480         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 481         case ICP_QAT_FW_LA_CMD_DELIMITER:
 482         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 483                 session->qat_cmd);
 484                 return -ENOTSUP;
 485         default:
 486         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 487                 session->qat_cmd);
 488                 return -ENOTSUP;
 489         }
 490
 491         return 0;
 492 }
 493
 494 int
 495 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 496                                 struct rte_crypto_sym_xform *xform,
 497                                 struct qat_session *session)
 498 {
 499         struct rte_crypto_auth_xform *auth_xform = qat_get_auth_xform(xform);
 500         struct qat_pmd_private *internals = dev->data->dev_private;
 501         uint8_t *key_data = auth_xform->key.data;
 502         uint8_t key_length = auth_xform->key.length;
 503
 504         switch (auth_xform->algo) {
 505         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 506                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 507                 break;
 508         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 509                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
 510                 break;
 511         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 512                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 513                 break;
 514         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 515                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
 516                 break;
 517         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 518                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 519                 break;
 520         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 521                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 522                 break;
 523         case RTE_CRYPTO_AUTH_AES_GMAC:
 524                 if (qat_alg_validate_aes_key(auth_xform->key.length,
 525                                 &session->qat_cipher_alg) != 0) {
 526                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 527                         return -EINVAL;
 528                 }
 529                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 530                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 531
 532                 break;
 533         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 534                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 535                 break;
 536         case RTE_CRYPTO_AUTH_MD5_HMAC:
 537                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
 538                 break;
 539         case RTE_CRYPTO_AUTH_NULL:
 540                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
 541                 break;
 542         case RTE_CRYPTO_AUTH_KASUMI_F9:
 543                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9;
 544                 break;
 545         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 546                 if (!qat_is_auth_alg_supported(auth_xform->algo, internals)) {
 547                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 548                                 rte_crypto_auth_algorithm_strings
 549                                 [auth_xform->algo]);
 550                         return -ENOTSUP;
 551                 }
 552                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3;
 553                 break;
 554         case RTE_CRYPTO_AUTH_SHA1:
 555         case RTE_CRYPTO_AUTH_SHA256:
 556         case RTE_CRYPTO_AUTH_SHA512:
 557         case RTE_CRYPTO_AUTH_SHA224:
 558         case RTE_CRYPTO_AUTH_SHA384:
 559         case RTE_CRYPTO_AUTH_MD5:
 560         case RTE_CRYPTO_AUTH_AES_CMAC:
 561         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 562                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 563                                 auth_xform->algo);
 564                 return -ENOTSUP;
 565         default:
 566                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 567                                 auth_xform->algo);
 568                 return -EINVAL;
 569         }
 570
 571         session->auth_iv.offset = auth_xform->iv.offset;
 572         session->auth_iv.length = auth_xform->iv.length;
 573
 574         if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 575                 if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 576                         session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 577                         session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 578                         /*
 579                          * It needs to create cipher desc content first,
 580                          * then authentication
 581                          */
 582                         if (qat_alg_aead_session_create_content_desc_cipher(
 583                                                 session,
 584                                                 auth_xform->key.data,
 585                                                 auth_xform->key.length))
 586                                 return -EINVAL;
 587
 588                         if (qat_alg_aead_session_create_content_desc_auth(
 589                                                 session,
 590                                                 key_data,
 591                                                 key_length,
 592                                                 0,
 593                                                 auth_xform->digest_length,
 594                                                 auth_xform->op))
 595                                 return -EINVAL;
 596                 } else {
 597                         session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 598                         session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 599                         /*
 600                          * It needs to create authentication desc content first,
 601                          * then cipher
 602                          */
 603                         if (qat_alg_aead_session_create_content_desc_auth(
 604                                         session,
 605                                         key_data,
 606                                         key_length,
 607                                         0,
 608                                         auth_xform->digest_length,
 609                                         auth_xform->op))
 610                                 return -EINVAL;
 611
 612                         if (qat_alg_aead_session_create_content_desc_cipher(
 613                                                 session,
 614                                                 auth_xform->key.data,
 615                                                 auth_xform->key.length))
 616                                 return -EINVAL;
 617                 }
 618                 /* Restore to authentication only only */
 619                 session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH;
 620         } else {
 621                 if (qat_alg_aead_session_create_content_desc_auth(session,
 622                                 key_data,
 623                                 key_length,
 624                                 0,
 625                                 auth_xform->digest_length,
 626                                 auth_xform->op))
 627                         return -EINVAL;
 628         }
 629
 630         session->digest_length = auth_xform->digest_length;
 631         return 0;
 632 }
 633
 634 int
 635 qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform,
 636                                 struct qat_session *session)
 637 {
 638         struct rte_crypto_aead_xform *aead_xform = &xform->aead;
 639         enum rte_crypto_auth_operation crypto_operation;
 640
 641         /*
 642          * Store AEAD IV parameters as cipher IV,
 643          * to avoid unnecessary memory usage
 644          */
 645         session->cipher_iv.offset = xform->aead.iv.offset;
 646         session->cipher_iv.length = xform->aead.iv.length;
 647
 648         switch (aead_xform->algo) {
 649         case RTE_CRYPTO_AEAD_AES_GCM:
 650                 if (qat_alg_validate_aes_key(aead_xform->key.length,
 651                                 &session->qat_cipher_alg) != 0) {
 652                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 653                         return -EINVAL;
 654                 }
 655                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 656                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 657                 break;
 658         case RTE_CRYPTO_AEAD_AES_CCM:
 659                 if (qat_alg_validate_aes_key(aead_xform->key.length,
 660                                 &session->qat_cipher_alg) != 0) {
 661                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 662                         return -EINVAL;
 663                 }
 664                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 665                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC;
 666                 break;
 667         default:
 668                 PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n",
 669                                 aead_xform->algo);
 670                 return -EINVAL;
 671         }
 672
 673         if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
 674                         aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) ||
 675                         (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
 676                         aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) {
 677                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 678                 /*
 679                  * It needs to create cipher desc content first,
 680                  * then authentication
 681                  */
 682                 crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
 683                         RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY;
 684
 685                 if (qat_alg_aead_session_create_content_desc_cipher(session,
 686                                         aead_xform->key.data,
 687                                         aead_xform->key.length))
 688                         return -EINVAL;
 689
 690                 if (qat_alg_aead_session_create_content_desc_auth(session,
 691                                         aead_xform->key.data,
 692                                         aead_xform->key.length,
 693                                         aead_xform->aad_length,
 694                                         aead_xform->digest_length,
 695                                         crypto_operation))
 696                         return -EINVAL;
 697         } else {
 698                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 699                 /*
 700                  * It needs to create authentication desc content first,
 701                  * then cipher
 702                  */
 703
 704                 crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
 705                         RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE;
 706
 707                 if (qat_alg_aead_session_create_content_desc_auth(session,
 708                                         aead_xform->key.data,
 709                                         aead_xform->key.length,
 710                                         aead_xform->aad_length,
 711                                         aead_xform->digest_length,
 712                                         crypto_operation))
 713                         return -EINVAL;
 714
 715                 if (qat_alg_aead_session_create_content_desc_cipher(session,
 716                                         aead_xform->key.data,
 717                                         aead_xform->key.length))
 718                         return -EINVAL;
 719         }
 720
 721         session->digest_length = aead_xform->digest_length;
 722         return 0;
 723 }
 724
 725 unsigned int qat_crypto_sym_get_session_private_size(
 726                 struct rte_cryptodev *dev __rte_unused)
 727 {
 728         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 729 }
 730
 731 /* returns block size in bytes per cipher algo */
 732 int qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg)
 733 {
 734         switch (qat_cipher_alg) {
 735         case ICP_QAT_HW_CIPHER_ALGO_DES:
 736                 return ICP_QAT_HW_DES_BLK_SZ;
 737         case ICP_QAT_HW_CIPHER_ALGO_3DES:
 738                 return ICP_QAT_HW_3DES_BLK_SZ;
 739         case ICP_QAT_HW_CIPHER_ALGO_AES128:
 740         case ICP_QAT_HW_CIPHER_ALGO_AES192:
 741         case ICP_QAT_HW_CIPHER_ALGO_AES256:
 742                 return ICP_QAT_HW_AES_BLK_SZ;
 743         default:
 744                 PMD_DRV_LOG(ERR, "invalid block cipher alg %u", qat_cipher_alg);
 745                 return -EFAULT;
 746         };
 747         return -EFAULT;
 748 }
 749
 750 /*
 751  * Returns size in bytes per hash algo for state1 size field in cd_ctrl
 752  * This is digest size rounded up to nearest quadword
 753  */
 754 static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 755 {
 756         switch (qat_hash_alg) {
 757         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 758                 return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA1_STATE1_SZ,
 759                                                 QAT_HW_DEFAULT_ALIGNMENT);
 760         case ICP_QAT_HW_AUTH_ALGO_SHA224:
 761                 return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA224_STATE1_SZ,
 762                                                 QAT_HW_DEFAULT_ALIGNMENT);
 763         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 764                 return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA256_STATE1_SZ,
 765                                                 QAT_HW_DEFAULT_ALIGNMENT);
 766         case ICP_QAT_HW_AUTH_ALGO_SHA384:
 767                 return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA384_STATE1_SZ,
 768                                                 QAT_HW_DEFAULT_ALIGNMENT);
 769         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 770                 return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ,
 771                                                 QAT_HW_DEFAULT_ALIGNMENT);
 772         case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC:
 773                 return QAT_HW_ROUND_UP(ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ,
 774                                                 QAT_HW_DEFAULT_ALIGNMENT);
 775         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 776         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 777                 return QAT_HW_ROUND_UP(ICP_QAT_HW_GALOIS_128_STATE1_SZ,
 778                                                 QAT_HW_DEFAULT_ALIGNMENT);
 779         case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
 780                 return QAT_HW_ROUND_UP(ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ,
 781                                                 QAT_HW_DEFAULT_ALIGNMENT);
 782         case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
 783                 return QAT_HW_ROUND_UP(ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ,
 784                                                 QAT_HW_DEFAULT_ALIGNMENT);
 785         case ICP_QAT_HW_AUTH_ALGO_MD5:
 786                 return QAT_HW_ROUND_UP(ICP_QAT_HW_MD5_STATE1_SZ,
 787                                                 QAT_HW_DEFAULT_ALIGNMENT);
 788         case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
 789                 return QAT_HW_ROUND_UP(ICP_QAT_HW_KASUMI_F9_STATE1_SZ,
 790                                                 QAT_HW_DEFAULT_ALIGNMENT);
 791         case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
 792                 return QAT_HW_ROUND_UP(ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ,
 793                                                 QAT_HW_DEFAULT_ALIGNMENT);
 794         case ICP_QAT_HW_AUTH_ALGO_NULL:
 795                 return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ,
 796                                                 QAT_HW_DEFAULT_ALIGNMENT);
 797         case ICP_QAT_HW_AUTH_ALGO_DELIMITER:
 798                 /* return maximum state1 size in this case */
 799                 return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ,
 800                                                 QAT_HW_DEFAULT_ALIGNMENT);
 801         default:
 802                 PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
 803                 return -EFAULT;
 804         };
 805         return -EFAULT;
 806 }
 807
 808 /* returns digest size in bytes  per hash algo */
 809 static int qat_hash_get_digest_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 810 {
 811         switch (qat_hash_alg) {
 812         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 813                 return ICP_QAT_HW_SHA1_STATE1_SZ;
 814         case ICP_QAT_HW_AUTH_ALGO_SHA224:
 815                 return ICP_QAT_HW_SHA224_STATE1_SZ;
 816         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 817                 return ICP_QAT_HW_SHA256_STATE1_SZ;
 818         case ICP_QAT_HW_AUTH_ALGO_SHA384:
 819                 return ICP_QAT_HW_SHA384_STATE1_SZ;
 820         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 821                 return ICP_QAT_HW_SHA512_STATE1_SZ;
 822         case ICP_QAT_HW_AUTH_ALGO_MD5:
 823                 return ICP_QAT_HW_MD5_STATE1_SZ;
 824         case ICP_QAT_HW_AUTH_ALGO_DELIMITER:
 825                 /* return maximum digest size in this case */
 826                 return ICP_QAT_HW_SHA512_STATE1_SZ;
 827         default:
 828                 PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
 829                 return -EFAULT;
 830         };
 831         return -EFAULT;
 832 }
 833
 834 /* returns block size in byes per hash algo */
 835 static int qat_hash_get_block_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 836 {
 837         switch (qat_hash_alg) {
 838         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 839                 return SHA_CBLOCK;
 840         case ICP_QAT_HW_AUTH_ALGO_SHA224:
 841                 return SHA256_CBLOCK;
 842         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 843                 return SHA256_CBLOCK;
 844         case ICP_QAT_HW_AUTH_ALGO_SHA384:
 845                 return SHA512_CBLOCK;
 846         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 847                 return SHA512_CBLOCK;
 848         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 849                 return 16;
 850         case ICP_QAT_HW_AUTH_ALGO_MD5:
 851                 return MD5_CBLOCK;
 852         case ICP_QAT_HW_AUTH_ALGO_DELIMITER:
 853                 /* return maximum block size in this case */
 854                 return SHA512_CBLOCK;
 855         default:
 856                 PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg);
 857                 return -EFAULT;
 858         };
 859         return -EFAULT;
 860 }
 861
 862 static int partial_hash_sha1(uint8_t *data_in, uint8_t *data_out)
 863 {
 864         SHA_CTX ctx;
 865
 866         if (!SHA1_Init(&ctx))
 867                 return -EFAULT;
 868         SHA1_Transform(&ctx, data_in);
 869         rte_memcpy(data_out, &ctx, SHA_DIGEST_LENGTH);
 870         return 0;
 871 }
 872
 873 static int partial_hash_sha224(uint8_t *data_in, uint8_t *data_out)
 874 {
 875         SHA256_CTX ctx;
 876
 877         if (!SHA224_Init(&ctx))
 878                 return -EFAULT;
 879         SHA256_Transform(&ctx, data_in);
 880         rte_memcpy(data_out, &ctx, SHA256_DIGEST_LENGTH);
 881         return 0;
 882 }
 883
 884 static int partial_hash_sha256(uint8_t *data_in, uint8_t *data_out)
 885 {
 886         SHA256_CTX ctx;
 887
 888         if (!SHA256_Init(&ctx))
 889                 return -EFAULT;
 890         SHA256_Transform(&ctx, data_in);
 891         rte_memcpy(data_out, &ctx, SHA256_DIGEST_LENGTH);
 892         return 0;
 893 }
 894
 895 static int partial_hash_sha384(uint8_t *data_in, uint8_t *data_out)
 896 {
 897         SHA512_CTX ctx;
 898
 899         if (!SHA384_Init(&ctx))
 900                 return -EFAULT;
 901         SHA512_Transform(&ctx, data_in);
 902         rte_memcpy(data_out, &ctx, SHA512_DIGEST_LENGTH);
 903         return 0;
 904 }
 905
 906 static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out)
 907 {
 908         SHA512_CTX ctx;
 909
 910         if (!SHA512_Init(&ctx))
 911                 return -EFAULT;
 912         SHA512_Transform(&ctx, data_in);
 913         rte_memcpy(data_out, &ctx, SHA512_DIGEST_LENGTH);
 914         return 0;
 915 }
 916
 917 static int partial_hash_md5(uint8_t *data_in, uint8_t *data_out)
 918 {
 919         MD5_CTX ctx;
 920
 921         if (!MD5_Init(&ctx))
 922                 return -EFAULT;
 923         MD5_Transform(&ctx, data_in);
 924         rte_memcpy(data_out, &ctx, MD5_DIGEST_LENGTH);
 925
 926         return 0;
 927 }
 928
 929 static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
 930                         uint8_t *data_in,
 931                         uint8_t *data_out)
 932 {
 933         int digest_size;
 934         uint8_t digest[qat_hash_get_digest_size(
 935                         ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
 936         uint32_t *hash_state_out_be32;
 937         uint64_t *hash_state_out_be64;
 938         int i;
 939
 940         PMD_INIT_FUNC_TRACE();
 941         digest_size = qat_hash_get_digest_size(hash_alg);
 942         if (digest_size <= 0)
 943                 return -EFAULT;
 944
 945         hash_state_out_be32 = (uint32_t *)data_out;
 946         hash_state_out_be64 = (uint64_t *)data_out;
 947
 948         switch (hash_alg) {
 949         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 950                 if (partial_hash_sha1(data_in, digest))
 951                         return -EFAULT;
 952                 for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++)
 953                         *hash_state_out_be32 =
 954                                 rte_bswap32(*(((uint32_t *)digest)+i));
 955                 break;
 956         case ICP_QAT_HW_AUTH_ALGO_SHA224:
 957                 if (partial_hash_sha224(data_in, digest))
 958                         return -EFAULT;
 959                 for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++)
 960                         *hash_state_out_be32 =
 961                                 rte_bswap32(*(((uint32_t *)digest)+i));
 962                 break;
 963         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 964                 if (partial_hash_sha256(data_in, digest))
 965                         return -EFAULT;
 966                 for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++)
 967                         *hash_state_out_be32 =
 968                                 rte_bswap32(*(((uint32_t *)digest)+i));
 969                 break;
 970         case ICP_QAT_HW_AUTH_ALGO_SHA384:
 971                 if (partial_hash_sha384(data_in, digest))
 972                         return -EFAULT;
 973                 for (i = 0; i < digest_size >> 3; i++, hash_state_out_be64++)
 974                         *hash_state_out_be64 =
 975                                 rte_bswap64(*(((uint64_t *)digest)+i));
 976                 break;
 977         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 978                 if (partial_hash_sha512(data_in, digest))
 979                         return -EFAULT;
 980                 for (i = 0; i < digest_size >> 3; i++, hash_state_out_be64++)
 981                         *hash_state_out_be64 =
 982                                 rte_bswap64(*(((uint64_t *)digest)+i));
 983                 break;
 984         case ICP_QAT_HW_AUTH_ALGO_MD5:
 985                 if (partial_hash_md5(data_in, data_out))
 986                         return -EFAULT;
 987                 break;
 988         default:
 989                 PMD_DRV_LOG(ERR, "invalid hash alg %u", hash_alg);
 990                 return -EFAULT;
 991         }
 992
 993         return 0;
 994 }
 995 #define HMAC_IPAD_VALUE 0x36
 996 #define HMAC_OPAD_VALUE 0x5c
 997 #define HASH_XCBC_PRECOMP_KEY_NUM 3
 998
 999 static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
1000                                 const uint8_t *auth_key,
1001                                 uint16_t auth_keylen,
1002                                 uint8_t *p_state_buf,
1003                                 uint16_t *p_state_len)
1004 {
1005         int block_size;
1006         uint8_t ipad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
1007         uint8_t opad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
1008         int i;
1009
1010         PMD_INIT_FUNC_TRACE();
1011         if (hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC) {
1012                 static uint8_t qat_aes_xcbc_key_seed[
1013                                         ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ] = {
1014                         0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
1015                         0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
1016                         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
1017                         0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
1018                         0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
1019                         0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
1020                 };
1021
1022                 uint8_t *in = NULL;
1023                 uint8_t *out = p_state_buf;
1024                 int x;
1025                 AES_KEY enc_key;
1026
1027                 in = rte_zmalloc("working mem for key",
1028                                 ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ, 16);
1029                 if (in == NULL) {
1030                         PMD_DRV_LOG(ERR, "Failed to alloc memory");
1031                         return -ENOMEM;
1032                 }
1033
1034                 rte_memcpy(in, qat_aes_xcbc_key_seed,
1035                                 ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
1036                 for (x = 0; x < HASH_XCBC_PRECOMP_KEY_NUM; x++) {
1037                         if (AES_set_encrypt_key(auth_key, auth_keylen << 3,
1038                                 &enc_key) != 0) {
1039                                 rte_free(in -
1040                                         (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ));
1041                                 memset(out -
1042                                         (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ),
1043                                         0, ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
1044                                 return -EFAULT;
1045                         }
1046                         AES_encrypt(in, out, &enc_key);
1047                         in += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
1048                         out += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
1049                 }
1050                 *p_state_len = ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ;
1051                 rte_free(in - x*ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ);
1052                 return 0;
1053         } else if ((hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
1054                 (hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) {
1055                 uint8_t *in = NULL;
1056                 uint8_t *out = p_state_buf;
1057                 AES_KEY enc_key;
1058
1059                 memset(p_state_buf, 0, ICP_QAT_HW_GALOIS_H_SZ +
1060                                 ICP_QAT_HW_GALOIS_LEN_A_SZ +
1061                                 ICP_QAT_HW_GALOIS_E_CTR0_SZ);
1062                 in = rte_zmalloc("working mem for key",
1063                                 ICP_QAT_HW_GALOIS_H_SZ, 16);
1064                 if (in == NULL) {
1065                         PMD_DRV_LOG(ERR, "Failed to alloc memory");
1066                         return -ENOMEM;
1067                 }
1068
1069                 memset(in, 0, ICP_QAT_HW_GALOIS_H_SZ);
1070                 if (AES_set_encrypt_key(auth_key, auth_keylen << 3,
1071                         &enc_key) != 0) {
1072                         return -EFAULT;
1073                 }
1074                 AES_encrypt(in, out, &enc_key);
1075                 *p_state_len = ICP_QAT_HW_GALOIS_H_SZ +
1076                                 ICP_QAT_HW_GALOIS_LEN_A_SZ +
1077                                 ICP_QAT_HW_GALOIS_E_CTR0_SZ;
1078                 rte_free(in);
1079                 return 0;
1080         }
1081
1082         block_size = qat_hash_get_block_size(hash_alg);
1083         if (block_size <= 0)
1084                 return -EFAULT;
1085         /* init ipad and opad from key and xor with fixed values */
1086         memset(ipad, 0, block_size);
1087         memset(opad, 0, block_size);
1088
1089         if (auth_keylen > (unsigned int)block_size) {
1090                 PMD_DRV_LOG(ERR, "invalid keylen %u", auth_keylen);
1091                 return -EFAULT;
1092         }
1093         rte_memcpy(ipad, auth_key, auth_keylen);
1094         rte_memcpy(opad, auth_key, auth_keylen);
1095
1096         for (i = 0; i < block_size; i++) {
1097                 uint8_t *ipad_ptr = ipad + i;
1098                 uint8_t *opad_ptr = opad + i;
1099                 *ipad_ptr ^= HMAC_IPAD_VALUE;
1100                 *opad_ptr ^= HMAC_OPAD_VALUE;
1101         }
1102
1103         /* do partial hash of ipad and copy to state1 */
1104         if (partial_hash_compute(hash_alg, ipad, p_state_buf)) {
1105                 memset(ipad, 0, block_size);
1106                 memset(opad, 0, block_size);
1107                 PMD_DRV_LOG(ERR, "ipad precompute failed");
1108                 return -EFAULT;
1109         }
1110
1111         /*
1112          * State len is a multiple of 8, so may be larger than the digest.
1113          * Put the partial hash of opad state_len bytes after state1
1114          */
1115         *p_state_len = qat_hash_get_state1_size(hash_alg);
1116         if (partial_hash_compute(hash_alg, opad, p_state_buf + *p_state_len)) {
1117                 memset(ipad, 0, block_size);
1118                 memset(opad, 0, block_size);
1119                 PMD_DRV_LOG(ERR, "opad precompute failed");
1120                 return -EFAULT;
1121         }
1122
1123         /*  don't leave data lying around */
1124         memset(ipad, 0, block_size);
1125         memset(opad, 0, block_size);
1126         return 0;
1127 }
1128
1129 void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
1130                 enum qat_crypto_proto_flag proto_flags)
1131 {
1132         PMD_INIT_FUNC_TRACE();
1133         header->hdr_flags =
1134                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
1135         header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
1136         header->comn_req_flags =
1137                 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
1138                                         QAT_COMN_PTR_TYPE_FLAT);
1139         ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
1140                                   ICP_QAT_FW_LA_PARTIAL_NONE);
1141         ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
1142                                            ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
1143
1144         switch (proto_flags)            {
1145         case QAT_CRYPTO_PROTO_FLAG_NONE:
1146                 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
1147                                         ICP_QAT_FW_LA_NO_PROTO);
1148                 break;
1149         case QAT_CRYPTO_PROTO_FLAG_CCM:
1150                 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
1151                                         ICP_QAT_FW_LA_CCM_PROTO);
1152                 break;
1153         case QAT_CRYPTO_PROTO_FLAG_GCM:
1154                 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
1155                                         ICP_QAT_FW_LA_GCM_PROTO);
1156                 break;
1157         case QAT_CRYPTO_PROTO_FLAG_SNOW3G:
1158                 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
1159                                         ICP_QAT_FW_LA_SNOW_3G_PROTO);
1160                 break;
1161         case QAT_CRYPTO_PROTO_FLAG_ZUC:
1162                 ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(header->serv_specif_flags,
1163                         ICP_QAT_FW_LA_ZUC_3G_PROTO);
1164                 break;
1165         }
1166
1167         ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
1168                                            ICP_QAT_FW_LA_NO_UPDATE_STATE);
1169         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
1170                                         ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
1171 }
1172
1173 /*
1174  *      Snow3G and ZUC should never use this function
1175  *      and set its protocol flag in both cipher and auth part of content
1176  *      descriptor building function
1177  */
1178 static enum qat_crypto_proto_flag
1179 qat_get_crypto_proto_flag(uint16_t flags)
1180 {
1181         int proto = ICP_QAT_FW_LA_PROTO_GET(flags);
1182         enum qat_crypto_proto_flag qat_proto_flag =
1183                         QAT_CRYPTO_PROTO_FLAG_NONE;
1184
1185         switch (proto) {
1186         case ICP_QAT_FW_LA_GCM_PROTO:
1187                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM;
1188                 break;
1189         case ICP_QAT_FW_LA_CCM_PROTO:
1190                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM;
1191                 break;
1192         }
1193
1194         return qat_proto_flag;
1195 }
1196
1197 int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc,
1198                                                 uint8_t *cipherkey,
1199                                                 uint32_t cipherkeylen)
1200 {
1201         struct icp_qat_hw_cipher_algo_blk *cipher;
1202         struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req;
1203         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
1204         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
1205         void *ptr = &req_tmpl->cd_ctrl;
1206         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
1207         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
1208         enum icp_qat_hw_cipher_convert key_convert;
1209         enum qat_crypto_proto_flag qat_proto_flag =
1210                 QAT_CRYPTO_PROTO_FLAG_NONE;
1211         uint32_t total_key_size;
1212         uint16_t cipher_offset, cd_size;
1213         uint32_t wordIndex  = 0;
1214         uint32_t *temp_key = NULL;
1215         PMD_INIT_FUNC_TRACE();
1216
1217         if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
1218                 cd_pars->u.s.content_desc_addr = cdesc->cd_paddr;
1219                 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl,
1220                                         ICP_QAT_FW_SLICE_CIPHER);
1221                 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl,
1222                                         ICP_QAT_FW_SLICE_DRAM_WR);
1223                 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
1224                                         ICP_QAT_FW_LA_NO_RET_AUTH_RES);
1225                 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
1226                                         ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
1227                 cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
1228         } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
1229                 cd_pars->u.s.content_desc_addr = cdesc->cd_paddr;
1230                 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl,
1231                                         ICP_QAT_FW_SLICE_CIPHER);
1232                 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl,
1233                                         ICP_QAT_FW_SLICE_AUTH);
1234                 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl,
1235                                         ICP_QAT_FW_SLICE_AUTH);
1236                 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl,
1237                                         ICP_QAT_FW_SLICE_DRAM_WR);
1238                 cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
1239         } else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
1240                 PMD_DRV_LOG(ERR, "Invalid param, must be a cipher command.");
1241                 return -EFAULT;
1242         }
1243
1244         if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE) {
1245                 /*
1246                  * CTR Streaming ciphers are a special case. Decrypt = encrypt
1247                  * Overriding default values previously set
1248                  */
1249                 cdesc->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
1250                 key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT;
1251         } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2
1252                 || cdesc->qat_cipher_alg ==
1253                         ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)
1254                 key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT;
1255         else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT)
1256                 key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT;
1257         else
1258                 key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT;
1259
1260         if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
1261                 total_key_size = ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ +
1262                         ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ;
1263                 cipher_cd_ctrl->cipher_state_sz =
1264                         ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3;
1265                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G;
1266
1267         } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) {
1268                 total_key_size = ICP_QAT_HW_KASUMI_F8_KEY_SZ;
1269                 cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_KASUMI_BLK_SZ >> 3;
1270                 cipher_cd_ctrl->cipher_padding_sz =
1271                                         (2 * ICP_QAT_HW_KASUMI_BLK_SZ) >> 3;
1272         } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) {
1273                 total_key_size = ICP_QAT_HW_3DES_KEY_SZ;
1274                 cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_3DES_BLK_SZ >> 3;
1275                 qat_proto_flag =
1276                         qat_get_crypto_proto_flag(header->serv_specif_flags);
1277         } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_DES) {
1278                 total_key_size = ICP_QAT_HW_DES_KEY_SZ;
1279                 cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_DES_BLK_SZ >> 3;
1280                 qat_proto_flag =
1281                         qat_get_crypto_proto_flag(header->serv_specif_flags);
1282         } else if (cdesc->qat_cipher_alg ==
1283                 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
1284                 total_key_size = ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ +
1285                         ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ;
1286                 cipher_cd_ctrl->cipher_state_sz =
1287                         ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3;
1288                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC;
1289                 cdesc->min_qat_dev_gen = QAT_GEN2;
1290         } else {
1291                 total_key_size = cipherkeylen;
1292                 cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3;
1293                 qat_proto_flag =
1294                         qat_get_crypto_proto_flag(header->serv_specif_flags);
1295         }
1296         cipher_cd_ctrl->cipher_key_sz = total_key_size >> 3;
1297         cipher_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd);
1298         cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3;
1299
1300         header->service_cmd_id = cdesc->qat_cmd;
1301         qat_alg_init_common_hdr(header, qat_proto_flag);
1302
1303         cipher = (struct icp_qat_hw_cipher_algo_blk *)cdesc->cd_cur_ptr;
1304         cipher->cipher_config.val =
1305             ICP_QAT_HW_CIPHER_CONFIG_BUILD(cdesc->qat_mode,
1306                                         cdesc->qat_cipher_alg, key_convert,
1307                                         cdesc->qat_dir);
1308
1309         if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) {
1310                 temp_key = (uint32_t *)(cdesc->cd_cur_ptr +
1311                                         sizeof(struct icp_qat_hw_cipher_config)
1312                                         + cipherkeylen);
1313                 memcpy(cipher->key, cipherkey, cipherkeylen);
1314                 memcpy(temp_key, cipherkey, cipherkeylen);
1315
1316                 /* XOR Key with KASUMI F8 key modifier at 4 bytes level */
1317                 for (wordIndex = 0; wordIndex < (cipherkeylen >> 2);
1318                                                                 wordIndex++)
1319                         temp_key[wordIndex] ^= KASUMI_F8_KEY_MODIFIER_4_BYTES;
1320
1321                 cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) +
1322                                         cipherkeylen + cipherkeylen;
1323         } else {
1324                 memcpy(cipher->key, cipherkey, cipherkeylen);
1325                 cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) +
1326                                         cipherkeylen;
1327         }
1328
1329         if (total_key_size > cipherkeylen) {
1330                 uint32_t padding_size =  total_key_size-cipherkeylen;
1331                 if ((cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES)
1332                         && (cipherkeylen == QAT_3DES_KEY_SZ_OPT2))
1333                         /* K3 not provided so use K1 = K3*/
1334                         memcpy(cdesc->cd_cur_ptr, cipherkey, padding_size);
1335                 else
1336                         memset(cdesc->cd_cur_ptr, 0, padding_size);
1337                 cdesc->cd_cur_ptr += padding_size;
1338         }
1339         cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd;
1340         cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3;
1341
1342         return 0;
1343 }
1344
1345 int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc,
1346                                                 uint8_t *authkey,
1347                                                 uint32_t authkeylen,
1348                                                 uint32_t aad_length,
1349                                                 uint32_t digestsize,
1350                                                 unsigned int operation)
1351 {
1352         struct icp_qat_hw_auth_setup *hash;
1353         struct icp_qat_hw_cipher_algo_blk *cipherconfig;
1354         struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req;
1355         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
1356         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
1357         void *ptr = &req_tmpl->cd_ctrl;
1358         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
1359         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
1360         struct icp_qat_fw_la_auth_req_params *auth_param =
1361                 (struct icp_qat_fw_la_auth_req_params *)
1362                 ((char *)&req_tmpl->serv_specif_rqpars +
1363                 sizeof(struct icp_qat_fw_la_cipher_req_params));
1364         uint16_t state1_size = 0, state2_size = 0;
1365         uint16_t hash_offset, cd_size;
1366         uint32_t *aad_len = NULL;
1367         uint32_t wordIndex  = 0;
1368         uint32_t *pTempKey;
1369         enum qat_crypto_proto_flag qat_proto_flag =
1370                 QAT_CRYPTO_PROTO_FLAG_NONE;
1371
1372         PMD_INIT_FUNC_TRACE();
1373
1374         if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
1375                 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl,
1376                                         ICP_QAT_FW_SLICE_AUTH);
1377                 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl,
1378                                         ICP_QAT_FW_SLICE_DRAM_WR);
1379                 cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
1380         } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
1381                 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl,
1382                                 ICP_QAT_FW_SLICE_AUTH);
1383                 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl,
1384                                 ICP_QAT_FW_SLICE_CIPHER);
1385                 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl,
1386                                 ICP_QAT_FW_SLICE_CIPHER);
1387                 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl,
1388                                 ICP_QAT_FW_SLICE_DRAM_WR);
1389                 cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd;
1390         } else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
1391                 PMD_DRV_LOG(ERR, "Invalid param, must be a hash command.");
1392                 return -EFAULT;
1393         }
1394
1395         if (operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1396                 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
1397                                 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
1398                 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
1399                                 ICP_QAT_FW_LA_CMP_AUTH_RES);
1400                 cdesc->auth_op = ICP_QAT_HW_AUTH_VERIFY;
1401         } else {
1402                 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
1403                                            ICP_QAT_FW_LA_RET_AUTH_RES);
1404                 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
1405                                            ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
1406                 cdesc->auth_op = ICP_QAT_HW_AUTH_GENERATE;
1407         }
1408
1409         /*
1410          * Setup the inner hash config
1411          */
1412         hash_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd);
1413         hash = (struct icp_qat_hw_auth_setup *)cdesc->cd_cur_ptr;
1414         hash->auth_config.reserved = 0;
1415         hash->auth_config.config =
1416                         ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
1417                                 cdesc->qat_hash_alg, digestsize);
1418
1419         if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2
1420                 || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9
1421                 || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3)
1422                 hash->auth_counter.counter = 0;
1423         else
1424                 hash->auth_counter.counter = rte_bswap32(
1425                                 qat_hash_get_block_size(cdesc->qat_hash_alg));
1426
1427         cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_auth_setup);
1428
1429         /*
1430          * cd_cur_ptr now points at the state1 information.
1431          */
1432         switch (cdesc->qat_hash_alg) {
1433         case ICP_QAT_HW_AUTH_ALGO_SHA1:
1434                 if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1,
1435                         authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
1436                         PMD_DRV_LOG(ERR, "(SHA)precompute failed");
1437                         return -EFAULT;
1438                 }
1439                 state2_size = RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
1440                 break;
1441         case ICP_QAT_HW_AUTH_ALGO_SHA224:
1442                 if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224,
1443                         authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
1444                         PMD_DRV_LOG(ERR, "(SHA)precompute failed");
1445                         return -EFAULT;
1446                 }
1447                 state2_size = ICP_QAT_HW_SHA224_STATE2_SZ;
1448                 break;
1449         case ICP_QAT_HW_AUTH_ALGO_SHA256:
1450                 if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256,
1451                         authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
1452                         PMD_DRV_LOG(ERR, "(SHA)precompute failed");
1453                         return -EFAULT;
1454                 }
1455                 state2_size = ICP_QAT_HW_SHA256_STATE2_SZ;
1456                 break;
1457         case ICP_QAT_HW_AUTH_ALGO_SHA384:
1458                 if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384,
1459                         authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
1460                         PMD_DRV_LOG(ERR, "(SHA)precompute failed");
1461                         return -EFAULT;
1462                 }
1463                 state2_size = ICP_QAT_HW_SHA384_STATE2_SZ;
1464                 break;
1465         case ICP_QAT_HW_AUTH_ALGO_SHA512:
1466                 if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512,
1467                         authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) {
1468                         PMD_DRV_LOG(ERR, "(SHA)precompute failed");
1469                         return -EFAULT;
1470                 }
1471                 state2_size = ICP_QAT_HW_SHA512_STATE2_SZ;
1472                 break;
1473         case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC:
1474                 state1_size = ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
1475                 if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
1476                         authkey, authkeylen, cdesc->cd_cur_ptr + state1_size,
1477                         &state2_size)) {
1478                         PMD_DRV_LOG(ERR, "(XCBC)precompute failed");
1479                         return -EFAULT;
1480                 }
1481                 break;
1482         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
1483         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
1484                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM;
1485                 state1_size = ICP_QAT_HW_GALOIS_128_STATE1_SZ;
1486                 if (qat_alg_do_precomputes(cdesc->qat_hash_alg,
1487                         authkey, authkeylen, cdesc->cd_cur_ptr + state1_size,
1488                         &state2_size)) {
1489                         PMD_DRV_LOG(ERR, "(GCM)precompute failed");
1490                         return -EFAULT;
1491                 }
1492                 /*
1493                  * Write (the length of AAD) into bytes 16-19 of state2
1494                  * in big-endian format. This field is 8 bytes
1495                  */
1496                 auth_param->u2.aad_sz =
1497                                 RTE_ALIGN_CEIL(aad_length, 16);
1498                 auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3;
1499
1500                 aad_len = (uint32_t *)(cdesc->cd_cur_ptr +
1501                                         ICP_QAT_HW_GALOIS_128_STATE1_SZ +
1502                                         ICP_QAT_HW_GALOIS_H_SZ);
1503                 *aad_len = rte_bswap32(aad_length);
1504                 cdesc->aad_len = aad_length;
1505                 break;
1506         case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
1507                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G;
1508                 state1_size = qat_hash_get_state1_size(
1509                                 ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2);
1510                 state2_size = ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ;
1511                 memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size);
1512
1513                 cipherconfig = (struct icp_qat_hw_cipher_algo_blk *)
1514                                 (cdesc->cd_cur_ptr + state1_size + state2_size);
1515                 cipherconfig->cipher_config.val =
1516                 ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_ECB_MODE,
1517                         ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2,
1518                         ICP_QAT_HW_CIPHER_KEY_CONVERT,
1519                         ICP_QAT_HW_CIPHER_ENCRYPT);
1520                 memcpy(cipherconfig->key, authkey, authkeylen);
1521                 memset(cipherconfig->key + authkeylen,
1522                                 0, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ);
1523                 cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) +
1524                                 authkeylen + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ;
1525                 auth_param->hash_state_sz = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3;
1526                 break;
1527         case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
1528                 hash->auth_config.config =
1529                         ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE0,
1530                                 cdesc->qat_hash_alg, digestsize);
1531                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC;
1532                 state1_size = qat_hash_get_state1_size(
1533                                 ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3);
1534                 state2_size = ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ;
1535                 memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size
1536                         + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ);
1537
1538                 memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen);
1539                 cdesc->cd_cur_ptr += state1_size + state2_size
1540                         + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ;
1541                 auth_param->hash_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3;
1542                 cdesc->min_qat_dev_gen = QAT_GEN2;
1543
1544                 break;
1545         case ICP_QAT_HW_AUTH_ALGO_MD5:
1546                 if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_MD5,
1547                         authkey, authkeylen, cdesc->cd_cur_ptr,
1548                         &state1_size)) {
1549                         PMD_DRV_LOG(ERR, "(MD5)precompute failed");
1550                         return -EFAULT;
1551                 }
1552                 state2_size = ICP_QAT_HW_MD5_STATE2_SZ;
1553                 break;
1554         case ICP_QAT_HW_AUTH_ALGO_NULL:
1555                 state1_size = qat_hash_get_state1_size(
1556                                 ICP_QAT_HW_AUTH_ALGO_NULL);
1557                 state2_size = ICP_QAT_HW_NULL_STATE2_SZ;
1558                 break;
1559         case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
1560                 qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM;
1561                 state1_size = qat_hash_get_state1_size(
1562                                 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC);
1563                 state2_size = ICP_QAT_HW_AES_CBC_MAC_KEY_SZ +
1564                                 ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ;
1565
1566                 if (aad_length > 0) {
1567                         aad_length += ICP_QAT_HW_CCM_AAD_B0_LEN +
1568                         ICP_QAT_HW_CCM_AAD_LEN_INFO;
1569                         auth_param->u2.aad_sz =
1570                         RTE_ALIGN_CEIL(aad_length,
1571                         ICP_QAT_HW_CCM_AAD_ALIGNMENT);
1572                 } else {
1573                         auth_param->u2.aad_sz = ICP_QAT_HW_CCM_AAD_B0_LEN;
1574                 }
1575                 cdesc->aad_len = aad_length;
1576                 hash->auth_counter.counter = 0;
1577
1578                 hash_cd_ctrl->outer_prefix_sz = digestsize;
1579                 auth_param->hash_state_sz = digestsize;
1580
1581                 memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen);
1582                 break;
1583         case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
1584                 state1_size = qat_hash_get_state1_size(
1585                                 ICP_QAT_HW_AUTH_ALGO_KASUMI_F9);
1586                 state2_size = ICP_QAT_HW_KASUMI_F9_STATE2_SZ;
1587                 memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size);
1588                 pTempKey = (uint32_t *)(cdesc->cd_cur_ptr + state1_size
1589                                                         + authkeylen);
1590                 /*
1591                 * The Inner Hash Initial State2 block must contain IK
1592                 * (Initialisation Key), followed by IK XOR-ed with KM
1593                 * (Key Modifier): IK||(IK^KM).
1594                 */
1595                 /* write the auth key */
1596                 memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen);
1597                 /* initialise temp key with auth key */
1598                 memcpy(pTempKey, authkey, authkeylen);
1599                 /* XOR Key with KASUMI F9 key modifier at 4 bytes level */
1600                 for (wordIndex = 0; wordIndex < (authkeylen >> 2); wordIndex++)
1601                         pTempKey[wordIndex] ^= KASUMI_F9_KEY_MODIFIER_4_BYTES;
1602                 break;
1603         default:
1604                 PMD_DRV_LOG(ERR, "Invalid HASH alg %u", cdesc->qat_hash_alg);
1605                 return -EFAULT;
1606         }
1607
1608         /* Request template setup */
1609         qat_alg_init_common_hdr(header, qat_proto_flag);
1610         header->service_cmd_id = cdesc->qat_cmd;
1611
1612         /* Auth CD config setup */
1613         hash_cd_ctrl->hash_cfg_offset = hash_offset >> 3;
1614         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
1615         hash_cd_ctrl->inner_res_sz = digestsize;
1616         hash_cd_ctrl->final_sz = digestsize;
1617         hash_cd_ctrl->inner_state1_sz = state1_size;
1618         auth_param->auth_res_sz = digestsize;
1619
1620         hash_cd_ctrl->inner_state2_sz  = state2_size;
1621         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
1622                         ((sizeof(struct icp_qat_hw_auth_setup) +
1623                          RTE_ALIGN_CEIL(hash_cd_ctrl->inner_state1_sz, 8))
1624                                         >> 3);
1625
1626         cdesc->cd_cur_ptr += state1_size + state2_size;
1627         cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd;
1628
1629         cd_pars->u.s.content_desc_addr = cdesc->cd_paddr;
1630         cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3;
1631
1632         return 0;
1633 }
1634
1635 int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
1636 {
1637         switch (key_len) {
1638         case ICP_QAT_HW_AES_128_KEY_SZ:
1639                 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
1640                 break;
1641         case ICP_QAT_HW_AES_192_KEY_SZ:
1642                 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
1643                 break;
1644         case ICP_QAT_HW_AES_256_KEY_SZ:
1645                 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
1646                 break;
1647         default:
1648                 return -EINVAL;
1649         }
1650         return 0;
1651 }
1652
1653 int qat_alg_validate_aes_docsisbpi_key(int key_len,
1654                 enum icp_qat_hw_cipher_algo *alg)
1655 {
1656         switch (key_len) {
1657         case ICP_QAT_HW_AES_128_KEY_SZ:
1658                 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
1659                 break;
1660         default:
1661                 return -EINVAL;
1662         }
1663         return 0;
1664 }
1665
1666 int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
1667 {
1668         switch (key_len) {
1669         case ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ:
1670                 *alg = ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2;
1671                 break;
1672         default:
1673                 return -EINVAL;
1674         }
1675         return 0;
1676 }
1677
1678 int qat_alg_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
1679 {
1680         switch (key_len) {
1681         case ICP_QAT_HW_KASUMI_KEY_SZ:
1682                 *alg = ICP_QAT_HW_CIPHER_ALGO_KASUMI;
1683                 break;
1684         default:
1685                 return -EINVAL;
1686         }
1687         return 0;
1688 }
1689
1690 int qat_alg_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
1691 {
1692         switch (key_len) {
1693         case ICP_QAT_HW_DES_KEY_SZ:
1694                 *alg = ICP_QAT_HW_CIPHER_ALGO_DES;
1695                 break;
1696         default:
1697                 return -EINVAL;
1698         }
1699         return 0;
1700 }
1701
1702 int qat_alg_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
1703 {
1704         switch (key_len) {
1705         case QAT_3DES_KEY_SZ_OPT1:
1706         case QAT_3DES_KEY_SZ_OPT2:
1707                 *alg = ICP_QAT_HW_CIPHER_ALGO_3DES;
1708                 break;
1709         default:
1710                 return -EINVAL;
1711         }
1712         return 0;
1713 }
1714
1715 int qat_alg_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg)
1716 {
1717         switch (key_len) {
1718         case ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ:
1719                 *alg = ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3;
1720                 break;
1721         default:
1722                 return -EINVAL;
1723         }
1724         return 0;
1725 }