drivers/crypto/qat/qat_crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *       * Redistributions of source code must retain the above copyright
  12  *         notice, this list of conditions and the following disclaimer.
  13  *       * Redistributions in binary form must reproduce the above copyright
  14  *         notice, this list of conditions and the following disclaimer in
  15  *         the documentation and/or other materials provided with the
  16  *         distribution.
  17  *       * Neither the name of Intel Corporation nor the names of its
  18  *         contributors may be used to endorse or promote products derived
  19  *         from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_tailq.h>
  49 #include <rte_malloc.h>
  50 #include <rte_launch.h>
  51 #include <rte_eal.h>
  52 #include <rte_per_lcore.h>
  53 #include <rte_lcore.h>
  54 #include <rte_branch_prediction.h>
  55 #include <rte_mempool.h>
  56 #include <rte_mbuf.h>
  57 #include <rte_string_fns.h>
  58 #include <rte_spinlock.h>
  59 #include <rte_hexdump.h>
  60 #include <rte_crypto_sym.h>
  61 #include <rte_cryptodev_pci.h>
  62 #include <rte_byteorder.h>
  63 #include <openssl/evp.h>
  64
  65 #include "qat_logs.h"
  66 #include "qat_algs.h"
  67 #include "qat_crypto.h"
  68 #include "adf_transport_access_macros.h"
  69
  70 #define BYTE_LENGTH    8
  71
  72 static int
  73 qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
  74                 struct qat_pmd_private *internals) {
  75         int i = 0;
  76         const struct rte_cryptodev_capabilities *capability;
  77
  78         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
  79                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
  80                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
  81                         continue;
  82
  83                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
  84                         continue;
  85
  86                 if (capability->sym.cipher.algo == algo)
  87                         return 1;
  88         }
  89         return 0;
  90 }
  91
  92 static int
  93 qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
  94                 struct qat_pmd_private *internals) {
  95         int i = 0;
  96         const struct rte_cryptodev_capabilities *capability;
  97
  98         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
  99                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 100                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
 101                         continue;
 102
 103                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
 104                         continue;
 105
 106                 if (capability->sym.auth.algo == algo)
 107                         return 1;
 108         }
 109         return 0;
 110 }
 111
 112 /** Encrypt a single partial block
 113  *  Depends on openssl libcrypto
 114  *  Uses ECB+XOR to do CFB encryption, same result, more performant
 115  */
 116 static inline int
 117 bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
 118                 uint8_t *iv, int ivlen, int srclen,
 119                 void *bpi_ctx)
 120 {
 121         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
 122         int encrypted_ivlen;
 123         uint8_t encrypted_iv[16];
 124         int i;
 125
 126         /* ECB method: encrypt the IV, then XOR this with plaintext */
 127         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
 128                                                                 <= 0)
 129                 goto cipher_encrypt_err;
 130
 131         for (i = 0; i < srclen; i++)
 132                 *(dst+i) = *(src+i)^(encrypted_iv[i]);
 133
 134         return 0;
 135
 136 cipher_encrypt_err:
 137         PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
 138         return -EINVAL;
 139 }
 140
 141 /** Decrypt a single partial block
 142  *  Depends on openssl libcrypto
 143  *  Uses ECB+XOR to do CFB encryption, same result, more performant
 144  */
 145 static inline int
 146 bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
 147                 uint8_t *iv, int ivlen, int srclen,
 148                 void *bpi_ctx)
 149 {
 150         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
 151         int encrypted_ivlen;
 152         uint8_t encrypted_iv[16];
 153         int i;
 154
 155         /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
 156         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
 157                                                                 <= 0)
 158                 goto cipher_decrypt_err;
 159
 160         for (i = 0; i < srclen; i++)
 161                 *(dst+i) = *(src+i)^(encrypted_iv[i]);
 162
 163         return 0;
 164
 165 cipher_decrypt_err:
 166         PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt for BPI IV failed");
 167         return -EINVAL;
 168 }
 169
 170 /** Creates a context in either AES or DES in ECB mode
 171  *  Depends on openssl libcrypto
 172  */
 173 static int
 174 bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
 175                 enum rte_crypto_cipher_operation direction __rte_unused,
 176                 uint8_t *key, void **ctx)
 177 {
 178         const EVP_CIPHER *algo = NULL;
 179         int ret;
 180         *ctx = EVP_CIPHER_CTX_new();
 181
 182         if (*ctx == NULL) {
 183                 ret = -ENOMEM;
 184                 goto ctx_init_err;
 185         }
 186
 187         if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI)
 188                 algo = EVP_des_ecb();
 189         else
 190                 algo = EVP_aes_128_ecb();
 191
 192         /* IV will be ECB encrypted whether direction is encrypt or decrypt*/
 193         if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) {
 194                 ret = -EINVAL;
 195                 goto ctx_init_err;
 196         }
 197
 198         return 0;
 199
 200 ctx_init_err:
 201         if (*ctx != NULL)
 202                 EVP_CIPHER_CTX_free(*ctx);
 203         return ret;
 204 }
 205
 206 /** Frees a context previously created
 207  *  Depends on openssl libcrypto
 208  */
 209 static void
 210 bpi_cipher_ctx_free(void *bpi_ctx)
 211 {
 212         if (bpi_ctx != NULL)
 213                 EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)bpi_ctx);
 214 }
 215
 216 static inline uint32_t
 217 adf_modulo(uint32_t data, uint32_t shift);
 218
 219 static inline int
 220 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
 221                 struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp);
 222
 223 void
 224 qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 225                 struct rte_cryptodev_sym_session *sess)
 226 {
 227         PMD_INIT_FUNC_TRACE();
 228         uint8_t index = dev->driver_id;
 229         void *sess_priv = get_session_private_data(sess, index);
 230         struct qat_session *s = (struct qat_session *)sess_priv;
 231
 232         if (sess_priv) {
 233                 if (s->bpi_ctx)
 234                         bpi_cipher_ctx_free(s->bpi_ctx);
 235                 memset(s, 0, qat_crypto_sym_get_session_private_size(dev));
 236                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 237                 set_session_private_data(sess, index, NULL);
 238                 rte_mempool_put(sess_mp, sess_priv);
 239         }
 240 }
 241
 242 static int
 243 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 244 {
 245         /* Cipher Only */
 246         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 247                 return ICP_QAT_FW_LA_CMD_CIPHER;
 248
 249         /* Authentication Only */
 250         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 251                 return ICP_QAT_FW_LA_CMD_AUTH;
 252
 253         /* AEAD */
 254         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 255                 /* AES-GCM and AES-CCM works with different direction
 256                  * GCM first encrypts and generate hash where AES-CCM
 257                  * first generate hash and encrypts. Similar relation
 258                  * applies to decryption.
 259                  */
 260                 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 261                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 262                                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 263                         else
 264                                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 265                 else
 266                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 267                                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 268                         else
 269                                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 270         }
 271
 272         if (xform->next == NULL)
 273                 return -1;
 274
 275         /* Cipher then Authenticate */
 276         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 277                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 278                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 279
 280         /* Authenticate then Cipher */
 281         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 282                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 283                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 284
 285         return -1;
 286 }
 287
 288 static struct rte_crypto_auth_xform *
 289 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 290 {
 291         do {
 292                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 293                         return &xform->auth;
 294
 295                 xform = xform->next;
 296         } while (xform);
 297
 298         return NULL;
 299 }
 300
 301 static struct rte_crypto_cipher_xform *
 302 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 303 {
 304         do {
 305                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 306                         return &xform->cipher;
 307
 308                 xform = xform->next;
 309         } while (xform);
 310
 311         return NULL;
 312 }
 313
 314 int
 315 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 316                 struct rte_crypto_sym_xform *xform,
 317                 struct qat_session *session)
 318 {
 319         struct qat_pmd_private *internals = dev->data->dev_private;
 320         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 321         int ret;
 322
 323         /* Get cipher xform from crypto xform chain */
 324         cipher_xform = qat_get_cipher_xform(xform);
 325
 326         session->cipher_iv.offset = cipher_xform->iv.offset;
 327         session->cipher_iv.length = cipher_xform->iv.length;
 328
 329         switch (cipher_xform->algo) {
 330         case RTE_CRYPTO_CIPHER_AES_CBC:
 331                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 332                                 &session->qat_cipher_alg) != 0) {
 333                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 334                         ret = -EINVAL;
 335                         goto error_out;
 336                 }
 337                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 338                 break;
 339         case RTE_CRYPTO_CIPHER_AES_CTR:
 340                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 341                                 &session->qat_cipher_alg) != 0) {
 342                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 343                         ret = -EINVAL;
 344                         goto error_out;
 345                 }
 346                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 347                 break;
 348         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 349                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 350                                         &session->qat_cipher_alg) != 0) {
 351                         PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size");
 352                         ret = -EINVAL;
 353                         goto error_out;
 354                 }
 355                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 356                 break;
 357         case RTE_CRYPTO_CIPHER_NULL:
 358                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 359                 break;
 360         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 361                 if (qat_alg_validate_kasumi_key(cipher_xform->key.length,
 362                                         &session->qat_cipher_alg) != 0) {
 363                         PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size");
 364                         ret = -EINVAL;
 365                         goto error_out;
 366                 }
 367                 session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE;
 368                 break;
 369         case RTE_CRYPTO_CIPHER_3DES_CBC:
 370                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 371                                 &session->qat_cipher_alg) != 0) {
 372                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 373                         ret = -EINVAL;
 374                         goto error_out;
 375                 }
 376                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 377                 break;
 378         case RTE_CRYPTO_CIPHER_DES_CBC:
 379                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 380                                 &session->qat_cipher_alg) != 0) {
 381                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 382                         ret = -EINVAL;
 383                         goto error_out;
 384                 }
 385                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 386                 break;
 387         case RTE_CRYPTO_CIPHER_3DES_CTR:
 388                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 389                                 &session->qat_cipher_alg) != 0) {
 390                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 391                         ret = -EINVAL;
 392                         goto error_out;
 393                 }
 394                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 395                 break;
 396         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 397                 ret = bpi_cipher_ctx_init(
 398                                         cipher_xform->algo,
 399                                         cipher_xform->op,
 400                                         cipher_xform->key.data,
 401                                         &session->bpi_ctx);
 402                 if (ret != 0) {
 403                         PMD_DRV_LOG(ERR, "failed to create DES BPI ctx");
 404                         goto error_out;
 405                 }
 406                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 407                                 &session->qat_cipher_alg) != 0) {
 408                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 409                         ret = -EINVAL;
 410                         goto error_out;
 411                 }
 412                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 413                 break;
 414         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 415                 ret = bpi_cipher_ctx_init(
 416                                         cipher_xform->algo,
 417                                         cipher_xform->op,
 418                                         cipher_xform->key.data,
 419                                         &session->bpi_ctx);
 420                 if (ret != 0) {
 421                         PMD_DRV_LOG(ERR, "failed to create AES BPI ctx");
 422                         goto error_out;
 423                 }
 424                 if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length,
 425                                 &session->qat_cipher_alg) != 0) {
 426                         PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size");
 427                         ret = -EINVAL;
 428                         goto error_out;
 429                 }
 430                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 431                 break;
 432         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 433                 if (!qat_is_cipher_alg_supported(
 434                         cipher_xform->algo, internals)) {
 435                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 436                                 rte_crypto_cipher_algorithm_strings
 437                                         [cipher_xform->algo]);
 438                         ret = -ENOTSUP;
 439                         goto error_out;
 440                 }
 441                 if (qat_alg_validate_zuc_key(cipher_xform->key.length,
 442                                 &session->qat_cipher_alg) != 0) {
 443                         PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size");
 444                         ret = -EINVAL;
 445                         goto error_out;
 446                 }
 447                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 448                 break;
 449         case RTE_CRYPTO_CIPHER_3DES_ECB:
 450         case RTE_CRYPTO_CIPHER_AES_ECB:
 451         case RTE_CRYPTO_CIPHER_AES_F8:
 452         case RTE_CRYPTO_CIPHER_AES_XTS:
 453         case RTE_CRYPTO_CIPHER_ARC4:
 454                 PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u",
 455                                 cipher_xform->algo);
 456                 ret = -ENOTSUP;
 457                 goto error_out;
 458         default:
 459                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 460                                 cipher_xform->algo);
 461                 ret = -EINVAL;
 462                 goto error_out;
 463         }
 464
 465         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 466                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 467         else
 468                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 469
 470         if (qat_alg_aead_session_create_content_desc_cipher(session,
 471                                                 cipher_xform->key.data,
 472                                                 cipher_xform->key.length)) {
 473                 ret = -EINVAL;
 474                 goto error_out;
 475         }
 476
 477         return 0;
 478
 479 error_out:
 480         if (session->bpi_ctx) {
 481                 bpi_cipher_ctx_free(session->bpi_ctx);
 482                 session->bpi_ctx = NULL;
 483         }
 484         return ret;
 485 }
 486
 487 int
 488 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 489                 struct rte_crypto_sym_xform *xform,
 490                 struct rte_cryptodev_sym_session *sess,
 491                 struct rte_mempool *mempool)
 492 {
 493         void *sess_private_data;
 494         int ret;
 495
 496         if (rte_mempool_get(mempool, &sess_private_data)) {
 497                 CDEV_LOG_ERR(
 498                         "Couldn't get object from session mempool");
 499                 return -ENOMEM;
 500         }
 501
 502         ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data);
 503         if (ret != 0) {
 504                 PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure "
 505                                 "session parameters");
 506
 507                 /* Return session to mempool */
 508                 rte_mempool_put(mempool, sess_private_data);
 509                 return ret;
 510         }
 511
 512         set_session_private_data(sess, dev->driver_id,
 513                 sess_private_data);
 514
 515         return 0;
 516 }
 517
 518 int
 519 qat_crypto_set_session_parameters(struct rte_cryptodev *dev,
 520                 struct rte_crypto_sym_xform *xform, void *session_private)
 521 {
 522         struct qat_session *session = session_private;
 523         int ret;
 524
 525         int qat_cmd_id;
 526         PMD_INIT_FUNC_TRACE();
 527
 528         /* Set context descriptor physical address */
 529         session->cd_paddr = rte_mempool_virt2phy(NULL, session) +
 530                         offsetof(struct qat_session, cd);
 531
 532         session->min_qat_dev_gen = QAT_GEN1;
 533
 534         /* Get requested QAT command id */
 535         qat_cmd_id = qat_get_cmd_id(xform);
 536         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 537                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 538                 return -ENOTSUP;
 539         }
 540         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 541         switch (session->qat_cmd) {
 542         case ICP_QAT_FW_LA_CMD_CIPHER:
 543                 ret = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 544                 if (ret < 0)
 545                         return ret;
 546                 break;
 547         case ICP_QAT_FW_LA_CMD_AUTH:
 548                 ret = qat_crypto_sym_configure_session_auth(dev, xform, session);
 549                 if (ret < 0)
 550                         return ret;
 551                 break;
 552         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 553                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 554                         ret = qat_crypto_sym_configure_session_aead(xform,
 555                                         session);
 556                         if (ret < 0)
 557                                 return ret;
 558                 } else {
 559                         ret = qat_crypto_sym_configure_session_cipher(dev,
 560                                         xform, session);
 561                         if (ret < 0)
 562                                 return ret;
 563                         ret = qat_crypto_sym_configure_session_auth(dev,
 564                                         xform, session);
 565                         if (ret < 0)
 566                                 return ret;
 567                 }
 568                 break;
 569         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 570                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 571                         ret = qat_crypto_sym_configure_session_aead(xform,
 572                                         session);
 573                         if (ret < 0)
 574                                 return ret;
 575                 } else {
 576                         ret = qat_crypto_sym_configure_session_auth(dev,
 577                                         xform, session);
 578                         if (ret < 0)
 579                                 return ret;
 580                         ret = qat_crypto_sym_configure_session_cipher(dev,
 581                                         xform, session);
 582                         if (ret < 0)
 583                                 return ret;
 584                 }
 585                 break;
 586         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 587         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 588         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 589         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 590         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 591         case ICP_QAT_FW_LA_CMD_MGF1:
 592         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 593         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 594         case ICP_QAT_FW_LA_CMD_DELIMITER:
 595         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 596                 session->qat_cmd);
 597                 return -ENOTSUP;
 598         default:
 599         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 600                 session->qat_cmd);
 601                 return -ENOTSUP;
 602         }
 603
 604         return 0;
 605 }
 606
 607 int
 608 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 609                                 struct rte_crypto_sym_xform *xform,
 610                                 struct qat_session *session)
 611 {
 612         struct rte_crypto_auth_xform *auth_xform = NULL;
 613         struct qat_pmd_private *internals = dev->data->dev_private;
 614         auth_xform = qat_get_auth_xform(xform);
 615         uint8_t *key_data = auth_xform->key.data;
 616         uint8_t key_length = auth_xform->key.length;
 617
 618         switch (auth_xform->algo) {
 619         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 620                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 621                 break;
 622         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 623                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
 624                 break;
 625         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 626                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 627                 break;
 628         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 629                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
 630                 break;
 631         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 632                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 633                 break;
 634         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 635                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 636                 break;
 637         case RTE_CRYPTO_AUTH_AES_GMAC:
 638                 if (qat_alg_validate_aes_key(auth_xform->key.length,
 639                                 &session->qat_cipher_alg) != 0) {
 640                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 641                         return -EINVAL;
 642                 }
 643                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 644                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 645
 646                 break;
 647         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 648                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 649                 break;
 650         case RTE_CRYPTO_AUTH_MD5_HMAC:
 651                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
 652                 break;
 653         case RTE_CRYPTO_AUTH_NULL:
 654                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
 655                 break;
 656         case RTE_CRYPTO_AUTH_KASUMI_F9:
 657                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9;
 658                 break;
 659         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 660                 if (!qat_is_auth_alg_supported(auth_xform->algo, internals)) {
 661                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 662                                 rte_crypto_auth_algorithm_strings
 663                                 [auth_xform->algo]);
 664                         return -ENOTSUP;
 665                 }
 666                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3;
 667                 break;
 668         case RTE_CRYPTO_AUTH_SHA1:
 669         case RTE_CRYPTO_AUTH_SHA256:
 670         case RTE_CRYPTO_AUTH_SHA512:
 671         case RTE_CRYPTO_AUTH_SHA224:
 672         case RTE_CRYPTO_AUTH_SHA384:
 673         case RTE_CRYPTO_AUTH_MD5:
 674         case RTE_CRYPTO_AUTH_AES_CMAC:
 675         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 676                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 677                                 auth_xform->algo);
 678                 return -ENOTSUP;
 679         default:
 680                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 681                                 auth_xform->algo);
 682                 return -EINVAL;
 683         }
 684
 685         session->auth_iv.offset = auth_xform->iv.offset;
 686         session->auth_iv.length = auth_xform->iv.length;
 687
 688         if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 689                 if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 690                         session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 691                         session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 692                         /*
 693                          * It needs to create cipher desc content first,
 694                          * then authentication
 695                          */
 696                         if (qat_alg_aead_session_create_content_desc_cipher(session,
 697                                                 auth_xform->key.data,
 698                                                 auth_xform->key.length))
 699                                 return -EINVAL;
 700
 701                         if (qat_alg_aead_session_create_content_desc_auth(session,
 702                                                 key_data,
 703                                                 key_length,
 704                                                 0,
 705                                                 auth_xform->digest_length,
 706                                                 auth_xform->op))
 707                                 return -EINVAL;
 708                 } else {
 709                         session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 710                         session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 711                         /*
 712                          * It needs to create authentication desc content first,
 713                          * then cipher
 714                          */
 715                         if (qat_alg_aead_session_create_content_desc_auth(session,
 716                                         key_data,
 717                                         key_length,
 718                                         0,
 719                                         auth_xform->digest_length,
 720                                         auth_xform->op))
 721                                 return -EINVAL;
 722
 723                         if (qat_alg_aead_session_create_content_desc_cipher(session,
 724                                                 auth_xform->key.data,
 725                                                 auth_xform->key.length))
 726                                 return -EINVAL;
 727                 }
 728                 /* Restore to authentication only only */
 729                 session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH;
 730         } else {
 731                 if (qat_alg_aead_session_create_content_desc_auth(session,
 732                                 key_data,
 733                                 key_length,
 734                                 0,
 735                                 auth_xform->digest_length,
 736                                 auth_xform->op))
 737                         return -EINVAL;
 738         }
 739
 740         session->digest_length = auth_xform->digest_length;
 741         return 0;
 742 }
 743
 744 int
 745 qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform,
 746                                 struct qat_session *session)
 747 {
 748         struct rte_crypto_aead_xform *aead_xform = &xform->aead;
 749         enum rte_crypto_auth_operation crypto_operation;
 750
 751         /*
 752          * Store AEAD IV parameters as cipher IV,
 753          * to avoid unnecessary memory usage
 754          */
 755         session->cipher_iv.offset = xform->aead.iv.offset;
 756         session->cipher_iv.length = xform->aead.iv.length;
 757
 758         switch (aead_xform->algo) {
 759         case RTE_CRYPTO_AEAD_AES_GCM:
 760                 if (qat_alg_validate_aes_key(aead_xform->key.length,
 761                                 &session->qat_cipher_alg) != 0) {
 762                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 763                         return -EINVAL;
 764                 }
 765                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 766                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 767                 break;
 768         case RTE_CRYPTO_AEAD_AES_CCM:
 769                 if (qat_alg_validate_aes_key(aead_xform->key.length,
 770                                 &session->qat_cipher_alg) != 0) {
 771                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 772                         return -EINVAL;
 773                 }
 774                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 775                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC;
 776                 break;
 777         default:
 778                 PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n",
 779                                 aead_xform->algo);
 780                 return -EINVAL;
 781         }
 782
 783         if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
 784                         aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) ||
 785                         (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
 786                         aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) {
 787                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 788                 /*
 789                  * It needs to create cipher desc content first,
 790                  * then authentication
 791                  */
 792
 793                 crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
 794                         RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY;
 795
 796                 if (qat_alg_aead_session_create_content_desc_cipher(session,
 797                                         aead_xform->key.data,
 798                                         aead_xform->key.length))
 799                         return -EINVAL;
 800
 801                 if (qat_alg_aead_session_create_content_desc_auth(session,
 802                                         aead_xform->key.data,
 803                                         aead_xform->key.length,
 804                                         aead_xform->aad_length,
 805                                         aead_xform->digest_length,
 806                                         crypto_operation))
 807                         return -EINVAL;
 808         } else {
 809                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 810                 /*
 811                  * It needs to create authentication desc content first,
 812                  * then cipher
 813                  */
 814
 815                 crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
 816                         RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE;
 817
 818                 if (qat_alg_aead_session_create_content_desc_auth(session,
 819                                         aead_xform->key.data,
 820                                         aead_xform->key.length,
 821                                         aead_xform->aad_length,
 822                                         aead_xform->digest_length,
 823                                         crypto_operation))
 824                         return -EINVAL;
 825
 826                 if (qat_alg_aead_session_create_content_desc_cipher(session,
 827                                         aead_xform->key.data,
 828                                         aead_xform->key.length))
 829                         return -EINVAL;
 830         }
 831
 832         session->digest_length = aead_xform->digest_length;
 833         return 0;
 834 }
 835
 836 unsigned qat_crypto_sym_get_session_private_size(
 837                 struct rte_cryptodev *dev __rte_unused)
 838 {
 839         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 840 }
 841
 842 static inline uint32_t
 843 qat_bpicipher_preprocess(struct qat_session *ctx,
 844                                 struct rte_crypto_op *op)
 845 {
 846         uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
 847         struct rte_crypto_sym_op *sym_op = op->sym;
 848         uint8_t last_block_len = block_len > 0 ?
 849                         sym_op->cipher.data.length % block_len : 0;
 850
 851         if (last_block_len &&
 852                         ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
 853
 854                 /* Decrypt last block */
 855                 uint8_t *last_block, *dst, *iv;
 856                 uint32_t last_block_offset = sym_op->cipher.data.offset +
 857                                 sym_op->cipher.data.length - last_block_len;
 858                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
 859                                 uint8_t *, last_block_offset);
 860
 861                 if (unlikely(sym_op->m_dst != NULL))
 862                         /* out-of-place operation (OOP) */
 863                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
 864                                                 uint8_t *, last_block_offset);
 865                 else
 866                         dst = last_block;
 867
 868                 if (last_block_len < sym_op->cipher.data.length)
 869                         /* use previous block ciphertext as IV */
 870                         iv = last_block - block_len;
 871                 else
 872                         /* runt block, i.e. less than one full block */
 873                         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 874                                         ctx->cipher_iv.offset);
 875
 876 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 877                 rte_hexdump(stdout, "BPI: src before pre-process:", last_block,
 878                         last_block_len);
 879                 if (sym_op->m_dst != NULL)
 880                         rte_hexdump(stdout, "BPI: dst before pre-process:", dst,
 881                                 last_block_len);
 882 #endif
 883                 bpi_cipher_decrypt(last_block, dst, iv, block_len,
 884                                 last_block_len, ctx->bpi_ctx);
 885 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 886                 rte_hexdump(stdout, "BPI: src after pre-process:", last_block,
 887                         last_block_len);
 888                 if (sym_op->m_dst != NULL)
 889                         rte_hexdump(stdout, "BPI: dst after pre-process:", dst,
 890                                 last_block_len);
 891 #endif
 892         }
 893
 894         return sym_op->cipher.data.length - last_block_len;
 895 }
 896
 897 static inline uint32_t
 898 qat_bpicipher_postprocess(struct qat_session *ctx,
 899                                 struct rte_crypto_op *op)
 900 {
 901         uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
 902         struct rte_crypto_sym_op *sym_op = op->sym;
 903         uint8_t last_block_len = block_len > 0 ?
 904                         sym_op->cipher.data.length % block_len : 0;
 905
 906         if (last_block_len > 0 &&
 907                         ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
 908
 909                 /* Encrypt last block */
 910                 uint8_t *last_block, *dst, *iv;
 911                 uint32_t last_block_offset;
 912
 913                 last_block_offset = sym_op->cipher.data.offset +
 914                                 sym_op->cipher.data.length - last_block_len;
 915                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
 916                                 uint8_t *, last_block_offset);
 917
 918                 if (unlikely(sym_op->m_dst != NULL))
 919                         /* out-of-place operation (OOP) */
 920                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
 921                                                 uint8_t *, last_block_offset);
 922                 else
 923                         dst = last_block;
 924
 925                 if (last_block_len < sym_op->cipher.data.length)
 926                         /* use previous block ciphertext as IV */
 927                         iv = dst - block_len;
 928                 else
 929                         /* runt block, i.e. less than one full block */
 930                         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 931                                         ctx->cipher_iv.offset);
 932
 933 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 934                 rte_hexdump(stdout, "BPI: src before post-process:", last_block,
 935                         last_block_len);
 936                 if (sym_op->m_dst != NULL)
 937                         rte_hexdump(stdout, "BPI: dst before post-process:",
 938                                         dst, last_block_len);
 939 #endif
 940                 bpi_cipher_encrypt(last_block, dst, iv, block_len,
 941                                 last_block_len, ctx->bpi_ctx);
 942 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 943                 rte_hexdump(stdout, "BPI: src after post-process:", last_block,
 944                         last_block_len);
 945                 if (sym_op->m_dst != NULL)
 946                         rte_hexdump(stdout, "BPI: dst after post-process:", dst,
 947                                 last_block_len);
 948 #endif
 949         }
 950         return sym_op->cipher.data.length - last_block_len;
 951 }
 952
 953 static inline void
 954 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 955         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 956                         q->hw_queue_number, q->tail);
 957         q->nb_pending_requests = 0;
 958         q->csr_tail = q->tail;
 959 }
 960
 961 uint16_t
 962 qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 963                 uint16_t nb_ops)
 964 {
 965         register struct qat_queue *queue;
 966         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 967         register uint32_t nb_ops_sent = 0;
 968         register struct rte_crypto_op **cur_op = ops;
 969         register int ret;
 970         uint16_t nb_ops_possible = nb_ops;
 971         register uint8_t *base_addr;
 972         register uint32_t tail;
 973         int overflow;
 974
 975         if (unlikely(nb_ops == 0))
 976                 return 0;
 977
 978         /* read params used a lot in main loop into registers */
 979         queue = &(tmp_qp->tx_q);
 980         base_addr = (uint8_t *)queue->base_addr;
 981         tail = queue->tail;
 982
 983         /* Find how many can actually fit on the ring */
 984         tmp_qp->inflights16 += nb_ops;
 985         overflow = tmp_qp->inflights16 - queue->max_inflights;
 986         if (overflow > 0) {
 987                 tmp_qp->inflights16 -= overflow;
 988                 nb_ops_possible = nb_ops - overflow;
 989                 if (nb_ops_possible == 0)
 990                         return 0;
 991         }
 992
 993         while (nb_ops_sent != nb_ops_possible) {
 994                 ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail,
 995                         tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp);
 996                 if (ret != 0) {
 997                         tmp_qp->stats.enqueue_err_count++;
 998                         /*
 999                          * This message cannot be enqueued,
1000                          * decrease number of ops that wasn't sent
1001                          */
1002                         tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
1003                         if (nb_ops_sent == 0)
1004                                 return 0;
1005                         goto kick_tail;
1006                 }
1007
1008                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
1009                 nb_ops_sent++;
1010                 cur_op++;
1011         }
1012 kick_tail:
1013         queue->tail = tail;
1014         tmp_qp->stats.enqueued_count += nb_ops_sent;
1015         queue->nb_pending_requests += nb_ops_sent;
1016         if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
1017                         queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
1018                 txq_write_tail(tmp_qp, queue);
1019         }
1020         return nb_ops_sent;
1021 }
1022
1023 static inline
1024 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
1025 {
1026         uint32_t old_head, new_head;
1027         uint32_t max_head;
1028
1029         old_head = q->csr_head;
1030         new_head = q->head;
1031         max_head = qp->nb_descriptors * q->msg_size;
1032
1033         /* write out free descriptors */
1034         void *cur_desc = (uint8_t *)q->base_addr + old_head;
1035
1036         if (new_head < old_head) {
1037                 memset(cur_desc, ADF_RING_EMPTY_SIG, max_head - old_head);
1038                 memset(q->base_addr, ADF_RING_EMPTY_SIG, new_head);
1039         } else {
1040                 memset(cur_desc, ADF_RING_EMPTY_SIG, new_head - old_head);
1041         }
1042         q->nb_processed_responses = 0;
1043         q->csr_head = new_head;
1044
1045         /* write current head to CSR */
1046         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
1047                             q->hw_queue_number, new_head);
1048 }
1049
1050 uint16_t
1051 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
1052                 uint16_t nb_ops)
1053 {
1054         struct qat_queue *rx_queue, *tx_queue;
1055         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
1056         uint32_t msg_counter = 0;
1057         struct rte_crypto_op *rx_op;
1058         struct icp_qat_fw_comn_resp *resp_msg;
1059         uint32_t head;
1060
1061         rx_queue = &(tmp_qp->rx_q);
1062         tx_queue = &(tmp_qp->tx_q);
1063         head = rx_queue->head;
1064         resp_msg = (struct icp_qat_fw_comn_resp *)
1065                         ((uint8_t *)rx_queue->base_addr + head);
1066
1067         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
1068                         msg_counter != nb_ops) {
1069                 rx_op = (struct rte_crypto_op *)(uintptr_t)
1070                                 (resp_msg->opaque_data);
1071
1072 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
1073                 rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
1074                         sizeof(struct icp_qat_fw_comn_resp));
1075 #endif
1076                 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
1077                                 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
1078                                         resp_msg->comn_hdr.comn_status)) {
1079                         rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1080                 } else {
1081                         struct qat_session *sess = (struct qat_session *)
1082                                         get_session_private_data(
1083                                         rx_op->sym->session,
1084                                         cryptodev_qat_driver_id);
1085
1086                         if (sess->bpi_ctx)
1087                                 qat_bpicipher_postprocess(sess, rx_op);
1088                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1089                 }
1090
1091                 head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo);
1092                 resp_msg = (struct icp_qat_fw_comn_resp *)
1093                                 ((uint8_t *)rx_queue->base_addr + head);
1094                 *ops = rx_op;
1095                 ops++;
1096                 msg_counter++;
1097         }
1098         if (msg_counter > 0) {
1099                 rx_queue->head = head;
1100                 tmp_qp->stats.dequeued_count += msg_counter;
1101                 rx_queue->nb_processed_responses += msg_counter;
1102                 tmp_qp->inflights16 -= msg_counter;
1103
1104                 if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
1105                         rxq_free_desc(tmp_qp, rx_queue);
1106         }
1107         /* also check if tail needs to be advanced */
1108         if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
1109                         tx_queue->tail != tx_queue->csr_tail) {
1110                 txq_write_tail(tmp_qp, tx_queue);
1111         }
1112         return msg_counter;
1113 }
1114
1115 static inline int
1116 qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start,
1117                 struct qat_alg_buf_list *list, uint32_t data_len)
1118 {
1119         int nr = 1;
1120
1121         uint32_t buf_len = rte_pktmbuf_mtophys(buf) -
1122                         buff_start + rte_pktmbuf_data_len(buf);
1123
1124         list->bufers[0].addr = buff_start;
1125         list->bufers[0].resrvd = 0;
1126         list->bufers[0].len = buf_len;
1127
1128         if (data_len <= buf_len) {
1129                 list->num_bufs = nr;
1130                 list->bufers[0].len = data_len;
1131                 return 0;
1132         }
1133
1134         buf = buf->next;
1135         while (buf) {
1136                 if (unlikely(nr == QAT_SGL_MAX_NUMBER)) {
1137                         PMD_DRV_LOG(ERR, "QAT PMD exceeded size of QAT SGL"
1138                                         " entry(%u)",
1139                                         QAT_SGL_MAX_NUMBER);
1140                         return -EINVAL;
1141                 }
1142
1143                 list->bufers[nr].len = rte_pktmbuf_data_len(buf);
1144                 list->bufers[nr].resrvd = 0;
1145                 list->bufers[nr].addr = rte_pktmbuf_mtophys(buf);
1146
1147                 buf_len += list->bufers[nr].len;
1148                 buf = buf->next;
1149
1150                 if (buf_len > data_len) {
1151                         list->bufers[nr].len -=
1152                                 buf_len - data_len;
1153                         buf = NULL;
1154                 }
1155                 ++nr;
1156         }
1157         list->num_bufs = nr;
1158
1159         return 0;
1160 }
1161
1162 static inline void
1163 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
1164                 struct icp_qat_fw_la_cipher_req_params *cipher_param,
1165                 struct rte_crypto_op *op,
1166                 struct icp_qat_fw_la_bulk_req *qat_req)
1167 {
1168         /* copy IV into request if it fits */
1169         if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) {
1170                 rte_memcpy(cipher_param->u.cipher_IV_array,
1171                                 rte_crypto_op_ctod_offset(op, uint8_t *,
1172                                         iv_offset),
1173                                 iv_length);
1174         } else {
1175                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
1176                                 qat_req->comn_hdr.serv_specif_flags,
1177                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
1178                 cipher_param->u.s.cipher_IV_ptr =
1179                                 rte_crypto_op_ctophys_offset(op,
1180                                         iv_offset);
1181         }
1182 }
1183
1184 /** Set IV for CCM is special case, 0th byte is set to q-1
1185  *  where q is padding of nonce in 16 byte block
1186  */
1187 static inline void
1188 set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset,
1189                 struct icp_qat_fw_la_cipher_req_params *cipher_param,
1190                 struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz)
1191 {
1192         rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) +
1193                         ICP_QAT_HW_CCM_NONCE_OFFSET,
1194                         rte_crypto_op_ctod_offset(op, uint8_t *,
1195                                 iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
1196                         iv_length);
1197         *(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
1198                         q - ICP_QAT_HW_CCM_NONCE_OFFSET;
1199
1200         if (aad_len_field_sz)
1201                 rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET],
1202                         rte_crypto_op_ctod_offset(op, uint8_t *,
1203                                 iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
1204                         iv_length);
1205 }
1206
1207 static inline int
1208 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
1209                 struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp)
1210 {
1211         int ret = 0;
1212         struct qat_session *ctx;
1213         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1214         struct icp_qat_fw_la_auth_req_params *auth_param;
1215         register struct icp_qat_fw_la_bulk_req *qat_req;
1216         uint8_t do_auth = 0, do_cipher = 0, do_aead = 0;
1217         uint32_t cipher_len = 0, cipher_ofs = 0;
1218         uint32_t auth_len = 0, auth_ofs = 0;
1219         uint32_t min_ofs = 0;
1220         uint64_t src_buf_start = 0, dst_buf_start = 0;
1221         uint8_t do_sgl = 0;
1222
1223 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
1224         if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
1225                 PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
1226                                 "operation requests, op (%p) is not a "
1227                                 "symmetric operation.", op);
1228                 return -EINVAL;
1229         }
1230 #endif
1231         if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
1232                 PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
1233                                 " requests, op (%p) is sessionless.", op);
1234                 return -EINVAL;
1235         }
1236
1237         ctx = (struct qat_session *)get_session_private_data(
1238                         op->sym->session, cryptodev_qat_driver_id);
1239
1240         if (unlikely(ctx == NULL)) {
1241                 PMD_DRV_LOG(ERR, "Session was not created for this device");
1242                 return -EINVAL;
1243         }
1244
1245         if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) {
1246                 PMD_DRV_LOG(ERR, "Session alg not supported on this device gen");
1247                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1248                 return -EINVAL;
1249         }
1250
1251
1252
1253         qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
1254         rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
1255         qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
1256         cipher_param = (void *)&qat_req->serv_specif_rqpars;
1257         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
1258
1259         if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
1260                         ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
1261                 /* AES-GCM or AES-CCM */
1262                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1263                                 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
1264                                 (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
1265                                 && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
1266                                 && ctx->qat_hash_alg ==
1267                                                 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
1268                         do_aead = 1;
1269                 } else {
1270                         do_auth = 1;
1271                         do_cipher = 1;
1272                 }
1273         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
1274                 do_auth = 1;
1275                 do_cipher = 0;
1276         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
1277                 do_auth = 0;
1278                 do_cipher = 1;
1279         }
1280
1281         if (do_cipher) {
1282
1283                 if (ctx->qat_cipher_alg ==
1284                                          ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
1285                         ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
1286                         ctx->qat_cipher_alg ==
1287                                 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
1288
1289                         if (unlikely(
1290                                 (cipher_param->cipher_length % BYTE_LENGTH != 0)
1291                                  || (cipher_param->cipher_offset
1292                                                         % BYTE_LENGTH != 0))) {
1293                                 PMD_DRV_LOG(ERR,
1294                   "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
1295                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1296                                 return -EINVAL;
1297                         }
1298                         cipher_len = op->sym->cipher.data.length >> 3;
1299                         cipher_ofs = op->sym->cipher.data.offset >> 3;
1300
1301                 } else if (ctx->bpi_ctx) {
1302                         /* DOCSIS - only send complete blocks to device
1303                          * Process any partial block using CFB mode.
1304                          * Even if 0 complete blocks, still send this to device
1305                          * to get into rx queue for post-process and dequeuing
1306                          */
1307                         cipher_len = qat_bpicipher_preprocess(ctx, op);
1308                         cipher_ofs = op->sym->cipher.data.offset;
1309                 } else {
1310                         cipher_len = op->sym->cipher.data.length;
1311                         cipher_ofs = op->sym->cipher.data.offset;
1312                 }
1313
1314                 set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset,
1315                                 cipher_param, op, qat_req);
1316                 min_ofs = cipher_ofs;
1317         }
1318
1319         if (do_auth) {
1320
1321                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
1322                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
1323                         ctx->qat_hash_alg ==
1324                                 ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
1325                         if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0)
1326                                 || (auth_param->auth_len % BYTE_LENGTH != 0))) {
1327                                 PMD_DRV_LOG(ERR,
1328                 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
1329                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1330                                 return -EINVAL;
1331                         }
1332                         auth_ofs = op->sym->auth.data.offset >> 3;
1333                         auth_len = op->sym->auth.data.length >> 3;
1334
1335                         auth_param->u1.aad_adr =
1336                                         rte_crypto_op_ctophys_offset(op,
1337                                                         ctx->auth_iv.offset);
1338
1339                 } else if (ctx->qat_hash_alg ==
1340                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1341                                 ctx->qat_hash_alg ==
1342                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
1343                         /* AES-GMAC */
1344                         set_cipher_iv(ctx->auth_iv.length,
1345                                 ctx->auth_iv.offset,
1346                                 cipher_param, op, qat_req);
1347                         auth_ofs = op->sym->auth.data.offset;
1348                         auth_len = op->sym->auth.data.length;
1349
1350                         auth_param->u1.aad_adr = 0;
1351                         auth_param->u2.aad_sz = 0;
1352
1353                         /*
1354                          * If len(iv)==12B fw computes J0
1355                          */
1356                         if (ctx->auth_iv.length == 12) {
1357                                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1358                                         qat_req->comn_hdr.serv_specif_flags,
1359                                         ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
1360
1361                         }
1362                 } else {
1363                         auth_ofs = op->sym->auth.data.offset;
1364                         auth_len = op->sym->auth.data.length;
1365
1366                 }
1367                 min_ofs = auth_ofs;
1368
1369                 auth_param->auth_res_addr = op->sym->auth.digest.phys_addr;
1370
1371         }
1372
1373         if (do_aead) {
1374                 /*
1375                  * This address may used for setting AAD physical pointer
1376                  * into IV offset from op
1377                  */
1378                 phys_addr_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr;
1379                 if (ctx->qat_hash_alg ==
1380                                 ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1381                                 ctx->qat_hash_alg ==
1382                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
1383                         /*
1384                          * If len(iv)==12B fw computes J0
1385                          */
1386                         if (ctx->cipher_iv.length == 12) {
1387                                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1388                                         qat_req->comn_hdr.serv_specif_flags,
1389                                         ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
1390                         }
1391
1392                         set_cipher_iv(ctx->cipher_iv.length,
1393                                         ctx->cipher_iv.offset,
1394                                         cipher_param, op, qat_req);
1395
1396                 } else if (ctx->qat_hash_alg ==
1397                                 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) {
1398
1399                         /* In case of AES-CCM this may point to user selected memory
1400                          * or iv offset in cypto_op
1401                          */
1402                         uint8_t *aad_data = op->sym->aead.aad.data;
1403                         /* This is true AAD length, it not includes 18 bytes of
1404                          * preceding data
1405                          */
1406                         uint8_t aad_ccm_real_len = 0;
1407
1408                         uint8_t aad_len_field_sz = 0;
1409                         uint32_t msg_len_be =
1410                                         rte_bswap32(op->sym->aead.data.length);
1411
1412                         if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
1413                                 aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
1414                                 aad_ccm_real_len = ctx->aad_len -
1415                                         ICP_QAT_HW_CCM_AAD_B0_LEN -
1416                                         ICP_QAT_HW_CCM_AAD_LEN_INFO;
1417                         } else {
1418                                 /*
1419                                  * aad_len not greater than 18, so no actual aad data,
1420                                  * then use IV after op for B0 block
1421                                  */
1422                                 aad_data = rte_crypto_op_ctod_offset(op, uint8_t *,
1423                                                 ctx->cipher_iv.offset);
1424                                 aad_phys_addr_aead =
1425                                                 rte_crypto_op_ctophys_offset(op,
1426                                                                 ctx->cipher_iv.offset);
1427                         }
1428
1429                         uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
1430
1431                         aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(aad_len_field_sz,
1432                                                         ctx->digest_length, q);
1433
1434                         if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
1435                                 memcpy(aad_data + ctx->cipher_iv.length +
1436                                         ICP_QAT_HW_CCM_NONCE_OFFSET
1437                                         + (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
1438                                         (uint8_t *)&msg_len_be,
1439                                         ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
1440                         } else {
1441                                 memcpy(aad_data + ctx->cipher_iv.length +
1442                                         ICP_QAT_HW_CCM_NONCE_OFFSET,
1443                                         (uint8_t *)&msg_len_be
1444                                         + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
1445                                         - q), q);
1446                         }
1447
1448                         if (aad_len_field_sz > 0) {
1449                                 *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN]
1450                                                 = rte_bswap16(aad_ccm_real_len);
1451
1452                                 if ((aad_ccm_real_len + aad_len_field_sz)
1453                                                 % ICP_QAT_HW_CCM_AAD_B0_LEN) {
1454                                         uint8_t pad_len = 0;
1455                                         uint8_t pad_idx = 0;
1456
1457                                         pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
1458                                                 ((aad_ccm_real_len + aad_len_field_sz) %
1459                                                         ICP_QAT_HW_CCM_AAD_B0_LEN);
1460                                         pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
1461                                                 aad_ccm_real_len + aad_len_field_sz;
1462                                         memset(&aad_data[pad_idx],
1463                                                         0, pad_len);
1464                                 }
1465
1466                         }
1467
1468                         set_cipher_iv_ccm(ctx->cipher_iv.length,
1469                                         ctx->cipher_iv.offset,
1470                                         cipher_param, op, q,
1471                                         aad_len_field_sz);
1472
1473                 }
1474
1475                 cipher_len = op->sym->aead.data.length;
1476                 cipher_ofs = op->sym->aead.data.offset;
1477                 auth_len = op->sym->aead.data.length;
1478                 auth_ofs = op->sym->aead.data.offset;
1479
1480                 auth_param->u1.aad_adr = aad_phys_addr_aead;
1481                 auth_param->auth_res_addr = op->sym->aead.digest.phys_addr;
1482                 min_ofs = op->sym->aead.data.offset;
1483         }
1484
1485         if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next))
1486                 do_sgl = 1;
1487
1488         /* adjust for chain case */
1489         if (do_cipher && do_auth)
1490                 min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
1491
1492         if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl))
1493                 min_ofs = 0;
1494
1495         if (unlikely(op->sym->m_dst != NULL)) {
1496                 /* Out-of-place operation (OOP)
1497                  * Don't align DMA start. DMA the minimum data-set
1498                  * so as not to overwrite data in dest buffer
1499                  */
1500                 src_buf_start =
1501                         rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs);
1502                 dst_buf_start =
1503                         rte_pktmbuf_mtophys_offset(op->sym->m_dst, min_ofs);
1504
1505         } else {
1506                 /* In-place operation
1507                  * Start DMA at nearest aligned address below min_ofs
1508                  */
1509                 src_buf_start =
1510                         rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs)
1511                                                 & QAT_64_BTYE_ALIGN_MASK;
1512
1513                 if (unlikely((rte_pktmbuf_mtophys(op->sym->m_src) -
1514                                         rte_pktmbuf_headroom(op->sym->m_src))
1515                                                         > src_buf_start)) {
1516                         /* alignment has pushed addr ahead of start of mbuf
1517                          * so revert and take the performance hit
1518                          */
1519                         src_buf_start =
1520                                 rte_pktmbuf_mtophys_offset(op->sym->m_src,
1521                                                                 min_ofs);
1522                 }
1523                 dst_buf_start = src_buf_start;
1524         }
1525
1526         if (do_cipher || do_aead) {
1527                 cipher_param->cipher_offset =
1528                                 (uint32_t)rte_pktmbuf_mtophys_offset(
1529                                 op->sym->m_src, cipher_ofs) - src_buf_start;
1530                 cipher_param->cipher_length = cipher_len;
1531         } else {
1532                 cipher_param->cipher_offset = 0;
1533                 cipher_param->cipher_length = 0;
1534         }
1535
1536         if (do_auth || do_aead) {
1537                 auth_param->auth_off = (uint32_t)rte_pktmbuf_mtophys_offset(
1538                                 op->sym->m_src, auth_ofs) - src_buf_start;
1539                 auth_param->auth_len = auth_len;
1540         } else {
1541                 auth_param->auth_off = 0;
1542                 auth_param->auth_len = 0;
1543         }
1544
1545         qat_req->comn_mid.dst_length =
1546                 qat_req->comn_mid.src_length =
1547                 (cipher_param->cipher_offset + cipher_param->cipher_length)
1548                 > (auth_param->auth_off + auth_param->auth_len) ?
1549                 (cipher_param->cipher_offset + cipher_param->cipher_length)
1550                 : (auth_param->auth_off + auth_param->auth_len);
1551
1552         if (do_sgl) {
1553
1554                 ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags,
1555                                 QAT_COMN_PTR_TYPE_SGL);
1556                 ret = qat_sgl_fill_array(op->sym->m_src, src_buf_start,
1557                                 &qat_op_cookie->qat_sgl_list_src,
1558                                 qat_req->comn_mid.src_length);
1559                 if (ret) {
1560                         PMD_DRV_LOG(ERR, "QAT PMD Cannot fill sgl array");
1561                         return ret;
1562                 }
1563
1564                 if (likely(op->sym->m_dst == NULL))
1565                         qat_req->comn_mid.dest_data_addr =
1566                                 qat_req->comn_mid.src_data_addr =
1567                                 qat_op_cookie->qat_sgl_src_phys_addr;
1568                 else {
1569                         ret = qat_sgl_fill_array(op->sym->m_dst,
1570                                         dst_buf_start,
1571                                         &qat_op_cookie->qat_sgl_list_dst,
1572                                                 qat_req->comn_mid.dst_length);
1573
1574                         if (ret) {
1575                                 PMD_DRV_LOG(ERR, "QAT PMD Cannot "
1576                                                 "fill sgl array");
1577                                 return ret;
1578                         }
1579
1580                         qat_req->comn_mid.src_data_addr =
1581                                 qat_op_cookie->qat_sgl_src_phys_addr;
1582                         qat_req->comn_mid.dest_data_addr =
1583                                         qat_op_cookie->qat_sgl_dst_phys_addr;
1584                 }
1585         } else {
1586                 qat_req->comn_mid.src_data_addr = src_buf_start;
1587                 qat_req->comn_mid.dest_data_addr = dst_buf_start;
1588         }
1589
1590 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
1591         rte_hexdump(stdout, "qat_req:", qat_req,
1592                         sizeof(struct icp_qat_fw_la_bulk_req));
1593         rte_hexdump(stdout, "src_data:",
1594                         rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
1595                         rte_pktmbuf_data_len(op->sym->m_src));
1596         if (do_cipher) {
1597                 uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op,
1598                                                 uint8_t *,
1599                                                 ctx->cipher_iv.offset);
1600                 rte_hexdump(stdout, "cipher iv:", cipher_iv_ptr,
1601                                 ctx->cipher_iv.length);
1602         }
1603
1604         if (do_auth) {
1605                 if (ctx->auth_iv.length) {
1606                         uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op,
1607                                                         uint8_t *,
1608                                                         ctx->auth_iv.offset);
1609                         rte_hexdump(stdout, "auth iv:", auth_iv_ptr,
1610                                                 ctx->auth_iv.length);
1611                 }
1612                 rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
1613                                 ctx->digest_length);
1614         }
1615
1616         if (do_aead) {
1617                 rte_hexdump(stdout, "digest:", op->sym->aead.digest.data,
1618                                 ctx->digest_length);
1619                 rte_hexdump(stdout, "aad:", op->sym->aead.aad.data,
1620                                 ctx->aad_len);
1621         }
1622 #endif
1623         return 0;
1624 }
1625
1626 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
1627 {
1628         uint32_t div = data >> shift;
1629         uint32_t mult = div << shift;
1630
1631         return data - mult;
1632 }
1633
1634 int qat_dev_config(__rte_unused struct rte_cryptodev *dev,
1635                 __rte_unused struct rte_cryptodev_config *config)
1636 {
1637         PMD_INIT_FUNC_TRACE();
1638         return 0;
1639 }
1640
1641 int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
1642 {
1643         PMD_INIT_FUNC_TRACE();
1644         return 0;
1645 }
1646
1647 void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
1648 {
1649         PMD_INIT_FUNC_TRACE();
1650 }
1651
1652 int qat_dev_close(struct rte_cryptodev *dev)
1653 {
1654         int i, ret;
1655
1656         PMD_INIT_FUNC_TRACE();
1657
1658         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1659                 ret = qat_crypto_sym_qp_release(dev, i);
1660                 if (ret < 0)
1661                         return ret;
1662         }
1663
1664         return 0;
1665 }
1666
1667 void qat_dev_info_get(struct rte_cryptodev *dev,
1668                         struct rte_cryptodev_info *info)
1669 {
1670         struct qat_pmd_private *internals = dev->data->dev_private;
1671
1672         PMD_INIT_FUNC_TRACE();
1673         if (info != NULL) {
1674                 info->max_nb_queue_pairs =
1675                                 ADF_NUM_SYM_QPS_PER_BUNDLE *
1676                                 ADF_NUM_BUNDLES_PER_DEV;
1677                 info->feature_flags = dev->feature_flags;
1678                 info->capabilities = internals->qat_dev_capabilities;
1679                 info->sym.max_nb_sessions = internals->max_nb_sessions;
1680                 info->driver_id = cryptodev_qat_driver_id;
1681                 info->pci_dev = RTE_DEV_TO_PCI(dev->device);
1682         }
1683 }
1684
1685 void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
1686                 struct rte_cryptodev_stats *stats)
1687 {
1688         int i;
1689         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1690
1691         PMD_INIT_FUNC_TRACE();
1692         if (stats == NULL) {
1693                 PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
1694                 return;
1695         }
1696         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1697                 if (qp[i] == NULL) {
1698                         PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
1699                         continue;
1700                 }
1701
1702                 stats->enqueued_count += qp[i]->stats.enqueued_count;
1703                 stats->dequeued_count += qp[i]->stats.dequeued_count;
1704                 stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
1705                 stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
1706         }
1707 }
1708
1709 void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
1710 {
1711         int i;
1712         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1713
1714         PMD_INIT_FUNC_TRACE();
1715         for (i = 0; i < dev->data->nb_queue_pairs; i++)
1716                 memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
1717         PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
1718 }