drivers/crypto/qat/qat_crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *       * Redistributions of source code must retain the above copyright
  12  *         notice, this list of conditions and the following disclaimer.
  13  *       * Redistributions in binary form must reproduce the above copyright
  14  *         notice, this list of conditions and the following disclaimer in
  15  *         the documentation and/or other materials provided with the
  16  *         distribution.
  17  *       * Neither the name of Intel Corporation nor the names of its
  18  *         contributors may be used to endorse or promote products derived
  19  *         from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_tailq.h>
  49 #include <rte_ether.h>
  50 #include <rte_malloc.h>
  51 #include <rte_launch.h>
  52 #include <rte_eal.h>
  53 #include <rte_per_lcore.h>
  54 #include <rte_lcore.h>
  55 #include <rte_atomic.h>
  56 #include <rte_branch_prediction.h>
  57 #include <rte_mempool.h>
  58 #include <rte_mbuf.h>
  59 #include <rte_string_fns.h>
  60 #include <rte_spinlock.h>
  61 #include <rte_hexdump.h>
  62
  63 #include "qat_logs.h"
  64 #include "qat_algs.h"
  65 #include "qat_crypto.h"
  66 #include "adf_transport_access_macros.h"
  67
  68 #define BYTE_LENGTH    8
  69
  70 static const struct rte_cryptodev_capabilities qat_pmd_capabilities[] = {
  71         {       /* SHA1 HMAC */
  72                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  73                 {.sym = {
  74                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  75                         {.auth = {
  76                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  77                                 .block_size = 64,
  78                                 .key_size = {
  79                                         .min = 64,
  80                                         .max = 64,
  81                                         .increment = 0
  82                                 },
  83                                 .digest_size = {
  84                                         .min = 20,
  85                                         .max = 20,
  86                                         .increment = 0
  87                                 },
  88                                 .aad_size = { 0 }
  89                         }, }
  90                 }, }
  91         },
  92         {       /* SHA256 HMAC */
  93                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  94                 {.sym = {
  95                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  96                         {.auth = {
  97                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
  98                                 .block_size = 64,
  99                                 .key_size = {
 100                                         .min = 64,
 101                                         .max = 64,
 102                                         .increment = 0
 103                                 },
 104                                 .digest_size = {
 105                                         .min = 32,
 106                                         .max = 32,
 107                                         .increment = 0
 108                                 },
 109                                 .aad_size = { 0 }
 110                         }, }
 111                 }, }
 112         },
 113         {       /* SHA512 HMAC */
 114                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 115                 {.sym = {
 116                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 117                         {.auth = {
 118                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 119                                 .block_size = 128,
 120                                 .key_size = {
 121                                         .min = 128,
 122                                         .max = 128,
 123                                         .increment = 0
 124                                 },
 125                                 .digest_size = {
 126                                         .min = 64,
 127                                         .max = 64,
 128                                         .increment = 0
 129                                 },
 130                                 .aad_size = { 0 }
 131                         }, }
 132                 }, }
 133         },
 134         {       /* MD5 HMAC */
 135                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 136                 {.sym = {
 137                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 138                         {.auth = {
 139                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
 140                                 .block_size = 64,
 141                                 .key_size = {
 142                                         .min = 8,
 143                                         .max = 64,
 144                                         .increment = 8
 145                                 },
 146                                 .digest_size = {
 147                                         .min = 16,
 148                                         .max = 16,
 149                                         .increment = 0
 150                                 },
 151                                 .aad_size = { 0 }
 152                         }, }
 153                 }, }
 154         },
 155         {       /* AES XCBC MAC */
 156                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 157                 {.sym = {
 158                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 159                         {.auth = {
 160                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 161                                 .block_size = 16,
 162                                 .key_size = {
 163                                         .min = 16,
 164                                         .max = 16,
 165                                         .increment = 0
 166                                 },
 167                                 .digest_size = {
 168                                         .min = 16,
 169                                         .max = 16,
 170                                         .increment = 0
 171                                 },
 172                                 .aad_size = { 0 }
 173                         }, }
 174                 }, }
 175         },
 176         {       /* AES GCM (AUTH) */
 177                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 178                 {.sym = {
 179                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 180                         {.auth = {
 181                                 .algo = RTE_CRYPTO_AUTH_AES_GCM,
 182                                 .block_size = 16,
 183                                 .key_size = {
 184                                         .min = 16,
 185                                         .max = 32,
 186                                         .increment = 8
 187                                 },
 188                                 .digest_size = {
 189                                         .min = 8,
 190                                         .max = 16,
 191                                         .increment = 4
 192                                 },
 193                                 .aad_size = {
 194                                         .min = 8,
 195                                         .max = 12,
 196                                         .increment = 4
 197                                 }
 198                         }, }
 199                 }, }
 200         },
 201         {       /* SNOW3G (UIA2) */
 202                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 203                 {.sym = {
 204                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 205                         {.auth = {
 206                                 .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
 207                                 .block_size = 16,
 208                                 .key_size = {
 209                                         .min = 16,
 210                                         .max = 16,
 211                                         .increment = 0
 212                                 },
 213                                 .digest_size = {
 214                                         .min = 4,
 215                                         .max = 4,
 216                                         .increment = 0
 217                                 },
 218                                 .aad_size = {
 219                                         .min = 16,
 220                                         .max = 16,
 221                                         .increment = 0
 222                                 }
 223                         }, }
 224                 }, }
 225         },
 226         {       /* AES GCM (CIPHER) */
 227                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 228                 {.sym = {
 229                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 230                         {.cipher = {
 231                                 .algo = RTE_CRYPTO_CIPHER_AES_GCM,
 232                                 .block_size = 16,
 233                                 .key_size = {
 234                                         .min = 16,
 235                                         .max = 32,
 236                                         .increment = 8
 237                                 },
 238                                 .iv_size = {
 239                                         .min = 16,
 240                                         .max = 16,
 241                                         .increment = 0
 242                                 }
 243                         }, }
 244                 }, }
 245         },
 246         {       /* AES CBC */
 247                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 248                 {.sym = {
 249                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 250                         {.cipher = {
 251                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 252                                 .block_size = 16,
 253                                 .key_size = {
 254                                         .min = 16,
 255                                         .max = 32,
 256                                         .increment = 8
 257                                 },
 258                                 .iv_size = {
 259                                         .min = 16,
 260                                         .max = 16,
 261                                         .increment = 0
 262                                 }
 263                         }, }
 264                 }, }
 265         },
 266         {       /* SNOW3G (UEA2) */
 267                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 268                 {.sym = {
 269                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 270                         {.cipher = {
 271                                 .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 272                                 .block_size = 16,
 273                                 .key_size = {
 274                                         .min = 16,
 275                                         .max = 16,
 276                                         .increment = 0
 277                                 },
 278                                 .iv_size = {
 279                                         .min = 16,
 280                                         .max = 16,
 281                                         .increment = 0
 282                                 }
 283                         }, }
 284                 }, }
 285         },
 286         {       /* AES CTR */
 287                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 288                 {.sym = {
 289                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 290                         {.cipher = {
 291                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 292                                 .block_size = 16,
 293                                 .key_size = {
 294                                         .min = 16,
 295                                         .max = 32,
 296                                         .increment = 8
 297                                 },
 298                                 .iv_size = {
 299                                         .min = 16,
 300                                         .max = 16,
 301                                         .increment = 0
 302                                 }
 303                         }, }
 304                 }, }
 305         },
 306         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 307 };
 308
 309 static inline uint32_t
 310 adf_modulo(uint32_t data, uint32_t shift);
 311
 312 static inline int
 313 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg);
 314
 315 void qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 316                 void *session)
 317 {
 318         struct qat_session *sess = session;
 319         phys_addr_t cd_paddr;
 320
 321         PMD_INIT_FUNC_TRACE();
 322         if (session) {
 323                 cd_paddr = sess->cd_paddr;
 324                 memset(sess, 0, qat_crypto_sym_get_session_private_size(dev));
 325                 sess->cd_paddr = cd_paddr;
 326         } else
 327                 PMD_DRV_LOG(ERR, "NULL session");
 328 }
 329
 330 static int
 331 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 332 {
 333         /* Cipher Only */
 334         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 335                 return ICP_QAT_FW_LA_CMD_CIPHER;
 336
 337         /* Authentication Only */
 338         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 339                 return ICP_QAT_FW_LA_CMD_AUTH;
 340
 341         if (xform->next == NULL)
 342                 return -1;
 343
 344         /* Cipher then Authenticate */
 345         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 346                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 347                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 348
 349         /* Authenticate then Cipher */
 350         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 351                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 352                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 353
 354         return -1;
 355 }
 356
 357 static struct rte_crypto_auth_xform *
 358 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 359 {
 360         do {
 361                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 362                         return &xform->auth;
 363
 364                 xform = xform->next;
 365         } while (xform);
 366
 367         return NULL;
 368 }
 369
 370 static struct rte_crypto_cipher_xform *
 371 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 372 {
 373         do {
 374                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 375                         return &xform->cipher;
 376
 377                 xform = xform->next;
 378         } while (xform);
 379
 380         return NULL;
 381 }
 382 void *
 383 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 384                 struct rte_crypto_sym_xform *xform, void *session_private)
 385 {
 386         struct qat_pmd_private *internals = dev->data->dev_private;
 387
 388         struct qat_session *session = session_private;
 389
 390         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 391
 392         /* Get cipher xform from crypto xform chain */
 393         cipher_xform = qat_get_cipher_xform(xform);
 394
 395         switch (cipher_xform->algo) {
 396         case RTE_CRYPTO_CIPHER_AES_CBC:
 397                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 398                                 &session->qat_cipher_alg) != 0) {
 399                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 400                         goto error_out;
 401                 }
 402                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 403                 break;
 404         case RTE_CRYPTO_CIPHER_AES_GCM:
 405                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 406                                 &session->qat_cipher_alg) != 0) {
 407                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 408                         goto error_out;
 409                 }
 410                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 411                 break;
 412         case RTE_CRYPTO_CIPHER_AES_CTR:
 413                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 414                                 &session->qat_cipher_alg) != 0) {
 415                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 416                         goto error_out;
 417                 }
 418                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 419                 break;
 420         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 421                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 422                                         &session->qat_cipher_alg) != 0) {
 423                         PMD_DRV_LOG(ERR, "Invalid SNOW3G cipher key size");
 424                         goto error_out;
 425                 }
 426                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 427                 break;
 428         case RTE_CRYPTO_CIPHER_NULL:
 429         case RTE_CRYPTO_CIPHER_3DES_ECB:
 430         case RTE_CRYPTO_CIPHER_3DES_CBC:
 431         case RTE_CRYPTO_CIPHER_AES_ECB:
 432         case RTE_CRYPTO_CIPHER_AES_CCM:
 433         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 434                 PMD_DRV_LOG(ERR, "Crypto: Unsupported Cipher alg %u",
 435                                 cipher_xform->algo);
 436                 goto error_out;
 437         default:
 438                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 439                                 cipher_xform->algo);
 440                 goto error_out;
 441         }
 442
 443         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 444                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 445         else
 446                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 447
 448         if (qat_alg_aead_session_create_content_desc_cipher(session,
 449                                                 cipher_xform->key.data,
 450                                                 cipher_xform->key.length))
 451                 goto error_out;
 452
 453         return session;
 454
 455 error_out:
 456         rte_mempool_put(internals->sess_mp, session);
 457         return NULL;
 458 }
 459
 460
 461 void *
 462 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 463                 struct rte_crypto_sym_xform *xform, void *session_private)
 464 {
 465         struct qat_pmd_private *internals = dev->data->dev_private;
 466
 467         struct qat_session *session = session_private;
 468
 469         int qat_cmd_id;
 470
 471         PMD_INIT_FUNC_TRACE();
 472
 473         /* Get requested QAT command id */
 474         qat_cmd_id = qat_get_cmd_id(xform);
 475         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 476                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 477                 goto error_out;
 478         }
 479         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 480         switch (session->qat_cmd) {
 481         case ICP_QAT_FW_LA_CMD_CIPHER:
 482         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 483                 break;
 484         case ICP_QAT_FW_LA_CMD_AUTH:
 485         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 486                 break;
 487         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 488         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 489         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 490                 break;
 491         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 492         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 493         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 494                 break;
 495         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 496         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 497         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 498         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 499         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 500         case ICP_QAT_FW_LA_CMD_MGF1:
 501         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 502         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 503         case ICP_QAT_FW_LA_CMD_DELIMITER:
 504         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 505                 session->qat_cmd);
 506                 goto error_out;
 507         default:
 508         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 509                 session->qat_cmd);
 510                 goto error_out;
 511         }
 512         return session;
 513
 514 error_out:
 515         rte_mempool_put(internals->sess_mp, session);
 516         return NULL;
 517 }
 518
 519 struct qat_session *
 520 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 521                                 struct rte_crypto_sym_xform *xform,
 522                                 struct qat_session *session_private)
 523 {
 524
 525         struct qat_pmd_private *internals = dev->data->dev_private;
 526         struct qat_session *session = session_private;
 527         struct rte_crypto_auth_xform *auth_xform = NULL;
 528         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 529         auth_xform = qat_get_auth_xform(xform);
 530
 531         switch (auth_xform->algo) {
 532         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 533                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 534                 break;
 535         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 536                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 537                 break;
 538         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 539                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 540                 break;
 541         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 542                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 543                 break;
 544         case RTE_CRYPTO_AUTH_AES_GCM:
 545                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 546                 break;
 547         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 548                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 549                 break;
 550         case RTE_CRYPTO_AUTH_MD5_HMAC:
 551                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
 552                 break;
 553         case RTE_CRYPTO_AUTH_NULL:
 554         case RTE_CRYPTO_AUTH_SHA1:
 555         case RTE_CRYPTO_AUTH_SHA256:
 556         case RTE_CRYPTO_AUTH_SHA512:
 557         case RTE_CRYPTO_AUTH_SHA224:
 558         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 559         case RTE_CRYPTO_AUTH_SHA384:
 560         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 561         case RTE_CRYPTO_AUTH_MD5:
 562         case RTE_CRYPTO_AUTH_AES_CCM:
 563         case RTE_CRYPTO_AUTH_AES_GMAC:
 564         case RTE_CRYPTO_AUTH_KASUMI_F9:
 565         case RTE_CRYPTO_AUTH_AES_CMAC:
 566         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 567         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 568                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 569                                 auth_xform->algo);
 570                 goto error_out;
 571         default:
 572                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 573                                 auth_xform->algo);
 574                 goto error_out;
 575         }
 576         cipher_xform = qat_get_cipher_xform(xform);
 577
 578         if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
 579                         (session->qat_hash_alg ==
 580                                 ICP_QAT_HW_AUTH_ALGO_GALOIS_64))  {
 581                 if (qat_alg_aead_session_create_content_desc_auth(session,
 582                                 cipher_xform->key.data,
 583                                 cipher_xform->key.length,
 584                                 auth_xform->add_auth_data_length,
 585                                 auth_xform->digest_length,
 586                                 auth_xform->op))
 587                         goto error_out;
 588         } else {
 589                 if (qat_alg_aead_session_create_content_desc_auth(session,
 590                                 auth_xform->key.data,
 591                                 auth_xform->key.length,
 592                                 auth_xform->add_auth_data_length,
 593                                 auth_xform->digest_length,
 594                                 auth_xform->op))
 595                         goto error_out;
 596         }
 597         return session;
 598
 599 error_out:
 600         if (internals->sess_mp != NULL)
 601                 rte_mempool_put(internals->sess_mp, session);
 602         return NULL;
 603 }
 604
 605 unsigned qat_crypto_sym_get_session_private_size(
 606                 struct rte_cryptodev *dev __rte_unused)
 607 {
 608         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 609 }
 610
 611
 612 uint16_t
 613 qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 614                 uint16_t nb_ops)
 615 {
 616         register struct qat_queue *queue;
 617         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 618         register uint32_t nb_ops_sent = 0;
 619         register struct rte_crypto_op **cur_op = ops;
 620         register int ret;
 621         uint16_t nb_ops_possible = nb_ops;
 622         register uint8_t *base_addr;
 623         register uint32_t tail;
 624         int overflow;
 625
 626         if (unlikely(nb_ops == 0))
 627                 return 0;
 628
 629         /* read params used a lot in main loop into registers */
 630         queue = &(tmp_qp->tx_q);
 631         base_addr = (uint8_t *)queue->base_addr;
 632         tail = queue->tail;
 633
 634         /* Find how many can actually fit on the ring */
 635         overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops)
 636                                 - queue->max_inflights;
 637         if (overflow > 0) {
 638                 rte_atomic16_sub(&tmp_qp->inflights16, overflow);
 639                 nb_ops_possible = nb_ops - overflow;
 640                 if (nb_ops_possible == 0)
 641                         return 0;
 642         }
 643
 644         while (nb_ops_sent != nb_ops_possible) {
 645                 ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail);
 646                 if (ret != 0) {
 647                         tmp_qp->stats.enqueue_err_count++;
 648                         if (nb_ops_sent == 0)
 649                                 return 0;
 650                         goto kick_tail;
 651                 }
 652
 653                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
 654                 nb_ops_sent++;
 655                 cur_op++;
 656         }
 657 kick_tail:
 658         WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number,
 659                         queue->hw_queue_number, tail);
 660         queue->tail = tail;
 661         tmp_qp->stats.enqueued_count += nb_ops_sent;
 662         return nb_ops_sent;
 663 }
 664
 665 uint16_t
 666 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 667                 uint16_t nb_ops)
 668 {
 669         struct qat_queue *queue;
 670         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 671         uint32_t msg_counter = 0;
 672         struct rte_crypto_op *rx_op;
 673         struct icp_qat_fw_comn_resp *resp_msg;
 674
 675         queue = &(tmp_qp->rx_q);
 676         resp_msg = (struct icp_qat_fw_comn_resp *)
 677                         ((uint8_t *)queue->base_addr + queue->head);
 678
 679         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 680                         msg_counter != nb_ops) {
 681                 rx_op = (struct rte_crypto_op *)(uintptr_t)
 682                                 (resp_msg->opaque_data);
 683
 684 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 685                 rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
 686                                 sizeof(struct icp_qat_fw_comn_resp));
 687 #endif
 688                 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
 689                                 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
 690                                         resp_msg->comn_hdr.comn_status)) {
 691                         rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 692                 } else {
 693                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 694                 }
 695                 *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG;
 696                 queue->head = adf_modulo(queue->head +
 697                                 queue->msg_size,
 698                                 ADF_RING_SIZE_MODULO(queue->queue_size));
 699                 resp_msg = (struct icp_qat_fw_comn_resp *)
 700                                         ((uint8_t *)queue->base_addr +
 701                                                         queue->head);
 702                 *ops = rx_op;
 703                 ops++;
 704                 msg_counter++;
 705         }
 706         if (msg_counter > 0) {
 707                 WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr,
 708                                         queue->hw_bundle_number,
 709                                         queue->hw_queue_number, queue->head);
 710                 rte_atomic16_sub(&tmp_qp->inflights16, msg_counter);
 711                 tmp_qp->stats.dequeued_count += msg_counter;
 712         }
 713         return msg_counter;
 714 }
 715
 716 static inline int
 717 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg)
 718 {
 719         struct qat_session *ctx;
 720         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 721         struct icp_qat_fw_la_auth_req_params *auth_param;
 722         register struct icp_qat_fw_la_bulk_req *qat_req;
 723
 724 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 725         if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
 726                 PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
 727                                 "operation requests, op (%p) is not a "
 728                                 "symmetric operation.", op);
 729                 return -EINVAL;
 730         }
 731 #endif
 732         if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) {
 733                 PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
 734                                 " requests, op (%p) is sessionless.", op);
 735                 return -EINVAL;
 736         }
 737
 738         if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) {
 739                 PMD_DRV_LOG(ERR, "Session was not created for this device");
 740                 return -EINVAL;
 741         }
 742
 743         ctx = (struct qat_session *)op->sym->session->_private;
 744         qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 745         rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 746         qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
 747
 748         qat_req->comn_mid.dst_length =
 749                 qat_req->comn_mid.src_length =
 750                                 rte_pktmbuf_data_len(op->sym->m_src);
 751
 752         qat_req->comn_mid.dest_data_addr =
 753                 qat_req->comn_mid.src_data_addr =
 754                             rte_pktmbuf_mtophys(op->sym->m_src);
 755
 756         if (unlikely(op->sym->m_dst != NULL)) {
 757                 qat_req->comn_mid.dest_data_addr =
 758                                 rte_pktmbuf_mtophys(op->sym->m_dst);
 759                 qat_req->comn_mid.dst_length =
 760                                 rte_pktmbuf_data_len(op->sym->m_dst);
 761         }
 762
 763         cipher_param = (void *)&qat_req->serv_specif_rqpars;
 764         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 765
 766         cipher_param->cipher_length = op->sym->cipher.data.length;
 767         cipher_param->cipher_offset = op->sym->cipher.data.offset;
 768         if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
 769                 if (unlikely((cipher_param->cipher_length % BYTE_LENGTH != 0) ||
 770                                 (cipher_param->cipher_offset
 771                                         % BYTE_LENGTH != 0))) {
 772                         PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only "
 773                                 "supports byte aligned values");
 774                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 775                         return -EINVAL;
 776                 }
 777                 cipher_param->cipher_length >>= 3;
 778                 cipher_param->cipher_offset >>= 3;
 779         }
 780
 781         if (op->sym->cipher.iv.length && (op->sym->cipher.iv.length <=
 782                         sizeof(cipher_param->u.cipher_IV_array))) {
 783                 rte_memcpy(cipher_param->u.cipher_IV_array,
 784                                 op->sym->cipher.iv.data,
 785                                 op->sym->cipher.iv.length);
 786         } else {
 787                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
 788                                 qat_req->comn_hdr.serv_specif_flags,
 789                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
 790                 cipher_param->u.s.cipher_IV_ptr = op->sym->cipher.iv.phys_addr;
 791         }
 792         if (op->sym->auth.digest.phys_addr) {
 793                 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
 794                                 qat_req->comn_hdr.serv_specif_flags,
 795                                 ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
 796                 auth_param->auth_res_addr = op->sym->auth.digest.phys_addr;
 797         }
 798         auth_param->auth_off = op->sym->auth.data.offset;
 799         auth_param->auth_len = op->sym->auth.data.length;
 800         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) {
 801                 if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0) ||
 802                                 (auth_param->auth_len % BYTE_LENGTH != 0))) {
 803                         PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only "
 804                                 "supports byte aligned values");
 805                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 806                         return -EINVAL;
 807                 }
 808                 auth_param->auth_off >>= 3;
 809                 auth_param->auth_len >>= 3;
 810         }
 811         auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr;
 812
 813         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
 814                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
 815                 if (op->sym->cipher.iv.length == 12) {
 816                         /*
 817                          * For GCM a 12 bit IV is allowed,
 818                          * but we need to inform the f/w
 819                          */
 820                         ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 821                                 qat_req->comn_hdr.serv_specif_flags,
 822                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 823                 }
 824         }
 825
 826 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 827         rte_hexdump(stdout, "qat_req:", qat_req,
 828                         sizeof(struct icp_qat_fw_la_bulk_req));
 829         rte_hexdump(stdout, "src_data:",
 830                         rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
 831                         rte_pktmbuf_data_len(op->sym->m_src));
 832         rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data,
 833                         op->sym->cipher.iv.length);
 834         rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
 835                         op->sym->auth.digest.length);
 836         rte_hexdump(stdout, "aad:", op->sym->auth.aad.data,
 837                         op->sym->auth.aad.length);
 838 #endif
 839         return 0;
 840 }
 841
 842 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
 843 {
 844         uint32_t div = data >> shift;
 845         uint32_t mult = div << shift;
 846
 847         return data - mult;
 848 }
 849
 850 void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess)
 851 {
 852         struct rte_cryptodev_sym_session *sess = sym_sess;
 853         struct qat_session *s = (void *)sess->_private;
 854
 855         PMD_INIT_FUNC_TRACE();
 856         s->cd_paddr = rte_mempool_virt2phy(mp, sess) +
 857                 offsetof(struct qat_session, cd) +
 858                 offsetof(struct rte_cryptodev_sym_session, _private);
 859 }
 860
 861 int qat_dev_config(__rte_unused struct rte_cryptodev *dev)
 862 {
 863         PMD_INIT_FUNC_TRACE();
 864         return -ENOTSUP;
 865 }
 866
 867 int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
 868 {
 869         PMD_INIT_FUNC_TRACE();
 870         return 0;
 871 }
 872
 873 void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
 874 {
 875         PMD_INIT_FUNC_TRACE();
 876 }
 877
 878 int qat_dev_close(struct rte_cryptodev *dev)
 879 {
 880         int i, ret;
 881
 882         PMD_INIT_FUNC_TRACE();
 883
 884         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 885                 ret = qat_crypto_sym_qp_release(dev, i);
 886                 if (ret < 0)
 887                         return ret;
 888         }
 889
 890         return 0;
 891 }
 892
 893 void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev,
 894                                 struct rte_cryptodev_info *info)
 895 {
 896         struct qat_pmd_private *internals = dev->data->dev_private;
 897
 898         PMD_INIT_FUNC_TRACE();
 899         if (info != NULL) {
 900                 info->max_nb_queue_pairs =
 901                                 ADF_NUM_SYM_QPS_PER_BUNDLE *
 902                                 ADF_NUM_BUNDLES_PER_DEV;
 903                 info->feature_flags = dev->feature_flags;
 904                 info->capabilities = qat_pmd_capabilities;
 905                 info->sym.max_nb_sessions = internals->max_nb_sessions;
 906                 info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD;
 907         }
 908 }
 909
 910 void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
 911                 struct rte_cryptodev_stats *stats)
 912 {
 913         int i;
 914         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
 915
 916         PMD_INIT_FUNC_TRACE();
 917         if (stats == NULL) {
 918                 PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
 919                 return;
 920         }
 921         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 922                 if (qp[i] == NULL) {
 923                         PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
 924                         continue;
 925                 }
 926
 927                 stats->enqueued_count += qp[i]->stats.enqueued_count;
 928                 stats->dequeued_count += qp[i]->stats.enqueued_count;
 929                 stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
 930                 stats->dequeue_err_count += qp[i]->stats.enqueue_err_count;
 931         }
 932 }
 933
 934 void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
 935 {
 936         int i;
 937         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
 938
 939         PMD_INIT_FUNC_TRACE();
 940         for (i = 0; i < dev->data->nb_queue_pairs; i++)
 941                 memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
 942         PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
 943 }