drivers/crypto/qat/qat_crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *       * Redistributions of source code must retain the above copyright
  12  *         notice, this list of conditions and the following disclaimer.
  13  *       * Redistributions in binary form must reproduce the above copyright
  14  *         notice, this list of conditions and the following disclaimer in
  15  *         the documentation and/or other materials provided with the
  16  *         distribution.
  17  *       * Neither the name of Intel Corporation nor the names of its
  18  *         contributors may be used to endorse or promote products derived
  19  *         from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_tailq.h>
  49 #include <rte_ether.h>
  50 #include <rte_malloc.h>
  51 #include <rte_launch.h>
  52 #include <rte_eal.h>
  53 #include <rte_per_lcore.h>
  54 #include <rte_lcore.h>
  55 #include <rte_atomic.h>
  56 #include <rte_branch_prediction.h>
  57 #include <rte_mempool.h>
  58 #include <rte_mbuf.h>
  59 #include <rte_string_fns.h>
  60 #include <rte_spinlock.h>
  61 #include <rte_hexdump.h>
  62
  63 #include "qat_logs.h"
  64 #include "qat_algs.h"
  65 #include "qat_crypto.h"
  66 #include "adf_transport_access_macros.h"
  67
  68 #define BYTE_LENGTH    8
  69
  70 static const struct rte_cryptodev_capabilities qat_pmd_capabilities[] = {
  71         {       /* SHA1 HMAC */
  72                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  73                 {.sym = {
  74                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  75                         {.auth = {
  76                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  77                                 .block_size = 64,
  78                                 .key_size = {
  79                                         .min = 64,
  80                                         .max = 64,
  81                                         .increment = 0
  82                                 },
  83                                 .digest_size = {
  84                                         .min = 20,
  85                                         .max = 20,
  86                                         .increment = 0
  87                                 },
  88                                 .aad_size = { 0 }
  89                         }, }
  90                 }, }
  91         },
  92         {       /* SHA224 HMAC */
  93                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  94                 {.sym = {
  95                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  96                         {.auth = {
  97                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
  98                                 .block_size = 64,
  99                                         .key_size = {
 100                                         .min = 64,
 101                                         .max = 64,
 102                                         .increment = 0
 103                                 },
 104                                 .digest_size = {
 105                                         .min = 28,
 106                                         .max = 28,
 107                                         .increment = 0
 108                                 },
 109                                 .aad_size = { 0 }
 110                         }, }
 111                 }, }
 112         },
 113         {       /* SHA256 HMAC */
 114                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 115                 {.sym = {
 116                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 117                         {.auth = {
 118                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 119                                 .block_size = 64,
 120                                 .key_size = {
 121                                         .min = 64,
 122                                         .max = 64,
 123                                         .increment = 0
 124                                 },
 125                                 .digest_size = {
 126                                         .min = 32,
 127                                         .max = 32,
 128                                         .increment = 0
 129                                 },
 130                                 .aad_size = { 0 }
 131                         }, }
 132                 }, }
 133         },
 134         {       /* SHA384 HMAC */
 135                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 136                 {.sym = {
 137                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 138                         {.auth = {
 139                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
 140                                 .block_size = 64,
 141                                 .key_size = {
 142                                         .min = 128,
 143                                         .max = 128,
 144                                         .increment = 0
 145                                 },
 146                                 .digest_size = {
 147                                         .min = 48,
 148                                         .max = 48,
 149                                         .increment = 0
 150                                         },
 151                                 .aad_size = { 0 }
 152                         }, }
 153                 }, }
 154         },
 155         {       /* SHA512 HMAC */
 156                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 157                 {.sym = {
 158                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 159                         {.auth = {
 160                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 161                                 .block_size = 128,
 162                                 .key_size = {
 163                                         .min = 128,
 164                                         .max = 128,
 165                                         .increment = 0
 166                                 },
 167                                 .digest_size = {
 168                                         .min = 64,
 169                                         .max = 64,
 170                                         .increment = 0
 171                                 },
 172                                 .aad_size = { 0 }
 173                         }, }
 174                 }, }
 175         },
 176         {       /* MD5 HMAC */
 177                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 178                 {.sym = {
 179                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 180                         {.auth = {
 181                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
 182                                 .block_size = 64,
 183                                 .key_size = {
 184                                         .min = 8,
 185                                         .max = 64,
 186                                         .increment = 8
 187                                 },
 188                                 .digest_size = {
 189                                         .min = 16,
 190                                         .max = 16,
 191                                         .increment = 0
 192                                 },
 193                                 .aad_size = { 0 }
 194                         }, }
 195                 }, }
 196         },
 197         {       /* AES XCBC MAC */
 198                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 199                 {.sym = {
 200                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 201                         {.auth = {
 202                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 203                                 .block_size = 16,
 204                                 .key_size = {
 205                                         .min = 16,
 206                                         .max = 16,
 207                                         .increment = 0
 208                                 },
 209                                 .digest_size = {
 210                                         .min = 16,
 211                                         .max = 16,
 212                                         .increment = 0
 213                                 },
 214                                 .aad_size = { 0 }
 215                         }, }
 216                 }, }
 217         },
 218         {       /* AES GCM (AUTH) */
 219                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 220                 {.sym = {
 221                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 222                         {.auth = {
 223                                 .algo = RTE_CRYPTO_AUTH_AES_GCM,
 224                                 .block_size = 16,
 225                                 .key_size = {
 226                                         .min = 16,
 227                                         .max = 32,
 228                                         .increment = 8
 229                                 },
 230                                 .digest_size = {
 231                                         .min = 8,
 232                                         .max = 16,
 233                                         .increment = 4
 234                                 },
 235                                 .aad_size = {
 236                                         .min = 8,
 237                                         .max = 12,
 238                                         .increment = 4
 239                                 }
 240                         }, }
 241                 }, }
 242         },
 243         {       /* SNOW3G (UIA2) */
 244                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 245                 {.sym = {
 246                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 247                         {.auth = {
 248                                 .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
 249                                 .block_size = 16,
 250                                 .key_size = {
 251                                         .min = 16,
 252                                         .max = 16,
 253                                         .increment = 0
 254                                 },
 255                                 .digest_size = {
 256                                         .min = 4,
 257                                         .max = 4,
 258                                         .increment = 0
 259                                 },
 260                                 .aad_size = {
 261                                         .min = 16,
 262                                         .max = 16,
 263                                         .increment = 0
 264                                 }
 265                         }, }
 266                 }, }
 267         },
 268         {       /* AES GCM (CIPHER) */
 269                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 270                 {.sym = {
 271                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 272                         {.cipher = {
 273                                 .algo = RTE_CRYPTO_CIPHER_AES_GCM,
 274                                 .block_size = 16,
 275                                 .key_size = {
 276                                         .min = 16,
 277                                         .max = 32,
 278                                         .increment = 8
 279                                 },
 280                                 .iv_size = {
 281                                         .min = 16,
 282                                         .max = 16,
 283                                         .increment = 0
 284                                 }
 285                         }, }
 286                 }, }
 287         },
 288         {       /* AES CBC */
 289                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 290                 {.sym = {
 291                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 292                         {.cipher = {
 293                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 294                                 .block_size = 16,
 295                                 .key_size = {
 296                                         .min = 16,
 297                                         .max = 32,
 298                                         .increment = 8
 299                                 },
 300                                 .iv_size = {
 301                                         .min = 16,
 302                                         .max = 16,
 303                                         .increment = 0
 304                                 }
 305                         }, }
 306                 }, }
 307         },
 308         {       /* SNOW3G (UEA2) */
 309                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 310                 {.sym = {
 311                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 312                         {.cipher = {
 313                                 .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 314                                 .block_size = 16,
 315                                 .key_size = {
 316                                         .min = 16,
 317                                         .max = 16,
 318                                         .increment = 0
 319                                 },
 320                                 .iv_size = {
 321                                         .min = 16,
 322                                         .max = 16,
 323                                         .increment = 0
 324                                 }
 325                         }, }
 326                 }, }
 327         },
 328         {       /* AES CTR */
 329                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 330                 {.sym = {
 331                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 332                         {.cipher = {
 333                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 334                                 .block_size = 16,
 335                                 .key_size = {
 336                                         .min = 16,
 337                                         .max = 32,
 338                                         .increment = 8
 339                                 },
 340                                 .iv_size = {
 341                                         .min = 16,
 342                                         .max = 16,
 343                                         .increment = 0
 344                                 }
 345                         }, }
 346                 }, }
 347         },
 348         {       /* NULL (AUTH) */
 349                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 350                 {.sym = {
 351                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 352                         {.auth = {
 353                                 .algo = RTE_CRYPTO_AUTH_NULL,
 354                                 .block_size = 1,
 355                                 .key_size = {
 356                                         .min = 0,
 357                                         .max = 0,
 358                                         .increment = 0
 359                                 },
 360                                 .digest_size = {
 361                                         .min = 0,
 362                                         .max = 0,
 363                                         .increment = 0
 364                                 },
 365                                 .aad_size = { 0 }
 366                         }, },
 367                 }, },
 368         },
 369         {       /* NULL (CIPHER) */
 370                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 371                 {.sym = {
 372                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 373                         {.cipher = {
 374                                 .algo = RTE_CRYPTO_CIPHER_NULL,
 375                                 .block_size = 1,
 376                                 .key_size = {
 377                                         .min = 0,
 378                                         .max = 0,
 379                                         .increment = 0
 380                                 },
 381                                 .iv_size = {
 382                                         .min = 0,
 383                                         .max = 0,
 384                                         .increment = 0
 385                                 }
 386                         }, },
 387                 }, }
 388         },
 389         {       /* KASUMI (F8) */
 390                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 391                 {.sym = {
 392                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 393                         {.cipher = {
 394                                 .algo = RTE_CRYPTO_CIPHER_KASUMI_F8,
 395                                 .block_size = 8,
 396                                 .key_size = {
 397                                         .min = 16,
 398                                         .max = 16,
 399                                         .increment = 0
 400                                 },
 401                                 .iv_size = {
 402                                         .min = 8,
 403                                         .max = 8,
 404                                         .increment = 0
 405                                 }
 406                         }, }
 407                 }, }
 408         },
 409         {       /* KASUMI (F9) */
 410                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 411                 {.sym = {
 412                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 413                         {.auth = {
 414                                 .algo = RTE_CRYPTO_AUTH_KASUMI_F9,
 415                                 .block_size = 8,
 416                                 .key_size = {
 417                                         .min = 16,
 418                                         .max = 16,
 419                                         .increment = 0
 420                                 },
 421                                 .digest_size = {
 422                                         .min = 4,
 423                                         .max = 4,
 424                                         .increment = 0
 425                                 },
 426                                 .aad_size = {
 427                                         .min = 8,
 428                                         .max = 8,
 429                                         .increment = 0
 430                                 }
 431                         }, }
 432                 }, }
 433         },
 434         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 435 };
 436
 437 static inline uint32_t
 438 adf_modulo(uint32_t data, uint32_t shift);
 439
 440 static inline int
 441 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg);
 442
 443 void qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 444                 void *session)
 445 {
 446         struct qat_session *sess = session;
 447         phys_addr_t cd_paddr;
 448
 449         PMD_INIT_FUNC_TRACE();
 450         if (session) {
 451                 cd_paddr = sess->cd_paddr;
 452                 memset(sess, 0, qat_crypto_sym_get_session_private_size(dev));
 453                 sess->cd_paddr = cd_paddr;
 454         } else
 455                 PMD_DRV_LOG(ERR, "NULL session");
 456 }
 457
 458 static int
 459 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 460 {
 461         /* Cipher Only */
 462         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 463                 return ICP_QAT_FW_LA_CMD_CIPHER;
 464
 465         /* Authentication Only */
 466         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 467                 return ICP_QAT_FW_LA_CMD_AUTH;
 468
 469         if (xform->next == NULL)
 470                 return -1;
 471
 472         /* Cipher then Authenticate */
 473         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 474                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 475                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 476
 477         /* Authenticate then Cipher */
 478         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 479                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 480                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 481
 482         return -1;
 483 }
 484
 485 static struct rte_crypto_auth_xform *
 486 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 487 {
 488         do {
 489                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 490                         return &xform->auth;
 491
 492                 xform = xform->next;
 493         } while (xform);
 494
 495         return NULL;
 496 }
 497
 498 static struct rte_crypto_cipher_xform *
 499 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 500 {
 501         do {
 502                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 503                         return &xform->cipher;
 504
 505                 xform = xform->next;
 506         } while (xform);
 507
 508         return NULL;
 509 }
 510 void *
 511 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 512                 struct rte_crypto_sym_xform *xform, void *session_private)
 513 {
 514         struct qat_pmd_private *internals = dev->data->dev_private;
 515
 516         struct qat_session *session = session_private;
 517
 518         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 519
 520         /* Get cipher xform from crypto xform chain */
 521         cipher_xform = qat_get_cipher_xform(xform);
 522
 523         switch (cipher_xform->algo) {
 524         case RTE_CRYPTO_CIPHER_AES_CBC:
 525                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 526                                 &session->qat_cipher_alg) != 0) {
 527                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 528                         goto error_out;
 529                 }
 530                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 531                 break;
 532         case RTE_CRYPTO_CIPHER_AES_GCM:
 533                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 534                                 &session->qat_cipher_alg) != 0) {
 535                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 536                         goto error_out;
 537                 }
 538                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 539                 break;
 540         case RTE_CRYPTO_CIPHER_AES_CTR:
 541                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 542                                 &session->qat_cipher_alg) != 0) {
 543                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 544                         goto error_out;
 545                 }
 546                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 547                 break;
 548         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 549                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 550                                         &session->qat_cipher_alg) != 0) {
 551                         PMD_DRV_LOG(ERR, "Invalid SNOW3G cipher key size");
 552                         goto error_out;
 553                 }
 554                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 555                 break;
 556         case RTE_CRYPTO_CIPHER_NULL:
 557                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 558                 break;
 559         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 560                 if (qat_alg_validate_kasumi_key(cipher_xform->key.length,
 561                                         &session->qat_cipher_alg) != 0) {
 562                         PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size");
 563                         goto error_out;
 564                 }
 565                 session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE;
 566                 break;
 567         case RTE_CRYPTO_CIPHER_3DES_ECB:
 568         case RTE_CRYPTO_CIPHER_3DES_CBC:
 569         case RTE_CRYPTO_CIPHER_AES_ECB:
 570         case RTE_CRYPTO_CIPHER_AES_CCM:
 571                 PMD_DRV_LOG(ERR, "Crypto: Unsupported Cipher alg %u",
 572                                 cipher_xform->algo);
 573                 goto error_out;
 574         default:
 575                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 576                                 cipher_xform->algo);
 577                 goto error_out;
 578         }
 579
 580         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 581                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 582         else
 583                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 584
 585         if (qat_alg_aead_session_create_content_desc_cipher(session,
 586                                                 cipher_xform->key.data,
 587                                                 cipher_xform->key.length))
 588                 goto error_out;
 589
 590         return session;
 591
 592 error_out:
 593         rte_mempool_put(internals->sess_mp, session);
 594         return NULL;
 595 }
 596
 597
 598 void *
 599 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 600                 struct rte_crypto_sym_xform *xform, void *session_private)
 601 {
 602         struct qat_pmd_private *internals = dev->data->dev_private;
 603
 604         struct qat_session *session = session_private;
 605
 606         int qat_cmd_id;
 607
 608         PMD_INIT_FUNC_TRACE();
 609
 610         /* Get requested QAT command id */
 611         qat_cmd_id = qat_get_cmd_id(xform);
 612         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 613                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 614                 goto error_out;
 615         }
 616         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 617         switch (session->qat_cmd) {
 618         case ICP_QAT_FW_LA_CMD_CIPHER:
 619         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 620                 break;
 621         case ICP_QAT_FW_LA_CMD_AUTH:
 622         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 623                 break;
 624         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 625         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 626         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 627                 break;
 628         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 629         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 630         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 631                 break;
 632         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 633         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 634         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 635         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 636         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 637         case ICP_QAT_FW_LA_CMD_MGF1:
 638         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 639         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 640         case ICP_QAT_FW_LA_CMD_DELIMITER:
 641         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 642                 session->qat_cmd);
 643                 goto error_out;
 644         default:
 645         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 646                 session->qat_cmd);
 647                 goto error_out;
 648         }
 649         return session;
 650
 651 error_out:
 652         rte_mempool_put(internals->sess_mp, session);
 653         return NULL;
 654 }
 655
 656 struct qat_session *
 657 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 658                                 struct rte_crypto_sym_xform *xform,
 659                                 struct qat_session *session_private)
 660 {
 661
 662         struct qat_pmd_private *internals = dev->data->dev_private;
 663         struct qat_session *session = session_private;
 664         struct rte_crypto_auth_xform *auth_xform = NULL;
 665         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 666         auth_xform = qat_get_auth_xform(xform);
 667
 668         switch (auth_xform->algo) {
 669         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 670                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 671                 break;
 672         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 673                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
 674                 break;
 675         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 676                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 677                 break;
 678         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 679                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
 680                 break;
 681         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 682                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 683                 break;
 684         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 685                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 686                 break;
 687         case RTE_CRYPTO_AUTH_AES_GCM:
 688                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 689                 break;
 690         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 691                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 692                 break;
 693         case RTE_CRYPTO_AUTH_MD5_HMAC:
 694                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
 695                 break;
 696         case RTE_CRYPTO_AUTH_NULL:
 697                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
 698                 break;
 699         case RTE_CRYPTO_AUTH_KASUMI_F9:
 700                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9;
 701                 break;
 702         case RTE_CRYPTO_AUTH_SHA1:
 703         case RTE_CRYPTO_AUTH_SHA256:
 704         case RTE_CRYPTO_AUTH_SHA512:
 705         case RTE_CRYPTO_AUTH_SHA224:
 706         case RTE_CRYPTO_AUTH_SHA384:
 707         case RTE_CRYPTO_AUTH_MD5:
 708         case RTE_CRYPTO_AUTH_AES_CCM:
 709         case RTE_CRYPTO_AUTH_AES_GMAC:
 710         case RTE_CRYPTO_AUTH_AES_CMAC:
 711         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 712         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 713                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 714                                 auth_xform->algo);
 715                 goto error_out;
 716         default:
 717                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 718                                 auth_xform->algo);
 719                 goto error_out;
 720         }
 721         cipher_xform = qat_get_cipher_xform(xform);
 722
 723         if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
 724                         (session->qat_hash_alg ==
 725                                 ICP_QAT_HW_AUTH_ALGO_GALOIS_64))  {
 726                 if (qat_alg_aead_session_create_content_desc_auth(session,
 727                                 cipher_xform->key.data,
 728                                 cipher_xform->key.length,
 729                                 auth_xform->add_auth_data_length,
 730                                 auth_xform->digest_length,
 731                                 auth_xform->op))
 732                         goto error_out;
 733         } else {
 734                 if (qat_alg_aead_session_create_content_desc_auth(session,
 735                                 auth_xform->key.data,
 736                                 auth_xform->key.length,
 737                                 auth_xform->add_auth_data_length,
 738                                 auth_xform->digest_length,
 739                                 auth_xform->op))
 740                         goto error_out;
 741         }
 742         return session;
 743
 744 error_out:
 745         if (internals->sess_mp != NULL)
 746                 rte_mempool_put(internals->sess_mp, session);
 747         return NULL;
 748 }
 749
 750 unsigned qat_crypto_sym_get_session_private_size(
 751                 struct rte_cryptodev *dev __rte_unused)
 752 {
 753         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 754 }
 755
 756
 757 uint16_t
 758 qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 759                 uint16_t nb_ops)
 760 {
 761         register struct qat_queue *queue;
 762         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 763         register uint32_t nb_ops_sent = 0;
 764         register struct rte_crypto_op **cur_op = ops;
 765         register int ret;
 766         uint16_t nb_ops_possible = nb_ops;
 767         register uint8_t *base_addr;
 768         register uint32_t tail;
 769         int overflow;
 770
 771         if (unlikely(nb_ops == 0))
 772                 return 0;
 773
 774         /* read params used a lot in main loop into registers */
 775         queue = &(tmp_qp->tx_q);
 776         base_addr = (uint8_t *)queue->base_addr;
 777         tail = queue->tail;
 778
 779         /* Find how many can actually fit on the ring */
 780         overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops)
 781                                 - queue->max_inflights;
 782         if (overflow > 0) {
 783                 rte_atomic16_sub(&tmp_qp->inflights16, overflow);
 784                 nb_ops_possible = nb_ops - overflow;
 785                 if (nb_ops_possible == 0)
 786                         return 0;
 787         }
 788
 789         while (nb_ops_sent != nb_ops_possible) {
 790                 ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail);
 791                 if (ret != 0) {
 792                         tmp_qp->stats.enqueue_err_count++;
 793                         if (nb_ops_sent == 0)
 794                                 return 0;
 795                         goto kick_tail;
 796                 }
 797
 798                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
 799                 nb_ops_sent++;
 800                 cur_op++;
 801         }
 802 kick_tail:
 803         WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number,
 804                         queue->hw_queue_number, tail);
 805         queue->tail = tail;
 806         tmp_qp->stats.enqueued_count += nb_ops_sent;
 807         return nb_ops_sent;
 808 }
 809
 810 uint16_t
 811 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 812                 uint16_t nb_ops)
 813 {
 814         struct qat_queue *queue;
 815         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 816         uint32_t msg_counter = 0;
 817         struct rte_crypto_op *rx_op;
 818         struct icp_qat_fw_comn_resp *resp_msg;
 819
 820         queue = &(tmp_qp->rx_q);
 821         resp_msg = (struct icp_qat_fw_comn_resp *)
 822                         ((uint8_t *)queue->base_addr + queue->head);
 823
 824         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 825                         msg_counter != nb_ops) {
 826                 rx_op = (struct rte_crypto_op *)(uintptr_t)
 827                                 (resp_msg->opaque_data);
 828
 829 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 830                 rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
 831                                 sizeof(struct icp_qat_fw_comn_resp));
 832 #endif
 833                 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
 834                                 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
 835                                         resp_msg->comn_hdr.comn_status)) {
 836                         rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 837                 } else {
 838                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 839                 }
 840                 *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG;
 841                 queue->head = adf_modulo(queue->head +
 842                                 queue->msg_size,
 843                                 ADF_RING_SIZE_MODULO(queue->queue_size));
 844                 resp_msg = (struct icp_qat_fw_comn_resp *)
 845                                         ((uint8_t *)queue->base_addr +
 846                                                         queue->head);
 847                 *ops = rx_op;
 848                 ops++;
 849                 msg_counter++;
 850         }
 851         if (msg_counter > 0) {
 852                 WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr,
 853                                         queue->hw_bundle_number,
 854                                         queue->hw_queue_number, queue->head);
 855                 rte_atomic16_sub(&tmp_qp->inflights16, msg_counter);
 856                 tmp_qp->stats.dequeued_count += msg_counter;
 857         }
 858         return msg_counter;
 859 }
 860
 861 static inline int
 862 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg)
 863 {
 864         struct qat_session *ctx;
 865         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 866         struct icp_qat_fw_la_auth_req_params *auth_param;
 867         register struct icp_qat_fw_la_bulk_req *qat_req;
 868
 869 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 870         if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
 871                 PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
 872                                 "operation requests, op (%p) is not a "
 873                                 "symmetric operation.", op);
 874                 return -EINVAL;
 875         }
 876 #endif
 877         if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) {
 878                 PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
 879                                 " requests, op (%p) is sessionless.", op);
 880                 return -EINVAL;
 881         }
 882
 883         if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) {
 884                 PMD_DRV_LOG(ERR, "Session was not created for this device");
 885                 return -EINVAL;
 886         }
 887
 888         ctx = (struct qat_session *)op->sym->session->_private;
 889         qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 890         rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 891         qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
 892
 893         qat_req->comn_mid.dst_length =
 894                 qat_req->comn_mid.src_length =
 895                                 rte_pktmbuf_data_len(op->sym->m_src);
 896
 897         qat_req->comn_mid.dest_data_addr =
 898                 qat_req->comn_mid.src_data_addr =
 899                             rte_pktmbuf_mtophys(op->sym->m_src);
 900
 901         if (unlikely(op->sym->m_dst != NULL)) {
 902                 qat_req->comn_mid.dest_data_addr =
 903                                 rte_pktmbuf_mtophys(op->sym->m_dst);
 904                 qat_req->comn_mid.dst_length =
 905                                 rte_pktmbuf_data_len(op->sym->m_dst);
 906         }
 907
 908         cipher_param = (void *)&qat_req->serv_specif_rqpars;
 909         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 910
 911         cipher_param->cipher_length = op->sym->cipher.data.length;
 912         cipher_param->cipher_offset = op->sym->cipher.data.offset;
 913         if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
 914                         ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) {
 915                 if (unlikely((cipher_param->cipher_length % BYTE_LENGTH != 0) ||
 916                                 (cipher_param->cipher_offset
 917                                         % BYTE_LENGTH != 0))) {
 918                         PMD_DRV_LOG(ERR, " For Snow3g/Kasumi, QAT PMD only "
 919                                 "supports byte aligned values");
 920                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 921                         return -EINVAL;
 922                 }
 923                 cipher_param->cipher_length >>= 3;
 924                 cipher_param->cipher_offset >>= 3;
 925         }
 926
 927         if (op->sym->cipher.iv.length && (op->sym->cipher.iv.length <=
 928                         sizeof(cipher_param->u.cipher_IV_array))) {
 929                 rte_memcpy(cipher_param->u.cipher_IV_array,
 930                                 op->sym->cipher.iv.data,
 931                                 op->sym->cipher.iv.length);
 932         } else {
 933                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
 934                                 qat_req->comn_hdr.serv_specif_flags,
 935                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
 936                 cipher_param->u.s.cipher_IV_ptr = op->sym->cipher.iv.phys_addr;
 937         }
 938         if (op->sym->auth.digest.phys_addr) {
 939                 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
 940                                 qat_req->comn_hdr.serv_specif_flags,
 941                                 ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
 942                 auth_param->auth_res_addr = op->sym->auth.digest.phys_addr;
 943         }
 944         auth_param->auth_off = op->sym->auth.data.offset;
 945         auth_param->auth_len = op->sym->auth.data.length;
 946         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) {
 947                 if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0) ||
 948                                 (auth_param->auth_len % BYTE_LENGTH != 0))) {
 949                         PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only "
 950                                 "supports byte aligned values");
 951                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 952                         return -EINVAL;
 953                 }
 954                 auth_param->auth_off >>= 3;
 955                 auth_param->auth_len >>= 3;
 956         }
 957         if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
 958                         ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&
 959                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9) {
 960                 auth_param->auth_len = (auth_param->auth_len >> 3)
 961                                 + (auth_param->auth_off >> 3)
 962                                 + (BYTE_LENGTH >> 3)
 963                                 - 8;
 964                 auth_param->auth_off = 8;
 965         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH
 966                         && ctx->qat_hash_alg ==
 967                                         ICP_QAT_HW_AUTH_ALGO_KASUMI_F9) {
 968                 auth_param->auth_len = (auth_param->auth_len >> 3)
 969                                 + (auth_param->auth_off >> 3)
 970                                 + (BYTE_LENGTH >> 3);
 971                 auth_param->auth_off = 0;
 972         }
 973         auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr;
 974
 975         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
 976                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
 977                 if (op->sym->cipher.iv.length == 12) {
 978                         /*
 979                          * For GCM a 12 bit IV is allowed,
 980                          * but we need to inform the f/w
 981                          */
 982                         ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 983                                 qat_req->comn_hdr.serv_specif_flags,
 984                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 985                 }
 986         }
 987
 988 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 989         rte_hexdump(stdout, "qat_req:", qat_req,
 990                         sizeof(struct icp_qat_fw_la_bulk_req));
 991         rte_hexdump(stdout, "src_data:",
 992                         rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
 993                         rte_pktmbuf_data_len(op->sym->m_src));
 994         rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data,
 995                         op->sym->cipher.iv.length);
 996         rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
 997                         op->sym->auth.digest.length);
 998         rte_hexdump(stdout, "aad:", op->sym->auth.aad.data,
 999                         op->sym->auth.aad.length);
1000 #endif
1001         return 0;
1002 }
1003
1004 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
1005 {
1006         uint32_t div = data >> shift;
1007         uint32_t mult = div << shift;
1008
1009         return data - mult;
1010 }
1011
1012 void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess)
1013 {
1014         struct rte_cryptodev_sym_session *sess = sym_sess;
1015         struct qat_session *s = (void *)sess->_private;
1016
1017         PMD_INIT_FUNC_TRACE();
1018         s->cd_paddr = rte_mempool_virt2phy(mp, sess) +
1019                 offsetof(struct qat_session, cd) +
1020                 offsetof(struct rte_cryptodev_sym_session, _private);
1021 }
1022
1023 int qat_dev_config(__rte_unused struct rte_cryptodev *dev)
1024 {
1025         PMD_INIT_FUNC_TRACE();
1026         return -ENOTSUP;
1027 }
1028
1029 int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
1030 {
1031         PMD_INIT_FUNC_TRACE();
1032         return 0;
1033 }
1034
1035 void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
1036 {
1037         PMD_INIT_FUNC_TRACE();
1038 }
1039
1040 int qat_dev_close(struct rte_cryptodev *dev)
1041 {
1042         int i, ret;
1043
1044         PMD_INIT_FUNC_TRACE();
1045
1046         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1047                 ret = qat_crypto_sym_qp_release(dev, i);
1048                 if (ret < 0)
1049                         return ret;
1050         }
1051
1052         return 0;
1053 }
1054
1055 void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev,
1056                                 struct rte_cryptodev_info *info)
1057 {
1058         struct qat_pmd_private *internals = dev->data->dev_private;
1059
1060         PMD_INIT_FUNC_TRACE();
1061         if (info != NULL) {
1062                 info->max_nb_queue_pairs =
1063                                 ADF_NUM_SYM_QPS_PER_BUNDLE *
1064                                 ADF_NUM_BUNDLES_PER_DEV;
1065                 info->feature_flags = dev->feature_flags;
1066                 info->capabilities = qat_pmd_capabilities;
1067                 info->sym.max_nb_sessions = internals->max_nb_sessions;
1068                 info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD;
1069         }
1070 }
1071
1072 void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
1073                 struct rte_cryptodev_stats *stats)
1074 {
1075         int i;
1076         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1077
1078         PMD_INIT_FUNC_TRACE();
1079         if (stats == NULL) {
1080                 PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
1081                 return;
1082         }
1083         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1084                 if (qp[i] == NULL) {
1085                         PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
1086                         continue;
1087                 }
1088
1089                 stats->enqueued_count += qp[i]->stats.enqueued_count;
1090                 stats->dequeued_count += qp[i]->stats.enqueued_count;
1091                 stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
1092                 stats->dequeue_err_count += qp[i]->stats.enqueue_err_count;
1093         }
1094 }
1095
1096 void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
1097 {
1098         int i;
1099         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1100
1101         PMD_INIT_FUNC_TRACE();
1102         for (i = 0; i < dev->data->nb_queue_pairs; i++)
1103                 memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
1104         PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
1105 }