examples/dpdk_qat/crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <strings.h>
  38 #include <string.h>
  39 #include <inttypes.h>
  40 #include <errno.h>
  41 #include <sys/queue.h>
  42 #include <stdarg.h>
  43
  44 #include <rte_common.h>
  45 #include <rte_log.h>
  46 #include <rte_debug.h>
  47 #include <rte_memory.h>
  48 #include <rte_memzone.h>
  49 #include <rte_tailq.h>
  50 #include <rte_ether.h>
  51 #include <rte_malloc.h>
  52 #include <rte_launch.h>
  53 #include <rte_eal.h>
  54 #include <rte_per_lcore.h>
  55 #include <rte_lcore.h>
  56 #include <rte_atomic.h>
  57 #include <rte_branch_prediction.h>
  58 #include <rte_ring.h>
  59 #include <rte_mempool.h>
  60 #include <rte_mbuf.h>
  61 #include <rte_string_fns.h>
  62
  63 #define CPA_CY_SYM_DP_TMP_WORKAROUND 1
  64
  65 #include "cpa.h"
  66 #include "cpa_types.h"
  67 #include "cpa_cy_sym_dp.h"
  68 #include "cpa_cy_common.h"
  69 #include "cpa_cy_im.h"
  70 #include "icp_sal_user.h"
  71 #include "icp_sal_poll.h"
  72
  73 #include "crypto.h"
  74
  75 #define NUM_HMAC        (10)
  76 #define NUM_CRYPTO      (7)
  77
  78
  79 /* CIPHER KEY LENGTHS */
  80 #define KEY_SIZE_64_IN_BYTES    (64 / 8)
  81 #define KEY_SIZE_56_IN_BYTES    (56 / 8)
  82 #define KEY_SIZE_128_IN_BYTES   (128 / 8)
  83 #define KEY_SIZE_168_IN_BYTES   (168 / 8)
  84 #define KEY_SIZE_192_IN_BYTES   (192 / 8)
  85 #define KEY_SIZE_256_IN_BYTES   (256 / 8)
  86
  87 /* HMAC AUTH KEY LENGTHS */
  88 #define AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES       (128 / 8)
  89 #define SHA1_AUTH_KEY_LENGTH_IN_BYTES           (160 / 8)
  90 #define SHA224_AUTH_KEY_LENGTH_IN_BYTES         (224 / 8)
  91 #define SHA256_AUTH_KEY_LENGTH_IN_BYTES         (256 / 8)
  92 #define SHA384_AUTH_KEY_LENGTH_IN_BYTES         (384 / 8)
  93 #define SHA512_AUTH_KEY_LENGTH_IN_BYTES         (512 / 8)
  94 #define MD5_AUTH_KEY_LENGTH_IN_BYTES            (128 / 8)
  95
  96 /* HASH DIGEST LENGHTS */
  97 #define AES_XCBC_DIGEST_LENGTH_IN_BYTES         (128 / 8)
  98 #define AES_XCBC_96_DIGEST_LENGTH_IN_BYTES      (96 / 8)
  99 #define MD5_DIGEST_LENGTH_IN_BYTES              (128 / 8)
 100 #define SHA1_DIGEST_LENGTH_IN_BYTES             (160 / 8)
 101 #define SHA1_96_DIGEST_LENGTH_IN_BYTES          (96 / 8)
 102 #define SHA224_DIGEST_LENGTH_IN_BYTES           (224 / 8)
 103 #define SHA256_DIGEST_LENGTH_IN_BYTES           (256 / 8)
 104 #define SHA384_DIGEST_LENGTH_IN_BYTES           (384 / 8)
 105 #define SHA512_DIGEST_LENGTH_IN_BYTES           (512 / 8)
 106
 107 #define IV_LENGTH_16_BYTES      (16)
 108 #define IV_LENGTH_8_BYTES       (8)
 109
 110
 111 /*
 112  * rte_memzone is used to allocate physically contiguous virtual memory.
 113  * In this application we allocate a single block and divide between variables
 114  * which require a virtual to physical mapping for use by the QAT driver.
 115  * Virt2phys is only performed during initialisation and not on the data-path.
 116  */
 117
 118 #define LCORE_MEMZONE_SIZE      (1 << 22)
 119
 120 struct lcore_memzone
 121 {
 122         const struct rte_memzone *memzone;
 123         void *next_free_address;
 124 };
 125
 126 /*
 127  * Size the qa software response queue.
 128  * Note: Head and Tail are 8 bit, therefore, the queue is
 129  * fixed to 256 entries.
 130  */
 131 #define CRYPTO_SOFTWARE_QUEUE_SIZE 256
 132
 133 struct qa_callbackQueue {
 134         uint8_t head;
 135         uint8_t tail;
 136         uint16_t numEntries;
 137         struct rte_mbuf *qaCallbackRing[CRYPTO_SOFTWARE_QUEUE_SIZE];
 138 };
 139
 140 struct qa_core_conf {
 141         CpaCySymDpSessionCtx *encryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 142         CpaCySymDpSessionCtx *decryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 143         CpaInstanceHandle instanceHandle;
 144         struct qa_callbackQueue callbackQueue;
 145         uint64_t qaOutstandingRequests;
 146         uint64_t numResponseAttempts;
 147         uint8_t kickFreq;
 148         void *pPacketIV;
 149         CpaPhysicalAddr packetIVPhy;
 150         struct lcore_memzone lcoreMemzone;
 151 } __rte_cache_aligned;
 152
 153 #define MAX_CORES   (RTE_MAX_LCORE)
 154
 155 static struct qa_core_conf qaCoreConf[MAX_CORES];
 156
 157 /*
 158  *Create maximum possible key size,
 159  *One for cipher and one for hash
 160  */
 161 struct glob_keys {
 162         uint8_t cipher_key[32];
 163         uint8_t hash_key[64];
 164         uint8_t iv[16];
 165 };
 166
 167 struct glob_keys g_crypto_hash_keys = {
 168         .cipher_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 169                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 170                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 171                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20},
 172         .hash_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 173                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 174                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 175                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,
 176                 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,
 177                 0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,
 178                 0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,
 179                 0x39,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50},
 180         .iv = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 181                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10}
 182 };
 183
 184 /*
 185  * Offsets from the start of the packet.
 186  *
 187  */
 188 #define PACKET_DATA_START_PHYS(p) \
 189                 ((p)->buf_physaddr + ((char *)p->pkt.data - (char *)p->buf_addr))
 190
 191 /*
 192  * A fixed offset to where the crypto is to be performed, which is the first
 193  * byte after the Ethernet(14 bytes) and IPv4 headers(20 bytes)
 194  */
 195 #define CRYPTO_START_OFFSET             (14+20)
 196 #define HASH_START_OFFSET               (14+20)
 197 #define CIPHER_BLOCK_DEFAULT_SIZE       (16)
 198 #define HASH_BLOCK_DEFAULT_SIZE         (16)
 199
 200 /*
 201  * Offset to the opdata from the start of the data portion of packet.
 202  * Assumption: The buffer is physically contiguous.
 203  * +18 takes this to the next cache line.
 204  */
 205
 206 #define CRYPTO_OFFSET_TO_OPDATA         (ETHER_MAX_LEN+18)
 207
 208 /*
 209  * Default number of requests to place on the hardware ring before kicking the
 210  * ring pointers.
 211  */
 212 #define CRYPTO_BURST_TX (16)
 213
 214 /*
 215  * Only call the qa poll function when the number responses in the software
 216  * queue drops below this number.
 217  */
 218 #define CRYPTO_QUEUED_RESP_POLL_THRESHOLD       (32)
 219
 220 /*
 221  * Limit the number of polls per call to get_next_response.
 222  */
 223 #define GET_NEXT_RESPONSE_FREQ  (32)
 224
 225 /*
 226  * Max number of responses to pull from the qa in one poll.
 227  */
 228 #define CRYPTO_MAX_RESPONSE_QUOTA \
 229                 (CRYPTO_SOFTWARE_QUEUE_SIZE-CRYPTO_QUEUED_RESP_POLL_THRESHOLD-1)
 230
 231 #if (CRYPTO_QUEUED_RESP_POLL_THRESHOLD + CRYPTO_MAX_RESPONSE_QUOTA >= \
 232                 CRYPTO_SOFTWARE_QUEUE_SIZE)
 233 #error Its possible to overflow the qa response Q with current poll and \
 234                 response quota.
 235 #endif
 236
 237 static void
 238 crypto_callback(CpaCySymDpOpData *pOpData,
 239                 __rte_unused CpaStatus status,
 240                 __rte_unused CpaBoolean verifyResult)
 241 {
 242         uint32_t lcore_id;
 243         lcore_id = rte_lcore_id();
 244         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 245
 246         /*
 247          * Received a completion from the QA hardware.
 248          * Place the response on the return queue.
 249          */
 250         callbackQ->qaCallbackRing[callbackQ->head] = pOpData->pCallbackTag;
 251         callbackQ->head++;
 252         callbackQ->numEntries++;
 253         qaCoreConf[lcore_id].qaOutstandingRequests--;
 254 }
 255
 256 static void
 257 qa_crypto_callback(CpaCySymDpOpData *pOpData, CpaStatus status,
 258                 CpaBoolean verifyResult)
 259 {
 260         crypto_callback(pOpData, status, verifyResult);
 261 }
 262
 263 /*
 264  * Each allocation from a particular memzone lasts for the life-time of
 265  * the application. No freeing of previous allocations will occur.
 266  */
 267 static void *
 268 alloc_memzone_region(uint32_t length, uint32_t lcore_id)
 269 {
 270         char *current_free_addr_ptr = NULL;
 271         struct lcore_memzone *lcore_memzone = &(qaCoreConf[lcore_id].lcoreMemzone);
 272
 273         current_free_addr_ptr  = lcore_memzone->next_free_address;
 274
 275         if (current_free_addr_ptr + length >=
 276                 (char *)lcore_memzone->memzone->addr + lcore_memzone->memzone->len) {
 277                 printf("Crypto: No memory available in memzone\n");
 278                 return NULL;
 279         }
 280         lcore_memzone->next_free_address = current_free_addr_ptr + length;
 281
 282         return (void *)current_free_addr_ptr;
 283 }
 284
 285 /*
 286  * Virtual to Physical Address translation is only executed during initialization
 287  * and not on the data-path.
 288  */
 289 static CpaPhysicalAddr
 290 qa_v2p(void *ptr)
 291 {
 292         const struct rte_memzone *memzone = NULL;
 293         uint32_t lcore_id = 0;
 294         RTE_LCORE_FOREACH(lcore_id) {
 295                 memzone = qaCoreConf[lcore_id].lcoreMemzone.memzone;
 296
 297                 if ((char*) ptr >= (char *) memzone->addr &&
 298                                 (char*) ptr < ((char*) memzone->addr + memzone->len)) {
 299                         return (CpaPhysicalAddr)
 300                                         (memzone->phys_addr + ((char *) ptr - (char*) memzone->addr));
 301                 }
 302         }
 303         printf("Crypto: Corresponding physical address not found in memzone\n");
 304         return (CpaPhysicalAddr) 0;
 305 }
 306
 307 static CpaStatus
 308 getCoreAffinity(Cpa32U *coreAffinity, const CpaInstanceHandle instanceHandle)
 309 {
 310         CpaInstanceInfo2 info;
 311         Cpa16U i = 0;
 312         CpaStatus status = CPA_STATUS_SUCCESS;
 313
 314         bzero(&info, sizeof(CpaInstanceInfo2));
 315
 316         status = cpaCyInstanceGetInfo2(instanceHandle, &info);
 317         if (CPA_STATUS_SUCCESS != status) {
 318                 printf("Crypto: Error getting instance info\n");
 319                 return CPA_STATUS_FAIL;
 320         }
 321         for (i = 0; i < MAX_CORES; i++) {
 322                 if (CPA_BITMAP_BIT_TEST(info.coreAffinity, i)) {
 323                         *coreAffinity = i;
 324                         return CPA_STATUS_SUCCESS;
 325                 }
 326         }
 327         return CPA_STATUS_FAIL;
 328 }
 329
 330 static CpaStatus
 331 get_crypto_instance_on_core(CpaInstanceHandle *pInstanceHandle,
 332                 uint32_t lcore_id)
 333 {
 334         Cpa16U numInstances = 0, i = 0;
 335         CpaStatus status = CPA_STATUS_FAIL;
 336         CpaInstanceHandle *pLocalInstanceHandles = NULL;
 337         Cpa32U coreAffinity = 0;
 338
 339         status = cpaCyGetNumInstances(&numInstances);
 340         if (CPA_STATUS_SUCCESS != status || numInstances == 0) {
 341                 return CPA_STATUS_FAIL;
 342         }
 343
 344         pLocalInstanceHandles = rte_malloc("pLocalInstanceHandles",
 345                         sizeof(CpaInstanceHandle) * numInstances, CACHE_LINE_SIZE);
 346
 347         if (NULL == pLocalInstanceHandles) {
 348                 return CPA_STATUS_FAIL;
 349         }
 350         status = cpaCyGetInstances(numInstances, pLocalInstanceHandles);
 351         if (CPA_STATUS_SUCCESS != status) {
 352                 printf("Crypto: cpaCyGetInstances failed with status: %"PRId32"\n", status);
 353                 rte_free((void *) pLocalInstanceHandles);
 354                 return CPA_STATUS_FAIL;
 355         }
 356
 357         for (i = 0; i < numInstances; i++) {
 358                 status = getCoreAffinity(&coreAffinity, pLocalInstanceHandles[i]);
 359                 if (CPA_STATUS_SUCCESS != status) {
 360                         rte_free((void *) pLocalInstanceHandles);
 361                         return CPA_STATUS_FAIL;
 362                 }
 363                 if (coreAffinity == lcore_id) {
 364                         printf("Crypto: instance found on core %d\n", i);
 365                         *pInstanceHandle = pLocalInstanceHandles[i];
 366                         return CPA_STATUS_SUCCESS;
 367                 }
 368         }
 369         /* core affinity not found */
 370         rte_free((void *) pLocalInstanceHandles);
 371         return CPA_STATUS_FAIL;
 372 }
 373
 374 static CpaStatus
 375 initCySymSession(const int pkt_cipher_alg,
 376                 const int pkt_hash_alg, const CpaCySymHashMode hashMode,
 377                 const CpaCySymCipherDirection crypto_direction,
 378                 CpaCySymSessionCtx **ppSessionCtx,
 379                 const CpaInstanceHandle cyInstanceHandle,
 380                 const uint32_t lcore_id)
 381 {
 382         Cpa32U sessionCtxSizeInBytes = 0;
 383         CpaStatus status = CPA_STATUS_FAIL;
 384         CpaBoolean isCrypto = CPA_TRUE, isHmac = CPA_TRUE;
 385         CpaCySymSessionSetupData sessionSetupData;
 386
 387         bzero(&sessionSetupData, sizeof(CpaCySymSessionSetupData));
 388
 389         /* Assumption: key length is set to each algorithm's max length */
 390         switch (pkt_cipher_alg) {
 391         case NO_CIPHER:
 392                 isCrypto = CPA_FALSE;
 393                 break;
 394         case CIPHER_DES:
 395                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 396                                 CPA_CY_SYM_CIPHER_DES_ECB;
 397                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 398                                 KEY_SIZE_64_IN_BYTES;
 399                 break;
 400         case CIPHER_DES_CBC:
 401                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 402                                 CPA_CY_SYM_CIPHER_DES_CBC;
 403                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 404                                 KEY_SIZE_64_IN_BYTES;
 405                 break;
 406         case CIPHER_DES3:
 407                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 408                                 CPA_CY_SYM_CIPHER_3DES_ECB;
 409                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 410                                 KEY_SIZE_192_IN_BYTES;
 411                 break;
 412         case CIPHER_DES3_CBC:
 413                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 414                                 CPA_CY_SYM_CIPHER_3DES_CBC;
 415                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 416                                 KEY_SIZE_192_IN_BYTES;
 417                 break;
 418         case CIPHER_AES:
 419                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 420                                 CPA_CY_SYM_CIPHER_AES_ECB;
 421                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 422                                 KEY_SIZE_128_IN_BYTES;
 423                 break;
 424         case CIPHER_AES_CBC_128:
 425                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 426                                 CPA_CY_SYM_CIPHER_AES_CBC;
 427                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 428                                 KEY_SIZE_128_IN_BYTES;
 429                 break;
 430         default:
 431                 printf("Crypto: Undefined Cipher specified\n");
 432                 break;
 433         }
 434         /* Set the cipher direction */
 435         if (isCrypto) {
 436                 sessionSetupData.cipherSetupData.cipherDirection = crypto_direction;
 437                 sessionSetupData.cipherSetupData.pCipherKey =
 438                                 g_crypto_hash_keys.cipher_key;
 439                 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
 440         }
 441
 442         /* Setup Hash common fields */
 443         switch (pkt_hash_alg) {
 444         case NO_HASH:
 445                 isHmac = CPA_FALSE;
 446                 break;
 447         case HASH_AES_XCBC:
 448                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 449                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 450                                 AES_XCBC_DIGEST_LENGTH_IN_BYTES;
 451                 break;
 452         case HASH_AES_XCBC_96:
 453                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 454                                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 455                                 AES_XCBC_96_DIGEST_LENGTH_IN_BYTES;
 456                 break;
 457         case HASH_MD5:
 458                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_MD5;
 459                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 460                                 MD5_DIGEST_LENGTH_IN_BYTES;
 461                 break;
 462         case HASH_SHA1:
 463                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 464                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 465                                 SHA1_DIGEST_LENGTH_IN_BYTES;
 466                 break;
 467         case HASH_SHA1_96:
 468                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 469                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 470                                 SHA1_96_DIGEST_LENGTH_IN_BYTES;
 471             break;
 472         case HASH_SHA224:
 473                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA224;
 474                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 475                                 SHA224_DIGEST_LENGTH_IN_BYTES;
 476                 break;
 477         case HASH_SHA256:
 478                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA256;
 479                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 480                                 SHA256_DIGEST_LENGTH_IN_BYTES;
 481                 break;
 482         case HASH_SHA384:
 483                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA384;
 484                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 485                                 SHA384_DIGEST_LENGTH_IN_BYTES;
 486                 break;
 487         case HASH_SHA512:
 488                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA512;
 489                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 490                                 SHA512_DIGEST_LENGTH_IN_BYTES;
 491                 break;
 492         default:
 493                 printf("Crypto: Undefined Hash specified\n");
 494                 break;
 495         }
 496         if (isHmac) {
 497                 sessionSetupData.hashSetupData.hashMode = hashMode;
 498                 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
 499                 /* If using authenticated hash setup key lengths */
 500                 if (CPA_CY_SYM_HASH_MODE_AUTH == hashMode) {
 501                         /* Use a common max length key */
 502                         sessionSetupData.hashSetupData.authModeSetupData.authKey =
 503                                         g_crypto_hash_keys.hash_key;
 504                         switch (pkt_hash_alg) {
 505                         case HASH_AES_XCBC:
 506                         case HASH_AES_XCBC_96:
 507                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 508                                                 AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES;
 509                                 break;
 510                         case HASH_MD5:
 511                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 512                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 513                                 break;
 514                         case HASH_SHA1:
 515                         case HASH_SHA1_96:
 516                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 517                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 518                                 break;
 519                         case HASH_SHA224:
 520                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 521                                                 SHA224_AUTH_KEY_LENGTH_IN_BYTES;
 522                                 break;
 523                         case HASH_SHA256:
 524                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 525                                                 SHA256_AUTH_KEY_LENGTH_IN_BYTES;
 526                                 break;
 527                         case HASH_SHA384:
 528                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 529                                                 SHA384_AUTH_KEY_LENGTH_IN_BYTES;
 530                                 break;
 531                         case HASH_SHA512:
 532                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 533                                                 SHA512_AUTH_KEY_LENGTH_IN_BYTES;
 534                                 break;
 535                         default:
 536                                 printf("Crypto: Undefined Hash specified\n");
 537                                 return CPA_STATUS_FAIL;
 538                         }
 539                 }
 540         }
 541
 542         /* Only high priority supported */
 543         sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
 544
 545         /* If chaining algorithms */
 546         if (isCrypto && isHmac) {
 547                 sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 548                 /* @assumption Alg Chain order is cipher then hash for encrypt
 549                  * and hash then cipher then has for decrypt*/
 550                 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == crypto_direction) {
 551                         sessionSetupData.algChainOrder =
 552                                         CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 553                 } else {
 554                         sessionSetupData.algChainOrder =
 555                                         CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 556                 }
 557         }
 558         if (!isCrypto && !isHmac) {
 559                 *ppSessionCtx = NULL;
 560                 return CPA_STATUS_SUCCESS;
 561         }
 562
 563         /* Get the session context size based on the crypto and/or hash operations*/
 564         status = cpaCySymDpSessionCtxGetSize(cyInstanceHandle, &sessionSetupData,
 565                         &sessionCtxSizeInBytes);
 566         if (CPA_STATUS_SUCCESS != status) {
 567                 printf("Crypto: cpaCySymDpSessionCtxGetSize error, status: %"PRId32"\n",
 568                                 status);
 569                 return CPA_STATUS_FAIL;
 570         }
 571
 572         *ppSessionCtx = alloc_memzone_region(sessionCtxSizeInBytes, lcore_id);
 573         if (NULL == *ppSessionCtx) {
 574                 printf("Crypto: Failed to allocate memory for Session Context\n");
 575                 return CPA_STATUS_FAIL;
 576         }
 577
 578         status = cpaCySymDpInitSession(cyInstanceHandle, &sessionSetupData,
 579                         CPA_TRUE,CPA_FALSE, *ppSessionCtx);
 580         if (CPA_STATUS_SUCCESS != status) {
 581                 printf("Crypto: cpaCySymDpInitSession failed with status %"PRId32"\n", status);
 582                 return CPA_STATUS_FAIL;
 583         }
 584         return CPA_STATUS_SUCCESS;
 585 }
 586
 587 static CpaStatus
 588 initSessionDataTables(struct qa_core_conf *qaCoreConf,uint32_t lcore_id)
 589 {
 590         Cpa32U i = 0, j = 0;
 591         CpaStatus status = CPA_STATUS_FAIL;
 592         for (i = 0; i < NUM_CRYPTO; i++) {
 593                 for (j = 0; j < NUM_HMAC; j++) {
 594                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 595                                         CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
 596                                         &qaCoreConf->encryptSessionHandleTbl[i][j],
 597                                         qaCoreConf->instanceHandle,
 598                                         lcore_id);
 599                         if (CPA_STATUS_SUCCESS != status) {
 600                                 printf("Crypto: Failed to initialize Encrypt sessions\n");
 601                                 return CPA_STATUS_FAIL;
 602                         }
 603                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 604                                         CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
 605                                         &qaCoreConf->decryptSessionHandleTbl[i][j],
 606                                         qaCoreConf->instanceHandle,
 607                                         lcore_id);
 608                         if (CPA_STATUS_SUCCESS != status) {
 609                                 printf("Crypto: Failed to initialize Decrypt sessions\n");
 610                                 return CPA_STATUS_FAIL;
 611                         }
 612                 }
 613         }
 614         return CPA_STATUS_SUCCESS;
 615 }
 616
 617 int
 618 crypto_init(void)
 619 {
 620         if (CPA_STATUS_SUCCESS != icp_sal_userStart("SSL")) {
 621                 printf("Crypto: Could not start sal for user space\n");
 622                 return CPA_STATUS_FAIL;
 623         }
 624         printf("Crypto: icp_sal_userStart(\"SSL\")\n");
 625         return 0;
 626 }
 627
 628 /*
 629  * Per core initialisation
 630  */
 631 int
 632 per_core_crypto_init(uint32_t lcore_id)
 633 {
 634         CpaStatus status = CPA_STATUS_FAIL;
 635         char memzone_name[RTE_MEMZONE_NAMESIZE];
 636
 637         int socketID = rte_lcore_to_socket_id(lcore_id);
 638
 639         /* Allocate software ring for response messages. */
 640
 641         qaCoreConf[lcore_id].callbackQueue.head = 0;
 642         qaCoreConf[lcore_id].callbackQueue.tail = 0;
 643         qaCoreConf[lcore_id].callbackQueue.numEntries = 0;
 644         qaCoreConf[lcore_id].kickFreq = 0;
 645         qaCoreConf[lcore_id].qaOutstandingRequests = 0;
 646         qaCoreConf[lcore_id].numResponseAttempts = 0;
 647
 648         /* Initialise and reserve lcore memzone for virt2phys translation */
 649         rte_snprintf(memzone_name,
 650                         RTE_MEMZONE_NAMESIZE,
 651                         "lcore_%u",
 652                         lcore_id);
 653
 654         qaCoreConf[lcore_id].lcoreMemzone.memzone = rte_memzone_reserve(
 655                         memzone_name,
 656                         LCORE_MEMZONE_SIZE,
 657                         socketID,
 658                         0);
 659         if (NULL == qaCoreConf[lcore_id].lcoreMemzone.memzone) {
 660                 printf("Crypto: Error allocating memzone on lcore %u\n",lcore_id);
 661                 return -1;
 662         }
 663         qaCoreConf[lcore_id].lcoreMemzone.next_free_address =
 664                                                         qaCoreConf[lcore_id].lcoreMemzone.memzone->addr;
 665
 666         qaCoreConf[lcore_id].pPacketIV = alloc_memzone_region(IV_LENGTH_16_BYTES,
 667                                                         lcore_id);
 668
 669         if (NULL == qaCoreConf[lcore_id].pPacketIV ) {
 670                 printf("Crypto: Failed to allocate memory for Initialization Vector\n");
 671                 return -1;
 672         }
 673
 674         memcpy(qaCoreConf[lcore_id].pPacketIV, &g_crypto_hash_keys.iv,
 675                         IV_LENGTH_16_BYTES);
 676
 677         qaCoreConf[lcore_id].packetIVPhy = qa_v2p(qaCoreConf[lcore_id].pPacketIV);
 678         if (0 == qaCoreConf[lcore_id].packetIVPhy) {
 679                 printf("Crypto: Invalid physical address for Initialization Vector\n");
 680                 return -1;
 681         }
 682
 683         /*
 684          * Obtain the instance handle that is mapped to the current lcore.
 685          * This can fail if an instance is not mapped to a bank which has been
 686          * affinitized to the current lcore.
 687          */
 688         status = get_crypto_instance_on_core(&(qaCoreConf[lcore_id].instanceHandle),
 689                         lcore_id);
 690         if (CPA_STATUS_SUCCESS != status) {
 691                 printf("Crypto: get_crypto_instance_on_core failed with status: %"PRId32"\n",
 692                                 status);
 693                 return -1;
 694         }
 695
 696         status = cpaCySymDpRegCbFunc(qaCoreConf[lcore_id].instanceHandle,
 697                         (CpaCySymDpCbFunc) qa_crypto_callback);
 698         if (CPA_STATUS_SUCCESS != status) {
 699                 printf("Crypto: cpaCySymDpRegCbFunc failed with status: %"PRId32"\n", status);
 700                 return -1;
 701         }
 702
 703         /*
 704          * Set the address translation callback for virtual to physcial address
 705          * mapping. This will be called by the QAT driver during initialisation only.
 706          */
 707         status = cpaCySetAddressTranslation(qaCoreConf[lcore_id].instanceHandle,
 708                         (CpaVirtualToPhysical) qa_v2p);
 709         if (CPA_STATUS_SUCCESS != status) {
 710                 printf("Crypto: cpaCySetAddressTranslation failed with status: %"PRId32"\n",
 711                                 status);
 712                 return -1;
 713         }
 714
 715         status = initSessionDataTables(&qaCoreConf[lcore_id],lcore_id);
 716         if (CPA_STATUS_SUCCESS != status) {
 717                 printf("Crypto: Failed to allocate all session tables.");
 718                 return -1;
 719         }
 720         return 0;
 721 }
 722
 723 static CpaStatus
 724 enqueueOp(CpaCySymDpOpData *opData, uint32_t lcore_id)
 725 {
 726
 727         CpaStatus status;
 728
 729         /*
 730          * Assumption is there is no requirement to do load balancing between
 731          * acceleration units - that is one acceleration unit is tied to a core.
 732          */
 733         opData->instanceHandle = qaCoreConf[lcore_id].instanceHandle;
 734
 735         if ((++qaCoreConf[lcore_id].kickFreq) % CRYPTO_BURST_TX == 0) {
 736                 status = cpaCySymDpEnqueueOp(opData, CPA_TRUE);
 737         } else {
 738                 status = cpaCySymDpEnqueueOp(opData, CPA_FALSE);
 739         }
 740
 741         qaCoreConf[lcore_id].qaOutstandingRequests++;
 742
 743         return status;
 744 }
 745
 746 void
 747 crypto_flush_tx_queue(uint32_t lcore_id)
 748 {
 749
 750         cpaCySymDpPerformOpNow(qaCoreConf[lcore_id].instanceHandle);
 751 }
 752
 753 enum crypto_result
 754 crypto_encrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 755 {
 756         CpaCySymDpOpData *opData =
 757                         (CpaCySymDpOpData *) ((char *) (rte_buff->pkt.data)
 758                                         + CRYPTO_OFFSET_TO_OPDATA);
 759         uint32_t lcore_id;
 760
 761         lcore_id = rte_lcore_id();
 762
 763         bzero(opData, sizeof(CpaCySymDpOpData));
 764
 765         opData->srcBuffer = opData->dstBuffer = PACKET_DATA_START_PHYS(rte_buff);
 766         opData->srcBufferLen = opData->dstBufferLen = rte_buff->pkt.data_len;
 767         opData->sessionCtx = qaCoreConf[lcore_id].encryptSessionHandleTbl[c][h];
 768         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 769                         + CRYPTO_OFFSET_TO_OPDATA;
 770         opData->pCallbackTag = rte_buff;
 771
 772         /* if no crypto or hash operations are specified return fail */
 773         if (NO_CIPHER == c && NO_HASH == h)
 774                 return CRYPTO_RESULT_FAIL;
 775
 776         if (NO_CIPHER != c) {
 777                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 778                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 779
 780                 if (CIPHER_AES_CBC_128 == c)
 781                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 782                 else
 783                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 784
 785                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 786                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 787                                 - CRYPTO_START_OFFSET;
 788                 /*
 789                  * Work around for padding, message length has to be a multiple of
 790                  * block size.
 791                  */
 792                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 793                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 794         }
 795
 796         if (NO_HASH != h) {
 797
 798                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 799                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 800                                 - HASH_START_OFFSET;
 801                 /*
 802                  * Work around for padding, message length has to be a multiple of block
 803                  * size.
 804                  */
 805                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 806                                 % HASH_BLOCK_DEFAULT_SIZE;
 807
 808                 /*
 809                  * Assumption: Ok ignore the passed digest pointer and place HMAC at end
 810                  * of packet.
 811                  */
 812                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 813         }
 814
 815         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 816                 /*
 817                  * Failed to place a packet on the hardware queue.
 818                  * Most likely because the QA hardware is busy.
 819                  */
 820                 return CRYPTO_RESULT_FAIL;
 821         }
 822         return CRYPTO_RESULT_IN_PROGRESS;
 823 }
 824
 825 enum crypto_result
 826 crypto_decrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 827 {
 828
 829         CpaCySymDpOpData *opData = (void*) (((char *) rte_buff->pkt.data)
 830                         + CRYPTO_OFFSET_TO_OPDATA);
 831         uint32_t lcore_id;
 832
 833         lcore_id = rte_lcore_id();
 834
 835         bzero(opData, sizeof(CpaCySymDpOpData));
 836
 837         opData->dstBuffer = opData->srcBuffer = PACKET_DATA_START_PHYS(rte_buff);
 838         opData->dstBufferLen = opData->srcBufferLen = rte_buff->pkt.data_len;
 839         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 840                         + CRYPTO_OFFSET_TO_OPDATA;
 841         opData->sessionCtx = qaCoreConf[lcore_id].decryptSessionHandleTbl[c][h];
 842         opData->pCallbackTag = rte_buff;
 843
 844         /* if no crypto or hmac operations are specified return fail */
 845         if (NO_CIPHER == c && NO_HASH == h)
 846                 return CRYPTO_RESULT_FAIL;
 847
 848         if (NO_CIPHER != c) {
 849                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 850                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 851
 852                 if (CIPHER_AES_CBC_128 == c)
 853                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 854                 else
 855                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 856
 857                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 858                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 859                                 - CRYPTO_START_OFFSET;
 860
 861                 /*
 862                  * Work around for padding, message length has to be a multiple of block
 863                  * size.
 864                  */
 865                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 866                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 867         }
 868         if (NO_HASH != h) {
 869                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 870                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 871                                 - HASH_START_OFFSET;
 872                 /*
 873                  * Work around for padding, message length has to be a multiple of block
 874                  * size.
 875                  */
 876                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 877                                 % HASH_BLOCK_DEFAULT_SIZE;
 878                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 879         }
 880
 881         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 882                 /*
 883                  * Failed to place a packet on the hardware queue.
 884                  * Most likely because the QA hardware is busy.
 885                  */
 886                 return CRYPTO_RESULT_FAIL;
 887         }
 888         return CRYPTO_RESULT_IN_PROGRESS;
 889 }
 890
 891 void *
 892 crypto_get_next_response(void)
 893 {
 894         uint32_t lcore_id;
 895         lcore_id = rte_lcore_id();
 896         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 897         void *entry = NULL;
 898
 899         if (callbackQ->numEntries) {
 900                 entry = callbackQ->qaCallbackRing[callbackQ->tail];
 901                 callbackQ->tail++;
 902                 callbackQ->numEntries--;
 903         }
 904
 905         /* If there are no outstanding requests no need to poll, return entry */
 906         if (qaCoreConf[lcore_id].qaOutstandingRequests == 0)
 907                 return entry;
 908
 909         if (callbackQ->numEntries < CRYPTO_QUEUED_RESP_POLL_THRESHOLD
 910                         && qaCoreConf[lcore_id].numResponseAttempts++
 911                                         % GET_NEXT_RESPONSE_FREQ == 0) {
 912                 /*
 913                  * Only poll the hardware when there is less than
 914                  * CRYPTO_QUEUED_RESP_POLL_THRESHOLD elements in the software queue
 915                  */
 916                 icp_sal_CyPollDpInstance(qaCoreConf[lcore_id].instanceHandle,
 917                                 CRYPTO_MAX_RESPONSE_QUOTA);
 918         }
 919         return entry;
 920 }