examples/dpdk_qat/crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <strings.h>
  38 #include <string.h>
  39 #include <inttypes.h>
  40 #include <errno.h>
  41 #include <sys/queue.h>
  42 #include <stdarg.h>
  43
  44 #include <rte_common.h>
  45 #include <rte_log.h>
  46 #include <rte_debug.h>
  47 #include <rte_memory.h>
  48 #include <rte_memzone.h>
  49 #include <rte_tailq.h>
  50 #include <rte_ether.h>
  51 #include <rte_malloc.h>
  52 #include <rte_launch.h>
  53 #include <rte_eal.h>
  54 #include <rte_per_lcore.h>
  55 #include <rte_lcore.h>
  56 #include <rte_atomic.h>
  57 #include <rte_branch_prediction.h>
  58 #include <rte_ring.h>
  59 #include <rte_mempool.h>
  60 #include <rte_mbuf.h>
  61 #include <rte_string_fns.h>
  62
  63 #define CPA_CY_SYM_DP_TMP_WORKAROUND 1
  64
  65 #include "cpa.h"
  66 #include "cpa_types.h"
  67 #include "cpa_cy_sym_dp.h"
  68 #include "cpa_cy_common.h"
  69 #include "cpa_cy_im.h"
  70 #include "icp_sal_user.h"
  71 #include "icp_sal_poll.h"
  72
  73 #include "crypto.h"
  74
  75 /* CIPHER KEY LENGTHS */
  76 #define KEY_SIZE_64_IN_BYTES    (64 / 8)
  77 #define KEY_SIZE_56_IN_BYTES    (56 / 8)
  78 #define KEY_SIZE_128_IN_BYTES   (128 / 8)
  79 #define KEY_SIZE_168_IN_BYTES   (168 / 8)
  80 #define KEY_SIZE_192_IN_BYTES   (192 / 8)
  81 #define KEY_SIZE_256_IN_BYTES   (256 / 8)
  82
  83 /* HMAC AUTH KEY LENGTHS */
  84 #define AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES       (128 / 8)
  85 #define SHA1_AUTH_KEY_LENGTH_IN_BYTES           (160 / 8)
  86 #define SHA224_AUTH_KEY_LENGTH_IN_BYTES         (224 / 8)
  87 #define SHA256_AUTH_KEY_LENGTH_IN_BYTES         (256 / 8)
  88 #define SHA384_AUTH_KEY_LENGTH_IN_BYTES         (384 / 8)
  89 #define SHA512_AUTH_KEY_LENGTH_IN_BYTES         (512 / 8)
  90 #define MD5_AUTH_KEY_LENGTH_IN_BYTES            (128 / 8)
  91
  92 /* HASH DIGEST LENGHTS */
  93 #define AES_XCBC_DIGEST_LENGTH_IN_BYTES         (128 / 8)
  94 #define AES_XCBC_96_DIGEST_LENGTH_IN_BYTES      (96 / 8)
  95 #define MD5_DIGEST_LENGTH_IN_BYTES              (128 / 8)
  96 #define SHA1_DIGEST_LENGTH_IN_BYTES             (160 / 8)
  97 #define SHA1_96_DIGEST_LENGTH_IN_BYTES          (96 / 8)
  98 #define SHA224_DIGEST_LENGTH_IN_BYTES           (224 / 8)
  99 #define SHA256_DIGEST_LENGTH_IN_BYTES           (256 / 8)
 100 #define SHA384_DIGEST_LENGTH_IN_BYTES           (384 / 8)
 101 #define SHA512_DIGEST_LENGTH_IN_BYTES           (512 / 8)
 102
 103 #define IV_LENGTH_16_BYTES      (16)
 104 #define IV_LENGTH_8_BYTES       (8)
 105
 106
 107 /*
 108  * rte_memzone is used to allocate physically contiguous virtual memory.
 109  * In this application we allocate a single block and divide between variables
 110  * which require a virtual to physical mapping for use by the QAT driver.
 111  * Virt2phys is only performed during initialisation and not on the data-path.
 112  */
 113
 114 #define LCORE_MEMZONE_SIZE      (1 << 22)
 115
 116 struct lcore_memzone
 117 {
 118         const struct rte_memzone *memzone;
 119         void *next_free_address;
 120 };
 121
 122 /*
 123  * Size the qa software response queue.
 124  * Note: Head and Tail are 8 bit, therefore, the queue is
 125  * fixed to 256 entries.
 126  */
 127 #define CRYPTO_SOFTWARE_QUEUE_SIZE 256
 128
 129 struct qa_callbackQueue {
 130         uint8_t head;
 131         uint8_t tail;
 132         uint16_t numEntries;
 133         struct rte_mbuf *qaCallbackRing[CRYPTO_SOFTWARE_QUEUE_SIZE];
 134 };
 135
 136 struct qa_core_conf {
 137         CpaCySymDpSessionCtx *encryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 138         CpaCySymDpSessionCtx *decryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 139         CpaInstanceHandle instanceHandle;
 140         struct qa_callbackQueue callbackQueue;
 141         uint64_t qaOutstandingRequests;
 142         uint64_t numResponseAttempts;
 143         uint8_t kickFreq;
 144         void *pPacketIV;
 145         CpaPhysicalAddr packetIVPhy;
 146         struct lcore_memzone lcoreMemzone;
 147 } __rte_cache_aligned;
 148
 149 #define MAX_CORES   (RTE_MAX_LCORE)
 150
 151 static struct qa_core_conf qaCoreConf[MAX_CORES];
 152
 153 /*
 154  *Create maximum possible key size,
 155  *One for cipher and one for hash
 156  */
 157 struct glob_keys {
 158         uint8_t cipher_key[32];
 159         uint8_t hash_key[64];
 160         uint8_t iv[16];
 161 };
 162
 163 struct glob_keys g_crypto_hash_keys = {
 164         .cipher_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 165                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 166                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 167                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20},
 168         .hash_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 169                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 170                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 171                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,
 172                 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,
 173                 0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,
 174                 0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,
 175                 0x39,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50},
 176         .iv = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 177                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10}
 178 };
 179
 180 /*
 181  * Offsets from the start of the packet.
 182  *
 183  */
 184 #define PACKET_DATA_START_PHYS(p) \
 185                 ((p)->buf_physaddr + ((char *)p->pkt.data - (char *)p->buf_addr))
 186
 187 /*
 188  * A fixed offset to where the crypto is to be performed, which is the first
 189  * byte after the Ethernet(14 bytes) and IPv4 headers(20 bytes)
 190  */
 191 #define CRYPTO_START_OFFSET             (14+20)
 192 #define HASH_START_OFFSET               (14+20)
 193 #define CIPHER_BLOCK_DEFAULT_SIZE       (16)
 194 #define HASH_BLOCK_DEFAULT_SIZE         (16)
 195
 196 /*
 197  * Offset to the opdata from the start of the data portion of packet.
 198  * Assumption: The buffer is physically contiguous.
 199  * +18 takes this to the next cache line.
 200  */
 201
 202 #define CRYPTO_OFFSET_TO_OPDATA         (ETHER_MAX_LEN+18)
 203
 204 /*
 205  * Default number of requests to place on the hardware ring before kicking the
 206  * ring pointers.
 207  */
 208 #define CRYPTO_BURST_TX (16)
 209
 210 /*
 211  * Only call the qa poll function when the number responses in the software
 212  * queue drops below this number.
 213  */
 214 #define CRYPTO_QUEUED_RESP_POLL_THRESHOLD       (32)
 215
 216 /*
 217  * Limit the number of polls per call to get_next_response.
 218  */
 219 #define GET_NEXT_RESPONSE_FREQ  (32)
 220
 221 /*
 222  * Max number of responses to pull from the qa in one poll.
 223  */
 224 #define CRYPTO_MAX_RESPONSE_QUOTA \
 225                 (CRYPTO_SOFTWARE_QUEUE_SIZE-CRYPTO_QUEUED_RESP_POLL_THRESHOLD-1)
 226
 227 #if (CRYPTO_QUEUED_RESP_POLL_THRESHOLD + CRYPTO_MAX_RESPONSE_QUOTA >= \
 228                 CRYPTO_SOFTWARE_QUEUE_SIZE)
 229 #error Its possible to overflow the qa response Q with current poll and \
 230                 response quota.
 231 #endif
 232
 233 static void
 234 crypto_callback(CpaCySymDpOpData *pOpData,
 235                 __rte_unused CpaStatus status,
 236                 __rte_unused CpaBoolean verifyResult)
 237 {
 238         uint32_t lcore_id;
 239         lcore_id = rte_lcore_id();
 240         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 241
 242         /*
 243          * Received a completion from the QA hardware.
 244          * Place the response on the return queue.
 245          */
 246         callbackQ->qaCallbackRing[callbackQ->head] = pOpData->pCallbackTag;
 247         callbackQ->head++;
 248         callbackQ->numEntries++;
 249         qaCoreConf[lcore_id].qaOutstandingRequests--;
 250 }
 251
 252 static void
 253 qa_crypto_callback(CpaCySymDpOpData *pOpData, CpaStatus status,
 254                 CpaBoolean verifyResult)
 255 {
 256         crypto_callback(pOpData, status, verifyResult);
 257 }
 258
 259 /*
 260  * Each allocation from a particular memzone lasts for the life-time of
 261  * the application. No freeing of previous allocations will occur.
 262  */
 263 static void *
 264 alloc_memzone_region(uint32_t length, uint32_t lcore_id)
 265 {
 266         char *current_free_addr_ptr = NULL;
 267         struct lcore_memzone *lcore_memzone = &(qaCoreConf[lcore_id].lcoreMemzone);
 268
 269         current_free_addr_ptr  = lcore_memzone->next_free_address;
 270
 271         if (current_free_addr_ptr + length >=
 272                 (char *)lcore_memzone->memzone->addr + lcore_memzone->memzone->len) {
 273                 printf("Crypto: No memory available in memzone\n");
 274                 return NULL;
 275         }
 276         lcore_memzone->next_free_address = current_free_addr_ptr + length;
 277
 278         return (void *)current_free_addr_ptr;
 279 }
 280
 281 /*
 282  * Virtual to Physical Address translation is only executed during initialization
 283  * and not on the data-path.
 284  */
 285 static CpaPhysicalAddr
 286 qa_v2p(void *ptr)
 287 {
 288         const struct rte_memzone *memzone = NULL;
 289         uint32_t lcore_id = 0;
 290         RTE_LCORE_FOREACH(lcore_id) {
 291                 memzone = qaCoreConf[lcore_id].lcoreMemzone.memzone;
 292
 293                 if ((char*) ptr >= (char *) memzone->addr &&
 294                                 (char*) ptr < ((char*) memzone->addr + memzone->len)) {
 295                         return (CpaPhysicalAddr)
 296                                         (memzone->phys_addr + ((char *) ptr - (char*) memzone->addr));
 297                 }
 298         }
 299         printf("Crypto: Corresponding physical address not found in memzone\n");
 300         return (CpaPhysicalAddr) 0;
 301 }
 302
 303 static CpaStatus
 304 getCoreAffinity(Cpa32U *coreAffinity, const CpaInstanceHandle instanceHandle)
 305 {
 306         CpaInstanceInfo2 info;
 307         Cpa16U i = 0;
 308         CpaStatus status = CPA_STATUS_SUCCESS;
 309
 310         bzero(&info, sizeof(CpaInstanceInfo2));
 311
 312         status = cpaCyInstanceGetInfo2(instanceHandle, &info);
 313         if (CPA_STATUS_SUCCESS != status) {
 314                 printf("Crypto: Error getting instance info\n");
 315                 return CPA_STATUS_FAIL;
 316         }
 317         for (i = 0; i < MAX_CORES; i++) {
 318                 if (CPA_BITMAP_BIT_TEST(info.coreAffinity, i)) {
 319                         *coreAffinity = i;
 320                         return CPA_STATUS_SUCCESS;
 321                 }
 322         }
 323         return CPA_STATUS_FAIL;
 324 }
 325
 326 static CpaStatus
 327 get_crypto_instance_on_core(CpaInstanceHandle *pInstanceHandle,
 328                 uint32_t lcore_id)
 329 {
 330         Cpa16U numInstances = 0, i = 0;
 331         CpaStatus status = CPA_STATUS_FAIL;
 332         CpaInstanceHandle *pLocalInstanceHandles = NULL;
 333         Cpa32U coreAffinity = 0;
 334
 335         status = cpaCyGetNumInstances(&numInstances);
 336         if (CPA_STATUS_SUCCESS != status || numInstances == 0) {
 337                 return CPA_STATUS_FAIL;
 338         }
 339
 340         pLocalInstanceHandles = rte_malloc("pLocalInstanceHandles",
 341                         sizeof(CpaInstanceHandle) * numInstances, CACHE_LINE_SIZE);
 342
 343         if (NULL == pLocalInstanceHandles) {
 344                 return CPA_STATUS_FAIL;
 345         }
 346         status = cpaCyGetInstances(numInstances, pLocalInstanceHandles);
 347         if (CPA_STATUS_SUCCESS != status) {
 348                 printf("Crypto: cpaCyGetInstances failed with status: %"PRId32"\n", status);
 349                 rte_free((void *) pLocalInstanceHandles);
 350                 return CPA_STATUS_FAIL;
 351         }
 352
 353         for (i = 0; i < numInstances; i++) {
 354                 status = getCoreAffinity(&coreAffinity, pLocalInstanceHandles[i]);
 355                 if (CPA_STATUS_SUCCESS != status) {
 356                         rte_free((void *) pLocalInstanceHandles);
 357                         return CPA_STATUS_FAIL;
 358                 }
 359                 if (coreAffinity == lcore_id) {
 360                         printf("Crypto: instance found on core %d\n", i);
 361                         *pInstanceHandle = pLocalInstanceHandles[i];
 362                         return CPA_STATUS_SUCCESS;
 363                 }
 364         }
 365         /* core affinity not found */
 366         rte_free((void *) pLocalInstanceHandles);
 367         return CPA_STATUS_FAIL;
 368 }
 369
 370 static CpaStatus
 371 initCySymSession(const int pkt_cipher_alg,
 372                 const int pkt_hash_alg, const CpaCySymHashMode hashMode,
 373                 const CpaCySymCipherDirection crypto_direction,
 374                 CpaCySymSessionCtx **ppSessionCtx,
 375                 const CpaInstanceHandle cyInstanceHandle,
 376                 const uint32_t lcore_id)
 377 {
 378         Cpa32U sessionCtxSizeInBytes = 0;
 379         CpaStatus status = CPA_STATUS_FAIL;
 380         CpaBoolean isCrypto = CPA_TRUE, isHmac = CPA_TRUE;
 381         CpaCySymSessionSetupData sessionSetupData;
 382
 383         bzero(&sessionSetupData, sizeof(CpaCySymSessionSetupData));
 384
 385         /* Assumption: key length is set to each algorithm's max length */
 386         switch (pkt_cipher_alg) {
 387         case NO_CIPHER:
 388                 isCrypto = CPA_FALSE;
 389                 break;
 390         case CIPHER_DES:
 391                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 392                                 CPA_CY_SYM_CIPHER_DES_ECB;
 393                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 394                                 KEY_SIZE_64_IN_BYTES;
 395                 break;
 396         case CIPHER_DES_CBC:
 397                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 398                                 CPA_CY_SYM_CIPHER_DES_CBC;
 399                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 400                                 KEY_SIZE_64_IN_BYTES;
 401                 break;
 402         case CIPHER_DES3:
 403                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 404                                 CPA_CY_SYM_CIPHER_3DES_ECB;
 405                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 406                                 KEY_SIZE_192_IN_BYTES;
 407                 break;
 408         case CIPHER_DES3_CBC:
 409                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 410                                 CPA_CY_SYM_CIPHER_3DES_CBC;
 411                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 412                                 KEY_SIZE_192_IN_BYTES;
 413                 break;
 414         case CIPHER_AES:
 415                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 416                                 CPA_CY_SYM_CIPHER_AES_ECB;
 417                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 418                                 KEY_SIZE_128_IN_BYTES;
 419                 break;
 420         case CIPHER_AES_CBC_128:
 421                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 422                                 CPA_CY_SYM_CIPHER_AES_CBC;
 423                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 424                                 KEY_SIZE_128_IN_BYTES;
 425                 break;
 426         default:
 427                 printf("Crypto: Undefined Cipher specified\n");
 428                 break;
 429         }
 430         /* Set the cipher direction */
 431         if (isCrypto) {
 432                 sessionSetupData.cipherSetupData.cipherDirection = crypto_direction;
 433                 sessionSetupData.cipherSetupData.pCipherKey =
 434                                 g_crypto_hash_keys.cipher_key;
 435                 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
 436         }
 437
 438         /* Setup Hash common fields */
 439         switch (pkt_hash_alg) {
 440         case NO_HASH:
 441                 isHmac = CPA_FALSE;
 442                 break;
 443         case HASH_AES_XCBC:
 444                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 445                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 446                                 AES_XCBC_DIGEST_LENGTH_IN_BYTES;
 447                 break;
 448         case HASH_AES_XCBC_96:
 449                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 450                                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 451                                 AES_XCBC_96_DIGEST_LENGTH_IN_BYTES;
 452                 break;
 453         case HASH_MD5:
 454                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_MD5;
 455                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 456                                 MD5_DIGEST_LENGTH_IN_BYTES;
 457                 break;
 458         case HASH_SHA1:
 459                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 460                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 461                                 SHA1_DIGEST_LENGTH_IN_BYTES;
 462                 break;
 463         case HASH_SHA1_96:
 464                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 465                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 466                                 SHA1_96_DIGEST_LENGTH_IN_BYTES;
 467             break;
 468         case HASH_SHA224:
 469                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA224;
 470                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 471                                 SHA224_DIGEST_LENGTH_IN_BYTES;
 472                 break;
 473         case HASH_SHA256:
 474                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA256;
 475                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 476                                 SHA256_DIGEST_LENGTH_IN_BYTES;
 477                 break;
 478         case HASH_SHA384:
 479                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA384;
 480                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 481                                 SHA384_DIGEST_LENGTH_IN_BYTES;
 482                 break;
 483         case HASH_SHA512:
 484                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA512;
 485                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 486                                 SHA512_DIGEST_LENGTH_IN_BYTES;
 487                 break;
 488         default:
 489                 printf("Crypto: Undefined Hash specified\n");
 490                 break;
 491         }
 492         if (isHmac) {
 493                 sessionSetupData.hashSetupData.hashMode = hashMode;
 494                 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
 495                 /* If using authenticated hash setup key lengths */
 496                 if (CPA_CY_SYM_HASH_MODE_AUTH == hashMode) {
 497                         /* Use a common max length key */
 498                         sessionSetupData.hashSetupData.authModeSetupData.authKey =
 499                                         g_crypto_hash_keys.hash_key;
 500                         switch (pkt_hash_alg) {
 501                         case HASH_AES_XCBC:
 502                         case HASH_AES_XCBC_96:
 503                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 504                                                 AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES;
 505                                 break;
 506                         case HASH_MD5:
 507                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 508                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 509                                 break;
 510                         case HASH_SHA1:
 511                         case HASH_SHA1_96:
 512                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 513                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 514                                 break;
 515                         case HASH_SHA224:
 516                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 517                                                 SHA224_AUTH_KEY_LENGTH_IN_BYTES;
 518                                 break;
 519                         case HASH_SHA256:
 520                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 521                                                 SHA256_AUTH_KEY_LENGTH_IN_BYTES;
 522                                 break;
 523                         case HASH_SHA384:
 524                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 525                                                 SHA384_AUTH_KEY_LENGTH_IN_BYTES;
 526                                 break;
 527                         case HASH_SHA512:
 528                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 529                                                 SHA512_AUTH_KEY_LENGTH_IN_BYTES;
 530                                 break;
 531                         default:
 532                                 printf("Crypto: Undefined Hash specified\n");
 533                                 return CPA_STATUS_FAIL;
 534                         }
 535                 }
 536         }
 537
 538         /* Only high priority supported */
 539         sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
 540
 541         /* If chaining algorithms */
 542         if (isCrypto && isHmac) {
 543                 sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 544                 /* @assumption Alg Chain order is cipher then hash for encrypt
 545                  * and hash then cipher then has for decrypt*/
 546                 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == crypto_direction) {
 547                         sessionSetupData.algChainOrder =
 548                                         CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 549                 } else {
 550                         sessionSetupData.algChainOrder =
 551                                         CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 552                 }
 553         }
 554         if (!isCrypto && !isHmac) {
 555                 *ppSessionCtx = NULL;
 556                 return CPA_STATUS_SUCCESS;
 557         }
 558
 559         /* Set flags for digest operations */
 560         sessionSetupData.digestIsAppended = CPA_FALSE;
 561         sessionSetupData.verifyDigest = CPA_TRUE;
 562
 563         /* Get the session context size based on the crypto and/or hash operations*/
 564         status = cpaCySymDpSessionCtxGetSize(cyInstanceHandle, &sessionSetupData,
 565                         &sessionCtxSizeInBytes);
 566         if (CPA_STATUS_SUCCESS != status) {
 567                 printf("Crypto: cpaCySymDpSessionCtxGetSize error, status: %"PRId32"\n",
 568                                 status);
 569                 return CPA_STATUS_FAIL;
 570         }
 571
 572         *ppSessionCtx = alloc_memzone_region(sessionCtxSizeInBytes, lcore_id);
 573         if (NULL == *ppSessionCtx) {
 574                 printf("Crypto: Failed to allocate memory for Session Context\n");
 575                 return CPA_STATUS_FAIL;
 576         }
 577
 578         status = cpaCySymDpInitSession(cyInstanceHandle, &sessionSetupData,
 579                         *ppSessionCtx);
 580         if (CPA_STATUS_SUCCESS != status) {
 581                 printf("Crypto: cpaCySymDpInitSession failed with status %"PRId32"\n", status);
 582                 return CPA_STATUS_FAIL;
 583         }
 584         return CPA_STATUS_SUCCESS;
 585 }
 586
 587 static CpaStatus
 588 initSessionDataTables(struct qa_core_conf *qaCoreConf,uint32_t lcore_id)
 589 {
 590         Cpa32U i = 0, j = 0;
 591         CpaStatus status = CPA_STATUS_FAIL;
 592         for (i = 0; i < NUM_CRYPTO; i++) {
 593                 for (j = 0; j < NUM_HMAC; j++) {
 594                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 595                                         CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
 596                                         &qaCoreConf->encryptSessionHandleTbl[i][j],
 597                                         qaCoreConf->instanceHandle,
 598                                         lcore_id);
 599                         if (CPA_STATUS_SUCCESS != status) {
 600                                 printf("Crypto: Failed to initialize Encrypt sessions\n");
 601                                 return CPA_STATUS_FAIL;
 602                         }
 603                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 604                                         CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
 605                                         &qaCoreConf->decryptSessionHandleTbl[i][j],
 606                                         qaCoreConf->instanceHandle,
 607                                         lcore_id);
 608                         if (CPA_STATUS_SUCCESS != status) {
 609                                 printf("Crypto: Failed to initialize Decrypt sessions\n");
 610                                 return CPA_STATUS_FAIL;
 611                         }
 612                 }
 613         }
 614         return CPA_STATUS_SUCCESS;
 615 }
 616
 617 int
 618 crypto_init(void)
 619 {
 620         if (CPA_STATUS_SUCCESS != icp_sal_userStartMultiProcess("SSL",CPA_FALSE)) {
 621                 printf("Crypto: Could not start sal for user space\n");
 622                 return CPA_STATUS_FAIL;
 623         }
 624         printf("Crypto: icp_sal_userStartMultiProcess(\"SSL\",CPA_FALSE)\n");
 625         return 0;
 626 }
 627
 628 /*
 629  * Per core initialisation
 630  */
 631 int
 632 per_core_crypto_init(uint32_t lcore_id)
 633 {
 634         CpaStatus status = CPA_STATUS_FAIL;
 635         char memzone_name[RTE_MEMZONE_NAMESIZE];
 636
 637         int socketID = rte_lcore_to_socket_id(lcore_id);
 638
 639         /* Allocate software ring for response messages. */
 640
 641         qaCoreConf[lcore_id].callbackQueue.head = 0;
 642         qaCoreConf[lcore_id].callbackQueue.tail = 0;
 643         qaCoreConf[lcore_id].callbackQueue.numEntries = 0;
 644         qaCoreConf[lcore_id].kickFreq = 0;
 645         qaCoreConf[lcore_id].qaOutstandingRequests = 0;
 646         qaCoreConf[lcore_id].numResponseAttempts = 0;
 647
 648         /* Initialise and reserve lcore memzone for virt2phys translation */
 649         rte_snprintf(memzone_name,
 650                         RTE_MEMZONE_NAMESIZE,
 651                         "lcore_%u",
 652                         lcore_id);
 653
 654         qaCoreConf[lcore_id].lcoreMemzone.memzone = rte_memzone_reserve(
 655                         memzone_name,
 656                         LCORE_MEMZONE_SIZE,
 657                         socketID,
 658                         0);
 659         if (NULL == qaCoreConf[lcore_id].lcoreMemzone.memzone) {
 660                 printf("Crypto: Error allocating memzone on lcore %u\n",lcore_id);
 661                 return -1;
 662         }
 663         qaCoreConf[lcore_id].lcoreMemzone.next_free_address =
 664                                                         qaCoreConf[lcore_id].lcoreMemzone.memzone->addr;
 665
 666         qaCoreConf[lcore_id].pPacketIV = alloc_memzone_region(IV_LENGTH_16_BYTES,
 667                                                         lcore_id);
 668
 669         if (NULL == qaCoreConf[lcore_id].pPacketIV ) {
 670                 printf("Crypto: Failed to allocate memory for Initialization Vector\n");
 671                 return -1;
 672         }
 673
 674         memcpy(qaCoreConf[lcore_id].pPacketIV, &g_crypto_hash_keys.iv,
 675                         IV_LENGTH_16_BYTES);
 676
 677         qaCoreConf[lcore_id].packetIVPhy = qa_v2p(qaCoreConf[lcore_id].pPacketIV);
 678         if (0 == qaCoreConf[lcore_id].packetIVPhy) {
 679                 printf("Crypto: Invalid physical address for Initialization Vector\n");
 680                 return -1;
 681         }
 682
 683         /*
 684          * Obtain the instance handle that is mapped to the current lcore.
 685          * This can fail if an instance is not mapped to a bank which has been
 686          * affinitized to the current lcore.
 687          */
 688         status = get_crypto_instance_on_core(&(qaCoreConf[lcore_id].instanceHandle),
 689                         lcore_id);
 690         if (CPA_STATUS_SUCCESS != status) {
 691                 printf("Crypto: get_crypto_instance_on_core failed with status: %"PRId32"\n",
 692                                 status);
 693                 return -1;
 694         }
 695
 696         status = cpaCySymDpRegCbFunc(qaCoreConf[lcore_id].instanceHandle,
 697                         (CpaCySymDpCbFunc) qa_crypto_callback);
 698         if (CPA_STATUS_SUCCESS != status) {
 699                 printf("Crypto: cpaCySymDpRegCbFunc failed with status: %"PRId32"\n", status);
 700                 return -1;
 701         }
 702
 703         /*
 704          * Set the address translation callback for virtual to physcial address
 705          * mapping. This will be called by the QAT driver during initialisation only.
 706          */
 707         status = cpaCySetAddressTranslation(qaCoreConf[lcore_id].instanceHandle,
 708                         (CpaVirtualToPhysical) qa_v2p);
 709         if (CPA_STATUS_SUCCESS != status) {
 710                 printf("Crypto: cpaCySetAddressTranslation failed with status: %"PRId32"\n",
 711                                 status);
 712                 return -1;
 713         }
 714
 715         status = initSessionDataTables(&qaCoreConf[lcore_id],lcore_id);
 716         if (CPA_STATUS_SUCCESS != status) {
 717                 printf("Crypto: Failed to allocate all session tables.");
 718                 return -1;
 719         }
 720         return 0;
 721 }
 722
 723 static CpaStatus
 724 enqueueOp(CpaCySymDpOpData *opData, uint32_t lcore_id)
 725 {
 726
 727         CpaStatus status;
 728
 729         /*
 730          * Assumption is there is no requirement to do load balancing between
 731          * acceleration units - that is one acceleration unit is tied to a core.
 732          */
 733         opData->instanceHandle = qaCoreConf[lcore_id].instanceHandle;
 734
 735         if ((++qaCoreConf[lcore_id].kickFreq) % CRYPTO_BURST_TX == 0) {
 736                 status = cpaCySymDpEnqueueOp(opData, CPA_TRUE);
 737         } else {
 738                 status = cpaCySymDpEnqueueOp(opData, CPA_FALSE);
 739         }
 740
 741         qaCoreConf[lcore_id].qaOutstandingRequests++;
 742
 743         return status;
 744 }
 745
 746 void
 747 crypto_flush_tx_queue(uint32_t lcore_id)
 748 {
 749
 750         cpaCySymDpPerformOpNow(qaCoreConf[lcore_id].instanceHandle);
 751 }
 752
 753 enum crypto_result
 754 crypto_encrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 755 {
 756         CpaCySymDpOpData *opData =
 757                         (CpaCySymDpOpData *) ((char *) (rte_buff->pkt.data)
 758                                         + CRYPTO_OFFSET_TO_OPDATA);
 759         uint32_t lcore_id;
 760
 761         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 762                 return CRYPTO_RESULT_FAIL;
 763
 764         lcore_id = rte_lcore_id();
 765
 766         bzero(opData, sizeof(CpaCySymDpOpData));
 767
 768         opData->srcBuffer = opData->dstBuffer = PACKET_DATA_START_PHYS(rte_buff);
 769         opData->srcBufferLen = opData->dstBufferLen = rte_buff->pkt.data_len;
 770         opData->sessionCtx = qaCoreConf[lcore_id].encryptSessionHandleTbl[c][h];
 771         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 772                         + CRYPTO_OFFSET_TO_OPDATA;
 773         opData->pCallbackTag = rte_buff;
 774
 775         /* if no crypto or hash operations are specified return fail */
 776         if (NO_CIPHER == c && NO_HASH == h)
 777                 return CRYPTO_RESULT_FAIL;
 778
 779         if (NO_CIPHER != c) {
 780                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 781                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 782
 783                 if (CIPHER_AES_CBC_128 == c)
 784                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 785                 else
 786                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 787
 788                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 789                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 790                                 - CRYPTO_START_OFFSET;
 791                 /*
 792                  * Work around for padding, message length has to be a multiple of
 793                  * block size.
 794                  */
 795                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 796                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 797         }
 798
 799         if (NO_HASH != h) {
 800
 801                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 802                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 803                                 - HASH_START_OFFSET;
 804                 /*
 805                  * Work around for padding, message length has to be a multiple of block
 806                  * size.
 807                  */
 808                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 809                                 % HASH_BLOCK_DEFAULT_SIZE;
 810
 811                 /*
 812                  * Assumption: Ok ignore the passed digest pointer and place HMAC at end
 813                  * of packet.
 814                  */
 815                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 816         }
 817
 818         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 819                 /*
 820                  * Failed to place a packet on the hardware queue.
 821                  * Most likely because the QA hardware is busy.
 822                  */
 823                 return CRYPTO_RESULT_FAIL;
 824         }
 825         return CRYPTO_RESULT_IN_PROGRESS;
 826 }
 827
 828 enum crypto_result
 829 crypto_decrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 830 {
 831
 832         CpaCySymDpOpData *opData = (void*) (((char *) rte_buff->pkt.data)
 833                         + CRYPTO_OFFSET_TO_OPDATA);
 834         uint32_t lcore_id;
 835
 836         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 837                 return CRYPTO_RESULT_FAIL;
 838
 839         lcore_id = rte_lcore_id();
 840
 841         bzero(opData, sizeof(CpaCySymDpOpData));
 842
 843         opData->dstBuffer = opData->srcBuffer = PACKET_DATA_START_PHYS(rte_buff);
 844         opData->dstBufferLen = opData->srcBufferLen = rte_buff->pkt.data_len;
 845         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 846                         + CRYPTO_OFFSET_TO_OPDATA;
 847         opData->sessionCtx = qaCoreConf[lcore_id].decryptSessionHandleTbl[c][h];
 848         opData->pCallbackTag = rte_buff;
 849
 850         /* if no crypto or hmac operations are specified return fail */
 851         if (NO_CIPHER == c && NO_HASH == h)
 852                 return CRYPTO_RESULT_FAIL;
 853
 854         if (NO_CIPHER != c) {
 855                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 856                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 857
 858                 if (CIPHER_AES_CBC_128 == c)
 859                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 860                 else
 861                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 862
 863                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 864                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 865                                 - CRYPTO_START_OFFSET;
 866
 867                 /*
 868                  * Work around for padding, message length has to be a multiple of block
 869                  * size.
 870                  */
 871                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 872                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 873         }
 874         if (NO_HASH != h) {
 875                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 876                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 877                                 - HASH_START_OFFSET;
 878                 /*
 879                  * Work around for padding, message length has to be a multiple of block
 880                  * size.
 881                  */
 882                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 883                                 % HASH_BLOCK_DEFAULT_SIZE;
 884                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 885         }
 886
 887         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 888                 /*
 889                  * Failed to place a packet on the hardware queue.
 890                  * Most likely because the QA hardware is busy.
 891                  */
 892                 return CRYPTO_RESULT_FAIL;
 893         }
 894         return CRYPTO_RESULT_IN_PROGRESS;
 895 }
 896
 897 void *
 898 crypto_get_next_response(void)
 899 {
 900         uint32_t lcore_id;
 901         lcore_id = rte_lcore_id();
 902         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 903         void *entry = NULL;
 904
 905         if (callbackQ->numEntries) {
 906                 entry = callbackQ->qaCallbackRing[callbackQ->tail];
 907                 callbackQ->tail++;
 908                 callbackQ->numEntries--;
 909         }
 910
 911         /* If there are no outstanding requests no need to poll, return entry */
 912         if (qaCoreConf[lcore_id].qaOutstandingRequests == 0)
 913                 return entry;
 914
 915         if (callbackQ->numEntries < CRYPTO_QUEUED_RESP_POLL_THRESHOLD
 916                         && qaCoreConf[lcore_id].numResponseAttempts++
 917                                         % GET_NEXT_RESPONSE_FREQ == 0) {
 918                 /*
 919                  * Only poll the hardware when there is less than
 920                  * CRYPTO_QUEUED_RESP_POLL_THRESHOLD elements in the software queue
 921                  */
 922                 icp_sal_CyPollDpInstance(qaCoreConf[lcore_id].instanceHandle,
 923                                 CRYPTO_MAX_RESPONSE_QUOTA);
 924         }
 925         return entry;
 926 }