examples/dpdk_qat/crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_tailq.h>
  49 #include <rte_ether.h>
  50 #include <rte_malloc.h>
  51 #include <rte_launch.h>
  52 #include <rte_eal.h>
  53 #include <rte_per_lcore.h>
  54 #include <rte_lcore.h>
  55 #include <rte_atomic.h>
  56 #include <rte_branch_prediction.h>
  57 #include <rte_ring.h>
  58 #include <rte_mempool.h>
  59 #include <rte_mbuf.h>
  60 #include <rte_string_fns.h>
  61
  62 #define CPA_CY_SYM_DP_TMP_WORKAROUND 1
  63
  64 #include "cpa.h"
  65 #include "cpa_types.h"
  66 #include "cpa_cy_sym_dp.h"
  67 #include "cpa_cy_common.h"
  68 #include "cpa_cy_im.h"
  69 #include "icp_sal_user.h"
  70 #include "icp_sal_poll.h"
  71
  72 #include "crypto.h"
  73
  74 /* CIPHER KEY LENGTHS */
  75 #define KEY_SIZE_64_IN_BYTES    (64 / 8)
  76 #define KEY_SIZE_56_IN_BYTES    (56 / 8)
  77 #define KEY_SIZE_128_IN_BYTES   (128 / 8)
  78 #define KEY_SIZE_168_IN_BYTES   (168 / 8)
  79 #define KEY_SIZE_192_IN_BYTES   (192 / 8)
  80 #define KEY_SIZE_256_IN_BYTES   (256 / 8)
  81
  82 /* HMAC AUTH KEY LENGTHS */
  83 #define AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES       (128 / 8)
  84 #define SHA1_AUTH_KEY_LENGTH_IN_BYTES           (160 / 8)
  85 #define SHA224_AUTH_KEY_LENGTH_IN_BYTES         (224 / 8)
  86 #define SHA256_AUTH_KEY_LENGTH_IN_BYTES         (256 / 8)
  87 #define SHA384_AUTH_KEY_LENGTH_IN_BYTES         (384 / 8)
  88 #define SHA512_AUTH_KEY_LENGTH_IN_BYTES         (512 / 8)
  89 #define MD5_AUTH_KEY_LENGTH_IN_BYTES            (128 / 8)
  90
  91 /* HASH DIGEST LENGHTS */
  92 #define AES_XCBC_DIGEST_LENGTH_IN_BYTES         (128 / 8)
  93 #define AES_XCBC_96_DIGEST_LENGTH_IN_BYTES      (96 / 8)
  94 #define MD5_DIGEST_LENGTH_IN_BYTES              (128 / 8)
  95 #define SHA1_DIGEST_LENGTH_IN_BYTES             (160 / 8)
  96 #define SHA1_96_DIGEST_LENGTH_IN_BYTES          (96 / 8)
  97 #define SHA224_DIGEST_LENGTH_IN_BYTES           (224 / 8)
  98 #define SHA256_DIGEST_LENGTH_IN_BYTES           (256 / 8)
  99 #define SHA384_DIGEST_LENGTH_IN_BYTES           (384 / 8)
 100 #define SHA512_DIGEST_LENGTH_IN_BYTES           (512 / 8)
 101
 102 #define IV_LENGTH_16_BYTES      (16)
 103 #define IV_LENGTH_8_BYTES       (8)
 104
 105
 106 /*
 107  * rte_memzone is used to allocate physically contiguous virtual memory.
 108  * In this application we allocate a single block and divide between variables
 109  * which require a virtual to physical mapping for use by the QAT driver.
 110  * Virt2phys is only performed during initialisation and not on the data-path.
 111  */
 112
 113 #define LCORE_MEMZONE_SIZE      (1 << 22)
 114
 115 struct lcore_memzone
 116 {
 117         const struct rte_memzone *memzone;
 118         void *next_free_address;
 119 };
 120
 121 /*
 122  * Size the qa software response queue.
 123  * Note: Head and Tail are 8 bit, therefore, the queue is
 124  * fixed to 256 entries.
 125  */
 126 #define CRYPTO_SOFTWARE_QUEUE_SIZE 256
 127
 128 struct qa_callbackQueue {
 129         uint8_t head;
 130         uint8_t tail;
 131         uint16_t numEntries;
 132         struct rte_mbuf *qaCallbackRing[CRYPTO_SOFTWARE_QUEUE_SIZE];
 133 };
 134
 135 struct qa_core_conf {
 136         CpaCySymDpSessionCtx *encryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 137         CpaCySymDpSessionCtx *decryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 138         CpaInstanceHandle instanceHandle;
 139         struct qa_callbackQueue callbackQueue;
 140         uint64_t qaOutstandingRequests;
 141         uint64_t numResponseAttempts;
 142         uint8_t kickFreq;
 143         void *pPacketIV;
 144         CpaPhysicalAddr packetIVPhy;
 145         struct lcore_memzone lcoreMemzone;
 146 } __rte_cache_aligned;
 147
 148 #define MAX_CORES   (RTE_MAX_LCORE)
 149
 150 static struct qa_core_conf qaCoreConf[MAX_CORES];
 151
 152 /*
 153  *Create maximum possible key size,
 154  *One for cipher and one for hash
 155  */
 156 struct glob_keys {
 157         uint8_t cipher_key[32];
 158         uint8_t hash_key[64];
 159         uint8_t iv[16];
 160 };
 161
 162 struct glob_keys g_crypto_hash_keys = {
 163         .cipher_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 164                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 165                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 166                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20},
 167         .hash_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 168                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 169                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 170                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,
 171                 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,
 172                 0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,
 173                 0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,
 174                 0x39,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50},
 175         .iv = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 176                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10}
 177 };
 178
 179 /*
 180  * Offsets from the start of the packet.
 181  *
 182  */
 183 #define PACKET_DATA_START_PHYS(p) \
 184                 ((p)->buf_physaddr + ((char *)p->pkt.data - (char *)p->buf_addr))
 185
 186 /*
 187  * A fixed offset to where the crypto is to be performed, which is the first
 188  * byte after the Ethernet(14 bytes) and IPv4 headers(20 bytes)
 189  */
 190 #define CRYPTO_START_OFFSET             (14+20)
 191 #define HASH_START_OFFSET               (14+20)
 192 #define CIPHER_BLOCK_DEFAULT_SIZE       (16)
 193 #define HASH_BLOCK_DEFAULT_SIZE         (16)
 194
 195 /*
 196  * Offset to the opdata from the start of the data portion of packet.
 197  * Assumption: The buffer is physically contiguous.
 198  * +18 takes this to the next cache line.
 199  */
 200
 201 #define CRYPTO_OFFSET_TO_OPDATA         (ETHER_MAX_LEN+18)
 202
 203 /*
 204  * Default number of requests to place on the hardware ring before kicking the
 205  * ring pointers.
 206  */
 207 #define CRYPTO_BURST_TX (16)
 208
 209 /*
 210  * Only call the qa poll function when the number responses in the software
 211  * queue drops below this number.
 212  */
 213 #define CRYPTO_QUEUED_RESP_POLL_THRESHOLD       (32)
 214
 215 /*
 216  * Limit the number of polls per call to get_next_response.
 217  */
 218 #define GET_NEXT_RESPONSE_FREQ  (32)
 219
 220 /*
 221  * Max number of responses to pull from the qa in one poll.
 222  */
 223 #define CRYPTO_MAX_RESPONSE_QUOTA \
 224                 (CRYPTO_SOFTWARE_QUEUE_SIZE-CRYPTO_QUEUED_RESP_POLL_THRESHOLD-1)
 225
 226 #if (CRYPTO_QUEUED_RESP_POLL_THRESHOLD + CRYPTO_MAX_RESPONSE_QUOTA >= \
 227                 CRYPTO_SOFTWARE_QUEUE_SIZE)
 228 #error Its possible to overflow the qa response Q with current poll and \
 229                 response quota.
 230 #endif
 231
 232 static void
 233 crypto_callback(CpaCySymDpOpData *pOpData,
 234                 __rte_unused CpaStatus status,
 235                 __rte_unused CpaBoolean verifyResult)
 236 {
 237         uint32_t lcore_id;
 238         lcore_id = rte_lcore_id();
 239         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 240
 241         /*
 242          * Received a completion from the QA hardware.
 243          * Place the response on the return queue.
 244          */
 245         callbackQ->qaCallbackRing[callbackQ->head] = pOpData->pCallbackTag;
 246         callbackQ->head++;
 247         callbackQ->numEntries++;
 248         qaCoreConf[lcore_id].qaOutstandingRequests--;
 249 }
 250
 251 static void
 252 qa_crypto_callback(CpaCySymDpOpData *pOpData, CpaStatus status,
 253                 CpaBoolean verifyResult)
 254 {
 255         crypto_callback(pOpData, status, verifyResult);
 256 }
 257
 258 /*
 259  * Each allocation from a particular memzone lasts for the life-time of
 260  * the application. No freeing of previous allocations will occur.
 261  */
 262 static void *
 263 alloc_memzone_region(uint32_t length, uint32_t lcore_id)
 264 {
 265         char *current_free_addr_ptr = NULL;
 266         struct lcore_memzone *lcore_memzone = &(qaCoreConf[lcore_id].lcoreMemzone);
 267
 268         current_free_addr_ptr  = lcore_memzone->next_free_address;
 269
 270         if (current_free_addr_ptr + length >=
 271                 (char *)lcore_memzone->memzone->addr + lcore_memzone->memzone->len) {
 272                 printf("Crypto: No memory available in memzone\n");
 273                 return NULL;
 274         }
 275         lcore_memzone->next_free_address = current_free_addr_ptr + length;
 276
 277         return (void *)current_free_addr_ptr;
 278 }
 279
 280 /*
 281  * Virtual to Physical Address translation is only executed during initialization
 282  * and not on the data-path.
 283  */
 284 static CpaPhysicalAddr
 285 qa_v2p(void *ptr)
 286 {
 287         const struct rte_memzone *memzone = NULL;
 288         uint32_t lcore_id = 0;
 289         RTE_LCORE_FOREACH(lcore_id) {
 290                 memzone = qaCoreConf[lcore_id].lcoreMemzone.memzone;
 291
 292                 if ((char*) ptr >= (char *) memzone->addr &&
 293                                 (char*) ptr < ((char*) memzone->addr + memzone->len)) {
 294                         return (CpaPhysicalAddr)
 295                                         (memzone->phys_addr + ((char *) ptr - (char*) memzone->addr));
 296                 }
 297         }
 298         printf("Crypto: Corresponding physical address not found in memzone\n");
 299         return (CpaPhysicalAddr) 0;
 300 }
 301
 302 static CpaStatus
 303 getCoreAffinity(Cpa32U *coreAffinity, const CpaInstanceHandle instanceHandle)
 304 {
 305         CpaInstanceInfo2 info;
 306         Cpa16U i = 0;
 307         CpaStatus status = CPA_STATUS_SUCCESS;
 308
 309         bzero(&info, sizeof(CpaInstanceInfo2));
 310
 311         status = cpaCyInstanceGetInfo2(instanceHandle, &info);
 312         if (CPA_STATUS_SUCCESS != status) {
 313                 printf("Crypto: Error getting instance info\n");
 314                 return CPA_STATUS_FAIL;
 315         }
 316         for (i = 0; i < MAX_CORES; i++) {
 317                 if (CPA_BITMAP_BIT_TEST(info.coreAffinity, i)) {
 318                         *coreAffinity = i;
 319                         return CPA_STATUS_SUCCESS;
 320                 }
 321         }
 322         return CPA_STATUS_FAIL;
 323 }
 324
 325 static CpaStatus
 326 get_crypto_instance_on_core(CpaInstanceHandle *pInstanceHandle,
 327                 uint32_t lcore_id)
 328 {
 329         Cpa16U numInstances = 0, i = 0;
 330         CpaStatus status = CPA_STATUS_FAIL;
 331         CpaInstanceHandle *pLocalInstanceHandles = NULL;
 332         Cpa32U coreAffinity = 0;
 333
 334         status = cpaCyGetNumInstances(&numInstances);
 335         if (CPA_STATUS_SUCCESS != status || numInstances == 0) {
 336                 return CPA_STATUS_FAIL;
 337         }
 338
 339         pLocalInstanceHandles = rte_malloc("pLocalInstanceHandles",
 340                         sizeof(CpaInstanceHandle) * numInstances, CACHE_LINE_SIZE);
 341
 342         if (NULL == pLocalInstanceHandles) {
 343                 return CPA_STATUS_FAIL;
 344         }
 345         status = cpaCyGetInstances(numInstances, pLocalInstanceHandles);
 346         if (CPA_STATUS_SUCCESS != status) {
 347                 printf("Crypto: cpaCyGetInstances failed with status: %"PRId32"\n", status);
 348                 rte_free((void *) pLocalInstanceHandles);
 349                 return CPA_STATUS_FAIL;
 350         }
 351
 352         for (i = 0; i < numInstances; i++) {
 353                 status = getCoreAffinity(&coreAffinity, pLocalInstanceHandles[i]);
 354                 if (CPA_STATUS_SUCCESS != status) {
 355                         rte_free((void *) pLocalInstanceHandles);
 356                         return CPA_STATUS_FAIL;
 357                 }
 358                 if (coreAffinity == lcore_id) {
 359                         printf("Crypto: instance found on core %d\n", i);
 360                         *pInstanceHandle = pLocalInstanceHandles[i];
 361                         return CPA_STATUS_SUCCESS;
 362                 }
 363         }
 364         /* core affinity not found */
 365         rte_free((void *) pLocalInstanceHandles);
 366         return CPA_STATUS_FAIL;
 367 }
 368
 369 static CpaStatus
 370 initCySymSession(const int pkt_cipher_alg,
 371                 const int pkt_hash_alg, const CpaCySymHashMode hashMode,
 372                 const CpaCySymCipherDirection crypto_direction,
 373                 CpaCySymSessionCtx **ppSessionCtx,
 374                 const CpaInstanceHandle cyInstanceHandle,
 375                 const uint32_t lcore_id)
 376 {
 377         Cpa32U sessionCtxSizeInBytes = 0;
 378         CpaStatus status = CPA_STATUS_FAIL;
 379         CpaBoolean isCrypto = CPA_TRUE, isHmac = CPA_TRUE;
 380         CpaCySymSessionSetupData sessionSetupData;
 381
 382         bzero(&sessionSetupData, sizeof(CpaCySymSessionSetupData));
 383
 384         /* Assumption: key length is set to each algorithm's max length */
 385         switch (pkt_cipher_alg) {
 386         case NO_CIPHER:
 387                 isCrypto = CPA_FALSE;
 388                 break;
 389         case CIPHER_DES:
 390                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 391                                 CPA_CY_SYM_CIPHER_DES_ECB;
 392                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 393                                 KEY_SIZE_64_IN_BYTES;
 394                 break;
 395         case CIPHER_DES_CBC:
 396                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 397                                 CPA_CY_SYM_CIPHER_DES_CBC;
 398                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 399                                 KEY_SIZE_64_IN_BYTES;
 400                 break;
 401         case CIPHER_DES3:
 402                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 403                                 CPA_CY_SYM_CIPHER_3DES_ECB;
 404                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 405                                 KEY_SIZE_192_IN_BYTES;
 406                 break;
 407         case CIPHER_DES3_CBC:
 408                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 409                                 CPA_CY_SYM_CIPHER_3DES_CBC;
 410                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 411                                 KEY_SIZE_192_IN_BYTES;
 412                 break;
 413         case CIPHER_AES:
 414                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 415                                 CPA_CY_SYM_CIPHER_AES_ECB;
 416                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 417                                 KEY_SIZE_128_IN_BYTES;
 418                 break;
 419         case CIPHER_AES_CBC_128:
 420                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 421                                 CPA_CY_SYM_CIPHER_AES_CBC;
 422                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 423                                 KEY_SIZE_128_IN_BYTES;
 424                 break;
 425         default:
 426                 printf("Crypto: Undefined Cipher specified\n");
 427                 break;
 428         }
 429         /* Set the cipher direction */
 430         if (isCrypto) {
 431                 sessionSetupData.cipherSetupData.cipherDirection = crypto_direction;
 432                 sessionSetupData.cipherSetupData.pCipherKey =
 433                                 g_crypto_hash_keys.cipher_key;
 434                 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
 435         }
 436
 437         /* Setup Hash common fields */
 438         switch (pkt_hash_alg) {
 439         case NO_HASH:
 440                 isHmac = CPA_FALSE;
 441                 break;
 442         case HASH_AES_XCBC:
 443                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 444                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 445                                 AES_XCBC_DIGEST_LENGTH_IN_BYTES;
 446                 break;
 447         case HASH_AES_XCBC_96:
 448                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 449                                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 450                                 AES_XCBC_96_DIGEST_LENGTH_IN_BYTES;
 451                 break;
 452         case HASH_MD5:
 453                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_MD5;
 454                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 455                                 MD5_DIGEST_LENGTH_IN_BYTES;
 456                 break;
 457         case HASH_SHA1:
 458                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 459                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 460                                 SHA1_DIGEST_LENGTH_IN_BYTES;
 461                 break;
 462         case HASH_SHA1_96:
 463                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 464                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 465                                 SHA1_96_DIGEST_LENGTH_IN_BYTES;
 466             break;
 467         case HASH_SHA224:
 468                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA224;
 469                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 470                                 SHA224_DIGEST_LENGTH_IN_BYTES;
 471                 break;
 472         case HASH_SHA256:
 473                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA256;
 474                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 475                                 SHA256_DIGEST_LENGTH_IN_BYTES;
 476                 break;
 477         case HASH_SHA384:
 478                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA384;
 479                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 480                                 SHA384_DIGEST_LENGTH_IN_BYTES;
 481                 break;
 482         case HASH_SHA512:
 483                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA512;
 484                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 485                                 SHA512_DIGEST_LENGTH_IN_BYTES;
 486                 break;
 487         default:
 488                 printf("Crypto: Undefined Hash specified\n");
 489                 break;
 490         }
 491         if (isHmac) {
 492                 sessionSetupData.hashSetupData.hashMode = hashMode;
 493                 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
 494                 /* If using authenticated hash setup key lengths */
 495                 if (CPA_CY_SYM_HASH_MODE_AUTH == hashMode) {
 496                         /* Use a common max length key */
 497                         sessionSetupData.hashSetupData.authModeSetupData.authKey =
 498                                         g_crypto_hash_keys.hash_key;
 499                         switch (pkt_hash_alg) {
 500                         case HASH_AES_XCBC:
 501                         case HASH_AES_XCBC_96:
 502                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 503                                                 AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES;
 504                                 break;
 505                         case HASH_MD5:
 506                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 507                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 508                                 break;
 509                         case HASH_SHA1:
 510                         case HASH_SHA1_96:
 511                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 512                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 513                                 break;
 514                         case HASH_SHA224:
 515                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 516                                                 SHA224_AUTH_KEY_LENGTH_IN_BYTES;
 517                                 break;
 518                         case HASH_SHA256:
 519                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 520                                                 SHA256_AUTH_KEY_LENGTH_IN_BYTES;
 521                                 break;
 522                         case HASH_SHA384:
 523                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 524                                                 SHA384_AUTH_KEY_LENGTH_IN_BYTES;
 525                                 break;
 526                         case HASH_SHA512:
 527                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 528                                                 SHA512_AUTH_KEY_LENGTH_IN_BYTES;
 529                                 break;
 530                         default:
 531                                 printf("Crypto: Undefined Hash specified\n");
 532                                 return CPA_STATUS_FAIL;
 533                         }
 534                 }
 535         }
 536
 537         /* Only high priority supported */
 538         sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
 539
 540         /* If chaining algorithms */
 541         if (isCrypto && isHmac) {
 542                 sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 543                 /* @assumption Alg Chain order is cipher then hash for encrypt
 544                  * and hash then cipher then has for decrypt*/
 545                 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == crypto_direction) {
 546                         sessionSetupData.algChainOrder =
 547                                         CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 548                 } else {
 549                         sessionSetupData.algChainOrder =
 550                                         CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 551                 }
 552         }
 553         if (!isCrypto && !isHmac) {
 554                 *ppSessionCtx = NULL;
 555                 return CPA_STATUS_SUCCESS;
 556         }
 557
 558         /* Set flags for digest operations */
 559         sessionSetupData.digestIsAppended = CPA_FALSE;
 560         sessionSetupData.verifyDigest = CPA_TRUE;
 561
 562         /* Get the session context size based on the crypto and/or hash operations*/
 563         status = cpaCySymDpSessionCtxGetSize(cyInstanceHandle, &sessionSetupData,
 564                         &sessionCtxSizeInBytes);
 565         if (CPA_STATUS_SUCCESS != status) {
 566                 printf("Crypto: cpaCySymDpSessionCtxGetSize error, status: %"PRId32"\n",
 567                                 status);
 568                 return CPA_STATUS_FAIL;
 569         }
 570
 571         *ppSessionCtx = alloc_memzone_region(sessionCtxSizeInBytes, lcore_id);
 572         if (NULL == *ppSessionCtx) {
 573                 printf("Crypto: Failed to allocate memory for Session Context\n");
 574                 return CPA_STATUS_FAIL;
 575         }
 576
 577         status = cpaCySymDpInitSession(cyInstanceHandle, &sessionSetupData,
 578                         *ppSessionCtx);
 579         if (CPA_STATUS_SUCCESS != status) {
 580                 printf("Crypto: cpaCySymDpInitSession failed with status %"PRId32"\n", status);
 581                 return CPA_STATUS_FAIL;
 582         }
 583         return CPA_STATUS_SUCCESS;
 584 }
 585
 586 static CpaStatus
 587 initSessionDataTables(struct qa_core_conf *qaCoreConf,uint32_t lcore_id)
 588 {
 589         Cpa32U i = 0, j = 0;
 590         CpaStatus status = CPA_STATUS_FAIL;
 591         for (i = 0; i < NUM_CRYPTO; i++) {
 592                 for (j = 0; j < NUM_HMAC; j++) {
 593                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 594                                         CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
 595                                         &qaCoreConf->encryptSessionHandleTbl[i][j],
 596                                         qaCoreConf->instanceHandle,
 597                                         lcore_id);
 598                         if (CPA_STATUS_SUCCESS != status) {
 599                                 printf("Crypto: Failed to initialize Encrypt sessions\n");
 600                                 return CPA_STATUS_FAIL;
 601                         }
 602                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 603                                         CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
 604                                         &qaCoreConf->decryptSessionHandleTbl[i][j],
 605                                         qaCoreConf->instanceHandle,
 606                                         lcore_id);
 607                         if (CPA_STATUS_SUCCESS != status) {
 608                                 printf("Crypto: Failed to initialize Decrypt sessions\n");
 609                                 return CPA_STATUS_FAIL;
 610                         }
 611                 }
 612         }
 613         return CPA_STATUS_SUCCESS;
 614 }
 615
 616 int
 617 crypto_init(void)
 618 {
 619         if (CPA_STATUS_SUCCESS != icp_sal_userStartMultiProcess("SSL",CPA_FALSE)) {
 620                 printf("Crypto: Could not start sal for user space\n");
 621                 return CPA_STATUS_FAIL;
 622         }
 623         printf("Crypto: icp_sal_userStartMultiProcess(\"SSL\",CPA_FALSE)\n");
 624         return 0;
 625 }
 626
 627 /*
 628  * Per core initialisation
 629  */
 630 int
 631 per_core_crypto_init(uint32_t lcore_id)
 632 {
 633         CpaStatus status = CPA_STATUS_FAIL;
 634         char memzone_name[RTE_MEMZONE_NAMESIZE];
 635
 636         int socketID = rte_lcore_to_socket_id(lcore_id);
 637
 638         /* Allocate software ring for response messages. */
 639
 640         qaCoreConf[lcore_id].callbackQueue.head = 0;
 641         qaCoreConf[lcore_id].callbackQueue.tail = 0;
 642         qaCoreConf[lcore_id].callbackQueue.numEntries = 0;
 643         qaCoreConf[lcore_id].kickFreq = 0;
 644         qaCoreConf[lcore_id].qaOutstandingRequests = 0;
 645         qaCoreConf[lcore_id].numResponseAttempts = 0;
 646
 647         /* Initialise and reserve lcore memzone for virt2phys translation */
 648         rte_snprintf(memzone_name,
 649                         RTE_MEMZONE_NAMESIZE,
 650                         "lcore_%u",
 651                         lcore_id);
 652
 653         qaCoreConf[lcore_id].lcoreMemzone.memzone = rte_memzone_reserve(
 654                         memzone_name,
 655                         LCORE_MEMZONE_SIZE,
 656                         socketID,
 657                         0);
 658         if (NULL == qaCoreConf[lcore_id].lcoreMemzone.memzone) {
 659                 printf("Crypto: Error allocating memzone on lcore %u\n",lcore_id);
 660                 return -1;
 661         }
 662         qaCoreConf[lcore_id].lcoreMemzone.next_free_address =
 663                                                         qaCoreConf[lcore_id].lcoreMemzone.memzone->addr;
 664
 665         qaCoreConf[lcore_id].pPacketIV = alloc_memzone_region(IV_LENGTH_16_BYTES,
 666                                                         lcore_id);
 667
 668         if (NULL == qaCoreConf[lcore_id].pPacketIV ) {
 669                 printf("Crypto: Failed to allocate memory for Initialization Vector\n");
 670                 return -1;
 671         }
 672
 673         memcpy(qaCoreConf[lcore_id].pPacketIV, &g_crypto_hash_keys.iv,
 674                         IV_LENGTH_16_BYTES);
 675
 676         qaCoreConf[lcore_id].packetIVPhy = qa_v2p(qaCoreConf[lcore_id].pPacketIV);
 677         if (0 == qaCoreConf[lcore_id].packetIVPhy) {
 678                 printf("Crypto: Invalid physical address for Initialization Vector\n");
 679                 return -1;
 680         }
 681
 682         /*
 683          * Obtain the instance handle that is mapped to the current lcore.
 684          * This can fail if an instance is not mapped to a bank which has been
 685          * affinitized to the current lcore.
 686          */
 687         status = get_crypto_instance_on_core(&(qaCoreConf[lcore_id].instanceHandle),
 688                         lcore_id);
 689         if (CPA_STATUS_SUCCESS != status) {
 690                 printf("Crypto: get_crypto_instance_on_core failed with status: %"PRId32"\n",
 691                                 status);
 692                 return -1;
 693         }
 694
 695         status = cpaCySymDpRegCbFunc(qaCoreConf[lcore_id].instanceHandle,
 696                         (CpaCySymDpCbFunc) qa_crypto_callback);
 697         if (CPA_STATUS_SUCCESS != status) {
 698                 printf("Crypto: cpaCySymDpRegCbFunc failed with status: %"PRId32"\n", status);
 699                 return -1;
 700         }
 701
 702         /*
 703          * Set the address translation callback for virtual to physcial address
 704          * mapping. This will be called by the QAT driver during initialisation only.
 705          */
 706         status = cpaCySetAddressTranslation(qaCoreConf[lcore_id].instanceHandle,
 707                         (CpaVirtualToPhysical) qa_v2p);
 708         if (CPA_STATUS_SUCCESS != status) {
 709                 printf("Crypto: cpaCySetAddressTranslation failed with status: %"PRId32"\n",
 710                                 status);
 711                 return -1;
 712         }
 713
 714         status = initSessionDataTables(&qaCoreConf[lcore_id],lcore_id);
 715         if (CPA_STATUS_SUCCESS != status) {
 716                 printf("Crypto: Failed to allocate all session tables.");
 717                 return -1;
 718         }
 719         return 0;
 720 }
 721
 722 static CpaStatus
 723 enqueueOp(CpaCySymDpOpData *opData, uint32_t lcore_id)
 724 {
 725
 726         CpaStatus status;
 727
 728         /*
 729          * Assumption is there is no requirement to do load balancing between
 730          * acceleration units - that is one acceleration unit is tied to a core.
 731          */
 732         opData->instanceHandle = qaCoreConf[lcore_id].instanceHandle;
 733
 734         if ((++qaCoreConf[lcore_id].kickFreq) % CRYPTO_BURST_TX == 0) {
 735                 status = cpaCySymDpEnqueueOp(opData, CPA_TRUE);
 736         } else {
 737                 status = cpaCySymDpEnqueueOp(opData, CPA_FALSE);
 738         }
 739
 740         qaCoreConf[lcore_id].qaOutstandingRequests++;
 741
 742         return status;
 743 }
 744
 745 void
 746 crypto_flush_tx_queue(uint32_t lcore_id)
 747 {
 748
 749         cpaCySymDpPerformOpNow(qaCoreConf[lcore_id].instanceHandle);
 750 }
 751
 752 enum crypto_result
 753 crypto_encrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 754 {
 755         CpaCySymDpOpData *opData =
 756                         (CpaCySymDpOpData *) ((char *) (rte_buff->pkt.data)
 757                                         + CRYPTO_OFFSET_TO_OPDATA);
 758         uint32_t lcore_id;
 759
 760         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 761                 return CRYPTO_RESULT_FAIL;
 762
 763         lcore_id = rte_lcore_id();
 764
 765         bzero(opData, sizeof(CpaCySymDpOpData));
 766
 767         opData->srcBuffer = opData->dstBuffer = PACKET_DATA_START_PHYS(rte_buff);
 768         opData->srcBufferLen = opData->dstBufferLen = rte_buff->pkt.data_len;
 769         opData->sessionCtx = qaCoreConf[lcore_id].encryptSessionHandleTbl[c][h];
 770         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 771                         + CRYPTO_OFFSET_TO_OPDATA;
 772         opData->pCallbackTag = rte_buff;
 773
 774         /* if no crypto or hash operations are specified return fail */
 775         if (NO_CIPHER == c && NO_HASH == h)
 776                 return CRYPTO_RESULT_FAIL;
 777
 778         if (NO_CIPHER != c) {
 779                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 780                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 781
 782                 if (CIPHER_AES_CBC_128 == c)
 783                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 784                 else
 785                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 786
 787                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 788                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 789                                 - CRYPTO_START_OFFSET;
 790                 /*
 791                  * Work around for padding, message length has to be a multiple of
 792                  * block size.
 793                  */
 794                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 795                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 796         }
 797
 798         if (NO_HASH != h) {
 799
 800                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 801                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 802                                 - HASH_START_OFFSET;
 803                 /*
 804                  * Work around for padding, message length has to be a multiple of block
 805                  * size.
 806                  */
 807                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 808                                 % HASH_BLOCK_DEFAULT_SIZE;
 809
 810                 /*
 811                  * Assumption: Ok ignore the passed digest pointer and place HMAC at end
 812                  * of packet.
 813                  */
 814                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 815         }
 816
 817         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 818                 /*
 819                  * Failed to place a packet on the hardware queue.
 820                  * Most likely because the QA hardware is busy.
 821                  */
 822                 return CRYPTO_RESULT_FAIL;
 823         }
 824         return CRYPTO_RESULT_IN_PROGRESS;
 825 }
 826
 827 enum crypto_result
 828 crypto_decrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 829 {
 830
 831         CpaCySymDpOpData *opData = (void*) (((char *) rte_buff->pkt.data)
 832                         + CRYPTO_OFFSET_TO_OPDATA);
 833         uint32_t lcore_id;
 834
 835         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 836                 return CRYPTO_RESULT_FAIL;
 837
 838         lcore_id = rte_lcore_id();
 839
 840         bzero(opData, sizeof(CpaCySymDpOpData));
 841
 842         opData->dstBuffer = opData->srcBuffer = PACKET_DATA_START_PHYS(rte_buff);
 843         opData->dstBufferLen = opData->srcBufferLen = rte_buff->pkt.data_len;
 844         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 845                         + CRYPTO_OFFSET_TO_OPDATA;
 846         opData->sessionCtx = qaCoreConf[lcore_id].decryptSessionHandleTbl[c][h];
 847         opData->pCallbackTag = rte_buff;
 848
 849         /* if no crypto or hmac operations are specified return fail */
 850         if (NO_CIPHER == c && NO_HASH == h)
 851                 return CRYPTO_RESULT_FAIL;
 852
 853         if (NO_CIPHER != c) {
 854                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 855                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 856
 857                 if (CIPHER_AES_CBC_128 == c)
 858                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 859                 else
 860                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 861
 862                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 863                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 864                                 - CRYPTO_START_OFFSET;
 865
 866                 /*
 867                  * Work around for padding, message length has to be a multiple of block
 868                  * size.
 869                  */
 870                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 871                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 872         }
 873         if (NO_HASH != h) {
 874                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 875                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 876                                 - HASH_START_OFFSET;
 877                 /*
 878                  * Work around for padding, message length has to be a multiple of block
 879                  * size.
 880                  */
 881                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 882                                 % HASH_BLOCK_DEFAULT_SIZE;
 883                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 884         }
 885
 886         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 887                 /*
 888                  * Failed to place a packet on the hardware queue.
 889                  * Most likely because the QA hardware is busy.
 890                  */
 891                 return CRYPTO_RESULT_FAIL;
 892         }
 893         return CRYPTO_RESULT_IN_PROGRESS;
 894 }
 895
 896 void *
 897 crypto_get_next_response(void)
 898 {
 899         uint32_t lcore_id;
 900         lcore_id = rte_lcore_id();
 901         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 902         void *entry = NULL;
 903
 904         if (callbackQ->numEntries) {
 905                 entry = callbackQ->qaCallbackRing[callbackQ->tail];
 906                 callbackQ->tail++;
 907                 callbackQ->numEntries--;
 908         }
 909
 910         /* If there are no outstanding requests no need to poll, return entry */
 911         if (qaCoreConf[lcore_id].qaOutstandingRequests == 0)
 912                 return entry;
 913
 914         if (callbackQ->numEntries < CRYPTO_QUEUED_RESP_POLL_THRESHOLD
 915                         && qaCoreConf[lcore_id].numResponseAttempts++
 916                                         % GET_NEXT_RESPONSE_FREQ == 0) {
 917                 /*
 918                  * Only poll the hardware when there is less than
 919                  * CRYPTO_QUEUED_RESP_POLL_THRESHOLD elements in the software queue
 920                  */
 921                 icp_sal_CyPollDpInstance(qaCoreConf[lcore_id].instanceHandle,
 922                                 CRYPTO_MAX_RESPONSE_QUOTA);
 923         }
 924         return entry;
 925 }