examples/dpdk_qat/crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_ether.h>
  49 #include <rte_malloc.h>
  50 #include <rte_launch.h>
  51 #include <rte_eal.h>
  52 #include <rte_per_lcore.h>
  53 #include <rte_lcore.h>
  54 #include <rte_atomic.h>
  55 #include <rte_branch_prediction.h>
  56 #include <rte_ring.h>
  57 #include <rte_mempool.h>
  58 #include <rte_mbuf.h>
  59 #include <rte_string_fns.h>
  60
  61 #define CPA_CY_SYM_DP_TMP_WORKAROUND 1
  62
  63 #include "cpa.h"
  64 #include "cpa_types.h"
  65 #include "cpa_cy_sym_dp.h"
  66 #include "cpa_cy_common.h"
  67 #include "cpa_cy_im.h"
  68 #include "icp_sal_user.h"
  69 #include "icp_sal_poll.h"
  70
  71 #include "crypto.h"
  72
  73 /* CIPHER KEY LENGTHS */
  74 #define KEY_SIZE_64_IN_BYTES    (64 / 8)
  75 #define KEY_SIZE_56_IN_BYTES    (56 / 8)
  76 #define KEY_SIZE_128_IN_BYTES   (128 / 8)
  77 #define KEY_SIZE_168_IN_BYTES   (168 / 8)
  78 #define KEY_SIZE_192_IN_BYTES   (192 / 8)
  79 #define KEY_SIZE_256_IN_BYTES   (256 / 8)
  80
  81 /* HMAC AUTH KEY LENGTHS */
  82 #define AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES       (128 / 8)
  83 #define SHA1_AUTH_KEY_LENGTH_IN_BYTES           (160 / 8)
  84 #define SHA224_AUTH_KEY_LENGTH_IN_BYTES         (224 / 8)
  85 #define SHA256_AUTH_KEY_LENGTH_IN_BYTES         (256 / 8)
  86 #define SHA384_AUTH_KEY_LENGTH_IN_BYTES         (384 / 8)
  87 #define SHA512_AUTH_KEY_LENGTH_IN_BYTES         (512 / 8)
  88 #define MD5_AUTH_KEY_LENGTH_IN_BYTES            (128 / 8)
  89 #define KASUMI_AUTH_KEY_LENGTH_IN_BYTES         (128 / 8)
  90
  91 /* HASH DIGEST LENGHTS */
  92 #define AES_XCBC_DIGEST_LENGTH_IN_BYTES         (128 / 8)
  93 #define AES_XCBC_96_DIGEST_LENGTH_IN_BYTES      (96 / 8)
  94 #define MD5_DIGEST_LENGTH_IN_BYTES              (128 / 8)
  95 #define SHA1_DIGEST_LENGTH_IN_BYTES             (160 / 8)
  96 #define SHA1_96_DIGEST_LENGTH_IN_BYTES          (96 / 8)
  97 #define SHA224_DIGEST_LENGTH_IN_BYTES           (224 / 8)
  98 #define SHA256_DIGEST_LENGTH_IN_BYTES           (256 / 8)
  99 #define SHA384_DIGEST_LENGTH_IN_BYTES           (384 / 8)
 100 #define SHA512_DIGEST_LENGTH_IN_BYTES           (512 / 8)
 101 #define KASUMI_DIGEST_LENGTH_IN_BYTES           (32 / 8)
 102
 103 #define IV_LENGTH_16_BYTES      (16)
 104 #define IV_LENGTH_8_BYTES       (8)
 105
 106
 107 /*
 108  * rte_memzone is used to allocate physically contiguous virtual memory.
 109  * In this application we allocate a single block and divide between variables
 110  * which require a virtual to physical mapping for use by the QAT driver.
 111  * Virt2phys is only performed during initialisation and not on the data-path.
 112  */
 113
 114 #define LCORE_MEMZONE_SIZE      (1 << 22)
 115
 116 struct lcore_memzone
 117 {
 118         const struct rte_memzone *memzone;
 119         void *next_free_address;
 120 };
 121
 122 /*
 123  * Size the qa software response queue.
 124  * Note: Head and Tail are 8 bit, therefore, the queue is
 125  * fixed to 256 entries.
 126  */
 127 #define CRYPTO_SOFTWARE_QUEUE_SIZE 256
 128
 129 struct qa_callbackQueue {
 130         uint8_t head;
 131         uint8_t tail;
 132         uint16_t numEntries;
 133         struct rte_mbuf *qaCallbackRing[CRYPTO_SOFTWARE_QUEUE_SIZE];
 134 };
 135
 136 struct qa_core_conf {
 137         CpaCySymDpSessionCtx *encryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 138         CpaCySymDpSessionCtx *decryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 139         CpaInstanceHandle instanceHandle;
 140         struct qa_callbackQueue callbackQueue;
 141         uint64_t qaOutstandingRequests;
 142         uint64_t numResponseAttempts;
 143         uint8_t kickFreq;
 144         void *pPacketIV;
 145         CpaPhysicalAddr packetIVPhy;
 146         struct lcore_memzone lcoreMemzone;
 147 } __rte_cache_aligned;
 148
 149 #define MAX_CORES   (RTE_MAX_LCORE)
 150
 151 static struct qa_core_conf qaCoreConf[MAX_CORES];
 152
 153 /*
 154  *Create maximum possible key size,
 155  *One for cipher and one for hash
 156  */
 157 struct glob_keys {
 158         uint8_t cipher_key[32];
 159         uint8_t hash_key[64];
 160         uint8_t iv[16];
 161 };
 162
 163 struct glob_keys g_crypto_hash_keys = {
 164         .cipher_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 165                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 166                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 167                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20},
 168         .hash_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 169                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 170                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 171                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,
 172                 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,
 173                 0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,
 174                 0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,
 175                 0x39,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50},
 176         .iv = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 177                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10}
 178 };
 179
 180 /*
 181  * Offsets from the start of the packet.
 182  *
 183  */
 184 #define PACKET_DATA_START_PHYS(p) \
 185                 ((p)->buf_physaddr + (p)->data_off)
 186
 187 /*
 188  * A fixed offset to where the crypto is to be performed, which is the first
 189  * byte after the Ethernet(14 bytes) and IPv4 headers(20 bytes)
 190  */
 191 #define CRYPTO_START_OFFSET             (14+20)
 192 #define HASH_START_OFFSET               (14+20)
 193 #define CIPHER_BLOCK_DEFAULT_SIZE       (16)
 194 #define HASH_BLOCK_DEFAULT_SIZE         (16)
 195
 196 /*
 197  * Offset to the opdata from the start of the data portion of packet.
 198  * Assumption: The buffer is physically contiguous.
 199  * +18 takes this to the next cache line.
 200  */
 201
 202 #define CRYPTO_OFFSET_TO_OPDATA         (ETHER_MAX_LEN+18)
 203
 204 /*
 205  * Default number of requests to place on the hardware ring before kicking the
 206  * ring pointers.
 207  */
 208 #define CRYPTO_BURST_TX (16)
 209
 210 /*
 211  * Only call the qa poll function when the number responses in the software
 212  * queue drops below this number.
 213  */
 214 #define CRYPTO_QUEUED_RESP_POLL_THRESHOLD       (32)
 215
 216 /*
 217  * Limit the number of polls per call to get_next_response.
 218  */
 219 #define GET_NEXT_RESPONSE_FREQ  (32)
 220
 221 /*
 222  * Max number of responses to pull from the qa in one poll.
 223  */
 224 #define CRYPTO_MAX_RESPONSE_QUOTA \
 225                 (CRYPTO_SOFTWARE_QUEUE_SIZE-CRYPTO_QUEUED_RESP_POLL_THRESHOLD-1)
 226
 227 #if (CRYPTO_QUEUED_RESP_POLL_THRESHOLD + CRYPTO_MAX_RESPONSE_QUOTA >= \
 228                 CRYPTO_SOFTWARE_QUEUE_SIZE)
 229 #error Its possible to overflow the qa response Q with current poll and \
 230                 response quota.
 231 #endif
 232
 233 static void
 234 crypto_callback(CpaCySymDpOpData *pOpData,
 235                 __rte_unused CpaStatus status,
 236                 __rte_unused CpaBoolean verifyResult)
 237 {
 238         uint32_t lcore_id;
 239         lcore_id = rte_lcore_id();
 240         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 241
 242         /*
 243          * Received a completion from the QA hardware.
 244          * Place the response on the return queue.
 245          */
 246         callbackQ->qaCallbackRing[callbackQ->head] = pOpData->pCallbackTag;
 247         callbackQ->head++;
 248         callbackQ->numEntries++;
 249         qaCoreConf[lcore_id].qaOutstandingRequests--;
 250 }
 251
 252 static void
 253 qa_crypto_callback(CpaCySymDpOpData *pOpData, CpaStatus status,
 254                 CpaBoolean verifyResult)
 255 {
 256         crypto_callback(pOpData, status, verifyResult);
 257 }
 258
 259 /*
 260  * Each allocation from a particular memzone lasts for the life-time of
 261  * the application. No freeing of previous allocations will occur.
 262  */
 263 static void *
 264 alloc_memzone_region(uint32_t length, uint32_t lcore_id)
 265 {
 266         char *current_free_addr_ptr = NULL;
 267         struct lcore_memzone *lcore_memzone = &(qaCoreConf[lcore_id].lcoreMemzone);
 268
 269         current_free_addr_ptr  = lcore_memzone->next_free_address;
 270
 271         if (current_free_addr_ptr + length >=
 272                 (char *)lcore_memzone->memzone->addr + lcore_memzone->memzone->len) {
 273                 printf("Crypto: No memory available in memzone\n");
 274                 return NULL;
 275         }
 276         lcore_memzone->next_free_address = current_free_addr_ptr + length;
 277
 278         return (void *)current_free_addr_ptr;
 279 }
 280
 281 /*
 282  * Virtual to Physical Address translation is only executed during initialization
 283  * and not on the data-path.
 284  */
 285 static CpaPhysicalAddr
 286 qa_v2p(void *ptr)
 287 {
 288         const struct rte_memzone *memzone = NULL;
 289         uint32_t lcore_id = 0;
 290         RTE_LCORE_FOREACH(lcore_id) {
 291                 memzone = qaCoreConf[lcore_id].lcoreMemzone.memzone;
 292
 293                 if ((char*) ptr >= (char *) memzone->addr &&
 294                                 (char*) ptr < ((char*) memzone->addr + memzone->len)) {
 295                         return (CpaPhysicalAddr)
 296                                         (memzone->phys_addr + ((char *) ptr - (char*) memzone->addr));
 297                 }
 298         }
 299         printf("Crypto: Corresponding physical address not found in memzone\n");
 300         return (CpaPhysicalAddr) 0;
 301 }
 302
 303 static CpaStatus
 304 getCoreAffinity(Cpa32U *coreAffinity, const CpaInstanceHandle instanceHandle)
 305 {
 306         CpaInstanceInfo2 info;
 307         Cpa16U i = 0;
 308         CpaStatus status = CPA_STATUS_SUCCESS;
 309
 310         bzero(&info, sizeof(CpaInstanceInfo2));
 311
 312         status = cpaCyInstanceGetInfo2(instanceHandle, &info);
 313         if (CPA_STATUS_SUCCESS != status) {
 314                 printf("Crypto: Error getting instance info\n");
 315                 return CPA_STATUS_FAIL;
 316         }
 317         for (i = 0; i < MAX_CORES; i++) {
 318                 if (CPA_BITMAP_BIT_TEST(info.coreAffinity, i)) {
 319                         *coreAffinity = i;
 320                         return CPA_STATUS_SUCCESS;
 321                 }
 322         }
 323         return CPA_STATUS_FAIL;
 324 }
 325
 326 static CpaStatus
 327 get_crypto_instance_on_core(CpaInstanceHandle *pInstanceHandle,
 328                 uint32_t lcore_id)
 329 {
 330         Cpa16U numInstances = 0, i = 0;
 331         CpaStatus status = CPA_STATUS_FAIL;
 332         CpaInstanceHandle *pLocalInstanceHandles = NULL;
 333         Cpa32U coreAffinity = 0;
 334
 335         status = cpaCyGetNumInstances(&numInstances);
 336         if (CPA_STATUS_SUCCESS != status || numInstances == 0) {
 337                 return CPA_STATUS_FAIL;
 338         }
 339
 340         pLocalInstanceHandles = rte_malloc("pLocalInstanceHandles",
 341                         sizeof(CpaInstanceHandle) * numInstances, RTE_CACHE_LINE_SIZE);
 342
 343         if (NULL == pLocalInstanceHandles) {
 344                 return CPA_STATUS_FAIL;
 345         }
 346         status = cpaCyGetInstances(numInstances, pLocalInstanceHandles);
 347         if (CPA_STATUS_SUCCESS != status) {
 348                 printf("Crypto: cpaCyGetInstances failed with status: %"PRId32"\n", status);
 349                 rte_free((void *) pLocalInstanceHandles);
 350                 return CPA_STATUS_FAIL;
 351         }
 352
 353         for (i = 0; i < numInstances; i++) {
 354                 status = getCoreAffinity(&coreAffinity, pLocalInstanceHandles[i]);
 355                 if (CPA_STATUS_SUCCESS != status) {
 356                         rte_free((void *) pLocalInstanceHandles);
 357                         return CPA_STATUS_FAIL;
 358                 }
 359                 if (coreAffinity == lcore_id) {
 360                         printf("Crypto: instance found on core %d\n", i);
 361                         *pInstanceHandle = pLocalInstanceHandles[i];
 362                         return CPA_STATUS_SUCCESS;
 363                 }
 364         }
 365         /* core affinity not found */
 366         rte_free((void *) pLocalInstanceHandles);
 367         return CPA_STATUS_FAIL;
 368 }
 369
 370 static CpaStatus
 371 initCySymSession(const int pkt_cipher_alg,
 372                 const int pkt_hash_alg, const CpaCySymHashMode hashMode,
 373                 const CpaCySymCipherDirection crypto_direction,
 374                 CpaCySymSessionCtx **ppSessionCtx,
 375                 const CpaInstanceHandle cyInstanceHandle,
 376                 const uint32_t lcore_id)
 377 {
 378         Cpa32U sessionCtxSizeInBytes = 0;
 379         CpaStatus status = CPA_STATUS_FAIL;
 380         CpaBoolean isCrypto = CPA_TRUE, isHmac = CPA_TRUE;
 381         CpaCySymSessionSetupData sessionSetupData;
 382
 383         bzero(&sessionSetupData, sizeof(CpaCySymSessionSetupData));
 384
 385         /* Assumption: key length is set to each algorithm's max length */
 386         switch (pkt_cipher_alg) {
 387         case NO_CIPHER:
 388                 isCrypto = CPA_FALSE;
 389                 break;
 390         case CIPHER_DES:
 391                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 392                                 CPA_CY_SYM_CIPHER_DES_ECB;
 393                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 394                                 KEY_SIZE_64_IN_BYTES;
 395                 break;
 396         case CIPHER_DES_CBC:
 397                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 398                                 CPA_CY_SYM_CIPHER_DES_CBC;
 399                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 400                                 KEY_SIZE_64_IN_BYTES;
 401                 break;
 402         case CIPHER_DES3:
 403                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 404                                 CPA_CY_SYM_CIPHER_3DES_ECB;
 405                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 406                                 KEY_SIZE_192_IN_BYTES;
 407                 break;
 408         case CIPHER_DES3_CBC:
 409                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 410                                 CPA_CY_SYM_CIPHER_3DES_CBC;
 411                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 412                                 KEY_SIZE_192_IN_BYTES;
 413                 break;
 414         case CIPHER_AES:
 415                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 416                                 CPA_CY_SYM_CIPHER_AES_ECB;
 417                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 418                                 KEY_SIZE_128_IN_BYTES;
 419                 break;
 420         case CIPHER_AES_CBC_128:
 421                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 422                                 CPA_CY_SYM_CIPHER_AES_CBC;
 423                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 424                                 KEY_SIZE_128_IN_BYTES;
 425                 break;
 426         case CIPHER_KASUMI_F8:
 427                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 428                                 CPA_CY_SYM_CIPHER_KASUMI_F8;
 429                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 430                                 KEY_SIZE_128_IN_BYTES;
 431                 break;
 432         default:
 433                 printf("Crypto: Undefined Cipher specified\n");
 434                 break;
 435         }
 436         /* Set the cipher direction */
 437         if (isCrypto) {
 438                 sessionSetupData.cipherSetupData.cipherDirection = crypto_direction;
 439                 sessionSetupData.cipherSetupData.pCipherKey =
 440                                 g_crypto_hash_keys.cipher_key;
 441                 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
 442         }
 443
 444         /* Setup Hash common fields */
 445         switch (pkt_hash_alg) {
 446         case NO_HASH:
 447                 isHmac = CPA_FALSE;
 448                 break;
 449         case HASH_AES_XCBC:
 450                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 451                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 452                                 AES_XCBC_DIGEST_LENGTH_IN_BYTES;
 453                 break;
 454         case HASH_AES_XCBC_96:
 455                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 456                                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 457                                 AES_XCBC_96_DIGEST_LENGTH_IN_BYTES;
 458                 break;
 459         case HASH_MD5:
 460                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_MD5;
 461                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 462                                 MD5_DIGEST_LENGTH_IN_BYTES;
 463                 break;
 464         case HASH_SHA1:
 465                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 466                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 467                                 SHA1_DIGEST_LENGTH_IN_BYTES;
 468                 break;
 469         case HASH_SHA1_96:
 470                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 471                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 472                                 SHA1_96_DIGEST_LENGTH_IN_BYTES;
 473             break;
 474         case HASH_SHA224:
 475                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA224;
 476                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 477                                 SHA224_DIGEST_LENGTH_IN_BYTES;
 478                 break;
 479         case HASH_SHA256:
 480                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA256;
 481                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 482                                 SHA256_DIGEST_LENGTH_IN_BYTES;
 483                 break;
 484         case HASH_SHA384:
 485                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA384;
 486                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 487                                 SHA384_DIGEST_LENGTH_IN_BYTES;
 488                 break;
 489         case HASH_SHA512:
 490                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA512;
 491                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 492                                 SHA512_DIGEST_LENGTH_IN_BYTES;
 493                 break;
 494         case HASH_KASUMI_F9:
 495                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_KASUMI_F9;
 496                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 497                                 KASUMI_DIGEST_LENGTH_IN_BYTES;
 498                 break;
 499         default:
 500                 printf("Crypto: Undefined Hash specified\n");
 501                 break;
 502         }
 503         if (isHmac) {
 504                 sessionSetupData.hashSetupData.hashMode = hashMode;
 505                 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
 506                 /* If using authenticated hash setup key lengths */
 507                 if (CPA_CY_SYM_HASH_MODE_AUTH == hashMode) {
 508                         /* Use a common max length key */
 509                         sessionSetupData.hashSetupData.authModeSetupData.authKey =
 510                                         g_crypto_hash_keys.hash_key;
 511                         switch (pkt_hash_alg) {
 512                         case HASH_AES_XCBC:
 513                         case HASH_AES_XCBC_96:
 514                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 515                                                 AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES;
 516                                 break;
 517                         case HASH_MD5:
 518                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 519                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 520                                 break;
 521                         case HASH_SHA1:
 522                         case HASH_SHA1_96:
 523                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 524                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 525                                 break;
 526                         case HASH_SHA224:
 527                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 528                                                 SHA224_AUTH_KEY_LENGTH_IN_BYTES;
 529                                 break;
 530                         case HASH_SHA256:
 531                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 532                                                 SHA256_AUTH_KEY_LENGTH_IN_BYTES;
 533                                 break;
 534                         case HASH_SHA384:
 535                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 536                                                 SHA384_AUTH_KEY_LENGTH_IN_BYTES;
 537                                 break;
 538                         case HASH_SHA512:
 539                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 540                                                 SHA512_AUTH_KEY_LENGTH_IN_BYTES;
 541                                 break;
 542                         case HASH_KASUMI_F9:
 543                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 544                                                 KASUMI_AUTH_KEY_LENGTH_IN_BYTES;
 545                                 break;
 546                         default:
 547                                 printf("Crypto: Undefined Hash specified\n");
 548                                 return CPA_STATUS_FAIL;
 549                         }
 550                 }
 551         }
 552
 553         /* Only high priority supported */
 554         sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
 555
 556         /* If chaining algorithms */
 557         if (isCrypto && isHmac) {
 558                 sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 559                 /* @assumption Alg Chain order is cipher then hash for encrypt
 560                  * and hash then cipher then has for decrypt*/
 561                 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == crypto_direction) {
 562                         sessionSetupData.algChainOrder =
 563                                         CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 564                 } else {
 565                         sessionSetupData.algChainOrder =
 566                                         CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 567                 }
 568         }
 569         if (!isCrypto && !isHmac) {
 570                 *ppSessionCtx = NULL;
 571                 return CPA_STATUS_SUCCESS;
 572         }
 573
 574         /* Set flags for digest operations */
 575         sessionSetupData.digestIsAppended = CPA_FALSE;
 576         sessionSetupData.verifyDigest = CPA_TRUE;
 577
 578         /* Get the session context size based on the crypto and/or hash operations*/
 579         status = cpaCySymDpSessionCtxGetSize(cyInstanceHandle, &sessionSetupData,
 580                         &sessionCtxSizeInBytes);
 581         if (CPA_STATUS_SUCCESS != status) {
 582                 printf("Crypto: cpaCySymDpSessionCtxGetSize error, status: %"PRId32"\n",
 583                                 status);
 584                 return CPA_STATUS_FAIL;
 585         }
 586
 587         *ppSessionCtx = alloc_memzone_region(sessionCtxSizeInBytes, lcore_id);
 588         if (NULL == *ppSessionCtx) {
 589                 printf("Crypto: Failed to allocate memory for Session Context\n");
 590                 return CPA_STATUS_FAIL;
 591         }
 592
 593         status = cpaCySymDpInitSession(cyInstanceHandle, &sessionSetupData,
 594                         *ppSessionCtx);
 595         if (CPA_STATUS_SUCCESS != status) {
 596                 printf("Crypto: cpaCySymDpInitSession failed with status %"PRId32"\n", status);
 597                 return CPA_STATUS_FAIL;
 598         }
 599         return CPA_STATUS_SUCCESS;
 600 }
 601
 602 static CpaStatus
 603 initSessionDataTables(struct qa_core_conf *qaCoreConf,uint32_t lcore_id)
 604 {
 605         Cpa32U i = 0, j = 0;
 606         CpaStatus status = CPA_STATUS_FAIL;
 607         for (i = 0; i < NUM_CRYPTO; i++) {
 608                 for (j = 0; j < NUM_HMAC; j++) {
 609                         if (((i == CIPHER_KASUMI_F8) && (j != NO_HASH) && (j != HASH_KASUMI_F9)) ||
 610                                 ((i != NO_CIPHER) && (i != CIPHER_KASUMI_F8) && (j == HASH_KASUMI_F9)))
 611                                 continue;
 612                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 613                                         CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
 614                                         &qaCoreConf->encryptSessionHandleTbl[i][j],
 615                                         qaCoreConf->instanceHandle,
 616                                         lcore_id);
 617                         if (CPA_STATUS_SUCCESS != status) {
 618                                 printf("Crypto: Failed to initialize Encrypt sessions\n");
 619                                 return CPA_STATUS_FAIL;
 620                         }
 621                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 622                                         CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
 623                                         &qaCoreConf->decryptSessionHandleTbl[i][j],
 624                                         qaCoreConf->instanceHandle,
 625                                         lcore_id);
 626                         if (CPA_STATUS_SUCCESS != status) {
 627                                 printf("Crypto: Failed to initialize Decrypt sessions\n");
 628                                 return CPA_STATUS_FAIL;
 629                         }
 630                 }
 631         }
 632         return CPA_STATUS_SUCCESS;
 633 }
 634
 635 int
 636 crypto_init(void)
 637 {
 638         if (CPA_STATUS_SUCCESS != icp_sal_userStartMultiProcess("SSL",CPA_FALSE)) {
 639                 printf("Crypto: Could not start sal for user space\n");
 640                 return CPA_STATUS_FAIL;
 641         }
 642         printf("Crypto: icp_sal_userStartMultiProcess(\"SSL\",CPA_FALSE)\n");
 643         return 0;
 644 }
 645
 646 /*
 647  * Per core initialisation
 648  */
 649 int
 650 per_core_crypto_init(uint32_t lcore_id)
 651 {
 652         CpaStatus status = CPA_STATUS_FAIL;
 653         char memzone_name[RTE_MEMZONE_NAMESIZE];
 654
 655         int socketID = rte_lcore_to_socket_id(lcore_id);
 656
 657         /* Allocate software ring for response messages. */
 658
 659         qaCoreConf[lcore_id].callbackQueue.head = 0;
 660         qaCoreConf[lcore_id].callbackQueue.tail = 0;
 661         qaCoreConf[lcore_id].callbackQueue.numEntries = 0;
 662         qaCoreConf[lcore_id].kickFreq = 0;
 663         qaCoreConf[lcore_id].qaOutstandingRequests = 0;
 664         qaCoreConf[lcore_id].numResponseAttempts = 0;
 665
 666         /* Initialise and reserve lcore memzone for virt2phys translation */
 667         snprintf(memzone_name,
 668                         RTE_MEMZONE_NAMESIZE,
 669                         "lcore_%u",
 670                         lcore_id);
 671
 672         qaCoreConf[lcore_id].lcoreMemzone.memzone = rte_memzone_reserve(
 673                         memzone_name,
 674                         LCORE_MEMZONE_SIZE,
 675                         socketID,
 676                         0);
 677         if (NULL == qaCoreConf[lcore_id].lcoreMemzone.memzone) {
 678                 printf("Crypto: Error allocating memzone on lcore %u\n",lcore_id);
 679                 return -1;
 680         }
 681         qaCoreConf[lcore_id].lcoreMemzone.next_free_address =
 682                                                         qaCoreConf[lcore_id].lcoreMemzone.memzone->addr;
 683
 684         qaCoreConf[lcore_id].pPacketIV = alloc_memzone_region(IV_LENGTH_16_BYTES,
 685                                                         lcore_id);
 686
 687         if (NULL == qaCoreConf[lcore_id].pPacketIV ) {
 688                 printf("Crypto: Failed to allocate memory for Initialization Vector\n");
 689                 return -1;
 690         }
 691
 692         memcpy(qaCoreConf[lcore_id].pPacketIV, &g_crypto_hash_keys.iv,
 693                         IV_LENGTH_16_BYTES);
 694
 695         qaCoreConf[lcore_id].packetIVPhy = qa_v2p(qaCoreConf[lcore_id].pPacketIV);
 696         if (0 == qaCoreConf[lcore_id].packetIVPhy) {
 697                 printf("Crypto: Invalid physical address for Initialization Vector\n");
 698                 return -1;
 699         }
 700
 701         /*
 702          * Obtain the instance handle that is mapped to the current lcore.
 703          * This can fail if an instance is not mapped to a bank which has been
 704          * affinitized to the current lcore.
 705          */
 706         status = get_crypto_instance_on_core(&(qaCoreConf[lcore_id].instanceHandle),
 707                         lcore_id);
 708         if (CPA_STATUS_SUCCESS != status) {
 709                 printf("Crypto: get_crypto_instance_on_core failed with status: %"PRId32"\n",
 710                                 status);
 711                 return -1;
 712         }
 713
 714         status = cpaCySymDpRegCbFunc(qaCoreConf[lcore_id].instanceHandle,
 715                         (CpaCySymDpCbFunc) qa_crypto_callback);
 716         if (CPA_STATUS_SUCCESS != status) {
 717                 printf("Crypto: cpaCySymDpRegCbFunc failed with status: %"PRId32"\n", status);
 718                 return -1;
 719         }
 720
 721         /*
 722          * Set the address translation callback for virtual to physcial address
 723          * mapping. This will be called by the QAT driver during initialisation only.
 724          */
 725         status = cpaCySetAddressTranslation(qaCoreConf[lcore_id].instanceHandle,
 726                         (CpaVirtualToPhysical) qa_v2p);
 727         if (CPA_STATUS_SUCCESS != status) {
 728                 printf("Crypto: cpaCySetAddressTranslation failed with status: %"PRId32"\n",
 729                                 status);
 730                 return -1;
 731         }
 732
 733         status = initSessionDataTables(&qaCoreConf[lcore_id],lcore_id);
 734         if (CPA_STATUS_SUCCESS != status) {
 735                 printf("Crypto: Failed to allocate all session tables.");
 736                 return -1;
 737         }
 738         return 0;
 739 }
 740
 741 static CpaStatus
 742 enqueueOp(CpaCySymDpOpData *opData, uint32_t lcore_id)
 743 {
 744
 745         CpaStatus status;
 746
 747         /*
 748          * Assumption is there is no requirement to do load balancing between
 749          * acceleration units - that is one acceleration unit is tied to a core.
 750          */
 751         opData->instanceHandle = qaCoreConf[lcore_id].instanceHandle;
 752
 753         if ((++qaCoreConf[lcore_id].kickFreq) % CRYPTO_BURST_TX == 0) {
 754                 status = cpaCySymDpEnqueueOp(opData, CPA_TRUE);
 755         } else {
 756                 status = cpaCySymDpEnqueueOp(opData, CPA_FALSE);
 757         }
 758
 759         qaCoreConf[lcore_id].qaOutstandingRequests++;
 760
 761         return status;
 762 }
 763
 764 void
 765 crypto_flush_tx_queue(uint32_t lcore_id)
 766 {
 767
 768         cpaCySymDpPerformOpNow(qaCoreConf[lcore_id].instanceHandle);
 769 }
 770
 771 enum crypto_result
 772 crypto_encrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 773 {
 774         CpaCySymDpOpData *opData =
 775                         rte_pktmbuf_mtod_offset(rte_buff, CpaCySymDpOpData *,
 776                                                 CRYPTO_OFFSET_TO_OPDATA);
 777         uint32_t lcore_id;
 778
 779         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 780                 return CRYPTO_RESULT_FAIL;
 781
 782         lcore_id = rte_lcore_id();
 783
 784         bzero(opData, sizeof(CpaCySymDpOpData));
 785
 786         opData->srcBuffer = opData->dstBuffer = PACKET_DATA_START_PHYS(rte_buff);
 787         opData->srcBufferLen = opData->dstBufferLen = rte_buff->data_len;
 788         opData->sessionCtx = qaCoreConf[lcore_id].encryptSessionHandleTbl[c][h];
 789         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 790                         + CRYPTO_OFFSET_TO_OPDATA;
 791         opData->pCallbackTag = rte_buff;
 792
 793         /* if no crypto or hash operations are specified return fail */
 794         if (NO_CIPHER == c && NO_HASH == h)
 795                 return CRYPTO_RESULT_FAIL;
 796
 797         if (NO_CIPHER != c) {
 798                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 799                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 800
 801                 if (CIPHER_AES_CBC_128 == c)
 802                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 803                 else
 804                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 805
 806                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 807                 opData->messageLenToCipherInBytes = rte_buff->data_len
 808                                 - CRYPTO_START_OFFSET;
 809                 /*
 810                  * Work around for padding, message length has to be a multiple of
 811                  * block size.
 812                  */
 813                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 814                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 815         }
 816
 817         if (NO_HASH != h) {
 818
 819                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 820                 opData->messageLenToHashInBytes = rte_buff->data_len
 821                                 - HASH_START_OFFSET;
 822                 /*
 823                  * Work around for padding, message length has to be a multiple of block
 824                  * size.
 825                  */
 826                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 827                                 % HASH_BLOCK_DEFAULT_SIZE;
 828
 829                 /*
 830                  * Assumption: Ok ignore the passed digest pointer and place HMAC at end
 831                  * of packet.
 832                  */
 833                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->data_len;
 834         }
 835
 836         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 837                 /*
 838                  * Failed to place a packet on the hardware queue.
 839                  * Most likely because the QA hardware is busy.
 840                  */
 841                 return CRYPTO_RESULT_FAIL;
 842         }
 843         return CRYPTO_RESULT_IN_PROGRESS;
 844 }
 845
 846 enum crypto_result
 847 crypto_decrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 848 {
 849
 850         CpaCySymDpOpData *opData = rte_pktmbuf_mtod_offset(rte_buff, void *,
 851                                                            CRYPTO_OFFSET_TO_OPDATA);
 852         uint32_t lcore_id;
 853
 854         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 855                 return CRYPTO_RESULT_FAIL;
 856
 857         lcore_id = rte_lcore_id();
 858
 859         bzero(opData, sizeof(CpaCySymDpOpData));
 860
 861         opData->dstBuffer = opData->srcBuffer = PACKET_DATA_START_PHYS(rte_buff);
 862         opData->dstBufferLen = opData->srcBufferLen = rte_buff->data_len;
 863         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 864                         + CRYPTO_OFFSET_TO_OPDATA;
 865         opData->sessionCtx = qaCoreConf[lcore_id].decryptSessionHandleTbl[c][h];
 866         opData->pCallbackTag = rte_buff;
 867
 868         /* if no crypto or hmac operations are specified return fail */
 869         if (NO_CIPHER == c && NO_HASH == h)
 870                 return CRYPTO_RESULT_FAIL;
 871
 872         if (NO_CIPHER != c) {
 873                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 874                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 875
 876                 if (CIPHER_AES_CBC_128 == c)
 877                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 878                 else
 879                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 880
 881                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 882                 opData->messageLenToCipherInBytes = rte_buff->data_len
 883                                 - CRYPTO_START_OFFSET;
 884
 885                 /*
 886                  * Work around for padding, message length has to be a multiple of block
 887                  * size.
 888                  */
 889                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 890                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 891         }
 892         if (NO_HASH != h) {
 893                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 894                 opData->messageLenToHashInBytes = rte_buff->data_len
 895                                 - HASH_START_OFFSET;
 896                 /*
 897                  * Work around for padding, message length has to be a multiple of block
 898                  * size.
 899                  */
 900                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 901                                 % HASH_BLOCK_DEFAULT_SIZE;
 902                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->data_len;
 903         }
 904
 905         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 906                 /*
 907                  * Failed to place a packet on the hardware queue.
 908                  * Most likely because the QA hardware is busy.
 909                  */
 910                 return CRYPTO_RESULT_FAIL;
 911         }
 912         return CRYPTO_RESULT_IN_PROGRESS;
 913 }
 914
 915 void *
 916 crypto_get_next_response(void)
 917 {
 918         uint32_t lcore_id;
 919         lcore_id = rte_lcore_id();
 920         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 921         void *entry = NULL;
 922
 923         if (callbackQ->numEntries) {
 924                 entry = callbackQ->qaCallbackRing[callbackQ->tail];
 925                 callbackQ->tail++;
 926                 callbackQ->numEntries--;
 927         }
 928
 929         /* If there are no outstanding requests no need to poll, return entry */
 930         if (qaCoreConf[lcore_id].qaOutstandingRequests == 0)
 931                 return entry;
 932
 933         if (callbackQ->numEntries < CRYPTO_QUEUED_RESP_POLL_THRESHOLD
 934                         && qaCoreConf[lcore_id].numResponseAttempts++
 935                                         % GET_NEXT_RESPONSE_FREQ == 0) {
 936                 /*
 937                  * Only poll the hardware when there is less than
 938                  * CRYPTO_QUEUED_RESP_POLL_THRESHOLD elements in the software queue
 939                  */
 940                 icp_sal_CyPollDpInstance(qaCoreConf[lcore_id].instanceHandle,
 941                                 CRYPTO_MAX_RESPONSE_QUOTA);
 942         }
 943         return entry;
 944 }