examples/dpdk_qat/crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  *  version: DPDK.L.1.2.3-3
  34  */
  35
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <strings.h>
  39 #include <string.h>
  40 #include <inttypes.h>
  41 #include <errno.h>
  42 #include <sys/queue.h>
  43 #include <stdarg.h>
  44
  45 #include <rte_common.h>
  46 #include <rte_log.h>
  47 #include <rte_debug.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_tailq.h>
  51 #include <rte_ether.h>
  52 #include <rte_malloc.h>
  53 #include <rte_launch.h>
  54 #include <rte_eal.h>
  55 #include <rte_per_lcore.h>
  56 #include <rte_lcore.h>
  57 #include <rte_atomic.h>
  58 #include <rte_branch_prediction.h>
  59 #include <rte_ring.h>
  60 #include <rte_mempool.h>
  61 #include <rte_mbuf.h>
  62 #include <rte_string_fns.h>
  63
  64 #define CPA_CY_SYM_DP_TMP_WORKAROUND 1
  65
  66 #include "cpa.h"
  67 #include "cpa_types.h"
  68 #include "cpa_cy_sym_dp.h"
  69 #include "cpa_cy_common.h"
  70 #include "cpa_cy_im.h"
  71 #include "icp_sal_user.h"
  72 #include "icp_sal_poll.h"
  73
  74 #include "crypto.h"
  75
  76 #define NUM_HMAC        (10)
  77 #define NUM_CRYPTO      (7)
  78
  79
  80 /* CIPHER KEY LENGTHS */
  81 #define KEY_SIZE_64_IN_BYTES    (64 / 8)
  82 #define KEY_SIZE_56_IN_BYTES    (56 / 8)
  83 #define KEY_SIZE_128_IN_BYTES   (128 / 8)
  84 #define KEY_SIZE_168_IN_BYTES   (168 / 8)
  85 #define KEY_SIZE_192_IN_BYTES   (192 / 8)
  86 #define KEY_SIZE_256_IN_BYTES   (256 / 8)
  87
  88 /* HMAC AUTH KEY LENGTHS */
  89 #define AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES       (128 / 8)
  90 #define SHA1_AUTH_KEY_LENGTH_IN_BYTES           (160 / 8)
  91 #define SHA224_AUTH_KEY_LENGTH_IN_BYTES         (224 / 8)
  92 #define SHA256_AUTH_KEY_LENGTH_IN_BYTES         (256 / 8)
  93 #define SHA384_AUTH_KEY_LENGTH_IN_BYTES         (384 / 8)
  94 #define SHA512_AUTH_KEY_LENGTH_IN_BYTES         (512 / 8)
  95 #define MD5_AUTH_KEY_LENGTH_IN_BYTES            (128 / 8)
  96
  97 /* HASH DIGEST LENGHTS */
  98 #define AES_XCBC_DIGEST_LENGTH_IN_BYTES         (128 / 8)
  99 #define AES_XCBC_96_DIGEST_LENGTH_IN_BYTES      (96 / 8)
 100 #define MD5_DIGEST_LENGTH_IN_BYTES              (128 / 8)
 101 #define SHA1_DIGEST_LENGTH_IN_BYTES             (160 / 8)
 102 #define SHA1_96_DIGEST_LENGTH_IN_BYTES          (96 / 8)
 103 #define SHA224_DIGEST_LENGTH_IN_BYTES           (224 / 8)
 104 #define SHA256_DIGEST_LENGTH_IN_BYTES           (256 / 8)
 105 #define SHA384_DIGEST_LENGTH_IN_BYTES           (384 / 8)
 106 #define SHA512_DIGEST_LENGTH_IN_BYTES           (512 / 8)
 107
 108 #define IV_LENGTH_16_BYTES      (16)
 109 #define IV_LENGTH_8_BYTES       (8)
 110
 111
 112 /*
 113  * rte_memzone is used to allocate physically contiguous virtual memory.
 114  * In this application we allocate a single block and divide between variables
 115  * which require a virtual to physical mapping for use by the QAT driver.
 116  * Virt2phys is only performed during initialisation and not on the data-path.
 117  */
 118
 119 #define LCORE_MEMZONE_SIZE      (1 << 22)
 120
 121 struct lcore_memzone
 122 {
 123         const struct rte_memzone *memzone;
 124         void *next_free_address;
 125 };
 126
 127 /*
 128  * Size the qa software response queue.
 129  * Note: Head and Tail are 8 bit, therefore, the queue is
 130  * fixed to 256 entries.
 131  */
 132 #define CRYPTO_SOFTWARE_QUEUE_SIZE 256
 133
 134 struct qa_callbackQueue {
 135         uint8_t head;
 136         uint8_t tail;
 137         uint16_t numEntries;
 138         struct rte_mbuf *qaCallbackRing[CRYPTO_SOFTWARE_QUEUE_SIZE];
 139 };
 140
 141 struct qa_core_conf {
 142         CpaCySymDpSessionCtx *encryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 143         CpaCySymDpSessionCtx *decryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 144         CpaInstanceHandle instanceHandle;
 145         struct qa_callbackQueue callbackQueue;
 146         uint64_t qaOutstandingRequests;
 147         uint64_t numResponseAttempts;
 148         uint8_t kickFreq;
 149         void *pPacketIV;
 150         CpaPhysicalAddr packetIVPhy;
 151         struct lcore_memzone lcoreMemzone;
 152 } __rte_cache_aligned;
 153
 154 #define MAX_CORES   (RTE_MAX_LCORE)
 155
 156 static struct qa_core_conf qaCoreConf[MAX_CORES];
 157
 158 /*
 159  *Create maximum possible key size,
 160  *One for cipher and one for hash
 161  */
 162 struct glob_keys {
 163         uint8_t cipher_key[32];
 164         uint8_t hash_key[64];
 165         uint8_t iv[16];
 166 };
 167
 168 struct glob_keys g_crypto_hash_keys = {
 169         .cipher_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 170                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 171                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 172                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20},
 173         .hash_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 174                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 175                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 176                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,
 177                 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,
 178                 0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,
 179                 0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,
 180                 0x39,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50},
 181         .iv = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 182                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10}
 183 };
 184
 185 /*
 186  * Offsets from the start of the packet.
 187  *
 188  */
 189 #define PACKET_DATA_START_PHYS(p) \
 190                 ((p)->buf_physaddr + ((char *)p->pkt.data - (char *)p->buf_addr))
 191
 192 /*
 193  * A fixed offset to where the crypto is to be performed, which is the first
 194  * byte after the Ethernet(14 bytes) and IPv4 headers(20 bytes)
 195  */
 196 #define CRYPTO_START_OFFSET             (14+20)
 197 #define HASH_START_OFFSET               (14+20)
 198 #define CIPHER_BLOCK_DEFAULT_SIZE       (16)
 199 #define HASH_BLOCK_DEFAULT_SIZE         (16)
 200
 201 /*
 202  * Offset to the opdata from the start of the data portion of packet.
 203  * Assumption: The buffer is physically contiguous.
 204  * +18 takes this to the next cache line.
 205  */
 206
 207 #define CRYPTO_OFFSET_TO_OPDATA         (ETHER_MAX_LEN+18)
 208
 209 /*
 210  * Default number of requests to place on the hardware ring before kicking the
 211  * ring pointers.
 212  */
 213 #define CRYPTO_BURST_TX (16)
 214
 215 /*
 216  * Only call the qa poll function when the number responses in the software
 217  * queue drops below this number.
 218  */
 219 #define CRYPTO_QUEUED_RESP_POLL_THRESHOLD       (32)
 220
 221 /*
 222  * Limit the number of polls per call to get_next_response.
 223  */
 224 #define GET_NEXT_RESPONSE_FREQ  (32)
 225
 226 /*
 227  * Max number of responses to pull from the qa in one poll.
 228  */
 229 #define CRYPTO_MAX_RESPONSE_QUOTA \
 230                 (CRYPTO_SOFTWARE_QUEUE_SIZE-CRYPTO_QUEUED_RESP_POLL_THRESHOLD-1)
 231
 232 #if (CRYPTO_QUEUED_RESP_POLL_THRESHOLD + CRYPTO_MAX_RESPONSE_QUOTA >= \
 233                 CRYPTO_SOFTWARE_QUEUE_SIZE)
 234 #error Its possible to overflow the qa response Q with current poll and \
 235                 response quota.
 236 #endif
 237
 238 static void
 239 crypto_callback(CpaCySymDpOpData *pOpData,
 240                 __rte_unused CpaStatus status,
 241                 __rte_unused CpaBoolean verifyResult)
 242 {
 243         uint32_t lcore_id;
 244         lcore_id = rte_lcore_id();
 245         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 246
 247         /*
 248          * Received a completion from the QA hardware.
 249          * Place the response on the return queue.
 250          */
 251         callbackQ->qaCallbackRing[callbackQ->head] = pOpData->pCallbackTag;
 252         callbackQ->head++;
 253         callbackQ->numEntries++;
 254         qaCoreConf[lcore_id].qaOutstandingRequests--;
 255 }
 256
 257 static void
 258 qa_crypto_callback(CpaCySymDpOpData *pOpData, CpaStatus status,
 259                 CpaBoolean verifyResult)
 260 {
 261         crypto_callback(pOpData, status, verifyResult);
 262 }
 263
 264 /*
 265  * Each allocation from a particular memzone lasts for the life-time of
 266  * the application. No freeing of previous allocations will occur.
 267  */
 268 static void *
 269 alloc_memzone_region(uint32_t length, uint32_t lcore_id)
 270 {
 271         char *current_free_addr_ptr = NULL;
 272         struct lcore_memzone *lcore_memzone = &(qaCoreConf[lcore_id].lcoreMemzone);
 273
 274         current_free_addr_ptr  = lcore_memzone->next_free_address;
 275
 276         if (current_free_addr_ptr + length >=
 277                 (char *)lcore_memzone->memzone->addr + lcore_memzone->memzone->len) {
 278                 printf("Crypto: No memory available in memzone\n");
 279                 return NULL;
 280         }
 281         lcore_memzone->next_free_address = current_free_addr_ptr + length;
 282
 283         return (void *)current_free_addr_ptr;
 284 }
 285
 286 /*
 287  * Virtual to Physical Address translation is only executed during initialization
 288  * and not on the data-path.
 289  */
 290 static CpaPhysicalAddr
 291 qa_v2p(void *ptr)
 292 {
 293         const struct rte_memzone *memzone = NULL;
 294         uint32_t lcore_id = 0;
 295         RTE_LCORE_FOREACH(lcore_id) {
 296                 memzone = qaCoreConf[lcore_id].lcoreMemzone.memzone;
 297
 298                 if ((char*) ptr >= (char *) memzone->addr &&
 299                                 (char*) ptr < ((char*) memzone->addr + memzone->len)) {
 300                         return (CpaPhysicalAddr)
 301                                         (memzone->phys_addr + ((char *) ptr - (char*) memzone->addr));
 302                 }
 303         }
 304         printf("Crypto: Corresponding physical address not found in memzone\n");
 305         return (CpaPhysicalAddr) 0;
 306 }
 307
 308 static CpaStatus
 309 getCoreAffinity(Cpa32U *coreAffinity, const CpaInstanceHandle instanceHandle)
 310 {
 311         CpaInstanceInfo2 info;
 312         Cpa16U i = 0;
 313         CpaStatus status = CPA_STATUS_SUCCESS;
 314
 315         bzero(&info, sizeof(CpaInstanceInfo2));
 316
 317         status = cpaCyInstanceGetInfo2(instanceHandle, &info);
 318         if (CPA_STATUS_SUCCESS != status) {
 319                 printf("Crypto: Error getting instance info\n");
 320                 return CPA_STATUS_FAIL;
 321         }
 322         for (i = 0; i < MAX_CORES; i++) {
 323                 if (CPA_BITMAP_BIT_TEST(info.coreAffinity, i)) {
 324                         *coreAffinity = i;
 325                         return CPA_STATUS_SUCCESS;
 326                 }
 327         }
 328         return CPA_STATUS_FAIL;
 329 }
 330
 331 static CpaStatus
 332 get_crypto_instance_on_core(CpaInstanceHandle *pInstanceHandle,
 333                 uint32_t lcore_id)
 334 {
 335         Cpa16U numInstances = 0, i = 0;
 336         CpaStatus status = CPA_STATUS_FAIL;
 337         CpaInstanceHandle *pLocalInstanceHandles = NULL;
 338         Cpa32U coreAffinity = 0;
 339
 340         status = cpaCyGetNumInstances(&numInstances);
 341         if (CPA_STATUS_SUCCESS != status || numInstances == 0) {
 342                 return CPA_STATUS_FAIL;
 343         }
 344
 345         pLocalInstanceHandles = rte_malloc("pLocalInstanceHandles",
 346                         sizeof(CpaInstanceHandle) * numInstances, CACHE_LINE_SIZE);
 347
 348         if (NULL == pLocalInstanceHandles) {
 349                 return CPA_STATUS_FAIL;
 350         }
 351         status = cpaCyGetInstances(numInstances, pLocalInstanceHandles);
 352         if (CPA_STATUS_SUCCESS != status) {
 353                 printf("Crypto: cpaCyGetInstances failed with status: %"PRId32"\n", status);
 354                 rte_free((void *) pLocalInstanceHandles);
 355                 return CPA_STATUS_FAIL;
 356         }
 357
 358         for (i = 0; i < numInstances; i++) {
 359                 status = getCoreAffinity(&coreAffinity, pLocalInstanceHandles[i]);
 360                 if (CPA_STATUS_SUCCESS != status) {
 361                         rte_free((void *) pLocalInstanceHandles);
 362                         return CPA_STATUS_FAIL;
 363                 }
 364                 if (coreAffinity == lcore_id) {
 365                         printf("Crypto: instance found on core %d\n", i);
 366                         *pInstanceHandle = pLocalInstanceHandles[i];
 367                         return CPA_STATUS_SUCCESS;
 368                 }
 369         }
 370         /* core affinity not found */
 371         rte_free((void *) pLocalInstanceHandles);
 372         return CPA_STATUS_FAIL;
 373 }
 374
 375 static CpaStatus
 376 initCySymSession(const int pkt_cipher_alg,
 377                 const int pkt_hash_alg, const CpaCySymHashMode hashMode,
 378                 const CpaCySymCipherDirection crypto_direction,
 379                 CpaCySymSessionCtx **ppSessionCtx,
 380                 const CpaInstanceHandle cyInstanceHandle,
 381                 const uint32_t lcore_id)
 382 {
 383         Cpa32U sessionCtxSizeInBytes = 0;
 384         CpaStatus status = CPA_STATUS_FAIL;
 385         CpaBoolean isCrypto = CPA_TRUE, isHmac = CPA_TRUE;
 386         CpaCySymSessionSetupData sessionSetupData;
 387
 388         bzero(&sessionSetupData, sizeof(CpaCySymSessionSetupData));
 389
 390         /* Assumption: key length is set to each algorithm's max length */
 391         switch (pkt_cipher_alg) {
 392         case NO_CIPHER:
 393                 isCrypto = CPA_FALSE;
 394                 break;
 395         case CIPHER_DES:
 396                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 397                                 CPA_CY_SYM_CIPHER_DES_ECB;
 398                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 399                                 KEY_SIZE_64_IN_BYTES;
 400                 break;
 401         case CIPHER_DES_CBC:
 402                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 403                                 CPA_CY_SYM_CIPHER_DES_CBC;
 404                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 405                                 KEY_SIZE_64_IN_BYTES;
 406                 break;
 407         case CIPHER_DES3:
 408                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 409                                 CPA_CY_SYM_CIPHER_3DES_ECB;
 410                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 411                                 KEY_SIZE_192_IN_BYTES;
 412                 break;
 413         case CIPHER_DES3_CBC:
 414                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 415                                 CPA_CY_SYM_CIPHER_3DES_CBC;
 416                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 417                                 KEY_SIZE_192_IN_BYTES;
 418                 break;
 419         case CIPHER_AES:
 420                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 421                                 CPA_CY_SYM_CIPHER_AES_ECB;
 422                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 423                                 KEY_SIZE_128_IN_BYTES;
 424                 break;
 425         case CIPHER_AES_CBC_128:
 426                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 427                                 CPA_CY_SYM_CIPHER_AES_CBC;
 428                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 429                                 KEY_SIZE_128_IN_BYTES;
 430                 break;
 431         default:
 432                 printf("Crypto: Undefined Cipher specified\n");
 433                 break;
 434         }
 435         /* Set the cipher direction */
 436         if (isCrypto) {
 437                 sessionSetupData.cipherSetupData.cipherDirection = crypto_direction;
 438                 sessionSetupData.cipherSetupData.pCipherKey =
 439                                 g_crypto_hash_keys.cipher_key;
 440                 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
 441         }
 442
 443         /* Setup Hash common fields */
 444         switch (pkt_hash_alg) {
 445         case NO_HASH:
 446                 isHmac = CPA_FALSE;
 447                 break;
 448         case HASH_AES_XCBC:
 449                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 450                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 451                                 AES_XCBC_DIGEST_LENGTH_IN_BYTES;
 452                 break;
 453         case HASH_AES_XCBC_96:
 454                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 455                                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 456                                 AES_XCBC_96_DIGEST_LENGTH_IN_BYTES;
 457                 break;
 458         case HASH_MD5:
 459                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_MD5;
 460                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 461                                 MD5_DIGEST_LENGTH_IN_BYTES;
 462                 break;
 463         case HASH_SHA1:
 464                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 465                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 466                                 SHA1_DIGEST_LENGTH_IN_BYTES;
 467                 break;
 468         case HASH_SHA1_96:
 469                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 470                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 471                                 SHA1_96_DIGEST_LENGTH_IN_BYTES;
 472             break;
 473         case HASH_SHA224:
 474                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA224;
 475                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 476                                 SHA224_DIGEST_LENGTH_IN_BYTES;
 477                 break;
 478         case HASH_SHA256:
 479                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA256;
 480                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 481                                 SHA256_DIGEST_LENGTH_IN_BYTES;
 482                 break;
 483         case HASH_SHA384:
 484                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA384;
 485                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 486                                 SHA384_DIGEST_LENGTH_IN_BYTES;
 487                 break;
 488         case HASH_SHA512:
 489                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA512;
 490                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 491                                 SHA512_DIGEST_LENGTH_IN_BYTES;
 492                 break;
 493         default:
 494                 printf("Crypto: Undefined Hash specified\n");
 495                 break;
 496         }
 497         if (isHmac) {
 498                 sessionSetupData.hashSetupData.hashMode = hashMode;
 499                 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
 500                 /* If using authenticated hash setup key lengths */
 501                 if (CPA_CY_SYM_HASH_MODE_AUTH == hashMode) {
 502                         /* Use a common max length key */
 503                         sessionSetupData.hashSetupData.authModeSetupData.authKey =
 504                                         g_crypto_hash_keys.hash_key;
 505                         switch (pkt_hash_alg) {
 506                         case HASH_AES_XCBC:
 507                         case HASH_AES_XCBC_96:
 508                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 509                                                 AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES;
 510                                 break;
 511                         case HASH_MD5:
 512                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 513                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 514                                 break;
 515                         case HASH_SHA1:
 516                         case HASH_SHA1_96:
 517                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 518                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 519                                 break;
 520                         case HASH_SHA224:
 521                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 522                                                 SHA224_AUTH_KEY_LENGTH_IN_BYTES;
 523                                 break;
 524                         case HASH_SHA256:
 525                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 526                                                 SHA256_AUTH_KEY_LENGTH_IN_BYTES;
 527                                 break;
 528                         case HASH_SHA384:
 529                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 530                                                 SHA384_AUTH_KEY_LENGTH_IN_BYTES;
 531                                 break;
 532                         case HASH_SHA512:
 533                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 534                                                 SHA512_AUTH_KEY_LENGTH_IN_BYTES;
 535                                 break;
 536                         default:
 537                                 printf("Crypto: Undefined Hash specified\n");
 538                                 return CPA_STATUS_FAIL;
 539                         }
 540                 }
 541         }
 542
 543         /* Only high priority supported */
 544         sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
 545
 546         /* If chaining algorithms */
 547         if (isCrypto && isHmac) {
 548                 sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 549                 /* @assumption Alg Chain order is cipher then hash for encrypt
 550                  * and hash then cipher then has for decrypt*/
 551                 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == crypto_direction) {
 552                         sessionSetupData.algChainOrder =
 553                                         CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 554                 } else {
 555                         sessionSetupData.algChainOrder =
 556                                         CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 557                 }
 558         }
 559         if (!isCrypto && !isHmac) {
 560                 *ppSessionCtx = NULL;
 561                 return CPA_STATUS_SUCCESS;
 562         }
 563
 564         /* Get the session context size based on the crypto and/or hash operations*/
 565         status = cpaCySymDpSessionCtxGetSize(cyInstanceHandle, &sessionSetupData,
 566                         &sessionCtxSizeInBytes);
 567         if (CPA_STATUS_SUCCESS != status) {
 568                 printf("Crypto: cpaCySymDpSessionCtxGetSize error, status: %"PRId32"\n",
 569                                 status);
 570                 return CPA_STATUS_FAIL;
 571         }
 572
 573         *ppSessionCtx = alloc_memzone_region(sessionCtxSizeInBytes, lcore_id);
 574         if (NULL == *ppSessionCtx) {
 575                 printf("Crypto: Failed to allocate memory for Session Context\n");
 576                 return CPA_STATUS_FAIL;
 577         }
 578
 579         status = cpaCySymDpInitSession(cyInstanceHandle, &sessionSetupData,
 580                         CPA_TRUE,CPA_FALSE, *ppSessionCtx);
 581         if (CPA_STATUS_SUCCESS != status) {
 582                 printf("Crypto: cpaCySymDpInitSession failed with status %"PRId32"\n", status);
 583                 return CPA_STATUS_FAIL;
 584         }
 585         return CPA_STATUS_SUCCESS;
 586 }
 587
 588 static CpaStatus
 589 initSessionDataTables(struct qa_core_conf *qaCoreConf,uint32_t lcore_id)
 590 {
 591         Cpa32U i = 0, j = 0;
 592         CpaStatus status = CPA_STATUS_FAIL;
 593         for (i = 0; i < NUM_CRYPTO; i++) {
 594                 for (j = 0; j < NUM_HMAC; j++) {
 595                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 596                                         CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
 597                                         &qaCoreConf->encryptSessionHandleTbl[i][j],
 598                                         qaCoreConf->instanceHandle,
 599                                         lcore_id);
 600                         if (CPA_STATUS_SUCCESS != status) {
 601                                 printf("Crypto: Failed to initialize Encrypt sessions\n");
 602                                 return CPA_STATUS_FAIL;
 603                         }
 604                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 605                                         CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
 606                                         &qaCoreConf->decryptSessionHandleTbl[i][j],
 607                                         qaCoreConf->instanceHandle,
 608                                         lcore_id);
 609                         if (CPA_STATUS_SUCCESS != status) {
 610                                 printf("Crypto: Failed to initialize Decrypt sessions\n");
 611                                 return CPA_STATUS_FAIL;
 612                         }
 613                 }
 614         }
 615         return CPA_STATUS_SUCCESS;
 616 }
 617
 618 int
 619 crypto_init(void)
 620 {
 621         if (CPA_STATUS_SUCCESS != icp_sal_userStart("SSL")) {
 622                 printf("Crypto: Could not start sal for user space\n");
 623                 return CPA_STATUS_FAIL;
 624         }
 625         printf("Crypto: icp_sal_userStart(\"SSL\")\n");
 626         return 0;
 627 }
 628
 629 /*
 630  * Per core initialisation
 631  */
 632 int
 633 per_core_crypto_init(uint32_t lcore_id)
 634 {
 635         CpaStatus status = CPA_STATUS_FAIL;
 636         char memzone_name[RTE_MEMZONE_NAMESIZE];
 637
 638         int socketID = rte_lcore_to_socket_id(lcore_id);
 639
 640         /* Allocate software ring for response messages. */
 641
 642         qaCoreConf[lcore_id].callbackQueue.head = 0;
 643         qaCoreConf[lcore_id].callbackQueue.tail = 0;
 644         qaCoreConf[lcore_id].callbackQueue.numEntries = 0;
 645         qaCoreConf[lcore_id].kickFreq = 0;
 646         qaCoreConf[lcore_id].qaOutstandingRequests = 0;
 647         qaCoreConf[lcore_id].numResponseAttempts = 0;
 648
 649         /* Initialise and reserve lcore memzone for virt2phys translation */
 650         rte_snprintf(memzone_name,
 651                         RTE_MEMZONE_NAMESIZE,
 652                         "lcore_%u",
 653                         lcore_id);
 654
 655         qaCoreConf[lcore_id].lcoreMemzone.memzone = rte_memzone_reserve(
 656                         memzone_name,
 657                         LCORE_MEMZONE_SIZE,
 658                         socketID,
 659                         0);
 660         if (NULL == qaCoreConf[lcore_id].lcoreMemzone.memzone) {
 661                 printf("Crypto: Error allocating memzone on lcore %u\n",lcore_id);
 662                 return -1;
 663         }
 664         qaCoreConf[lcore_id].lcoreMemzone.next_free_address =
 665                                                         qaCoreConf[lcore_id].lcoreMemzone.memzone->addr;
 666
 667         qaCoreConf[lcore_id].pPacketIV = alloc_memzone_region(IV_LENGTH_16_BYTES,
 668                                                         lcore_id);
 669
 670         if (NULL == qaCoreConf[lcore_id].pPacketIV ) {
 671                 printf("Crypto: Failed to allocate memory for Initialization Vector\n");
 672                 return -1;
 673         }
 674
 675         memcpy(qaCoreConf[lcore_id].pPacketIV, &g_crypto_hash_keys.iv,
 676                         IV_LENGTH_16_BYTES);
 677
 678         qaCoreConf[lcore_id].packetIVPhy = qa_v2p(qaCoreConf[lcore_id].pPacketIV);
 679         if (0 == qaCoreConf[lcore_id].packetIVPhy) {
 680                 printf("Crypto: Invalid physical address for Initialization Vector\n");
 681                 return -1;
 682         }
 683
 684         /*
 685          * Obtain the instance handle that is mapped to the current lcore.
 686          * This can fail if an instance is not mapped to a bank which has been
 687          * affinitized to the current lcore.
 688          */
 689         status = get_crypto_instance_on_core(&(qaCoreConf[lcore_id].instanceHandle),
 690                         lcore_id);
 691         if (CPA_STATUS_SUCCESS != status) {
 692                 printf("Crypto: get_crypto_instance_on_core failed with status: %"PRId32"\n",
 693                                 status);
 694                 return -1;
 695         }
 696
 697         status = cpaCySymDpRegCbFunc(qaCoreConf[lcore_id].instanceHandle,
 698                         (CpaCySymDpCbFunc) qa_crypto_callback);
 699         if (CPA_STATUS_SUCCESS != status) {
 700                 printf("Crypto: cpaCySymDpRegCbFunc failed with status: %"PRId32"\n", status);
 701                 return -1;
 702         }
 703
 704         /*
 705          * Set the address translation callback for virtual to physcial address
 706          * mapping. This will be called by the QAT driver during initialisation only.
 707          */
 708         status = cpaCySetAddressTranslation(qaCoreConf[lcore_id].instanceHandle,
 709                         (CpaVirtualToPhysical) qa_v2p);
 710         if (CPA_STATUS_SUCCESS != status) {
 711                 printf("Crypto: cpaCySetAddressTranslation failed with status: %"PRId32"\n",
 712                                 status);
 713                 return -1;
 714         }
 715
 716         status = initSessionDataTables(&qaCoreConf[lcore_id],lcore_id);
 717         if (CPA_STATUS_SUCCESS != status) {
 718                 printf("Crypto: Failed to allocate all session tables.");
 719                 return -1;
 720         }
 721         return 0;
 722 }
 723
 724 static CpaStatus
 725 enqueueOp(CpaCySymDpOpData *opData, uint32_t lcore_id)
 726 {
 727
 728         CpaStatus status;
 729
 730         /*
 731          * Assumption is there is no requirement to do load balancing between
 732          * acceleration units - that is one acceleration unit is tied to a core.
 733          */
 734         opData->instanceHandle = qaCoreConf[lcore_id].instanceHandle;
 735
 736         if ((++qaCoreConf[lcore_id].kickFreq) % CRYPTO_BURST_TX == 0) {
 737                 status = cpaCySymDpEnqueueOp(opData, CPA_TRUE);
 738         } else {
 739                 status = cpaCySymDpEnqueueOp(opData, CPA_FALSE);
 740         }
 741
 742         qaCoreConf[lcore_id].qaOutstandingRequests++;
 743
 744         return status;
 745 }
 746
 747 void
 748 crypto_flush_tx_queue(uint32_t lcore_id)
 749 {
 750
 751         cpaCySymDpPerformOpNow(qaCoreConf[lcore_id].instanceHandle);
 752 }
 753
 754 enum crypto_result
 755 crypto_encrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 756 {
 757         CpaCySymDpOpData *opData =
 758                         (CpaCySymDpOpData *) ((char *) (rte_buff->pkt.data)
 759                                         + CRYPTO_OFFSET_TO_OPDATA);
 760         uint32_t lcore_id;
 761
 762         lcore_id = rte_lcore_id();
 763
 764         bzero(opData, sizeof(CpaCySymDpOpData));
 765
 766         opData->srcBuffer = opData->dstBuffer = PACKET_DATA_START_PHYS(rte_buff);
 767         opData->srcBufferLen = opData->dstBufferLen = rte_buff->pkt.data_len;
 768         opData->sessionCtx = qaCoreConf[lcore_id].encryptSessionHandleTbl[c][h];
 769         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 770                         + CRYPTO_OFFSET_TO_OPDATA;
 771         opData->pCallbackTag = rte_buff;
 772
 773         /* if no crypto or hash operations are specified return fail */
 774         if (NO_CIPHER == c && NO_HASH == h)
 775                 return CRYPTO_RESULT_FAIL;
 776
 777         if (NO_CIPHER != c) {
 778                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 779                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 780
 781                 if (CIPHER_AES_CBC_128 == c)
 782                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 783                 else
 784                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 785
 786                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 787                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 788                                 - CRYPTO_START_OFFSET;
 789                 /*
 790                  * Work around for padding, message length has to be a multiple of
 791                  * block size.
 792                  */
 793                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 794                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 795         }
 796
 797         if (NO_HASH != h) {
 798
 799                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 800                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 801                                 - HASH_START_OFFSET;
 802                 /*
 803                  * Work around for padding, message length has to be a multiple of block
 804                  * size.
 805                  */
 806                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 807                                 % HASH_BLOCK_DEFAULT_SIZE;
 808
 809                 /*
 810                  * Assumption: Ok ignore the passed digest pointer and place HMAC at end
 811                  * of packet.
 812                  */
 813                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 814         }
 815
 816         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 817                 /*
 818                  * Failed to place a packet on the hardware queue.
 819                  * Most likely because the QA hardware is busy.
 820                  */
 821                 return CRYPTO_RESULT_FAIL;
 822         }
 823         return CRYPTO_RESULT_IN_PROGRESS;
 824 }
 825
 826 enum crypto_result
 827 crypto_decrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 828 {
 829
 830         CpaCySymDpOpData *opData = (void*) (((char *) rte_buff->pkt.data)
 831                         + CRYPTO_OFFSET_TO_OPDATA);
 832         uint32_t lcore_id;
 833
 834         lcore_id = rte_lcore_id();
 835
 836         bzero(opData, sizeof(CpaCySymDpOpData));
 837
 838         opData->dstBuffer = opData->srcBuffer = PACKET_DATA_START_PHYS(rte_buff);
 839         opData->dstBufferLen = opData->srcBufferLen = rte_buff->pkt.data_len;
 840         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 841                         + CRYPTO_OFFSET_TO_OPDATA;
 842         opData->sessionCtx = qaCoreConf[lcore_id].decryptSessionHandleTbl[c][h];
 843         opData->pCallbackTag = rte_buff;
 844
 845         /* if no crypto or hmac operations are specified return fail */
 846         if (NO_CIPHER == c && NO_HASH == h)
 847                 return CRYPTO_RESULT_FAIL;
 848
 849         if (NO_CIPHER != c) {
 850                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 851                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 852
 853                 if (CIPHER_AES_CBC_128 == c)
 854                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 855                 else
 856                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 857
 858                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 859                 opData->messageLenToCipherInBytes = rte_buff->pkt.data_len
 860                                 - CRYPTO_START_OFFSET;
 861
 862                 /*
 863                  * Work around for padding, message length has to be a multiple of block
 864                  * size.
 865                  */
 866                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 867                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 868         }
 869         if (NO_HASH != h) {
 870                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 871                 opData->messageLenToHashInBytes = rte_buff->pkt.data_len
 872                                 - HASH_START_OFFSET;
 873                 /*
 874                  * Work around for padding, message length has to be a multiple of block
 875                  * size.
 876                  */
 877                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 878                                 % HASH_BLOCK_DEFAULT_SIZE;
 879                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->pkt.data_len;
 880         }
 881
 882         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 883                 /*
 884                  * Failed to place a packet on the hardware queue.
 885                  * Most likely because the QA hardware is busy.
 886                  */
 887                 return CRYPTO_RESULT_FAIL;
 888         }
 889         return CRYPTO_RESULT_IN_PROGRESS;
 890 }
 891
 892 void *
 893 crypto_get_next_response(void)
 894 {
 895         uint32_t lcore_id;
 896         lcore_id = rte_lcore_id();
 897         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 898         void *entry = NULL;
 899
 900         if (callbackQ->numEntries) {
 901                 entry = callbackQ->qaCallbackRing[callbackQ->tail];
 902                 callbackQ->tail++;
 903                 callbackQ->numEntries--;
 904         }
 905
 906         /* If there are no outstanding requests no need to poll, return entry */
 907         if (qaCoreConf[lcore_id].qaOutstandingRequests == 0)
 908                 return entry;
 909
 910         if (callbackQ->numEntries < CRYPTO_QUEUED_RESP_POLL_THRESHOLD
 911                         && qaCoreConf[lcore_id].numResponseAttempts++
 912                                         % GET_NEXT_RESPONSE_FREQ == 0) {
 913                 /*
 914                  * Only poll the hardware when there is less than
 915                  * CRYPTO_QUEUED_RESP_POLL_THRESHOLD elements in the software queue
 916                  */
 917                 icp_sal_CyPollDpInstance(qaCoreConf[lcore_id].instanceHandle,
 918                                 CRYPTO_MAX_RESPONSE_QUOTA);
 919         }
 920         return entry;
 921 }