drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_common.h>
  34 #include <rte_hexdump.h>
  35 #include <rte_cryptodev.h>
  36 #include <rte_cryptodev_pmd.h>
  37 #include <rte_vdev.h>
  38 #include <rte_malloc.h>
  39 #include <rte_cpuflags.h>
  40
  41 #include "rte_aesni_mb_pmd_private.h"
  42
  43 /**
  44  * Global static parameter used to create a unique name for each AES-NI multi
  45  * buffer crypto device.
  46  */
  47 static unsigned unique_name_id;
  48
  49 static inline int
  50 create_unique_device_name(char *name, size_t size)
  51 {
  52         int ret;
  53
  54         if (name == NULL)
  55                 return -EINVAL;
  56
  57         ret = snprintf(name, size, "%s_%u", RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD),
  58                         unique_name_id++);
  59         if (ret < 0)
  60                 return ret;
  61         return 0;
  62 }
  63
  64 typedef void (*hash_one_block_t)(void *data, void *digest);
  65 typedef void (*aes_keyexp_t)(void *key, void *enc_exp_keys, void *dec_exp_keys);
  66
  67 /**
  68  * Calculate the authentication pre-computes
  69  *
  70  * @param one_block_hash        Function pointer to calculate digest on ipad/opad
  71  * @param ipad                  Inner pad output byte array
  72  * @param opad                  Outer pad output byte array
  73  * @param hkey                  Authentication key
  74  * @param hkey_len              Authentication key length
  75  * @param blocksize             Block size of selected hash algo
  76  */
  77 static void
  78 calculate_auth_precomputes(hash_one_block_t one_block_hash,
  79                 uint8_t *ipad, uint8_t *opad,
  80                 uint8_t *hkey, uint16_t hkey_len,
  81                 uint16_t blocksize)
  82 {
  83         unsigned i, length;
  84
  85         uint8_t ipad_buf[blocksize] __rte_aligned(16);
  86         uint8_t opad_buf[blocksize] __rte_aligned(16);
  87
  88         /* Setup inner and outer pads */
  89         memset(ipad_buf, HMAC_IPAD_VALUE, blocksize);
  90         memset(opad_buf, HMAC_OPAD_VALUE, blocksize);
  91
  92         /* XOR hash key with inner and outer pads */
  93         length = hkey_len > blocksize ? blocksize : hkey_len;
  94
  95         for (i = 0; i < length; i++) {
  96                 ipad_buf[i] ^= hkey[i];
  97                 opad_buf[i] ^= hkey[i];
  98         }
  99
 100         /* Compute partial hashes */
 101         (*one_block_hash)(ipad_buf, ipad);
 102         (*one_block_hash)(opad_buf, opad);
 103
 104         /* Clean up stack */
 105         memset(ipad_buf, 0, blocksize);
 106         memset(opad_buf, 0, blocksize);
 107 }
 108
 109 /** Get xform chain order */
 110 static enum aesni_mb_operation
 111 aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
 112 {
 113         if (xform == NULL)
 114                 return AESNI_MB_OP_NOT_SUPPORTED;
 115
 116         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 117                 if (xform->next == NULL)
 118                         return AESNI_MB_OP_CIPHER_ONLY;
 119                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 120                         return AESNI_MB_OP_CIPHER_HASH;
 121         }
 122
 123         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 124                 if (xform->next == NULL)
 125                         return AESNI_MB_OP_HASH_ONLY;
 126                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 127                         return AESNI_MB_OP_HASH_CIPHER;
 128         }
 129
 130         return AESNI_MB_OP_NOT_SUPPORTED;
 131 }
 132
 133 /** Set session authentication parameters */
 134 static int
 135 aesni_mb_set_session_auth_parameters(const struct aesni_mb_ops *mb_ops,
 136                 struct aesni_mb_session *sess,
 137                 const struct rte_crypto_sym_xform *xform)
 138 {
 139         hash_one_block_t hash_oneblock_fn;
 140
 141         if (xform == NULL) {
 142                 sess->auth.algo = NULL_HASH;
 143                 return 0;
 144         }
 145
 146         if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
 147                 MB_LOG_ERR("Crypto xform struct not of type auth");
 148                 return -1;
 149         }
 150
 151         /* Select auth generate/verify */
 152         sess->auth.operation = xform->auth.op;
 153
 154         /* Set Authentication Parameters */
 155         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_XCBC_MAC) {
 156                 sess->auth.algo = AES_XCBC;
 157                 (*mb_ops->aux.keyexp.aes_xcbc)(xform->auth.key.data,
 158                                 sess->auth.xcbc.k1_expanded,
 159                                 sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 160                 return 0;
 161         }
 162
 163         switch (xform->auth.algo) {
 164         case RTE_CRYPTO_AUTH_MD5_HMAC:
 165                 sess->auth.algo = MD5;
 166                 hash_oneblock_fn = mb_ops->aux.one_block.md5;
 167                 break;
 168         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 169                 sess->auth.algo = SHA1;
 170                 hash_oneblock_fn = mb_ops->aux.one_block.sha1;
 171                 break;
 172         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 173                 sess->auth.algo = SHA_224;
 174                 hash_oneblock_fn = mb_ops->aux.one_block.sha224;
 175                 break;
 176         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 177                 sess->auth.algo = SHA_256;
 178                 hash_oneblock_fn = mb_ops->aux.one_block.sha256;
 179                 break;
 180         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 181                 sess->auth.algo = SHA_384;
 182                 hash_oneblock_fn = mb_ops->aux.one_block.sha384;
 183                 break;
 184         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 185                 sess->auth.algo = SHA_512;
 186                 hash_oneblock_fn = mb_ops->aux.one_block.sha512;
 187                 break;
 188         default:
 189                 MB_LOG_ERR("Unsupported authentication algorithm selection");
 190                 return -1;
 191         }
 192
 193         /* Calculate Authentication precomputes */
 194         calculate_auth_precomputes(hash_oneblock_fn,
 195                         sess->auth.pads.inner, sess->auth.pads.outer,
 196                         xform->auth.key.data,
 197                         xform->auth.key.length,
 198                         get_auth_algo_blocksize(sess->auth.algo));
 199
 200         return 0;
 201 }
 202
 203 /** Set session cipher parameters */
 204 static int
 205 aesni_mb_set_session_cipher_parameters(const struct aesni_mb_ops *mb_ops,
 206                 struct aesni_mb_session *sess,
 207                 const struct rte_crypto_sym_xform *xform)
 208 {
 209         aes_keyexp_t aes_keyexp_fn;
 210
 211         if (xform == NULL) {
 212                 sess->cipher.mode = NULL_CIPHER;
 213                 return 0;
 214         }
 215
 216         if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 217                 MB_LOG_ERR("Crypto xform struct not of type cipher");
 218                 return -1;
 219         }
 220
 221         /* Select cipher direction */
 222         switch (xform->cipher.op) {
 223         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
 224                 sess->cipher.direction = ENCRYPT;
 225                 break;
 226         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
 227                 sess->cipher.direction = DECRYPT;
 228                 break;
 229         default:
 230                 MB_LOG_ERR("Unsupported cipher operation parameter");
 231                 return -1;
 232         }
 233
 234         /* Select cipher mode */
 235         switch (xform->cipher.algo) {
 236         case RTE_CRYPTO_CIPHER_AES_CBC:
 237                 sess->cipher.mode = CBC;
 238                 break;
 239         case RTE_CRYPTO_CIPHER_AES_CTR:
 240                 sess->cipher.mode = CNTR;
 241                 break;
 242         default:
 243                 MB_LOG_ERR("Unsupported cipher mode parameter");
 244                 return -1;
 245         }
 246
 247         /* Check key length and choose key expansion function */
 248         switch (xform->cipher.key.length) {
 249         case AES_128_BYTES:
 250                 sess->cipher.key_length_in_bytes = AES_128_BYTES;
 251                 aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
 252                 break;
 253         case AES_192_BYTES:
 254                 sess->cipher.key_length_in_bytes = AES_192_BYTES;
 255                 aes_keyexp_fn = mb_ops->aux.keyexp.aes192;
 256                 break;
 257         case AES_256_BYTES:
 258                 sess->cipher.key_length_in_bytes = AES_256_BYTES;
 259                 aes_keyexp_fn = mb_ops->aux.keyexp.aes256;
 260                 break;
 261         default:
 262                 MB_LOG_ERR("Unsupported cipher key length");
 263                 return -1;
 264         }
 265
 266         /* Expanded cipher keys */
 267         (*aes_keyexp_fn)(xform->cipher.key.data,
 268                         sess->cipher.expanded_aes_keys.encode,
 269                         sess->cipher.expanded_aes_keys.decode);
 270
 271         return 0;
 272 }
 273
 274 /** Parse crypto xform chain and set private session parameters */
 275 int
 276 aesni_mb_set_session_parameters(const struct aesni_mb_ops *mb_ops,
 277                 struct aesni_mb_session *sess,
 278                 const struct rte_crypto_sym_xform *xform)
 279 {
 280         const struct rte_crypto_sym_xform *auth_xform = NULL;
 281         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 282
 283         /* Select Crypto operation - hash then cipher / cipher then hash */
 284         switch (aesni_mb_get_chain_order(xform)) {
 285         case AESNI_MB_OP_HASH_CIPHER:
 286                 sess->chain_order = HASH_CIPHER;
 287                 auth_xform = xform;
 288                 cipher_xform = xform->next;
 289                 break;
 290         case AESNI_MB_OP_CIPHER_HASH:
 291                 sess->chain_order = CIPHER_HASH;
 292                 auth_xform = xform->next;
 293                 cipher_xform = xform;
 294                 break;
 295         case AESNI_MB_OP_HASH_ONLY:
 296                 sess->chain_order = HASH_CIPHER;
 297                 auth_xform = xform;
 298                 cipher_xform = NULL;
 299                 break;
 300         case AESNI_MB_OP_CIPHER_ONLY:
 301                 /*
 302                  * Multi buffer library operates only at two modes,
 303                  * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
 304                  * chain order depends on cipher operation: encryption is always
 305                  * the first operation and decryption the last one.
 306                  */
 307                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 308                         sess->chain_order = CIPHER_HASH;
 309                 else
 310                         sess->chain_order = HASH_CIPHER;
 311                 auth_xform = NULL;
 312                 cipher_xform = xform;
 313                 break;
 314         case AESNI_MB_OP_NOT_SUPPORTED:
 315         default:
 316                 MB_LOG_ERR("Unsupported operation chain order parameter");
 317                 return -1;
 318         }
 319
 320         if (aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform)) {
 321                 MB_LOG_ERR("Invalid/unsupported authentication parameters");
 322                 return -1;
 323         }
 324
 325         if (aesni_mb_set_session_cipher_parameters(mb_ops, sess,
 326                         cipher_xform)) {
 327                 MB_LOG_ERR("Invalid/unsupported cipher parameters");
 328                 return -1;
 329         }
 330         return 0;
 331 }
 332
 333 /** Get multi buffer session */
 334 static struct aesni_mb_session *
 335 get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 336 {
 337         struct aesni_mb_session *sess = NULL;
 338
 339         if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) {
 340                 if (unlikely(op->sym->session->dev_type !=
 341                                 RTE_CRYPTODEV_AESNI_MB_PMD))
 342                         return NULL;
 343
 344                 sess = (struct aesni_mb_session *)op->sym->session->_private;
 345         } else  {
 346                 void *_sess = NULL;
 347
 348                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 349                         return NULL;
 350
 351                 sess = (struct aesni_mb_session *)
 352                         ((struct rte_cryptodev_sym_session *)_sess)->_private;
 353
 354                 if (unlikely(aesni_mb_set_session_parameters(qp->ops,
 355                                 sess, op->sym->xform) != 0)) {
 356                         rte_mempool_put(qp->sess_mp, _sess);
 357                         sess = NULL;
 358                 }
 359                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 360         }
 361
 362         return sess;
 363 }
 364
 365 /**
 366  * Process a crypto operation and complete a JOB_AES_HMAC job structure for
 367  * submission to the multi buffer library for processing.
 368  *
 369  * @param       qp      queue pair
 370  * @param       job     JOB_AES_HMAC structure to fill
 371  * @param       m       mbuf to process
 372  *
 373  * @return
 374  * - Completed JOB_AES_HMAC structure pointer on success
 375  * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
 376  */
 377 static JOB_AES_HMAC *
 378 process_crypto_op(struct aesni_mb_qp *qp, struct rte_crypto_op *op,
 379                 struct aesni_mb_session *session)
 380 {
 381         JOB_AES_HMAC *job;
 382
 383         struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
 384         uint16_t m_offset = 0;
 385
 386         job = (*qp->ops->job.get_next)(&qp->mb_mgr);
 387         if (unlikely(job == NULL))
 388                 return job;
 389
 390         /* Set crypto operation */
 391         job->chain_order = session->chain_order;
 392
 393         /* Set cipher parameters */
 394         job->cipher_direction = session->cipher.direction;
 395         job->cipher_mode = session->cipher.mode;
 396
 397         job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
 398         job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
 399         job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
 400
 401
 402         /* Set authentication parameters */
 403         job->hash_alg = session->auth.algo;
 404         if (job->hash_alg == AES_XCBC) {
 405                 job->_k1_expanded = session->auth.xcbc.k1_expanded;
 406                 job->_k2 = session->auth.xcbc.k2;
 407                 job->_k3 = session->auth.xcbc.k3;
 408         } else {
 409                 job->hashed_auth_key_xor_ipad = session->auth.pads.inner;
 410                 job->hashed_auth_key_xor_opad = session->auth.pads.outer;
 411         }
 412
 413         /* Mutable crypto operation parameters */
 414         if (op->sym->m_dst) {
 415                 m_src = m_dst = op->sym->m_dst;
 416
 417                 /* append space for output data to mbuf */
 418                 char *odata = rte_pktmbuf_append(m_dst,
 419                                 rte_pktmbuf_data_len(op->sym->m_src));
 420                 if (odata == NULL) {
 421                         MB_LOG_ERR("failed to allocate space in destination "
 422                                         "mbuf for source data");
 423                         return NULL;
 424                 }
 425
 426                 memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*),
 427                                 rte_pktmbuf_data_len(op->sym->m_src));
 428         } else {
 429                 m_dst = m_src;
 430                 m_offset = op->sym->cipher.data.offset;
 431         }
 432
 433         /* Set digest output location */
 434         if (job->hash_alg != NULL_HASH &&
 435                         session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
 436                 job->auth_tag_output = (uint8_t *)rte_pktmbuf_append(m_dst,
 437                                 get_digest_byte_length(job->hash_alg));
 438
 439                 if (job->auth_tag_output == NULL) {
 440                         MB_LOG_ERR("failed to allocate space in output mbuf "
 441                                         "for temp digest");
 442                         return NULL;
 443                 }
 444
 445                 memset(job->auth_tag_output, 0,
 446                                 sizeof(get_digest_byte_length(job->hash_alg)));
 447
 448         } else {
 449                 job->auth_tag_output = op->sym->auth.digest.data;
 450         }
 451
 452         /*
 453          * Multi-buffer library current only support returning a truncated
 454          * digest length as specified in the relevant IPsec RFCs
 455          */
 456         job->auth_tag_output_len_in_bytes =
 457                         get_truncated_digest_byte_length(job->hash_alg);
 458
 459         /* Set IV parameters */
 460         job->iv = op->sym->cipher.iv.data;
 461         job->iv_len_in_bytes = op->sym->cipher.iv.length;
 462
 463         /* Data  Parameter */
 464         job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
 465         job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
 466
 467         job->cipher_start_src_offset_in_bytes = op->sym->cipher.data.offset;
 468         job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
 469
 470         job->hash_start_src_offset_in_bytes = op->sym->auth.data.offset;
 471         job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
 472
 473         /* Set user data to be crypto operation data struct */
 474         job->user_data = op;
 475         job->user_data2 = m_dst;
 476
 477         return job;
 478 }
 479
 480 /**
 481  * Process a completed job and return rte_mbuf which job processed
 482  *
 483  * @param job   JOB_AES_HMAC job to process
 484  *
 485  * @return
 486  * - Returns processed mbuf which is trimmed of output digest used in
 487  * verification of supplied digest in the case of a HASH_CIPHER operation
 488  * - Returns NULL on invalid job
 489  */
 490 static struct rte_crypto_op *
 491 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 492 {
 493         struct rte_crypto_op *op =
 494                         (struct rte_crypto_op *)job->user_data;
 495         struct rte_mbuf *m_dst =
 496                         (struct rte_mbuf *)job->user_data2;
 497         struct aesni_mb_session *sess;
 498
 499         if (op == NULL || m_dst == NULL)
 500                 return NULL;
 501
 502         /* set status as successful by default */
 503         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 504
 505         /* check if job has been processed  */
 506         if (unlikely(job->status != STS_COMPLETED)) {
 507                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 508                 return op;
 509         } else if (job->hash_alg != NULL_HASH) {
 510                 sess = (struct aesni_mb_session *)op->sym->session->_private;
 511                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
 512                         /* Verify digest if required */
 513                         if (memcmp(job->auth_tag_output,
 514                                         op->sym->auth.digest.data,
 515                                         job->auth_tag_output_len_in_bytes) != 0)
 516                                 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 517
 518                         /* trim area used for digest from mbuf */
 519                         rte_pktmbuf_trim(m_dst,
 520                                         get_digest_byte_length(job->hash_alg));
 521                 }
 522         }
 523
 524         /* Free session if a session-less crypto op */
 525         if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) {
 526                 rte_mempool_put(qp->sess_mp, op->sym->session);
 527                 op->sym->session = NULL;
 528         }
 529
 530         return op;
 531 }
 532
 533 /**
 534  * Process a completed JOB_AES_HMAC job and keep processing jobs until
 535  * get_completed_job return NULL
 536  *
 537  * @param qp            Queue Pair to process
 538  * @param job           JOB_AES_HMAC job
 539  *
 540  * @return
 541  * - Number of processed jobs
 542  */
 543 static unsigned
 544 handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 545 {
 546         struct rte_crypto_op *op = NULL;
 547         unsigned processed_jobs = 0;
 548
 549         while (job) {
 550                 processed_jobs++;
 551                 op = post_process_mb_job(qp, job);
 552                 if (op)
 553                         rte_ring_enqueue(qp->processed_ops, (void *)op);
 554                 else
 555                         qp->stats.dequeue_err_count++;
 556                 job = (*qp->ops->job.get_completed_job)(&qp->mb_mgr);
 557         }
 558
 559         return processed_jobs;
 560 }
 561
 562 static uint16_t
 563 aesni_mb_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 564                 uint16_t nb_ops)
 565 {
 566         struct aesni_mb_session *sess;
 567         struct aesni_mb_qp *qp = queue_pair;
 568
 569         JOB_AES_HMAC *job = NULL;
 570
 571         int i, processed_jobs = 0;
 572
 573         for (i = 0; i < nb_ops; i++) {
 574 #ifdef RTE_LIBRTE_AESNI_MB_DEBUG
 575                 if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
 576                         MB_LOG_ERR("PMD only supports symmetric crypto "
 577                                 "operation requests, op (%p) is not a "
 578                                 "symmetric operation.", op);
 579                         qp->stats.enqueue_err_count++;
 580                         goto flush_jobs;
 581                 }
 582 #endif
 583                 sess = get_session(qp, ops[i]);
 584                 if (unlikely(sess == NULL)) {
 585                         qp->stats.enqueue_err_count++;
 586                         goto flush_jobs;
 587                 }
 588
 589                 job = process_crypto_op(qp, ops[i], sess);
 590                 if (unlikely(job == NULL)) {
 591                         qp->stats.enqueue_err_count++;
 592                         goto flush_jobs;
 593                 }
 594
 595                 /* Submit Job */
 596                 job = (*qp->ops->job.submit)(&qp->mb_mgr);
 597
 598                 /*
 599                  * If submit returns a processed job then handle it,
 600                  * before submitting subsequent jobs
 601                  */
 602                 if (job)
 603                         processed_jobs += handle_completed_jobs(qp, job);
 604         }
 605
 606         if (processed_jobs == 0)
 607                 goto flush_jobs;
 608         else
 609                 qp->stats.enqueued_count += processed_jobs;
 610         return i;
 611
 612 flush_jobs:
 613         /*
 614          * If we haven't processed any jobs in submit loop, then flush jobs
 615          * queue to stop the output stalling
 616          */
 617         job = (*qp->ops->job.flush_job)(&qp->mb_mgr);
 618         if (job)
 619                 qp->stats.enqueued_count += handle_completed_jobs(qp, job);
 620
 621         return i;
 622 }
 623
 624 static uint16_t
 625 aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 626                 uint16_t nb_ops)
 627 {
 628         struct aesni_mb_qp *qp = queue_pair;
 629
 630         unsigned nb_dequeued;
 631
 632         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
 633                         (void **)ops, nb_ops);
 634         qp->stats.dequeued_count += nb_dequeued;
 635
 636         return nb_dequeued;
 637 }
 638
 639
 640 static int cryptodev_aesni_mb_remove(const char *name);
 641
 642 static int
 643 cryptodev_aesni_mb_create(const char *name,
 644                 struct rte_crypto_vdev_init_params *init_params)
 645 {
 646         struct rte_cryptodev *dev;
 647         char crypto_dev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 648         struct aesni_mb_private *internals;
 649         enum aesni_mb_vector_mode vector_mode;
 650
 651         /* Check CPU for support for AES instruction set */
 652         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
 653                 MB_LOG_ERR("AES instructions not supported by CPU");
 654                 return -EFAULT;
 655         }
 656
 657         /* Check CPU for supported vector instruction set */
 658         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
 659                 vector_mode = RTE_AESNI_MB_AVX512;
 660         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
 661                 vector_mode = RTE_AESNI_MB_AVX2;
 662         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
 663                 vector_mode = RTE_AESNI_MB_AVX;
 664         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
 665                 vector_mode = RTE_AESNI_MB_SSE;
 666         else {
 667                 MB_LOG_ERR("Vector instructions are not supported by CPU");
 668                 return -EFAULT;
 669         }
 670
 671         /* create a unique device name */
 672         if (create_unique_device_name(crypto_dev_name,
 673                         RTE_CRYPTODEV_NAME_MAX_LEN) != 0) {
 674                 MB_LOG_ERR("failed to create unique cryptodev name");
 675                 return -EINVAL;
 676         }
 677
 678
 679         dev = rte_cryptodev_pmd_virtual_dev_init(crypto_dev_name,
 680                         sizeof(struct aesni_mb_private), init_params->socket_id);
 681         if (dev == NULL) {
 682                 MB_LOG_ERR("failed to create cryptodev vdev");
 683                 goto init_error;
 684         }
 685
 686         dev->dev_type = RTE_CRYPTODEV_AESNI_MB_PMD;
 687         dev->dev_ops = rte_aesni_mb_pmd_ops;
 688
 689         /* register rx/tx burst functions for data path */
 690         dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
 691         dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
 692
 693         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 694                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 695                         RTE_CRYPTODEV_FF_CPU_AESNI;
 696
 697         switch (vector_mode) {
 698         case RTE_AESNI_MB_SSE:
 699                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 700                 break;
 701         case RTE_AESNI_MB_AVX:
 702                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 703                 break;
 704         case RTE_AESNI_MB_AVX2:
 705                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 706                 break;
 707         case RTE_AESNI_MB_AVX512:
 708                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
 709                 break;
 710         default:
 711                 break;
 712         }
 713
 714         /* Set vector instructions mode supported */
 715         internals = dev->data->dev_private;
 716
 717         internals->vector_mode = vector_mode;
 718         internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
 719         internals->max_nb_sessions = init_params->max_nb_sessions;
 720
 721         return 0;
 722 init_error:
 723         MB_LOG_ERR("driver %s: cryptodev_aesni_create failed", name);
 724
 725         cryptodev_aesni_mb_remove(crypto_dev_name);
 726         return -EFAULT;
 727 }
 728
 729
 730 static int
 731 cryptodev_aesni_mb_probe(const char *name,
 732                 const char *input_args)
 733 {
 734         struct rte_crypto_vdev_init_params init_params = {
 735                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
 736                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
 737                 rte_socket_id()
 738         };
 739
 740         rte_cryptodev_parse_vdev_init_params(&init_params, input_args);
 741
 742         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
 743                         init_params.socket_id);
 744         RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
 745                         init_params.max_nb_queue_pairs);
 746         RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
 747                         init_params.max_nb_sessions);
 748
 749         return cryptodev_aesni_mb_create(name, &init_params);
 750 }
 751
 752 static int
 753 cryptodev_aesni_mb_remove(const char *name)
 754 {
 755         if (name == NULL)
 756                 return -EINVAL;
 757
 758         RTE_LOG(INFO, PMD, "Closing AESNI crypto device %s on numa socket %u\n",
 759                         name, rte_socket_id());
 760
 761         return 0;
 762 }
 763
 764 static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
 765         .probe = cryptodev_aesni_mb_probe,
 766         .remove = cryptodev_aesni_mb_remove
 767 };
 768
 769 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
 770 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
 771 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
 772         "max_nb_queue_pairs=<int> "
 773         "max_nb_sessions=<int> "
 774         "socket_id=<int>");