drivers/crypto/mvsam/rte_mrvl_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Marvell International Ltd.
   3  * Copyright(c) 2017 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_common.h>
   8 #include <rte_hexdump.h>
   9 #include <rte_cryptodev.h>
  10 #include <rte_cryptodev_pmd.h>
  11 #include <rte_bus_vdev.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cpuflags.h>
  14 #include <rte_kvargs.h>
  15 #include <rte_mvep_common.h>
  16
  17 #include "mrvl_pmd_private.h"
  18
  19 #define MRVL_PMD_MAX_NB_SESS_ARG                ("max_nb_sessions")
  20 #define MRVL_PMD_DEFAULT_MAX_NB_SESSIONS        2048
  21
  22 int mrvl_logtype_driver;
  23 static uint8_t cryptodev_driver_id;
  24
  25 struct mrvl_pmd_init_params {
  26         struct rte_cryptodev_pmd_init_params common;
  27         uint32_t max_nb_sessions;
  28 };
  29
  30 const char *mrvl_pmd_valid_params[] = {
  31         RTE_CRYPTODEV_PMD_NAME_ARG,
  32         RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG,
  33         RTE_CRYPTODEV_PMD_SOCKET_ID_ARG,
  34         MRVL_PMD_MAX_NB_SESS_ARG
  35 };
  36
  37 /**
  38  * Flag if particular crypto algorithm is supported by PMD/MUSDK.
  39  *
  40  * The idea is to have Not Supported value as default (0).
  41  * This way we need only to define proper map sizes,
  42  * non-initialized entries will be by default not supported.
  43  */
  44 enum algo_supported {
  45         ALGO_NOT_SUPPORTED = 0,
  46         ALGO_SUPPORTED = 1,
  47 };
  48
  49 /** Map elements for cipher mapping.*/
  50 struct cipher_params_mapping {
  51         enum algo_supported  supported;   /**< On/Off switch */
  52         enum sam_cipher_alg  cipher_alg;  /**< Cipher algorithm */
  53         enum sam_cipher_mode cipher_mode; /**< Cipher mode */
  54         unsigned int max_key_len;         /**< Maximum key length (in bytes)*/
  55 }
  56 /* We want to squeeze in multiple maps into the cache line. */
  57 __rte_aligned(32);
  58
  59 /** Map elements for auth mapping.*/
  60 struct auth_params_mapping {
  61         enum algo_supported supported;  /**< On/off switch */
  62         enum sam_auth_alg   auth_alg;   /**< Auth algorithm */
  63 }
  64 /* We want to squeeze in multiple maps into the cache line. */
  65 __rte_aligned(32);
  66
  67 /**
  68  * Map of supported cipher algorithms.
  69  */
  70 static const
  71 struct cipher_params_mapping cipher_map[RTE_CRYPTO_CIPHER_LIST_END] = {
  72         [RTE_CRYPTO_CIPHER_NULL] = {
  73                 .supported = ALGO_SUPPORTED,
  74                 .cipher_alg = SAM_CIPHER_NONE },
  75         [RTE_CRYPTO_CIPHER_3DES_CBC] = {
  76                 .supported = ALGO_SUPPORTED,
  77                 .cipher_alg = SAM_CIPHER_3DES,
  78                 .cipher_mode = SAM_CIPHER_CBC,
  79                 .max_key_len = BITS2BYTES(192) },
  80         [RTE_CRYPTO_CIPHER_3DES_CTR] = {
  81                 .supported = ALGO_SUPPORTED,
  82                 .cipher_alg = SAM_CIPHER_3DES,
  83                 .cipher_mode = SAM_CIPHER_CTR,
  84                 .max_key_len = BITS2BYTES(192) },
  85         [RTE_CRYPTO_CIPHER_3DES_ECB] = {
  86                 .supported = ALGO_SUPPORTED,
  87                 .cipher_alg = SAM_CIPHER_3DES,
  88                 .cipher_mode = SAM_CIPHER_ECB,
  89                 .max_key_len = BITS2BYTES(192) },
  90         [RTE_CRYPTO_CIPHER_AES_CBC] = {
  91                 .supported = ALGO_SUPPORTED,
  92                 .cipher_alg = SAM_CIPHER_AES,
  93                 .cipher_mode = SAM_CIPHER_CBC,
  94                 .max_key_len = BITS2BYTES(256) },
  95         [RTE_CRYPTO_CIPHER_AES_CTR] = {
  96                 .supported = ALGO_SUPPORTED,
  97                 .cipher_alg = SAM_CIPHER_AES,
  98                 .cipher_mode = SAM_CIPHER_CTR,
  99                 .max_key_len = BITS2BYTES(256) },
 100         [RTE_CRYPTO_CIPHER_AES_ECB] = {
 101                 .supported = ALGO_SUPPORTED,
 102                 .cipher_alg = SAM_CIPHER_AES,
 103                 .cipher_mode = SAM_CIPHER_ECB,
 104                 .max_key_len = BITS2BYTES(256) },
 105 };
 106
 107 /**
 108  * Map of supported auth algorithms.
 109  */
 110 static const
 111 struct auth_params_mapping auth_map[RTE_CRYPTO_AUTH_LIST_END] = {
 112         [RTE_CRYPTO_AUTH_NULL] = {
 113                 .supported = ALGO_SUPPORTED,
 114                 .auth_alg = SAM_AUTH_NONE },
 115         [RTE_CRYPTO_AUTH_MD5_HMAC] = {
 116                 .supported = ALGO_SUPPORTED,
 117                 .auth_alg = SAM_AUTH_HMAC_MD5 },
 118         [RTE_CRYPTO_AUTH_MD5] = {
 119                 .supported = ALGO_SUPPORTED,
 120                 .auth_alg = SAM_AUTH_HASH_MD5 },
 121         [RTE_CRYPTO_AUTH_SHA1_HMAC] = {
 122                 .supported = ALGO_SUPPORTED,
 123                 .auth_alg = SAM_AUTH_HMAC_SHA1 },
 124         [RTE_CRYPTO_AUTH_SHA1] = {
 125                 .supported = ALGO_SUPPORTED,
 126                 .auth_alg = SAM_AUTH_HASH_SHA1 },
 127         [RTE_CRYPTO_AUTH_SHA224_HMAC] = {
 128                 .supported = ALGO_SUPPORTED,
 129                 .auth_alg = SAM_AUTH_HMAC_SHA2_224 },
 130         [RTE_CRYPTO_AUTH_SHA224] = {
 131                 .supported = ALGO_SUPPORTED,
 132                 .auth_alg = SAM_AUTH_HASH_SHA2_224 },
 133         [RTE_CRYPTO_AUTH_SHA256_HMAC] = {
 134                 .supported = ALGO_SUPPORTED,
 135                 .auth_alg = SAM_AUTH_HMAC_SHA2_256 },
 136         [RTE_CRYPTO_AUTH_SHA256] = {
 137                 .supported = ALGO_SUPPORTED,
 138                 .auth_alg = SAM_AUTH_HASH_SHA2_256 },
 139         [RTE_CRYPTO_AUTH_SHA384_HMAC] = {
 140                 .supported = ALGO_SUPPORTED,
 141                 .auth_alg = SAM_AUTH_HMAC_SHA2_384 },
 142         [RTE_CRYPTO_AUTH_SHA384] = {
 143                 .supported = ALGO_SUPPORTED,
 144                 .auth_alg = SAM_AUTH_HASH_SHA2_384 },
 145         [RTE_CRYPTO_AUTH_SHA512_HMAC] = {
 146                 .supported = ALGO_SUPPORTED,
 147                 .auth_alg = SAM_AUTH_HMAC_SHA2_512 },
 148         [RTE_CRYPTO_AUTH_SHA512] = {
 149                 .supported = ALGO_SUPPORTED,
 150                 .auth_alg = SAM_AUTH_HASH_SHA2_512 },
 151         [RTE_CRYPTO_AUTH_AES_GMAC] = {
 152                 .supported = ALGO_SUPPORTED,
 153                 .auth_alg = SAM_AUTH_AES_GMAC },
 154 };
 155
 156 /**
 157  * Map of supported aead algorithms.
 158  */
 159 static const
 160 struct cipher_params_mapping aead_map[RTE_CRYPTO_AEAD_LIST_END] = {
 161         [RTE_CRYPTO_AEAD_AES_GCM] = {
 162                 .supported = ALGO_SUPPORTED,
 163                 .cipher_alg = SAM_CIPHER_AES,
 164                 .cipher_mode = SAM_CIPHER_GCM,
 165                 .max_key_len = BITS2BYTES(256) },
 166 };
 167
 168 /*
 169  *-----------------------------------------------------------------------------
 170  * Forward declarations.
 171  *-----------------------------------------------------------------------------
 172  */
 173 static int cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev);
 174
 175 /*
 176  *-----------------------------------------------------------------------------
 177  * Session Preparation.
 178  *-----------------------------------------------------------------------------
 179  */
 180
 181 /**
 182  * Get xform chain order.
 183  *
 184  * @param xform Pointer to configuration structure chain for crypto operations.
 185  * @returns Order of crypto operations.
 186  */
 187 static enum mrvl_crypto_chain_order
 188 mrvl_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform)
 189 {
 190         /* Currently, Marvell supports max 2 operations in chain */
 191         if (xform->next != NULL && xform->next->next != NULL)
 192                 return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED;
 193
 194         if (xform->next != NULL) {
 195                 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
 196                         (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER))
 197                         return MRVL_CRYPTO_CHAIN_AUTH_CIPHER;
 198
 199                 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
 200                         (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH))
 201                         return MRVL_CRYPTO_CHAIN_CIPHER_AUTH;
 202         } else {
 203                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 204                         return MRVL_CRYPTO_CHAIN_AUTH_ONLY;
 205
 206                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 207                         return MRVL_CRYPTO_CHAIN_CIPHER_ONLY;
 208
 209                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 210                         return MRVL_CRYPTO_CHAIN_COMBINED;
 211         }
 212         return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED;
 213 }
 214
 215 /**
 216  * Set session parameters for cipher part.
 217  *
 218  * @param sess Crypto session pointer.
 219  * @param cipher_xform Pointer to configuration structure for cipher operations.
 220  * @returns 0 in case of success, negative value otherwise.
 221  */
 222 static int
 223 mrvl_crypto_set_cipher_session_parameters(struct mrvl_crypto_session *sess,
 224                 const struct rte_crypto_sym_xform *cipher_xform)
 225 {
 226         uint8_t *cipher_key;
 227
 228         /* Make sure we've got proper struct */
 229         if (cipher_xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
 230                 MRVL_LOG(ERR, "Wrong xform struct provided!");
 231                 return -EINVAL;
 232         }
 233
 234         /* See if map data is present and valid */
 235         if ((cipher_xform->cipher.algo > RTE_DIM(cipher_map)) ||
 236                 (cipher_map[cipher_xform->cipher.algo].supported
 237                         != ALGO_SUPPORTED)) {
 238                 MRVL_LOG(ERR, "Cipher algorithm not supported!");
 239                 return -EINVAL;
 240         }
 241
 242         sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
 243
 244         sess->sam_sess_params.dir =
 245                 (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
 246                 SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT;
 247         sess->sam_sess_params.cipher_alg =
 248                 cipher_map[cipher_xform->cipher.algo].cipher_alg;
 249         sess->sam_sess_params.cipher_mode =
 250                 cipher_map[cipher_xform->cipher.algo].cipher_mode;
 251
 252         /* Assume IV will be passed together with data. */
 253         sess->sam_sess_params.cipher_iv = NULL;
 254
 255         /* Get max key length. */
 256         if (cipher_xform->cipher.key.length >
 257                 cipher_map[cipher_xform->cipher.algo].max_key_len) {
 258                 MRVL_LOG(ERR, "Wrong key length!");
 259                 return -EINVAL;
 260         }
 261
 262         cipher_key = malloc(cipher_xform->cipher.key.length);
 263         if (cipher_key == NULL) {
 264                 MRVL_LOG(ERR, "Insufficient memory!");
 265                 return -ENOMEM;
 266         }
 267
 268         memcpy(cipher_key, cipher_xform->cipher.key.data,
 269                         cipher_xform->cipher.key.length);
 270
 271         sess->sam_sess_params.cipher_key_len = cipher_xform->cipher.key.length;
 272         sess->sam_sess_params.cipher_key = cipher_key;
 273
 274         return 0;
 275 }
 276
 277 /**
 278  * Set session parameters for authentication part.
 279  *
 280  * @param sess Crypto session pointer.
 281  * @param auth_xform Pointer to configuration structure for auth operations.
 282  * @returns 0 in case of success, negative value otherwise.
 283  */
 284 static int
 285 mrvl_crypto_set_auth_session_parameters(struct mrvl_crypto_session *sess,
 286                 const struct rte_crypto_sym_xform *auth_xform)
 287 {
 288         uint8_t *auth_key = NULL;
 289
 290         /* Make sure we've got proper struct */
 291         if (auth_xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
 292                 MRVL_LOG(ERR, "Wrong xform struct provided!");
 293                 return -EINVAL;
 294         }
 295
 296         /* See if map data is present and valid */
 297         if ((auth_xform->auth.algo > RTE_DIM(auth_map)) ||
 298                 (auth_map[auth_xform->auth.algo].supported != ALGO_SUPPORTED)) {
 299                 MRVL_LOG(ERR, "Auth algorithm not supported!");
 300                 return -EINVAL;
 301         }
 302
 303         sess->sam_sess_params.dir =
 304                 (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
 305                 SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT;
 306         sess->sam_sess_params.auth_alg =
 307                 auth_map[auth_xform->auth.algo].auth_alg;
 308         sess->sam_sess_params.u.basic.auth_icv_len =
 309                 auth_xform->auth.digest_length;
 310
 311         if (auth_xform->auth.key.length > 0) {
 312                 auth_key = malloc(auth_xform->auth.key.length);
 313                 if (auth_key == NULL) {
 314                         MRVL_LOG(ERR, "Not enough memory!");
 315                         return -EINVAL;
 316                 }
 317
 318                 memcpy(auth_key, auth_xform->auth.key.data,
 319                                 auth_xform->auth.key.length);
 320         }
 321
 322         /* auth_key must be NULL if auth algorithm does not use HMAC */
 323         sess->sam_sess_params.auth_key = auth_key;
 324         sess->sam_sess_params.auth_key_len = auth_xform->auth.key.length;
 325
 326         return 0;
 327 }
 328
 329 /**
 330  * Set session parameters for aead part.
 331  *
 332  * @param sess Crypto session pointer.
 333  * @param aead_xform Pointer to configuration structure for aead operations.
 334  * @returns 0 in case of success, negative value otherwise.
 335  */
 336 static int
 337 mrvl_crypto_set_aead_session_parameters(struct mrvl_crypto_session *sess,
 338                 const struct rte_crypto_sym_xform *aead_xform)
 339 {
 340         uint8_t *aead_key;
 341
 342         /* Make sure we've got proper struct */
 343         if (aead_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) {
 344                 MRVL_LOG(ERR, "Wrong xform struct provided!");
 345                 return -EINVAL;
 346         }
 347
 348         /* See if map data is present and valid */
 349         if ((aead_xform->aead.algo > RTE_DIM(aead_map)) ||
 350                 (aead_map[aead_xform->aead.algo].supported
 351                         != ALGO_SUPPORTED)) {
 352                 MRVL_LOG(ERR, "AEAD algorithm not supported!");
 353                 return -EINVAL;
 354         }
 355
 356         sess->sam_sess_params.dir =
 357                 (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
 358                 SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT;
 359         sess->sam_sess_params.cipher_alg =
 360                 aead_map[aead_xform->aead.algo].cipher_alg;
 361         sess->sam_sess_params.cipher_mode =
 362                 aead_map[aead_xform->aead.algo].cipher_mode;
 363
 364         /* Assume IV will be passed together with data. */
 365         sess->sam_sess_params.cipher_iv = NULL;
 366
 367         /* Get max key length. */
 368         if (aead_xform->aead.key.length >
 369                 aead_map[aead_xform->aead.algo].max_key_len) {
 370                 MRVL_LOG(ERR, "Wrong key length!");
 371                 return -EINVAL;
 372         }
 373
 374         aead_key = malloc(aead_xform->aead.key.length);
 375         if (aead_key == NULL) {
 376                 MRVL_LOG(ERR, "Insufficient memory!");
 377                 return -ENOMEM;
 378         }
 379
 380         memcpy(aead_key, aead_xform->aead.key.data,
 381                         aead_xform->aead.key.length);
 382
 383         sess->sam_sess_params.cipher_key = aead_key;
 384         sess->sam_sess_params.cipher_key_len = aead_xform->aead.key.length;
 385
 386         if (sess->sam_sess_params.cipher_mode == SAM_CIPHER_GCM)
 387                 sess->sam_sess_params.auth_alg = SAM_AUTH_AES_GCM;
 388
 389         sess->sam_sess_params.u.basic.auth_icv_len =
 390                 aead_xform->aead.digest_length;
 391
 392         sess->sam_sess_params.u.basic.auth_aad_len =
 393                 aead_xform->aead.aad_length;
 394
 395         return 0;
 396 }
 397
 398 /**
 399  * Parse crypto transform chain and setup session parameters.
 400  *
 401  * @param dev Pointer to crypto device
 402  * @param sess Poiner to crypto session
 403  * @param xform Pointer to configuration structure chain for crypto operations.
 404  * @returns 0 in case of success, negative value otherwise.
 405  */
 406 int
 407 mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess,
 408                 const struct rte_crypto_sym_xform *xform)
 409 {
 410         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 411         const struct rte_crypto_sym_xform *auth_xform = NULL;
 412         const struct rte_crypto_sym_xform *aead_xform = NULL;
 413
 414         /* Filter out spurious/broken requests */
 415         if (xform == NULL)
 416                 return -EINVAL;
 417
 418         sess->chain_order = mrvl_crypto_get_chain_order(xform);
 419         switch (sess->chain_order) {
 420         case MRVL_CRYPTO_CHAIN_CIPHER_AUTH:
 421                 cipher_xform = xform;
 422                 auth_xform = xform->next;
 423                 break;
 424         case MRVL_CRYPTO_CHAIN_AUTH_CIPHER:
 425                 auth_xform = xform;
 426                 cipher_xform = xform->next;
 427                 break;
 428         case MRVL_CRYPTO_CHAIN_CIPHER_ONLY:
 429                 cipher_xform = xform;
 430                 break;
 431         case MRVL_CRYPTO_CHAIN_AUTH_ONLY:
 432                 auth_xform = xform;
 433                 break;
 434         case MRVL_CRYPTO_CHAIN_COMBINED:
 435                 aead_xform = xform;
 436                 break;
 437         default:
 438                 return -EINVAL;
 439         }
 440
 441         if ((cipher_xform != NULL) &&
 442                 (mrvl_crypto_set_cipher_session_parameters(
 443                         sess, cipher_xform) < 0)) {
 444                 MRVL_LOG(ERR, "Invalid/unsupported cipher parameters!");
 445                 return -EINVAL;
 446         }
 447
 448         if ((auth_xform != NULL) &&
 449                 (mrvl_crypto_set_auth_session_parameters(
 450                         sess, auth_xform) < 0)) {
 451                 MRVL_LOG(ERR, "Invalid/unsupported auth parameters!");
 452                 return -EINVAL;
 453         }
 454
 455         if ((aead_xform != NULL) &&
 456                 (mrvl_crypto_set_aead_session_parameters(
 457                         sess, aead_xform) < 0)) {
 458                 MRVL_LOG(ERR, "Invalid/unsupported aead parameters!");
 459                 return -EINVAL;
 460         }
 461
 462         return 0;
 463 }
 464
 465 /*
 466  *-----------------------------------------------------------------------------
 467  * Process Operations
 468  *-----------------------------------------------------------------------------
 469  */
 470
 471 /**
 472  * Prepare a single request.
 473  *
 474  * This function basically translates DPDK crypto request into one
 475  * understandable by MUDSK's SAM. If this is a first request in a session,
 476  * it starts the session.
 477  *
 478  * @param request Pointer to pre-allocated && reset request buffer [Out].
 479  * @param src_bd Pointer to pre-allocated source descriptor [Out].
 480  * @param dst_bd Pointer to pre-allocated destination descriptor [Out].
 481  * @param op Pointer to DPDK crypto operation struct [In].
 482  */
 483 static inline int
 484 mrvl_request_prepare(struct sam_cio_op_params *request,
 485                 struct sam_buf_info *src_bd,
 486                 struct sam_buf_info *dst_bd,
 487                 struct rte_crypto_op *op)
 488 {
 489         struct mrvl_crypto_session *sess;
 490         struct rte_mbuf *src_mbuf, *dst_mbuf;
 491         uint16_t segments_nb;
 492         uint8_t *digest;
 493         int i;
 494
 495         if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
 496                 MRVL_LOG(ERR, "MRVL CRYPTO PMD only supports session "
 497                                 "oriented requests, op (%p) is sessionless!",
 498                                 op);
 499                 return -EINVAL;
 500         }
 501
 502         sess = (struct mrvl_crypto_session *)get_sym_session_private_data(
 503                         op->sym->session, cryptodev_driver_id);
 504         if (unlikely(sess == NULL)) {
 505                 MRVL_LOG(ERR, "Session was not created for this device!");
 506                 return -EINVAL;
 507         }
 508
 509         request->sa = sess->sam_sess;
 510         request->cookie = op;
 511
 512         src_mbuf = op->sym->m_src;
 513         segments_nb = src_mbuf->nb_segs;
 514         /* The following conditions must be met:
 515          * - Destination buffer is required when segmented source buffer
 516          * - Segmented destination buffer is not supported
 517          */
 518         if ((segments_nb > 1) && (!op->sym->m_dst)) {
 519                 MRVL_LOG(ERR, "op->sym->m_dst = NULL!");
 520                 return -1;
 521         }
 522         /* For non SG case:
 523          * If application delivered us null dst buffer, it means it expects
 524          * us to deliver the result in src buffer.
 525          */
 526         dst_mbuf = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
 527
 528         if (!rte_pktmbuf_is_contiguous(dst_mbuf)) {
 529                 MRVL_LOG(ERR, "Segmented destination buffer not supported!");
 530                 return -1;
 531         }
 532
 533         request->num_bufs = segments_nb;
 534         for (i = 0; i < segments_nb; i++) {
 535                 /* Empty source. */
 536                 if (rte_pktmbuf_data_len(src_mbuf) == 0) {
 537                         /* EIP does not support 0 length buffers. */
 538                         MRVL_LOG(ERR, "Buffer length == 0 not supported!");
 539                         return -1;
 540                 }
 541                 src_bd[i].vaddr = rte_pktmbuf_mtod(src_mbuf, void *);
 542                 src_bd[i].paddr = rte_pktmbuf_iova(src_mbuf);
 543                 src_bd[i].len = rte_pktmbuf_data_len(src_mbuf);
 544
 545                 src_mbuf = src_mbuf->next;
 546         }
 547         request->src = src_bd;
 548
 549         /* Empty destination. */
 550         if (rte_pktmbuf_data_len(dst_mbuf) == 0) {
 551                 /* Make dst buffer fit at least source data. */
 552                 if (rte_pktmbuf_append(dst_mbuf,
 553                         rte_pktmbuf_data_len(op->sym->m_src)) == NULL) {
 554                         MRVL_LOG(ERR, "Unable to set big enough dst buffer!");
 555                         return -1;
 556                 }
 557         }
 558
 559         request->dst = dst_bd;
 560         dst_bd->vaddr = rte_pktmbuf_mtod(dst_mbuf, void *);
 561         dst_bd->paddr = rte_pktmbuf_iova(dst_mbuf);
 562
 563         /*
 564          * We can use all available space in dst_mbuf,
 565          * not only what's used currently.
 566          */
 567         dst_bd->len = dst_mbuf->buf_len - rte_pktmbuf_headroom(dst_mbuf);
 568
 569         if (sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED) {
 570                 request->cipher_len = op->sym->aead.data.length;
 571                 request->cipher_offset = op->sym->aead.data.offset;
 572                 request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 573                         sess->cipher_iv_offset);
 574
 575                 request->auth_aad = op->sym->aead.aad.data;
 576                 request->auth_offset = request->cipher_offset;
 577                 request->auth_len = request->cipher_len;
 578         } else {
 579                 request->cipher_len = op->sym->cipher.data.length;
 580                 request->cipher_offset = op->sym->cipher.data.offset;
 581                 request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 582                                 sess->cipher_iv_offset);
 583
 584                 request->auth_offset = op->sym->auth.data.offset;
 585                 request->auth_len = op->sym->auth.data.length;
 586         }
 587
 588         digest = sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED ?
 589                 op->sym->aead.digest.data : op->sym->auth.digest.data;
 590         if (digest == NULL) {
 591                 /* No auth - no worry. */
 592                 return 0;
 593         }
 594
 595         request->auth_icv_offset = request->auth_offset + request->auth_len;
 596
 597         /*
 598          * EIP supports only scenarios where ICV(digest buffer) is placed at
 599          * auth_icv_offset.
 600          */
 601         if (sess->sam_sess_params.dir == SAM_DIR_ENCRYPT) {
 602                 /*
 603                  * This should be the most common case anyway,
 604                  * EIP will overwrite DST buffer at auth_icv_offset.
 605                  */
 606                 if (rte_pktmbuf_mtod_offset(
 607                                 dst_mbuf, uint8_t *,
 608                                 request->auth_icv_offset) == digest)
 609                         return 0;
 610         } else {/* sess->sam_sess_params.dir == SAM_DIR_DECRYPT */
 611                 /*
 612                  * EIP will look for digest at auth_icv_offset
 613                  * offset in SRC buffer. It must be placed in the last
 614                  * segment and the offset must be set to reach digest
 615                  * in the last segment
 616                  */
 617                 struct rte_mbuf *last_seg =  op->sym->m_src;
 618                 uint32_t d_offset = request->auth_icv_offset;
 619                 u32 d_size = sess->sam_sess_params.u.basic.auth_icv_len;
 620                 unsigned char *d_ptr;
 621
 622                 /* Find the last segment and the offset for the last segment */
 623                 while ((last_seg->next != NULL) &&
 624                                 (d_offset >= last_seg->data_len)) {
 625                         d_offset -= last_seg->data_len;
 626                         last_seg = last_seg->next;
 627                 }
 628
 629                 if (rte_pktmbuf_mtod_offset(last_seg, uint8_t *,
 630                                             d_offset) == digest)
 631                         return 0;
 632
 633                 /* copy digest to last segment */
 634                 if (last_seg->buf_len >= (d_size + d_offset)) {
 635                         d_ptr = (unsigned char *)last_seg->buf_addr +
 636                                  d_offset;
 637                         rte_memcpy(d_ptr, digest, d_size);
 638                         return 0;
 639                 }
 640         }
 641
 642         /*
 643          * If we landed here it means that digest pointer is
 644          * at different than expected place.
 645          */
 646         return -1;
 647 }
 648
 649 /*
 650  *-----------------------------------------------------------------------------
 651  * PMD Framework handlers
 652  *-----------------------------------------------------------------------------
 653  */
 654
 655 /**
 656  * Enqueue burst.
 657  *
 658  * @param queue_pair Pointer to queue pair.
 659  * @param ops Pointer to ops requests array.
 660  * @param nb_ops Number of elements in ops requests array.
 661  * @returns Number of elements consumed from ops.
 662  */
 663 static uint16_t
 664 mrvl_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 665                 uint16_t nb_ops)
 666 {
 667         uint16_t iter_ops = 0;
 668         uint16_t to_enq = 0;
 669         uint16_t consumed = 0;
 670         int ret;
 671         struct sam_cio_op_params requests[nb_ops];
 672         /*
 673          * SAM does not store bd pointers, so on-stack scope will be enough.
 674          */
 675         struct mrvl_crypto_src_table src_bd[nb_ops];
 676         struct sam_buf_info          dst_bd[nb_ops];
 677         struct mrvl_crypto_qp *qp = (struct mrvl_crypto_qp *)queue_pair;
 678
 679         if (nb_ops == 0)
 680                 return 0;
 681
 682         /* Prepare the burst. */
 683         memset(&requests, 0, sizeof(requests));
 684         memset(&src_bd, 0, sizeof(src_bd));
 685
 686         /* Iterate through */
 687         for (; iter_ops < nb_ops; ++iter_ops) {
 688                 /* store the op id for debug */
 689                 src_bd[iter_ops].iter_ops = iter_ops;
 690                 if (mrvl_request_prepare(&requests[iter_ops],
 691                                         src_bd[iter_ops].src_bd,
 692                                         &dst_bd[iter_ops],
 693                                         ops[iter_ops]) < 0) {
 694                         MRVL_LOG(ERR, "Error while preparing parameters!");
 695                         qp->stats.enqueue_err_count++;
 696                         ops[iter_ops]->status = RTE_CRYPTO_OP_STATUS_ERROR;
 697
 698                         /*
 699                          * Number of handled ops is increased
 700                          * (even if the result of handling is error).
 701                          */
 702                         ++consumed;
 703                         break;
 704                 }
 705
 706                 ops[iter_ops]->status =
 707                         RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 708
 709                 /* Increase the number of ops to enqueue. */
 710                 ++to_enq;
 711         } /* for (; iter_ops < nb_ops;... */
 712
 713         if (to_enq > 0) {
 714                 /* Send the burst */
 715                 ret = sam_cio_enq(qp->cio, requests, &to_enq);
 716                 consumed += to_enq;
 717                 if (ret < 0) {
 718                         /*
 719                          * Trust SAM that in this case returned value will be at
 720                          * some point correct (now it is returned unmodified).
 721                          */
 722                         qp->stats.enqueue_err_count += to_enq;
 723                         for (iter_ops = 0; iter_ops < to_enq; ++iter_ops)
 724                                 ops[iter_ops]->status =
 725                                         RTE_CRYPTO_OP_STATUS_ERROR;
 726                 }
 727         }
 728
 729         qp->stats.enqueued_count += to_enq;
 730         return consumed;
 731 }
 732
 733 /**
 734  * Dequeue burst.
 735  *
 736  * @param queue_pair Pointer to queue pair.
 737  * @param ops Pointer to ops requests array.
 738  * @param nb_ops Number of elements in ops requests array.
 739  * @returns Number of elements dequeued.
 740  */
 741 static uint16_t
 742 mrvl_crypto_pmd_dequeue_burst(void *queue_pair,
 743                 struct rte_crypto_op **ops,
 744                 uint16_t nb_ops)
 745 {
 746         int ret;
 747         struct mrvl_crypto_qp *qp = queue_pair;
 748         struct sam_cio *cio = qp->cio;
 749         struct sam_cio_op_result results[nb_ops];
 750         uint16_t i;
 751
 752         ret = sam_cio_deq(cio, results, &nb_ops);
 753         if (ret < 0) {
 754                 /* Count all dequeued as error. */
 755                 qp->stats.dequeue_err_count += nb_ops;
 756
 757                 /* But act as they were dequeued anyway*/
 758                 qp->stats.dequeued_count += nb_ops;
 759
 760                 return 0;
 761         }
 762
 763         /* Unpack and check results. */
 764         for (i = 0; i < nb_ops; ++i) {
 765                 ops[i] = results[i].cookie;
 766
 767                 switch (results[i].status) {
 768                 case SAM_CIO_OK:
 769                         ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 770                         break;
 771                 case SAM_CIO_ERR_ICV:
 772                         MRVL_LOG(DEBUG, "CIO returned SAM_CIO_ERR_ICV.");
 773                         ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 774                         break;
 775                 default:
 776                         MRVL_LOG(DEBUG,
 777                                 "CIO returned Error: %d.", results[i].status);
 778                         ops[i]->status = RTE_CRYPTO_OP_STATUS_ERROR;
 779                         break;
 780                 }
 781         }
 782
 783         qp->stats.dequeued_count += nb_ops;
 784         return nb_ops;
 785 }
 786
 787 /**
 788  * Create a new crypto device.
 789  *
 790  * @param name Driver name.
 791  * @param vdev Pointer to device structure.
 792  * @param init_params Pointer to initialization parameters.
 793  * @returns 0 in case of success, negative value otherwise.
 794  */
 795 static int
 796 cryptodev_mrvl_crypto_create(const char *name,
 797                 struct rte_vdev_device *vdev,
 798                 struct mrvl_pmd_init_params *init_params)
 799 {
 800         struct rte_cryptodev *dev;
 801         struct mrvl_crypto_private *internals;
 802         struct sam_init_params  sam_params;
 803         int ret = -EINVAL;
 804
 805         dev = rte_cryptodev_pmd_create(name, &vdev->device,
 806                         &init_params->common);
 807         if (dev == NULL) {
 808                 MRVL_LOG(ERR, "Failed to create cryptodev vdev!");
 809                 goto init_error;
 810         }
 811
 812         dev->driver_id = cryptodev_driver_id;
 813         dev->dev_ops = rte_mrvl_crypto_pmd_ops;
 814
 815         /* Register rx/tx burst functions for data path. */
 816         dev->enqueue_burst = mrvl_crypto_pmd_enqueue_burst;
 817         dev->dequeue_burst = mrvl_crypto_pmd_dequeue_burst;
 818
 819         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 820                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 821                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
 822                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 823                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 824
 825         internals = dev->data->dev_private;
 826
 827         internals->max_nb_qpairs = init_params->common.max_nb_queue_pairs;
 828         internals->max_nb_sessions = init_params->max_nb_sessions;
 829
 830         ret = rte_mvep_init(MVEP_MOD_T_SAM, NULL);
 831         if (ret)
 832                 goto init_error;
 833
 834         sam_params.max_num_sessions = internals->max_nb_sessions;
 835
 836         /* sam_set_debug_flags(3); */
 837
 838         ret = sam_init(&sam_params);
 839         if (ret)
 840                 goto init_error;
 841
 842         return 0;
 843
 844 init_error:
 845         MRVL_LOG(ERR,
 846                 "Driver %s: %s failed!", init_params->common.name, __func__);
 847
 848         cryptodev_mrvl_crypto_uninit(vdev);
 849         return ret;
 850 }
 851
 852 /** Parse integer from integer argument */
 853 static int
 854 parse_integer_arg(const char *key __rte_unused,
 855                 const char *value, void *extra_args)
 856 {
 857         int *i = (int *) extra_args;
 858
 859         *i = atoi(value);
 860         if (*i < 0) {
 861                 MRVL_LOG(ERR, "Argument has to be positive!");
 862                 return -EINVAL;
 863         }
 864
 865         return 0;
 866 }
 867
 868 /** Parse name */
 869 static int
 870 parse_name_arg(const char *key __rte_unused,
 871                 const char *value, void *extra_args)
 872 {
 873         struct rte_cryptodev_pmd_init_params *params = extra_args;
 874
 875         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 876                 MRVL_LOG(ERR, "Invalid name %s, should be less than %u bytes!",
 877                          value, RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 878                 return -EINVAL;
 879         }
 880
 881         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 882
 883         return 0;
 884 }
 885
 886 static int
 887 mrvl_pmd_parse_input_args(struct mrvl_pmd_init_params *params,
 888                          const char *input_args)
 889 {
 890         struct rte_kvargs *kvlist = NULL;
 891         int ret = 0;
 892
 893         if (params == NULL)
 894                 return -EINVAL;
 895
 896         if (input_args) {
 897                 kvlist = rte_kvargs_parse(input_args,
 898                                           mrvl_pmd_valid_params);
 899                 if (kvlist == NULL)
 900                         return -1;
 901
 902                 /* Common VDEV parameters */
 903                 ret = rte_kvargs_process(kvlist,
 904                                          RTE_CRYPTODEV_PMD_MAX_NB_QP_ARG,
 905                                          &parse_integer_arg,
 906                                          &params->common.max_nb_queue_pairs);
 907                 if (ret < 0)
 908                         goto free_kvlist;
 909
 910                 ret = rte_kvargs_process(kvlist,
 911                                          RTE_CRYPTODEV_PMD_SOCKET_ID_ARG,
 912                                          &parse_integer_arg,
 913                                          &params->common.socket_id);
 914                 if (ret < 0)
 915                         goto free_kvlist;
 916
 917                 ret = rte_kvargs_process(kvlist,
 918                                          RTE_CRYPTODEV_PMD_NAME_ARG,
 919                                          &parse_name_arg,
 920                                          &params->common);
 921                 if (ret < 0)
 922                         goto free_kvlist;
 923
 924                 ret = rte_kvargs_process(kvlist,
 925                                          MRVL_PMD_MAX_NB_SESS_ARG,
 926                                          &parse_integer_arg,
 927                                          params);
 928                 if (ret < 0)
 929                         goto free_kvlist;
 930
 931         }
 932
 933 free_kvlist:
 934         rte_kvargs_free(kvlist);
 935         return ret;
 936 }
 937
 938 /**
 939  * Initialize the crypto device.
 940  *
 941  * @param vdev Pointer to device structure.
 942  * @returns 0 in case of success, negative value otherwise.
 943  */
 944 static int
 945 cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev)
 946 {
 947         struct mrvl_pmd_init_params init_params = {
 948                 .common = {
 949                         .name = "",
 950                         .private_data_size =
 951                                 sizeof(struct mrvl_crypto_private),
 952                         .max_nb_queue_pairs =
 953                                 sam_get_num_inst() * sam_get_num_cios(0),
 954                         .socket_id = rte_socket_id()
 955                 },
 956                 .max_nb_sessions = MRVL_PMD_DEFAULT_MAX_NB_SESSIONS
 957         };
 958
 959         const char *name, *args;
 960         int ret;
 961
 962         name = rte_vdev_device_name(vdev);
 963         if (name == NULL)
 964                 return -EINVAL;
 965         args = rte_vdev_device_args(vdev);
 966
 967         ret = mrvl_pmd_parse_input_args(&init_params, args);
 968         if (ret) {
 969                 MRVL_LOG(ERR, "Failed to parse initialisation arguments[%s]!",
 970                          args);
 971                 return -EINVAL;
 972         }
 973
 974         return cryptodev_mrvl_crypto_create(name, vdev, &init_params);
 975 }
 976
 977 /**
 978  * Uninitialize the crypto device
 979  *
 980  * @param vdev Pointer to device structure.
 981  * @returns 0 in case of success, negative value otherwise.
 982  */
 983 static int
 984 cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev)
 985 {
 986         struct rte_cryptodev *cryptodev;
 987         const char *name = rte_vdev_device_name(vdev);
 988
 989         if (name == NULL)
 990                 return -EINVAL;
 991
 992         MRVL_LOG(INFO, "Closing Marvell crypto device %s on numa socket %u.",
 993                  name, rte_socket_id());
 994
 995         sam_deinit();
 996         rte_mvep_deinit(MVEP_MOD_T_SAM);
 997
 998         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
 999         if (cryptodev == NULL)
1000                 return -ENODEV;
1001
1002         return rte_cryptodev_pmd_destroy(cryptodev);
1003 }
1004
1005 /**
1006  * Basic driver handlers for use in the constructor.
1007  */
1008 static struct rte_vdev_driver cryptodev_mrvl_pmd_drv = {
1009         .probe = cryptodev_mrvl_crypto_init,
1010         .remove = cryptodev_mrvl_crypto_uninit
1011 };
1012
1013 static struct cryptodev_driver mrvl_crypto_drv;
1014
1015 /* Register the driver in constructor. */
1016 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_MRVL_PMD, cryptodev_mrvl_pmd_drv);
1017 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_MRVL_PMD,
1018         "max_nb_queue_pairs=<int> "
1019         "max_nb_sessions=<int> "
1020         "socket_id=<int>");
1021 RTE_PMD_REGISTER_CRYPTO_DRIVER(mrvl_crypto_drv, cryptodev_mrvl_pmd_drv.driver,
1022                 cryptodev_driver_id);
1023
1024 RTE_INIT(crypto_mrvl_init_log)
1025 {
1026         mrvl_logtype_driver = rte_log_register("pmd.crypto.mvsam");
1027 }