crypto/mrvl: add mrvl crypto driver
authorTomasz Duszynski <tdu@semihalf.com>
Tue, 10 Oct 2017 12:17:19 +0000 (14:17 +0200)
committerPablo de Lara <pablo.de.lara.guarch@intel.com>
Thu, 12 Oct 2017 14:22:39 +0000 (15:22 +0100)
Add support for the Marvell Security Crypto Accelerator EIP197.
Driver is based on external, publicly available, Marvell MUSDK
library that provides access to the hardware with minimum overhead
and high performance.

Driver comes with support for the following features:

* Symmetric crypto
* Sym operation chaining
* AES CBC (128)
* AES CBC (192)
* AES CBC (256)
* AES CTR (128)
* AES CTR (192)
* AES CTR (256)
* 3DES CBC
* 3DES CTR
* MD5
* MD5 HMAC
* SHA1
* SHA1 HMAC
* SHA256
* SHA256 HMAC
* SHA384
* SHA384 HMAC
* SHA512
* SHA512 HMAC
* AES GCM (128)

Driver was engineered cooperatively by Semihalf and Marvell teams.

Semihalf:
Jacek Siuda <jck@semihalf.com>
Tomasz Duszynski <tdu@semihalf.com>

Marvell:
Dmitri Epshtein <dima@marvell.com>
Natalie Samsonov <nsamsono@marvell.com>

Signed-off-by: Jacek Siuda <jck@semihalf.com>
Signed-off-by: Tomasz Duszynski <tdu@semihalf.com>
13 files changed:
MAINTAINERS
config/common_base
devtools/test-build.sh
doc/guides/rel_notes/release_17_11.rst
doc/guides/tools/cryptoperf.rst
drivers/crypto/Makefile
drivers/crypto/mrvl/Makefile [new file with mode: 0644]
drivers/crypto/mrvl/rte_mrvl_compat.h [new file with mode: 0644]
drivers/crypto/mrvl/rte_mrvl_pmd.c [new file with mode: 0644]
drivers/crypto/mrvl/rte_mrvl_pmd_ops.c [new file with mode: 0644]
drivers/crypto/mrvl/rte_mrvl_pmd_private.h [new file with mode: 0644]
drivers/crypto/mrvl/rte_pmd_mrvl_version.map [new file with mode: 0644]
mk/rte.app.mk

index 206afda..0b1bf96 100644 (file)
@@ -576,6 +576,16 @@ F: drivers/crypto/qat/
 F: doc/guides/cryptodevs/qat.rst
 F: doc/guides/cryptodevs/features/qat.ini
 
+Marvell Mrvl
+M: Jacek Siuda <jck@semihalf.com>
+M: Tomasz Duszynski <tdu@semihalf.com>
+M: Dmitri Epshtein <dima@marvell.com>
+M: Natalie Samsonov <nsamsono@marvell.com>
+M: Jianbo Liu <jianbo.liu@arm.org>
+F: drivers/crypto/mrvl/
+F: doc/guides/cryptodevs/mrvl.rst
+F: doc/guides/cryptodevs/features/mrvl.ini
+
 NXP DPAA_SEC
 M: Akhil Goyal <akhil.goyal@nxp.com>
 M: Hemant Agrawal <hemant.agrawal@nxp.com>
index ee40ce5..6b9e7da 100644 (file)
@@ -542,6 +542,12 @@ CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER_DEBUG=n
 #
 CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO=y
 
+#
+# Compile PMD for Marvell Crypto device
+#
+CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO=n
+CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG=n
+
 #
 # Compile generic event device library
 #
index c6dfaf0..0242d50 100755 (executable)
@@ -47,6 +47,7 @@ default_path=$PATH
 # - DPDK_DEP_ZLIB (y/[n])
 # - DPDK_MAKE_JOBS (int)
 # - DPDK_NOTIFY (notify-send)
+# - LIBMUSDK_PATH
 # - LIBSSO_SNOW3G_PATH
 # - LIBSSO_KASUMI_PATH
 # - LIBSSO_ZUC_PATH
@@ -129,6 +130,7 @@ reset_env ()
        unset DPDK_DEP_ZLIB
        unset AESNI_MULTI_BUFFER_LIB_PATH
        unset ARMV8_CRYPTO_LIB_PATH
+       unset LIBMUSDK_PATH
        unset LIBSSO_SNOW3G_PATH
        unset LIBSSO_KASUMI_PATH
        unset LIBSSO_ZUC_PATH
@@ -193,6 +195,8 @@ config () # <directory> <target> <options>
                test "$DPDK_DEP_SSL" != y || \
                sed -ri            's,(PMD_QAT=)n,\1y,' $1/.config
                sed -ri           's,(SCHED_.*=)n,\1y,' $1/.config
+               test -z "$LIBMUSDK_PATH" || \
+               sed -ri    's,(PMD_MRVL_CRYPTO=)n,\1y,' $1/.config
                build_config_hook $1 $2 $3
 
                # Explicit enabler/disabler (uppercase)
index 092c3a1..8db35f5 100644 (file)
@@ -116,6 +116,11 @@ New Features
   added. See the "Crypto Device Drivers" document for more details on this
   driver.
 
+* **Added MRVL crypto PMD.**
+
+  A new crypto PMD has been added, which provides several ciphering and hashing
+  algorithms. All cryptography operations use the MUSDK library crypto API.
+
 * **Add new benchmarking mode to dpdk-test-crypto-perf application.**
 
   Added new "PMD cyclecount" benchmark mode to dpdk-test-crypto-perf application
index 2b3a5b6..7e12677 100644 (file)
@@ -194,6 +194,7 @@ The following are the appication command-line options:
            crypto_dpaa2_sec
            crypto_armv8
            crypto_scheduler
+           crypto_mrvl
 
 * ``--optype <name>``
 
index 8ca5f18..d8c8740 100644 (file)
@@ -51,6 +51,8 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += kasumi
 DEPDIRS-kasumi = $(core-libs)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += zuc
 DEPDIRS-zuc = $(core-libs)
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += mrvl
+DEPDIRS-mrvl = $(core-libs)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += null
 DEPDIRS-null = $(core-libs)
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += dpaa2_sec
diff --git a/drivers/crypto/mrvl/Makefile b/drivers/crypto/mrvl/Makefile
new file mode 100644 (file)
index 0000000..a00a19e
--- /dev/null
@@ -0,0 +1,63 @@
+#   BSD LICENSE
+#
+#   Copyright (C) Semihalf 2017. All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Semihalf nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(MAKECMDGOALS),config)
+ifeq ($(LIBMUSDK_PATH),)
+$(error "Please define LIBMUSDK_PATH environment variable")
+endif
+endif
+endif
+
+# library name
+LIB = librte_pmd_mrvl_crypto.a
+
+# build flags
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(LIBMUSDK_PATH)/include
+CFLAGS += -DMVCONF_ARCH_DMA_ADDR_T_64BIT
+
+# library version
+LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_mrvl_pmd_version.map
+
+# external library dependencies
+LDLIBS += -L$(LIBMUSDK_PATH)/lib -lmusdk
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += rte_mrvl_pmd.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += rte_mrvl_pmd_ops.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/crypto/mrvl/rte_mrvl_compat.h b/drivers/crypto/mrvl/rte_mrvl_compat.h
new file mode 100644 (file)
index 0000000..11d53fc
--- /dev/null
@@ -0,0 +1,48 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Semihalf 2017. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Semihalf nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MRVL_COMPAT_H_
+#define _RTE_MRVL_COMPAT_H_
+
+/* Unluckily, container_of is defined by both DPDK and MUSDK,
+ * we'll declare only one version.
+ *
+ * Note that it is not used in this PMD anyway.
+ */
+#ifdef container_of
+#undef container_of
+#endif
+#include "drivers/mv_sam.h"
+#include "drivers/mv_sam_cio.h"
+#include "drivers/mv_sam_session.h"
+
+#endif /* _RTE_MRVL_COMPAT_H_ */
diff --git a/drivers/crypto/mrvl/rte_mrvl_pmd.c b/drivers/crypto/mrvl/rte_mrvl_pmd.c
new file mode 100644 (file)
index 0000000..0c540fa
--- /dev/null
@@ -0,0 +1,872 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Semihalf 2017. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Semihalf nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_cryptodev_vdev.h>
+#include <rte_vdev.h>
+#include <rte_malloc.h>
+#include <rte_cpuflags.h>
+
+#include "rte_mrvl_pmd_private.h"
+
+#ifndef RTE_MRVL_MUSDK_DMA_MEMSIZE
+#define RTE_MRVL_MUSDK_DMA_MEMSIZE 41943040
+#endif
+
+static uint8_t cryptodev_driver_id;
+
+/**
+ * Flag if particular crypto algorithm is supported by PMD/MUSDK.
+ *
+ * The idea is to have Not Supported value as default (0).
+ * This way we need only to define proper map sizes,
+ * non-initialized entries will be by default not supported.
+ */
+enum algo_supported {
+       ALGO_NOT_SUPPORTED = 0,
+       ALGO_SUPPORTED = 1,
+};
+
+/** Map elements for cipher mapping.*/
+struct cipher_params_mapping {
+       enum algo_supported  supported;   /**< On/Off switch */
+       enum sam_cipher_alg  cipher_alg;  /**< Cipher algorithm */
+       enum sam_cipher_mode cipher_mode; /**< Cipher mode */
+       unsigned int max_key_len;         /**< Maximum key length (in bytes)*/
+}
+/* We want to squeeze in multiple maps into the cache line. */
+__rte_aligned(32);
+
+/** Map elements for auth mapping.*/
+struct auth_params_mapping {
+       enum algo_supported supported;  /**< On/off switch */
+       enum sam_auth_alg   auth_alg;   /**< Auth algorithm */
+}
+/* We want to squeeze in multiple maps into the cache line. */
+__rte_aligned(32);
+
+/**
+ * Map of supported cipher algorithms.
+ */
+static const
+struct cipher_params_mapping cipher_map[RTE_CRYPTO_CIPHER_LIST_END] = {
+       [RTE_CRYPTO_CIPHER_3DES_CBC] = {
+               .supported = ALGO_SUPPORTED,
+               .cipher_alg = SAM_CIPHER_3DES,
+               .cipher_mode = SAM_CIPHER_CBC,
+               .max_key_len = BITS2BYTES(192) },
+       [RTE_CRYPTO_CIPHER_3DES_CTR] = {
+               .supported = ALGO_SUPPORTED,
+               .cipher_alg = SAM_CIPHER_3DES,
+               .cipher_mode = SAM_CIPHER_CTR,
+               .max_key_len = BITS2BYTES(192) },
+       [RTE_CRYPTO_CIPHER_3DES_ECB] = {
+               .supported = ALGO_SUPPORTED,
+               .cipher_alg = SAM_CIPHER_3DES,
+               .cipher_mode = SAM_CIPHER_ECB,
+               .max_key_len = BITS2BYTES(192) },
+       [RTE_CRYPTO_CIPHER_AES_CBC] = {
+               .supported = ALGO_SUPPORTED,
+               .cipher_alg = SAM_CIPHER_AES,
+               .cipher_mode = SAM_CIPHER_CBC,
+               .max_key_len = BITS2BYTES(256) },
+       [RTE_CRYPTO_CIPHER_AES_CTR] = {
+               .supported = ALGO_SUPPORTED,
+               .cipher_alg = SAM_CIPHER_AES,
+               .cipher_mode = SAM_CIPHER_CTR,
+               .max_key_len = BITS2BYTES(256) },
+};
+
+/**
+ * Map of supported auth algorithms.
+ */
+static const
+struct auth_params_mapping auth_map[RTE_CRYPTO_AUTH_LIST_END] = {
+       [RTE_CRYPTO_AUTH_MD5_HMAC] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HMAC_MD5 },
+       [RTE_CRYPTO_AUTH_MD5] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HASH_MD5 },
+       [RTE_CRYPTO_AUTH_SHA1_HMAC] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HMAC_SHA1 },
+       [RTE_CRYPTO_AUTH_SHA1] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HASH_SHA1 },
+       [RTE_CRYPTO_AUTH_SHA224] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HASH_SHA2_224 },
+       [RTE_CRYPTO_AUTH_SHA256_HMAC] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HMAC_SHA2_256 },
+       [RTE_CRYPTO_AUTH_SHA256] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HASH_SHA2_256 },
+       [RTE_CRYPTO_AUTH_SHA384_HMAC] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HMAC_SHA2_384 },
+       [RTE_CRYPTO_AUTH_SHA384] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HASH_SHA2_384 },
+       [RTE_CRYPTO_AUTH_SHA512_HMAC] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HMAC_SHA2_512 },
+       [RTE_CRYPTO_AUTH_SHA512] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_HASH_SHA2_512 },
+       [RTE_CRYPTO_AUTH_AES_GMAC] = {
+               .supported = ALGO_SUPPORTED,
+               .auth_alg = SAM_AUTH_AES_GMAC },
+};
+
+/**
+ * Map of supported aead algorithms.
+ */
+static const
+struct cipher_params_mapping aead_map[RTE_CRYPTO_AEAD_LIST_END] = {
+       [RTE_CRYPTO_AEAD_AES_GCM] = {
+               .supported = ALGO_SUPPORTED,
+               .cipher_alg = SAM_CIPHER_AES,
+               .cipher_mode = SAM_CIPHER_GCM,
+               .max_key_len = BITS2BYTES(256) },
+};
+
+/*
+ *-----------------------------------------------------------------------------
+ * Forward declarations.
+ *-----------------------------------------------------------------------------
+ */
+static int cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev);
+
+/*
+ *-----------------------------------------------------------------------------
+ * Session Preparation.
+ *-----------------------------------------------------------------------------
+ */
+
+/**
+ * Get xform chain order.
+ *
+ * @param xform Pointer to configuration structure chain for crypto operations.
+ * @returns Order of crypto operations.
+ */
+static enum mrvl_crypto_chain_order
+mrvl_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform)
+{
+       /* Currently, Marvell supports max 2 operations in chain */
+       if (xform->next != NULL && xform->next->next != NULL)
+               return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED;
+
+       if (xform->next != NULL) {
+               if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
+                       (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER))
+                       return MRVL_CRYPTO_CHAIN_AUTH_CIPHER;
+
+               if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
+                       (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH))
+                       return MRVL_CRYPTO_CHAIN_CIPHER_AUTH;
+       } else {
+               if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
+                       return MRVL_CRYPTO_CHAIN_AUTH_ONLY;
+
+               if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
+                       return MRVL_CRYPTO_CHAIN_CIPHER_ONLY;
+
+               if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
+                       return MRVL_CRYPTO_CHAIN_COMBINED;
+       }
+       return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED;
+}
+
+/**
+ * Set session parameters for cipher part.
+ *
+ * @param sess Crypto session pointer.
+ * @param cipher_xform Pointer to configuration structure for cipher operations.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+static int
+mrvl_crypto_set_cipher_session_parameters(struct mrvl_crypto_session *sess,
+               const struct rte_crypto_sym_xform *cipher_xform)
+{
+       /* Make sure we've got proper struct */
+       if (cipher_xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
+               MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!");
+               return -EINVAL;
+       }
+
+       /* See if map data is present and valid */
+       if ((cipher_xform->cipher.algo > RTE_DIM(cipher_map)) ||
+               (cipher_map[cipher_xform->cipher.algo].supported
+                       != ALGO_SUPPORTED)) {
+               MRVL_CRYPTO_LOG_ERR("Cipher algorithm not supported!");
+               return -EINVAL;
+       }
+
+       sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
+
+       sess->sam_sess_params.dir =
+               (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
+               SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT;
+       sess->sam_sess_params.cipher_alg =
+               cipher_map[cipher_xform->cipher.algo].cipher_alg;
+       sess->sam_sess_params.cipher_mode =
+               cipher_map[cipher_xform->cipher.algo].cipher_mode;
+
+       /* Assume IV will be passed together with data. */
+       sess->sam_sess_params.cipher_iv = NULL;
+
+       /* Get max key length. */
+       if (cipher_xform->cipher.key.length >
+               cipher_map[cipher_xform->cipher.algo].max_key_len) {
+               MRVL_CRYPTO_LOG_ERR("Wrong key length!");
+               return -EINVAL;
+       }
+
+       sess->sam_sess_params.cipher_key_len = cipher_xform->cipher.key.length;
+       sess->sam_sess_params.cipher_key = cipher_xform->cipher.key.data;
+
+       return 0;
+}
+
+/**
+ * Set session parameters for authentication part.
+ *
+ * @param sess Crypto session pointer.
+ * @param auth_xform Pointer to configuration structure for auth operations.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+static int
+mrvl_crypto_set_auth_session_parameters(struct mrvl_crypto_session *sess,
+               const struct rte_crypto_sym_xform *auth_xform)
+{
+       /* Make sure we've got proper struct */
+       if (auth_xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
+               MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!");
+               return -EINVAL;
+       }
+
+       /* See if map data is present and valid */
+       if ((auth_xform->auth.algo > RTE_DIM(auth_map)) ||
+               (auth_map[auth_xform->auth.algo].supported != ALGO_SUPPORTED)) {
+               MRVL_CRYPTO_LOG_ERR("Auth algorithm not supported!");
+               return -EINVAL;
+       }
+
+       sess->sam_sess_params.dir =
+               (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
+               SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT;
+       sess->sam_sess_params.auth_alg =
+               auth_map[auth_xform->auth.algo].auth_alg;
+       sess->sam_sess_params.u.basic.auth_icv_len =
+               auth_xform->auth.digest_length;
+       /* auth_key must be NULL if auth algorithm does not use HMAC */
+       sess->sam_sess_params.auth_key = auth_xform->auth.key.length ?
+                                        auth_xform->auth.key.data : NULL;
+       sess->sam_sess_params.auth_key_len = auth_xform->auth.key.length;
+
+       return 0;
+}
+
+/**
+ * Set session parameters for aead part.
+ *
+ * @param sess Crypto session pointer.
+ * @param aead_xform Pointer to configuration structure for aead operations.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+static int
+mrvl_crypto_set_aead_session_parameters(struct mrvl_crypto_session *sess,
+               const struct rte_crypto_sym_xform *aead_xform)
+{
+       /* Make sure we've got proper struct */
+       if (aead_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) {
+               MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!");
+               return -EINVAL;
+       }
+
+       /* See if map data is present and valid */
+       if ((aead_xform->aead.algo > RTE_DIM(aead_map)) ||
+               (aead_map[aead_xform->aead.algo].supported
+                       != ALGO_SUPPORTED)) {
+               MRVL_CRYPTO_LOG_ERR("AEAD algorithm not supported!");
+               return -EINVAL;
+       }
+
+       sess->sam_sess_params.dir =
+               (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
+               SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT;
+       sess->sam_sess_params.cipher_alg =
+               aead_map[aead_xform->aead.algo].cipher_alg;
+       sess->sam_sess_params.cipher_mode =
+               aead_map[aead_xform->aead.algo].cipher_mode;
+
+       /* Assume IV will be passed together with data. */
+       sess->sam_sess_params.cipher_iv = NULL;
+
+       /* Get max key length. */
+       if (aead_xform->aead.key.length >
+               aead_map[aead_xform->aead.algo].max_key_len) {
+               MRVL_CRYPTO_LOG_ERR("Wrong key length!");
+               return -EINVAL;
+       }
+
+       sess->sam_sess_params.cipher_key = aead_xform->aead.key.data;
+       sess->sam_sess_params.cipher_key_len = aead_xform->aead.key.length;
+
+       if (sess->sam_sess_params.cipher_mode == SAM_CIPHER_GCM)
+               sess->sam_sess_params.auth_alg = SAM_AUTH_AES_GCM;
+
+       sess->sam_sess_params.u.basic.auth_icv_len =
+               aead_xform->aead.digest_length;
+
+       sess->sam_sess_params.u.basic.auth_aad_len =
+               aead_xform->aead.aad_length;
+
+       return 0;
+}
+
+/**
+ * Parse crypto transform chain and setup session parameters.
+ *
+ * @param dev Pointer to crypto device
+ * @param sess Poiner to crypto session
+ * @param xform Pointer to configuration structure chain for crypto operations.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+int
+mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess,
+               const struct rte_crypto_sym_xform *xform)
+{
+       const struct rte_crypto_sym_xform *cipher_xform = NULL;
+       const struct rte_crypto_sym_xform *auth_xform = NULL;
+       const struct rte_crypto_sym_xform *aead_xform = NULL;
+
+       /* Filter out spurious/broken requests */
+       if (xform == NULL)
+               return -EINVAL;
+
+       sess->chain_order = mrvl_crypto_get_chain_order(xform);
+       switch (sess->chain_order) {
+       case MRVL_CRYPTO_CHAIN_CIPHER_AUTH:
+               cipher_xform = xform;
+               auth_xform = xform->next;
+               break;
+       case MRVL_CRYPTO_CHAIN_AUTH_CIPHER:
+               auth_xform = xform;
+               cipher_xform = xform->next;
+               break;
+       case MRVL_CRYPTO_CHAIN_CIPHER_ONLY:
+               cipher_xform = xform;
+               break;
+       case MRVL_CRYPTO_CHAIN_AUTH_ONLY:
+               auth_xform = xform;
+               break;
+       case MRVL_CRYPTO_CHAIN_COMBINED:
+               aead_xform = xform;
+               break;
+       default:
+               return -EINVAL;
+       }
+
+       if ((cipher_xform != NULL) &&
+               (mrvl_crypto_set_cipher_session_parameters(
+                       sess, cipher_xform) < 0)) {
+               MRVL_CRYPTO_LOG_ERR("Invalid/unsupported cipher parameters");
+               return -EINVAL;
+       }
+
+       if ((auth_xform != NULL) &&
+               (mrvl_crypto_set_auth_session_parameters(
+                       sess, auth_xform) < 0)) {
+               MRVL_CRYPTO_LOG_ERR("Invalid/unsupported auth parameters");
+               return -EINVAL;
+       }
+
+       if ((aead_xform != NULL) &&
+               (mrvl_crypto_set_aead_session_parameters(
+                       sess, aead_xform) < 0)) {
+               MRVL_CRYPTO_LOG_ERR("Invalid/unsupported aead parameters");
+               return -EINVAL;
+       }
+
+       return 0;
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ * Process Operations
+ *-----------------------------------------------------------------------------
+ */
+
+/**
+ * Prepare a single request.
+ *
+ * This function basically translates DPDK crypto request into one
+ * understandable by MUDSK's SAM. If this is a first request in a session,
+ * it starts the session.
+ *
+ * @param request Pointer to pre-allocated && reset request buffer [Out].
+ * @param src_bd Pointer to pre-allocated source descriptor [Out].
+ * @param dst_bd Pointer to pre-allocated destination descriptor [Out].
+ * @param op Pointer to DPDK crypto operation struct [In].
+ */
+static inline int
+mrvl_request_prepare(struct sam_cio_op_params *request,
+               struct sam_buf_info *src_bd,
+               struct sam_buf_info *dst_bd,
+               struct rte_crypto_op *op)
+{
+       struct mrvl_crypto_session *sess;
+       struct rte_mbuf *dst_mbuf;
+       uint8_t *digest;
+
+       if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+               MRVL_CRYPTO_LOG_ERR("MRVL CRYPTO PMD only supports session "
+                               "oriented requests, op (%p) is sessionless.",
+                               op);
+               return -EINVAL;
+       }
+
+       sess = (struct mrvl_crypto_session *)get_session_private_data(
+                       op->sym->session, cryptodev_driver_id);
+       if (unlikely(sess == NULL)) {
+               MRVL_CRYPTO_LOG_ERR("Session was not created for this device");
+               return -EINVAL;
+       }
+
+       /*
+        * If application delivered us null dst buffer, it means it expects
+        * us to deliver the result in src buffer.
+        */
+       dst_mbuf = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
+
+       request->sa = sess->sam_sess;
+       request->cookie = op;
+
+       /* Single buffers only, sorry. */
+       request->num_bufs = 1;
+       request->src = src_bd;
+       src_bd->vaddr = rte_pktmbuf_mtod(op->sym->m_src, void *);
+       src_bd->paddr = rte_pktmbuf_mtophys(op->sym->m_src);
+       src_bd->len = rte_pktmbuf_data_len(op->sym->m_src);
+
+       /* Empty source. */
+       if (rte_pktmbuf_data_len(op->sym->m_src) == 0) {
+               /* EIP does not support 0 length buffers. */
+               MRVL_CRYPTO_LOG_ERR("Buffer length == 0 not supported!");
+               return -1;
+       }
+
+       /* Empty destination. */
+       if (rte_pktmbuf_data_len(dst_mbuf) == 0) {
+               /* Make dst buffer fit at least source data. */
+               if (rte_pktmbuf_append(dst_mbuf,
+                       rte_pktmbuf_data_len(op->sym->m_src)) == NULL) {
+                       MRVL_CRYPTO_LOG_ERR("Unable to set big enough dst buffer!");
+                       return -1;
+               }
+       }
+
+       request->dst = dst_bd;
+       dst_bd->vaddr = rte_pktmbuf_mtod(dst_mbuf, void *);
+       dst_bd->paddr = rte_pktmbuf_mtophys(dst_mbuf);
+
+       /*
+        * We can use all available space in dst_mbuf,
+        * not only what's used currently.
+        */
+       dst_bd->len = dst_mbuf->buf_len - rte_pktmbuf_headroom(dst_mbuf);
+
+       if (sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED) {
+               request->cipher_len = op->sym->aead.data.length;
+               request->cipher_offset = op->sym->aead.data.offset;
+               request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+                       sess->cipher_iv_offset);
+
+               request->auth_aad = op->sym->aead.aad.data;
+               request->auth_offset = request->cipher_offset;
+               request->auth_len = request->cipher_len;
+       } else {
+               request->cipher_len = op->sym->cipher.data.length;
+               request->cipher_offset = op->sym->cipher.data.offset;
+               request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+                               sess->cipher_iv_offset);
+
+               request->auth_offset = op->sym->auth.data.offset;
+               request->auth_len = op->sym->auth.data.length;
+       }
+
+       digest = sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED ?
+               op->sym->aead.digest.data : op->sym->auth.digest.data;
+       if (digest == NULL) {
+               /* No auth - no worry. */
+               return 0;
+       }
+
+       request->auth_icv_offset = request->auth_offset + request->auth_len;
+
+       /*
+        * EIP supports only scenarios where ICV(digest buffer) is placed at
+        * auth_icv_offset. Any other placement means risking errors.
+        */
+       if (sess->sam_sess_params.dir == SAM_DIR_ENCRYPT) {
+               /*
+                * This should be the most common case anyway,
+                * EIP will overwrite DST buffer at auth_icv_offset.
+                */
+               if (rte_pktmbuf_mtod_offset(
+                               dst_mbuf, uint8_t *,
+                               request->auth_icv_offset) == digest) {
+                       return 0;
+               }
+       } else {/* sess->sam_sess_params.dir == SAM_DIR_DECRYPT */
+               /*
+                * EIP will look for digest at auth_icv_offset
+                * offset in SRC buffer.
+                */
+               if (rte_pktmbuf_mtod_offset(
+                               op->sym->m_src, uint8_t *,
+                               request->auth_icv_offset) == digest) {
+                       return 0;
+               }
+       }
+
+       /*
+        * If we landed here it means that digest pointer is
+        * at different than expected place.
+        */
+       return -1;
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ * PMD Framework handlers
+ *-----------------------------------------------------------------------------
+ */
+
+/**
+ * Enqueue burst.
+ *
+ * @param queue_pair Pointer to queue pair.
+ * @param ops Pointer to ops requests array.
+ * @param nb_ops Number of elements in ops requests array.
+ * @returns Number of elements consumed from ops.
+ */
+static uint16_t
+mrvl_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+               uint16_t nb_ops)
+{
+       uint16_t iter_ops = 0;
+       uint16_t to_enq = 0;
+       uint16_t consumed = 0;
+       int ret;
+       struct sam_cio_op_params requests[nb_ops];
+       /*
+        * DPDK uses single fragment buffers, so we can KISS descriptors.
+        * SAM does not store bd pointers, so on-stack scope will be enough.
+        */
+       struct sam_buf_info src_bd[nb_ops];
+       struct sam_buf_info dst_bd[nb_ops];
+       struct mrvl_crypto_qp *qp = (struct mrvl_crypto_qp *)queue_pair;
+
+       if (nb_ops == 0)
+               return 0;
+
+       /* Prepare the burst. */
+       memset(&requests, 0, sizeof(requests));
+
+       /* Iterate through */
+       for (; iter_ops < nb_ops; ++iter_ops) {
+               if (mrvl_request_prepare(&requests[iter_ops],
+                                       &src_bd[iter_ops],
+                                       &dst_bd[iter_ops],
+                                       ops[iter_ops]) < 0) {
+                       MRVL_CRYPTO_LOG_ERR(
+                               "Error while parameters preparation!");
+                       qp->stats.enqueue_err_count++;
+                       ops[iter_ops]->status = RTE_CRYPTO_OP_STATUS_ERROR;
+
+                       /*
+                        * Number of handled ops is increased
+                        * (even if the result of handling is error).
+                        */
+                       ++consumed;
+                       break;
+               }
+
+               ops[iter_ops]->status =
+                       RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+
+               /* Increase the number of ops to enqueue. */
+               ++to_enq;
+       } /* for (; iter_ops < nb_ops;... */
+
+       if (to_enq > 0) {
+               /* Send the burst */
+               ret = sam_cio_enq(qp->cio, requests, &to_enq);
+               consumed += to_enq;
+               if (ret < 0) {
+                       /*
+                        * Trust SAM that in this case returned value will be at
+                        * some point correct (now it is returned unmodified).
+                        */
+                       qp->stats.enqueue_err_count += to_enq;
+                       for (iter_ops = 0; iter_ops < to_enq; ++iter_ops)
+                               ops[iter_ops]->status =
+                                       RTE_CRYPTO_OP_STATUS_ERROR;
+               }
+       }
+
+       qp->stats.enqueued_count += to_enq;
+       return consumed;
+}
+
+/**
+ * Dequeue burst.
+ *
+ * @param queue_pair Pointer to queue pair.
+ * @param ops Pointer to ops requests array.
+ * @param nb_ops Number of elements in ops requests array.
+ * @returns Number of elements dequeued.
+ */
+static uint16_t
+mrvl_crypto_pmd_dequeue_burst(void *queue_pair,
+               struct rte_crypto_op **ops,
+               uint16_t nb_ops)
+{
+       int ret;
+       struct mrvl_crypto_qp *qp = queue_pair;
+       struct sam_cio *cio = qp->cio;
+       struct sam_cio_op_result results[nb_ops];
+       uint16_t i;
+
+       ret = sam_cio_deq(cio, results, &nb_ops);
+       if (ret < 0) {
+               /* Count all dequeued as error. */
+               qp->stats.dequeue_err_count += nb_ops;
+
+               /* But act as they were dequeued anyway*/
+               qp->stats.dequeued_count += nb_ops;
+
+               return 0;
+       }
+
+       /* Unpack and check results. */
+       for (i = 0; i < nb_ops; ++i) {
+               ops[i] = results[i].cookie;
+
+               switch (results[i].status) {
+               case SAM_CIO_OK:
+                       ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+                       break;
+               case SAM_CIO_ERR_ICV:
+                       MRVL_CRYPTO_LOG_DBG("CIO returned SAM_CIO_ERR_ICV.");
+                       ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+                       break;
+               default:
+                       MRVL_CRYPTO_LOG_DBG(
+                               "CIO returned Error: %d", results[i].status);
+                       ops[i]->status = RTE_CRYPTO_OP_STATUS_ERROR;
+                       break;
+               }
+       }
+
+       qp->stats.dequeued_count += nb_ops;
+       return nb_ops;
+}
+
+/**
+ * Create a new crypto device.
+ *
+ * @param name Driver name.
+ * @param vdev Pointer to device structure.
+ * @param init_params Pointer to initialization parameters.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+static int
+cryptodev_mrvl_crypto_create(const char *name,
+               struct rte_vdev_device *vdev,
+               struct rte_crypto_vdev_init_params *init_params)
+{
+       struct rte_cryptodev *dev;
+       struct mrvl_crypto_private *internals;
+       struct sam_init_params  sam_params;
+       int ret;
+
+       if (init_params->name[0] == '\0') {
+               ret = rte_cryptodev_pmd_create_dev_name(
+                               init_params->name, name);
+
+               if (ret < 0) {
+                       MRVL_CRYPTO_LOG_ERR("failed to create unique name");
+                       return ret;
+               }
+       }
+
+       dev = rte_cryptodev_vdev_pmd_init(init_params->name,
+                               sizeof(struct mrvl_crypto_private),
+                               init_params->socket_id, vdev);
+       if (dev == NULL) {
+               MRVL_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
+               goto init_error;
+       }
+
+       dev->driver_id = cryptodev_driver_id;
+       dev->dev_ops = rte_mrvl_crypto_pmd_ops;
+
+       /* Register rx/tx burst functions for data path. */
+       dev->enqueue_burst = mrvl_crypto_pmd_enqueue_burst;
+       dev->dequeue_burst = mrvl_crypto_pmd_dequeue_burst;
+
+       dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+                       RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+                       RTE_CRYPTODEV_FF_HW_ACCELERATED;
+
+       /* Set vector instructions mode supported */
+       internals = dev->data->dev_private;
+
+       internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
+       internals->max_nb_sessions = init_params->max_nb_sessions;
+
+       /*
+        * ret == -EEXIST is correct, it means DMA
+        * has been already initialized.
+        */
+       ret = mv_sys_dma_mem_init(RTE_MRVL_MUSDK_DMA_MEMSIZE);
+       if ((ret < 0) && (ret != -EEXIST))
+               return ret;
+
+       sam_params.max_num_sessions = internals->max_nb_sessions;
+
+       return sam_init(&sam_params);
+
+init_error:
+       MRVL_CRYPTO_LOG_ERR(
+               "driver %s: %s failed", init_params->name, __func__);
+
+       cryptodev_mrvl_crypto_uninit(vdev);
+       return -EFAULT;
+}
+
+/**
+ * Initialize the crypto device.
+ *
+ * @param vdev Pointer to device structure.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+static int
+cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev)
+{
+       struct rte_crypto_vdev_init_params init_params = { };
+       const char *name;
+       const char *input_args;
+       int ret;
+
+       name = rte_vdev_device_name(vdev);
+       if (name == NULL)
+               return -EINVAL;
+       input_args = rte_vdev_device_args(vdev);
+
+       if (!input_args)
+               return -EINVAL;
+
+       init_params.max_nb_queue_pairs = sam_get_num_inst() * SAM_HW_RING_NUM;
+       init_params.max_nb_sessions =
+               RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS;
+       init_params.socket_id = rte_socket_id();
+
+       ret = rte_cryptodev_vdev_parse_init_params(&init_params, input_args);
+       if (ret) {
+               RTE_LOG(ERR, PMD, "Failed to parse input arguments\n");
+               return ret;
+       }
+
+       RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
+                       init_params.socket_id);
+       if (init_params.name[0] != '\0') {
+               RTE_LOG(INFO, PMD, "  User defined name = %s\n",
+                       init_params.name);
+       }
+       RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
+                       init_params.max_nb_queue_pairs);
+       RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
+                       init_params.max_nb_sessions);
+
+       return cryptodev_mrvl_crypto_create(name, vdev, &init_params);
+}
+
+/**
+ * Uninitialize the crypto device
+ *
+ * @param vdev Pointer to device structure.
+ * @returns 0 in case of success, negative value otherwise.
+ */
+static int
+cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev)
+{
+       const char *name = rte_vdev_device_name(vdev);
+
+       if (name == NULL)
+               return -EINVAL;
+
+       RTE_LOG(INFO, PMD,
+               "Closing Marvell crypto device %s on numa socket %u\n",
+               name, rte_socket_id());
+
+       sam_deinit();
+
+       return 0;
+}
+
+/**
+ * Basic driver handlers for use in the constructor.
+ */
+static struct rte_vdev_driver cryptodev_mrvl_pmd_drv = {
+       .probe = cryptodev_mrvl_crypto_init,
+       .remove = cryptodev_mrvl_crypto_uninit
+};
+
+static struct cryptodev_driver mrvl_crypto_drv;
+
+/* Register the driver in constructor. */
+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_MRVL_PMD, cryptodev_mrvl_pmd_drv);
+RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_MRVL_PMD,
+       "max_nb_queue_pairs=<int> "
+       "max_nb_sessions=<int> "
+       "socket_id=<int>");
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mrvl_crypto_drv, cryptodev_mrvl_pmd_drv,
+               cryptodev_driver_id);
diff --git a/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c b/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c
new file mode 100644 (file)
index 0000000..495fec6
--- /dev/null
@@ -0,0 +1,776 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Semihalf 2017. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Semihalf nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev_pmd.h>
+
+#include "rte_mrvl_pmd_private.h"
+
+/**
+ * Capabilities list to be used in reporting to DPDK.
+ */
+static const struct rte_cryptodev_capabilities
+       mrvl_crypto_pmd_capabilities[] = {
+       {       /* MD5 HMAC */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
+                               .block_size = 64,
+                               .key_size = {
+                                       .min = 1,
+                                       .max = 64,
+                                       .increment = 1
+                               },
+                               .digest_size = {
+                                       .min = 16,
+                                       .max = 16,
+                                       .increment = 0
+                               },
+                       }, }
+               }, }
+       },
+       {       /* MD5 */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+                       {.sym = {
+                               .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                               {.auth = {
+                                       .algo = RTE_CRYPTO_AUTH_MD5,
+                                       .block_size = 64,
+                                       .key_size = {
+                                               .min = 0,
+                                               .max = 0,
+                                               .increment = 0
+                                       },
+                                       .digest_size = {
+                                               .min = 16,
+                                               .max = 16,
+                                               .increment = 0
+                                       },
+                               }, }
+                       }, }
+       },
+       {       /* SHA1 HMAC */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+                       {.sym = {
+                               .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                               {.auth = {
+                                       .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
+                                       .block_size = 64,
+                                       .key_size = {
+                                               .min = 1,
+                                               .max = 64,
+                                               .increment = 1
+                                       },
+                                       .digest_size = {
+                                               .min = 20,
+                                               .max = 20,
+                                               .increment = 0
+                                       },
+                               }, }
+                       }, }
+       },
+       {       /* SHA1 */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_SHA1,
+                               .block_size = 64,
+                               .key_size = {
+                                       .min = 0,
+                                       .max = 0,
+                                       .increment = 0
+                               },
+                               .digest_size = {
+                                       .min = 20,
+                                       .max = 20,
+                                       .increment = 0
+                               },
+                       }, }
+               }, }
+       },
+       {       /* SHA224 */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_SHA224,
+                               .block_size = 64,
+                               .key_size = {
+                                       .min = 0,
+                                       .max = 0,
+                                       .increment = 0
+                               },
+                               .digest_size = {
+                                       .min = 28,
+                                       .max = 28,
+                                       .increment = 0
+                               },
+                       }, }
+               }, }
+       },
+       {       /* SHA256 HMAC */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+                       {.sym = {
+                               .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                               {.auth = {
+                                       .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
+                                       .block_size = 64,
+                                       .key_size = {
+                                               .min = 1,
+                                               .max = 64,
+                                               .increment = 1
+                                       },
+                                       .digest_size = {
+                                               .min = 32,
+                                               .max = 32,
+                                               .increment = 0
+                                       },
+                               }, }
+                       }, }
+       },
+       {       /* SHA256 */
+                       .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+                       {.sym = {
+                               .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                               {.auth = {
+                                       .algo = RTE_CRYPTO_AUTH_SHA256,
+                                       .block_size = 64,
+                                       .key_size = {
+                                               .min = 0,
+                                               .max = 0,
+                                               .increment = 0
+                                       },
+                                       .digest_size = {
+                                               .min = 32,
+                                               .max = 32,
+                                               .increment = 0
+                                       },
+                               }, }
+                       }, }
+               },
+       {       /* SHA384 HMAC */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
+                               .block_size = 128,
+                               .key_size = {
+                                       .min = 1,
+                                       .max = 128,
+                                       .increment = 1
+                               },
+                               .digest_size = {
+                                       .min = 48,
+                                       .max = 48,
+                                       .increment = 0
+                               },
+                       }, }
+               }, }
+       },
+       {       /* SHA384 */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_SHA384,
+                               .block_size = 128,
+                               .key_size = {
+                                       .min = 0,
+                                       .max = 0,
+                                       .increment = 0
+                               },
+                               .digest_size = {
+                                       .min = 48,
+                                       .max = 48,
+                                       .increment = 0
+                               },
+                       }, }
+               }, }
+       },
+       {       /* SHA512 HMAC */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
+                               .block_size = 128,
+                               .key_size = {
+                                       .min = 1,
+                                       .max = 128,
+                                       .increment = 1
+                               },
+                               .digest_size = {
+                                       .min = 64,
+                                       .max = 64,
+                                       .increment = 0
+                               },
+                       }, }
+               }, }
+       },
+       {       /* SHA512  */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_SHA512,
+                               .block_size = 128,
+                               .key_size = {
+                                       .min = 0,
+                                       .max = 0,
+                                       .increment = 0
+                               },
+                               .digest_size = {
+                                       .min = 64,
+                                       .max = 64,
+                                       .increment = 0
+                               },
+                       }, }
+               }, }
+       },
+       {       /* AES CBC */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+                       {.sym = {
+                               .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+                               {.cipher = {
+                                       .algo = RTE_CRYPTO_CIPHER_AES_CBC,
+                                       .block_size = 16,
+                                       .key_size = {
+                                               .min = 16,
+                                               .max = 32,
+                                               .increment = 8
+                                       },
+                                       .iv_size = {
+                                               .min = 16,
+                                               .max = 16,
+                                               .increment = 0
+                                       }
+                               }, }
+                       }, }
+       },
+       {       /* AES CTR */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+                       {.cipher = {
+                               .algo = RTE_CRYPTO_CIPHER_AES_CTR,
+                               .block_size = 16,
+                               .key_size = {
+                                       .min = 16,
+                                       .max = 32,
+                                       .increment = 8
+                               },
+                               .iv_size = {
+                                       .min = 16,
+                                       .max = 16,
+                                       .increment = 0
+                               }
+                       }, }
+               }, }
+       },
+       {       /* AES GCM */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+                       {.aead = {
+                               .algo = RTE_CRYPTO_AEAD_AES_GCM,
+                               .block_size = 16,
+                               .key_size = {
+                                       .min = 16,
+                                       .max = 32,
+                                       .increment = 8
+                               },
+                               .digest_size = {
+                                       .min = 16,
+                                       .max = 16,
+                                       .increment = 0
+                               },
+                               .aad_size = {
+                                       .min = 8,
+                                       .max = 12,
+                                       .increment = 4
+                               },
+                               .iv_size = {
+                                       .min = 12,
+                                       .max = 16,
+                                       .increment = 4
+                               }
+                       }, }
+               }, }
+       },
+       {       /* AES GMAC (AUTH) */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+                       {.auth = {
+                               .algo = RTE_CRYPTO_AUTH_AES_GMAC,
+                               .block_size = 16,
+                               .key_size = {
+                                       .min = 16,
+                                       .max = 32,
+                                       .increment = 8
+                               },
+                               .digest_size = {
+                                       .min = 16,
+                                       .max = 16,
+                                       .increment = 0
+                               },
+                               .iv_size = {
+                                       .min = 8,
+                                       .max = 65532,
+                                       .increment = 4
+                               }
+                       }, }
+               }, }
+       },
+       {       /* 3DES CBC */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+                       {.cipher = {
+                               .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+                               .block_size = 8,
+                               .key_size = {
+                                       .min = 24,
+                                       .max = 24,
+                                       .increment = 0
+                               },
+                               .iv_size = {
+                                       .min = 8,
+                                       .max = 8,
+                                       .increment = 0
+                               }
+                       }, }
+               }, }
+       },
+       {       /* 3DES CTR */
+               .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+               {.sym = {
+                       .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+                       {.cipher = {
+                               .algo = RTE_CRYPTO_CIPHER_3DES_CTR,
+                               .block_size = 8,
+                               .key_size = {
+                                       .min = 24,
+                                       .max = 24,
+                                       .increment = 0
+                               },
+                               .iv_size = {
+                                       .min = 8,
+                                       .max = 8,
+                                       .increment = 0
+                               }
+                       }, }
+               }, }
+       },
+
+       RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+
+/**
+ * Configure device (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @param config Pointer to configuration structure.
+ * @returns 0. Always.
+ */
+static int
+mrvl_crypto_pmd_config(__rte_unused struct rte_cryptodev *dev,
+               __rte_unused struct rte_cryptodev_config *config)
+{
+       return 0;
+}
+
+/**
+ * Start device (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @returns 0. Always.
+ */
+static int
+mrvl_crypto_pmd_start(__rte_unused struct rte_cryptodev *dev)
+{
+       return 0;
+}
+
+/**
+ * Stop device (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @returns 0. Always.
+ */
+static void
+mrvl_crypto_pmd_stop(__rte_unused struct rte_cryptodev *dev)
+{
+}
+
+/**
+ * Get device statistics (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @param stats Pointer to statistics structure [out].
+ */
+static void
+mrvl_crypto_pmd_stats_get(struct rte_cryptodev *dev,
+               struct rte_cryptodev_stats *stats)
+{
+       int qp_id;
+
+       for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+               struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+               stats->enqueued_count += qp->stats.enqueued_count;
+               stats->dequeued_count += qp->stats.dequeued_count;
+
+               stats->enqueue_err_count += qp->stats.enqueue_err_count;
+               stats->dequeue_err_count += qp->stats.dequeue_err_count;
+       }
+}
+
+/**
+ * Reset device statistics (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ */
+static void
+mrvl_crypto_pmd_stats_reset(struct rte_cryptodev *dev)
+{
+       int qp_id;
+
+       for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+               struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+               memset(&qp->stats, 0, sizeof(qp->stats));
+       }
+}
+
+/**
+ * Get device info (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @param dev_info Pointer to the device info structure [out].
+ */
+static void
+mrvl_crypto_pmd_info_get(struct rte_cryptodev *dev,
+               struct rte_cryptodev_info *dev_info)
+{
+       struct mrvl_crypto_private *internals = dev->data->dev_private;
+
+       if (dev_info != NULL) {
+               dev_info->driver_id = dev->driver_id;
+               dev_info->feature_flags = dev->feature_flags;
+               dev_info->capabilities = mrvl_crypto_pmd_capabilities;
+               dev_info->max_nb_queue_pairs = internals->max_nb_qpairs;
+               dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
+       }
+}
+
+/**
+ * Release queue pair (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @param qp_id ID of Queue Pair to release.
+ * @returns 0. Always.
+ */
+static int
+mrvl_crypto_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+       struct mrvl_crypto_qp *qp =
+                       (struct mrvl_crypto_qp *)dev->data->queue_pairs[qp_id];
+
+       if (dev->data->queue_pairs[qp_id] != NULL) {
+               sam_cio_flush(qp->cio);
+               sam_cio_deinit(qp->cio);
+               rte_free(dev->data->queue_pairs[qp_id]);
+               dev->data->queue_pairs[qp_id] = NULL;
+       }
+
+       return 0;
+}
+
+/**
+ * Close device (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @returns 0. Always.
+ */
+static int
+mrvl_crypto_pmd_close(struct rte_cryptodev *dev)
+{
+       int qp_id;
+
+       for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++)
+               mrvl_crypto_pmd_qp_release(dev, qp_id);
+
+       return 0;
+}
+
+/**
+ * Setup a queue pair (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @param qp_id ID of the Queue Pair.
+ * @param qp_conf Queue pair configuration (nb of descriptors).
+ * @param socket_id NUMA socket to allocate memory on.
+ * @returns 0 upon success, negative value otherwise.
+ */
+static int
+mrvl_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+               const struct rte_cryptodev_qp_conf *qp_conf,
+               int socket_id, struct rte_mempool *session_pool)
+{
+       struct mrvl_crypto_qp *qp = NULL;
+       char match[RTE_CRYPTODEV_NAME_LEN];
+       unsigned int n;
+
+       /* Allocate the queue pair data structure. */
+       qp = rte_zmalloc_socket("MRVL Crypto PMD Queue Pair", sizeof(*qp),
+                                       RTE_CACHE_LINE_SIZE, socket_id);
+       if (qp == NULL)
+               return -ENOMEM;
+
+       /* Free old qp prior setup if needed. */
+       if (dev->data->queue_pairs[qp_id] != NULL)
+               mrvl_crypto_pmd_qp_release(dev, qp_id);
+
+       do { /* Error handling block */
+
+               /*
+                * This extra check is necessary due to a bug in
+                * crypto library.
+                */
+               int num = sam_get_num_inst();
+               if (num == 0) {
+                       MRVL_CRYPTO_LOG_ERR("No crypto engines detected.\n");
+                       return -1;
+               }
+
+               /*
+                * In case two crypto engines are enabled qps will
+                * be evenly spread among them. Even and odd qps will
+                * be handled by cio-0 and cio-1 respectively. qp-cio mapping
+                * will look as follows:
+                *
+                * qp:      0        1        2        3
+                * cio-x:y: cio-0:0, cio-1:0, cio-0:1, cio-1:1
+                *
+                * qp:      4        5        6        7
+                * cio-x:y: cio-0:2, cio-1:2, cio-0:3, cio-1:3
+                *
+                * In case just one engine is enabled mapping will look as
+                * follows:
+                * qp:      0        1        2        3
+                * cio-x:y: cio-0:0, cio-0:1, cio-0:2, cio-0:3
+                */
+               n = snprintf(match, sizeof(match), "cio-%u:%u",
+                               qp_id % num, qp_id / num);
+
+               if (n >= sizeof(match))
+                       break;
+
+               qp->cio_params.match = match;
+               qp->cio_params.size = qp_conf->nb_descriptors;
+
+               if (sam_cio_init(&qp->cio_params, &qp->cio) < 0)
+                       break;
+
+               qp->sess_mp = session_pool;
+
+               memset(&qp->stats, 0, sizeof(qp->stats));
+               dev->data->queue_pairs[qp_id] = qp;
+               return 0;
+       } while (0);
+
+       rte_free(qp);
+       return -1;
+}
+
+/** Start queue pair (PMD ops callback) - not supported.
+ *
+ * @param dev Pointer to the device structure.
+ * @param qp_id ID of the Queue Pair.
+ * @returns -ENOTSUP. Always.
+ */
+static int
+mrvl_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
+               __rte_unused uint16_t queue_pair_id)
+{
+       return -ENOTSUP;
+}
+
+/** Stop queue pair (PMD ops callback) - not supported.
+ *
+ * @param dev Pointer to the device structure.
+ * @param qp_id ID of the Queue Pair.
+ * @returns -ENOTSUP. Always.
+ */
+static int
+mrvl_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
+               __rte_unused uint16_t queue_pair_id)
+{
+       return -ENOTSUP;
+}
+
+/** Return the number of allocated queue pairs (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @returns Number of allocated queue pairs.
+ */
+static uint32_t
+mrvl_crypto_pmd_qp_count(struct rte_cryptodev *dev)
+{
+       return dev->data->nb_queue_pairs;
+}
+
+/** Returns the size of the session structure (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure [Unused].
+ * @returns Size of Marvell crypto session.
+ */
+static unsigned
+mrvl_crypto_pmd_session_get_size(__rte_unused struct rte_cryptodev *dev)
+{
+       return sizeof(struct mrvl_crypto_session);
+}
+
+/** Configure the session from a crypto xform chain (PMD ops callback).
+ *
+ * @param dev Pointer to the device structure.
+ * @param xform Pointer to the crytpo configuration structure.
+ * @param sess Pointer to the empty session structure.
+ * @returns 0 upon success, negative value otherwise.
+ */
+static int
+mrvl_crypto_pmd_session_configure(__rte_unused struct rte_cryptodev *dev,
+               struct rte_crypto_sym_xform *xform,
+               struct rte_cryptodev_sym_session *sess,
+               struct rte_mempool *mp)
+{
+       struct mrvl_crypto_session *mrvl_sess;
+       void *sess_private_data;
+       int ret;
+
+       if (sess == NULL) {
+               MRVL_CRYPTO_LOG_ERR("Invalid session struct.");
+               return -EINVAL;
+       }
+
+       if (rte_mempool_get(mp, &sess_private_data)) {
+               CDEV_LOG_ERR("Couldn't get object from session mempool.");
+               return -ENOMEM;
+       }
+
+       ret = mrvl_crypto_set_session_parameters(sess_private_data, xform);
+       if (ret != 0) {
+               MRVL_CRYPTO_LOG_ERR("Failed to configure session parameters.");
+
+               /* Return session to mempool */
+               rte_mempool_put(mp, sess_private_data);
+               return ret;
+       }
+
+       set_session_private_data(sess, dev->driver_id, sess_private_data);
+
+       mrvl_sess = (struct mrvl_crypto_session *)sess_private_data;
+       if (sam_session_create(&mrvl_sess->sam_sess_params,
+                               &mrvl_sess->sam_sess) < 0) {
+               MRVL_CRYPTO_LOG_DBG("Failed to create session!");
+               return -EIO;
+       }
+
+       return 0;
+}
+
+/**
+ * Clear the memory of session so it doesn't leave key material behind.
+ *
+ * @param dev Pointer to the device structure.
+ * @returns 0. Always.
+ */
+static void
+mrvl_crypto_pmd_session_clear(struct rte_cryptodev *dev,
+               struct rte_cryptodev_sym_session *sess)
+{
+
+       uint8_t index = dev->driver_id;
+       void *sess_priv = get_session_private_data(sess, index);
+
+       /* Zero out the whole structure */
+       if (sess_priv) {
+               struct mrvl_crypto_session *mrvl_sess =
+                       (struct mrvl_crypto_session *)sess_priv;
+
+               if (mrvl_sess->sam_sess &&
+                   sam_session_destroy(mrvl_sess->sam_sess) < 0) {
+                       MRVL_CRYPTO_LOG_INFO("Error while destroying session!");
+               }
+
+               memset(sess, 0, sizeof(struct mrvl_crypto_session));
+               struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+               set_session_private_data(sess, index, NULL);
+               rte_mempool_put(sess_mp, sess_priv);
+       }
+}
+
+/**
+ * PMD handlers for crypto ops.
+ */
+static struct rte_cryptodev_ops mrvl_crypto_pmd_ops = {
+               .dev_configure          = mrvl_crypto_pmd_config,
+               .dev_start              = mrvl_crypto_pmd_start,
+               .dev_stop               = mrvl_crypto_pmd_stop,
+               .dev_close              = mrvl_crypto_pmd_close,
+
+               .dev_infos_get          = mrvl_crypto_pmd_info_get,
+
+               .stats_get              = mrvl_crypto_pmd_stats_get,
+               .stats_reset            = mrvl_crypto_pmd_stats_reset,
+
+               .queue_pair_setup       = mrvl_crypto_pmd_qp_setup,
+               .queue_pair_release     = mrvl_crypto_pmd_qp_release,
+               .queue_pair_start       = mrvl_crypto_pmd_qp_start,
+               .queue_pair_stop        = mrvl_crypto_pmd_qp_stop,
+               .queue_pair_count       = mrvl_crypto_pmd_qp_count,
+
+               .session_get_size       = mrvl_crypto_pmd_session_get_size,
+               .session_configure      = mrvl_crypto_pmd_session_configure,
+               .session_clear          = mrvl_crypto_pmd_session_clear
+};
+
+struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops = &mrvl_crypto_pmd_ops;
diff --git a/drivers/crypto/mrvl/rte_mrvl_pmd_private.h b/drivers/crypto/mrvl/rte_mrvl_pmd_private.h
new file mode 100644 (file)
index 0000000..2da14b8
--- /dev/null
@@ -0,0 +1,121 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Semihalf 2017. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Semihalf nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MRVL_PMD_PRIVATE_H_
+#define _RTE_MRVL_PMD_PRIVATE_H_
+
+#include "rte_mrvl_compat.h"
+
+#define CRYPTODEV_NAME_MRVL_PMD crypto_mrvl
+/**< Marvell PMD device name */
+
+#define MRVL_CRYPTO_LOG_ERR(fmt, args...) \
+       RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n",  \
+                       RTE_STR(CRYPTODEV_NAME_MRVL_CRYPTO_PMD), \
+                       __func__, __LINE__, ## args)
+
+#ifdef RTE_LIBRTE_MRVL_CRYPTO_DEBUG
+#define MRVL_CRYPTO_LOG_INFO(fmt, args...) \
+       RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
+                       RTE_STR(CRYPTODEV_NAME_MRVL_CRYPTO_PMD), \
+                       __func__, __LINE__, ## args)
+
+#define MRVL_CRYPTO_LOG_DBG(fmt, args...) \
+       RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \
+                       RTE_STR(CRYPTODEV_NAME_MRVL_CRYPTO_PMD), \
+                       __func__, __LINE__, ## args)
+
+#else
+#define MRVL_CRYPTO_LOG_INFO(fmt, args...)
+#define MRVL_CRYPTO_LOG_DBG(fmt, args...)
+#endif
+
+/**
+ * Handy bits->bytes conversion macro.
+ */
+#define BITS2BYTES(x) ((x) >> 3)
+
+/** The operation order mode enumerator. */
+enum mrvl_crypto_chain_order {
+       MRVL_CRYPTO_CHAIN_CIPHER_ONLY,
+       MRVL_CRYPTO_CHAIN_AUTH_ONLY,
+       MRVL_CRYPTO_CHAIN_CIPHER_AUTH,
+       MRVL_CRYPTO_CHAIN_AUTH_CIPHER,
+       MRVL_CRYPTO_CHAIN_COMBINED,
+       MRVL_CRYPTO_CHAIN_NOT_SUPPORTED,
+};
+
+/** Private data structure for each crypto device. */
+struct mrvl_crypto_private {
+       unsigned int max_nb_qpairs;     /**< Max number of queue pairs */
+       unsigned int max_nb_sessions;   /**< Max number of sessions */
+};
+
+/** MRVL crypto queue pair structure. */
+struct mrvl_crypto_qp {
+       /** SAM CIO (MUSDK Queue Pair equivalent).*/
+       struct sam_cio *cio;
+
+       /** Session Mempool. */
+       struct rte_mempool *sess_mp;
+
+       /** Queue pair statistics. */
+       struct rte_cryptodev_stats stats;
+
+       /** CIO initialization parameters.*/
+       struct sam_cio_params cio_params;
+} __rte_cache_aligned;
+
+/** MRVL crypto private session structure. */
+struct mrvl_crypto_session {
+       /** Crypto operations chain order. */
+       enum mrvl_crypto_chain_order chain_order;
+
+       /** Session initialization parameters. */
+       struct sam_session_params sam_sess_params;
+
+       /** SAM session pointer. */
+       struct sam_sa *sam_sess;
+
+       /** Cipher IV offset. */
+       uint16_t cipher_iv_offset;
+} __rte_cache_aligned;
+
+/** Set and validate MRVL crypto session parameters */
+extern int
+mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess,
+               const struct rte_crypto_sym_xform *xform);
+
+/** device specific operations function pointer structure */
+extern struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops;
+
+#endif /* _RTE_MRVL_PMD_PRIVATE_H_ */
diff --git a/drivers/crypto/mrvl/rte_pmd_mrvl_version.map b/drivers/crypto/mrvl/rte_pmd_mrvl_version.map
new file mode 100644 (file)
index 0000000..a753031
--- /dev/null
@@ -0,0 +1,3 @@
+DPDK_17.11 {
+       local: *;
+};
index 019f442..384dc06 100644 (file)
@@ -172,6 +172,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ZUC)         += -lrte_pmd_zuc
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ZUC)         += -L$(LIBSSO_ZUC_PATH)/build -lsso_zuc
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO)    += -lrte_pmd_armv8
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO)    += -L$(ARMV8_CRYPTO_LIB_PATH) -larmv8_crypto
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += -L$(LIBMUSDK_PATH)/lib -lrte_pmd_mrvl_crypto -lmusdk
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += -lrte_pmd_crypto_scheduler
 ifeq ($(CONFIG_RTE_LIBRTE_FSLMC_BUS),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC)   += -lrte_pmd_dpaa2_sec