crypto/dpaa2_sec: add per dev mempool to store FLE

rte_malloc uses common memory area for all cores.

Now rte_malloc are replaced by per device mempool to allocate
space for FLE. This removes contention and improves performance.

Signed-off-by: Akhil Goyal <akhil.goyal@nxp.com>

diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
index 3620751..dbdaf46 100644 (file)
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c
@@ -71,6 +71,13 @@
 #define NO_PREFETCH 0
 #define TDES_CBC_IV_LEN 8
 #define AES_CBC_IV_LEN 16
+#define AES_CTR_IV_LEN 16
+#define AES_GCM_IV_LEN 12
+/* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */
+#define FLE_POOL_NUM_BUFS      32000
+#define FLE_POOL_BUF_SIZE      256
+#define FLE_POOL_CACHE_SIZE    512
+
 enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8;
 
 static inline int
@@ -84,9 +91,8 @@ build_authenc_fd(dpaa2_sec_session *sess,
        struct sec_flow_context *flc;
        uint32_t auth_only_len = sym_op->auth.data.length -
                                sym_op->cipher.data.length;
-       int icv_len = sess->digest_length;
+       int icv_len = sess->digest_length, retval;
        uint8_t *old_icv;
-       uint32_t mem_len = (7 * sizeof(struct qbman_fle)) + icv_len;
        uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
                        sess->iv.offset);
 
@@ -98,12 +104,14 @@ build_authenc_fd(dpaa2_sec_session *sess,
         * to get the MBUF Addr from the previous FLE.
         * We can have a better approach to use the inline Mbuf
         */
-       fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE);
-       if (!fle) {
+       retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
+       if (retval) {
                RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n");
                return -1;
        }
+       memset(fle, 0, FLE_POOL_BUF_SIZE);
        DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op));
+       DPAA2_FLE_SAVE_CTXT(fle, priv);
        fle = fle + 1;
        sge = fle + 2;
        if (likely(bpid < MAX_BPID)) {
@@ -214,21 +222,19 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
 {
        struct rte_crypto_sym_op *sym_op = op->sym;
        struct qbman_fle *fle, *sge;
-       uint32_t mem_len = (sess->dir == DIR_ENC) ?
-                          (3 * sizeof(struct qbman_fle)) :
-                          (5 * sizeof(struct qbman_fle) +
-                           sess->digest_length);
        struct sec_flow_context *flc;
        struct ctxt_priv *priv = sess->ctxt;
        uint8_t *old_digest;
+       int retval;
 
        PMD_INIT_FUNC_TRACE();
 
-       fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE);
-       if (!fle) {
-               RTE_LOG(ERR, PMD, "Memory alloc failed for FLE\n");
+       retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
+       if (retval) {
+               RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n");
                return -1;
        }
+       memset(fle, 0, FLE_POOL_BUF_SIZE);
        /* TODO we are using the first FLE entry to store Mbuf.
         * Currently we donot know which FLE has the mbuf stored.
         * So while retreiving we can go back 1 FLE from the FD -ADDR
@@ -236,6 +242,7 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
         * We can have a better approach to use the inline Mbuf
         */
        DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op));
+       DPAA2_FLE_SAVE_CTXT(fle, priv);
        fle = fle + 1;
 
        if (likely(bpid < MAX_BPID)) {
@@ -306,7 +313,7 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
 {
        struct rte_crypto_sym_op *sym_op = op->sym;
        struct qbman_fle *fle, *sge;
-       uint32_t mem_len = (5 * sizeof(struct qbman_fle));
+       int retval;
        struct sec_flow_context *flc;
        struct ctxt_priv *priv = sess->ctxt;
        uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
@@ -314,12 +321,12 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
 
        PMD_INIT_FUNC_TRACE();
 
-       /* todo - we can use some mempool to avoid malloc here */
-       fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE);
-       if (!fle) {
+       retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
+       if (retval) {
                RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n");
                return -1;
        }
+       memset(fle, 0, FLE_POOL_BUF_SIZE);
        /* TODO we are using the first FLE entry to store Mbuf.
         * Currently we donot know which FLE has the mbuf stored.
         * So while retreiving we can go back 1 FLE from the FD -ADDR
@@ -327,6 +334,7 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
         * We can have a better approach to use the inline Mbuf
         */
        DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op));
+       DPAA2_FLE_SAVE_CTXT(fle, priv);
        fle = fle + 1;
        sge = fle + 2;
 
@@ -499,6 +507,7 @@ sec_fd_to_mbuf(const struct qbman_fd *fd)
 {
        struct qbman_fle *fle;
        struct rte_crypto_op *op;
+       struct ctxt_priv *priv;
 
        fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 
@@ -534,7 +543,8 @@ sec_fd_to_mbuf(const struct qbman_fd *fd)
                   DPAA2_GET_FD_LEN(fd));
 
        /* free the fle memory */
-       rte_free(fle - 1);
+       priv = (struct ctxt_priv *)DPAA2_GET_FLE_CTXT(fle - 1);
+       rte_mempool_put(priv->fle_pool, (void *)(fle - 1));
 
        return op;
 }
@@ -764,6 +774,7 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev,
                      dpaa2_sec_session *session)
 {
        struct dpaa2_sec_cipher_ctxt *ctxt = &session->ext_params.cipher_ctxt;
+       struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
        struct alginfo cipherdata;
        int bufsize, i;
        struct ctxt_priv *priv;
@@ -780,6 +791,8 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev,
                return -1;
        }
 
+       priv->fle_pool = dev_priv->fle_pool;
+
        flc = &priv->flc_desc[0].flc;
 
        session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
@@ -875,8 +888,9 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev,
                    dpaa2_sec_session *session)
 {
        struct dpaa2_sec_auth_ctxt *ctxt = &session->ext_params.auth_ctxt;
+       struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
        struct alginfo authdata;
-       unsigned int bufsize;
+       unsigned int bufsize, i;
        struct ctxt_priv *priv;
        struct sec_flow_context *flc;
 
@@ -892,6 +906,7 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev,
                return -1;
        }
 
+       priv->fle_pool = dev_priv->fle_pool;
        flc = &priv->flc_desc[DESC_INITFINAL].flc;
 
        session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
@@ -980,6 +995,9 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev,
                        (uint64_t)&(((struct dpaa2_sec_qp *)
                        dev->data->queue_pairs[0])->rx_vq));
        session->ctxt = priv;
+       for (i = 0; i < bufsize; i++)
+               PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n",
+                           i, priv->flc_desc[0].desc[i]);
 
        return 0;
 
@@ -995,8 +1013,9 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev,
                    dpaa2_sec_session *session)
 {
        struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
+       struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
        struct alginfo authdata, cipherdata;
-       unsigned int bufsize;
+       unsigned int bufsize, i;
        struct ctxt_priv *priv;
        struct sec_flow_context *flc;
        struct rte_crypto_cipher_xform *cipher_xform;
@@ -1032,6 +1051,7 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev,
                return -1;
        }
 
+       priv->fle_pool = dev_priv->fle_pool;
        flc = &priv->flc_desc[0].flc;
 
        session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
@@ -1199,6 +1219,10 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev,
                        (uint64_t)&(((struct dpaa2_sec_qp *)
                        dev->data->queue_pairs[0])->rx_vq));
        session->ctxt = priv;
+       for (i = 0; i < bufsize; i++)
+               PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n",
+                           i, priv->flc_desc[DESC_INITFINAL].desc[i]);
+
 
        return 0;
 
@@ -1496,6 +1520,10 @@ static struct rte_cryptodev_ops crypto_ops = {
 static int
 dpaa2_sec_uninit(const struct rte_cryptodev *dev)
 {
+       struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
+
+       rte_mempool_free(internals->fle_pool);
+
        PMD_INIT_LOG(INFO, "Closing DPAA2_SEC device %s on numa socket %u\n",
                     dev->data->name, rte_socket_id());
 
@@ -1512,6 +1540,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
        uint16_t token;
        struct dpseci_attr attr;
        int retcode, hw_id;
+       char str[20];
 
        PMD_INIT_FUNC_TRACE();
        dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
@@ -1572,6 +1601,20 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
        internals->hw = dpseci;
        internals->token = token;
 
+       sprintf(str, "fle_pool_%d", cryptodev->data->dev_id);
+       internals->fle_pool = rte_mempool_create((const char *)str,
+                       FLE_POOL_NUM_BUFS,
+                       FLE_POOL_BUF_SIZE,
+                       FLE_POOL_CACHE_SIZE, 0,
+                       NULL, NULL, NULL, NULL,
+                       SOCKET_ID_ANY, 0);
+       if (!internals->fle_pool) {
+               RTE_LOG(ERR, PMD, "%s create failed", str);
+               goto init_error;
+       } else
+               RTE_LOG(INFO, PMD, "%s created: %p\n", str,
+                               internals->fle_pool);
+
        PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name);
        return 0;
 

diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
index eda2eec..b4dfe24 100644 (file)
--- a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
+++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h
@@ -40,6 +40,7 @@
 struct dpaa2_sec_dev_private {
        void *mc_portal; /**< MC Portal for configuring this device */
        void *hw; /**< Hardware handle for this device.Used by NADK framework */
+       struct rte_mempool *fle_pool; /* per device memory pool for FLE */
        int32_t hw_id; /**< An unique ID of this device instance */
        int32_t vfio_fd; /**< File descriptor received via VFIO */
        uint16_t token; /**< Token required by DPxxx objects */
@@ -128,6 +129,7 @@ struct sec_flc_desc {
 };
 
 struct ctxt_priv {
+       struct rte_mempool *fle_pool; /* per device memory pool for FLE */
        struct sec_flc_desc flc_desc[0];
 };