From b23d4e898adecb596f5483d12f7224cc4114c374 Mon Sep 17 00:00:00 2001 From: Akhil Goyal Date: Mon, 3 Jul 2017 18:01:36 +0530 Subject: [PATCH] crypto/dpaa2_sec: add per dev mempool to store FLE rte_malloc uses common memory area for all cores. Now rte_malloc are replaced by per device mempool to allocate space for FLE. This removes contention and improves performance. Signed-off-by: Akhil Goyal --- drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c | 79 ++++++++++++++++----- drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h | 2 + 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 36207515c6..dbdaf4619f 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -71,6 +71,13 @@ #define NO_PREFETCH 0 #define TDES_CBC_IV_LEN 8 #define AES_CBC_IV_LEN 16 +#define AES_CTR_IV_LEN 16 +#define AES_GCM_IV_LEN 12 +/* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */ +#define FLE_POOL_NUM_BUFS 32000 +#define FLE_POOL_BUF_SIZE 256 +#define FLE_POOL_CACHE_SIZE 512 + enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8; static inline int @@ -84,9 +91,8 @@ build_authenc_fd(dpaa2_sec_session *sess, struct sec_flow_context *flc; uint32_t auth_only_len = sym_op->auth.data.length - sym_op->cipher.data.length; - int icv_len = sess->digest_length; + int icv_len = sess->digest_length, retval; uint8_t *old_icv; - uint32_t mem_len = (7 * sizeof(struct qbman_fle)) + icv_len; uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv.offset); @@ -98,12 +104,14 @@ build_authenc_fd(dpaa2_sec_session *sess, * to get the MBUF Addr from the previous FLE. * We can have a better approach to use the inline Mbuf */ - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; sge = fle + 2; if (likely(bpid < MAX_BPID)) { @@ -214,21 +222,19 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, { struct rte_crypto_sym_op *sym_op = op->sym; struct qbman_fle *fle, *sge; - uint32_t mem_len = (sess->dir == DIR_ENC) ? - (3 * sizeof(struct qbman_fle)) : - (5 * sizeof(struct qbman_fle) + - sess->digest_length); struct sec_flow_context *flc; struct ctxt_priv *priv = sess->ctxt; uint8_t *old_digest; + int retval; PMD_INIT_FUNC_TRACE(); - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { - RTE_LOG(ERR, PMD, "Memory alloc failed for FLE\n"); + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); /* TODO we are using the first FLE entry to store Mbuf. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -236,6 +242,7 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, * We can have a better approach to use the inline Mbuf */ DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; if (likely(bpid < MAX_BPID)) { @@ -306,7 +313,7 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, { struct rte_crypto_sym_op *sym_op = op->sym; struct qbman_fle *fle, *sge; - uint32_t mem_len = (5 * sizeof(struct qbman_fle)); + int retval; struct sec_flow_context *flc; struct ctxt_priv *priv = sess->ctxt; uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, @@ -314,12 +321,12 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, PMD_INIT_FUNC_TRACE(); - /* todo - we can use some mempool to avoid malloc here */ - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); /* TODO we are using the first FLE entry to store Mbuf. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -327,6 +334,7 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, * We can have a better approach to use the inline Mbuf */ DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; sge = fle + 2; @@ -499,6 +507,7 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) { struct qbman_fle *fle; struct rte_crypto_op *op; + struct ctxt_priv *priv; fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); @@ -534,7 +543,8 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) DPAA2_GET_FD_LEN(fd)); /* free the fle memory */ - rte_free(fle - 1); + priv = (struct ctxt_priv *)DPAA2_GET_FLE_CTXT(fle - 1); + rte_mempool_put(priv->fle_pool, (void *)(fle - 1)); return op; } @@ -764,6 +774,7 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, dpaa2_sec_session *session) { struct dpaa2_sec_cipher_ctxt *ctxt = &session->ext_params.cipher_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo cipherdata; int bufsize, i; struct ctxt_priv *priv; @@ -780,6 +791,8 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, return -1; } + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[0].flc; session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, @@ -875,8 +888,9 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, dpaa2_sec_session *session) { struct dpaa2_sec_auth_ctxt *ctxt = &session->ext_params.auth_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo authdata; - unsigned int bufsize; + unsigned int bufsize, i; struct ctxt_priv *priv; struct sec_flow_context *flc; @@ -892,6 +906,7 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, return -1; } + priv->fle_pool = dev_priv->fle_pool; flc = &priv->flc_desc[DESC_INITFINAL].flc; session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, @@ -980,6 +995,9 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, (uint64_t)&(((struct dpaa2_sec_qp *) dev->data->queue_pairs[0])->rx_vq)); session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); return 0; @@ -995,8 +1013,9 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, dpaa2_sec_session *session) { struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo authdata, cipherdata; - unsigned int bufsize; + unsigned int bufsize, i; struct ctxt_priv *priv; struct sec_flow_context *flc; struct rte_crypto_cipher_xform *cipher_xform; @@ -1032,6 +1051,7 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, return -1; } + priv->fle_pool = dev_priv->fle_pool; flc = &priv->flc_desc[0].flc; session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, @@ -1199,6 +1219,10 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, (uint64_t)&(((struct dpaa2_sec_qp *) dev->data->queue_pairs[0])->rx_vq)); session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[DESC_INITFINAL].desc[i]); + return 0; @@ -1496,6 +1520,10 @@ static struct rte_cryptodev_ops crypto_ops = { static int dpaa2_sec_uninit(const struct rte_cryptodev *dev) { + struct dpaa2_sec_dev_private *internals = dev->data->dev_private; + + rte_mempool_free(internals->fle_pool); + PMD_INIT_LOG(INFO, "Closing DPAA2_SEC device %s on numa socket %u\n", dev->data->name, rte_socket_id()); @@ -1512,6 +1540,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) uint16_t token; struct dpseci_attr attr; int retcode, hw_id; + char str[20]; PMD_INIT_FUNC_TRACE(); dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device); @@ -1572,6 +1601,20 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) internals->hw = dpseci; internals->token = token; + sprintf(str, "fle_pool_%d", cryptodev->data->dev_id); + internals->fle_pool = rte_mempool_create((const char *)str, + FLE_POOL_NUM_BUFS, + FLE_POOL_BUF_SIZE, + FLE_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!internals->fle_pool) { + RTE_LOG(ERR, PMD, "%s create failed", str); + goto init_error; + } else + RTE_LOG(INFO, PMD, "%s created: %p\n", str, + internals->fle_pool); + PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); return 0; diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h index eda2eec985..b4dfe2423d 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h @@ -40,6 +40,7 @@ struct dpaa2_sec_dev_private { void *mc_portal; /**< MC Portal for configuring this device */ void *hw; /**< Hardware handle for this device.Used by NADK framework */ + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ int32_t hw_id; /**< An unique ID of this device instance */ int32_t vfio_fd; /**< File descriptor received via VFIO */ uint16_t token; /**< Token required by DPxxx objects */ @@ -128,6 +129,7 @@ struct sec_flc_desc { }; struct ctxt_priv { + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ struct sec_flc_desc flc_desc[0]; }; -- 2.20.1