X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fdpaa_sec%2Fdpaa_sec.c;h=27a9226767243bf2550389affbea406034389f08;hb=98e84273902045640b2c6ecc438412c4f1202e15;hp=8305f19a312e4be8266bab0fffaa07eebf8e6b6c;hpb=e621d970006787e28e0f9d9593a9f48a71071d3d;p=dpdk.git diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c index 8305f19a31..27a9226767 100644 --- a/drivers/crypto/dpaa_sec/dpaa_sec.c +++ b/drivers/crypto/dpaa_sec/dpaa_sec.c @@ -37,7 +37,9 @@ #include #include +#include #include +#include enum rta_sec_era rta_sec_era; @@ -60,18 +62,17 @@ dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status); ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; } - - /* report op status to sym->op and then free the ctx memeory */ - rte_mempool_put(ctx->ctx_pool, (void *)ctx); } static inline struct dpaa_sec_op_ctx * -dpaa_sec_alloc_ctx(dpaa_sec_session *ses) +dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count) { struct dpaa_sec_op_ctx *ctx; - int retval; + int i, retval; - retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); + retval = rte_mempool_get( + ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool, + (void **)(&ctx)); if (!ctx || retval) { DPAA_SEC_DP_WARN("Alloc sec descriptor failed!"); return NULL; @@ -82,14 +83,11 @@ dpaa_sec_alloc_ctx(dpaa_sec_session *ses) * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for * each packet, memset is costlier than dcbz_64(). */ - dcbz_64(&ctx->job.sg[SG_CACHELINE_0]); - dcbz_64(&ctx->job.sg[SG_CACHELINE_1]); - dcbz_64(&ctx->job.sg[SG_CACHELINE_2]); - dcbz_64(&ctx->job.sg[SG_CACHELINE_3]); + for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4) + dcbz_64(&ctx->job.sg[i]); - ctx->ctx_pool = ses->ctx_pool; - ctx->vtop_offset = (size_t) ctx - - rte_mempool_virt2iova(ctx); + ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool; + ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx); return ctx; } @@ -100,8 +98,10 @@ dpaa_mem_vtop(void *vaddr) const struct rte_memseg *ms; ms = rte_mem_virt2memseg(vaddr, NULL); - if (ms) + if (ms) { + dpaax_iova_table_update(ms->iova, ms->addr, ms->len); return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr); + } return (size_t)NULL; } @@ -188,12 +188,18 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { struct qm_sg_entry *sg_out; uint32_t len; + struct rte_mbuf *mbuf = (ctx->op->sym->m_dst == NULL) ? + ctx->op->sym->m_src : ctx->op->sym->m_dst; sg_out = &job->sg[0]; hw_sg_to_cpu(sg_out); len = sg_out->length; - ctx->op->sym->m_src->pkt_len = len; - ctx->op->sym->m_src->data_len = len; + mbuf->pkt_len = len; + while (mbuf->next != NULL) { + len -= mbuf->data_len; + mbuf = mbuf->next; + } + mbuf->data_len = len; } dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; dpaa_sec_op_ending(ctx); @@ -259,7 +265,9 @@ static inline int is_auth_cipher(dpaa_sec_session *ses) { return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) && - (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC)); + (ses->proto_alg != RTE_SECURITY_PROTOCOL_PDCP) && + (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC) && + (ses->aead_alg == 0)); } static inline int is_proto_ipsec(dpaa_sec_session *ses) @@ -383,6 +391,7 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) { struct alginfo authdata = {0}, cipherdata = {0}; struct sec_cdb *cdb = &ses->cdb; + struct alginfo *p_authdata = NULL; int32_t shared_desc_len = 0; int err; #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN @@ -415,7 +424,11 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) cipherdata.key_enc_flags = 0; cipherdata.key_type = RTA_DATA_IMM; - if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) { + cdb->sh_desc[0] = cipherdata.keylen; + cdb->sh_desc[1] = 0; + cdb->sh_desc[2] = 0; + + if (ses->auth_alg) { switch (ses->auth_alg) { case RTE_CRYPTO_AUTH_SNOW3G_UIA2: authdata.algtype = PDCP_AUTH_TYPE_SNOW; @@ -440,36 +453,41 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) authdata.key_enc_flags = 0; authdata.key_type = RTA_DATA_IMM; - cdb->sh_desc[0] = cipherdata.keylen; + p_authdata = &authdata; + cdb->sh_desc[1] = authdata.keylen; - err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, - (unsigned int *)cdb->sh_desc, - &cdb->sh_desc[2], 2); + } - if (err < 0) { - DPAA_SEC_ERR("Crypto: Incorrect key lengths"); - return err; - } - if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) { - cipherdata.key = (size_t)dpaa_mem_vtop( - (void *)(size_t)cipherdata.key); - cipherdata.key_type = RTA_DATA_PTR; - } - if (!(cdb->sh_desc[2] & (1<<1)) && authdata.keylen) { - authdata.key = (size_t)dpaa_mem_vtop( - (void *)(size_t)authdata.key); - authdata.key_type = RTA_DATA_PTR; - } + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)cdb->sh_desc, + &cdb->sh_desc[2], 2); + if (err < 0) { + DPAA_SEC_ERR("Crypto: Incorrect key lengths"); + return err; + } - cdb->sh_desc[0] = 0; - cdb->sh_desc[1] = 0; - cdb->sh_desc[2] = 0; + if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) { + cipherdata.key = + (size_t)dpaa_mem_vtop((void *)(size_t)cipherdata.key); + cipherdata.key_type = RTA_DATA_PTR; + } + if (!(cdb->sh_desc[2] & (1 << 1)) && authdata.keylen) { + authdata.key = + (size_t)dpaa_mem_vtop((void *)(size_t)authdata.key); + authdata.key_type = RTA_DATA_PTR; + } + cdb->sh_desc[0] = 0; + cdb->sh_desc[1] = 0; + cdb->sh_desc[2] = 0; + + if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) { if (ses->dir == DIR_ENC) shared_desc_len = cnstr_shdsc_pdcp_c_plane_encap( cdb->sh_desc, 1, swap, ses->pdcp.hfn, + ses->pdcp.sn_size, ses->pdcp.bearer, ses->pdcp.pkt_dir, ses->pdcp.hfn_threshold, @@ -479,31 +497,13 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) shared_desc_len = cnstr_shdsc_pdcp_c_plane_decap( cdb->sh_desc, 1, swap, ses->pdcp.hfn, + ses->pdcp.sn_size, ses->pdcp.bearer, ses->pdcp.pkt_dir, ses->pdcp.hfn_threshold, &cipherdata, &authdata, 0); } else { - cdb->sh_desc[0] = cipherdata.keylen; - err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, - (unsigned int *)cdb->sh_desc, - &cdb->sh_desc[2], 1); - - if (err < 0) { - DPAA_SEC_ERR("Crypto: Incorrect key lengths"); - return err; - } - if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) { - cipherdata.key = (size_t)dpaa_mem_vtop( - (void *)(size_t)cipherdata.key); - cipherdata.key_type = RTA_DATA_PTR; - } - cdb->sh_desc[0] = 0; - cdb->sh_desc[1] = 0; - cdb->sh_desc[2] = 0; - if (ses->dir == DIR_ENC) shared_desc_len = cnstr_shdsc_pdcp_u_plane_encap( cdb->sh_desc, 1, swap, @@ -512,7 +512,7 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) ses->pdcp.bearer, ses->pdcp.pkt_dir, ses->pdcp.hfn_threshold, - &cipherdata, 0); + &cipherdata, p_authdata, 0); else if (ses->dir == DIR_DEC) shared_desc_len = cnstr_shdsc_pdcp_u_plane_decap( cdb->sh_desc, 1, swap, @@ -521,7 +521,7 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) ses->pdcp.bearer, ses->pdcp.pkt_dir, ses->pdcp.hfn_threshold, - &cipherdata, 0); + &cipherdata, p_authdata, 0); } return shared_desc_len; @@ -809,12 +809,18 @@ dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { struct qm_sg_entry *sg_out; uint32_t len; + struct rte_mbuf *mbuf = (op->sym->m_dst == NULL) ? + op->sym->m_src : op->sym->m_dst; sg_out = &job->sg[0]; hw_sg_to_cpu(sg_out); len = sg_out->length; - op->sym->m_src->pkt_len = len; - op->sym->m_src->data_len = len; + mbuf->pkt_len = len; + while (mbuf->next != NULL) { + len -= mbuf->data_len; + mbuf = mbuf->next; + } + mbuf->data_len = len; } if (!ctx->fd_status) { op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -849,12 +855,12 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) else extra_segs = 2; - if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) { + if (mbuf->nb_segs > MAX_SG_ENTRIES) { DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d", MAX_SG_ENTRIES); return NULL; } - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, mbuf->nb_segs + extra_segs); if (!ctx) return NULL; @@ -932,7 +938,7 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) rte_iova_t start_addr; uint8_t *old_digest; - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, 4); if (!ctx) return NULL; @@ -1002,13 +1008,13 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) req_segs = mbuf->nb_segs * 2 + 3; } - if (req_segs > MAX_SG_ENTRIES) { + if (mbuf->nb_segs > MAX_SG_ENTRIES) { DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d", MAX_SG_ENTRIES); return NULL; } - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, req_segs); if (!ctx) return NULL; @@ -1088,7 +1094,7 @@ build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, ses->iv.offset); - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, 4); if (!ctx) return NULL; @@ -1155,13 +1161,13 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) if (ses->auth_only_len) req_segs++; - if (req_segs > MAX_SG_ENTRIES) { + if (mbuf->nb_segs > MAX_SG_ENTRIES) { DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d", MAX_SG_ENTRIES); return NULL; } - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, req_segs); if (!ctx) return NULL; @@ -1290,7 +1296,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) else dst_start_addr = src_start_addr; - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, 7); if (!ctx) return NULL; @@ -1403,13 +1409,13 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) req_segs = mbuf->nb_segs * 2 + 4; } - if (req_segs > MAX_SG_ENTRIES) { + if (mbuf->nb_segs > MAX_SG_ENTRIES) { DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d", MAX_SG_ENTRIES); return NULL; } - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, req_segs); if (!ctx) return NULL; @@ -1527,7 +1533,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) else dst_start_addr = src_start_addr; - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, 7); if (!ctx) return NULL; @@ -1613,7 +1619,7 @@ build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) struct qm_sg_entry *sg; phys_addr_t src_start_addr, dst_start_addr; - ctx = dpaa_sec_alloc_ctx(ses); + ctx = dpaa_sec_alloc_ctx(ses, 2); if (!ctx) return NULL; cf = &ctx->job; @@ -1643,6 +1649,99 @@ build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) return cf; } +static inline struct dpaa_sec_job * +build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg, *out_sg, *in_sg; + struct rte_mbuf *mbuf; + uint8_t req_segs; + uint32_t in_len = 0, out_len = 0; + + if (sym->m_dst) + mbuf = sym->m_dst; + else + mbuf = sym->m_src; + + req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2; + if (mbuf->nb_segs > MAX_SG_ENTRIES) { + DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d", + MAX_SG_ENTRIES); + return NULL; + } + + ctx = dpaa_sec_alloc_ctx(ses, req_segs); + if (!ctx) + return NULL; + cf = &ctx->job; + ctx->op = op; + /* output */ + out_sg = &cf->sg[0]; + out_sg->extension = 1; + qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2])); + + /* 1st seg */ + sg = &cf->sg[2]; + qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + sg->offset = 0; + + /* Successive segs */ + while (mbuf->next) { + sg->length = mbuf->data_len; + out_len += sg->length; + mbuf = mbuf->next; + cpu_to_hw_sg(sg); + sg++; + qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + sg->offset = 0; + } + sg->length = mbuf->buf_len - mbuf->data_off; + out_len += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + + out_sg->length = out_len; + cpu_to_hw_sg(out_sg); + + /* input */ + mbuf = sym->m_src; + in_sg = &cf->sg[1]; + in_sg->extension = 1; + in_sg->final = 1; + in_len = mbuf->data_len; + + sg++; + qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + + /* 1st seg */ + qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + sg->length = mbuf->data_len; + sg->offset = 0; + + /* Successive segs */ + mbuf = mbuf->next; + while (mbuf) { + cpu_to_hw_sg(sg); + sg++; + qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + sg->length = mbuf->data_len; + sg->offset = 0; + in_len += sg->length; + mbuf = mbuf->next; + } + sg->final = 1; + cpu_to_hw_sg(sg); + + in_sg->length = in_len; + cpu_to_hw_sg(in_sg); + + sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK; + + return cf; +} + static uint16_t dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -1656,7 +1755,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, struct rte_crypto_op *op; struct dpaa_sec_job *cf; dpaa_sec_session *ses; - uint32_t auth_only_len; + uint32_t auth_only_len, index, flags[DPAA_SEC_BURST] = {0}; struct qman_fq *inq[DPAA_SEC_BURST]; while (nb_ops) { @@ -1664,6 +1763,18 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, DPAA_SEC_BURST : nb_ops; for (loop = 0; loop < frames_to_send; loop++) { op = *(ops++); + if (op->sym->m_src->seqn != 0) { + index = op->sym->m_src->seqn - 1; + if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { + /* QM_EQCR_DCA_IDXMASK = 0x0f */ + flags[loop] = ((index & 0x0f) << 8); + flags[loop] |= QMAN_ENQUEUE_FLAG_DCA; + DPAA_PER_LCORE_DQRR_SIZE--; + DPAA_PER_LCORE_DQRR_HELD &= + ~(1 << index); + } + } + switch (op->sess_type) { case RTE_CRYPTO_OP_WITH_SESSION: ses = (dpaa_sec_session *) @@ -1702,7 +1813,9 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, auth_only_len = op->sym->auth.data.length - op->sym->cipher.data.length; - if (rte_pktmbuf_is_contiguous(op->sym->m_src)) { + if (rte_pktmbuf_is_contiguous(op->sym->m_src) && + ((op->sym->m_dst == NULL) || + rte_pktmbuf_is_contiguous(op->sym->m_dst))) { if (is_proto_ipsec(ses)) { cf = build_proto(op, ses); } else if (is_proto_pdcp(ses)) { @@ -1723,7 +1836,9 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, goto send_pkts; } } else { - if (is_auth_only(ses)) { + if (is_proto_pdcp(ses) || is_proto_ipsec(ses)) { + cf = build_proto_sg(op, ses); + } else if (is_auth_only(ses)) { cf = build_auth_only_sg(op, ses); } else if (is_cipher_only(ses)) { cf = build_cipher_only_sg(op, ses); @@ -1759,12 +1874,26 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, if (auth_only_len) fd->cmd = 0x80000000 | auth_only_len; + /* In case of PDCP, per packet HFN is stored in + * mbuf priv after sym_op. + */ + if (is_proto_pdcp(ses) && ses->pdcp.hfn_ovd) { + fd->cmd = 0x80000000 | + *((uint32_t *)((uint8_t *)op + + ses->pdcp.hfn_ovd_offset)); + DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u,%u\n", + *((uint32_t *)((uint8_t *)op + + ses->pdcp.hfn_ovd_offset)), + ses->pdcp.hfn_ovd, + is_proto_pdcp(ses)); + } + } send_pkts: loop = 0; while (loop < frames_to_send) { loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop], - frames_to_send - loop); + &flags[loop], frames_to_send - loop); } nb_ops -= frames_to_send; num_tx += frames_to_send; @@ -1813,6 +1942,7 @@ dpaa_sec_queue_pair_release(struct rte_cryptodev *dev, } qp = &internals->qps[qp_id]; + rte_mempool_free(qp->ctx_pool); qp->internals = NULL; dev->data->queue_pairs[qp_id] = NULL; @@ -1827,6 +1957,7 @@ dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, { struct dpaa_sec_dev_private *internals; struct dpaa_sec_qp *qp = NULL; + char str[20]; DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf); @@ -1839,6 +1970,22 @@ dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, qp = &internals->qps[qp_id]; qp->internals = internals; + snprintf(str, sizeof(str), "ctx_pool_d%d_qp%d", + dev->data->dev_id, qp_id); + if (!qp->ctx_pool) { + qp->ctx_pool = rte_mempool_create((const char *)str, + CTX_POOL_NUM_BUFS, + CTX_POOL_BUF_SIZE, + CTX_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!qp->ctx_pool) { + DPAA_SEC_ERR("%s create failed\n", str); + return -ENOMEM; + } + } else + DPAA_SEC_INFO("mempool already created for dev_id : %d, qp: %d", + dev->data->dev_id, qp_id); dev->data->queue_pairs[qp_id] = qp; return 0; @@ -2055,7 +2202,6 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, DPAA_SEC_ERR("Invalid crypto type"); return -EINVAL; } - session->ctx_pool = internals->ctx_pool; rte_spinlock_lock(&internals->lock); for (i = 0; i < MAX_DPAA_CORES; i++) { session->inq[i] = dpaa_sec_attach_rxq(internals); @@ -2109,33 +2255,38 @@ dpaa_sec_sym_session_configure(struct rte_cryptodev *dev, return 0; } +static inline void +free_session_memory(struct rte_cryptodev *dev, dpaa_sec_session *s) +{ + struct dpaa_sec_dev_private *qi = dev->data->dev_private; + struct rte_mempool *sess_mp = rte_mempool_from_obj((void *)s); + uint8_t i; + + for (i = 0; i < MAX_DPAA_CORES; i++) { + if (s->inq[i]) + dpaa_sec_detach_rxq(qi, s->inq[i]); + s->inq[i] = NULL; + s->qp[i] = NULL; + } + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(s, 0, sizeof(dpaa_sec_session)); + rte_mempool_put(sess_mp, (void *)s); +} + /** Clear the memory of session so it doesn't leave key material behind */ static void dpaa_sec_sym_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess) { - struct dpaa_sec_dev_private *qi = dev->data->dev_private; - uint8_t index = dev->driver_id, i; - void *sess_priv = get_sym_session_private_data(sess, index); - PMD_INIT_FUNC_TRACE(); - + uint8_t index = dev->driver_id; + void *sess_priv = get_sym_session_private_data(sess, index); dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; if (sess_priv) { - struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); - - for (i = 0; i < MAX_DPAA_CORES; i++) { - if (s->inq[i]) - dpaa_sec_detach_rxq(qi, s->inq[i]); - s->inq[i] = NULL; - s->qp[i] = NULL; - } - rte_free(s->cipher_key.data); - rte_free(s->auth_key.data); - memset(s, 0, sizeof(dpaa_sec_session)); + free_session_memory(dev, s); set_sym_session_private_data(sess, index, NULL); - rte_mempool_put(sess_mp, sess_priv); } } @@ -2230,44 +2381,81 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, } if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { - memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) + + if (ipsec_xform->tunnel.type == + RTE_SECURITY_IPSEC_TUNNEL_IPV4) { + memset(&session->encap_pdb, 0, + sizeof(struct ipsec_encap_pdb) + sizeof(session->ip4_hdr)); - session->ip4_hdr.ip_v = IPVERSION; - session->ip4_hdr.ip_hl = 5; - session->ip4_hdr.ip_len = rte_cpu_to_be_16( + session->ip4_hdr.ip_v = IPVERSION; + session->ip4_hdr.ip_hl = 5; + session->ip4_hdr.ip_len = rte_cpu_to_be_16( sizeof(session->ip4_hdr)); - session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; - session->ip4_hdr.ip_id = 0; - session->ip4_hdr.ip_off = 0; - session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; - session->ip4_hdr.ip_p = (ipsec_xform->proto == - RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP - : IPPROTO_AH; - session->ip4_hdr.ip_sum = 0; - session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; - session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; - session->ip4_hdr.ip_sum = calc_chksum((uint16_t *) + session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; + session->ip4_hdr.ip_id = 0; + session->ip4_hdr.ip_off = 0; + session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; + session->ip4_hdr.ip_p = (ipsec_xform->proto == + RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? + IPPROTO_ESP : IPPROTO_AH; + session->ip4_hdr.ip_sum = 0; + session->ip4_hdr.ip_src = + ipsec_xform->tunnel.ipv4.src_ip; + session->ip4_hdr.ip_dst = + ipsec_xform->tunnel.ipv4.dst_ip; + session->ip4_hdr.ip_sum = calc_chksum((uint16_t *) (void *)&session->ip4_hdr, sizeof(struct ip)); - + session->encap_pdb.ip_hdr_len = sizeof(struct ip); + } else if (ipsec_xform->tunnel.type == + RTE_SECURITY_IPSEC_TUNNEL_IPV6) { + memset(&session->encap_pdb, 0, + sizeof(struct ipsec_encap_pdb) + + sizeof(session->ip6_hdr)); + session->ip6_hdr.vtc_flow = rte_cpu_to_be_32( + DPAA_IPv6_DEFAULT_VTC_FLOW | + ((ipsec_xform->tunnel.ipv6.dscp << + RTE_IPV6_HDR_TC_SHIFT) & + RTE_IPV6_HDR_TC_MASK) | + ((ipsec_xform->tunnel.ipv6.flabel << + RTE_IPV6_HDR_FL_SHIFT) & + RTE_IPV6_HDR_FL_MASK)); + /* Payload length will be updated by HW */ + session->ip6_hdr.payload_len = 0; + session->ip6_hdr.hop_limits = + ipsec_xform->tunnel.ipv6.hlimit; + session->ip6_hdr.proto = (ipsec_xform->proto == + RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? + IPPROTO_ESP : IPPROTO_AH; + memcpy(&session->ip6_hdr.src_addr, + &ipsec_xform->tunnel.ipv6.src_addr, 16); + memcpy(&session->ip6_hdr.dst_addr, + &ipsec_xform->tunnel.ipv6.dst_addr, 16); + session->encap_pdb.ip_hdr_len = + sizeof(struct rte_ipv6_hdr); + } session->encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | PDBOPTS_ESP_OIHI_PDB_INL | PDBOPTS_ESP_IVSRC | PDBHMO_ESP_ENCAP_DTTL | PDBHMO_ESP_SNR; + if (ipsec_xform->options.esn) + session->encap_pdb.options |= PDBOPTS_ESP_ESN; session->encap_pdb.spi = ipsec_xform->spi; - session->encap_pdb.ip_hdr_len = sizeof(struct ip); - session->dir = DIR_ENC; } else if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); - session->decap_pdb.options = sizeof(struct ip) << 16; + if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) + session->decap_pdb.options = sizeof(struct ip) << 16; + else + session->decap_pdb.options = + sizeof(struct rte_ipv6_hdr) << 16; + if (ipsec_xform->options.esn) + session->decap_pdb.options |= PDBOPTS_ESP_ESN; session->dir = DIR_DEC; } else goto out; - session->ctx_pool = internals->ctx_pool; rte_spinlock_lock(&internals->lock); for (i = 0; i < MAX_DPAA_CORES; i++) { session->inq[i] = dpaa_sec_attach_rxq(internals); @@ -2341,44 +2529,43 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, session->dir = DIR_ENC; } - /* Auth is only applicable for control mode operation. */ if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) { - if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5) { + if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5 && + pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_12) { DPAA_SEC_ERR( - "PDCP Seq Num size should be 5 bits for cmode"); + "PDCP Seq Num size should be 5/12 bits for cmode"); goto out; } - if (auth_xform) { - session->auth_key.data = rte_zmalloc(NULL, - auth_xform->key.length, - RTE_CACHE_LINE_SIZE); - if (session->auth_key.data == NULL && - auth_xform->key.length > 0) { - DPAA_SEC_ERR("No Memory for auth key"); - rte_free(session->cipher_key.data); - return -ENOMEM; - } - session->auth_key.length = auth_xform->key.length; - memcpy(session->auth_key.data, auth_xform->key.data, - auth_xform->key.length); - session->auth_alg = auth_xform->algo; - } else { - session->auth_key.data = NULL; - session->auth_key.length = 0; - session->auth_alg = RTE_CRYPTO_AUTH_NULL; + } + + if (auth_xform) { + session->auth_key.data = rte_zmalloc(NULL, + auth_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (!session->auth_key.data && + auth_xform->key.length > 0) { + DPAA_SEC_ERR("No Memory for auth key"); + rte_free(session->cipher_key.data); + return -ENOMEM; } + session->auth_key.length = auth_xform->key.length; + memcpy(session->auth_key.data, auth_xform->key.data, + auth_xform->key.length); + session->auth_alg = auth_xform->algo; + } else { + session->auth_key.data = NULL; + session->auth_key.length = 0; + session->auth_alg = 0; } session->pdcp.domain = pdcp_xform->domain; session->pdcp.bearer = pdcp_xform->bearer; session->pdcp.pkt_dir = pdcp_xform->pkt_dir; session->pdcp.sn_size = pdcp_xform->sn_size; -#ifdef ENABLE_HFN_OVERRIDE - session->pdcp.hfn_ovd = pdcp_xform->hfn_ovd; -#endif session->pdcp.hfn = pdcp_xform->hfn; session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold; + session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd; + session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset; - session->ctx_pool = dev_priv->ctx_pool; rte_spinlock_lock(&dev_priv->lock); for (i = 0; i < MAX_DPAA_CORES; i++) { session->inq[i] = dpaa_sec_attach_rxq(dev_priv); @@ -2445,49 +2632,21 @@ dpaa_sec_security_session_destroy(void *dev __rte_unused, { PMD_INIT_FUNC_TRACE(); void *sess_priv = get_sec_session_private_data(sess); - dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; if (sess_priv) { - struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); - - rte_free(s->cipher_key.data); - rte_free(s->auth_key.data); - memset(sess, 0, sizeof(dpaa_sec_session)); + free_session_memory((struct rte_cryptodev *)dev, s); set_sec_session_private_data(sess, NULL); - rte_mempool_put(sess_mp, sess_priv); } return 0; } - static int -dpaa_sec_dev_configure(struct rte_cryptodev *dev, +dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, struct rte_cryptodev_config *config __rte_unused) { - - char str[20]; - struct dpaa_sec_dev_private *internals; - PMD_INIT_FUNC_TRACE(); - internals = dev->data->dev_private; - snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id); - if (!internals->ctx_pool) { - internals->ctx_pool = rte_mempool_create((const char *)str, - CTX_POOL_NUM_BUFS, - CTX_POOL_BUF_SIZE, - CTX_POOL_CACHE_SIZE, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - if (!internals->ctx_pool) { - DPAA_SEC_ERR("%s create failed\n", str); - return -ENOMEM; - } - } else - DPAA_SEC_INFO("mempool already created for dev_id : %d", - dev->data->dev_id); - return 0; } @@ -2507,17 +2666,11 @@ dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused) static int dpaa_sec_dev_close(struct rte_cryptodev *dev) { - struct dpaa_sec_dev_private *internals; - PMD_INIT_FUNC_TRACE(); if (dev == NULL) return -ENOMEM; - internals = dev->data->dev_private; - rte_mempool_free(internals->ctx_pool); - internals->ctx_pool = NULL; - return 0; } @@ -2537,6 +2690,188 @@ dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, } } +static enum qman_cb_dqrr_result +dpaa_sec_process_parallel_event(void *event, + struct qman_portal *qm __always_unused, + struct qman_fq *outq, + const struct qm_dqrr_entry *dqrr, + void **bufs) +{ + const struct qm_fd *fd; + struct dpaa_sec_job *job; + struct dpaa_sec_op_ctx *ctx; + struct rte_event *ev = (struct rte_event *)event; + + fd = &dqrr->fd; + + /* sg is embedded in an op ctx, + * sg[0] is for output + * sg[1] for input + */ + job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + + ctx = container_of(job, struct dpaa_sec_op_ctx, job); + ctx->fd_status = fd->status; + if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { + struct qm_sg_entry *sg_out; + uint32_t len; + + sg_out = &job->sg[0]; + hw_sg_to_cpu(sg_out); + len = sg_out->length; + ctx->op->sym->m_src->pkt_len = len; + ctx->op->sym->m_src->data_len = len; + } + if (!ctx->fd_status) { + ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } else { + DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status); + ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; + } + ev->event_ptr = (void *)ctx->op; + + ev->flow_id = outq->ev.flow_id; + ev->sub_event_type = outq->ev.sub_event_type; + ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; + ev->op = RTE_EVENT_OP_NEW; + ev->sched_type = outq->ev.sched_type; + ev->queue_id = outq->ev.queue_id; + ev->priority = outq->ev.priority; + *bufs = (void *)ctx->op; + + rte_mempool_put(ctx->ctx_pool, (void *)ctx); + + return qman_cb_dqrr_consume; +} + +static enum qman_cb_dqrr_result +dpaa_sec_process_atomic_event(void *event, + struct qman_portal *qm __rte_unused, + struct qman_fq *outq, + const struct qm_dqrr_entry *dqrr, + void **bufs) +{ + u8 index; + const struct qm_fd *fd; + struct dpaa_sec_job *job; + struct dpaa_sec_op_ctx *ctx; + struct rte_event *ev = (struct rte_event *)event; + + fd = &dqrr->fd; + + /* sg is embedded in an op ctx, + * sg[0] is for output + * sg[1] for input + */ + job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + + ctx = container_of(job, struct dpaa_sec_op_ctx, job); + ctx->fd_status = fd->status; + if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { + struct qm_sg_entry *sg_out; + uint32_t len; + + sg_out = &job->sg[0]; + hw_sg_to_cpu(sg_out); + len = sg_out->length; + ctx->op->sym->m_src->pkt_len = len; + ctx->op->sym->m_src->data_len = len; + } + if (!ctx->fd_status) { + ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } else { + DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status); + ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; + } + ev->event_ptr = (void *)ctx->op; + ev->flow_id = outq->ev.flow_id; + ev->sub_event_type = outq->ev.sub_event_type; + ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; + ev->op = RTE_EVENT_OP_NEW; + ev->sched_type = outq->ev.sched_type; + ev->queue_id = outq->ev.queue_id; + ev->priority = outq->ev.priority; + + /* Save active dqrr entries */ + index = ((uintptr_t)dqrr >> 6) & (16/*QM_DQRR_SIZE*/ - 1); + DPAA_PER_LCORE_DQRR_SIZE++; + DPAA_PER_LCORE_DQRR_HELD |= 1 << index; + DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src; + ev->impl_opaque = index + 1; + ctx->op->sym->m_src->seqn = (uint32_t)index + 1; + *bufs = (void *)ctx->op; + + rte_mempool_put(ctx->ctx_pool, (void *)ctx); + + return qman_cb_dqrr_defer; +} + +int +dpaa_sec_eventq_attach(const struct rte_cryptodev *dev, + int qp_id, + uint16_t ch_id, + const struct rte_event *event) +{ + struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id]; + struct qm_mcc_initfq opts = {0}; + + int ret; + + opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; + opts.fqd.dest.channel = ch_id; + + switch (event->sched_type) { + case RTE_SCHED_TYPE_ATOMIC: + opts.fqd.fq_ctrl |= QM_FQCTRL_HOLDACTIVE; + /* Reset FQCTRL_AVOIDBLOCK bit as it is unnecessary + * configuration with HOLD_ACTIVE setting + */ + opts.fqd.fq_ctrl &= (~QM_FQCTRL_AVOIDBLOCK); + qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_atomic_event; + break; + case RTE_SCHED_TYPE_ORDERED: + DPAA_SEC_ERR("Ordered queue schedule type is not supported\n"); + return -1; + default: + opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK; + qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_parallel_event; + break; + } + + ret = qman_init_fq(&qp->outq, QMAN_INITFQ_FLAG_SCHED, &opts); + if (unlikely(ret)) { + DPAA_SEC_ERR("unable to init caam source fq!"); + return ret; + } + + memcpy(&qp->outq.ev, event, sizeof(struct rte_event)); + + return 0; +} + +int +dpaa_sec_eventq_detach(const struct rte_cryptodev *dev, + int qp_id) +{ + struct qm_mcc_initfq opts = {0}; + int ret; + struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id]; + + opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; + qp->outq.cb.dqrr = dqrr_out_fq_cb_rx; + qp->outq.cb.ern = ern_sec_fq_handler; + qman_retire_fq(&qp->outq, NULL); + qman_oos_fq(&qp->outq); + ret = qman_init_fq(&qp->outq, 0, &opts); + if (ret) + RTE_LOG(ERR, PMD, "Error in qman_init_fq: ret: %d\n", ret); + qp->outq.cb.dqrr = NULL; + + return ret; +} + static struct rte_cryptodev_ops crypto_ops = { .dev_configure = dpaa_sec_dev_configure, .dev_start = dpaa_sec_dev_start, @@ -2577,8 +2912,6 @@ dpaa_sec_uninit(struct rte_cryptodev *dev) internals = dev->data->dev_private; rte_free(dev->security_ctx); - /* In case close has been called, internals->ctx_pool would be NULL */ - rte_mempool_free(internals->ctx_pool); rte_free(internals); DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u", @@ -2678,8 +3011,7 @@ cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused, int retval; - snprintf(cryptodev_name, sizeof(cryptodev_name), "dpaa_sec-%d", - dpaa_dev->id.dev_id); + snprintf(cryptodev_name, sizeof(cryptodev_name), "%s", dpaa_dev->name); cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); if (cryptodev == NULL)