/* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- * Copyright 2017-2018 NXP
+ * Copyright 2017-2019 NXP
*
*/
#include <hw/desc/common.h>
#include <hw/desc/algo.h>
#include <hw/desc/ipsec.h>
+#include <hw/desc/pdcp.h>
#include <rte_dpaa_bus.h>
#include <dpaa_sec.h>
+#include <dpaa_sec_event.h>
#include <dpaa_sec_log.h>
+#include <dpaax_iova_table.h>
enum rta_sec_era rta_sec_era;
DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
}
-
- /* report op status to sym->op and then free the ctx memeory */
- rte_mempool_put(ctx->ctx_pool, (void *)ctx);
}
static inline struct dpaa_sec_op_ctx *
-dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
+dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count)
{
struct dpaa_sec_op_ctx *ctx;
- int retval;
+ int i, retval;
- retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
+ retval = rte_mempool_get(
+ ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool,
+ (void **)(&ctx));
if (!ctx || retval) {
DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
return NULL;
* to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
* each packet, memset is costlier than dcbz_64().
*/
- dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
- dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
- dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
- dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
+ for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4)
+ dcbz_64(&ctx->job.sg[i]);
- ctx->ctx_pool = ses->ctx_pool;
- ctx->vtop_offset = (size_t) ctx
- - rte_mempool_virt2iova(ctx);
+ ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool;
+ ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
return ctx;
}
const struct rte_memseg *ms;
ms = rte_mem_virt2memseg(vaddr, NULL);
- if (ms)
+ if (ms) {
+ dpaax_iova_table_update(ms->iova, ms->addr, ms->len);
return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
+ }
return (size_t)NULL;
}
static inline void *
dpaa_mem_ptov(rte_iova_t paddr)
{
+ void *va;
+
+ va = (void *)dpaax_iova_table_get_va(paddr);
+ if (likely(va))
+ return va;
+
return rte_mem_iova2virt(paddr);
}
if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
struct qm_sg_entry *sg_out;
uint32_t len;
+ struct rte_mbuf *mbuf = (ctx->op->sym->m_dst == NULL) ?
+ ctx->op->sym->m_src : ctx->op->sym->m_dst;
sg_out = &job->sg[0];
hw_sg_to_cpu(sg_out);
len = sg_out->length;
- ctx->op->sym->m_src->pkt_len = len;
- ctx->op->sym->m_src->data_len = len;
+ mbuf->pkt_len = len;
+ while (mbuf->next != NULL) {
+ len -= mbuf->data_len;
+ mbuf = mbuf->next;
+ }
+ mbuf->data_len = len;
}
dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
dpaa_sec_op_ending(ctx);
{
return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
(ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
+ (ses->proto_alg != RTE_SECURITY_PROTOCOL_PDCP) &&
(ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
}
return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
}
+static inline int is_proto_pdcp(dpaa_sec_session *ses)
+{
+ return (ses->proto_alg == RTE_SECURITY_PROTOCOL_PDCP);
+}
+
static inline int is_encode(dpaa_sec_session *ses)
{
return ses->dir == DIR_ENC;
{
switch (ses->auth_alg) {
case RTE_CRYPTO_AUTH_NULL:
+ alginfo_a->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_HMAC_NULL : 0;
ses->digest_length = 0;
break;
case RTE_CRYPTO_AUTH_MD5_HMAC:
{
switch (ses->cipher_alg) {
case RTE_CRYPTO_CIPHER_NULL:
+ alginfo_c->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_NULL : 0;
break;
case RTE_CRYPTO_CIPHER_AES_CBC:
alginfo_c->algtype =
}
}
+static int
+dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses)
+{
+ struct alginfo authdata = {0}, cipherdata = {0};
+ struct sec_cdb *cdb = &ses->cdb;
+ struct alginfo *p_authdata = NULL;
+ int32_t shared_desc_len = 0;
+ int err;
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ int swap = false;
+#else
+ int swap = true;
+#endif
+
+ switch (ses->cipher_alg) {
+ case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+ cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
+ break;
+ case RTE_CRYPTO_CIPHER_ZUC_EEA3:
+ cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
+ break;
+ case RTE_CRYPTO_CIPHER_AES_CTR:
+ cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
+ break;
+ case RTE_CRYPTO_CIPHER_NULL:
+ cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
+ break;
+ default:
+ DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
+ ses->cipher_alg);
+ return -1;
+ }
+
+ cipherdata.key = (size_t)ses->cipher_key.data;
+ cipherdata.keylen = ses->cipher_key.length;
+ cipherdata.key_enc_flags = 0;
+ cipherdata.key_type = RTA_DATA_IMM;
+
+ cdb->sh_desc[0] = cipherdata.keylen;
+ cdb->sh_desc[1] = 0;
+ cdb->sh_desc[2] = 0;
+
+ if (ses->auth_alg) {
+ switch (ses->auth_alg) {
+ case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
+ authdata.algtype = PDCP_AUTH_TYPE_SNOW;
+ break;
+ case RTE_CRYPTO_AUTH_ZUC_EIA3:
+ authdata.algtype = PDCP_AUTH_TYPE_ZUC;
+ break;
+ case RTE_CRYPTO_AUTH_AES_CMAC:
+ authdata.algtype = PDCP_AUTH_TYPE_AES;
+ break;
+ case RTE_CRYPTO_AUTH_NULL:
+ authdata.algtype = PDCP_AUTH_TYPE_NULL;
+ break;
+ default:
+ DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
+ ses->auth_alg);
+ return -1;
+ }
+
+ authdata.key = (size_t)ses->auth_key.data;
+ authdata.keylen = ses->auth_key.length;
+ authdata.key_enc_flags = 0;
+ authdata.key_type = RTA_DATA_IMM;
+
+ p_authdata = &authdata;
+
+ cdb->sh_desc[1] = authdata.keylen;
+ }
+
+ err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
+ MIN_JOB_DESC_SIZE,
+ (unsigned int *)cdb->sh_desc,
+ &cdb->sh_desc[2], 2);
+ if (err < 0) {
+ DPAA_SEC_ERR("Crypto: Incorrect key lengths");
+ return err;
+ }
+
+ if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) {
+ cipherdata.key =
+ (size_t)dpaa_mem_vtop((void *)(size_t)cipherdata.key);
+ cipherdata.key_type = RTA_DATA_PTR;
+ }
+ if (!(cdb->sh_desc[2] & (1 << 1)) && authdata.keylen) {
+ authdata.key =
+ (size_t)dpaa_mem_vtop((void *)(size_t)authdata.key);
+ authdata.key_type = RTA_DATA_PTR;
+ }
+
+ cdb->sh_desc[0] = 0;
+ cdb->sh_desc[1] = 0;
+ cdb->sh_desc[2] = 0;
+
+ if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
+ if (ses->dir == DIR_ENC)
+ shared_desc_len = cnstr_shdsc_pdcp_c_plane_encap(
+ cdb->sh_desc, 1, swap,
+ ses->pdcp.hfn,
+ ses->pdcp.sn_size,
+ ses->pdcp.bearer,
+ ses->pdcp.pkt_dir,
+ ses->pdcp.hfn_threshold,
+ &cipherdata, &authdata,
+ 0);
+ else if (ses->dir == DIR_DEC)
+ shared_desc_len = cnstr_shdsc_pdcp_c_plane_decap(
+ cdb->sh_desc, 1, swap,
+ ses->pdcp.hfn,
+ ses->pdcp.sn_size,
+ ses->pdcp.bearer,
+ ses->pdcp.pkt_dir,
+ ses->pdcp.hfn_threshold,
+ &cipherdata, &authdata,
+ 0);
+ } else {
+ if (ses->dir == DIR_ENC)
+ shared_desc_len = cnstr_shdsc_pdcp_u_plane_encap(
+ cdb->sh_desc, 1, swap,
+ ses->pdcp.sn_size,
+ ses->pdcp.hfn,
+ ses->pdcp.bearer,
+ ses->pdcp.pkt_dir,
+ ses->pdcp.hfn_threshold,
+ &cipherdata, p_authdata, 0);
+ else if (ses->dir == DIR_DEC)
+ shared_desc_len = cnstr_shdsc_pdcp_u_plane_decap(
+ cdb->sh_desc, 1, swap,
+ ses->pdcp.sn_size,
+ ses->pdcp.hfn,
+ ses->pdcp.bearer,
+ ses->pdcp.pkt_dir,
+ ses->pdcp.hfn_threshold,
+ &cipherdata, p_authdata, 0);
+ }
+
+ return shared_desc_len;
+}
+
+/* prepare ipsec proto command block of the session */
+static int
+dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses)
+{
+ struct alginfo cipherdata = {0}, authdata = {0};
+ struct sec_cdb *cdb = &ses->cdb;
+ int32_t shared_desc_len = 0;
+ int err;
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ int swap = false;
+#else
+ int swap = true;
+#endif
+
+ caam_cipher_alg(ses, &cipherdata);
+ if (cipherdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
+ DPAA_SEC_ERR("not supported cipher alg");
+ return -ENOTSUP;
+ }
+
+ cipherdata.key = (size_t)ses->cipher_key.data;
+ cipherdata.keylen = ses->cipher_key.length;
+ cipherdata.key_enc_flags = 0;
+ cipherdata.key_type = RTA_DATA_IMM;
+
+ caam_auth_alg(ses, &authdata);
+ if (authdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
+ DPAA_SEC_ERR("not supported auth alg");
+ return -ENOTSUP;
+ }
+
+ authdata.key = (size_t)ses->auth_key.data;
+ authdata.keylen = ses->auth_key.length;
+ authdata.key_enc_flags = 0;
+ authdata.key_type = RTA_DATA_IMM;
+
+ cdb->sh_desc[0] = cipherdata.keylen;
+ cdb->sh_desc[1] = authdata.keylen;
+ err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
+ MIN_JOB_DESC_SIZE,
+ (unsigned int *)cdb->sh_desc,
+ &cdb->sh_desc[2], 2);
+
+ if (err < 0) {
+ DPAA_SEC_ERR("Crypto: Incorrect key lengths");
+ return err;
+ }
+ if (cdb->sh_desc[2] & 1)
+ cipherdata.key_type = RTA_DATA_IMM;
+ else {
+ cipherdata.key = (size_t)dpaa_mem_vtop(
+ (void *)(size_t)cipherdata.key);
+ cipherdata.key_type = RTA_DATA_PTR;
+ }
+ if (cdb->sh_desc[2] & (1<<1))
+ authdata.key_type = RTA_DATA_IMM;
+ else {
+ authdata.key = (size_t)dpaa_mem_vtop(
+ (void *)(size_t)authdata.key);
+ authdata.key_type = RTA_DATA_PTR;
+ }
+
+ cdb->sh_desc[0] = 0;
+ cdb->sh_desc[1] = 0;
+ cdb->sh_desc[2] = 0;
+ if (ses->dir == DIR_ENC) {
+ shared_desc_len = cnstr_shdsc_ipsec_new_encap(
+ cdb->sh_desc,
+ true, swap, SHR_SERIAL,
+ &ses->encap_pdb,
+ (uint8_t *)&ses->ip4_hdr,
+ &cipherdata, &authdata);
+ } else if (ses->dir == DIR_DEC) {
+ shared_desc_len = cnstr_shdsc_ipsec_new_decap(
+ cdb->sh_desc,
+ true, swap, SHR_SERIAL,
+ &ses->decap_pdb,
+ &cipherdata, &authdata);
+ }
+ return shared_desc_len;
+}
/* prepare command block of the session */
static int
memset(cdb, 0, sizeof(struct sec_cdb));
- if (is_cipher_only(ses)) {
+ if (is_proto_ipsec(ses)) {
+ shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses);
+ } else if (is_proto_pdcp(ses)) {
+ shared_desc_len = dpaa_sec_prep_pdcp_cdb(ses);
+ } else if (is_cipher_only(ses)) {
caam_cipher_alg(ses, &alginfo_c);
if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
DPAA_SEC_ERR("not supported cipher alg");
shared_desc_len = cnstr_shdsc_blkcipher(
cdb->sh_desc, true,
- swap, &alginfo_c,
+ swap, SHR_NEVER, &alginfo_c,
NULL,
ses->iv.length,
ses->dir);
alginfo_a.key_type = RTA_DATA_IMM;
shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
- swap, &alginfo_a,
+ swap, SHR_NEVER, &alginfo_a,
!ses->dir,
ses->digest_length);
} else if (is_aead(ses)) {
if (ses->dir == DIR_ENC)
shared_desc_len = cnstr_shdsc_gcm_encap(
- cdb->sh_desc, true, swap,
+ cdb->sh_desc, true, swap, SHR_NEVER,
&alginfo,
ses->iv.length,
ses->digest_length);
else
shared_desc_len = cnstr_shdsc_gcm_decap(
- cdb->sh_desc, true, swap,
+ cdb->sh_desc, true, swap, SHR_NEVER,
&alginfo,
ses->iv.length,
ses->digest_length);
cdb->sh_desc[0] = 0;
cdb->sh_desc[1] = 0;
cdb->sh_desc[2] = 0;
- if (is_proto_ipsec(ses)) {
- if (ses->dir == DIR_ENC) {
- shared_desc_len = cnstr_shdsc_ipsec_new_encap(
- cdb->sh_desc,
- true, swap, SHR_SERIAL,
- &ses->encap_pdb,
- (uint8_t *)&ses->ip4_hdr,
- &alginfo_c, &alginfo_a);
- } else if (ses->dir == DIR_DEC) {
- shared_desc_len = cnstr_shdsc_ipsec_new_decap(
- cdb->sh_desc,
- true, swap, SHR_SERIAL,
- &ses->decap_pdb,
- &alginfo_c, &alginfo_a);
- }
- } else {
- /* Auth_only_len is set as 0 here and it will be
- * overwritten in fd for each packet.
- */
- shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
- true, swap, &alginfo_c, &alginfo_a,
- ses->iv.length, 0,
- ses->digest_length, ses->dir);
- }
+ /* Auth_only_len is set as 0 here and it will be
+ * overwritten in fd for each packet.
+ */
+ shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
+ true, swap, SHR_SERIAL, &alginfo_c, &alginfo_a,
+ ses->iv.length, 0,
+ ses->digest_length, ses->dir);
}
if (shared_desc_len < 0) {
if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
struct qm_sg_entry *sg_out;
uint32_t len;
+ struct rte_mbuf *mbuf = (op->sym->m_dst == NULL) ?
+ op->sym->m_src : op->sym->m_dst;
sg_out = &job->sg[0];
hw_sg_to_cpu(sg_out);
len = sg_out->length;
- op->sym->m_src->pkt_len = len;
- op->sym->m_src->data_len = len;
+ mbuf->pkt_len = len;
+ while (mbuf->next != NULL) {
+ len -= mbuf->data_len;
+ mbuf = mbuf->next;
+ }
+ mbuf->data_len = len;
}
if (!ctx->fd_status) {
op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
else
extra_segs = 2;
- if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, mbuf->nb_segs + extra_segs);
if (!ctx)
return NULL;
rte_iova_t start_addr;
uint8_t *old_digest;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 4);
if (!ctx)
return NULL;
req_segs = mbuf->nb_segs * 2 + 3;
}
- if (req_segs > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
if (!ctx)
return NULL;
uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
ses->iv.offset);
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 4);
if (!ctx)
return NULL;
if (ses->auth_only_len)
req_segs++;
- if (req_segs > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
if (!ctx)
return NULL;
else
dst_start_addr = src_start_addr;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 7);
if (!ctx)
return NULL;
req_segs = mbuf->nb_segs * 2 + 4;
}
- if (req_segs > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
if (!ctx)
return NULL;
else
dst_start_addr = src_start_addr;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 7);
if (!ctx)
return NULL;
struct qm_sg_entry *sg;
phys_addr_t src_start_addr, dst_start_addr;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 2);
if (!ctx)
return NULL;
cf = &ctx->job;
return cf;
}
+static inline struct dpaa_sec_job *
+build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct dpaa_sec_job *cf;
+ struct dpaa_sec_op_ctx *ctx;
+ struct qm_sg_entry *sg, *out_sg, *in_sg;
+ struct rte_mbuf *mbuf;
+ uint8_t req_segs;
+ uint32_t in_len = 0, out_len = 0;
+
+ if (sym->m_dst)
+ mbuf = sym->m_dst;
+ else
+ mbuf = sym->m_src;
+
+ req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
+ DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d",
+ MAX_SG_ENTRIES);
+ return NULL;
+ }
+
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
+ if (!ctx)
+ return NULL;
+ cf = &ctx->job;
+ ctx->op = op;
+ /* output */
+ out_sg = &cf->sg[0];
+ out_sg->extension = 1;
+ qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
+
+ /* 1st seg */
+ sg = &cf->sg[2];
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->offset = 0;
+
+ /* Successive segs */
+ while (mbuf->next) {
+ sg->length = mbuf->data_len;
+ out_len += sg->length;
+ mbuf = mbuf->next;
+ cpu_to_hw_sg(sg);
+ sg++;
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->offset = 0;
+ }
+ sg->length = mbuf->buf_len - mbuf->data_off;
+ out_len += sg->length;
+ sg->final = 1;
+ cpu_to_hw_sg(sg);
+
+ out_sg->length = out_len;
+ cpu_to_hw_sg(out_sg);
+
+ /* input */
+ mbuf = sym->m_src;
+ in_sg = &cf->sg[1];
+ in_sg->extension = 1;
+ in_sg->final = 1;
+ in_len = mbuf->data_len;
+
+ sg++;
+ qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
+
+ /* 1st seg */
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->length = mbuf->data_len;
+ sg->offset = 0;
+
+ /* Successive segs */
+ mbuf = mbuf->next;
+ while (mbuf) {
+ cpu_to_hw_sg(sg);
+ sg++;
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->length = mbuf->data_len;
+ sg->offset = 0;
+ in_len += sg->length;
+ mbuf = mbuf->next;
+ }
+ sg->final = 1;
+ cpu_to_hw_sg(sg);
+
+ in_sg->length = in_len;
+ cpu_to_hw_sg(in_sg);
+
+ sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
+
+ return cf;
+}
+
static uint16_t
dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
uint16_t nb_ops)
struct rte_crypto_op *op;
struct dpaa_sec_job *cf;
dpaa_sec_session *ses;
- uint32_t auth_only_len;
+ uint32_t auth_only_len, index, flags[DPAA_SEC_BURST] = {0};
struct qman_fq *inq[DPAA_SEC_BURST];
while (nb_ops) {
DPAA_SEC_BURST : nb_ops;
for (loop = 0; loop < frames_to_send; loop++) {
op = *(ops++);
+ if (op->sym->m_src->seqn != 0) {
+ index = op->sym->m_src->seqn - 1;
+ if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
+ /* QM_EQCR_DCA_IDXMASK = 0x0f */
+ flags[loop] = ((index & 0x0f) << 8);
+ flags[loop] |= QMAN_ENQUEUE_FLAG_DCA;
+ DPAA_PER_LCORE_DQRR_SIZE--;
+ DPAA_PER_LCORE_DQRR_HELD &=
+ ~(1 << index);
+ }
+ }
+
switch (op->sess_type) {
case RTE_CRYPTO_OP_WITH_SESSION:
ses = (dpaa_sec_session *)
nb_ops = loop;
goto send_pkts;
}
- if (unlikely(!ses->qp)) {
+ if (unlikely(!ses->qp[rte_lcore_id() % MAX_DPAA_CORES])) {
if (dpaa_sec_attach_sess_q(qp, ses)) {
frames_to_send = loop;
nb_ops = loop;
goto send_pkts;
}
- } else if (unlikely(ses->qp != qp)) {
+ } else if (unlikely(ses->qp[rte_lcore_id() %
+ MAX_DPAA_CORES] != qp)) {
DPAA_SEC_DP_ERR("Old:sess->qp = %p"
- " New qp = %p\n", ses->qp, qp);
+ " New qp = %p\n",
+ ses->qp[rte_lcore_id() %
+ MAX_DPAA_CORES], qp);
frames_to_send = loop;
nb_ops = loop;
goto send_pkts;
auth_only_len = op->sym->auth.data.length -
op->sym->cipher.data.length;
- if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
- if (is_auth_only(ses)) {
+ if (rte_pktmbuf_is_contiguous(op->sym->m_src) &&
+ ((op->sym->m_dst == NULL) ||
+ rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
+ if (is_proto_ipsec(ses)) {
+ cf = build_proto(op, ses);
+ } else if (is_proto_pdcp(ses)) {
+ cf = build_proto(op, ses);
+ } else if (is_auth_only(ses)) {
cf = build_auth_only(op, ses);
} else if (is_cipher_only(ses)) {
cf = build_cipher_only(op, ses);
auth_only_len = ses->auth_only_len;
} else if (is_auth_cipher(ses)) {
cf = build_cipher_auth(op, ses);
- } else if (is_proto_ipsec(ses)) {
- cf = build_proto(op, ses);
} else {
DPAA_SEC_DP_ERR("not supported ops");
frames_to_send = loop;
goto send_pkts;
}
} else {
- if (is_auth_only(ses)) {
+ if (is_proto_pdcp(ses) || is_proto_ipsec(ses)) {
+ cf = build_proto_sg(op, ses);
+ } else if (is_auth_only(ses)) {
cf = build_auth_only_sg(op, ses);
} else if (is_cipher_only(ses)) {
cf = build_cipher_only_sg(op, ses);
}
fd = &fds[loop];
- inq[loop] = ses->inq;
+ inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES];
fd->opaque_addr = 0;
fd->cmd = 0;
qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg));
if (auth_only_len)
fd->cmd = 0x80000000 | auth_only_len;
+ /* In case of PDCP, per packet HFN is stored in
+ * mbuf priv after sym_op.
+ */
+ if (is_proto_pdcp(ses) && ses->pdcp.hfn_ovd) {
+ fd->cmd = 0x80000000 |
+ *((uint32_t *)((uint8_t *)op +
+ ses->pdcp.hfn_ovd_offset));
+ DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u,%u\n",
+ *((uint32_t *)((uint8_t *)op +
+ ses->pdcp.hfn_ovd_offset)),
+ ses->pdcp.hfn_ovd,
+ is_proto_pdcp(ses));
+ }
+
}
send_pkts:
loop = 0;
while (loop < frames_to_send) {
loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
- frames_to_send - loop);
+ &flags[loop], frames_to_send - loop);
}
nb_ops -= frames_to_send;
num_tx += frames_to_send;
}
qp = &internals->qps[qp_id];
+ rte_mempool_free(qp->ctx_pool);
qp->internals = NULL;
dev->data->queue_pairs[qp_id] = NULL;
static int
dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
__rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
- __rte_unused int socket_id,
- __rte_unused struct rte_mempool *session_pool)
+ __rte_unused int socket_id)
{
struct dpaa_sec_dev_private *internals;
struct dpaa_sec_qp *qp = NULL;
+ char str[20];
DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
qp = &internals->qps[qp_id];
qp->internals = internals;
+ snprintf(str, sizeof(str), "ctx_pool_d%d_qp%d",
+ dev->data->dev_id, qp_id);
+ if (!qp->ctx_pool) {
+ qp->ctx_pool = rte_mempool_create((const char *)str,
+ CTX_POOL_NUM_BUFS,
+ CTX_POOL_BUF_SIZE,
+ CTX_POOL_CACHE_SIZE, 0,
+ NULL, NULL, NULL, NULL,
+ SOCKET_ID_ANY, 0);
+ if (!qp->ctx_pool) {
+ DPAA_SEC_ERR("%s create failed\n", str);
+ return -ENOMEM;
+ }
+ } else
+ DPAA_SEC_INFO("mempool already created for dev_id : %d, qp: %d",
+ dev->data->dev_id, qp_id);
dev->data->queue_pairs[qp_id] = qp;
return 0;
{
unsigned int i;
- for (i = 0; i < qi->max_nb_sessions; i++) {
+ for (i = 0; i < qi->max_nb_sessions * MAX_DPAA_CORES; i++) {
if (qi->inq_attach[i] == 0) {
qi->inq_attach[i] = 1;
return &qi->inq[i];
}
}
- DPAA_SEC_WARN("All ses session in use %x", qi->max_nb_sessions);
+ DPAA_SEC_WARN("All session in use %u", qi->max_nb_sessions);
return NULL;
}
{
int ret;
- sess->qp = qp;
+ sess->qp[rte_lcore_id() % MAX_DPAA_CORES] = qp;
ret = dpaa_sec_prep_cdb(sess);
if (ret) {
DPAA_SEC_ERR("Unable to prepare sec cdb");
return ret;
}
}
- ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
+ ret = dpaa_sec_init_rx(sess->inq[rte_lcore_id() % MAX_DPAA_CORES],
+ dpaa_mem_vtop(&sess->cdb),
qman_fq_fqid(&qp->outq));
if (ret)
DPAA_SEC_ERR("Unable to init sec queue");
{
struct dpaa_sec_dev_private *internals = dev->data->dev_private;
dpaa_sec_session *session = sess;
+ uint32_t i;
PMD_INIT_FUNC_TRACE();
DPAA_SEC_ERR("Invalid crypto type");
return -EINVAL;
}
- session->ctx_pool = internals->ctx_pool;
rte_spinlock_lock(&internals->lock);
- session->inq = dpaa_sec_attach_rxq(internals);
- rte_spinlock_unlock(&internals->lock);
- if (session->inq == NULL) {
- DPAA_SEC_ERR("unable to attach sec queue");
- goto err1;
+ for (i = 0; i < MAX_DPAA_CORES; i++) {
+ session->inq[i] = dpaa_sec_attach_rxq(internals);
+ if (session->inq[i] == NULL) {
+ DPAA_SEC_ERR("unable to attach sec queue");
+ rte_spinlock_unlock(&internals->lock);
+ goto err1;
+ }
}
+ rte_spinlock_unlock(&internals->lock);
return 0;
return 0;
}
+static inline void
+free_session_memory(struct rte_cryptodev *dev, dpaa_sec_session *s)
+{
+ struct dpaa_sec_dev_private *qi = dev->data->dev_private;
+ struct rte_mempool *sess_mp = rte_mempool_from_obj((void *)s);
+ uint8_t i;
+
+ for (i = 0; i < MAX_DPAA_CORES; i++) {
+ if (s->inq[i])
+ dpaa_sec_detach_rxq(qi, s->inq[i]);
+ s->inq[i] = NULL;
+ s->qp[i] = NULL;
+ }
+ rte_free(s->cipher_key.data);
+ rte_free(s->auth_key.data);
+ memset(s, 0, sizeof(dpaa_sec_session));
+ rte_mempool_put(sess_mp, (void *)s);
+}
+
/** Clear the memory of session so it doesn't leave key material behind */
static void
dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
struct rte_cryptodev_sym_session *sess)
{
- struct dpaa_sec_dev_private *qi = dev->data->dev_private;
+ PMD_INIT_FUNC_TRACE();
uint8_t index = dev->driver_id;
void *sess_priv = get_sym_session_private_data(sess, index);
-
- PMD_INIT_FUNC_TRACE();
-
dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
if (sess_priv) {
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-
- if (s->inq)
- dpaa_sec_detach_rxq(qi, s->inq);
- rte_free(s->cipher_key.data);
- rte_free(s->auth_key.data);
- memset(s, 0, sizeof(dpaa_sec_session));
+ free_session_memory(dev, s);
set_sym_session_private_data(sess, index, NULL);
- rte_mempool_put(sess_mp, sess_priv);
}
}
{
struct dpaa_sec_dev_private *internals = dev->data->dev_private;
struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
- struct rte_crypto_auth_xform *auth_xform;
- struct rte_crypto_cipher_xform *cipher_xform;
+ struct rte_crypto_auth_xform *auth_xform = NULL;
+ struct rte_crypto_cipher_xform *cipher_xform = NULL;
dpaa_sec_session *session = (dpaa_sec_session *)sess;
+ uint32_t i;
PMD_INIT_FUNC_TRACE();
memset(session, 0, sizeof(dpaa_sec_session));
if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
cipher_xform = &conf->crypto_xform->cipher;
- auth_xform = &conf->crypto_xform->next->auth;
+ if (conf->crypto_xform->next)
+ auth_xform = &conf->crypto_xform->next->auth;
} else {
auth_xform = &conf->crypto_xform->auth;
- cipher_xform = &conf->crypto_xform->next->cipher;
+ if (conf->crypto_xform->next)
+ cipher_xform = &conf->crypto_xform->next->cipher;
}
session->proto_alg = conf->protocol;
- session->cipher_key.data = rte_zmalloc(NULL,
- cipher_xform->key.length,
- RTE_CACHE_LINE_SIZE);
- if (session->cipher_key.data == NULL &&
- cipher_xform->key.length > 0) {
- DPAA_SEC_ERR("No Memory for cipher key");
- return -ENOMEM;
- }
- session->cipher_key.length = cipher_xform->key.length;
- session->auth_key.data = rte_zmalloc(NULL,
- auth_xform->key.length,
- RTE_CACHE_LINE_SIZE);
- if (session->auth_key.data == NULL &&
- auth_xform->key.length > 0) {
- DPAA_SEC_ERR("No Memory for auth key");
- rte_free(session->cipher_key.data);
- return -ENOMEM;
+ if (cipher_xform && cipher_xform->algo != RTE_CRYPTO_CIPHER_NULL) {
+ session->cipher_key.data = rte_zmalloc(NULL,
+ cipher_xform->key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->cipher_key.data == NULL &&
+ cipher_xform->key.length > 0) {
+ DPAA_SEC_ERR("No Memory for cipher key");
+ return -ENOMEM;
+ }
+ memcpy(session->cipher_key.data, cipher_xform->key.data,
+ cipher_xform->key.length);
+ session->cipher_key.length = cipher_xform->key.length;
+
+ switch (cipher_xform->algo) {
+ case RTE_CRYPTO_CIPHER_AES_CBC:
+ case RTE_CRYPTO_CIPHER_3DES_CBC:
+ case RTE_CRYPTO_CIPHER_AES_CTR:
+ break;
+ default:
+ DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
+ cipher_xform->algo);
+ goto out;
+ }
+ session->cipher_alg = cipher_xform->algo;
+ } else {
+ session->cipher_key.data = NULL;
+ session->cipher_key.length = 0;
+ session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
}
- session->auth_key.length = auth_xform->key.length;
- memcpy(session->cipher_key.data, cipher_xform->key.data,
- cipher_xform->key.length);
- memcpy(session->auth_key.data, auth_xform->key.data,
- auth_xform->key.length);
- switch (auth_xform->algo) {
- case RTE_CRYPTO_AUTH_SHA1_HMAC:
- session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
- break;
- case RTE_CRYPTO_AUTH_MD5_HMAC:
- session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
- break;
- case RTE_CRYPTO_AUTH_SHA256_HMAC:
- session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
- break;
- case RTE_CRYPTO_AUTH_SHA384_HMAC:
- session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
- break;
- case RTE_CRYPTO_AUTH_SHA512_HMAC:
- session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
- break;
- case RTE_CRYPTO_AUTH_AES_CMAC:
- session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
- break;
- case RTE_CRYPTO_AUTH_NULL:
+ if (auth_xform && auth_xform->algo != RTE_CRYPTO_AUTH_NULL) {
+ session->auth_key.data = rte_zmalloc(NULL,
+ auth_xform->key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->auth_key.data == NULL &&
+ auth_xform->key.length > 0) {
+ DPAA_SEC_ERR("No Memory for auth key");
+ rte_free(session->cipher_key.data);
+ return -ENOMEM;
+ }
+ memcpy(session->auth_key.data, auth_xform->key.data,
+ auth_xform->key.length);
+ session->auth_key.length = auth_xform->key.length;
+
+ switch (auth_xform->algo) {
+ case RTE_CRYPTO_AUTH_SHA1_HMAC:
+ case RTE_CRYPTO_AUTH_MD5_HMAC:
+ case RTE_CRYPTO_AUTH_SHA256_HMAC:
+ case RTE_CRYPTO_AUTH_SHA384_HMAC:
+ case RTE_CRYPTO_AUTH_SHA512_HMAC:
+ case RTE_CRYPTO_AUTH_AES_CMAC:
+ break;
+ default:
+ DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
+ auth_xform->algo);
+ goto out;
+ }
+ session->auth_alg = auth_xform->algo;
+ } else {
+ session->auth_key.data = NULL;
+ session->auth_key.length = 0;
session->auth_alg = RTE_CRYPTO_AUTH_NULL;
- break;
- case RTE_CRYPTO_AUTH_SHA224_HMAC:
- case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
- case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
- case RTE_CRYPTO_AUTH_SHA1:
- case RTE_CRYPTO_AUTH_SHA256:
- case RTE_CRYPTO_AUTH_SHA512:
- case RTE_CRYPTO_AUTH_SHA224:
- case RTE_CRYPTO_AUTH_SHA384:
- case RTE_CRYPTO_AUTH_MD5:
- case RTE_CRYPTO_AUTH_AES_GMAC:
- case RTE_CRYPTO_AUTH_KASUMI_F9:
- case RTE_CRYPTO_AUTH_AES_CBC_MAC:
- case RTE_CRYPTO_AUTH_ZUC_EIA3:
- DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
- auth_xform->algo);
- goto out;
- default:
- DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
- auth_xform->algo);
- goto out;
- }
-
- switch (cipher_xform->algo) {
- case RTE_CRYPTO_CIPHER_AES_CBC:
- session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
- break;
- case RTE_CRYPTO_CIPHER_3DES_CBC:
- session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
- break;
- case RTE_CRYPTO_CIPHER_AES_CTR:
- session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
- break;
- case RTE_CRYPTO_CIPHER_NULL:
- case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
- case RTE_CRYPTO_CIPHER_3DES_ECB:
- case RTE_CRYPTO_CIPHER_AES_ECB:
- case RTE_CRYPTO_CIPHER_KASUMI_F8:
- DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
- cipher_xform->algo);
- goto out;
- default:
- DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
- cipher_xform->algo);
- goto out;
}
if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
- memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
+ if (ipsec_xform->tunnel.type ==
+ RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
+ memset(&session->encap_pdb, 0,
+ sizeof(struct ipsec_encap_pdb) +
sizeof(session->ip4_hdr));
- session->ip4_hdr.ip_v = IPVERSION;
- session->ip4_hdr.ip_hl = 5;
- session->ip4_hdr.ip_len = rte_cpu_to_be_16(
+ session->ip4_hdr.ip_v = IPVERSION;
+ session->ip4_hdr.ip_hl = 5;
+ session->ip4_hdr.ip_len = rte_cpu_to_be_16(
sizeof(session->ip4_hdr));
- session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
- session->ip4_hdr.ip_id = 0;
- session->ip4_hdr.ip_off = 0;
- session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
- session->ip4_hdr.ip_p = (ipsec_xform->proto ==
- RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
- : IPPROTO_AH;
- session->ip4_hdr.ip_sum = 0;
- session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
- session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
- session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
+ session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
+ session->ip4_hdr.ip_id = 0;
+ session->ip4_hdr.ip_off = 0;
+ session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
+ session->ip4_hdr.ip_p = (ipsec_xform->proto ==
+ RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
+ IPPROTO_ESP : IPPROTO_AH;
+ session->ip4_hdr.ip_sum = 0;
+ session->ip4_hdr.ip_src =
+ ipsec_xform->tunnel.ipv4.src_ip;
+ session->ip4_hdr.ip_dst =
+ ipsec_xform->tunnel.ipv4.dst_ip;
+ session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
(void *)&session->ip4_hdr,
sizeof(struct ip));
-
+ session->encap_pdb.ip_hdr_len = sizeof(struct ip);
+ } else if (ipsec_xform->tunnel.type ==
+ RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
+ memset(&session->encap_pdb, 0,
+ sizeof(struct ipsec_encap_pdb) +
+ sizeof(session->ip6_hdr));
+ session->ip6_hdr.vtc_flow = rte_cpu_to_be_32(
+ DPAA_IPv6_DEFAULT_VTC_FLOW |
+ ((ipsec_xform->tunnel.ipv6.dscp <<
+ RTE_IPV6_HDR_TC_SHIFT) &
+ RTE_IPV6_HDR_TC_MASK) |
+ ((ipsec_xform->tunnel.ipv6.flabel <<
+ RTE_IPV6_HDR_FL_SHIFT) &
+ RTE_IPV6_HDR_FL_MASK));
+ /* Payload length will be updated by HW */
+ session->ip6_hdr.payload_len = 0;
+ session->ip6_hdr.hop_limits =
+ ipsec_xform->tunnel.ipv6.hlimit;
+ session->ip6_hdr.proto = (ipsec_xform->proto ==
+ RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
+ IPPROTO_ESP : IPPROTO_AH;
+ memcpy(&session->ip6_hdr.src_addr,
+ &ipsec_xform->tunnel.ipv6.src_addr, 16);
+ memcpy(&session->ip6_hdr.dst_addr,
+ &ipsec_xform->tunnel.ipv6.dst_addr, 16);
+ session->encap_pdb.ip_hdr_len =
+ sizeof(struct rte_ipv6_hdr);
+ }
session->encap_pdb.options =
(IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
PDBOPTS_ESP_OIHI_PDB_INL |
PDBOPTS_ESP_IVSRC |
- PDBHMO_ESP_ENCAP_DTTL;
+ PDBHMO_ESP_ENCAP_DTTL |
+ PDBHMO_ESP_SNR;
+ if (ipsec_xform->options.esn)
+ session->encap_pdb.options |= PDBOPTS_ESP_ESN;
session->encap_pdb.spi = ipsec_xform->spi;
- session->encap_pdb.ip_hdr_len = sizeof(struct ip);
-
session->dir = DIR_ENC;
} else if (ipsec_xform->direction ==
RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
- session->decap_pdb.options = sizeof(struct ip) << 16;
+ if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4)
+ session->decap_pdb.options = sizeof(struct ip) << 16;
+ else
+ session->decap_pdb.options =
+ sizeof(struct rte_ipv6_hdr) << 16;
+ if (ipsec_xform->options.esn)
+ session->decap_pdb.options |= PDBOPTS_ESP_ESN;
session->dir = DIR_DEC;
} else
goto out;
- session->ctx_pool = internals->ctx_pool;
rte_spinlock_lock(&internals->lock);
- session->inq = dpaa_sec_attach_rxq(internals);
+ for (i = 0; i < MAX_DPAA_CORES; i++) {
+ session->inq[i] = dpaa_sec_attach_rxq(internals);
+ if (session->inq[i] == NULL) {
+ DPAA_SEC_ERR("unable to attach sec queue");
+ rte_spinlock_unlock(&internals->lock);
+ goto out;
+ }
+ }
rte_spinlock_unlock(&internals->lock);
- if (session->inq == NULL) {
- DPAA_SEC_ERR("unable to attach sec queue");
- goto out;
+
+ return 0;
+out:
+ rte_free(session->auth_key.data);
+ rte_free(session->cipher_key.data);
+ memset(session, 0, sizeof(dpaa_sec_session));
+ return -1;
+}
+
+static int
+dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev,
+ struct rte_security_session_conf *conf,
+ void *sess)
+{
+ struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
+ struct rte_crypto_sym_xform *xform = conf->crypto_xform;
+ struct rte_crypto_auth_xform *auth_xform = NULL;
+ struct rte_crypto_cipher_xform *cipher_xform = NULL;
+ dpaa_sec_session *session = (dpaa_sec_session *)sess;
+ struct dpaa_sec_dev_private *dev_priv = dev->data->dev_private;
+ uint32_t i;
+
+ PMD_INIT_FUNC_TRACE();
+
+ memset(session, 0, sizeof(dpaa_sec_session));
+
+ /* find xfrm types */
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+ cipher_xform = &xform->cipher;
+ if (xform->next != NULL)
+ auth_xform = &xform->next->auth;
+ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+ auth_xform = &xform->auth;
+ if (xform->next != NULL)
+ cipher_xform = &xform->next->cipher;
+ } else {
+ DPAA_SEC_ERR("Invalid crypto type");
+ return -EINVAL;
+ }
+
+ session->proto_alg = conf->protocol;
+ if (cipher_xform) {
+ session->cipher_key.data = rte_zmalloc(NULL,
+ cipher_xform->key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->cipher_key.data == NULL &&
+ cipher_xform->key.length > 0) {
+ DPAA_SEC_ERR("No Memory for cipher key");
+ return -ENOMEM;
+ }
+ session->cipher_key.length = cipher_xform->key.length;
+ memcpy(session->cipher_key.data, cipher_xform->key.data,
+ cipher_xform->key.length);
+ session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
+ DIR_ENC : DIR_DEC;
+ session->cipher_alg = cipher_xform->algo;
+ } else {
+ session->cipher_key.data = NULL;
+ session->cipher_key.length = 0;
+ session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
+ session->dir = DIR_ENC;
}
+ if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
+ if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5 &&
+ pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_12) {
+ DPAA_SEC_ERR(
+ "PDCP Seq Num size should be 5/12 bits for cmode");
+ goto out;
+ }
+ }
+ if (auth_xform) {
+ session->auth_key.data = rte_zmalloc(NULL,
+ auth_xform->key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (!session->auth_key.data &&
+ auth_xform->key.length > 0) {
+ DPAA_SEC_ERR("No Memory for auth key");
+ rte_free(session->cipher_key.data);
+ return -ENOMEM;
+ }
+ session->auth_key.length = auth_xform->key.length;
+ memcpy(session->auth_key.data, auth_xform->key.data,
+ auth_xform->key.length);
+ session->auth_alg = auth_xform->algo;
+ } else {
+ session->auth_key.data = NULL;
+ session->auth_key.length = 0;
+ session->auth_alg = 0;
+ }
+ session->pdcp.domain = pdcp_xform->domain;
+ session->pdcp.bearer = pdcp_xform->bearer;
+ session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
+ session->pdcp.sn_size = pdcp_xform->sn_size;
+ session->pdcp.hfn = pdcp_xform->hfn;
+ session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
+ session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
+ session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
+
+ rte_spinlock_lock(&dev_priv->lock);
+ for (i = 0; i < MAX_DPAA_CORES; i++) {
+ session->inq[i] = dpaa_sec_attach_rxq(dev_priv);
+ if (session->inq[i] == NULL) {
+ DPAA_SEC_ERR("unable to attach sec queue");
+ rte_spinlock_unlock(&dev_priv->lock);
+ goto out;
+ }
+ }
+ rte_spinlock_unlock(&dev_priv->lock);
return 0;
out:
rte_free(session->auth_key.data);
ret = dpaa_sec_set_ipsec_session(cdev, conf,
sess_private_data);
break;
+ case RTE_SECURITY_PROTOCOL_PDCP:
+ ret = dpaa_sec_set_pdcp_session(cdev, conf,
+ sess_private_data);
+ break;
case RTE_SECURITY_PROTOCOL_MACSEC:
return -ENOTSUP;
default:
{
PMD_INIT_FUNC_TRACE();
void *sess_priv = get_sec_session_private_data(sess);
-
dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
if (sess_priv) {
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-
- rte_free(s->cipher_key.data);
- rte_free(s->auth_key.data);
- memset(sess, 0, sizeof(dpaa_sec_session));
+ free_session_memory((struct rte_cryptodev *)dev, s);
set_sec_session_private_data(sess, NULL);
- rte_mempool_put(sess_mp, sess_priv);
}
return 0;
}
-
static int
-dpaa_sec_dev_configure(struct rte_cryptodev *dev,
+dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
struct rte_cryptodev_config *config __rte_unused)
{
-
- char str[20];
- struct dpaa_sec_dev_private *internals;
-
PMD_INIT_FUNC_TRACE();
- internals = dev->data->dev_private;
- sprintf(str, "ctx_pool_%d", dev->data->dev_id);
- if (!internals->ctx_pool) {
- internals->ctx_pool = rte_mempool_create((const char *)str,
- CTX_POOL_NUM_BUFS,
- CTX_POOL_BUF_SIZE,
- CTX_POOL_CACHE_SIZE, 0,
- NULL, NULL, NULL, NULL,
- SOCKET_ID_ANY, 0);
- if (!internals->ctx_pool) {
- DPAA_SEC_ERR("%s create failed\n", str);
- return -ENOMEM;
- }
- } else
- DPAA_SEC_INFO("mempool already created for dev_id : %d",
- dev->data->dev_id);
-
return 0;
}
static int
dpaa_sec_dev_close(struct rte_cryptodev *dev)
{
- struct dpaa_sec_dev_private *internals;
-
PMD_INIT_FUNC_TRACE();
if (dev == NULL)
return -ENOMEM;
- internals = dev->data->dev_private;
- rte_mempool_free(internals->ctx_pool);
- internals->ctx_pool = NULL;
-
return 0;
}
}
}
+static enum qman_cb_dqrr_result
+dpaa_sec_process_parallel_event(void *event,
+ struct qman_portal *qm __always_unused,
+ struct qman_fq *outq,
+ const struct qm_dqrr_entry *dqrr,
+ void **bufs)
+{
+ const struct qm_fd *fd;
+ struct dpaa_sec_job *job;
+ struct dpaa_sec_op_ctx *ctx;
+ struct rte_event *ev = (struct rte_event *)event;
+
+ fd = &dqrr->fd;
+
+ /* sg is embedded in an op ctx,
+ * sg[0] is for output
+ * sg[1] for input
+ */
+ job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+
+ ctx = container_of(job, struct dpaa_sec_op_ctx, job);
+ ctx->fd_status = fd->status;
+ if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+ struct qm_sg_entry *sg_out;
+ uint32_t len;
+
+ sg_out = &job->sg[0];
+ hw_sg_to_cpu(sg_out);
+ len = sg_out->length;
+ ctx->op->sym->m_src->pkt_len = len;
+ ctx->op->sym->m_src->data_len = len;
+ }
+ if (!ctx->fd_status) {
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ } else {
+ DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ }
+ ev->event_ptr = (void *)ctx->op;
+
+ ev->flow_id = outq->ev.flow_id;
+ ev->sub_event_type = outq->ev.sub_event_type;
+ ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
+ ev->op = RTE_EVENT_OP_NEW;
+ ev->sched_type = outq->ev.sched_type;
+ ev->queue_id = outq->ev.queue_id;
+ ev->priority = outq->ev.priority;
+ *bufs = (void *)ctx->op;
+
+ rte_mempool_put(ctx->ctx_pool, (void *)ctx);
+
+ return qman_cb_dqrr_consume;
+}
+
+static enum qman_cb_dqrr_result
+dpaa_sec_process_atomic_event(void *event,
+ struct qman_portal *qm __rte_unused,
+ struct qman_fq *outq,
+ const struct qm_dqrr_entry *dqrr,
+ void **bufs)
+{
+ u8 index;
+ const struct qm_fd *fd;
+ struct dpaa_sec_job *job;
+ struct dpaa_sec_op_ctx *ctx;
+ struct rte_event *ev = (struct rte_event *)event;
+
+ fd = &dqrr->fd;
+
+ /* sg is embedded in an op ctx,
+ * sg[0] is for output
+ * sg[1] for input
+ */
+ job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+
+ ctx = container_of(job, struct dpaa_sec_op_ctx, job);
+ ctx->fd_status = fd->status;
+ if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+ struct qm_sg_entry *sg_out;
+ uint32_t len;
+
+ sg_out = &job->sg[0];
+ hw_sg_to_cpu(sg_out);
+ len = sg_out->length;
+ ctx->op->sym->m_src->pkt_len = len;
+ ctx->op->sym->m_src->data_len = len;
+ }
+ if (!ctx->fd_status) {
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ } else {
+ DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ }
+ ev->event_ptr = (void *)ctx->op;
+ ev->flow_id = outq->ev.flow_id;
+ ev->sub_event_type = outq->ev.sub_event_type;
+ ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
+ ev->op = RTE_EVENT_OP_NEW;
+ ev->sched_type = outq->ev.sched_type;
+ ev->queue_id = outq->ev.queue_id;
+ ev->priority = outq->ev.priority;
+
+ /* Save active dqrr entries */
+ index = ((uintptr_t)dqrr >> 6) & (16/*QM_DQRR_SIZE*/ - 1);
+ DPAA_PER_LCORE_DQRR_SIZE++;
+ DPAA_PER_LCORE_DQRR_HELD |= 1 << index;
+ DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src;
+ ev->impl_opaque = index + 1;
+ ctx->op->sym->m_src->seqn = (uint32_t)index + 1;
+ *bufs = (void *)ctx->op;
+
+ rte_mempool_put(ctx->ctx_pool, (void *)ctx);
+
+ return qman_cb_dqrr_defer;
+}
+
+int
+dpaa_sec_eventq_attach(const struct rte_cryptodev *dev,
+ int qp_id,
+ uint16_t ch_id,
+ const struct rte_event *event)
+{
+ struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
+ struct qm_mcc_initfq opts = {0};
+
+ int ret;
+
+ opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
+ QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
+ opts.fqd.dest.channel = ch_id;
+
+ switch (event->sched_type) {
+ case RTE_SCHED_TYPE_ATOMIC:
+ opts.fqd.fq_ctrl |= QM_FQCTRL_HOLDACTIVE;
+ /* Reset FQCTRL_AVOIDBLOCK bit as it is unnecessary
+ * configuration with HOLD_ACTIVE setting
+ */
+ opts.fqd.fq_ctrl &= (~QM_FQCTRL_AVOIDBLOCK);
+ qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_atomic_event;
+ break;
+ case RTE_SCHED_TYPE_ORDERED:
+ DPAA_SEC_ERR("Ordered queue schedule type is not supported\n");
+ return -1;
+ default:
+ opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK;
+ qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_parallel_event;
+ break;
+ }
+
+ ret = qman_init_fq(&qp->outq, QMAN_INITFQ_FLAG_SCHED, &opts);
+ if (unlikely(ret)) {
+ DPAA_SEC_ERR("unable to init caam source fq!");
+ return ret;
+ }
+
+ memcpy(&qp->outq.ev, event, sizeof(struct rte_event));
+
+ return 0;
+}
+
+int
+dpaa_sec_eventq_detach(const struct rte_cryptodev *dev,
+ int qp_id)
+{
+ struct qm_mcc_initfq opts = {0};
+ int ret;
+ struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
+
+ opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
+ QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
+ qp->outq.cb.dqrr = dqrr_out_fq_cb_rx;
+ qp->outq.cb.ern = ern_sec_fq_handler;
+ qman_retire_fq(&qp->outq, NULL);
+ qman_oos_fq(&qp->outq);
+ ret = qman_init_fq(&qp->outq, 0, &opts);
+ if (ret)
+ RTE_LOG(ERR, PMD, "Error in qman_init_fq: ret: %d\n", ret);
+ qp->outq.cb.dqrr = NULL;
+
+ return ret;
+}
+
static struct rte_cryptodev_ops crypto_ops = {
.dev_configure = dpaa_sec_dev_configure,
.dev_start = dpaa_sec_dev_start,
return dpaa_sec_security_cap;
}
-struct rte_security_ops dpaa_sec_security_ops = {
+static const struct rte_security_ops dpaa_sec_security_ops = {
.session_create = dpaa_sec_security_session_create,
.session_update = NULL,
.session_stats_get = NULL,
internals = dev->data->dev_private;
rte_free(dev->security_ctx);
- /* In case close has been called, internals->ctx_pool would be NULL */
- rte_mempool_free(internals->ctx_pool);
rte_free(internals);
DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
QMAN_FQ_FLAG_TO_DCPORTAL;
- for (i = 0; i < internals->max_nb_sessions; i++) {
+ for (i = 0; i < MAX_DPAA_CORES * internals->max_nb_sessions; i++) {
/* create rx qman fq for sessions*/
ret = qman_create_fq(0, flags, &internals->inq[i]);
if (unlikely(ret != 0)) {
int retval;
- sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id);
+ snprintf(cryptodev_name, sizeof(cryptodev_name), "dpaa_sec-%d",
+ dpaa_dev->id.dev_id);
cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
if (cryptodev == NULL)
git.droids-corp.org / dpdk.git / blobdiff