#include <rte_dpaa_bus.h>
#include <dpaa_sec.h>
+#include <dpaa_sec_event.h>
#include <dpaa_sec_log.h>
+#include <dpaax_iova_table.h>
enum rta_sec_era rta_sec_era;
DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
}
-
- /* report op status to sym->op and then free the ctx memeory */
- rte_mempool_put(ctx->ctx_pool, (void *)ctx);
}
static inline struct dpaa_sec_op_ctx *
-dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
+dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count)
{
struct dpaa_sec_op_ctx *ctx;
- int retval;
+ int i, retval;
- retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
+ retval = rte_mempool_get(
+ ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool,
+ (void **)(&ctx));
if (!ctx || retval) {
DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
return NULL;
* to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
* each packet, memset is costlier than dcbz_64().
*/
- dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
- dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
- dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
- dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
+ for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4)
+ dcbz_64(&ctx->job.sg[i]);
- ctx->ctx_pool = ses->ctx_pool;
- ctx->vtop_offset = (size_t) ctx
- - rte_mempool_virt2iova(ctx);
+ ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool;
+ ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
return ctx;
}
const struct rte_memseg *ms;
ms = rte_mem_virt2memseg(vaddr, NULL);
- if (ms)
+ if (ms) {
+ dpaax_iova_table_update(ms->iova, ms->addr, ms->len);
return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
+ }
return (size_t)NULL;
}
if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
struct qm_sg_entry *sg_out;
uint32_t len;
+ struct rte_mbuf *mbuf = (ctx->op->sym->m_dst == NULL) ?
+ ctx->op->sym->m_src : ctx->op->sym->m_dst;
sg_out = &job->sg[0];
hw_sg_to_cpu(sg_out);
len = sg_out->length;
- ctx->op->sym->m_src->pkt_len = len;
- ctx->op->sym->m_src->data_len = len;
+ mbuf->pkt_len = len;
+ while (mbuf->next != NULL) {
+ len -= mbuf->data_len;
+ mbuf = mbuf->next;
+ }
+ mbuf->data_len = len;
}
dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
dpaa_sec_op_ending(ctx);
{
return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
(ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
- (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
+ (ses->proto_alg != RTE_SECURITY_PROTOCOL_PDCP) &&
+ (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC) &&
+ (ses->aead_alg == 0));
}
static inline int is_proto_ipsec(dpaa_sec_session *ses)
{
struct alginfo authdata = {0}, cipherdata = {0};
struct sec_cdb *cdb = &ses->cdb;
+ struct alginfo *p_authdata = NULL;
int32_t shared_desc_len = 0;
int err;
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
cipherdata.key_enc_flags = 0;
cipherdata.key_type = RTA_DATA_IMM;
- if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
+ cdb->sh_desc[0] = cipherdata.keylen;
+ cdb->sh_desc[1] = 0;
+ cdb->sh_desc[2] = 0;
+
+ if (ses->auth_alg) {
switch (ses->auth_alg) {
case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
authdata.algtype = PDCP_AUTH_TYPE_SNOW;
authdata.key_enc_flags = 0;
authdata.key_type = RTA_DATA_IMM;
- cdb->sh_desc[0] = cipherdata.keylen;
+ p_authdata = &authdata;
+
cdb->sh_desc[1] = authdata.keylen;
- err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
- MIN_JOB_DESC_SIZE,
- (unsigned int *)cdb->sh_desc,
- &cdb->sh_desc[2], 2);
+ }
- if (err < 0) {
- DPAA_SEC_ERR("Crypto: Incorrect key lengths");
- return err;
- }
- if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) {
- cipherdata.key = (size_t)dpaa_mem_vtop(
- (void *)(size_t)cipherdata.key);
- cipherdata.key_type = RTA_DATA_PTR;
- }
- if (!(cdb->sh_desc[2] & (1<<1)) && authdata.keylen) {
- authdata.key = (size_t)dpaa_mem_vtop(
- (void *)(size_t)authdata.key);
- authdata.key_type = RTA_DATA_PTR;
- }
+ err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
+ MIN_JOB_DESC_SIZE,
+ (unsigned int *)cdb->sh_desc,
+ &cdb->sh_desc[2], 2);
+ if (err < 0) {
+ DPAA_SEC_ERR("Crypto: Incorrect key lengths");
+ return err;
+ }
- cdb->sh_desc[0] = 0;
- cdb->sh_desc[1] = 0;
- cdb->sh_desc[2] = 0;
+ if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) {
+ cipherdata.key =
+ (size_t)dpaa_mem_vtop((void *)(size_t)cipherdata.key);
+ cipherdata.key_type = RTA_DATA_PTR;
+ }
+ if (!(cdb->sh_desc[2] & (1 << 1)) && authdata.keylen) {
+ authdata.key =
+ (size_t)dpaa_mem_vtop((void *)(size_t)authdata.key);
+ authdata.key_type = RTA_DATA_PTR;
+ }
+ cdb->sh_desc[0] = 0;
+ cdb->sh_desc[1] = 0;
+ cdb->sh_desc[2] = 0;
+
+ if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
if (ses->dir == DIR_ENC)
shared_desc_len = cnstr_shdsc_pdcp_c_plane_encap(
cdb->sh_desc, 1, swap,
ses->pdcp.hfn,
+ ses->pdcp.sn_size,
ses->pdcp.bearer,
ses->pdcp.pkt_dir,
ses->pdcp.hfn_threshold,
shared_desc_len = cnstr_shdsc_pdcp_c_plane_decap(
cdb->sh_desc, 1, swap,
ses->pdcp.hfn,
+ ses->pdcp.sn_size,
ses->pdcp.bearer,
ses->pdcp.pkt_dir,
ses->pdcp.hfn_threshold,
&cipherdata, &authdata,
0);
} else {
- cdb->sh_desc[0] = cipherdata.keylen;
- err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
- MIN_JOB_DESC_SIZE,
- (unsigned int *)cdb->sh_desc,
- &cdb->sh_desc[2], 1);
-
- if (err < 0) {
- DPAA_SEC_ERR("Crypto: Incorrect key lengths");
- return err;
- }
- if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) {
- cipherdata.key = (size_t)dpaa_mem_vtop(
- (void *)(size_t)cipherdata.key);
- cipherdata.key_type = RTA_DATA_PTR;
- }
- cdb->sh_desc[0] = 0;
- cdb->sh_desc[1] = 0;
- cdb->sh_desc[2] = 0;
-
if (ses->dir == DIR_ENC)
shared_desc_len = cnstr_shdsc_pdcp_u_plane_encap(
cdb->sh_desc, 1, swap,
ses->pdcp.bearer,
ses->pdcp.pkt_dir,
ses->pdcp.hfn_threshold,
- &cipherdata, 0);
+ &cipherdata, p_authdata, 0);
else if (ses->dir == DIR_DEC)
shared_desc_len = cnstr_shdsc_pdcp_u_plane_decap(
cdb->sh_desc, 1, swap,
ses->pdcp.bearer,
ses->pdcp.pkt_dir,
ses->pdcp.hfn_threshold,
- &cipherdata, 0);
+ &cipherdata, p_authdata, 0);
}
return shared_desc_len;
if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
struct qm_sg_entry *sg_out;
uint32_t len;
+ struct rte_mbuf *mbuf = (op->sym->m_dst == NULL) ?
+ op->sym->m_src : op->sym->m_dst;
sg_out = &job->sg[0];
hw_sg_to_cpu(sg_out);
len = sg_out->length;
- op->sym->m_src->pkt_len = len;
- op->sym->m_src->data_len = len;
+ mbuf->pkt_len = len;
+ while (mbuf->next != NULL) {
+ len -= mbuf->data_len;
+ mbuf = mbuf->next;
+ }
+ mbuf->data_len = len;
}
if (!ctx->fd_status) {
op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
else
extra_segs = 2;
- if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, mbuf->nb_segs + extra_segs);
if (!ctx)
return NULL;
rte_iova_t start_addr;
uint8_t *old_digest;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 4);
if (!ctx)
return NULL;
req_segs = mbuf->nb_segs * 2 + 3;
}
- if (req_segs > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
if (!ctx)
return NULL;
uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
ses->iv.offset);
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 4);
if (!ctx)
return NULL;
if (ses->auth_only_len)
req_segs++;
- if (req_segs > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
if (!ctx)
return NULL;
else
dst_start_addr = src_start_addr;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 7);
if (!ctx)
return NULL;
req_segs = mbuf->nb_segs * 2 + 4;
}
- if (req_segs > MAX_SG_ENTRIES) {
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
MAX_SG_ENTRIES);
return NULL;
}
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
if (!ctx)
return NULL;
else
dst_start_addr = src_start_addr;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 7);
if (!ctx)
return NULL;
struct qm_sg_entry *sg;
phys_addr_t src_start_addr, dst_start_addr;
- ctx = dpaa_sec_alloc_ctx(ses);
+ ctx = dpaa_sec_alloc_ctx(ses, 2);
if (!ctx)
return NULL;
cf = &ctx->job;
return cf;
}
+static inline struct dpaa_sec_job *
+build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct dpaa_sec_job *cf;
+ struct dpaa_sec_op_ctx *ctx;
+ struct qm_sg_entry *sg, *out_sg, *in_sg;
+ struct rte_mbuf *mbuf;
+ uint8_t req_segs;
+ uint32_t in_len = 0, out_len = 0;
+
+ if (sym->m_dst)
+ mbuf = sym->m_dst;
+ else
+ mbuf = sym->m_src;
+
+ req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
+ if (mbuf->nb_segs > MAX_SG_ENTRIES) {
+ DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d",
+ MAX_SG_ENTRIES);
+ return NULL;
+ }
+
+ ctx = dpaa_sec_alloc_ctx(ses, req_segs);
+ if (!ctx)
+ return NULL;
+ cf = &ctx->job;
+ ctx->op = op;
+ /* output */
+ out_sg = &cf->sg[0];
+ out_sg->extension = 1;
+ qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
+
+ /* 1st seg */
+ sg = &cf->sg[2];
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->offset = 0;
+
+ /* Successive segs */
+ while (mbuf->next) {
+ sg->length = mbuf->data_len;
+ out_len += sg->length;
+ mbuf = mbuf->next;
+ cpu_to_hw_sg(sg);
+ sg++;
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->offset = 0;
+ }
+ sg->length = mbuf->buf_len - mbuf->data_off;
+ out_len += sg->length;
+ sg->final = 1;
+ cpu_to_hw_sg(sg);
+
+ out_sg->length = out_len;
+ cpu_to_hw_sg(out_sg);
+
+ /* input */
+ mbuf = sym->m_src;
+ in_sg = &cf->sg[1];
+ in_sg->extension = 1;
+ in_sg->final = 1;
+ in_len = mbuf->data_len;
+
+ sg++;
+ qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
+
+ /* 1st seg */
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->length = mbuf->data_len;
+ sg->offset = 0;
+
+ /* Successive segs */
+ mbuf = mbuf->next;
+ while (mbuf) {
+ cpu_to_hw_sg(sg);
+ sg++;
+ qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
+ sg->length = mbuf->data_len;
+ sg->offset = 0;
+ in_len += sg->length;
+ mbuf = mbuf->next;
+ }
+ sg->final = 1;
+ cpu_to_hw_sg(sg);
+
+ in_sg->length = in_len;
+ cpu_to_hw_sg(in_sg);
+
+ sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
+
+ return cf;
+}
+
static uint16_t
dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
uint16_t nb_ops)
struct rte_crypto_op *op;
struct dpaa_sec_job *cf;
dpaa_sec_session *ses;
- uint32_t auth_only_len;
+ uint32_t auth_only_len, index, flags[DPAA_SEC_BURST] = {0};
struct qman_fq *inq[DPAA_SEC_BURST];
while (nb_ops) {
DPAA_SEC_BURST : nb_ops;
for (loop = 0; loop < frames_to_send; loop++) {
op = *(ops++);
+ if (op->sym->m_src->seqn != 0) {
+ index = op->sym->m_src->seqn - 1;
+ if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
+ /* QM_EQCR_DCA_IDXMASK = 0x0f */
+ flags[loop] = ((index & 0x0f) << 8);
+ flags[loop] |= QMAN_ENQUEUE_FLAG_DCA;
+ DPAA_PER_LCORE_DQRR_SIZE--;
+ DPAA_PER_LCORE_DQRR_HELD &=
+ ~(1 << index);
+ }
+ }
+
switch (op->sess_type) {
case RTE_CRYPTO_OP_WITH_SESSION:
ses = (dpaa_sec_session *)
auth_only_len = op->sym->auth.data.length -
op->sym->cipher.data.length;
- if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
+ if (rte_pktmbuf_is_contiguous(op->sym->m_src) &&
+ ((op->sym->m_dst == NULL) ||
+ rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
if (is_proto_ipsec(ses)) {
cf = build_proto(op, ses);
} else if (is_proto_pdcp(ses)) {
goto send_pkts;
}
} else {
- if (is_auth_only(ses)) {
+ if (is_proto_pdcp(ses) || is_proto_ipsec(ses)) {
+ cf = build_proto_sg(op, ses);
+ } else if (is_auth_only(ses)) {
cf = build_auth_only_sg(op, ses);
} else if (is_cipher_only(ses)) {
cf = build_cipher_only_sg(op, ses);
if (auth_only_len)
fd->cmd = 0x80000000 | auth_only_len;
+ /* In case of PDCP, per packet HFN is stored in
+ * mbuf priv after sym_op.
+ */
+ if (is_proto_pdcp(ses) && ses->pdcp.hfn_ovd) {
+ fd->cmd = 0x80000000 |
+ *((uint32_t *)((uint8_t *)op +
+ ses->pdcp.hfn_ovd_offset));
+ DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u,%u\n",
+ *((uint32_t *)((uint8_t *)op +
+ ses->pdcp.hfn_ovd_offset)),
+ ses->pdcp.hfn_ovd,
+ is_proto_pdcp(ses));
+ }
+
}
send_pkts:
loop = 0;
while (loop < frames_to_send) {
loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
- frames_to_send - loop);
+ &flags[loop], frames_to_send - loop);
}
nb_ops -= frames_to_send;
num_tx += frames_to_send;
}
qp = &internals->qps[qp_id];
+ rte_mempool_free(qp->ctx_pool);
qp->internals = NULL;
dev->data->queue_pairs[qp_id] = NULL;
{
struct dpaa_sec_dev_private *internals;
struct dpaa_sec_qp *qp = NULL;
+ char str[20];
DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
qp = &internals->qps[qp_id];
qp->internals = internals;
+ snprintf(str, sizeof(str), "ctx_pool_d%d_qp%d",
+ dev->data->dev_id, qp_id);
+ if (!qp->ctx_pool) {
+ qp->ctx_pool = rte_mempool_create((const char *)str,
+ CTX_POOL_NUM_BUFS,
+ CTX_POOL_BUF_SIZE,
+ CTX_POOL_CACHE_SIZE, 0,
+ NULL, NULL, NULL, NULL,
+ SOCKET_ID_ANY, 0);
+ if (!qp->ctx_pool) {
+ DPAA_SEC_ERR("%s create failed\n", str);
+ return -ENOMEM;
+ }
+ } else
+ DPAA_SEC_INFO("mempool already created for dev_id : %d, qp: %d",
+ dev->data->dev_id, qp_id);
dev->data->queue_pairs[qp_id] = qp;
return 0;
DPAA_SEC_ERR("Invalid crypto type");
return -EINVAL;
}
- session->ctx_pool = internals->ctx_pool;
rte_spinlock_lock(&internals->lock);
for (i = 0; i < MAX_DPAA_CORES; i++) {
session->inq[i] = dpaa_sec_attach_rxq(internals);
return 0;
}
+static inline void
+free_session_memory(struct rte_cryptodev *dev, dpaa_sec_session *s)
+{
+ struct dpaa_sec_dev_private *qi = dev->data->dev_private;
+ struct rte_mempool *sess_mp = rte_mempool_from_obj((void *)s);
+ uint8_t i;
+
+ for (i = 0; i < MAX_DPAA_CORES; i++) {
+ if (s->inq[i])
+ dpaa_sec_detach_rxq(qi, s->inq[i]);
+ s->inq[i] = NULL;
+ s->qp[i] = NULL;
+ }
+ rte_free(s->cipher_key.data);
+ rte_free(s->auth_key.data);
+ memset(s, 0, sizeof(dpaa_sec_session));
+ rte_mempool_put(sess_mp, (void *)s);
+}
+
/** Clear the memory of session so it doesn't leave key material behind */
static void
dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
struct rte_cryptodev_sym_session *sess)
{
- struct dpaa_sec_dev_private *qi = dev->data->dev_private;
- uint8_t index = dev->driver_id, i;
- void *sess_priv = get_sym_session_private_data(sess, index);
-
PMD_INIT_FUNC_TRACE();
-
+ uint8_t index = dev->driver_id;
+ void *sess_priv = get_sym_session_private_data(sess, index);
dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
if (sess_priv) {
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-
- for (i = 0; i < MAX_DPAA_CORES; i++) {
- if (s->inq[i])
- dpaa_sec_detach_rxq(qi, s->inq[i]);
- s->inq[i] = NULL;
- s->qp[i] = NULL;
- }
- rte_free(s->cipher_key.data);
- rte_free(s->auth_key.data);
- memset(s, 0, sizeof(dpaa_sec_session));
+ free_session_memory(dev, s);
set_sym_session_private_data(sess, index, NULL);
- rte_mempool_put(sess_mp, sess_priv);
}
}
}
if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
- memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
+ if (ipsec_xform->tunnel.type ==
+ RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
+ memset(&session->encap_pdb, 0,
+ sizeof(struct ipsec_encap_pdb) +
sizeof(session->ip4_hdr));
- session->ip4_hdr.ip_v = IPVERSION;
- session->ip4_hdr.ip_hl = 5;
- session->ip4_hdr.ip_len = rte_cpu_to_be_16(
+ session->ip4_hdr.ip_v = IPVERSION;
+ session->ip4_hdr.ip_hl = 5;
+ session->ip4_hdr.ip_len = rte_cpu_to_be_16(
sizeof(session->ip4_hdr));
- session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
- session->ip4_hdr.ip_id = 0;
- session->ip4_hdr.ip_off = 0;
- session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
- session->ip4_hdr.ip_p = (ipsec_xform->proto ==
- RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
- : IPPROTO_AH;
- session->ip4_hdr.ip_sum = 0;
- session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
- session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
- session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
+ session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
+ session->ip4_hdr.ip_id = 0;
+ session->ip4_hdr.ip_off = 0;
+ session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
+ session->ip4_hdr.ip_p = (ipsec_xform->proto ==
+ RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
+ IPPROTO_ESP : IPPROTO_AH;
+ session->ip4_hdr.ip_sum = 0;
+ session->ip4_hdr.ip_src =
+ ipsec_xform->tunnel.ipv4.src_ip;
+ session->ip4_hdr.ip_dst =
+ ipsec_xform->tunnel.ipv4.dst_ip;
+ session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
(void *)&session->ip4_hdr,
sizeof(struct ip));
-
+ session->encap_pdb.ip_hdr_len = sizeof(struct ip);
+ } else if (ipsec_xform->tunnel.type ==
+ RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
+ memset(&session->encap_pdb, 0,
+ sizeof(struct ipsec_encap_pdb) +
+ sizeof(session->ip6_hdr));
+ session->ip6_hdr.vtc_flow = rte_cpu_to_be_32(
+ DPAA_IPv6_DEFAULT_VTC_FLOW |
+ ((ipsec_xform->tunnel.ipv6.dscp <<
+ RTE_IPV6_HDR_TC_SHIFT) &
+ RTE_IPV6_HDR_TC_MASK) |
+ ((ipsec_xform->tunnel.ipv6.flabel <<
+ RTE_IPV6_HDR_FL_SHIFT) &
+ RTE_IPV6_HDR_FL_MASK));
+ /* Payload length will be updated by HW */
+ session->ip6_hdr.payload_len = 0;
+ session->ip6_hdr.hop_limits =
+ ipsec_xform->tunnel.ipv6.hlimit;
+ session->ip6_hdr.proto = (ipsec_xform->proto ==
+ RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
+ IPPROTO_ESP : IPPROTO_AH;
+ memcpy(&session->ip6_hdr.src_addr,
+ &ipsec_xform->tunnel.ipv6.src_addr, 16);
+ memcpy(&session->ip6_hdr.dst_addr,
+ &ipsec_xform->tunnel.ipv6.dst_addr, 16);
+ session->encap_pdb.ip_hdr_len =
+ sizeof(struct rte_ipv6_hdr);
+ }
session->encap_pdb.options =
(IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
PDBOPTS_ESP_OIHI_PDB_INL |
PDBOPTS_ESP_IVSRC |
PDBHMO_ESP_ENCAP_DTTL |
PDBHMO_ESP_SNR;
+ if (ipsec_xform->options.esn)
+ session->encap_pdb.options |= PDBOPTS_ESP_ESN;
session->encap_pdb.spi = ipsec_xform->spi;
- session->encap_pdb.ip_hdr_len = sizeof(struct ip);
-
session->dir = DIR_ENC;
} else if (ipsec_xform->direction ==
RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
- session->decap_pdb.options = sizeof(struct ip) << 16;
+ if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4)
+ session->decap_pdb.options = sizeof(struct ip) << 16;
+ else
+ session->decap_pdb.options =
+ sizeof(struct rte_ipv6_hdr) << 16;
+ if (ipsec_xform->options.esn)
+ session->decap_pdb.options |= PDBOPTS_ESP_ESN;
session->dir = DIR_DEC;
} else
goto out;
- session->ctx_pool = internals->ctx_pool;
rte_spinlock_lock(&internals->lock);
for (i = 0; i < MAX_DPAA_CORES; i++) {
session->inq[i] = dpaa_sec_attach_rxq(internals);
session->dir = DIR_ENC;
}
- /* Auth is only applicable for control mode operation. */
if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
- if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5) {
+ if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5 &&
+ pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_12) {
DPAA_SEC_ERR(
- "PDCP Seq Num size should be 5 bits for cmode");
+ "PDCP Seq Num size should be 5/12 bits for cmode");
goto out;
}
- if (auth_xform) {
- session->auth_key.data = rte_zmalloc(NULL,
- auth_xform->key.length,
- RTE_CACHE_LINE_SIZE);
- if (session->auth_key.data == NULL &&
- auth_xform->key.length > 0) {
- DPAA_SEC_ERR("No Memory for auth key");
- rte_free(session->cipher_key.data);
- return -ENOMEM;
- }
- session->auth_key.length = auth_xform->key.length;
- memcpy(session->auth_key.data, auth_xform->key.data,
- auth_xform->key.length);
- session->auth_alg = auth_xform->algo;
- } else {
- session->auth_key.data = NULL;
- session->auth_key.length = 0;
- session->auth_alg = RTE_CRYPTO_AUTH_NULL;
+ }
+
+ if (auth_xform) {
+ session->auth_key.data = rte_zmalloc(NULL,
+ auth_xform->key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (!session->auth_key.data &&
+ auth_xform->key.length > 0) {
+ DPAA_SEC_ERR("No Memory for auth key");
+ rte_free(session->cipher_key.data);
+ return -ENOMEM;
}
+ session->auth_key.length = auth_xform->key.length;
+ memcpy(session->auth_key.data, auth_xform->key.data,
+ auth_xform->key.length);
+ session->auth_alg = auth_xform->algo;
+ } else {
+ session->auth_key.data = NULL;
+ session->auth_key.length = 0;
+ session->auth_alg = 0;
}
session->pdcp.domain = pdcp_xform->domain;
session->pdcp.bearer = pdcp_xform->bearer;
session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
session->pdcp.sn_size = pdcp_xform->sn_size;
-#ifdef ENABLE_HFN_OVERRIDE
- session->pdcp.hfn_ovd = pdcp_xform->hfn_ovd;
-#endif
session->pdcp.hfn = pdcp_xform->hfn;
session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
+ session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
+ session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
- session->ctx_pool = dev_priv->ctx_pool;
rte_spinlock_lock(&dev_priv->lock);
for (i = 0; i < MAX_DPAA_CORES; i++) {
session->inq[i] = dpaa_sec_attach_rxq(dev_priv);
{
PMD_INIT_FUNC_TRACE();
void *sess_priv = get_sec_session_private_data(sess);
-
dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
if (sess_priv) {
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
-
- rte_free(s->cipher_key.data);
- rte_free(s->auth_key.data);
- memset(sess, 0, sizeof(dpaa_sec_session));
+ free_session_memory((struct rte_cryptodev *)dev, s);
set_sec_session_private_data(sess, NULL);
- rte_mempool_put(sess_mp, sess_priv);
}
return 0;
}
-
static int
-dpaa_sec_dev_configure(struct rte_cryptodev *dev,
+dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
struct rte_cryptodev_config *config __rte_unused)
{
-
- char str[20];
- struct dpaa_sec_dev_private *internals;
-
PMD_INIT_FUNC_TRACE();
- internals = dev->data->dev_private;
- snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id);
- if (!internals->ctx_pool) {
- internals->ctx_pool = rte_mempool_create((const char *)str,
- CTX_POOL_NUM_BUFS,
- CTX_POOL_BUF_SIZE,
- CTX_POOL_CACHE_SIZE, 0,
- NULL, NULL, NULL, NULL,
- SOCKET_ID_ANY, 0);
- if (!internals->ctx_pool) {
- DPAA_SEC_ERR("%s create failed\n", str);
- return -ENOMEM;
- }
- } else
- DPAA_SEC_INFO("mempool already created for dev_id : %d",
- dev->data->dev_id);
-
return 0;
}
static int
dpaa_sec_dev_close(struct rte_cryptodev *dev)
{
- struct dpaa_sec_dev_private *internals;
-
PMD_INIT_FUNC_TRACE();
if (dev == NULL)
return -ENOMEM;
- internals = dev->data->dev_private;
- rte_mempool_free(internals->ctx_pool);
- internals->ctx_pool = NULL;
-
return 0;
}
}
}
+static enum qman_cb_dqrr_result
+dpaa_sec_process_parallel_event(void *event,
+ struct qman_portal *qm __always_unused,
+ struct qman_fq *outq,
+ const struct qm_dqrr_entry *dqrr,
+ void **bufs)
+{
+ const struct qm_fd *fd;
+ struct dpaa_sec_job *job;
+ struct dpaa_sec_op_ctx *ctx;
+ struct rte_event *ev = (struct rte_event *)event;
+
+ fd = &dqrr->fd;
+
+ /* sg is embedded in an op ctx,
+ * sg[0] is for output
+ * sg[1] for input
+ */
+ job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+
+ ctx = container_of(job, struct dpaa_sec_op_ctx, job);
+ ctx->fd_status = fd->status;
+ if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+ struct qm_sg_entry *sg_out;
+ uint32_t len;
+
+ sg_out = &job->sg[0];
+ hw_sg_to_cpu(sg_out);
+ len = sg_out->length;
+ ctx->op->sym->m_src->pkt_len = len;
+ ctx->op->sym->m_src->data_len = len;
+ }
+ if (!ctx->fd_status) {
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ } else {
+ DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ }
+ ev->event_ptr = (void *)ctx->op;
+
+ ev->flow_id = outq->ev.flow_id;
+ ev->sub_event_type = outq->ev.sub_event_type;
+ ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
+ ev->op = RTE_EVENT_OP_NEW;
+ ev->sched_type = outq->ev.sched_type;
+ ev->queue_id = outq->ev.queue_id;
+ ev->priority = outq->ev.priority;
+ *bufs = (void *)ctx->op;
+
+ rte_mempool_put(ctx->ctx_pool, (void *)ctx);
+
+ return qman_cb_dqrr_consume;
+}
+
+static enum qman_cb_dqrr_result
+dpaa_sec_process_atomic_event(void *event,
+ struct qman_portal *qm __rte_unused,
+ struct qman_fq *outq,
+ const struct qm_dqrr_entry *dqrr,
+ void **bufs)
+{
+ u8 index;
+ const struct qm_fd *fd;
+ struct dpaa_sec_job *job;
+ struct dpaa_sec_op_ctx *ctx;
+ struct rte_event *ev = (struct rte_event *)event;
+
+ fd = &dqrr->fd;
+
+ /* sg is embedded in an op ctx,
+ * sg[0] is for output
+ * sg[1] for input
+ */
+ job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
+
+ ctx = container_of(job, struct dpaa_sec_op_ctx, job);
+ ctx->fd_status = fd->status;
+ if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+ struct qm_sg_entry *sg_out;
+ uint32_t len;
+
+ sg_out = &job->sg[0];
+ hw_sg_to_cpu(sg_out);
+ len = sg_out->length;
+ ctx->op->sym->m_src->pkt_len = len;
+ ctx->op->sym->m_src->data_len = len;
+ }
+ if (!ctx->fd_status) {
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ } else {
+ DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ }
+ ev->event_ptr = (void *)ctx->op;
+ ev->flow_id = outq->ev.flow_id;
+ ev->sub_event_type = outq->ev.sub_event_type;
+ ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
+ ev->op = RTE_EVENT_OP_NEW;
+ ev->sched_type = outq->ev.sched_type;
+ ev->queue_id = outq->ev.queue_id;
+ ev->priority = outq->ev.priority;
+
+ /* Save active dqrr entries */
+ index = ((uintptr_t)dqrr >> 6) & (16/*QM_DQRR_SIZE*/ - 1);
+ DPAA_PER_LCORE_DQRR_SIZE++;
+ DPAA_PER_LCORE_DQRR_HELD |= 1 << index;
+ DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src;
+ ev->impl_opaque = index + 1;
+ ctx->op->sym->m_src->seqn = (uint32_t)index + 1;
+ *bufs = (void *)ctx->op;
+
+ rte_mempool_put(ctx->ctx_pool, (void *)ctx);
+
+ return qman_cb_dqrr_defer;
+}
+
+int
+dpaa_sec_eventq_attach(const struct rte_cryptodev *dev,
+ int qp_id,
+ uint16_t ch_id,
+ const struct rte_event *event)
+{
+ struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
+ struct qm_mcc_initfq opts = {0};
+
+ int ret;
+
+ opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
+ QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
+ opts.fqd.dest.channel = ch_id;
+
+ switch (event->sched_type) {
+ case RTE_SCHED_TYPE_ATOMIC:
+ opts.fqd.fq_ctrl |= QM_FQCTRL_HOLDACTIVE;
+ /* Reset FQCTRL_AVOIDBLOCK bit as it is unnecessary
+ * configuration with HOLD_ACTIVE setting
+ */
+ opts.fqd.fq_ctrl &= (~QM_FQCTRL_AVOIDBLOCK);
+ qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_atomic_event;
+ break;
+ case RTE_SCHED_TYPE_ORDERED:
+ DPAA_SEC_ERR("Ordered queue schedule type is not supported\n");
+ return -1;
+ default:
+ opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK;
+ qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_parallel_event;
+ break;
+ }
+
+ ret = qman_init_fq(&qp->outq, QMAN_INITFQ_FLAG_SCHED, &opts);
+ if (unlikely(ret)) {
+ DPAA_SEC_ERR("unable to init caam source fq!");
+ return ret;
+ }
+
+ memcpy(&qp->outq.ev, event, sizeof(struct rte_event));
+
+ return 0;
+}
+
+int
+dpaa_sec_eventq_detach(const struct rte_cryptodev *dev,
+ int qp_id)
+{
+ struct qm_mcc_initfq opts = {0};
+ int ret;
+ struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
+
+ opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
+ QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
+ qp->outq.cb.dqrr = dqrr_out_fq_cb_rx;
+ qp->outq.cb.ern = ern_sec_fq_handler;
+ qman_retire_fq(&qp->outq, NULL);
+ qman_oos_fq(&qp->outq);
+ ret = qman_init_fq(&qp->outq, 0, &opts);
+ if (ret)
+ RTE_LOG(ERR, PMD, "Error in qman_init_fq: ret: %d\n", ret);
+ qp->outq.cb.dqrr = NULL;
+
+ return ret;
+}
+
static struct rte_cryptodev_ops crypto_ops = {
.dev_configure = dpaa_sec_dev_configure,
.dev_start = dpaa_sec_dev_start,
internals = dev->data->dev_private;
rte_free(dev->security_ctx);
- /* In case close has been called, internals->ctx_pool would be NULL */
- rte_mempool_free(internals->ctx_pool);
rte_free(internals);
DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
int retval;
- snprintf(cryptodev_name, sizeof(cryptodev_name), "dpaa_sec-%d",
- dpaa_dev->id.dev_id);
+ snprintf(cryptodev_name, sizeof(cryptodev_name), "%s", dpaa_dev->name);
cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
if (cryptodev == NULL)
git.droids-corp.org / dpdk.git / blobdiff