-/*-
- * BSD LICENSE
+/* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- * Copyright 2017 NXP.
+ * Copyright 2017 NXP
*
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of NXP nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <fcntl.h>
static __thread struct rte_crypto_op **dpaa_sec_ops;
static __thread int dpaa_sec_op_nb;
+static int
+dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
+
static inline void
dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
{
dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
ctx->ctx_pool = ses->ctx_pool;
+ ctx->vtop_offset = (uint64_t) ctx
+ - rte_mempool_virt2iova(ctx);
return ctx;
}
-static inline phys_addr_t
+static inline rte_iova_t
dpaa_mem_vtop(void *vaddr)
{
const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
if (vaddr_64 >= memseg[i].addr_64 &&
vaddr_64 < memseg[i].addr_64 + memseg[i].len) {
- paddr = memseg[i].phys_addr +
+ paddr = memseg[i].iova +
(vaddr_64 - memseg[i].addr_64);
- return (phys_addr_t)paddr;
+ return (rte_iova_t)paddr;
}
}
- return (phys_addr_t)(NULL);
+ return (rte_iova_t)(NULL);
+}
+
+/* virtual address conversin when mempool support is available for ctx */
+static inline phys_addr_t
+dpaa_mem_vtop_ctx(struct dpaa_sec_op_ctx *ctx, void *vaddr)
+{
+ return (uint64_t)vaddr - ctx->vtop_offset;
}
static inline void *
-dpaa_mem_ptov(phys_addr_t paddr)
+dpaa_mem_ptov(rte_iova_t paddr)
{
const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
int i;
for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
- if (paddr >= memseg[i].phys_addr &&
- (char *)paddr < (char *)memseg[i].phys_addr + memseg[i].len)
+ if (paddr >= memseg[i].iova &&
+ (char *)paddr < (char *)memseg[i].iova + memseg[i].len)
return (void *)(memseg[i].addr_64 +
- (paddr - memseg[i].phys_addr));
+ (paddr - memseg[i].iova));
}
return NULL;
}
* all the packets in this queue could be dispatched into caam
*/
static int
-dpaa_sec_init_rx(struct qman_fq *fq_in, phys_addr_t hwdesc,
+dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
uint32_t fqid_out)
{
struct qm_mcc_initfq fq_opts;
/* Clear FQ options */
memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
- flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
- QMAN_FQ_FLAG_TO_DCPORTAL;
-
- ret = qman_create_fq(0, flags, fq_in);
- if (unlikely(ret != 0)) {
- PMD_INIT_LOG(ERR, "qman_create_fq failed");
- return ret;
- }
-
flags = QMAN_INITFQ_FLAG_SCHED;
fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
QM_INITFQ_WE_CONTEXTB;
fq_in->cb.ern = ern_sec_fq_handler;
+ PMD_INIT_LOG(DEBUG, "in-%x out-%x", fq_in->fqid, fqid_out);
+
ret = qman_init_fq(fq_in, flags, &fq_opts);
if (unlikely(ret != 0))
- PMD_INIT_LOG(ERR, "qman_init_fq failed");
+ PMD_INIT_LOG(ERR, "qman_init_fq failed %d", ret);
return ret;
}
{
struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
uint32_t shared_desc_len = 0;
- struct sec_cdb *cdb = &ses->qp->cdb;
+ struct sec_cdb *cdb = &ses->cdb;
int err;
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
int swap = false;
struct dpaa_sec_job *cf;
struct dpaa_sec_op_ctx *ctx;
struct qm_sg_entry *sg;
- phys_addr_t start_addr;
+ rte_iova_t start_addr;
uint8_t *old_digest;
ctx = dpaa_sec_alloc_ctx(ses);
ctx->op = op;
old_digest = ctx->digest;
- start_addr = rte_pktmbuf_mtophys(mbuf);
+ start_addr = rte_pktmbuf_iova(mbuf);
/* output */
sg = &cf->sg[0];
qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
if (is_decode(ses)) {
/* need to extend the input to a compound frame */
sg->extension = 1;
- qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
+ qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
sg->length = sym->auth.data.length + ses->digest_length;
sg->final = 1;
cpu_to_hw_sg(sg);
/* hash result or digest, save digest first */
rte_memcpy(old_digest, sym->auth.digest.data,
ses->digest_length);
- memset(sym->auth.digest.data, 0, ses->digest_length);
qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
sg->length = sym->auth.data.length;
cpu_to_hw_sg(sg);
/* let's check digest by hw */
- start_addr = dpaa_mem_vtop(old_digest);
+ start_addr = dpaa_mem_vtop_ctx(ctx, old_digest);
sg++;
qm_sg_entry_set64(sg, start_addr);
sg->length = ses->digest_length;
build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
{
struct rte_crypto_sym_op *sym = op->sym;
- struct rte_mbuf *mbuf = sym->m_src;
struct dpaa_sec_job *cf;
struct dpaa_sec_op_ctx *ctx;
struct qm_sg_entry *sg;
- phys_addr_t start_addr;
+ rte_iova_t src_start_addr, dst_start_addr;
uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
ses->iv.offset);
cf = &ctx->job;
ctx->op = op;
- start_addr = rte_pktmbuf_mtophys(mbuf);
+
+ src_start_addr = rte_pktmbuf_iova(sym->m_src);
+
+ if (sym->m_dst)
+ dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
+ else
+ dst_start_addr = src_start_addr;
/* output */
sg = &cf->sg[0];
- qm_sg_entry_set64(sg, start_addr + sym->cipher.data.offset);
+ qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
sg->length = sym->cipher.data.length + ses->iv.length;
cpu_to_hw_sg(sg);
sg->extension = 1;
sg->final = 1;
sg->length = sym->cipher.data.length + ses->iv.length;
- qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
+ qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
cpu_to_hw_sg(sg);
sg = &cf->sg[2];
cpu_to_hw_sg(sg);
sg++;
- qm_sg_entry_set64(sg, start_addr + sym->cipher.data.offset);
+ qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset);
sg->length = sym->cipher.data.length;
sg->final = 1;
cpu_to_hw_sg(sg);
build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
{
struct rte_crypto_sym_op *sym = op->sym;
- struct rte_mbuf *mbuf = sym->m_src;
struct dpaa_sec_job *cf;
struct dpaa_sec_op_ctx *ctx;
struct qm_sg_entry *sg;
- phys_addr_t start_addr;
uint32_t length = 0;
+ rte_iova_t src_start_addr, dst_start_addr;
uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
ses->iv.offset);
- start_addr = mbuf->buf_physaddr + mbuf->data_off;
+ src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
+
+ if (sym->m_dst)
+ dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
+ else
+ dst_start_addr = src_start_addr;
ctx = dpaa_sec_alloc_ctx(ses);
if (!ctx)
/* input */
rte_prefetch0(cf->sg);
sg = &cf->sg[2];
- qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
+ qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg));
if (is_encode(ses)) {
qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
sg->length = ses->iv.length;
cpu_to_hw_sg(sg);
sg++;
}
- qm_sg_entry_set64(sg, start_addr + sym->aead.data.offset);
+ qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
sg->length = sym->aead.data.length;
length += sg->length;
sg->final = 1;
cpu_to_hw_sg(sg);
sg++;
}
- qm_sg_entry_set64(sg, start_addr + sym->aead.data.offset);
+ qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
sg->length = sym->aead.data.length;
length += sg->length;
cpu_to_hw_sg(sg);
memcpy(ctx->digest, sym->aead.digest.data,
ses->digest_length);
- memset(sym->aead.digest.data, 0, ses->digest_length);
sg++;
- qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
+ qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
sg->length = ses->digest_length;
length += sg->length;
sg->final = 1;
/* output */
sg++;
- qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
+ qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg));
qm_sg_entry_set64(sg,
- start_addr + sym->aead.data.offset - ses->auth_only_len);
+ dst_start_addr + sym->aead.data.offset - ses->auth_only_len);
sg->length = sym->aead.data.length + ses->auth_only_len;
length = sg->length;
if (is_encode(ses)) {
build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
{
struct rte_crypto_sym_op *sym = op->sym;
- struct rte_mbuf *mbuf = sym->m_src;
struct dpaa_sec_job *cf;
struct dpaa_sec_op_ctx *ctx;
struct qm_sg_entry *sg;
- phys_addr_t start_addr;
+ rte_iova_t src_start_addr, dst_start_addr;
uint32_t length = 0;
uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
ses->iv.offset);
- start_addr = mbuf->buf_physaddr + mbuf->data_off;
+ src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
+ if (sym->m_dst)
+ dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
+ else
+ dst_start_addr = src_start_addr;
ctx = dpaa_sec_alloc_ctx(ses);
if (!ctx)
/* input */
rte_prefetch0(cf->sg);
sg = &cf->sg[2];
- qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
+ qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg));
if (is_encode(ses)) {
qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
sg->length = ses->iv.length;
cpu_to_hw_sg(sg);
sg++;
- qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
+ qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
sg->length = sym->auth.data.length;
length += sg->length;
sg->final = 1;
sg++;
- qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
+ qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
sg->length = sym->auth.data.length;
length += sg->length;
cpu_to_hw_sg(sg);
memcpy(ctx->digest, sym->auth.digest.data,
ses->digest_length);
- memset(sym->auth.digest.data, 0, ses->digest_length);
sg++;
- qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
+ qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
sg->length = ses->digest_length;
length += sg->length;
sg->final = 1;
/* output */
sg++;
- qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
- qm_sg_entry_set64(sg, start_addr + sym->cipher.data.offset);
+ qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg));
+ qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
sg->length = sym->cipher.data.length;
length = sg->length;
if (is_encode(ses)) {
ses = (dpaa_sec_session *)get_session_private_data(op->sym->session,
cryptodev_driver_id);
- if (unlikely(!qp->ses || qp->ses != ses)) {
- qp->ses = ses;
- ses->qp = qp;
- ret = dpaa_sec_prep_cdb(ses);
- if (ret)
- return ret;
+ if (unlikely(!ses->qp || ses->qp != qp)) {
+ PMD_INIT_LOG(DEBUG, "sess->qp - %p qp %p", ses->qp, qp);
+ if (dpaa_sec_attach_sess_q(qp, ses))
+ return -1;
}
+ /*
+ * Segmented buffer is not supported.
+ */
+ if (!rte_pktmbuf_is_contiguous(op->sym->m_src)) {
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return -ENOTSUP;
+ }
if (is_auth_only(ses)) {
cf = build_auth_only(op, ses);
} else if (is_cipher_only(ses)) {
if (auth_only_len)
fd.cmd = 0x80000000 | auth_only_len;
do {
- ret = qman_enqueue(&qp->inq, &fd, 0);
+ ret = qman_enqueue(ses->inq, &fd, 0);
} while (ret != 0);
return 0;
return 0;
}
-static int
-dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses)
+static struct qman_fq *
+dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
{
- dpaa_sec_session *sess = ses;
- struct dpaa_sec_qp *qp;
+ unsigned int i;
- PMD_INIT_FUNC_TRACE();
+ for (i = 0; i < qi->max_nb_sessions; i++) {
+ if (qi->inq_attach[i] == 0) {
+ qi->inq_attach[i] = 1;
+ return &qi->inq[i];
+ }
+ }
+ PMD_DRV_LOG(ERR, "All ses session in use %x", qi->max_nb_sessions);
+
+ return NULL;
+}
- qp = dev->data->queue_pairs[qp_id];
- if (qp->ses != NULL) {
- PMD_INIT_LOG(ERR, "qp in-use by another session\n");
- return -EBUSY;
+static int
+dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
+{
+ unsigned int i;
+
+ for (i = 0; i < qi->max_nb_sessions; i++) {
+ if (&qi->inq[i] == fq) {
+ qi->inq_attach[i] = 0;
+ return 0;
+ }
}
+ return -1;
+}
+
+static int
+dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
+{
+ int ret;
- qp->ses = sess;
sess->qp = qp;
+ ret = dpaa_sec_prep_cdb(sess);
+ if (ret) {
+ PMD_DRV_LOG(ERR, "Unable to prepare sec cdb");
+ return -1;
+ }
+
+ ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
+ qman_fq_fqid(&qp->outq));
+ if (ret)
+ PMD_DRV_LOG(ERR, "Unable to init sec queue");
- return dpaa_sec_prep_cdb(sess);
+ return ret;
}
static int
-dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses)
+dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev __rte_unused,
+ uint16_t qp_id __rte_unused,
+ void *ses __rte_unused)
+{
+ PMD_INIT_FUNC_TRACE();
+ return 0;
+}
+
+static int
+dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev,
+ uint16_t qp_id __rte_unused,
+ void *ses)
{
dpaa_sec_session *sess = ses;
- struct dpaa_sec_qp *qp;
+ struct dpaa_sec_dev_private *qi = dev->data->dev_private;
PMD_INIT_FUNC_TRACE();
- qp = dev->data->queue_pairs[qp_id];
- if (qp->ses != NULL) {
- qp->ses = NULL;
- sess->qp = NULL;
- return 0;
- }
+ if (sess->inq)
+ dpaa_sec_detach_rxq(qi, sess->inq);
+ sess->inq = NULL;
- PMD_DRV_LOG(ERR, "No session attached to qp");
- return -EINVAL;
+ sess->qp = NULL;
+
+ return 0;
}
static int
return -EINVAL;
}
session->ctx_pool = internals->ctx_pool;
+ session->inq = dpaa_sec_attach_rxq(internals);
+ if (session->inq == NULL) {
+ PMD_DRV_LOG(ERR, "unable to attach sec queue");
+ goto err1;
+ }
return 0;
+
+err1:
+ rte_free(session->cipher_key.data);
+ rte_free(session->auth_key.data);
+ memset(session, 0, sizeof(dpaa_sec_session));
+
+ return -EINVAL;
}
static int
set_session_private_data(sess, dev->driver_id,
sess_private_data);
+
return 0;
}
dpaa_sec_session_clear(struct rte_cryptodev *dev,
struct rte_cryptodev_sym_session *sess)
{
- PMD_INIT_FUNC_TRACE();
+ struct dpaa_sec_dev_private *qi = dev->data->dev_private;
uint8_t index = dev->driver_id;
void *sess_priv = get_session_private_data(sess, index);
+
+ PMD_INIT_FUNC_TRACE();
+
dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
if (sess_priv) {
+ struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+
+ if (s->inq)
+ dpaa_sec_detach_rxq(qi, s->inq);
rte_free(s->cipher_key.data);
rte_free(s->auth_key.data);
memset(s, 0, sizeof(dpaa_sec_session));
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
set_session_private_data(sess, index, NULL);
rte_mempool_put(sess_mp, sess_priv);
}
info->capabilities = dpaa_sec_capabilities;
info->sym.max_nb_sessions = internals->max_nb_sessions;
info->sym.max_nb_sessions_per_qp =
- RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS / RTE_MAX_NB_SEC_QPS;
+ RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS /
+ RTE_DPAA_MAX_NB_SEC_QPS;
info->driver_id = cryptodev_driver_id;
}
}
{
struct dpaa_sec_dev_private *internals;
struct dpaa_sec_qp *qp;
- uint32_t i;
+ uint32_t i, flags;
int ret;
char str[20];
RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING;
internals = cryptodev->data->dev_private;
- internals->max_nb_queue_pairs = RTE_MAX_NB_SEC_QPS;
+ internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
for (i = 0; i < internals->max_nb_queue_pairs; i++) {
PMD_INIT_LOG(ERR, "config tx of queue pair %d", i);
goto init_error;
}
- ret = dpaa_sec_init_rx(&qp->inq, dpaa_mem_vtop(&qp->cdb),
- qman_fq_fqid(&qp->outq));
- if (ret) {
- PMD_INIT_LOG(ERR, "config rx of queue pair %d", i);
+ }
+
+ flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
+ QMAN_FQ_FLAG_TO_DCPORTAL;
+ for (i = 0; i < internals->max_nb_sessions; i++) {
+ /* create rx qman fq for sessions*/
+ ret = qman_create_fq(0, flags, &internals->inq[i]);
+ if (unlikely(ret != 0)) {
+ PMD_INIT_LOG(ERR, "sec qman_create_fq failed");
goto init_error;
}
}
if (ret)
return ret;
- /* free crypto device */
- rte_cryptodev_pmd_release_device(cryptodev);
-
- if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- rte_free(cryptodev->data->dev_private);
-
- PMD_INIT_LOG(INFO, "Closing dpaa crypto device %s",
- cryptodev->data->name);
-
- cryptodev->device = NULL;
- cryptodev->data = NULL;
-
- return 0;
+ return rte_cryptodev_pmd_destroy(cryptodev);
}
static struct rte_dpaa_driver rte_dpaa_sec_driver = {
git.droids-corp.org / dpdk.git / blobdiff