*/
#include <rte_crypto.h>
-#include <rte_cryptodev.h>
+#include <cryptodev_pmd.h>
#include <rte_cycles.h>
#include <rte_errno.h>
}
static void
-softreq_copy_iv(struct nitrox_softreq *sr)
+softreq_copy_iv(struct nitrox_softreq *sr, uint8_t salt_size)
{
- sr->iv.virt = rte_crypto_op_ctod_offset(sr->op, uint8_t *,
- sr->ctx->iv.offset);
- sr->iv.iova = rte_crypto_op_ctophys_offset(sr->op, sr->ctx->iv.offset);
- sr->iv.len = sr->ctx->iv.length;
-}
-
-static int
-extract_cipher_auth_digest(struct nitrox_softreq *sr,
- struct nitrox_sglist *digest)
-{
- struct rte_crypto_op *op = sr->op;
- struct rte_mbuf *mdst = op->sym->m_dst ? op->sym->m_dst :
- op->sym->m_src;
-
- if (sr->ctx->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY &&
- unlikely(!op->sym->auth.digest.data))
- return -EINVAL;
-
- digest->len = sr->ctx->digest_length;
- if (op->sym->auth.digest.data) {
- digest->iova = op->sym->auth.digest.phys_addr;
- digest->virt = op->sym->auth.digest.data;
- return 0;
- }
-
- if (unlikely(rte_pktmbuf_data_len(mdst) < op->sym->auth.data.offset +
- op->sym->auth.data.length + digest->len))
- return -EINVAL;
+ uint16_t offset = sr->ctx->iv.offset + salt_size;
- digest->iova = rte_pktmbuf_mtophys_offset(mdst,
- op->sym->auth.data.offset +
- op->sym->auth.data.length);
- digest->virt = rte_pktmbuf_mtod_offset(mdst, uint8_t *,
- op->sym->auth.data.offset +
- op->sym->auth.data.length);
- return 0;
+ sr->iv.virt = rte_crypto_op_ctod_offset(sr->op, uint8_t *, offset);
+ sr->iv.iova = rte_crypto_op_ctophys_offset(sr->op, offset);
+ sr->iv.len = sr->ctx->iv.length - salt_size;
}
static void
if (datalen <= mlen)
mlen = datalen;
sglist[cnt].len = mlen;
- sglist[cnt].iova = rte_pktmbuf_mtophys_offset(m, off);
+ sglist[cnt].iova = rte_pktmbuf_iova_offset(m, off);
sglist[cnt].virt = rte_pktmbuf_mtod_offset(m, uint8_t *, off);
sgtbl->total_bytes += mlen;
cnt++;
mlen = rte_pktmbuf_data_len(m) < datalen ?
rte_pktmbuf_data_len(m) : datalen;
sglist[cnt].len = mlen;
- sglist[cnt].iova = rte_pktmbuf_mtophys(m);
+ sglist[cnt].iova = rte_pktmbuf_iova(m);
sglist[cnt].virt = rte_pktmbuf_mtod(m, uint8_t *);
sgtbl->total_bytes += mlen;
cnt++;
return 0;
}
+static void
+create_sgcomp(struct nitrox_sgtable *sgtbl)
+{
+ int i, j, nr_sgcomp;
+ struct nitrox_sgcomp *sgcomp = sgtbl->sgcomp;
+ struct nitrox_sglist *sglist = sgtbl->sglist;
+
+ nr_sgcomp = RTE_ALIGN_MUL_CEIL(sgtbl->map_bufs_cnt, 4) / 4;
+ sgtbl->nr_sgcomp = nr_sgcomp;
+ for (i = 0; i < nr_sgcomp; i++, sgcomp++) {
+ for (j = 0; j < 4; j++, sglist++) {
+ sgcomp->len[j] = rte_cpu_to_be_16(sglist->len);
+ sgcomp->iova[j] = rte_cpu_to_be_64(sglist->iova);
+ }
+ }
+}
+
+static int
+create_cipher_inbuf(struct nitrox_softreq *sr)
+{
+ int err;
+ struct rte_crypto_op *op = sr->op;
+
+ fill_sglist(&sr->in, sr->iv.len, sr->iv.iova, sr->iv.virt);
+ err = create_sglist_from_mbuf(&sr->in, op->sym->m_src,
+ op->sym->cipher.data.offset,
+ op->sym->cipher.data.length);
+ if (unlikely(err))
+ return err;
+
+ create_sgcomp(&sr->in);
+ sr->dptr = sr->iova + offsetof(struct nitrox_softreq, in.sgcomp);
+
+ return 0;
+}
+
+static int
+create_cipher_outbuf(struct nitrox_softreq *sr)
+{
+ struct rte_crypto_op *op = sr->op;
+ int err, cnt = 0;
+ struct rte_mbuf *m_dst = op->sym->m_dst ? op->sym->m_dst :
+ op->sym->m_src;
+
+ sr->resp.orh = PENDING_SIG;
+ sr->out.sglist[cnt].len = sizeof(sr->resp.orh);
+ sr->out.sglist[cnt].iova = sr->iova + offsetof(struct nitrox_softreq,
+ resp.orh);
+ sr->out.sglist[cnt].virt = &sr->resp.orh;
+ cnt++;
+
+ sr->out.map_bufs_cnt = cnt;
+ fill_sglist(&sr->out, sr->iv.len, sr->iv.iova, sr->iv.virt);
+ err = create_sglist_from_mbuf(&sr->out, m_dst,
+ op->sym->cipher.data.offset,
+ op->sym->cipher.data.length);
+ if (unlikely(err))
+ return err;
+
+ cnt = sr->out.map_bufs_cnt;
+ sr->resp.completion = PENDING_SIG;
+ sr->out.sglist[cnt].len = sizeof(sr->resp.completion);
+ sr->out.sglist[cnt].iova = sr->iova + offsetof(struct nitrox_softreq,
+ resp.completion);
+ sr->out.sglist[cnt].virt = &sr->resp.completion;
+ cnt++;
+
+ RTE_VERIFY(cnt <= MAX_SGBUF_CNT);
+ sr->out.map_bufs_cnt = cnt;
+
+ create_sgcomp(&sr->out);
+ sr->rptr = sr->iova + offsetof(struct nitrox_softreq, out.sgcomp);
+
+ return 0;
+}
+
+static void
+create_cipher_gph(uint32_t cryptlen, uint16_t ivlen, struct gphdr *gph)
+{
+ gph->param0 = rte_cpu_to_be_16(cryptlen);
+ gph->param1 = 0;
+ gph->param2 = rte_cpu_to_be_16(ivlen);
+ gph->param3 = 0;
+}
+
+static int
+process_cipher_data(struct nitrox_softreq *sr)
+{
+ struct rte_crypto_op *op = sr->op;
+ int err;
+
+ softreq_copy_iv(sr, 0);
+ err = create_cipher_inbuf(sr);
+ if (unlikely(err))
+ return err;
+
+ err = create_cipher_outbuf(sr);
+ if (unlikely(err))
+ return err;
+
+ create_cipher_gph(op->sym->cipher.data.length, sr->iv.len, &sr->gph);
+
+ return 0;
+}
+
+static int
+extract_cipher_auth_digest(struct nitrox_softreq *sr,
+ struct nitrox_sglist *digest)
+{
+ struct rte_crypto_op *op = sr->op;
+ struct rte_mbuf *mdst = op->sym->m_dst ? op->sym->m_dst :
+ op->sym->m_src;
+
+ if (sr->ctx->req_op == NITROX_OP_DECRYPT &&
+ unlikely(!op->sym->auth.digest.data))
+ return -EINVAL;
+
+ digest->len = sr->ctx->digest_length;
+ if (op->sym->auth.digest.data) {
+ digest->iova = op->sym->auth.digest.phys_addr;
+ digest->virt = op->sym->auth.digest.data;
+ return 0;
+ }
+
+ if (unlikely(rte_pktmbuf_data_len(mdst) < op->sym->auth.data.offset +
+ op->sym->auth.data.length + digest->len))
+ return -EINVAL;
+
+ digest->iova = rte_pktmbuf_iova_offset(mdst,
+ op->sym->auth.data.offset +
+ op->sym->auth.data.length);
+ digest->virt = rte_pktmbuf_mtod_offset(mdst, uint8_t *,
+ op->sym->auth.data.offset +
+ op->sym->auth.data.length);
+ return 0;
+}
+
static int
create_cipher_auth_sglist(struct nitrox_softreq *sr,
struct nitrox_sgtable *sgtbl, struct rte_mbuf *mbuf)
if (unlikely(auth_only_len < 0))
return -EINVAL;
+ if (unlikely(
+ op->sym->cipher.data.offset + op->sym->cipher.data.length !=
+ op->sym->auth.data.offset + op->sym->auth.data.length)) {
+ NITROX_LOG(ERR, "Auth only data after cipher data not supported\n");
+ return -ENOTSUP;
+ }
+
err = create_sglist_from_mbuf(sgtbl, mbuf, op->sym->auth.data.offset,
auth_only_len);
if (unlikely(err))
return 0;
}
-static void
-create_sgcomp(struct nitrox_sgtable *sgtbl)
+static int
+create_combined_sglist(struct nitrox_softreq *sr, struct nitrox_sgtable *sgtbl,
+ struct rte_mbuf *mbuf)
{
- int i, j, nr_sgcomp;
- struct nitrox_sgcomp *sgcomp = sgtbl->sgcomp;
- struct nitrox_sglist *sglist = sgtbl->sglist;
+ struct rte_crypto_op *op = sr->op;
- nr_sgcomp = RTE_ALIGN_MUL_CEIL(sgtbl->map_bufs_cnt, 4) / 4;
- sgtbl->nr_sgcomp = nr_sgcomp;
- for (i = 0; i < nr_sgcomp; i++, sgcomp++) {
- for (j = 0; j < 4; j++, sglist++) {
- sgcomp->len[j] = rte_cpu_to_be_16(sglist->len);
- sgcomp->iova[j] = rte_cpu_to_be_64(sglist->iova);
- }
+ fill_sglist(sgtbl, sr->iv.len, sr->iv.iova, sr->iv.virt);
+ fill_sglist(sgtbl, sr->ctx->aad_length, op->sym->aead.aad.phys_addr,
+ op->sym->aead.aad.data);
+ return create_sglist_from_mbuf(sgtbl, mbuf, op->sym->cipher.data.offset,
+ op->sym->cipher.data.length);
+}
+
+static int
+create_aead_sglist(struct nitrox_softreq *sr, struct nitrox_sgtable *sgtbl,
+ struct rte_mbuf *mbuf)
+{
+ int err;
+
+ switch (sr->ctx->nitrox_chain) {
+ case NITROX_CHAIN_CIPHER_AUTH:
+ case NITROX_CHAIN_AUTH_CIPHER:
+ err = create_cipher_auth_sglist(sr, sgtbl, mbuf);
+ break;
+ case NITROX_CHAIN_COMBINED:
+ err = create_combined_sglist(sr, sgtbl, mbuf);
+ break;
+ default:
+ err = -EINVAL;
+ break;
}
+
+ return err;
}
static int
-create_cipher_auth_inbuf(struct nitrox_softreq *sr,
- struct nitrox_sglist *digest)
+create_aead_inbuf(struct nitrox_softreq *sr, struct nitrox_sglist *digest)
{
int err;
struct nitrox_crypto_ctx *ctx = sr->ctx;
- err = create_cipher_auth_sglist(sr, &sr->in, sr->op->sym->m_src);
+ err = create_aead_sglist(sr, &sr->in, sr->op->sym->m_src);
if (unlikely(err))
return err;
- if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
+ if (ctx->req_op == NITROX_OP_DECRYPT)
fill_sglist(&sr->in, digest->len, digest->iova, digest->virt);
create_sgcomp(&sr->in);
}
static int
-create_cipher_auth_oop_outbuf(struct nitrox_softreq *sr,
- struct nitrox_sglist *digest)
+create_aead_oop_outbuf(struct nitrox_softreq *sr, struct nitrox_sglist *digest)
{
int err;
struct nitrox_crypto_ctx *ctx = sr->ctx;
- err = create_cipher_auth_sglist(sr, &sr->out, sr->op->sym->m_dst);
+ err = create_aead_sglist(sr, &sr->out, sr->op->sym->m_dst);
if (unlikely(err))
return err;
- if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE)
+ if (ctx->req_op == NITROX_OP_ENCRYPT)
fill_sglist(&sr->out, digest->len, digest->iova, digest->virt);
return 0;
}
static void
-create_cipher_auth_inplace_outbuf(struct nitrox_softreq *sr,
- struct nitrox_sglist *digest)
+create_aead_inplace_outbuf(struct nitrox_softreq *sr,
+ struct nitrox_sglist *digest)
{
int i, cnt;
struct nitrox_crypto_ctx *ctx = sr->ctx;
}
sr->out.map_bufs_cnt = cnt;
- if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) {
+ if (ctx->req_op == NITROX_OP_ENCRYPT) {
fill_sglist(&sr->out, digest->len, digest->iova,
digest->virt);
- } else if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
+ } else if (ctx->req_op == NITROX_OP_DECRYPT) {
sr->out.map_bufs_cnt--;
}
}
static int
-create_cipher_auth_outbuf(struct nitrox_softreq *sr,
- struct nitrox_sglist *digest)
+create_aead_outbuf(struct nitrox_softreq *sr, struct nitrox_sglist *digest)
{
struct rte_crypto_op *op = sr->op;
int cnt = 0;
if (op->sym->m_dst) {
int err;
- err = create_cipher_auth_oop_outbuf(sr, digest);
+ err = create_aead_oop_outbuf(sr, digest);
if (unlikely(err))
return err;
} else {
- create_cipher_auth_inplace_outbuf(sr, digest);
+ create_aead_inplace_outbuf(sr, digest);
}
cnt = sr->out.map_bufs_cnt;
int err;
struct nitrox_sglist digest;
- softreq_copy_iv(sr);
+ softreq_copy_iv(sr, 0);
err = extract_cipher_auth_digest(sr, &digest);
if (unlikely(err))
return err;
- err = create_cipher_auth_inbuf(sr, &digest);
+ err = create_aead_inbuf(sr, &digest);
if (unlikely(err))
return err;
- err = create_cipher_auth_outbuf(sr, &digest);
+ err = create_aead_outbuf(sr, &digest);
if (unlikely(err))
return err;
return 0;
}
+static int
+softreq_copy_salt(struct nitrox_softreq *sr)
+{
+ struct nitrox_crypto_ctx *ctx = sr->ctx;
+ uint8_t *addr;
+
+ if (unlikely(ctx->iv.length < AES_GCM_SALT_SIZE)) {
+ NITROX_LOG(ERR, "Invalid IV length %d\n", ctx->iv.length);
+ return -EINVAL;
+ }
+
+ addr = rte_crypto_op_ctod_offset(sr->op, uint8_t *, ctx->iv.offset);
+ if (!memcmp(ctx->salt, addr, AES_GCM_SALT_SIZE))
+ return 0;
+
+ memcpy(ctx->salt, addr, AES_GCM_SALT_SIZE);
+ memcpy(ctx->fctx.crypto.iv, addr, AES_GCM_SALT_SIZE);
+ return 0;
+}
+
+static int
+extract_combined_digest(struct nitrox_softreq *sr, struct nitrox_sglist *digest)
+{
+ struct rte_crypto_op *op = sr->op;
+ struct rte_mbuf *mdst = op->sym->m_dst ? op->sym->m_dst :
+ op->sym->m_src;
+
+ digest->len = sr->ctx->digest_length;
+ if (op->sym->aead.digest.data) {
+ digest->iova = op->sym->aead.digest.phys_addr;
+ digest->virt = op->sym->aead.digest.data;
+
+ return 0;
+ }
+
+ if (unlikely(rte_pktmbuf_data_len(mdst) < op->sym->aead.data.offset +
+ op->sym->aead.data.length + digest->len))
+ return -EINVAL;
+
+ digest->iova = rte_pktmbuf_iova_offset(mdst,
+ op->sym->aead.data.offset +
+ op->sym->aead.data.length);
+ digest->virt = rte_pktmbuf_mtod_offset(mdst, uint8_t *,
+ op->sym->aead.data.offset +
+ op->sym->aead.data.length);
+
+ return 0;
+}
+
+static int
+process_combined_data(struct nitrox_softreq *sr)
+{
+ int err;
+ struct nitrox_sglist digest;
+ struct rte_crypto_op *op = sr->op;
+
+ err = softreq_copy_salt(sr);
+ if (unlikely(err))
+ return err;
+
+ softreq_copy_iv(sr, AES_GCM_SALT_SIZE);
+ err = extract_combined_digest(sr, &digest);
+ if (unlikely(err))
+ return err;
+
+ err = create_aead_inbuf(sr, &digest);
+ if (unlikely(err))
+ return err;
+
+ err = create_aead_outbuf(sr, &digest);
+ if (unlikely(err))
+ return err;
+
+ create_aead_gph(op->sym->aead.data.length, sr->iv.len,
+ op->sym->aead.data.length + sr->ctx->aad_length,
+ &sr->gph);
+
+ return 0;
+}
+
static int
process_softreq(struct nitrox_softreq *sr)
{
int err = 0;
switch (ctx->nitrox_chain) {
+ case NITROX_CHAIN_CIPHER_ONLY:
+ err = process_cipher_data(sr);
+ break;
case NITROX_CHAIN_CIPHER_AUTH:
case NITROX_CHAIN_AUTH_CIPHER:
err = process_cipher_auth_data(sr);
break;
+ case NITROX_CHAIN_COMBINED:
+ err = process_combined_data(sr);
+ break;
default:
err = -EINVAL;
break;
struct nitrox_crypto_ctx *ctx,
struct nitrox_softreq *sr)
{
+ int err;
+
softreq_init(sr, sr->iova);
sr->ctx = ctx;
sr->op = op;
- process_softreq(sr);
+ err = process_softreq(sr);
+ if (unlikely(err))
+ return err;
+
create_se_instr(sr, qno);
sr->timeout = rte_get_timer_cycles() + CMD_TIMEOUT * rte_get_timer_hz();
return 0;
cc = *(volatile uint64_t *)(&sr->resp.completion);
orh = *(volatile uint64_t *)(&sr->resp.orh);
if (cc != PENDING_SIG)
- err = 0;
+ err = orh & 0xff;
else if ((orh != PENDING_SIG) && (orh & 0xff))
err = orh & 0xff;
else if (rte_get_timer_cycles() >= sr->timeout)
git.droids-corp.org / dpdk.git / blobdiff