X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fnitrox%2Fnitrox_sym_reqmgr.c;h=d9b42677640bde66c5a2b0e94b88a92046fc5989;hb=446c3781c46865de8fc18f738cd44b2ab9f13948;hp=42d67317c7542b7eebbd214a708e4737e926d256;hpb=3b1fa94a022b861425a40ebc8e74dd75772e4bcb;p=dpdk.git

diff --git a/drivers/crypto/nitrox/nitrox_sym_reqmgr.c b/drivers/crypto/nitrox/nitrox_sym_reqmgr.c
index 42d67317c7..d9b4267764 100644
--- a/drivers/crypto/nitrox/nitrox_sym_reqmgr.c
+++ b/drivers/crypto/nitrox/nitrox_sym_reqmgr.c
@@ -4,12 +4,162 @@
 
 #include <rte_crypto.h>
 #include <rte_cryptodev.h>
+#include <rte_cycles.h>
 #include <rte_errno.h>
 
 #include "nitrox_sym_reqmgr.h"
 #include "nitrox_logs.h"
 
+#define MAX_SGBUF_CNT 16
+#define MAX_SGCOMP_CNT 5
+/* SLC_STORE_INFO */
+#define MIN_UDD_LEN 16
+/* PKT_IN_HDR + SLC_STORE_INFO */
+#define FDATA_SIZE 32
+/* Base destination port for the solicited requests */
+#define SOLICIT_BASE_DPORT 256
+#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
+#define CMD_TIMEOUT 2
+
+struct gphdr {
+	uint16_t param0;
+	uint16_t param1;
+	uint16_t param2;
+	uint16_t param3;
+};
+
+union pkt_instr_hdr {
+	uint64_t value;
+	struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+		uint64_t raz_48_63 : 16;
+		uint64_t g : 1;
+		uint64_t gsz : 7;
+		uint64_t ihi : 1;
+		uint64_t ssz : 7;
+		uint64_t raz_30_31 : 2;
+		uint64_t fsz : 6;
+		uint64_t raz_16_23 : 8;
+		uint64_t tlen : 16;
+#else
+		uint64_t tlen : 16;
+		uint64_t raz_16_23 : 8;
+		uint64_t fsz : 6;
+		uint64_t raz_30_31 : 2;
+		uint64_t ssz : 7;
+		uint64_t ihi : 1;
+		uint64_t gsz : 7;
+		uint64_t g : 1;
+		uint64_t raz_48_63 : 16;
+#endif
+	} s;
+};
+
+union pkt_hdr {
+	uint64_t value[2];
+	struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+		uint64_t opcode : 8;
+		uint64_t arg : 8;
+		uint64_t ctxc : 2;
+		uint64_t unca : 1;
+		uint64_t raz_44 : 1;
+		uint64_t info : 3;
+		uint64_t destport : 9;
+		uint64_t unc : 8;
+		uint64_t raz_19_23 : 5;
+		uint64_t grp : 3;
+		uint64_t raz_15 : 1;
+		uint64_t ctxl : 7;
+		uint64_t uddl : 8;
+#else
+		uint64_t uddl : 8;
+		uint64_t ctxl : 7;
+		uint64_t raz_15 : 1;
+		uint64_t grp : 3;
+		uint64_t raz_19_23 : 5;
+		uint64_t unc : 8;
+		uint64_t destport : 9;
+		uint64_t info : 3;
+		uint64_t raz_44 : 1;
+		uint64_t unca : 1;
+		uint64_t ctxc : 2;
+		uint64_t arg : 8;
+		uint64_t opcode : 8;
+#endif
+		uint64_t ctxp;
+	} s;
+};
+
+union slc_store_info {
+	uint64_t value[2];
+	struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+		uint64_t raz_39_63 : 25;
+		uint64_t ssz : 7;
+		uint64_t raz_0_31 : 32;
+#else
+		uint64_t raz_0_31 : 32;
+		uint64_t ssz : 7;
+		uint64_t raz_39_63 : 25;
+#endif
+		uint64_t rptr;
+	} s;
+};
+
+struct nps_pkt_instr {
+	uint64_t dptr0;
+	union pkt_instr_hdr ih;
+	union pkt_hdr irh;
+	union slc_store_info slc;
+	uint64_t fdata[2];
+};
+
+struct resp_hdr {
+	uint64_t orh;
+	uint64_t completion;
+};
+
+struct nitrox_sglist {
+	uint16_t len;
+	uint16_t raz0;
+	uint32_t raz1;
+	rte_iova_t iova;
+	void *virt;
+};
+
+struct nitrox_sgcomp {
+	uint16_t len[4];
+	uint64_t iova[4];
+};
+
+struct nitrox_sgtable {
+	uint8_t map_bufs_cnt;
+	uint8_t nr_sgcomp;
+	uint16_t total_bytes;
+
+	struct nitrox_sglist sglist[MAX_SGBUF_CNT];
+	struct nitrox_sgcomp sgcomp[MAX_SGCOMP_CNT];
+};
+
+struct iv {
+	uint8_t *virt;
+	rte_iova_t iova;
+	uint16_t len;
+};
+
 struct nitrox_softreq {
+	struct nitrox_crypto_ctx *ctx;
+	struct rte_crypto_op *op;
+	struct gphdr gph;
+	struct nps_pkt_instr instr;
+	struct resp_hdr resp;
+	struct nitrox_sgtable in;
+	struct nitrox_sgtable out;
+	struct iv iv;
+	uint64_t timeout;
+	rte_iova_t dptr;
+	rte_iova_t rptr;
 	rte_iova_t iova;
 };
 
@@ -20,6 +170,435 @@ softreq_init(struct nitrox_softreq *sr, rte_iova_t iova)
 	sr->iova = iova;
 }
 
+/*
+ * 64-Byte Instruction Format
+ *
+ *  ----------------------
+ *  |      DPTR0         | 8 bytes
+ *  ----------------------
+ *  |  PKT_IN_INSTR_HDR  | 8 bytes
+ *  ----------------------
+ *  |    PKT_IN_HDR      | 16 bytes
+ *  ----------------------
+ *  |    SLC_INFO        | 16 bytes
+ *  ----------------------
+ *  |   Front data       | 16 bytes
+ *  ----------------------
+ */
+static void
+create_se_instr(struct nitrox_softreq *sr, uint8_t qno)
+{
+	struct nitrox_crypto_ctx *ctx = sr->ctx;
+	rte_iova_t ctx_handle;
+
+	/* fill the packet instruction */
+	/* word 0 */
+	sr->instr.dptr0 = rte_cpu_to_be_64(sr->dptr);
+
+	/* word 1 */
+	sr->instr.ih.value = 0;
+	sr->instr.ih.s.g = 1;
+	sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
+	sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
+	sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
+	sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
+	sr->instr.ih.value = rte_cpu_to_be_64(sr->instr.ih.value);
+
+	/* word 2 */
+	sr->instr.irh.value[0] = 0;
+	sr->instr.irh.s.uddl = MIN_UDD_LEN;
+	/* context length in 64-bit words */
+	sr->instr.irh.s.ctxl = RTE_ALIGN_MUL_CEIL(sizeof(ctx->fctx), 8) / 8;
+	/* offset from solicit base port 256 */
+	sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
+	/* Invalid context cache */
+	sr->instr.irh.s.ctxc = 0x3;
+	sr->instr.irh.s.arg = ctx->req_op;
+	sr->instr.irh.s.opcode = ctx->opcode;
+	sr->instr.irh.value[0] = rte_cpu_to_be_64(sr->instr.irh.value[0]);
+
+	/* word 3 */
+	ctx_handle = ctx->iova + offsetof(struct nitrox_crypto_ctx, fctx);
+	sr->instr.irh.s.ctxp = rte_cpu_to_be_64(ctx_handle);
+
+	/* word 4 */
+	sr->instr.slc.value[0] = 0;
+	sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
+	sr->instr.slc.value[0] = rte_cpu_to_be_64(sr->instr.slc.value[0]);
+
+	/* word 5 */
+	sr->instr.slc.s.rptr = rte_cpu_to_be_64(sr->rptr);
+	/*
+	 * No conversion for front data,
+	 * It goes into payload
+	 * put GP Header in front data
+	 */
+	memcpy(&sr->instr.fdata[0], &sr->gph, sizeof(sr->instr.fdata[0]));
+	sr->instr.fdata[1] = 0;
+}
+
+static void
+softreq_copy_iv(struct nitrox_softreq *sr)
+{
+	sr->iv.virt = rte_crypto_op_ctod_offset(sr->op, uint8_t *,
+						sr->ctx->iv.offset);
+	sr->iv.iova = rte_crypto_op_ctophys_offset(sr->op, sr->ctx->iv.offset);
+	sr->iv.len = sr->ctx->iv.length;
+}
+
+static int
+extract_cipher_auth_digest(struct nitrox_softreq *sr,
+			   struct nitrox_sglist *digest)
+{
+	struct rte_crypto_op *op = sr->op;
+	struct rte_mbuf *mdst = op->sym->m_dst ? op->sym->m_dst :
+					op->sym->m_src;
+
+	if (sr->ctx->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY &&
+	    unlikely(!op->sym->auth.digest.data))
+		return -EINVAL;
+
+	digest->len = sr->ctx->digest_length;
+	if (op->sym->auth.digest.data) {
+		digest->iova = op->sym->auth.digest.phys_addr;
+		digest->virt = op->sym->auth.digest.data;
+		return 0;
+	}
+
+	if (unlikely(rte_pktmbuf_data_len(mdst) < op->sym->auth.data.offset +
+	       op->sym->auth.data.length + digest->len))
+		return -EINVAL;
+
+	digest->iova = rte_pktmbuf_mtophys_offset(mdst,
+					op->sym->auth.data.offset +
+					op->sym->auth.data.length);
+	digest->virt = rte_pktmbuf_mtod_offset(mdst, uint8_t *,
+					op->sym->auth.data.offset +
+					op->sym->auth.data.length);
+	return 0;
+}
+
+static void
+fill_sglist(struct nitrox_sgtable *sgtbl, uint16_t len, rte_iova_t iova,
+	    void *virt)
+{
+	struct nitrox_sglist *sglist = sgtbl->sglist;
+	uint8_t cnt = sgtbl->map_bufs_cnt;
+
+	if (unlikely(!len))
+		return;
+
+	sglist[cnt].len = len;
+	sglist[cnt].iova = iova;
+	sglist[cnt].virt = virt;
+	sgtbl->total_bytes += len;
+	cnt++;
+	sgtbl->map_bufs_cnt = cnt;
+}
+
+static int
+create_sglist_from_mbuf(struct nitrox_sgtable *sgtbl, struct rte_mbuf *mbuf,
+			uint32_t off, int datalen)
+{
+	struct nitrox_sglist *sglist = sgtbl->sglist;
+	uint8_t cnt = sgtbl->map_bufs_cnt;
+	struct rte_mbuf *m;
+	int mlen;
+
+	if (unlikely(datalen <= 0))
+		return 0;
+
+	for (m = mbuf; m && off > rte_pktmbuf_data_len(m); m = m->next)
+		off -= rte_pktmbuf_data_len(m);
+
+	if (unlikely(!m))
+		return -EIO;
+
+	mlen = rte_pktmbuf_data_len(m) - off;
+	if (datalen <= mlen)
+		mlen = datalen;
+	sglist[cnt].len = mlen;
+	sglist[cnt].iova = rte_pktmbuf_mtophys_offset(m, off);
+	sglist[cnt].virt = rte_pktmbuf_mtod_offset(m, uint8_t *, off);
+	sgtbl->total_bytes += mlen;
+	cnt++;
+	datalen -= mlen;
+	for (m = m->next; m && datalen; m = m->next) {
+		mlen = rte_pktmbuf_data_len(m) < datalen ?
+			rte_pktmbuf_data_len(m) : datalen;
+		sglist[cnt].len = mlen;
+		sglist[cnt].iova = rte_pktmbuf_mtophys(m);
+		sglist[cnt].virt = rte_pktmbuf_mtod(m, uint8_t *);
+		sgtbl->total_bytes += mlen;
+		cnt++;
+		datalen -= mlen;
+	}
+
+	RTE_VERIFY(cnt <= MAX_SGBUF_CNT);
+	sgtbl->map_bufs_cnt = cnt;
+	return 0;
+}
+
+static int
+create_cipher_auth_sglist(struct nitrox_softreq *sr,
+			  struct nitrox_sgtable *sgtbl, struct rte_mbuf *mbuf)
+{
+	struct rte_crypto_op *op = sr->op;
+	int auth_only_len;
+	int err;
+
+	fill_sglist(sgtbl, sr->iv.len, sr->iv.iova, sr->iv.virt);
+	auth_only_len = op->sym->auth.data.length - op->sym->cipher.data.length;
+	if (unlikely(auth_only_len < 0))
+		return -EINVAL;
+
+	err = create_sglist_from_mbuf(sgtbl, mbuf, op->sym->auth.data.offset,
+				      auth_only_len);
+	if (unlikely(err))
+		return err;
+
+	err = create_sglist_from_mbuf(sgtbl, mbuf, op->sym->cipher.data.offset,
+				      op->sym->cipher.data.length);
+	if (unlikely(err))
+		return err;
+
+	return 0;
+}
+
+static void
+create_sgcomp(struct nitrox_sgtable *sgtbl)
+{
+	int i, j, nr_sgcomp;
+	struct nitrox_sgcomp *sgcomp = sgtbl->sgcomp;
+	struct nitrox_sglist *sglist = sgtbl->sglist;
+
+	nr_sgcomp = RTE_ALIGN_MUL_CEIL(sgtbl->map_bufs_cnt, 4) / 4;
+	sgtbl->nr_sgcomp = nr_sgcomp;
+	for (i = 0; i < nr_sgcomp; i++, sgcomp++) {
+		for (j = 0; j < 4; j++, sglist++) {
+			sgcomp->len[j] = rte_cpu_to_be_16(sglist->len);
+			sgcomp->iova[j] = rte_cpu_to_be_64(sglist->iova);
+		}
+	}
+}
+
+static int
+create_cipher_auth_inbuf(struct nitrox_softreq *sr,
+			 struct nitrox_sglist *digest)
+{
+	int err;
+	struct nitrox_crypto_ctx *ctx = sr->ctx;
+
+	err = create_cipher_auth_sglist(sr, &sr->in, sr->op->sym->m_src);
+	if (unlikely(err))
+		return err;
+
+	if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
+		fill_sglist(&sr->in, digest->len, digest->iova, digest->virt);
+
+	create_sgcomp(&sr->in);
+	sr->dptr = sr->iova + offsetof(struct nitrox_softreq, in.sgcomp);
+	return 0;
+}
+
+static int
+create_cipher_auth_oop_outbuf(struct nitrox_softreq *sr,
+			      struct nitrox_sglist *digest)
+{
+	int err;
+	struct nitrox_crypto_ctx *ctx = sr->ctx;
+
+	err = create_cipher_auth_sglist(sr, &sr->out, sr->op->sym->m_dst);
+	if (unlikely(err))
+		return err;
+
+	if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE)
+		fill_sglist(&sr->out, digest->len, digest->iova, digest->virt);
+
+	return 0;
+}
+
+static void
+create_cipher_auth_inplace_outbuf(struct nitrox_softreq *sr,
+				  struct nitrox_sglist *digest)
+{
+	int i, cnt;
+	struct nitrox_crypto_ctx *ctx = sr->ctx;
+
+	cnt = sr->out.map_bufs_cnt;
+	for (i = 0; i < sr->in.map_bufs_cnt; i++, cnt++) {
+		sr->out.sglist[cnt].len = sr->in.sglist[i].len;
+		sr->out.sglist[cnt].iova = sr->in.sglist[i].iova;
+		sr->out.sglist[cnt].virt = sr->in.sglist[i].virt;
+	}
+
+	sr->out.map_bufs_cnt = cnt;
+	if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE) {
+		fill_sglist(&sr->out, digest->len, digest->iova,
+			    digest->virt);
+	} else if (ctx->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
+		sr->out.map_bufs_cnt--;
+	}
+}
+
+static int
+create_cipher_auth_outbuf(struct nitrox_softreq *sr,
+			  struct nitrox_sglist *digest)
+{
+	struct rte_crypto_op *op = sr->op;
+	int cnt = 0;
+
+	sr->resp.orh = PENDING_SIG;
+	sr->out.sglist[cnt].len = sizeof(sr->resp.orh);
+	sr->out.sglist[cnt].iova = sr->iova + offsetof(struct nitrox_softreq,
+						       resp.orh);
+	sr->out.sglist[cnt].virt = &sr->resp.orh;
+	cnt++;
+	sr->out.map_bufs_cnt = cnt;
+	if (op->sym->m_dst) {
+		int err;
+
+		err = create_cipher_auth_oop_outbuf(sr, digest);
+		if (unlikely(err))
+			return err;
+	} else {
+		create_cipher_auth_inplace_outbuf(sr, digest);
+	}
+
+	cnt = sr->out.map_bufs_cnt;
+	sr->resp.completion = PENDING_SIG;
+	sr->out.sglist[cnt].len = sizeof(sr->resp.completion);
+	sr->out.sglist[cnt].iova = sr->iova + offsetof(struct nitrox_softreq,
+						     resp.completion);
+	sr->out.sglist[cnt].virt = &sr->resp.completion;
+	cnt++;
+	RTE_VERIFY(cnt <= MAX_SGBUF_CNT);
+	sr->out.map_bufs_cnt = cnt;
+
+	create_sgcomp(&sr->out);
+	sr->rptr = sr->iova + offsetof(struct nitrox_softreq, out.sgcomp);
+	return 0;
+}
+
+static void
+create_aead_gph(uint32_t cryptlen, uint16_t ivlen, uint32_t authlen,
+		struct gphdr *gph)
+{
+	int auth_only_len;
+	union {
+		struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+			uint16_t iv_offset : 8;
+			uint16_t auth_offset	: 8;
+#else
+			uint16_t auth_offset	: 8;
+			uint16_t iv_offset : 8;
+#endif
+		};
+		uint16_t value;
+	} param3;
+
+	gph->param0 = rte_cpu_to_be_16(cryptlen);
+	gph->param1 = rte_cpu_to_be_16(authlen);
+
+	auth_only_len = authlen - cryptlen;
+	gph->param2 = rte_cpu_to_be_16(ivlen + auth_only_len);
+
+	param3.iv_offset = 0;
+	param3.auth_offset = ivlen;
+	gph->param3 = rte_cpu_to_be_16(param3.value);
+}
+
+static int
+process_cipher_auth_data(struct nitrox_softreq *sr)
+{
+	struct rte_crypto_op *op = sr->op;
+	int err;
+	struct nitrox_sglist digest;
+
+	softreq_copy_iv(sr);
+	err = extract_cipher_auth_digest(sr, &digest);
+	if (unlikely(err))
+		return err;
+
+	err = create_cipher_auth_inbuf(sr, &digest);
+	if (unlikely(err))
+		return err;
+
+	err = create_cipher_auth_outbuf(sr, &digest);
+	if (unlikely(err))
+		return err;
+
+	create_aead_gph(op->sym->cipher.data.length, sr->iv.len,
+			op->sym->auth.data.length, &sr->gph);
+	return 0;
+}
+
+static int
+process_softreq(struct nitrox_softreq *sr)
+{
+	struct nitrox_crypto_ctx *ctx = sr->ctx;
+	int err = 0;
+
+	switch (ctx->nitrox_chain) {
+	case NITROX_CHAIN_CIPHER_AUTH:
+	case NITROX_CHAIN_AUTH_CIPHER:
+		err = process_cipher_auth_data(sr);
+		break;
+	default:
+		err = -EINVAL;
+		break;
+	}
+
+	return err;
+}
+
+int
+nitrox_process_se_req(uint16_t qno, struct rte_crypto_op *op,
+		      struct nitrox_crypto_ctx *ctx,
+		      struct nitrox_softreq *sr)
+{
+	softreq_init(sr, sr->iova);
+	sr->ctx = ctx;
+	sr->op = op;
+	process_softreq(sr);
+	create_se_instr(sr, qno);
+	sr->timeout = rte_get_timer_cycles() + CMD_TIMEOUT * rte_get_timer_hz();
+	return 0;
+}
+
+int
+nitrox_check_se_req(struct nitrox_softreq *sr, struct rte_crypto_op **op)
+{
+	uint64_t cc;
+	uint64_t orh;
+	int err;
+
+	cc = *(volatile uint64_t *)(&sr->resp.completion);
+	orh = *(volatile uint64_t *)(&sr->resp.orh);
+	if (cc != PENDING_SIG)
+		err = 0;
+	else if ((orh != PENDING_SIG) && (orh & 0xff))
+		err = orh & 0xff;
+	else if (rte_get_timer_cycles() >= sr->timeout)
+		err = 0xff;
+	else
+		return -EAGAIN;
+
+	if (unlikely(err))
+		NITROX_LOG(ERR, "Request err 0x%x, orh 0x%"PRIx64"\n", err,
+			   sr->resp.orh);
+
+	*op = sr->op;
+	return err;
+}
+
+void *
+nitrox_sym_instr_addr(struct nitrox_softreq *sr)
+{
+	return &sr->instr;
+}
+
 static void
 req_pool_obj_init(__rte_unused struct rte_mempool *mp,
 		  __rte_unused void *opaque, void *obj,