From 89f1a8d6a057295037dd06e9601371887dc55379 Mon Sep 17 00:00:00 2001 From: Tejasree Kondoj Date: Tue, 9 Oct 2018 14:37:54 +0530 Subject: [PATCH] crypto/octeontx: add burst dequeue Signed-off-by: Ankur Dwivedi Signed-off-by: Anoob Joseph Signed-off-by: Murthy NSSR Signed-off-by: Nithin Dabilpuram Signed-off-by: Ragothaman Jayaraman Signed-off-by: Srisivasubramanian S Signed-off-by: Tejasree Kondoj --- drivers/common/cpt/cpt_common.h | 2 + drivers/common/cpt/cpt_mcode_defines.h | 15 ++++ drivers/common/cpt/cpt_request_mgr.h | 39 +++++++++++ drivers/common/cpt/cpt_ucode.h | 26 +++++++ .../crypto/octeontx/otx_cryptodev_hw_access.h | 69 +++++++++++++++++-- drivers/crypto/octeontx/otx_cryptodev_ops.c | 47 ++++++++++++- 6 files changed, 192 insertions(+), 6 deletions(-) diff --git a/drivers/common/cpt/cpt_common.h b/drivers/common/cpt/cpt_common.h index 1823939204..8461cd604a 100644 --- a/drivers/common/cpt/cpt_common.h +++ b/drivers/common/cpt/cpt_common.h @@ -15,6 +15,8 @@ */ #define CRYPTO_OCTEONTX 0x1 +#define TIME_IN_RESET_COUNT 5 + /* Default command timeout in seconds */ #define DEFAULT_COMMAND_TIMEOUT 4 diff --git a/drivers/common/cpt/cpt_mcode_defines.h b/drivers/common/cpt/cpt_mcode_defines.h index 60be8b3553..becc14f4ae 100644 --- a/drivers/common/cpt/cpt_mcode_defines.h +++ b/drivers/common/cpt/cpt_mcode_defines.h @@ -175,6 +175,21 @@ typedef enum { ERR_ENODEV, } mc_error_code_t; +/** + * Enumeration cpt_comp_e + * + * CPT Completion Enumeration + * Enumerates the values of CPT_RES_S[COMPCODE]. + */ +typedef enum { + CPT_8X_COMP_E_NOTDONE = (0x00), + CPT_8X_COMP_E_GOOD = (0x01), + CPT_8X_COMP_E_FAULT = (0x02), + CPT_8X_COMP_E_SWERR = (0x03), + CPT_8X_COMP_E_HWERR = (0x04), + CPT_8X_COMP_E_LAST_ENTRY = (0xFF) +} cpt_comp_e_t; + typedef struct sglist_comp { union { uint64_t len; diff --git a/drivers/common/cpt/cpt_request_mgr.h b/drivers/common/cpt/cpt_request_mgr.h index dea89f8157..4463cfbed4 100644 --- a/drivers/common/cpt/cpt_request_mgr.h +++ b/drivers/common/cpt/cpt_request_mgr.h @@ -143,4 +143,43 @@ req_fail: return ret; } +static __rte_always_inline int32_t __hot +cpt_dequeue_burst(struct cpt_instance *instance, uint16_t cnt, + void *resp[], uint8_t cc[], struct pending_queue *pqueue) +{ + struct cpt_request_info *user_req; + struct rid *rid_e; + int i, count, pcount; + uint8_t ret; + + pcount = pqueue->pending_count; + count = (cnt > pcount) ? pcount : cnt; + + for (i = 0; i < count; i++) { + rid_e = &pqueue->rid_queue[pqueue->deq_head]; + user_req = (struct cpt_request_info *)(rid_e->rid); + + if (likely((i+1) < count)) + rte_prefetch_non_temporal((void *)rid_e[1].rid); + + ret = check_nb_command_id(user_req, instance); + + if (unlikely(ret == ERR_REQ_PENDING)) { + /* Stop checking for completions */ + break; + } + + /* Return completion code and op handle */ + cc[i] = (uint8_t)ret; + resp[i] = user_req->op; + CPT_LOG_DP_DEBUG("Request %p Op %p completed with code %d", + user_req, user_req->op, ret); + + MOD_INC(pqueue->deq_head, DEFAULT_CMD_QLEN); + pqueue->pending_count -= 1; + } + + return i; +} + #endif /* _CPT_REQUEST_MGR_H_ */ diff --git a/drivers/common/cpt/cpt_ucode.h b/drivers/common/cpt/cpt_ucode.h index 4d892f2dd7..c5a9f34b4b 100644 --- a/drivers/common/cpt/cpt_ucode.h +++ b/drivers/common/cpt/cpt_ucode.h @@ -3373,6 +3373,32 @@ fill_fc_params(struct rte_crypto_op *cop, return prep_req; } +static __rte_always_inline void +compl_auth_verify(struct rte_crypto_op *op, + uint8_t *gen_mac, + uint64_t mac_len) +{ + uint8_t *mac; + struct rte_crypto_sym_op *sym_op = op->sym; + + if (sym_op->auth.digest.data) + mac = sym_op->auth.digest.data; + else + mac = rte_pktmbuf_mtod_offset(sym_op->m_src, + uint8_t *, + sym_op->auth.data.length + + sym_op->auth.data.offset); + if (!mac) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return; + } + + if (memcmp(mac, gen_mac, mac_len)) + op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + else + op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; +} + static __rte_always_inline int instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) { diff --git a/drivers/crypto/octeontx/otx_cryptodev_hw_access.h b/drivers/crypto/octeontx/otx_cryptodev_hw_access.h index 6f4d6e1d56..82b15eea80 100644 --- a/drivers/crypto/octeontx/otx_cryptodev_hw_access.h +++ b/drivers/crypto/octeontx/otx_cryptodev_hw_access.h @@ -1,19 +1,20 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2018 Cavium, Inc */ - #ifndef _OTX_CRYPTODEV_HW_ACCESS_H_ #define _OTX_CRYPTODEV_HW_ACCESS_H_ #include #include +#include #include #include #include #include "cpt_common.h" #include "cpt_hw_types.h" +#include "cpt_mcode_defines.h" #include "cpt_pmd_logs.h" #define CPT_INTR_POLL_INTERVAL_MS (50) @@ -252,10 +253,68 @@ static __rte_always_inline uint8_t check_nb_command_id(struct cpt_request_info *user_req, struct cpt_instance *instance) { - /* Required for dequeue operation. Adding a dummy routine for now */ - RTE_SET_USED(user_req); - RTE_SET_USED(instance); - return 0; + uint8_t ret = ERR_REQ_PENDING; + struct cpt_vf *cptvf = (struct cpt_vf *)instance; + volatile cpt_res_s_t *cptres; + + cptres = (volatile cpt_res_s_t *)user_req->completion_addr; + + if (unlikely(cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE)) { + /* + * Wait for some time for this command to get completed + * before timing out + */ + if (rte_get_timer_cycles() < user_req->time_out) + return ret; + /* + * TODO: See if alternate caddr can be used to not loop + * longer than needed. + */ + if ((cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE) && + (user_req->extra_time < TIME_IN_RESET_COUNT)) { + user_req->extra_time++; + return ret; + } + + if (cptres->s8x.compcode != CPT_8X_COMP_E_NOTDONE) + goto complete; + + ret = ERR_REQ_TIMEOUT; + CPT_LOG_DP_ERR("Request %p timedout", user_req); + otx_cpt_poll_misc(cptvf); + goto exit; + } + +complete: + if (likely(cptres->s8x.compcode == CPT_8X_COMP_E_GOOD)) { + ret = 0; /* success */ + if (unlikely((uint8_t)*user_req->alternate_caddr)) { + ret = (uint8_t)*user_req->alternate_caddr; + CPT_LOG_DP_ERR("Request %p : failed with microcode" + " error, MC completion code : 0x%x", user_req, + ret); + } + CPT_LOG_DP_DEBUG("MC status %.8x\n", + *((volatile uint32_t *)user_req->alternate_caddr)); + CPT_LOG_DP_DEBUG("HW status %.8x\n", + *((volatile uint32_t *)user_req->completion_addr)); + } else if ((cptres->s8x.compcode == CPT_8X_COMP_E_SWERR) || + (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT)) { + ret = (uint8_t)*user_req->alternate_caddr; + if (!ret) + ret = ERR_BAD_ALT_CCODE; + CPT_LOG_DP_DEBUG("Request %p : failed with %s : err code :%x", + user_req, + (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT) ? + "DMA Fault" : "Software error", ret); + } else { + CPT_LOG_DP_ERR("Request %p : unexpected completion code %d", + user_req, cptres->s8x.compcode); + ret = (uint8_t)*user_req->alternate_caddr; + } + +exit: + return ret; } #endif /* _OTX_CRYPTODEV_HW_ACCESS_H_ */ diff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.c b/drivers/crypto/octeontx/otx_cryptodev_ops.c index 42400513c8..23f9659138 100644 --- a/drivers/crypto/octeontx/otx_cryptodev_ops.c +++ b/drivers/crypto/octeontx/otx_cryptodev_ops.c @@ -366,6 +366,51 @@ otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops) return count; } +static uint16_t +otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct cpt_instance *instance = (struct cpt_instance *)qptr; + struct cpt_vf *cptvf = (struct cpt_vf *)instance; + struct pending_queue *pqueue = &cptvf->pqueue; + uint16_t nb_completed, i = 0; + uint8_t compcode[nb_ops]; + + nb_completed = cpt_dequeue_burst(instance, nb_ops, + (void **)ops, compcode, pqueue); + while (likely(i < nb_completed)) { + struct rte_crypto_op *cop; + void *metabuf; + uintptr_t *rsp; + uint8_t status; + + rsp = (void *)ops[i]; + status = compcode[i]; + if (likely((i + 1) < nb_completed)) + rte_prefetch0(ops[i+1]); + metabuf = (void *)rsp[0]; + cop = (void *)rsp[1]; + + ops[i] = cop; + + if (likely(status == 0)) { + if (likely(!rsp[2])) + cop->status = + RTE_CRYPTO_OP_STATUS_SUCCESS; + else + compl_auth_verify(cop, (uint8_t *)rsp[2], + rsp[3]); + } else if (status == ERR_GC_ICV_MISCOMPARE) { + /*auth data mismatch */ + cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + } else { + cop->status = RTE_CRYPTO_OP_STATUS_ERROR; + } + free_op_meta(metabuf, cptvf->meta_info.cptvf_meta_pool); + i++; + } + return nb_completed; +} + static struct rte_cryptodev_ops cptvf_ops = { /* Device related operations */ .dev_configure = otx_cpt_dev_config, @@ -458,7 +503,7 @@ otx_cpt_dev_create(struct rte_cryptodev *c_dev) c_dev->dev_ops = &cptvf_ops; c_dev->enqueue_burst = otx_cpt_pkt_enqueue; - c_dev->dequeue_burst = NULL; + c_dev->dequeue_burst = otx_cpt_pkt_dequeue; c_dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | -- 2.20.1