1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Cavium, Inc
4 #ifndef _OTX_CRYPTODEV_HW_ACCESS_H_
5 #define _OTX_CRYPTODEV_HW_ACCESS_H_
9 #include <rte_branch_prediction.h>
10 #include <cryptodev_pmd.h>
11 #include <rte_cycles.h>
13 #include <rte_memory.h>
14 #include <rte_prefetch.h>
16 #include "otx_cryptodev.h"
18 #include "cpt_common.h"
19 #include "cpt_hw_types.h"
20 #include "cpt_mcode_defines.h"
21 #include "cpt_pmd_logs.h"
23 #define CPT_INTR_POLL_INTERVAL_MS (50)
25 /* Default command queue length */
26 #define DEFAULT_CMD_QLEN 2048
27 #define DEFAULT_CMD_QCHUNKS 2
29 /* Instruction memory benefits from being 1023, so introduce
30 * reserved entries so we can't overrun the instruction queue
32 #define DEFAULT_CMD_QRSVD_SLOTS DEFAULT_CMD_QCHUNKS
33 #define DEFAULT_CMD_QCHUNK_SIZE \
34 ((DEFAULT_CMD_QLEN - DEFAULT_CMD_QRSVD_SLOTS) / \
37 #define CPT_CSR_REG_BASE(cpt) ((cpt)->reg_base)
39 /* Read hw register */
40 #define CPT_READ_CSR(__hw_addr, __offset) \
41 rte_read64_relaxed((uint8_t *)__hw_addr + __offset)
43 /* Write hw register */
44 #define CPT_WRITE_CSR(__hw_addr, __offset, __val) \
45 rte_write64_relaxed((__val), ((uint8_t *)__hw_addr + __offset))
51 struct rte_mempool *sess_mp;
52 struct rte_mempool *sess_mp_priv;
53 struct cpt_qp_meta_info meta_info;
57 struct command_chunk {
58 /** 128-byte aligned real_vaddr */
60 /** 128-byte aligned real_dma_addr */
65 * Command queue structure
67 struct command_queue {
68 /** Command queue host write idx */
70 /** Command queue chunk */
72 /** Command queue head; instructions are inserted here */
74 /** Command chunk list head */
75 struct command_chunk chead[DEFAULT_CMD_QCHUNKS];
79 * CPT VF device structure
83 struct cpt_instance instance;
84 /** Register start address */
86 /** Command queue information */
87 struct command_queue cqueue;
88 /** Pending queue information */
89 struct pending_queue pqueue;
91 /** Below fields are accessed only in control path */
93 /** Env specific pdev representing the pci dev */
95 /** Calculated queue size */
97 /** Device index (0...CPT_MAX_VQ_NUM)*/
99 /** VF type of cpt_vf_type_t (SE_TYPE(2) or AE_TYPE(1) */
101 /** VF group (0 - 8) */
103 /** Operating node: Bits (46:44) in BAR0 address */
106 /** VF-PF mailbox communication */
110 /** Flag if not acked */
115 } __rte_cache_aligned;
118 * CPT Registers map for 81xx
122 #define CPTX_VQX_CTL(a, b) (0x0000100ll + 0x1000000000ll * \
123 ((a) & 0x0) + 0x100000ll * (b))
124 #define CPTX_VQX_SADDR(a, b) (0x0000200ll + 0x1000000000ll * \
125 ((a) & 0x0) + 0x100000ll * (b))
126 #define CPTX_VQX_DONE_WAIT(a, b) (0x0000400ll + 0x1000000000ll * \
127 ((a) & 0x0) + 0x100000ll * (b))
128 #define CPTX_VQX_INPROG(a, b) (0x0000410ll + 0x1000000000ll * \
129 ((a) & 0x0) + 0x100000ll * (b))
130 #define CPTX_VQX_DONE(a, b) (0x0000420ll + 0x1000000000ll * \
131 ((a) & 0x1) + 0x100000ll * (b))
132 #define CPTX_VQX_DONE_ACK(a, b) (0x0000440ll + 0x1000000000ll * \
133 ((a) & 0x1) + 0x100000ll * (b))
134 #define CPTX_VQX_DONE_INT_W1S(a, b) (0x0000460ll + 0x1000000000ll * \
135 ((a) & 0x1) + 0x100000ll * (b))
136 #define CPTX_VQX_DONE_INT_W1C(a, b) (0x0000468ll + 0x1000000000ll * \
137 ((a) & 0x1) + 0x100000ll * (b))
138 #define CPTX_VQX_DONE_ENA_W1S(a, b) (0x0000470ll + 0x1000000000ll * \
139 ((a) & 0x1) + 0x100000ll * (b))
140 #define CPTX_VQX_DONE_ENA_W1C(a, b) (0x0000478ll + 0x1000000000ll * \
141 ((a) & 0x1) + 0x100000ll * (b))
142 #define CPTX_VQX_MISC_INT(a, b) (0x0000500ll + 0x1000000000ll * \
143 ((a) & 0x1) + 0x100000ll * (b))
144 #define CPTX_VQX_MISC_INT_W1S(a, b) (0x0000508ll + 0x1000000000ll * \
145 ((a) & 0x1) + 0x100000ll * (b))
146 #define CPTX_VQX_MISC_ENA_W1S(a, b) (0x0000510ll + 0x1000000000ll * \
147 ((a) & 0x1) + 0x100000ll * (b))
148 #define CPTX_VQX_MISC_ENA_W1C(a, b) (0x0000518ll + 0x1000000000ll * \
149 ((a) & 0x1) + 0x100000ll * (b))
150 #define CPTX_VQX_DOORBELL(a, b) (0x0000600ll + 0x1000000000ll * \
151 ((a) & 0x1) + 0x100000ll * (b))
152 #define CPTX_VFX_PF_MBOXX(a, b, c) (0x0001000ll + 0x1000000000ll * \
153 ((a) & 0x1) + 0x100000ll * (b) + \
156 /* VF HAL functions */
159 otx_cpt_poll_misc(struct cpt_vf *cptvf);
162 otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name);
165 otx_cpt_deinit_device(void *dev);
168 otx_cpt_get_resource(const struct rte_cryptodev *dev, uint8_t group,
169 struct cpt_instance **instance, uint16_t qp_id);
172 otx_cpt_put_resource(struct cpt_instance *instance);
175 otx_cpt_start_device(void *cptvf);
178 otx_cpt_stop_device(void *cptvf);
180 /* Write to VQX_DOORBELL register
182 static __rte_always_inline void
183 otx_cpt_write_vq_doorbell(struct cpt_vf *cptvf, uint32_t val)
185 cptx_vqx_doorbell_t vqx_dbell;
188 vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
189 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
190 CPTX_VQX_DOORBELL(0, 0), vqx_dbell.u);
193 static __rte_always_inline uint32_t
194 otx_cpt_read_vq_doorbell(struct cpt_vf *cptvf)
196 cptx_vqx_doorbell_t vqx_dbell;
198 vqx_dbell.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
199 CPTX_VQX_DOORBELL(0, 0));
200 return vqx_dbell.s.dbell_cnt;
203 static __rte_always_inline void
204 otx_cpt_ring_dbell(struct cpt_instance *instance, uint16_t count)
206 struct cpt_vf *cptvf = (struct cpt_vf *)instance;
207 /* Memory barrier to flush pending writes */
209 otx_cpt_write_vq_doorbell(cptvf, count);
212 static __rte_always_inline void *
213 get_cpt_inst(struct command_queue *cqueue)
215 CPT_LOG_DP_DEBUG("CPT queue idx %u\n", cqueue->idx);
216 return &cqueue->qhead[cqueue->idx * CPT_INST_SIZE];
219 static __rte_always_inline void
220 fill_cpt_inst(struct cpt_instance *instance, void *req, uint64_t ucmd_w3)
222 struct command_queue *cqueue;
223 cpt_inst_s_t *cpt_ist_p;
224 struct cpt_vf *cptvf = (struct cpt_vf *)instance;
225 struct cpt_request_info *user_req = (struct cpt_request_info *)req;
226 cqueue = &cptvf->cqueue;
227 cpt_ist_p = get_cpt_inst(cqueue);
228 rte_prefetch_non_temporal(cpt_ist_p);
230 /* EI0, EI1, EI2, EI3 are already prepared */
234 cpt_ist_p->s8x.res_addr = user_req->comp_baddr;
238 cpt_ist_p->s8x.wq_ptr = 0;
241 cpt_ist_p->s8x.ei0 = user_req->ist.ei0;
243 cpt_ist_p->s8x.ei1 = user_req->ist.ei1;
245 cpt_ist_p->s8x.ei2 = user_req->ist.ei2;
247 cpt_ist_p->s8x.ei3 = ucmd_w3;
250 static __rte_always_inline void
251 mark_cpt_inst(struct cpt_instance *instance)
253 struct cpt_vf *cptvf = (struct cpt_vf *)instance;
254 struct command_queue *queue = &cptvf->cqueue;
255 if (unlikely(++queue->idx >= DEFAULT_CMD_QCHUNK_SIZE)) {
256 uint32_t cchunk = queue->cchunk;
257 MOD_INC(cchunk, DEFAULT_CMD_QCHUNKS);
258 queue->qhead = queue->chead[cchunk].head;
260 queue->cchunk = cchunk;
264 static __rte_always_inline uint8_t
265 check_nb_command_id(struct cpt_request_info *user_req,
266 struct cpt_instance *instance)
268 uint8_t ret = ERR_REQ_PENDING;
269 struct cpt_vf *cptvf = (struct cpt_vf *)instance;
270 volatile cpt_res_s_t *cptres;
272 cptres = (volatile cpt_res_s_t *)user_req->completion_addr;
274 if (unlikely(cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE)) {
276 * Wait for some time for this command to get completed
279 if (rte_get_timer_cycles() < user_req->time_out)
282 * TODO: See if alternate caddr can be used to not loop
283 * longer than needed.
285 if ((cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE) &&
286 (user_req->extra_time < TIME_IN_RESET_COUNT)) {
287 user_req->extra_time++;
291 if (cptres->s8x.compcode != CPT_8X_COMP_E_NOTDONE)
294 ret = ERR_REQ_TIMEOUT;
295 CPT_LOG_DP_ERR("Request %p timedout", user_req);
296 otx_cpt_poll_misc(cptvf);
301 if (likely(cptres->s8x.compcode == CPT_8X_COMP_E_GOOD)) {
302 ret = 0; /* success */
303 if (unlikely((uint8_t)*user_req->alternate_caddr)) {
304 ret = (uint8_t)*user_req->alternate_caddr;
305 CPT_LOG_DP_ERR("Request %p : failed with microcode"
306 " error, MC completion code : 0x%x", user_req,
309 CPT_LOG_DP_DEBUG("MC status %.8x\n",
310 *((volatile uint32_t *)user_req->alternate_caddr));
311 CPT_LOG_DP_DEBUG("HW status %.8x\n",
312 *((volatile uint32_t *)user_req->completion_addr));
313 } else if ((cptres->s8x.compcode == CPT_8X_COMP_E_SWERR) ||
314 (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT)) {
315 ret = (uint8_t)*user_req->alternate_caddr;
317 ret = ERR_BAD_ALT_CCODE;
318 CPT_LOG_DP_DEBUG("Request %p : failed with %s : err code :%x",
320 (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT) ?
321 "DMA Fault" : "Software error", ret);
323 CPT_LOG_DP_ERR("Request %p : unexpected completion code %d",
324 user_req, cptres->s8x.compcode);
325 ret = (uint8_t)*user_req->alternate_caddr;
332 #endif /* _OTX_CRYPTODEV_HW_ACCESS_H_ */