1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Cavium, Inc
8 #include <rte_branch_prediction.h>
9 #include <rte_common.h>
10 #include <rte_errno.h>
11 #include <rte_memzone.h>
13 #include "otx_cryptodev_hw_access.h"
14 #include "otx_cryptodev_mbox.h"
16 #include "cpt_pmd_logs.h"
17 #include "cpt_hw_types.h"
21 * Access its own BAR0/4 registers by passing VF number as 0.
22 * OS/PCI maps them accordingly.
26 otx_cpt_vf_init(struct cpt_vf *cptvf)
30 /* Check ready with PF */
31 /* Gets chip ID / device Id from PF if ready */
32 ret = otx_cpt_check_pf_ready(cptvf);
34 CPT_LOG_ERR("%s: PF not responding to READY msg",
40 CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__);
47 * Read Interrupt status of the VF
49 * @param cptvf cptvf structure
52 otx_cpt_read_vf_misc_intr_status(struct cpt_vf *cptvf)
54 return CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf), CPTX_VQX_MISC_INT(0, 0));
58 * Clear mailbox interrupt of the VF
60 * @param cptvf cptvf structure
63 otx_cpt_clear_mbox_intr(struct cpt_vf *cptvf)
65 cptx_vqx_misc_int_t vqx_misc_int;
67 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
68 CPTX_VQX_MISC_INT(0, 0));
70 vqx_misc_int.s.mbox = 1;
71 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
72 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
76 * Clear instruction NCB read error interrupt of the VF
78 * @param cptvf cptvf structure
81 otx_cpt_clear_irde_intr(struct cpt_vf *cptvf)
83 cptx_vqx_misc_int_t vqx_misc_int;
85 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
86 CPTX_VQX_MISC_INT(0, 0));
88 vqx_misc_int.s.irde = 1;
89 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
90 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
94 * Clear NCB result write response error interrupt of the VF
96 * @param cptvf cptvf structure
99 otx_cpt_clear_nwrp_intr(struct cpt_vf *cptvf)
101 cptx_vqx_misc_int_t vqx_misc_int;
103 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
104 CPTX_VQX_MISC_INT(0, 0));
106 vqx_misc_int.s.nwrp = 1;
107 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
108 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
112 * Clear swerr interrupt of the VF
114 * @param cptvf cptvf structure
117 otx_cpt_clear_swerr_intr(struct cpt_vf *cptvf)
119 cptx_vqx_misc_int_t vqx_misc_int;
121 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
122 CPTX_VQX_MISC_INT(0, 0));
124 vqx_misc_int.s.swerr = 1;
125 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
126 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
130 * Clear hwerr interrupt of the VF
132 * @param cptvf cptvf structure
135 otx_cpt_clear_hwerr_intr(struct cpt_vf *cptvf)
137 cptx_vqx_misc_int_t vqx_misc_int;
139 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
140 CPTX_VQX_MISC_INT(0, 0));
142 vqx_misc_int.s.hwerr = 1;
143 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
144 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
148 * Clear translation fault interrupt of the VF
150 * @param cptvf cptvf structure
153 otx_cpt_clear_fault_intr(struct cpt_vf *cptvf)
155 cptx_vqx_misc_int_t vqx_misc_int;
157 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
158 CPTX_VQX_MISC_INT(0, 0));
160 vqx_misc_int.s.fault = 1;
161 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
162 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
166 * Clear doorbell overflow interrupt of the VF
168 * @param cptvf cptvf structure
171 otx_cpt_clear_dovf_intr(struct cpt_vf *cptvf)
173 cptx_vqx_misc_int_t vqx_misc_int;
175 vqx_misc_int.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
176 CPTX_VQX_MISC_INT(0, 0));
178 vqx_misc_int.s.dovf = 1;
179 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
180 CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
183 /* Write to VQX_CTL register
186 otx_cpt_write_vq_ctl(struct cpt_vf *cptvf, bool val)
188 cptx_vqx_ctl_t vqx_ctl;
190 vqx_ctl.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
193 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
194 CPTX_VQX_CTL(0, 0), vqx_ctl.u);
197 /* Write to VQX_INPROG register
200 otx_cpt_write_vq_inprog(struct cpt_vf *cptvf, uint8_t val)
202 cptx_vqx_inprog_t vqx_inprg;
204 vqx_inprg.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
205 CPTX_VQX_INPROG(0, 0));
206 vqx_inprg.s.inflight = val;
207 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
208 CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
211 /* Write to VQX_DONE_WAIT NUMWAIT register
214 otx_cpt_write_vq_done_numwait(struct cpt_vf *cptvf, uint32_t val)
216 cptx_vqx_done_wait_t vqx_dwait;
218 vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
219 CPTX_VQX_DONE_WAIT(0, 0));
220 vqx_dwait.s.num_wait = val;
221 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
222 CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u);
225 /* Write to VQX_DONE_WAIT NUM_WAIT register
228 otx_cpt_write_vq_done_timewait(struct cpt_vf *cptvf, uint16_t val)
230 cptx_vqx_done_wait_t vqx_dwait;
232 vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
233 CPTX_VQX_DONE_WAIT(0, 0));
234 vqx_dwait.s.time_wait = val;
235 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
236 CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u);
239 /* Write to VQX_SADDR register
242 otx_cpt_write_vq_saddr(struct cpt_vf *cptvf, uint64_t val)
244 cptx_vqx_saddr_t vqx_saddr;
247 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
248 CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
252 otx_cpt_vfvq_init(struct cpt_vf *cptvf)
254 uint64_t base_addr = 0;
257 otx_cpt_write_vq_ctl(cptvf, 0);
259 /* Reset the doorbell */
260 otx_cpt_write_vq_doorbell(cptvf, 0);
262 otx_cpt_write_vq_inprog(cptvf, 0);
265 base_addr = (uint64_t)(cptvf->cqueue.chead[0].dma_addr);
266 otx_cpt_write_vq_saddr(cptvf, base_addr);
268 /* Configure timerhold / coalescence */
269 otx_cpt_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
270 otx_cpt_write_vq_done_numwait(cptvf, CPT_COUNT_THOLD);
273 otx_cpt_write_vq_ctl(cptvf, 1);
277 cpt_vq_init(struct cpt_vf *cptvf, uint8_t group)
281 /* Convey VQ LEN to PF */
282 err = otx_cpt_send_vq_size_msg(cptvf);
284 CPT_LOG_ERR("%s: PF not responding to QLEN msg",
290 /* CPT VF device initialization */
291 otx_cpt_vfvq_init(cptvf);
293 /* Send msg to PF to assign currnet Q to required group */
294 cptvf->vfgrp = group;
295 err = otx_cpt_send_vf_grp_msg(cptvf, group);
297 CPT_LOG_ERR("%s: PF not responding to VF_GRP msg",
303 CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__);
311 otx_cpt_poll_misc(struct cpt_vf *cptvf)
315 intr = otx_cpt_read_vf_misc_intr_status(cptvf);
320 /* Check for MISC interrupt types */
321 if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
322 CPT_LOG_DP_DEBUG("%s: Mailbox interrupt 0x%lx on CPT VF %d",
323 cptvf->dev_name, (unsigned int long)intr, cptvf->vfid);
324 otx_cpt_handle_mbox_intr(cptvf);
325 otx_cpt_clear_mbox_intr(cptvf);
326 } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
327 otx_cpt_clear_irde_intr(cptvf);
328 CPT_LOG_DP_DEBUG("%s: Instruction NCB read error interrupt "
329 "0x%lx on CPT VF %d", cptvf->dev_name,
330 (unsigned int long)intr, cptvf->vfid);
331 } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
332 otx_cpt_clear_nwrp_intr(cptvf);
333 CPT_LOG_DP_DEBUG("%s: NCB response write error interrupt 0x%lx"
334 " on CPT VF %d", cptvf->dev_name,
335 (unsigned int long)intr, cptvf->vfid);
336 } else if (unlikely(intr & CPT_VF_INTR_SWERR_MASK)) {
337 otx_cpt_clear_swerr_intr(cptvf);
338 CPT_LOG_DP_DEBUG("%s: Software error interrupt 0x%lx on CPT VF "
339 "%d", cptvf->dev_name, (unsigned int long)intr,
341 } else if (unlikely(intr & CPT_VF_INTR_HWERR_MASK)) {
342 otx_cpt_clear_hwerr_intr(cptvf);
343 CPT_LOG_DP_DEBUG("%s: Hardware error interrupt 0x%lx on CPT VF "
344 "%d", cptvf->dev_name, (unsigned int long)intr,
346 } else if (unlikely(intr & CPT_VF_INTR_FAULT_MASK)) {
347 otx_cpt_clear_fault_intr(cptvf);
348 CPT_LOG_DP_DEBUG("%s: Translation fault interrupt 0x%lx on CPT VF "
349 "%d", cptvf->dev_name, (unsigned int long)intr,
351 } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
352 otx_cpt_clear_dovf_intr(cptvf);
353 CPT_LOG_DP_DEBUG("%s: Doorbell overflow interrupt 0x%lx on CPT VF "
354 "%d", cptvf->dev_name, (unsigned int long)intr,
357 CPT_LOG_DP_ERR("%s: Unhandled interrupt 0x%lx in CPT VF %d",
358 cptvf->dev_name, (unsigned int long)intr,
363 otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name)
365 memset(cptvf, 0, sizeof(struct cpt_vf));
367 /* Bar0 base address */
368 cptvf->reg_base = reg_base;
369 strncpy(cptvf->dev_name, name, 32);
373 /* To clear if there are any pending mbox msgs */
374 otx_cpt_poll_misc(cptvf);
376 if (otx_cpt_vf_init(cptvf)) {
377 CPT_LOG_ERR("Failed to initialize CPT VF device");
385 otx_cpt_deinit_device(void *dev)
387 struct cpt_vf *cptvf = (struct cpt_vf *)dev;
389 /* Do misc work one last time */
390 otx_cpt_poll_misc(cptvf);
396 otx_cpt_get_resource(void *dev, uint8_t group, struct cpt_instance **instance)
398 int ret = -ENOENT, len, qlen, i;
399 int chunk_len, chunks, chunk_size;
400 struct cpt_vf *cptvf = (struct cpt_vf *)dev;
401 struct cpt_instance *cpt_instance;
402 struct command_chunk *chunk_head = NULL, *chunk_prev = NULL;
403 struct command_chunk *chunk = NULL;
405 const struct rte_memzone *rz;
406 uint64_t dma_addr = 0, alloc_len, used_len;
408 uint64_t pg_sz = sysconf(_SC_PAGESIZE);
410 CPT_LOG_DP_DEBUG("Initializing cpt resource %s", cptvf->dev_name);
412 cpt_instance = &cptvf->instance;
414 memset(&cptvf->cqueue, 0, sizeof(cptvf->cqueue));
415 memset(&cptvf->pqueue, 0, sizeof(cptvf->pqueue));
417 /* Chunks are of fixed size buffers */
418 chunks = DEFAULT_CMD_QCHUNKS;
419 chunk_len = DEFAULT_CMD_QCHUNK_SIZE;
421 qlen = chunks * chunk_len;
422 /* Chunk size includes 8 bytes of next chunk ptr */
423 chunk_size = chunk_len * CPT_INST_SIZE + CPT_NEXT_CHUNK_PTR_SIZE;
425 /* For command chunk structures */
426 len = chunks * RTE_ALIGN(sizeof(struct command_chunk), 8);
428 /* For pending queue */
429 len += qlen * RTE_ALIGN(sizeof(struct rid), 8);
431 /* So that instruction queues start as pg size aligned */
432 len = RTE_ALIGN(len, pg_sz);
434 /* For Instruction queues */
435 len += chunks * RTE_ALIGN(chunk_size, 128);
437 /* Wastage after instruction queues */
438 len = RTE_ALIGN(len, pg_sz);
440 rz = rte_memzone_reserve_aligned(cptvf->dev_name, len, cptvf->node,
441 RTE_MEMZONE_SIZE_HINT_ONLY |
443 RTE_CACHE_LINE_SIZE);
450 dma_addr = rz->phys_addr;
455 cpt_instance->rsvd = (uintptr_t)rz;
457 /* Pending queue setup */
458 cptvf->pqueue.rid_queue = (struct rid *)mem;
459 cptvf->pqueue.enq_tail = 0;
460 cptvf->pqueue.deq_head = 0;
461 cptvf->pqueue.pending_count = 0;
463 mem += qlen * RTE_ALIGN(sizeof(struct rid), 8);
464 len -= qlen * RTE_ALIGN(sizeof(struct rid), 8);
465 dma_addr += qlen * RTE_ALIGN(sizeof(struct rid), 8);
467 /* Alignment wastage */
468 used_len = alloc_len - len;
469 mem += RTE_ALIGN(used_len, pg_sz) - used_len;
470 len -= RTE_ALIGN(used_len, pg_sz) - used_len;
471 dma_addr += RTE_ALIGN(used_len, pg_sz) - used_len;
473 /* Init instruction queues */
474 chunk_head = &cptvf->cqueue.chead[0];
478 for (i = 0; i < DEFAULT_CMD_QCHUNKS; i++) {
481 chunk = &cptvf->cqueue.chead[i];
483 chunk->dma_addr = dma_addr;
485 csize = RTE_ALIGN(chunk_size, 128);
491 next_ptr = (uint64_t *)(chunk_prev->head +
493 *next_ptr = (uint64_t)chunk->dma_addr;
498 next_ptr = (uint64_t *)(chunk_prev->head + chunk_size - 8);
499 *next_ptr = (uint64_t)chunk_head->dma_addr;
503 /* This is used for CPT(0)_PF_Q(0..15)_CTL.size config */
504 cptvf->qsize = chunk_size / 8;
505 cptvf->cqueue.qhead = chunk_head->head;
506 cptvf->cqueue.idx = 0;
507 cptvf->cqueue.cchunk = 0;
509 if (cpt_vq_init(cptvf, group)) {
510 CPT_LOG_ERR("Failed to initialize CPT VQ of device %s",
516 *instance = cpt_instance;
518 CPT_LOG_DP_DEBUG("Crypto device (%s) initialized", cptvf->dev_name);
522 rte_memzone_free(rz);
528 otx_cpt_put_resource(struct cpt_instance *instance)
530 struct cpt_vf *cptvf = (struct cpt_vf *)instance;
531 struct rte_memzone *rz;
534 CPT_LOG_ERR("Invalid CPTVF handle");
538 CPT_LOG_DP_DEBUG("Releasing cpt device %s", cptvf->dev_name);
540 rz = (struct rte_memzone *)instance->rsvd;
541 rte_memzone_free(rz);
546 otx_cpt_start_device(void *dev)
549 struct cpt_vf *cptvf = (struct cpt_vf *)dev;
551 rc = otx_cpt_send_vf_up(cptvf);
553 CPT_LOG_ERR("Failed to mark CPT VF device %s UP, rc = %d",
554 cptvf->dev_name, rc);
558 if ((cptvf->vftype != SE_TYPE) && (cptvf->vftype != AE_TYPE)) {
559 CPT_LOG_ERR("Fatal error, unexpected vf type %u, for CPT VF "
560 "device %s", cptvf->vftype, cptvf->dev_name);
568 otx_cpt_stop_device(void *dev)
571 uint32_t pending, retries = 5;
572 struct cpt_vf *cptvf = (struct cpt_vf *)dev;
574 /* Wait for pending entries to complete */
575 pending = otx_cpt_read_vq_doorbell(cptvf);
577 CPT_LOG_DP_DEBUG("%s: Waiting for pending %u cmds to complete",
578 cptvf->dev_name, pending);
580 pending = otx_cpt_read_vq_doorbell(cptvf);
586 if (!retries && pending) {
587 CPT_LOG_ERR("%s: Timeout waiting for commands(%u)",
588 cptvf->dev_name, pending);
592 rc = otx_cpt_send_vf_down(cptvf);
594 CPT_LOG_ERR("Failed to bring down vf %s, rc %d",
595 cptvf->dev_name, rc);