1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
8 #define CPT_IQ_FC_LEN 128
9 #define CPT_IQ_GRP_LEN 16
11 #define CPT_IQ_NB_DESC_MULTIPLIER 40
13 /* The effective queue size to software is (CPT_LF_Q_SIZE[SIZE_DIV40] - 1 - 8).
15 * CPT requires 320 free entries (+8). And 40 entries are required for
16 * allowing CPT to discard packet when the queues are full (+1).
18 #define CPT_IQ_NB_DESC_SIZE_DIV40(nb_desc) \
19 (PLT_DIV_CEIL(nb_desc, CPT_IQ_NB_DESC_MULTIPLIER) + 1 + 8)
21 #define CPT_IQ_GRP_SIZE(nb_desc) \
22 (CPT_IQ_NB_DESC_SIZE_DIV40(nb_desc) * CPT_IQ_GRP_LEN)
24 #define CPT_LF_MAX_NB_DESC 128000
25 #define CPT_LF_DEFAULT_NB_DESC 1024
28 cpt_lf_misc_intr_enb_dis(struct roc_cpt_lf *lf, bool enb)
30 /* Enable all cpt lf error irqs except RQ_DISABLED and CQ_DISABLED */
32 plt_write64((BIT_ULL(6) | BIT_ULL(5) | BIT_ULL(3) | BIT_ULL(2) |
34 lf->rbase + CPT_LF_MISC_INT_ENA_W1S);
36 plt_write64((BIT_ULL(6) | BIT_ULL(5) | BIT_ULL(3) | BIT_ULL(2) |
38 lf->rbase + CPT_LF_MISC_INT_ENA_W1C);
42 cpt_lf_misc_irq(void *param)
44 struct roc_cpt_lf *lf = (struct roc_cpt_lf *)param;
45 struct dev *dev = lf->dev;
48 intr = plt_read64(lf->rbase + CPT_LF_MISC_INT);
52 plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
55 plt_write64(intr, lf->rbase + CPT_LF_MISC_INT);
59 cpt_lf_register_misc_irq(struct roc_cpt_lf *lf)
61 struct plt_pci_device *pci_dev = lf->pci_dev;
62 struct plt_intr_handle *handle;
65 handle = &pci_dev->intr_handle;
67 vec = lf->msixoff + CPT_LF_INT_VEC_MISC;
68 /* Clear err interrupt */
69 cpt_lf_misc_intr_enb_dis(lf, false);
70 /* Set used interrupt vectors */
71 rc = dev_irq_register(handle, cpt_lf_misc_irq, lf, vec);
72 /* Enable all dev interrupt except for RQ_DISABLED */
73 cpt_lf_misc_intr_enb_dis(lf, true);
79 cpt_lf_unregister_misc_irq(struct roc_cpt_lf *lf)
81 struct plt_pci_device *pci_dev = lf->pci_dev;
82 struct plt_intr_handle *handle;
85 handle = &pci_dev->intr_handle;
87 vec = lf->msixoff + CPT_LF_INT_VEC_MISC;
88 /* Clear err interrupt */
89 cpt_lf_misc_intr_enb_dis(lf, false);
90 dev_irq_unregister(handle, cpt_lf_misc_irq, lf, vec);
94 cpt_lf_done_intr_enb_dis(struct roc_cpt_lf *lf, bool enb)
97 plt_write64(0x1, lf->rbase + CPT_LF_DONE_INT_ENA_W1S);
99 plt_write64(0x1, lf->rbase + CPT_LF_DONE_INT_ENA_W1C);
103 cpt_lf_done_irq(void *param)
105 struct roc_cpt_lf *lf = param;
109 /* Read the number of completed requests */
110 intr = plt_read64(lf->rbase + CPT_LF_DONE);
114 done_wait = plt_read64(lf->rbase + CPT_LF_DONE_WAIT);
116 /* Acknowledge the number of completed requests */
117 plt_write64(intr, lf->rbase + CPT_LF_DONE_ACK);
119 plt_write64(done_wait, lf->rbase + CPT_LF_DONE_WAIT);
123 cpt_lf_register_done_irq(struct roc_cpt_lf *lf)
125 struct plt_pci_device *pci_dev = lf->pci_dev;
126 struct plt_intr_handle *handle;
129 handle = &pci_dev->intr_handle;
131 vec = lf->msixoff + CPT_LF_INT_VEC_DONE;
133 /* Clear done interrupt */
134 cpt_lf_done_intr_enb_dis(lf, false);
136 /* Set used interrupt vectors */
137 rc = dev_irq_register(handle, cpt_lf_done_irq, lf, vec);
139 /* Enable done interrupt */
140 cpt_lf_done_intr_enb_dis(lf, true);
146 cpt_lf_unregister_done_irq(struct roc_cpt_lf *lf)
148 struct plt_pci_device *pci_dev = lf->pci_dev;
149 struct plt_intr_handle *handle;
152 handle = &pci_dev->intr_handle;
154 vec = lf->msixoff + CPT_LF_INT_VEC_DONE;
156 /* Clear done interrupt */
157 cpt_lf_done_intr_enb_dis(lf, false);
158 dev_irq_unregister(handle, cpt_lf_done_irq, lf, vec);
162 cpt_lf_register_irqs(struct roc_cpt_lf *lf)
166 if (lf->msixoff == MSIX_VECTOR_INVALID) {
167 plt_err("Invalid CPTLF MSIX vector offset vector: 0x%x",
172 /* Register lf err interrupt */
173 rc = cpt_lf_register_misc_irq(lf);
175 plt_err("Error registering IRQs");
177 rc = cpt_lf_register_done_irq(lf);
179 plt_err("Error registering IRQs");
185 cpt_lf_unregister_irqs(struct roc_cpt_lf *lf)
187 cpt_lf_unregister_misc_irq(lf);
188 cpt_lf_unregister_done_irq(lf);
192 cpt_lf_dump(struct roc_cpt_lf *lf)
194 plt_cpt_dbg("CPT LF");
195 plt_cpt_dbg("RBASE: 0x%016" PRIx64, lf->rbase);
196 plt_cpt_dbg("LMT_BASE: 0x%016" PRIx64, lf->lmt_base);
197 plt_cpt_dbg("MSIXOFF: 0x%x", lf->msixoff);
198 plt_cpt_dbg("LF_ID: 0x%x", lf->lf_id);
199 plt_cpt_dbg("NB DESC: %d", lf->nb_desc);
200 plt_cpt_dbg("FC_ADDR: 0x%016" PRIx64, (uintptr_t)lf->fc_addr);
201 plt_cpt_dbg("CQ.VADDR: 0x%016" PRIx64, (uintptr_t)lf->iq_vaddr);
203 plt_cpt_dbg("CPT LF REG:");
204 plt_cpt_dbg("LF_CTL[0x%016llx]: 0x%016" PRIx64, CPT_LF_CTL,
205 plt_read64(lf->rbase + CPT_LF_CTL));
206 plt_cpt_dbg("Q_SIZE[0x%016llx]: 0x%016" PRIx64, CPT_LF_INPROG,
207 plt_read64(lf->rbase + CPT_LF_INPROG));
209 plt_cpt_dbg("Q_BASE[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_BASE,
210 plt_read64(lf->rbase + CPT_LF_Q_BASE));
211 plt_cpt_dbg("Q_SIZE[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_SIZE,
212 plt_read64(lf->rbase + CPT_LF_Q_SIZE));
213 plt_cpt_dbg("Q_INST_PTR[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_INST_PTR,
214 plt_read64(lf->rbase + CPT_LF_Q_INST_PTR));
215 plt_cpt_dbg("Q_GRP_PTR[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_GRP_PTR,
216 plt_read64(lf->rbase + CPT_LF_Q_GRP_PTR));
220 cpt_lf_outb_cfg(struct dev *dev, uint16_t sso_pf_func, uint16_t nix_pf_func,
221 uint8_t lf_id, bool ena)
223 struct cpt_inline_ipsec_cfg_msg *req;
224 struct mbox *mbox = dev->mbox;
226 req = mbox_alloc_msg_cpt_inline_ipsec_cfg(mbox);
230 req->dir = CPT_INLINE_OUTBOUND;
234 req->sso_pf_func = sso_pf_func;
235 req->nix_pf_func = nix_pf_func;
240 return mbox_process(mbox);
244 roc_cpt_inline_ipsec_cfg(struct dev *cpt_dev, uint8_t lf_id,
245 struct roc_nix *roc_nix)
247 bool ena = roc_nix ? true : false;
248 uint16_t nix_pf_func = 0;
249 uint16_t sso_pf_func = 0;
252 nix_pf_func = roc_nix_get_pf_func(roc_nix);
253 sso_pf_func = idev_sso_pffunc_get();
256 return cpt_lf_outb_cfg(cpt_dev, sso_pf_func, nix_pf_func, lf_id, ena);
260 roc_cpt_inline_ipsec_inb_cfg(struct roc_cpt *roc_cpt, uint16_t param1,
263 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt);
264 struct cpt_rx_inline_lf_cfg_msg *req;
267 mbox = cpt->dev.mbox;
269 req = mbox_alloc_msg_cpt_rx_inline_lf_cfg(mbox);
273 req->sso_pf_func = idev_sso_pffunc_get();
274 req->param1 = param1;
275 req->param2 = param2;
277 return mbox_process(mbox);
281 roc_cpt_rxc_time_cfg(struct roc_cpt *roc_cpt, struct roc_cpt_rxc_time_cfg *cfg)
283 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt);
284 struct cpt_rxc_time_cfg_req *req;
285 struct dev *dev = &cpt->dev;
287 req = mbox_alloc_msg_cpt_rxc_time_cfg(dev->mbox);
293 /* The step value is in microseconds. */
294 req->step = cfg->step;
296 /* The timeout will be: limit * step microseconds */
297 req->zombie_limit = cfg->zombie_limit;
298 req->zombie_thres = cfg->zombie_thres;
300 /* The timeout will be: limit * step microseconds */
301 req->active_limit = cfg->active_limit;
302 req->active_thres = cfg->active_thres;
304 return mbox_process(dev->mbox);
308 cpt_get_msix_offset(struct dev *dev, struct msix_offset_rsp **msix_rsp)
310 struct mbox *mbox = dev->mbox;
313 /* Get MSIX vector offsets */
314 mbox_alloc_msg_msix_offset(mbox);
315 rc = mbox_process_msg(mbox, (void *)msix_rsp);
321 cpt_lfs_attach(struct dev *dev, uint8_t blkaddr, bool modify, uint16_t nb_lf)
323 struct mbox *mbox = dev->mbox;
324 struct rsrc_attach_req *req;
326 if (blkaddr != RVU_BLOCK_ADDR_CPT0 && blkaddr != RVU_BLOCK_ADDR_CPT1)
330 req = mbox_alloc_msg_attach_resources(mbox);
335 req->modify = modify;
336 req->cpt_blkaddr = blkaddr;
338 return mbox_process(mbox);
342 cpt_lfs_detach(struct dev *dev)
344 struct mbox *mbox = dev->mbox;
345 struct rsrc_detach_req *req;
347 req = mbox_alloc_msg_detach_resources(mbox);
354 return mbox_process(mbox);
358 cpt_available_lfs_get(struct dev *dev, uint16_t *nb_lf)
360 struct mbox *mbox = dev->mbox;
361 struct free_rsrcs_rsp *rsp;
364 mbox_alloc_msg_free_rsrc_cnt(mbox);
366 rc = mbox_process_msg(mbox, (void *)&rsp);
375 cpt_lfs_alloc(struct dev *dev, uint8_t eng_grpmsk, uint8_t blkaddr,
378 struct cpt_lf_alloc_req_msg *req;
379 struct mbox *mbox = dev->mbox;
381 if (blkaddr != RVU_BLOCK_ADDR_CPT0 && blkaddr != RVU_BLOCK_ADDR_CPT1)
384 PLT_SET_USED(inl_dev_sso);
386 req = mbox_alloc_msg_cpt_lf_alloc(mbox);
387 req->nix_pf_func = 0;
388 req->sso_pf_func = idev_sso_pffunc_get();
389 req->eng_grpmsk = eng_grpmsk;
390 req->blkaddr = blkaddr;
392 return mbox_process(mbox);
396 cpt_lfs_free(struct dev *dev)
398 mbox_alloc_msg_cpt_lf_free(dev->mbox);
400 return mbox_process(dev->mbox);
404 cpt_hardware_caps_get(struct dev *dev, union cpt_eng_caps *hw_caps)
406 struct cpt_caps_rsp_msg *rsp;
409 mbox_alloc_msg_cpt_caps_get(dev->mbox);
411 ret = mbox_process_msg(dev->mbox, (void *)&rsp);
415 mbox_memcpy(hw_caps, rsp->eng_caps,
416 sizeof(union cpt_eng_caps) * CPT_MAX_ENG_TYPES);
422 cpt_lf_iq_mem_calc(uint32_t nb_desc)
426 /* Space for instruction group memory */
427 len = CPT_IQ_GRP_SIZE(nb_desc);
430 len = PLT_ALIGN(len, ROC_ALIGN);
433 len += CPT_IQ_FC_LEN;
435 /* For instruction queues */
436 len += CPT_IQ_NB_DESC_SIZE_DIV40(nb_desc) * CPT_IQ_NB_DESC_MULTIPLIER *
437 sizeof(struct cpt_inst_s);
443 cpt_iq_init(struct roc_cpt_lf *lf)
445 union cpt_lf_q_size lf_q_size = {.u = 0x0};
446 union cpt_lf_q_base lf_q_base = {.u = 0x0};
449 lf->io_addr = lf->rbase + CPT_LF_NQX(0);
451 /* Disable command queue */
452 roc_cpt_iq_disable(lf);
454 /* Set command queue base address */
455 addr = (uintptr_t)lf->iq_vaddr +
456 PLT_ALIGN(CPT_IQ_GRP_SIZE(lf->nb_desc), ROC_ALIGN);
460 plt_write64(lf_q_base.u, lf->rbase + CPT_LF_Q_BASE);
462 /* Set command queue size */
463 lf_q_size.s.size_div40 = CPT_IQ_NB_DESC_SIZE_DIV40(lf->nb_desc);
464 plt_write64(lf_q_size.u, lf->rbase + CPT_LF_Q_SIZE);
466 lf->fc_addr = (uint64_t *)addr;
467 lf->fc_hyst_bits = plt_log2_u32(lf->nb_desc) / 2;
468 lf->fc_thresh = lf->nb_desc - (lf->nb_desc % (1 << lf->fc_hyst_bits));
472 roc_cpt_dev_configure(struct roc_cpt *roc_cpt, int nb_lf)
474 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt);
475 uint8_t blkaddr = RVU_BLOCK_ADDR_CPT0;
476 struct msix_offset_rsp *rsp;
480 /* Request LF resources */
481 rc = cpt_lfs_attach(&cpt->dev, blkaddr, true, nb_lf);
485 eng_grpmsk = (1 << roc_cpt->eng_grp[CPT_ENG_TYPE_AE]) |
486 (1 << roc_cpt->eng_grp[CPT_ENG_TYPE_SE]) |
487 (1 << roc_cpt->eng_grp[CPT_ENG_TYPE_IE]);
489 rc = cpt_lfs_alloc(&cpt->dev, eng_grpmsk, blkaddr, false);
493 rc = cpt_get_msix_offset(&cpt->dev, &rsp);
497 for (i = 0; i < nb_lf; i++)
498 cpt->lf_msix_off[i] =
499 (cpt->lf_blkaddr[i] == RVU_BLOCK_ADDR_CPT1) ?
500 rsp->cpt1_lf_msixoff[i] :
501 rsp->cptlf_msixoff[i];
503 roc_cpt->nb_lf = nb_lf;
508 cpt_lfs_free(&cpt->dev);
510 cpt_lfs_detach(&cpt->dev);
515 cpt_get_blkaddr(struct dev *dev)
520 /* Reading the discovery register to know which CPT is the LF
521 * attached to. Assume CPT LF's of only one block are attached
525 off = RVU_VF_BLOCK_ADDRX_DISC(RVU_BLOCK_ADDR_CPT1);
527 off = RVU_PF_BLOCK_ADDRX_DISC(RVU_BLOCK_ADDR_CPT1);
529 reg = plt_read64(dev->bar2 + off);
531 return reg & 0x1FFULL ? RVU_BLOCK_ADDR_CPT1 : RVU_BLOCK_ADDR_CPT0;
535 cpt_lf_init(struct roc_cpt_lf *lf)
537 struct dev *dev = lf->dev;
542 if (lf->nb_desc == 0 || lf->nb_desc > CPT_LF_MAX_NB_DESC)
543 lf->nb_desc = CPT_LF_DEFAULT_NB_DESC;
545 /* Allocate memory for instruction queue for CPT LF. */
546 iq_mem = plt_zmalloc(cpt_lf_iq_mem_calc(lf->nb_desc), ROC_ALIGN);
550 blkaddr = cpt_get_blkaddr(dev);
551 lf->rbase = dev->bar2 + ((blkaddr << 20) | (lf->lf_id << 12));
552 lf->iq_vaddr = iq_mem;
553 lf->lmt_base = dev->lmt_base;
554 lf->pf_func = dev->pf_func;
556 /* Initialize instruction queue */
559 rc = cpt_lf_register_irqs(lf);
566 roc_cpt_iq_disable(lf);
572 roc_cpt_lf_init(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf)
574 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt);
578 lf->roc_cpt = roc_cpt;
579 lf->msixoff = cpt->lf_msix_off[lf->lf_id];
580 lf->pci_dev = cpt->pci_dev;
582 rc = cpt_lf_init(lf);
586 /* LF init successful */
587 roc_cpt->lf[lf->lf_id] = lf;
592 roc_cpt_dev_init(struct roc_cpt *roc_cpt)
594 struct plt_pci_device *pci_dev;
595 uint16_t nb_lf_avail;
600 if (roc_cpt == NULL || roc_cpt->pci_dev == NULL)
603 PLT_STATIC_ASSERT(sizeof(struct cpt) <= ROC_CPT_MEM_SZ);
605 cpt = roc_cpt_to_cpt_priv(roc_cpt);
606 memset(cpt, 0, sizeof(*cpt));
607 pci_dev = roc_cpt->pci_dev;
610 /* Initialize device */
611 rc = dev_init(dev, pci_dev);
613 plt_err("Failed to init roc device");
617 cpt->pci_dev = pci_dev;
618 roc_cpt->lmt_base = dev->lmt_base;
620 rc = cpt_hardware_caps_get(dev, roc_cpt->hw_caps);
622 plt_err("Could not determine hardware capabilities");
626 rc = cpt_available_lfs_get(&cpt->dev, &nb_lf_avail);
628 plt_err("Could not get available lfs");
632 /* Reserve 1 CPT LF for inline inbound */
633 nb_lf_avail = PLT_MIN(nb_lf_avail, ROC_CPT_MAX_LFS - 1);
635 roc_cpt->nb_lf_avail = nb_lf_avail;
637 dev->roc_cpt = roc_cpt;
639 /* Set it to idev if not already present */
640 if (!roc_idev_cpt_get())
641 roc_idev_cpt_set(roc_cpt);
650 roc_cpt_lf_ctx_flush(struct roc_cpt_lf *lf, uint64_t cptr)
652 union cpt_lf_ctx_flush reg;
658 reg.s.pf_func = lf->pf_func;
662 plt_write64(reg.u, lf->rbase + CPT_LF_CTX_FLUSH);
668 cpt_lf_fini(struct roc_cpt_lf *lf)
670 /* Unregister IRQ's */
671 cpt_lf_unregister_irqs(lf);
674 roc_cpt_iq_disable(lf);
677 plt_free(lf->iq_vaddr);
682 roc_cpt_lf_fini(struct roc_cpt_lf *lf)
686 lf->roc_cpt->lf[lf->lf_id] = NULL;
691 roc_cpt_dev_fini(struct roc_cpt *roc_cpt)
693 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt);
698 /* Remove idev references */
699 if (roc_idev_cpt_get() == roc_cpt)
700 roc_idev_cpt_set(NULL);
702 roc_cpt->nb_lf_avail = 0;
704 roc_cpt->lmt_base = 0;
706 return dev_fini(&cpt->dev, cpt->pci_dev);
710 roc_cpt_dev_clear(struct roc_cpt *roc_cpt)
712 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt);
718 for (i = 0; i < roc_cpt->nb_lf; i++)
719 cpt->lf_msix_off[i] = 0;
723 cpt_lfs_free(&cpt->dev);
725 cpt_lfs_detach(&cpt->dev);
729 roc_cpt_eng_grp_add(struct roc_cpt *roc_cpt, enum cpt_eng_type eng_type)
731 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt);
732 struct dev *dev = &cpt->dev;
733 struct cpt_eng_grp_req *req;
734 struct cpt_eng_grp_rsp *rsp;
737 req = mbox_alloc_msg_cpt_eng_grp_get(dev->mbox);
742 case CPT_ENG_TYPE_AE:
743 case CPT_ENG_TYPE_SE:
744 case CPT_ENG_TYPE_IE:
750 req->eng_type = eng_type;
751 ret = mbox_process_msg(dev->mbox, (void *)&rsp);
755 if (rsp->eng_grp_num > 8) {
756 plt_err("Invalid CPT engine group");
760 roc_cpt->eng_grp[eng_type] = rsp->eng_grp_num;
762 return rsp->eng_grp_num;
766 roc_cpt_iq_disable(struct roc_cpt_lf *lf)
768 union cpt_lf_ctl lf_ctl = {.u = 0x0};
769 union cpt_lf_inprog lf_inprog;
772 /* Disable instructions enqueuing */
773 plt_write64(lf_ctl.u, lf->rbase + CPT_LF_CTL);
775 /* Wait for instruction queue to become empty */
777 lf_inprog.u = plt_read64(lf->rbase + CPT_LF_INPROG);
778 if (!lf_inprog.s.inflight)
783 plt_err("CPT LF %d is still busy", lf->lf_id);
789 /* Disable executions in the LF's queue.
790 * The queue should be empty at this point
792 lf_inprog.s.eena = 0x0;
793 plt_write64(lf_inprog.u, lf->rbase + CPT_LF_INPROG);
797 roc_cpt_iq_enable(struct roc_cpt_lf *lf)
799 union cpt_lf_inprog lf_inprog;
800 union cpt_lf_ctl lf_ctl;
802 /* Disable command queue */
803 roc_cpt_iq_disable(lf);
805 /* Enable command queue execution */
806 lf_inprog.u = plt_read64(lf->rbase + CPT_LF_INPROG);
807 lf_inprog.s.eena = 1;
808 plt_write64(lf_inprog.u, lf->rbase + CPT_LF_INPROG);
810 /* Enable instruction queue enqueuing */
811 lf_ctl.u = plt_read64(lf->rbase + CPT_LF_CTL);
814 lf_ctl.s.fc_up_crossing = 0;
815 lf_ctl.s.fc_hyst_bits = lf->fc_hyst_bits;
816 plt_write64(lf_ctl.u, lf->rbase + CPT_LF_CTL);
822 roc_cpt_lmtline_init(struct roc_cpt *roc_cpt, struct roc_cpt_lmtline *lmtline,
825 struct roc_cpt_lf *lf;
827 lf = roc_cpt->lf[lf_id];
831 lmtline->io_addr = lf->io_addr;
832 if (roc_model_is_cn10k())
833 lmtline->io_addr |= ROC_CN10K_CPT_INST_DW_M1 << 4;
835 lmtline->fc_addr = lf->fc_addr;
836 lmtline->lmt_base = lf->lmt_base;