1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
8 PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ ==
9 1UL << ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ_LOG2);
10 PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ == 512);
11 PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ ==
12 1UL << ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ_LOG2);
13 PLT_STATIC_ASSERT(ROC_NIX_INL_OT_IPSEC_INB_SA_SZ ==
14 1UL << ROC_NIX_INL_OT_IPSEC_INB_SA_SZ_LOG2);
15 PLT_STATIC_ASSERT(ROC_NIX_INL_OT_IPSEC_INB_SA_SZ == 1024);
16 PLT_STATIC_ASSERT(ROC_NIX_INL_OT_IPSEC_OUTB_SA_SZ ==
17 1UL << ROC_NIX_INL_OT_IPSEC_OUTB_SA_SZ_LOG2);
20 nix_inl_inb_sa_tbl_setup(struct roc_nix *roc_nix)
22 uint16_t ipsec_in_max_spi = roc_nix->ipsec_in_max_spi;
23 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
24 struct roc_nix_ipsec_cfg cfg;
29 /* CN9K SA size is different */
30 if (roc_model_is_cn9k())
31 inb_sa_sz = ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ;
33 inb_sa_sz = ROC_NIX_INL_OT_IPSEC_INB_SA_SZ;
35 /* Alloc contiguous memory for Inbound SA's */
36 nix->inb_sa_sz = inb_sa_sz;
37 nix->inb_sa_base = plt_zmalloc(inb_sa_sz * ipsec_in_max_spi,
38 ROC_NIX_INL_SA_BASE_ALIGN);
39 if (!nix->inb_sa_base) {
40 plt_err("Failed to allocate memory for Inbound SA");
43 if (roc_model_is_cn10k()) {
44 for (i = 0; i < ipsec_in_max_spi; i++) {
45 sa = ((uint8_t *)nix->inb_sa_base) + (i * inb_sa_sz);
46 roc_nix_inl_inb_sa_init(sa);
50 memset(&cfg, 0, sizeof(cfg));
51 cfg.sa_size = inb_sa_sz;
52 cfg.iova = (uintptr_t)nix->inb_sa_base;
53 cfg.max_sa = ipsec_in_max_spi + 1;
54 cfg.tt = SSO_TT_ORDERED;
56 /* Setup device specific inb SA table */
57 rc = roc_nix_lf_inl_ipsec_cfg(roc_nix, &cfg, true);
59 plt_err("Failed to setup NIX Inbound SA conf, rc=%d", rc);
65 plt_free(nix->inb_sa_base);
66 nix->inb_sa_base = NULL;
71 nix_inl_sa_tbl_release(struct roc_nix *roc_nix)
73 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
76 rc = roc_nix_lf_inl_ipsec_cfg(roc_nix, NULL, false);
78 plt_err("Failed to disable Inbound inline ipsec, rc=%d", rc);
82 plt_free(nix->inb_sa_base);
83 nix->inb_sa_base = NULL;
88 roc_nix_inl_outb_lf_base_get(struct roc_nix *roc_nix)
90 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
92 /* NIX Inline config needs to be done */
93 if (!nix->inl_outb_ena || !nix->cpt_lf_base)
96 return (struct roc_cpt_lf *)nix->cpt_lf_base;
100 roc_nix_inl_outb_sa_base_get(struct roc_nix *roc_nix)
102 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
104 return (uintptr_t)nix->outb_sa_base;
108 roc_nix_inl_inb_sa_base_get(struct roc_nix *roc_nix, bool inb_inl_dev)
110 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
111 struct idev_cfg *idev = idev_get_cfg();
112 struct nix_inl_dev *inl_dev;
117 if (!nix->inl_inb_ena)
120 inl_dev = idev->nix_inl_dev;
122 /* Return inline dev sa base */
124 return (uintptr_t)inl_dev->inb_sa_base;
128 return (uintptr_t)nix->inb_sa_base;
132 roc_nix_inl_inb_sa_max_spi(struct roc_nix *roc_nix, bool inb_inl_dev)
134 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
135 struct idev_cfg *idev = idev_get_cfg();
136 struct nix_inl_dev *inl_dev;
141 if (!nix->inl_inb_ena)
144 inl_dev = idev->nix_inl_dev;
147 return inl_dev->ipsec_in_max_spi;
151 return roc_nix->ipsec_in_max_spi;
155 roc_nix_inl_inb_sa_sz(struct roc_nix *roc_nix, bool inl_dev_sa)
157 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
158 struct idev_cfg *idev = idev_get_cfg();
159 struct nix_inl_dev *inl_dev;
165 return nix->inb_sa_sz;
167 inl_dev = idev->nix_inl_dev;
168 if (inl_dev_sa && inl_dev)
169 return inl_dev->inb_sa_sz;
176 roc_nix_inl_inb_sa_get(struct roc_nix *roc_nix, bool inb_inl_dev, uint32_t spi)
182 sa_base = roc_nix_inl_inb_sa_base_get(roc_nix, inb_inl_dev);
183 /* Check if SA base exists */
187 /* Check if SPI is in range */
188 max_spi = roc_nix_inl_inb_sa_max_spi(roc_nix, inb_inl_dev);
190 plt_err("Inbound SA SPI %u exceeds max %u", spi, max_spi);
195 sz = roc_nix_inl_inb_sa_sz(roc_nix, inb_inl_dev);
199 /* Basic logic of SPI->SA for now */
200 return (sa_base + (spi * sz));
204 roc_nix_inl_inb_init(struct roc_nix *roc_nix)
206 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
207 struct idev_cfg *idev = idev_get_cfg();
208 struct roc_cpt *roc_cpt;
215 /* Unless we have another mechanism to trigger
216 * onetime Inline config in CPTPF, we cannot
217 * support without CPT being probed.
221 plt_err("Cannot support inline inbound, cryptodev not probed");
225 if (roc_model_is_cn9k()) {
226 param1 = ROC_ONF_IPSEC_INB_MAX_L2_SZ;
228 union roc_ot_ipsec_inb_param1 u;
231 u.s.esp_trailer_disable = 1;
235 /* Do onetime Inbound Inline config in CPTPF */
236 rc = roc_cpt_inline_ipsec_inb_cfg(roc_cpt, param1, 0);
237 if (rc && rc != -EEXIST) {
238 plt_err("Failed to setup inbound lf, rc=%d", rc);
242 /* Setup Inbound SA table */
243 rc = nix_inl_inb_sa_tbl_setup(roc_nix);
247 nix->inl_inb_ena = true;
252 roc_nix_inl_inb_fini(struct roc_nix *roc_nix)
254 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
256 if (!nix->inl_inb_ena)
259 nix->inl_inb_ena = false;
261 /* Flush Inbound CTX cache entries */
262 roc_nix_cpt_ctx_cache_sync(roc_nix);
264 /* Disable Inbound SA */
265 return nix_inl_sa_tbl_release(roc_nix);
269 roc_nix_inl_outb_init(struct roc_nix *roc_nix)
271 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
272 struct idev_cfg *idev = idev_get_cfg();
273 struct roc_cpt_lf *lf_base, *lf;
274 struct dev *dev = &nix->dev;
275 struct msix_offset_rsp *rsp;
276 struct nix_inl_dev *inl_dev;
289 nb_lf = roc_nix->outb_nb_crypto_qs;
290 blkaddr = nix->is_nix1 ? RVU_BLOCK_ADDR_CPT1 : RVU_BLOCK_ADDR_CPT0;
292 /* Retrieve inline device if present */
293 inl_dev = idev->nix_inl_dev;
294 sso_pffunc = inl_dev ? inl_dev->dev.pf_func : idev_sso_pffunc_get();
296 plt_err("Failed to setup inline outb, need either "
297 "inline device or sso device");
301 /* Attach CPT LF for outbound */
302 rc = cpt_lfs_attach(dev, blkaddr, true, nb_lf);
304 plt_err("Failed to attach CPT LF for inline outb, rc=%d", rc);
309 eng_grpmask = (1ULL << ROC_CPT_DFLT_ENG_GRP_SE |
310 1ULL << ROC_CPT_DFLT_ENG_GRP_SE_IE |
311 1ULL << ROC_CPT_DFLT_ENG_GRP_AE);
312 rc = cpt_lfs_alloc(dev, eng_grpmask, blkaddr, true);
314 plt_err("Failed to alloc CPT LF resources, rc=%d", rc);
318 /* Get msix offsets */
319 rc = cpt_get_msix_offset(dev, &rsp);
321 plt_err("Failed to get CPT LF msix offset, rc=%d", rc);
325 mbox_memcpy(nix->cpt_msixoff,
326 nix->is_nix1 ? rsp->cpt1_lf_msixoff : rsp->cptlf_msixoff,
327 sizeof(nix->cpt_msixoff));
329 /* Alloc required num of cpt lfs */
330 lf_base = plt_zmalloc(nb_lf * sizeof(struct roc_cpt_lf), 0);
332 plt_err("Failed to alloc cpt lf memory");
337 /* Initialize CPT LF's */
338 for (i = 0; i < nb_lf; i++) {
342 lf->nb_desc = roc_nix->outb_nb_desc;
344 lf->msixoff = nix->cpt_msixoff[i];
345 lf->pci_dev = nix->pci_dev;
347 /* Setup CPT LF instruction queue */
348 rc = cpt_lf_init(lf);
350 plt_err("Failed to initialize CPT LF, rc=%d", rc);
354 /* Associate this CPT LF with NIX PFFUNC */
355 rc = cpt_lf_outb_cfg(dev, sso_pffunc, nix->dev.pf_func, i,
358 plt_err("Failed to setup CPT LF->(NIX,SSO) link, rc=%d",
364 roc_cpt_iq_enable(lf);
367 if (!roc_nix->ipsec_out_max_sa)
370 /* CN9K SA size is different */
371 if (roc_model_is_cn9k())
372 sa_sz = ROC_NIX_INL_ONF_IPSEC_OUTB_SA_SZ;
374 sa_sz = ROC_NIX_INL_OT_IPSEC_OUTB_SA_SZ;
375 /* Alloc contiguous memory of outbound SA */
376 sa_base = plt_zmalloc(sa_sz * roc_nix->ipsec_out_max_sa,
377 ROC_NIX_INL_SA_BASE_ALIGN);
379 plt_err("Outbound SA base alloc failed");
382 if (roc_model_is_cn10k()) {
383 for (i = 0; i < roc_nix->ipsec_out_max_sa; i++) {
384 sa = ((uint8_t *)sa_base) + (i * sa_sz);
385 roc_nix_inl_outb_sa_init(sa);
388 nix->outb_sa_base = sa_base;
389 nix->outb_sa_sz = sa_sz;
393 nix->cpt_lf_base = lf_base;
394 nix->nb_cpt_lf = nb_lf;
395 nix->outb_err_sso_pffunc = sso_pffunc;
396 nix->inl_outb_ena = true;
400 for (j = i - 1; j >= 0; j--)
401 cpt_lf_fini(&lf_base[j]);
404 rc |= cpt_lfs_free(dev);
406 rc |= cpt_lfs_detach(dev);
411 roc_nix_inl_outb_fini(struct roc_nix *roc_nix)
413 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
414 struct roc_cpt_lf *lf_base = nix->cpt_lf_base;
415 struct dev *dev = &nix->dev;
418 if (!nix->inl_outb_ena)
421 nix->inl_outb_ena = false;
423 /* Cleanup CPT LF instruction queue */
424 for (i = 0; i < nix->nb_cpt_lf; i++)
425 cpt_lf_fini(&lf_base[i]);
427 /* Free LF resources */
428 rc = cpt_lfs_free(dev);
430 plt_err("Failed to free CPT LF resources, rc=%d", rc);
434 rc = cpt_lfs_detach(dev);
436 plt_err("Failed to detach CPT LF, rc=%d", rc);
440 nix->cpt_lf_base = NULL;
443 /* Free outbound SA base */
444 plt_free(nix->outb_sa_base);
445 nix->outb_sa_base = NULL;
452 roc_nix_inl_dev_is_probed(void)
454 struct idev_cfg *idev = idev_get_cfg();
459 return !!idev->nix_inl_dev;
463 roc_nix_inl_inb_is_enabled(struct roc_nix *roc_nix)
465 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
467 return nix->inl_inb_ena;
471 roc_nix_inl_outb_is_enabled(struct roc_nix *roc_nix)
473 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
475 return nix->inl_outb_ena;
479 roc_nix_inl_dev_rq_get(struct roc_nix_rq *rq)
481 struct idev_cfg *idev = idev_get_cfg();
482 struct nix_inl_dev *inl_dev;
483 struct roc_nix_rq *inl_rq;
490 inl_dev = idev->nix_inl_dev;
491 /* Nothing to do if no inline device */
495 /* Just take reference if already inited */
496 if (inl_dev->rq_refs) {
498 rq->inl_dev_ref = true;
503 inl_rq = &inl_dev->rq;
504 memset(inl_rq, 0, sizeof(struct roc_nix_rq));
506 /* Take RQ pool attributes from the first ethdev RQ */
508 inl_rq->aura_handle = rq->aura_handle;
509 inl_rq->first_skip = rq->first_skip;
510 inl_rq->later_skip = rq->later_skip;
511 inl_rq->lpb_size = rq->lpb_size;
513 if (!roc_model_is_cn9k()) {
514 uint64_t aura_limit =
515 roc_npa_aura_op_limit_get(inl_rq->aura_handle);
516 uint64_t aura_shift = plt_log2_u32(aura_limit);
521 aura_shift = aura_shift - 8;
523 /* Set first pass RQ to drop when half of the buffers are in
524 * use to avoid metabuf alloc failure. This is needed as long
525 * as we cannot use different
527 inl_rq->red_pass = (aura_limit / 2) >> aura_shift;
528 inl_rq->red_drop = ((aura_limit / 2) - 1) >> aura_shift;
532 inl_rq->ipsech_ena = true;
534 inl_rq->flow_tag_width = 20;
535 /* Special tag mask */
536 inl_rq->tag_mask = 0xFFF00000;
537 inl_rq->tt = SSO_TT_ORDERED;
539 inl_rq->wqe_skip = 1;
540 inl_rq->sso_ena = true;
542 /* Prepare and send RQ init mbox */
543 if (roc_model_is_cn9k())
544 rc = nix_rq_cn9k_cfg(dev, inl_rq, inl_dev->qints, false, true);
546 rc = nix_rq_cfg(dev, inl_rq, inl_dev->qints, false, true);
548 plt_err("Failed to prepare aq_enq msg, rc=%d", rc);
552 rc = mbox_process(dev->mbox);
554 plt_err("Failed to send aq_enq msg, rc=%d", rc);
559 rq->inl_dev_ref = true;
564 roc_nix_inl_dev_rq_put(struct roc_nix_rq *rq)
566 struct idev_cfg *idev = idev_get_cfg();
567 struct nix_inl_dev *inl_dev;
568 struct roc_nix_rq *inl_rq;
575 if (!rq->inl_dev_ref)
578 inl_dev = idev->nix_inl_dev;
579 /* Inline device should be there if we have ref */
581 plt_err("Failed to find inline device with refs");
585 rq->inl_dev_ref = false;
587 if (inl_dev->rq_refs)
591 inl_rq = &inl_dev->rq;
592 /* There are no more references, disable RQ */
593 rc = nix_rq_ena_dis(dev, inl_rq, false);
595 plt_err("Failed to disable inline device rq, rc=%d", rc);
597 /* Flush NIX LF for CN10K */
598 nix_rq_vwqe_flush(rq, inl_dev->vwqe_interval);
604 roc_nix_inl_dev_rq_limit_get(void)
606 struct idev_cfg *idev = idev_get_cfg();
607 struct nix_inl_dev *inl_dev;
608 struct roc_nix_rq *inl_rq;
610 if (!idev || !idev->nix_inl_dev)
613 inl_dev = idev->nix_inl_dev;
614 if (!inl_dev->rq_refs)
617 inl_rq = &inl_dev->rq;
619 return roc_npa_aura_op_limit_get(inl_rq->aura_handle);
623 roc_nix_inb_mode_set(struct roc_nix *roc_nix, bool use_inl_dev)
625 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
627 /* Info used by NPC flow rule add */
628 nix->inb_inl_dev = use_inl_dev;
632 roc_nix_inb_is_with_inl_dev(struct roc_nix *roc_nix)
634 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
636 return nix->inb_inl_dev;
640 roc_nix_inl_dev_rq(void)
642 struct idev_cfg *idev = idev_get_cfg();
643 struct nix_inl_dev *inl_dev;
646 inl_dev = idev->nix_inl_dev;
647 if (inl_dev != NULL && inl_dev->rq_refs)
655 roc_nix_inl_outb_sso_pffunc_get(struct roc_nix *roc_nix)
657 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
659 return nix->outb_err_sso_pffunc;
663 roc_nix_inl_cb_register(roc_nix_inl_sso_work_cb_t cb, void *args)
665 struct idev_cfg *idev = idev_get_cfg();
666 struct nix_inl_dev *inl_dev;
671 inl_dev = idev->nix_inl_dev;
675 /* Be silent if registration called with same cb and args */
676 if (inl_dev->work_cb == cb && inl_dev->cb_args == args)
679 /* Don't allow registration again if registered with different cb */
680 if (inl_dev->work_cb)
683 inl_dev->work_cb = cb;
684 inl_dev->cb_args = args;
689 roc_nix_inl_cb_unregister(roc_nix_inl_sso_work_cb_t cb, void *args)
691 struct idev_cfg *idev = idev_get_cfg();
692 struct nix_inl_dev *inl_dev;
697 inl_dev = idev->nix_inl_dev;
701 if (inl_dev->work_cb != cb || inl_dev->cb_args != args)
704 inl_dev->work_cb = NULL;
705 inl_dev->cb_args = NULL;
710 roc_nix_inl_inb_tag_update(struct roc_nix *roc_nix, uint32_t tag_const,
713 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
714 struct roc_nix_ipsec_cfg cfg;
716 /* Be silent if inline inbound not enabled */
717 if (!nix->inl_inb_ena)
720 memset(&cfg, 0, sizeof(cfg));
721 cfg.sa_size = nix->inb_sa_sz;
722 cfg.iova = (uintptr_t)nix->inb_sa_base;
723 cfg.max_sa = roc_nix->ipsec_in_max_spi + 1;
725 cfg.tag_const = tag_const;
727 return roc_nix_lf_inl_ipsec_cfg(roc_nix, &cfg, true);
731 roc_nix_inl_sa_sync(struct roc_nix *roc_nix, void *sa, bool inb,
732 enum roc_nix_inl_sa_sync_op op)
734 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
735 struct roc_cpt_lf *outb_lf = nix->cpt_lf_base;
736 struct idev_cfg *idev = idev_get_cfg();
737 struct nix_inl_dev *inl_dev = NULL;
738 union cpt_lf_ctx_reload reload;
739 union cpt_lf_ctx_flush flush;
742 /* Nothing much to do on cn9k */
743 if (roc_model_is_cn9k()) {
744 plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
748 if (inb && nix->inb_inl_dev) {
751 inl_dev = idev->nix_inl_dev;
753 outb_lf = &inl_dev->cpt_lf;
757 rbase = outb_lf->rbase;
762 case ROC_NIX_INL_SA_OP_FLUSH_INVAL:
765 case ROC_NIX_INL_SA_OP_FLUSH:
766 flush.s.cptr = ((uintptr_t)sa) >> 7;
767 plt_write64(flush.u, rbase + CPT_LF_CTX_FLUSH);
769 case ROC_NIX_INL_SA_OP_RELOAD:
770 reload.s.cptr = ((uintptr_t)sa) >> 7;
771 plt_write64(reload.u, rbase + CPT_LF_CTX_RELOAD);
778 plt_err("Could not get CPT LF for SA sync");
783 roc_nix_inl_ctx_write(struct roc_nix *roc_nix, void *sa_dptr, void *sa_cptr,
784 bool inb, uint16_t sa_len)
786 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
787 struct roc_cpt_lf *outb_lf = nix->cpt_lf_base;
788 struct idev_cfg *idev = idev_get_cfg();
789 struct nix_inl_dev *inl_dev = NULL;
790 union cpt_lf_ctx_flush flush;
794 /* Nothing much to do on cn9k */
795 if (roc_model_is_cn9k()) {
796 plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
800 if (inb && nix->inb_inl_dev) {
803 inl_dev = idev->nix_inl_dev;
804 if (inl_dev && inl_dev->attach_cptlf)
805 outb_lf = &inl_dev->cpt_lf;
809 rbase = outb_lf->rbase;
812 rc = roc_cpt_ctx_write(outb_lf, sa_dptr, sa_cptr, sa_len);
815 /* Trigger CTX flush to write dirty data back to DRAM */
816 flush.s.cptr = ((uintptr_t)sa_cptr) >> 7;
817 plt_write64(flush.u, rbase + CPT_LF_CTX_FLUSH);
821 plt_nix_dbg("Could not get CPT LF for CTX write");
826 roc_nix_inl_inb_sa_init(struct roc_ot_ipsec_inb_sa *sa)
830 memset(sa, 0, sizeof(struct roc_ot_ipsec_inb_sa));
832 offset = offsetof(struct roc_ot_ipsec_inb_sa, ctx);
833 sa->w0.s.hw_ctx_off = offset / ROC_CTX_UNIT_8B;
834 sa->w0.s.ctx_push_size = sa->w0.s.hw_ctx_off + 1;
835 sa->w0.s.ctx_size = ROC_IE_OT_CTX_ILEN;
836 sa->w0.s.aop_valid = 1;
840 roc_nix_inl_outb_sa_init(struct roc_ot_ipsec_outb_sa *sa)
844 memset(sa, 0, sizeof(struct roc_ot_ipsec_outb_sa));
846 offset = offsetof(struct roc_ot_ipsec_outb_sa, ctx);
847 sa->w0.s.ctx_push_size = (offset / ROC_CTX_UNIT_8B);
848 sa->w0.s.ctx_size = ROC_IE_OT_CTX_ILEN;
849 sa->w0.s.aop_valid = 1;
853 roc_nix_inl_dev_lock(void)
855 struct idev_cfg *idev = idev_get_cfg();
858 plt_spinlock_lock(&idev->nix_inl_dev_lock);
862 roc_nix_inl_dev_unlock(void)
864 struct idev_cfg *idev = idev_get_cfg();
867 plt_spinlock_unlock(&idev->nix_inl_dev_lock);