From: Nithin Dabilpuram Date: Fri, 1 Oct 2021 13:39:59 +0000 (+0530) Subject: common/cnxk: support NIX inline device IRQ X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=cfb4f96406022146ce3fd5b906106ef8811fd9b1;p=dpdk.git common/cnxk: support NIX inline device IRQ Add API to setup NIX inline device IRQ's. This registers IRQ's for errors in case of NIX, CPT LF, SSOW and get wor interrupt in case of SSO. Signed-off-by: Nithin Dabilpuram Acked-by: Jerin Jacob --- diff --git a/drivers/common/cnxk/meson.build b/drivers/common/cnxk/meson.build index 258429d54b..3e836cec2b 100644 --- a/drivers/common/cnxk/meson.build +++ b/drivers/common/cnxk/meson.build @@ -28,6 +28,7 @@ sources = files( 'roc_nix_debug.c', 'roc_nix_fc.c', 'roc_nix_irq.c', + 'roc_nix_inl_dev_irq.c', 'roc_nix_mac.c', 'roc_nix_mcast.c', 'roc_nix_npc.c', diff --git a/drivers/common/cnxk/roc_api.h b/drivers/common/cnxk/roc_api.h index 7dec8453b4..c1af95e52e 100644 --- a/drivers/common/cnxk/roc_api.h +++ b/drivers/common/cnxk/roc_api.h @@ -129,4 +129,7 @@ /* HASH computation */ #include "roc_hash.h" +/* NIX Inline dev */ +#include "roc_nix_inl.h" + #endif /* _ROC_API_H_ */ diff --git a/drivers/common/cnxk/roc_irq.c b/drivers/common/cnxk/roc_irq.c index 4c2b4c30d7..28fe691932 100644 --- a/drivers/common/cnxk/roc_irq.c +++ b/drivers/common/cnxk/roc_irq.c @@ -138,9 +138,10 @@ dev_irq_register(struct plt_intr_handle *intr_handle, plt_intr_callback_fn cb, irq_init(intr_handle); } - if (vec > intr_handle->max_intr) { - plt_err("Vector=%d greater than max_intr=%d", vec, - intr_handle->max_intr); + if (vec > intr_handle->max_intr || vec >= PLT_DIM(intr_handle->efds)) { + plt_err("Vector=%d greater than max_intr=%d or " + "max_efd=%" PRIu64, + vec, intr_handle->max_intr, PLT_DIM(intr_handle->efds)); return -EINVAL; } diff --git a/drivers/common/cnxk/roc_nix_inl.h b/drivers/common/cnxk/roc_nix_inl.h new file mode 100644 index 0000000000..1ec3dda0f8 --- /dev/null +++ b/drivers/common/cnxk/roc_nix_inl.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2021 Marvell. + */ +#ifndef _ROC_NIX_INL_H_ +#define _ROC_NIX_INL_H_ + +/* Inline device SSO Work callback */ +typedef void (*roc_nix_inl_sso_work_cb_t)(uint64_t *gw, void *args); + +#endif /* _ROC_NIX_INL_H_ */ diff --git a/drivers/common/cnxk/roc_nix_inl_dev_irq.c b/drivers/common/cnxk/roc_nix_inl_dev_irq.c new file mode 100644 index 0000000000..25ed42f875 --- /dev/null +++ b/drivers/common/cnxk/roc_nix_inl_dev_irq.c @@ -0,0 +1,359 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2021 Marvell. + */ + +#include "roc_api.h" +#include "roc_priv.h" + +static void +nix_inl_sso_work_cb(struct nix_inl_dev *inl_dev) +{ + uintptr_t getwrk_op = inl_dev->ssow_base + SSOW_LF_GWS_OP_GET_WORK0; + uintptr_t tag_wqe_op = inl_dev->ssow_base + SSOW_LF_GWS_WQE0; + uint32_t wdata = BIT(16) | 1; + union { + __uint128_t get_work; + uint64_t u64[2]; + } gw; + uint64_t work; + +again: + /* Try to do get work */ + gw.get_work = wdata; + plt_write64(gw.u64[0], getwrk_op); + do { + roc_load_pair(gw.u64[0], gw.u64[1], tag_wqe_op); + } while (gw.u64[0] & BIT_ULL(63)); + + work = gw.u64[1]; + /* Do we have any work? */ + if (work) { + if (inl_dev->work_cb) + inl_dev->work_cb(gw.u64, inl_dev->cb_args); + else + plt_warn("Undelivered inl dev work gw0: %p gw1: %p", + (void *)gw.u64[0], (void *)gw.u64[1]); + goto again; + } + + plt_atomic_thread_fence(__ATOMIC_ACQ_REL); +} + +static int +nix_inl_nix_reg_dump(struct nix_inl_dev *inl_dev) +{ + uintptr_t nix_base = inl_dev->nix_base; + + /* General registers */ + nix_lf_gen_reg_dump(nix_base, NULL); + + /* Rx, Tx stat registers */ + nix_lf_stat_reg_dump(nix_base, NULL, inl_dev->lf_tx_stats, + inl_dev->lf_rx_stats); + + /* Intr registers */ + nix_lf_int_reg_dump(nix_base, NULL, inl_dev->qints, inl_dev->cints); + + return 0; +} + +static void +nix_inl_sso_hwgrp_irq(void *param) +{ + struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; + uintptr_t sso_base = inl_dev->sso_base; + uint64_t intr; + + intr = plt_read64(sso_base + SSO_LF_GGRP_INT); + if (intr == 0) + return; + + /* Check for work executable interrupt */ + if (intr & BIT(1)) + nix_inl_sso_work_cb(inl_dev); + + if (!(intr & BIT(1))) + plt_err("GGRP 0 GGRP_INT=0x%" PRIx64 "", intr); + + /* Clear interrupt */ + plt_write64(intr, sso_base + SSO_LF_GGRP_INT); +} + +static void +nix_inl_sso_hws_irq(void *param) +{ + struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; + uintptr_t ssow_base = inl_dev->ssow_base; + uint64_t intr; + + intr = plt_read64(ssow_base + SSOW_LF_GWS_INT); + if (intr == 0) + return; + + plt_err("GWS 0 GWS_INT=0x%" PRIx64 "", intr); + + /* Clear interrupt */ + plt_write64(intr, ssow_base + SSOW_LF_GWS_INT); +} + +int +nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev) +{ + struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle; + uintptr_t ssow_base = inl_dev->ssow_base; + uintptr_t sso_base = inl_dev->sso_base; + uint16_t sso_msixoff, ssow_msixoff; + int rc; + + ssow_msixoff = inl_dev->ssow_msixoff; + sso_msixoff = inl_dev->sso_msixoff; + if (sso_msixoff == MSIX_VECTOR_INVALID || + ssow_msixoff == MSIX_VECTOR_INVALID) { + plt_err("Invalid SSO/SSOW MSIX offsets (0x%x, 0x%x)", + sso_msixoff, ssow_msixoff); + return -EINVAL; + } + + /* + * Setup SSOW interrupt + */ + + /* Clear SSOW interrupt enable */ + plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C); + /* Register interrupt with vfio */ + rc = dev_irq_register(handle, nix_inl_sso_hws_irq, inl_dev, + ssow_msixoff + SSOW_LF_INT_VEC_IOP); + /* Set SSOW interrupt enable */ + plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1S); + + /* + * Setup SSO/HWGRP interrupt + */ + + /* Clear SSO interrupt enable */ + plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C); + /* Register IRQ */ + rc |= dev_irq_register(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev, + sso_msixoff + SSO_LF_INT_VEC_GRP); + /* Enable hw interrupt */ + plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1S); + + /* Setup threshold for work exec interrupt to 1 wqe in IAQ */ + plt_write64(0x1ull, sso_base + SSO_LF_GGRP_INT_THR); + + return rc; +} + +void +nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev) +{ + struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle; + uintptr_t ssow_base = inl_dev->ssow_base; + uintptr_t sso_base = inl_dev->sso_base; + uint16_t sso_msixoff, ssow_msixoff; + + ssow_msixoff = inl_dev->ssow_msixoff; + sso_msixoff = inl_dev->sso_msixoff; + + /* Clear SSOW interrupt enable */ + plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C); + /* Clear SSO/HWGRP interrupt enable */ + plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C); + /* Clear SSO threshold */ + plt_write64(0, sso_base + SSO_LF_GGRP_INT_THR); + + /* Unregister IRQ */ + dev_irq_unregister(handle, nix_inl_sso_hws_irq, (void *)inl_dev, + ssow_msixoff + SSOW_LF_INT_VEC_IOP); + dev_irq_unregister(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev, + sso_msixoff + SSO_LF_INT_VEC_GRP); +} + +static void +nix_inl_nix_q_irq(void *param) +{ + struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; + uintptr_t nix_base = inl_dev->nix_base; + struct dev *dev = &inl_dev->dev; + volatile void *ctx; + uint64_t reg, intr; + uint8_t irq; + int rc; + + intr = plt_read64(nix_base + NIX_LF_QINTX_INT(0)); + if (intr == 0) + return; + + plt_err("Queue_intr=0x%" PRIx64 " qintx 0 pf=%d, vf=%d", intr, dev->pf, + dev->vf); + + /* Get and clear RQ0 interrupt */ + reg = roc_atomic64_add_nosync(0, + (int64_t *)(nix_base + NIX_LF_RQ_OP_INT)); + if (reg & BIT_ULL(42) /* OP_ERR */) { + plt_err("Failed to get rq_int"); + return; + } + irq = reg & 0xff; + plt_write64(0 | irq, nix_base + NIX_LF_RQ_OP_INT); + + if (irq & BIT_ULL(NIX_RQINT_DROP)) + plt_err("RQ=0 NIX_RQINT_DROP"); + + if (irq & BIT_ULL(NIX_RQINT_RED)) + plt_err("RQ=0 NIX_RQINT_RED"); + + /* Clear interrupt */ + plt_write64(intr, nix_base + NIX_LF_QINTX_INT(0)); + + /* Dump registers to std out */ + nix_inl_nix_reg_dump(inl_dev); + + /* Dump RQ 0 */ + rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx); + if (rc) { + plt_err("Failed to get rq context"); + return; + } + nix_lf_rq_dump(ctx); +} + +static void +nix_inl_nix_ras_irq(void *param) +{ + struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; + uintptr_t nix_base = inl_dev->nix_base; + struct dev *dev = &inl_dev->dev; + volatile void *ctx; + uint64_t intr; + int rc; + + intr = plt_read64(nix_base + NIX_LF_RAS); + if (intr == 0) + return; + + plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf); + /* Clear interrupt */ + plt_write64(intr, nix_base + NIX_LF_RAS); + + /* Dump registers to std out */ + nix_inl_nix_reg_dump(inl_dev); + + /* Dump RQ 0 */ + rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx); + if (rc) { + plt_err("Failed to get rq context"); + return; + } + nix_lf_rq_dump(ctx); +} + +static void +nix_inl_nix_err_irq(void *param) +{ + struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; + uintptr_t nix_base = inl_dev->nix_base; + struct dev *dev = &inl_dev->dev; + volatile void *ctx; + uint64_t intr; + int rc; + + intr = plt_read64(nix_base + NIX_LF_ERR_INT); + if (intr == 0) + return; + + plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf); + + /* Clear interrupt */ + plt_write64(intr, nix_base + NIX_LF_ERR_INT); + + /* Dump registers to std out */ + nix_inl_nix_reg_dump(inl_dev); + + /* Dump RQ 0 */ + rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx); + if (rc) { + plt_err("Failed to get rq context"); + return; + } + nix_lf_rq_dump(ctx); +} + +int +nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev) +{ + struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle; + uintptr_t nix_base = inl_dev->nix_base; + uint16_t msixoff; + int rc; + + msixoff = inl_dev->nix_msixoff; + if (msixoff == MSIX_VECTOR_INVALID) { + plt_err("Invalid NIXLF MSIX vector offset: 0x%x", msixoff); + return -EINVAL; + } + + /* Disable err interrupts */ + plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C); + /* DIsable RAS interrupts */ + plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C); + + /* Register err irq */ + rc = dev_irq_register(handle, nix_inl_nix_err_irq, inl_dev, + msixoff + NIX_LF_INT_VEC_ERR_INT); + rc |= dev_irq_register(handle, nix_inl_nix_ras_irq, inl_dev, + msixoff + NIX_LF_INT_VEC_POISON); + + /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */ + plt_write64(~(BIT_ULL(11) | BIT_ULL(24)), + nix_base + NIX_LF_ERR_INT_ENA_W1S); + /* Enable RAS interrupts */ + plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1S); + + /* Setup queue irq for RQ 0 */ + + /* Clear QINT CNT, interrupt */ + plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0)); + plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0)); + + /* Register queue irq vector */ + rc |= dev_irq_register(handle, nix_inl_nix_q_irq, inl_dev, + msixoff + NIX_LF_INT_VEC_QINT_START); + + plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0)); + plt_write64(0, nix_base + NIX_LF_QINTX_INT(0)); + /* Enable QINT interrupt */ + plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1S(0)); + + return rc; +} + +void +nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev) +{ + struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle; + uintptr_t nix_base = inl_dev->nix_base; + uint16_t msixoff; + + msixoff = inl_dev->nix_msixoff; + /* Disable err interrupts */ + plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C); + /* DIsable RAS interrupts */ + plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C); + + dev_irq_unregister(handle, nix_inl_nix_err_irq, inl_dev, + msixoff + NIX_LF_INT_VEC_ERR_INT); + dev_irq_unregister(handle, nix_inl_nix_ras_irq, inl_dev, + msixoff + NIX_LF_INT_VEC_POISON); + + /* Clear QINT CNT */ + plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0)); + plt_write64(0, nix_base + NIX_LF_QINTX_INT(0)); + + /* Disable QINT interrupt */ + plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0)); + + /* Unregister queue irq vector */ + dev_irq_unregister(handle, nix_inl_nix_q_irq, inl_dev, + msixoff + NIX_LF_INT_VEC_QINT_START); +} diff --git a/drivers/common/cnxk/roc_nix_inl_priv.h b/drivers/common/cnxk/roc_nix_inl_priv.h new file mode 100644 index 0000000000..f42400986d --- /dev/null +++ b/drivers/common/cnxk/roc_nix_inl_priv.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2021 Marvell. + */ +#ifndef _ROC_NIX_INL_PRIV_H_ +#define _ROC_NIX_INL_PRIV_H_ + +struct nix_inl_dev { + /* Base device object */ + struct dev dev; + + /* PCI device */ + struct plt_pci_device *pci_dev; + + /* LF specific BAR2 regions */ + uintptr_t nix_base; + uintptr_t ssow_base; + uintptr_t sso_base; + + /* MSIX vector offsets */ + uint16_t nix_msixoff; + uint16_t ssow_msixoff; + uint16_t sso_msixoff; + + /* SSO data */ + uint32_t xaq_buf_size; + uint32_t xae_waes; + uint32_t iue; + uint64_t xaq_aura; + void *xaq_mem; + roc_nix_inl_sso_work_cb_t work_cb; + void *cb_args; + + /* NIX data */ + uint8_t lf_tx_stats; + uint8_t lf_rx_stats; + uint16_t cints; + uint16_t qints; + struct roc_nix_rq rq; + uint16_t rq_refs; + bool is_nix1; + + /* NIX/CPT data */ + void *inb_sa_base; + uint16_t inb_sa_sz; + + /* Device arguments */ + uint8_t selftest; + uint16_t ipsec_in_max_spi; +}; + +int nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev); +void nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev); + +int nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev); +void nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev); + +#endif /* _ROC_NIX_INL_PRIV_H_ */ diff --git a/drivers/common/cnxk/roc_platform.h b/drivers/common/cnxk/roc_platform.h index 285b24b82d..177db3de01 100644 --- a/drivers/common/cnxk/roc_platform.h +++ b/drivers/common/cnxk/roc_platform.h @@ -113,10 +113,11 @@ #define plt_write64(val, addr) \ rte_write64_relaxed((val), (volatile void *)(addr)) -#define plt_wmb() rte_wmb() -#define plt_rmb() rte_rmb() -#define plt_io_wmb() rte_io_wmb() -#define plt_io_rmb() rte_io_rmb() +#define plt_wmb() rte_wmb() +#define plt_rmb() rte_rmb() +#define plt_io_wmb() rte_io_wmb() +#define plt_io_rmb() rte_io_rmb() +#define plt_atomic_thread_fence rte_atomic_thread_fence #define plt_mmap mmap #define PLT_PROT_READ PROT_READ diff --git a/drivers/common/cnxk/roc_priv.h b/drivers/common/cnxk/roc_priv.h index 7494b8d2a7..f72bbd568f 100644 --- a/drivers/common/cnxk/roc_priv.h +++ b/drivers/common/cnxk/roc_priv.h @@ -38,4 +38,7 @@ /* CPT */ #include "roc_cpt_priv.h" +/* NIX Inline dev */ +#include "roc_nix_inl_priv.h" + #endif /* _ROC_PRIV_H_ */