1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
9 nix_err_intr_enb_dis(struct nix *nix, bool enb)
11 /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */
13 plt_write64(~(BIT_ULL(11) | BIT_ULL(24)),
14 nix->base + NIX_LF_ERR_INT_ENA_W1S);
16 plt_write64(~0ull, nix->base + NIX_LF_ERR_INT_ENA_W1C);
20 nix_ras_intr_enb_dis(struct nix *nix, bool enb)
23 plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1S);
25 plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1C);
29 roc_nix_rx_queue_intr_enable(struct roc_nix *roc_nix, uint16_t rx_queue_id)
31 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
33 /* Enable CINT interrupt */
34 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1S(rx_queue_id));
38 roc_nix_rx_queue_intr_disable(struct roc_nix *roc_nix, uint16_t rx_queue_id)
40 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
42 /* Clear and disable CINT interrupt */
43 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(rx_queue_id));
47 roc_nix_err_intr_ena_dis(struct roc_nix *roc_nix, bool enb)
49 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
51 return nix_err_intr_enb_dis(nix, enb);
55 roc_nix_ras_intr_ena_dis(struct roc_nix *roc_nix, bool enb)
57 struct nix *nix = roc_nix_to_nix_priv(roc_nix);
59 return nix_ras_intr_enb_dis(nix, enb);
63 nix_lf_err_irq(void *param)
65 struct nix *nix = (struct nix *)param;
66 struct dev *dev = &nix->dev;
69 intr = plt_read64(nix->base + NIX_LF_ERR_INT);
73 plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
76 plt_write64(intr, nix->base + NIX_LF_ERR_INT);
77 /* Dump registers to std out */
78 roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL);
79 roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix));
83 nix_lf_register_err_irq(struct nix *nix)
85 struct plt_intr_handle *handle = nix->pci_dev->intr_handle;
88 vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT;
89 /* Clear err interrupt */
90 nix_err_intr_enb_dis(nix, false);
91 /* Set used interrupt vectors */
92 rc = dev_irq_register(handle, nix_lf_err_irq, nix, vec);
93 /* Enable all dev interrupt except for RQ_DISABLED */
94 nix_err_intr_enb_dis(nix, true);
100 nix_lf_unregister_err_irq(struct nix *nix)
102 struct plt_intr_handle *handle = nix->pci_dev->intr_handle;
105 vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT;
106 /* Clear err interrupt */
107 nix_err_intr_enb_dis(nix, false);
108 dev_irq_unregister(handle, nix_lf_err_irq, nix, vec);
112 nix_lf_ras_irq(void *param)
114 struct nix *nix = (struct nix *)param;
115 struct dev *dev = &nix->dev;
118 intr = plt_read64(nix->base + NIX_LF_RAS);
122 plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
123 /* Clear interrupt */
124 plt_write64(intr, nix->base + NIX_LF_RAS);
126 /* Dump registers to std out */
127 roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL);
128 roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix));
132 nix_lf_register_ras_irq(struct nix *nix)
134 struct plt_intr_handle *handle = nix->pci_dev->intr_handle;
137 vec = nix->msixoff + NIX_LF_INT_VEC_POISON;
138 /* Clear err interrupt */
139 nix_ras_intr_enb_dis(nix, false);
140 /* Set used interrupt vectors */
141 rc = dev_irq_register(handle, nix_lf_ras_irq, nix, vec);
142 /* Enable dev interrupt */
143 nix_ras_intr_enb_dis(nix, true);
149 nix_lf_unregister_ras_irq(struct nix *nix)
151 struct plt_intr_handle *handle = nix->pci_dev->intr_handle;
154 vec = nix->msixoff + NIX_LF_INT_VEC_POISON;
155 /* Clear err interrupt */
156 nix_ras_intr_enb_dis(nix, false);
157 dev_irq_unregister(handle, nix_lf_ras_irq, nix, vec);
160 static inline uint8_t
161 nix_lf_q_irq_get_and_clear(struct nix *nix, uint16_t q, uint32_t off,
167 wdata = (uint64_t)q << 44;
168 reg = roc_atomic64_add_nosync(wdata, (int64_t *)(nix->base + off));
170 if (reg & BIT_ULL(42) /* OP_ERR */) {
171 plt_err("Failed execute irq get off=0x%x", off);
176 plt_write64(wdata | qint, nix->base + off);
181 static inline uint8_t
182 nix_lf_rq_irq_get_and_clear(struct nix *nix, uint16_t rq)
184 return nix_lf_q_irq_get_and_clear(nix, rq, NIX_LF_RQ_OP_INT, ~0xff00);
187 static inline uint8_t
188 nix_lf_cq_irq_get_and_clear(struct nix *nix, uint16_t cq)
190 return nix_lf_q_irq_get_and_clear(nix, cq, NIX_LF_CQ_OP_INT, ~0xff00);
193 static inline uint8_t
194 nix_lf_sq_irq_get_and_clear(struct nix *nix, uint16_t sq)
196 return nix_lf_q_irq_get_and_clear(nix, sq, NIX_LF_SQ_OP_INT, ~0x1ff00);
200 nix_lf_sq_debug_reg(struct nix *nix, uint32_t off)
204 reg = plt_read64(nix->base + off);
205 if (reg & BIT_ULL(44)) {
206 plt_err("SQ=%d err_code=0x%x", (int)((reg >> 8) & 0xfffff),
207 (uint8_t)(reg & 0xff));
208 /* Clear valid bit */
209 plt_write64(BIT_ULL(44), nix->base + off);
214 nix_lf_cq_irq(void *param)
216 struct nix_qint *cint = (struct nix_qint *)param;
217 struct nix *nix = cint->nix;
219 /* Clear interrupt */
220 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_INT(cint->qintx));
224 nix_lf_q_irq(void *param)
226 struct nix_qint *qint = (struct nix_qint *)param;
227 uint8_t irq, qintx = qint->qintx;
228 struct nix *nix = qint->nix;
229 struct dev *dev = &nix->dev;
233 intr = plt_read64(nix->base + NIX_LF_QINTX_INT(qintx));
237 plt_err("Queue_intr=0x%" PRIx64 " qintx=%d pf=%d, vf=%d", intr, qintx,
240 /* Handle RQ interrupts */
241 for (q = 0; q < nix->nb_rx_queues; q++) {
243 irq = nix_lf_rq_irq_get_and_clear(nix, rq);
245 if (irq & BIT_ULL(NIX_RQINT_DROP))
246 plt_err("RQ=%d NIX_RQINT_DROP", rq);
248 if (irq & BIT_ULL(NIX_RQINT_RED))
249 plt_err("RQ=%d NIX_RQINT_RED", rq);
252 /* Handle CQ interrupts */
253 for (q = 0; q < nix->nb_rx_queues; q++) {
255 irq = nix_lf_cq_irq_get_and_clear(nix, cq);
257 if (irq & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
258 plt_err("CQ=%d NIX_CQERRINT_DOOR_ERR", cq);
260 if (irq & BIT_ULL(NIX_CQERRINT_WR_FULL))
261 plt_err("CQ=%d NIX_CQERRINT_WR_FULL", cq);
263 if (irq & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
264 plt_err("CQ=%d NIX_CQERRINT_CQE_FAULT", cq);
267 /* Handle SQ interrupts */
268 for (q = 0; q < nix->nb_tx_queues; q++) {
270 irq = nix_lf_sq_irq_get_and_clear(nix, sq);
272 if (irq & BIT_ULL(NIX_SQINT_LMT_ERR)) {
273 plt_err("SQ=%d NIX_SQINT_LMT_ERR", sq);
274 nix_lf_sq_debug_reg(nix, NIX_LF_SQ_OP_ERR_DBG);
276 if (irq & BIT_ULL(NIX_SQINT_MNQ_ERR)) {
277 plt_err("SQ=%d NIX_SQINT_MNQ_ERR", sq);
278 nix_lf_sq_debug_reg(nix, NIX_LF_MNQ_ERR_DBG);
280 if (irq & BIT_ULL(NIX_SQINT_SEND_ERR)) {
281 plt_err("SQ=%d NIX_SQINT_SEND_ERR", sq);
282 nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG);
284 if (irq & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) {
285 plt_err("SQ=%d NIX_SQINT_SQB_ALLOC_FAIL", sq);
286 nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG);
290 /* Clear interrupt */
291 plt_write64(intr, nix->base + NIX_LF_QINTX_INT(qintx));
293 /* Dump registers to std out */
294 roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL);
295 roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix));
299 roc_nix_register_queue_irqs(struct roc_nix *roc_nix)
301 int vec, q, sqs, rqs, qs, rc = 0;
302 struct plt_intr_handle *handle;
305 nix = roc_nix_to_nix_priv(roc_nix);
306 handle = nix->pci_dev->intr_handle;
308 /* Figure out max qintx required */
309 rqs = PLT_MIN(nix->qints, nix->nb_rx_queues);
310 sqs = PLT_MIN(nix->qints, nix->nb_tx_queues);
311 qs = PLT_MAX(rqs, sqs);
313 nix->configured_qints = qs;
316 plt_zmalloc(nix->configured_qints * sizeof(struct nix_qint), 0);
317 if (nix->qints_mem == NULL)
320 for (q = 0; q < qs; q++) {
321 vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q;
324 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
326 /* Clear interrupt */
327 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q));
329 nix->qints_mem[q].nix = nix;
330 nix->qints_mem[q].qintx = q;
332 /* Sync qints_mem update */
335 /* Register queue irq vector */
336 rc = dev_irq_register(handle, nix_lf_q_irq, &nix->qints_mem[q],
341 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
342 plt_write64(0, nix->base + NIX_LF_QINTX_INT(q));
343 /* Enable QINT interrupt */
344 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1S(q));
351 roc_nix_unregister_queue_irqs(struct roc_nix *roc_nix)
353 struct plt_intr_handle *handle;
357 nix = roc_nix_to_nix_priv(roc_nix);
358 handle = nix->pci_dev->intr_handle;
360 for (q = 0; q < nix->configured_qints; q++) {
361 vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q;
364 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
365 plt_write64(0, nix->base + NIX_LF_QINTX_INT(q));
367 /* Clear interrupt */
368 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q));
370 /* Unregister queue irq vector */
371 dev_irq_unregister(handle, nix_lf_q_irq, &nix->qints_mem[q],
374 nix->configured_qints = 0;
376 plt_free(nix->qints_mem);
377 nix->qints_mem = NULL;
381 roc_nix_register_cq_irqs(struct roc_nix *roc_nix)
383 struct plt_intr_handle *handle;
384 uint8_t rc = 0, vec, q;
387 nix = roc_nix_to_nix_priv(roc_nix);
388 handle = nix->pci_dev->intr_handle;
390 nix->configured_cints = PLT_MIN(nix->cints, nix->nb_rx_queues);
393 plt_zmalloc(nix->configured_cints * sizeof(struct nix_qint), 0);
394 if (nix->cints_mem == NULL)
397 for (q = 0; q < nix->configured_cints; q++) {
398 vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q;
401 plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q));
403 /* Clear interrupt */
404 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q));
406 nix->cints_mem[q].nix = nix;
407 nix->cints_mem[q].qintx = q;
409 /* Sync cints_mem update */
412 /* Register queue irq vector */
413 rc = dev_irq_register(handle, nix_lf_cq_irq, &nix->cints_mem[q],
416 plt_err("Fail to register CQ irq, rc=%d", rc);
420 rc = plt_intr_vec_list_alloc(handle, "cnxk",
421 nix->configured_cints);
423 plt_err("Fail to allocate intr vec list, rc=%d",
427 /* VFIO vector zero is reserved for misc interrupt so
428 * doing required adjustment. (b13bfab4cd)
430 if (plt_intr_vec_list_index_set(handle, q,
431 PLT_INTR_VEC_RXTX_OFFSET + vec))
434 /* Configure CQE interrupt coalescing parameters */
435 plt_write64(((CQ_CQE_THRESH_DEFAULT) |
436 (CQ_CQE_THRESH_DEFAULT << 32) |
437 (CQ_TIMER_THRESH_DEFAULT << 48)),
438 nix->base + NIX_LF_CINTX_WAIT((q)));
440 /* Keeping the CQ interrupt disabled as the rx interrupt
441 * feature needs to be enabled/disabled on demand.
449 roc_nix_unregister_cq_irqs(struct roc_nix *roc_nix)
451 struct plt_intr_handle *handle;
455 nix = roc_nix_to_nix_priv(roc_nix);
456 handle = nix->pci_dev->intr_handle;
458 for (q = 0; q < nix->configured_cints; q++) {
459 vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q;
462 plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q));
464 /* Clear interrupt */
465 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q));
467 /* Unregister queue irq vector */
468 dev_irq_unregister(handle, nix_lf_cq_irq, &nix->cints_mem[q],
472 plt_intr_vec_list_free(handle);
473 plt_free(nix->cints_mem);
477 nix_register_irqs(struct nix *nix)
481 if (nix->msixoff == MSIX_VECTOR_INVALID) {
482 plt_err("Invalid NIXLF MSIX vector offset vector: 0x%x",
484 return NIX_ERR_PARAM;
487 /* Register lf err interrupt */
488 rc = nix_lf_register_err_irq(nix);
489 /* Register RAS interrupt */
490 rc |= nix_lf_register_ras_irq(nix);
496 nix_unregister_irqs(struct nix *nix)
498 nix_lf_unregister_err_irq(nix);
499 nix_lf_unregister_ras_irq(nix);