drivers/common/cnxk/roc_nix_irq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include "roc_api.h"
   6 #include "roc_priv.h"
   7
   8 static void
   9 nix_err_intr_enb_dis(struct nix *nix, bool enb)
  10 {
  11         /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */
  12         if (enb)
  13                 plt_write64(~(BIT_ULL(11) | BIT_ULL(24)),
  14                             nix->base + NIX_LF_ERR_INT_ENA_W1S);
  15         else
  16                 plt_write64(~0ull, nix->base + NIX_LF_ERR_INT_ENA_W1C);
  17 }
  18
  19 static void
  20 nix_ras_intr_enb_dis(struct nix *nix, bool enb)
  21 {
  22         if (enb)
  23                 plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1S);
  24         else
  25                 plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1C);
  26 }
  27
  28 void
  29 roc_nix_rx_queue_intr_enable(struct roc_nix *roc_nix, uint16_t rx_queue_id)
  30 {
  31         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  32
  33         /* Enable CINT interrupt */
  34         plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1S(rx_queue_id));
  35 }
  36
  37 void
  38 roc_nix_rx_queue_intr_disable(struct roc_nix *roc_nix, uint16_t rx_queue_id)
  39 {
  40         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  41
  42         /* Clear and disable CINT interrupt */
  43         plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(rx_queue_id));
  44 }
  45
  46 void
  47 roc_nix_err_intr_ena_dis(struct roc_nix *roc_nix, bool enb)
  48 {
  49         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  50
  51         return nix_err_intr_enb_dis(nix, enb);
  52 }
  53
  54 void
  55 roc_nix_ras_intr_ena_dis(struct roc_nix *roc_nix, bool enb)
  56 {
  57         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  58
  59         return nix_ras_intr_enb_dis(nix, enb);
  60 }
  61
  62 static void
  63 nix_lf_err_irq(void *param)
  64 {
  65         struct nix *nix = (struct nix *)param;
  66         struct dev *dev = &nix->dev;
  67         uint64_t intr;
  68
  69         intr = plt_read64(nix->base + NIX_LF_ERR_INT);
  70         if (intr == 0)
  71                 return;
  72
  73         plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
  74
  75         /* Clear interrupt */
  76         plt_write64(intr, nix->base + NIX_LF_ERR_INT);
  77 }
  78
  79 static int
  80 nix_lf_register_err_irq(struct nix *nix)
  81 {
  82         struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
  83         int rc, vec;
  84
  85         vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT;
  86         /* Clear err interrupt */
  87         nix_err_intr_enb_dis(nix, false);
  88         /* Set used interrupt vectors */
  89         rc = dev_irq_register(handle, nix_lf_err_irq, nix, vec);
  90         /* Enable all dev interrupt except for RQ_DISABLED */
  91         nix_err_intr_enb_dis(nix, true);
  92
  93         return rc;
  94 }
  95
  96 static void
  97 nix_lf_unregister_err_irq(struct nix *nix)
  98 {
  99         struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
 100         int vec;
 101
 102         vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT;
 103         /* Clear err interrupt */
 104         nix_err_intr_enb_dis(nix, false);
 105         dev_irq_unregister(handle, nix_lf_err_irq, nix, vec);
 106 }
 107
 108 static void
 109 nix_lf_ras_irq(void *param)
 110 {
 111         struct nix *nix = (struct nix *)param;
 112         struct dev *dev = &nix->dev;
 113         uint64_t intr;
 114
 115         intr = plt_read64(nix->base + NIX_LF_RAS);
 116         if (intr == 0)
 117                 return;
 118
 119         plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
 120         /* Clear interrupt */
 121         plt_write64(intr, nix->base + NIX_LF_RAS);
 122 }
 123
 124 static int
 125 nix_lf_register_ras_irq(struct nix *nix)
 126 {
 127         struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
 128         int rc, vec;
 129
 130         vec = nix->msixoff + NIX_LF_INT_VEC_POISON;
 131         /* Clear err interrupt */
 132         nix_ras_intr_enb_dis(nix, false);
 133         /* Set used interrupt vectors */
 134         rc = dev_irq_register(handle, nix_lf_ras_irq, nix, vec);
 135         /* Enable dev interrupt */
 136         nix_ras_intr_enb_dis(nix, true);
 137
 138         return rc;
 139 }
 140
 141 static void
 142 nix_lf_unregister_ras_irq(struct nix *nix)
 143 {
 144         struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
 145         int vec;
 146
 147         vec = nix->msixoff + NIX_LF_INT_VEC_POISON;
 148         /* Clear err interrupt */
 149         nix_ras_intr_enb_dis(nix, false);
 150         dev_irq_unregister(handle, nix_lf_ras_irq, nix, vec);
 151 }
 152
 153 static inline uint8_t
 154 nix_lf_q_irq_get_and_clear(struct nix *nix, uint16_t q, uint32_t off,
 155                            uint64_t mask)
 156 {
 157         uint64_t reg, wdata;
 158         uint8_t qint;
 159
 160         wdata = (uint64_t)q << 44;
 161         reg = roc_atomic64_add_nosync(wdata, (int64_t *)(nix->base + off));
 162
 163         if (reg & BIT_ULL(42) /* OP_ERR */) {
 164                 plt_err("Failed execute irq get off=0x%x", off);
 165                 return 0;
 166         }
 167         qint = reg & 0xff;
 168         wdata &= mask;
 169         plt_write64(wdata | qint, nix->base + off);
 170
 171         return qint;
 172 }
 173
 174 static inline uint8_t
 175 nix_lf_rq_irq_get_and_clear(struct nix *nix, uint16_t rq)
 176 {
 177         return nix_lf_q_irq_get_and_clear(nix, rq, NIX_LF_RQ_OP_INT, ~0xff00);
 178 }
 179
 180 static inline uint8_t
 181 nix_lf_cq_irq_get_and_clear(struct nix *nix, uint16_t cq)
 182 {
 183         return nix_lf_q_irq_get_and_clear(nix, cq, NIX_LF_CQ_OP_INT, ~0xff00);
 184 }
 185
 186 static inline uint8_t
 187 nix_lf_sq_irq_get_and_clear(struct nix *nix, uint16_t sq)
 188 {
 189         return nix_lf_q_irq_get_and_clear(nix, sq, NIX_LF_SQ_OP_INT, ~0x1ff00);
 190 }
 191
 192 static inline void
 193 nix_lf_sq_debug_reg(struct nix *nix, uint32_t off)
 194 {
 195         uint64_t reg;
 196
 197         reg = plt_read64(nix->base + off);
 198         if (reg & BIT_ULL(44))
 199                 plt_err("SQ=%d err_code=0x%x", (int)((reg >> 8) & 0xfffff),
 200                         (uint8_t)(reg & 0xff));
 201 }
 202
 203 static void
 204 nix_lf_cq_irq(void *param)
 205 {
 206         struct nix_qint *cint = (struct nix_qint *)param;
 207         struct nix *nix = cint->nix;
 208
 209         /* Clear interrupt */
 210         plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_INT(cint->qintx));
 211 }
 212
 213 static void
 214 nix_lf_q_irq(void *param)
 215 {
 216         struct nix_qint *qint = (struct nix_qint *)param;
 217         uint8_t irq, qintx = qint->qintx;
 218         struct nix *nix = qint->nix;
 219         struct dev *dev = &nix->dev;
 220         int q, cq, rq, sq;
 221         uint64_t intr;
 222
 223         intr = plt_read64(nix->base + NIX_LF_QINTX_INT(qintx));
 224         if (intr == 0)
 225                 return;
 226
 227         plt_err("Queue_intr=0x%" PRIx64 " qintx=%d pf=%d, vf=%d", intr, qintx,
 228                 dev->pf, dev->vf);
 229
 230         /* Handle RQ interrupts */
 231         for (q = 0; q < nix->nb_rx_queues; q++) {
 232                 rq = q % nix->qints;
 233                 irq = nix_lf_rq_irq_get_and_clear(nix, rq);
 234
 235                 if (irq & BIT_ULL(NIX_RQINT_DROP))
 236                         plt_err("RQ=%d NIX_RQINT_DROP", rq);
 237
 238                 if (irq & BIT_ULL(NIX_RQINT_RED))
 239                         plt_err("RQ=%d NIX_RQINT_RED", rq);
 240         }
 241
 242         /* Handle CQ interrupts */
 243         for (q = 0; q < nix->nb_rx_queues; q++) {
 244                 cq = q % nix->qints;
 245                 irq = nix_lf_cq_irq_get_and_clear(nix, cq);
 246
 247                 if (irq & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
 248                         plt_err("CQ=%d NIX_CQERRINT_DOOR_ERR", cq);
 249
 250                 if (irq & BIT_ULL(NIX_CQERRINT_WR_FULL))
 251                         plt_err("CQ=%d NIX_CQERRINT_WR_FULL", cq);
 252
 253                 if (irq & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
 254                         plt_err("CQ=%d NIX_CQERRINT_CQE_FAULT", cq);
 255         }
 256
 257         /* Handle SQ interrupts */
 258         for (q = 0; q < nix->nb_tx_queues; q++) {
 259                 sq = q % nix->qints;
 260                 irq = nix_lf_sq_irq_get_and_clear(nix, sq);
 261
 262                 if (irq & BIT_ULL(NIX_SQINT_LMT_ERR)) {
 263                         plt_err("SQ=%d NIX_SQINT_LMT_ERR", sq);
 264                         nix_lf_sq_debug_reg(nix, NIX_LF_SQ_OP_ERR_DBG);
 265                 }
 266                 if (irq & BIT_ULL(NIX_SQINT_MNQ_ERR)) {
 267                         plt_err("SQ=%d NIX_SQINT_MNQ_ERR", sq);
 268                         nix_lf_sq_debug_reg(nix, NIX_LF_MNQ_ERR_DBG);
 269                 }
 270                 if (irq & BIT_ULL(NIX_SQINT_SEND_ERR)) {
 271                         plt_err("SQ=%d NIX_SQINT_SEND_ERR", sq);
 272                         nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG);
 273                 }
 274                 if (irq & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) {
 275                         plt_err("SQ=%d NIX_SQINT_SQB_ALLOC_FAIL", sq);
 276                         nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG);
 277                 }
 278         }
 279
 280         /* Clear interrupt */
 281         plt_write64(intr, nix->base + NIX_LF_QINTX_INT(qintx));
 282 }
 283
 284 int
 285 roc_nix_register_queue_irqs(struct roc_nix *roc_nix)
 286 {
 287         int vec, q, sqs, rqs, qs, rc = 0;
 288         struct plt_intr_handle *handle;
 289         struct nix *nix;
 290
 291         nix = roc_nix_to_nix_priv(roc_nix);
 292         handle = &nix->pci_dev->intr_handle;
 293
 294         /* Figure out max qintx required */
 295         rqs = PLT_MIN(nix->qints, nix->nb_rx_queues);
 296         sqs = PLT_MIN(nix->qints, nix->nb_tx_queues);
 297         qs = PLT_MAX(rqs, sqs);
 298
 299         nix->configured_qints = qs;
 300
 301         nix->qints_mem =
 302                 plt_zmalloc(nix->configured_qints * sizeof(struct nix_qint), 0);
 303         if (nix->qints_mem == NULL)
 304                 return -ENOMEM;
 305
 306         for (q = 0; q < qs; q++) {
 307                 vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q;
 308
 309                 /* Clear QINT CNT */
 310                 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
 311
 312                 /* Clear interrupt */
 313                 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q));
 314
 315                 nix->qints_mem[q].nix = nix;
 316                 nix->qints_mem[q].qintx = q;
 317
 318                 /* Sync qints_mem update */
 319                 plt_wmb();
 320
 321                 /* Register queue irq vector */
 322                 rc = dev_irq_register(handle, nix_lf_q_irq, &nix->qints_mem[q],
 323                                       vec);
 324                 if (rc)
 325                         break;
 326
 327                 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
 328                 plt_write64(0, nix->base + NIX_LF_QINTX_INT(q));
 329                 /* Enable QINT interrupt */
 330                 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1S(q));
 331         }
 332
 333         return rc;
 334 }
 335
 336 void
 337 roc_nix_unregister_queue_irqs(struct roc_nix *roc_nix)
 338 {
 339         struct plt_intr_handle *handle;
 340         struct nix *nix;
 341         int vec, q;
 342
 343         nix = roc_nix_to_nix_priv(roc_nix);
 344         handle = &nix->pci_dev->intr_handle;
 345
 346         for (q = 0; q < nix->configured_qints; q++) {
 347                 vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q;
 348
 349                 /* Clear QINT CNT */
 350                 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
 351                 plt_write64(0, nix->base + NIX_LF_QINTX_INT(q));
 352
 353                 /* Clear interrupt */
 354                 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q));
 355
 356                 /* Unregister queue irq vector */
 357                 dev_irq_unregister(handle, nix_lf_q_irq, &nix->qints_mem[q],
 358                                    vec);
 359         }
 360         nix->configured_qints = 0;
 361
 362         plt_free(nix->qints_mem);
 363         nix->qints_mem = NULL;
 364 }
 365
 366 int
 367 roc_nix_register_cq_irqs(struct roc_nix *roc_nix)
 368 {
 369         struct plt_intr_handle *handle;
 370         uint8_t rc = 0, vec, q;
 371         struct nix *nix;
 372
 373         nix = roc_nix_to_nix_priv(roc_nix);
 374         handle = &nix->pci_dev->intr_handle;
 375
 376         nix->configured_cints = PLT_MIN(nix->cints, nix->nb_rx_queues);
 377
 378         nix->cints_mem =
 379                 plt_zmalloc(nix->configured_cints * sizeof(struct nix_qint), 0);
 380         if (nix->cints_mem == NULL)
 381                 return -ENOMEM;
 382
 383         for (q = 0; q < nix->configured_cints; q++) {
 384                 vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q;
 385
 386                 /* Clear CINT CNT */
 387                 plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q));
 388
 389                 /* Clear interrupt */
 390                 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q));
 391
 392                 nix->cints_mem[q].nix = nix;
 393                 nix->cints_mem[q].qintx = q;
 394
 395                 /* Sync cints_mem update */
 396                 plt_wmb();
 397
 398                 /* Register queue irq vector */
 399                 rc = dev_irq_register(handle, nix_lf_cq_irq, &nix->cints_mem[q],
 400                                       vec);
 401                 if (rc) {
 402                         plt_err("Fail to register CQ irq, rc=%d", rc);
 403                         return rc;
 404                 }
 405
 406                 if (!handle->intr_vec) {
 407                         handle->intr_vec = plt_zmalloc(
 408                                 nix->configured_cints * sizeof(int), 0);
 409                         if (!handle->intr_vec) {
 410                                 plt_err("Failed to allocate %d rx intr_vec",
 411                                         nix->configured_cints);
 412                                 return -ENOMEM;
 413                         }
 414                 }
 415                 /* VFIO vector zero is resereved for misc interrupt so
 416                  * doing required adjustment. (b13bfab4cd)
 417                  */
 418                 handle->intr_vec[q] = PLT_INTR_VEC_RXTX_OFFSET + vec;
 419
 420                 /* Configure CQE interrupt coalescing parameters */
 421                 plt_write64(((CQ_CQE_THRESH_DEFAULT) |
 422                              (CQ_CQE_THRESH_DEFAULT << 32) |
 423                              (CQ_TIMER_THRESH_DEFAULT << 48)),
 424                             nix->base + NIX_LF_CINTX_WAIT((q)));
 425
 426                 /* Keeping the CQ interrupt disabled as the rx interrupt
 427                  * feature needs to be enabled/disabled on demand.
 428                  */
 429         }
 430
 431         return rc;
 432 }
 433
 434 void
 435 roc_nix_unregister_cq_irqs(struct roc_nix *roc_nix)
 436 {
 437         struct plt_intr_handle *handle;
 438         struct nix *nix;
 439         int vec, q;
 440
 441         nix = roc_nix_to_nix_priv(roc_nix);
 442         handle = &nix->pci_dev->intr_handle;
 443
 444         for (q = 0; q < nix->configured_cints; q++) {
 445                 vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q;
 446
 447                 /* Clear CINT CNT */
 448                 plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q));
 449
 450                 /* Clear interrupt */
 451                 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q));
 452
 453                 /* Unregister queue irq vector */
 454                 dev_irq_unregister(handle, nix_lf_cq_irq, &nix->cints_mem[q],
 455                                    vec);
 456         }
 457         plt_free(nix->cints_mem);
 458 }
 459
 460 int
 461 nix_register_irqs(struct nix *nix)
 462 {
 463         int rc;
 464
 465         if (nix->msixoff == MSIX_VECTOR_INVALID) {
 466                 plt_err("Invalid NIXLF MSIX vector offset vector: 0x%x",
 467                         nix->msixoff);
 468                 return NIX_ERR_PARAM;
 469         }
 470
 471         /* Register lf err interrupt */
 472         rc = nix_lf_register_err_irq(nix);
 473         /* Register RAS interrupt */
 474         rc |= nix_lf_register_ras_irq(nix);
 475
 476         return rc;
 477 }
 478
 479 void
 480 nix_unregister_irqs(struct nix *nix)
 481 {
 482         nix_lf_unregister_err_irq(nix);
 483         nix_lf_unregister_ras_irq(nix);
 484 }