drivers/common/cnxk/roc_nix_inl_dev_irq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include "roc_api.h"
   6 #include "roc_priv.h"
   7
   8 #define WORK_LIMIT 1000
   9
  10 static void
  11 nix_inl_sso_work_cb(struct nix_inl_dev *inl_dev)
  12 {
  13         uintptr_t getwrk_op = inl_dev->ssow_base + SSOW_LF_GWS_OP_GET_WORK0;
  14         uintptr_t tag_wqe_op = inl_dev->ssow_base + SSOW_LF_GWS_WQE0;
  15         uint32_t wdata = BIT(16) | 1;
  16         union {
  17                 __uint128_t get_work;
  18                 uint64_t u64[2];
  19         } gw;
  20         uint16_t cnt = 0;
  21         uint64_t work;
  22
  23 again:
  24         /* Try to do get work */
  25         gw.get_work = wdata;
  26         plt_write64(gw.u64[0], getwrk_op);
  27         do {
  28                 roc_load_pair(gw.u64[0], gw.u64[1], tag_wqe_op);
  29         } while (gw.u64[0] & BIT_ULL(63));
  30
  31         work = gw.u64[1];
  32         /* Do we have any work? */
  33         if (work) {
  34                 if (inl_dev->work_cb)
  35                         inl_dev->work_cb(gw.u64, inl_dev->cb_args, false);
  36                 else
  37                         plt_warn("Undelivered inl dev work gw0: %p gw1: %p",
  38                                  (void *)gw.u64[0], (void *)gw.u64[1]);
  39                 cnt++;
  40                 if (cnt < WORK_LIMIT)
  41                         goto again;
  42         }
  43
  44         plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
  45 }
  46
  47 static int
  48 nix_inl_nix_reg_dump(struct nix_inl_dev *inl_dev)
  49 {
  50         uintptr_t nix_base = inl_dev->nix_base;
  51
  52         /* General registers */
  53         nix_lf_gen_reg_dump(nix_base, NULL);
  54
  55         /* Rx, Tx stat registers */
  56         nix_lf_stat_reg_dump(nix_base, NULL, inl_dev->lf_tx_stats,
  57                              inl_dev->lf_rx_stats);
  58
  59         /* Intr registers */
  60         nix_lf_int_reg_dump(nix_base, NULL, inl_dev->qints, inl_dev->cints);
  61
  62         return 0;
  63 }
  64
  65 static void
  66 nix_inl_sso_hwgrp_irq(void *param)
  67 {
  68         struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
  69         uintptr_t sso_base = inl_dev->sso_base;
  70         uint64_t intr;
  71
  72         intr = plt_read64(sso_base + SSO_LF_GGRP_INT);
  73         if (intr == 0)
  74                 return;
  75
  76         /* Check for work executable interrupt */
  77         if (intr & BIT(1))
  78                 nix_inl_sso_work_cb(inl_dev);
  79
  80         if (intr & ~BIT(1))
  81                 plt_err("GGRP 0 GGRP_INT=0x%" PRIx64 "", intr);
  82
  83         /* Clear interrupt */
  84         plt_write64(intr, sso_base + SSO_LF_GGRP_INT);
  85 }
  86
  87 static void
  88 nix_inl_sso_hws_irq(void *param)
  89 {
  90         struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
  91         uintptr_t ssow_base = inl_dev->ssow_base;
  92         uint64_t intr;
  93
  94         intr = plt_read64(ssow_base + SSOW_LF_GWS_INT);
  95         if (intr == 0)
  96                 return;
  97
  98         plt_err("GWS 0 GWS_INT=0x%" PRIx64 "", intr);
  99
 100         /* Clear interrupt */
 101         plt_write64(intr, ssow_base + SSOW_LF_GWS_INT);
 102 }
 103
 104 int
 105 nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev)
 106 {
 107         struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle;
 108         uintptr_t ssow_base = inl_dev->ssow_base;
 109         uintptr_t sso_base = inl_dev->sso_base;
 110         uint16_t sso_msixoff, ssow_msixoff;
 111         int rc;
 112
 113         ssow_msixoff = inl_dev->ssow_msixoff;
 114         sso_msixoff = inl_dev->sso_msixoff;
 115         if (sso_msixoff == MSIX_VECTOR_INVALID ||
 116             ssow_msixoff == MSIX_VECTOR_INVALID) {
 117                 plt_err("Invalid SSO/SSOW MSIX offsets (0x%x, 0x%x)",
 118                         sso_msixoff, ssow_msixoff);
 119                 return -EINVAL;
 120         }
 121
 122         /*
 123          * Setup SSOW interrupt
 124          */
 125
 126         /* Clear SSOW interrupt enable */
 127         plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C);
 128         /* Register interrupt with vfio */
 129         rc = dev_irq_register(handle, nix_inl_sso_hws_irq, inl_dev,
 130                               ssow_msixoff + SSOW_LF_INT_VEC_IOP);
 131         /* Set SSOW interrupt enable */
 132         plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1S);
 133
 134         /*
 135          * Setup SSO/HWGRP interrupt
 136          */
 137
 138         /* Clear SSO interrupt enable */
 139         plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C);
 140         /* Register IRQ */
 141         rc |= dev_irq_register(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev,
 142                                sso_msixoff + SSO_LF_INT_VEC_GRP);
 143         /* Enable hw interrupt */
 144         plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1S);
 145
 146         /* Setup threshold for work exec interrupt to 100us timeout
 147          * based on time counter.
 148          */
 149         plt_write64(BIT_ULL(63) | 10ULL << 48, sso_base + SSO_LF_GGRP_INT_THR);
 150
 151         return rc;
 152 }
 153
 154 void
 155 nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev)
 156 {
 157         struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle;
 158         uintptr_t ssow_base = inl_dev->ssow_base;
 159         uintptr_t sso_base = inl_dev->sso_base;
 160         uint16_t sso_msixoff, ssow_msixoff;
 161
 162         ssow_msixoff = inl_dev->ssow_msixoff;
 163         sso_msixoff = inl_dev->sso_msixoff;
 164
 165         /* Clear SSOW interrupt enable */
 166         plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C);
 167         /* Clear SSO/HWGRP interrupt enable */
 168         plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C);
 169         /* Clear SSO threshold */
 170         plt_write64(0, sso_base + SSO_LF_GGRP_INT_THR);
 171
 172         /* Unregister IRQ */
 173         dev_irq_unregister(handle, nix_inl_sso_hws_irq, (void *)inl_dev,
 174                            ssow_msixoff + SSOW_LF_INT_VEC_IOP);
 175         dev_irq_unregister(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev,
 176                            sso_msixoff + SSO_LF_INT_VEC_GRP);
 177 }
 178
 179 static void
 180 nix_inl_nix_q_irq(void *param)
 181 {
 182         struct nix_inl_qint *qints_mem = (struct nix_inl_qint *)param;
 183         struct nix_inl_dev *inl_dev = qints_mem->inl_dev;
 184         uintptr_t nix_base = inl_dev->nix_base;
 185         struct dev *dev = &inl_dev->dev;
 186         uint16_t qint = qints_mem->qint;
 187         volatile void *ctx;
 188         uint64_t reg, intr;
 189         uint64_t wdata;
 190         uint8_t irq;
 191         int rc, q;
 192
 193         intr = plt_read64(nix_base + NIX_LF_QINTX_INT(qint));
 194         if (intr == 0)
 195                 return;
 196
 197         plt_err("Queue_intr=0x%" PRIx64 " qintx 0 pf=%d, vf=%d", intr, dev->pf,
 198                 dev->vf);
 199
 200         /* Handle RQ interrupts */
 201         for (q = 0; q < inl_dev->nb_rqs; q++) {
 202                 /* Get and clear RQ interrupts */
 203                 wdata = (uint64_t)q << 44;
 204                 reg = roc_atomic64_add_nosync(wdata,
 205                                               (int64_t *)(nix_base + NIX_LF_RQ_OP_INT));
 206                 if (reg & BIT_ULL(42) /* OP_ERR */) {
 207                         plt_err("Failed to get rq_int");
 208                         return;
 209                 }
 210                 irq = reg & 0xff;
 211                 plt_write64(wdata | irq, nix_base + NIX_LF_RQ_OP_INT);
 212
 213                 if (irq & BIT_ULL(NIX_RQINT_DROP))
 214                         plt_err("RQ=0 NIX_RQINT_DROP");
 215
 216                 if (irq & BIT_ULL(NIX_RQINT_RED))
 217                         plt_err("RQ=0 NIX_RQINT_RED");
 218         }
 219
 220         /* Clear interrupt */
 221         plt_write64(intr, nix_base + NIX_LF_QINTX_INT(qint));
 222
 223         /* Dump registers to std out */
 224         nix_inl_nix_reg_dump(inl_dev);
 225
 226         /* Dump RQs */
 227         for (q = 0; q < inl_dev->nb_rqs; q++) {
 228                 rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, q, &ctx);
 229                 if (rc) {
 230                         plt_err("Failed to get rq %d context, rc=%d", q, rc);
 231                         continue;
 232                 }
 233                 nix_lf_rq_dump(ctx);
 234         }
 235 }
 236
 237 static void
 238 nix_inl_nix_ras_irq(void *param)
 239 {
 240         struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
 241         uintptr_t nix_base = inl_dev->nix_base;
 242         struct dev *dev = &inl_dev->dev;
 243         volatile void *ctx;
 244         uint64_t intr;
 245         int rc, q;
 246
 247         intr = plt_read64(nix_base + NIX_LF_RAS);
 248         if (intr == 0)
 249                 return;
 250
 251         plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
 252         /* Clear interrupt */
 253         plt_write64(intr, nix_base + NIX_LF_RAS);
 254
 255         /* Dump registers to std out */
 256         nix_inl_nix_reg_dump(inl_dev);
 257
 258         /* Dump RQs */
 259         for (q = 0; q < inl_dev->nb_rqs; q++) {
 260                 rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, q, &ctx);
 261                 if (rc) {
 262                         plt_err("Failed to get rq %d context, rc=%d", q, rc);
 263                         continue;
 264                 }
 265                 nix_lf_rq_dump(ctx);
 266         }
 267 }
 268
 269 static void
 270 nix_inl_nix_err_irq(void *param)
 271 {
 272         struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
 273         uintptr_t nix_base = inl_dev->nix_base;
 274         struct dev *dev = &inl_dev->dev;
 275         volatile void *ctx;
 276         uint64_t intr;
 277         int rc, q;
 278
 279         intr = plt_read64(nix_base + NIX_LF_ERR_INT);
 280         if (intr == 0)
 281                 return;
 282
 283         plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
 284
 285         /* Clear interrupt */
 286         plt_write64(intr, nix_base + NIX_LF_ERR_INT);
 287
 288         /* Dump registers to std out */
 289         nix_inl_nix_reg_dump(inl_dev);
 290
 291         /* Dump RQs */
 292         for (q = 0; q < inl_dev->nb_rqs; q++) {
 293                 rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, q, &ctx);
 294                 if (rc) {
 295                         plt_err("Failed to get rq %d context, rc=%d", q, rc);
 296                         continue;
 297                 }
 298                 nix_lf_rq_dump(ctx);
 299         }
 300 }
 301
 302 int
 303 nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev)
 304 {
 305         struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle;
 306         uintptr_t nix_base = inl_dev->nix_base;
 307         struct nix_inl_qint *qints_mem;
 308         int rc, q, ret = 0;
 309         uint16_t msixoff;
 310         int qints;
 311
 312         msixoff = inl_dev->nix_msixoff;
 313         if (msixoff == MSIX_VECTOR_INVALID) {
 314                 plt_err("Invalid NIXLF MSIX vector offset: 0x%x", msixoff);
 315                 return -EINVAL;
 316         }
 317
 318         /* Disable err interrupts */
 319         plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C);
 320         /* DIsable RAS interrupts */
 321         plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C);
 322
 323         /* Register err irq */
 324         rc = dev_irq_register(handle, nix_inl_nix_err_irq, inl_dev,
 325                               msixoff + NIX_LF_INT_VEC_ERR_INT);
 326         rc |= dev_irq_register(handle, nix_inl_nix_ras_irq, inl_dev,
 327                                msixoff + NIX_LF_INT_VEC_POISON);
 328
 329         /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */
 330         plt_write64(~(BIT_ULL(11) | BIT_ULL(24)),
 331                     nix_base + NIX_LF_ERR_INT_ENA_W1S);
 332         /* Enable RAS interrupts */
 333         plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1S);
 334
 335         /* Setup queue irq for RQ's */
 336         qints = PLT_MIN(inl_dev->nb_rqs, inl_dev->qints);
 337         qints_mem = plt_zmalloc(sizeof(struct nix_inl_qint) * qints, 0);
 338         if (!qints_mem) {
 339                 plt_err("Failed to allocate memory for %u qints", qints);
 340                 return -ENOMEM;
 341         }
 342
 343         inl_dev->configured_qints = qints;
 344         inl_dev->qints_mem = qints_mem;
 345
 346         for (q = 0; q < qints; q++) {
 347                 /* Clear QINT CNT, interrupt */
 348                 plt_write64(0, nix_base + NIX_LF_QINTX_CNT(q));
 349                 plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(q));
 350
 351                 /* Register queue irq vector */
 352                 ret = dev_irq_register(handle, nix_inl_nix_q_irq, &qints_mem[q],
 353                                        msixoff + NIX_LF_INT_VEC_QINT_START + q);
 354                 if (ret)
 355                         break;
 356
 357                 plt_write64(0, nix_base + NIX_LF_QINTX_CNT(q));
 358                 plt_write64(0, nix_base + NIX_LF_QINTX_INT(q));
 359                 /* Enable QINT interrupt */
 360                 plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1S(q));
 361
 362                 qints_mem[q].inl_dev = inl_dev;
 363                 qints_mem[q].qint = q;
 364         }
 365
 366         rc |= ret;
 367         return rc;
 368 }
 369
 370 void
 371 nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev)
 372 {
 373         struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle;
 374         struct nix_inl_qint *qints_mem = inl_dev->qints_mem;
 375         uintptr_t nix_base = inl_dev->nix_base;
 376         uint16_t msixoff;
 377         int q;
 378
 379         msixoff = inl_dev->nix_msixoff;
 380         /* Disable err interrupts */
 381         plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C);
 382         /* DIsable RAS interrupts */
 383         plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C);
 384
 385         dev_irq_unregister(handle, nix_inl_nix_err_irq, inl_dev,
 386                            msixoff + NIX_LF_INT_VEC_ERR_INT);
 387         dev_irq_unregister(handle, nix_inl_nix_ras_irq, inl_dev,
 388                            msixoff + NIX_LF_INT_VEC_POISON);
 389
 390         for (q = 0; q < inl_dev->configured_qints; q++) {
 391                 /* Clear QINT CNT */
 392                 plt_write64(0, nix_base + NIX_LF_QINTX_CNT(q));
 393                 plt_write64(0, nix_base + NIX_LF_QINTX_INT(q));
 394
 395                 /* Disable QINT interrupt */
 396                 plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(q));
 397
 398                 /* Unregister queue irq vector */
 399                 dev_irq_unregister(handle, nix_inl_nix_q_irq, &qints_mem[q],
 400                                    msixoff + NIX_LF_INT_VEC_QINT_START + q);
 401         }
 402
 403         plt_free(inl_dev->qints_mem);
 404         inl_dev->qints_mem = NULL;
 405 }