drivers/common/sfc_efx/base/efx_ev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2020 Xilinx, Inc.
   4  * Copyright(c) 2007-2019 Solarflare Communications Inc.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9 #if EFSYS_OPT_MON_MCDI
  10 #include "mcdi_mon.h"
  11 #endif
  12
  13 #define EFX_EV_PRESENT(_qword)                                          \
  14         (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&        \
  15         EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
  16
  17
  18
  19 #if EFSYS_OPT_SIENA
  20
  21 static  __checkReturn   efx_rc_t
  22 siena_ev_init(
  23         __in            efx_nic_t *enp);
  24
  25 static                  void
  26 siena_ev_fini(
  27         __in            efx_nic_t *enp);
  28
  29 static  __checkReturn   efx_rc_t
  30 siena_ev_qcreate(
  31         __in            efx_nic_t *enp,
  32         __in            unsigned int index,
  33         __in            efsys_mem_t *esmp,
  34         __in            size_t ndescs,
  35         __in            uint32_t id,
  36         __in            uint32_t us,
  37         __in            uint32_t flags,
  38         __in            efx_evq_t *eep);
  39
  40 static                  void
  41 siena_ev_qdestroy(
  42         __in            efx_evq_t *eep);
  43
  44 static  __checkReturn   efx_rc_t
  45 siena_ev_qprime(
  46         __in            efx_evq_t *eep,
  47         __in            unsigned int count);
  48
  49 static                  void
  50 siena_ev_qpost(
  51         __in    efx_evq_t *eep,
  52         __in    uint16_t data);
  53
  54 static  __checkReturn   efx_rc_t
  55 siena_ev_qmoderate(
  56         __in            efx_evq_t *eep,
  57         __in            unsigned int us);
  58
  59 #if EFSYS_OPT_QSTATS
  60 static                  void
  61 siena_ev_qstats_update(
  62         __in                            efx_evq_t *eep,
  63         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat);
  64
  65 #endif
  66
  67 #endif /* EFSYS_OPT_SIENA */
  68
  69 #if EFX_OPTS_EF10() || EFSYS_OPT_SIENA
  70
  71 static                  void
  72 siena_ef10_ev_qpoll(
  73         __in            efx_evq_t *eep,
  74         __inout         unsigned int *countp,
  75         __in            const efx_ev_callbacks_t *eecp,
  76         __in_opt        void *arg);
  77
  78 #endif  /* EFX_OPTS_EF10() || EFSYS_OPT_SIENA */
  79
  80 #if EFSYS_OPT_SIENA
  81 static const efx_ev_ops_t       __efx_ev_siena_ops = {
  82         siena_ev_init,                          /* eevo_init */
  83         siena_ev_fini,                          /* eevo_fini */
  84         siena_ev_qcreate,                       /* eevo_qcreate */
  85         siena_ev_qdestroy,                      /* eevo_qdestroy */
  86         siena_ev_qprime,                        /* eevo_qprime */
  87         siena_ev_qpost,                         /* eevo_qpost */
  88         siena_ef10_ev_qpoll,                    /* eevo_qpoll */
  89         siena_ev_qmoderate,                     /* eevo_qmoderate */
  90 #if EFSYS_OPT_QSTATS
  91         siena_ev_qstats_update,                 /* eevo_qstats_update */
  92 #endif
  93 };
  94 #endif /* EFSYS_OPT_SIENA */
  95
  96 #if EFX_OPTS_EF10()
  97 static const efx_ev_ops_t       __efx_ev_ef10_ops = {
  98         ef10_ev_init,                           /* eevo_init */
  99         ef10_ev_fini,                           /* eevo_fini */
 100         ef10_ev_qcreate,                        /* eevo_qcreate */
 101         ef10_ev_qdestroy,                       /* eevo_qdestroy */
 102         ef10_ev_qprime,                         /* eevo_qprime */
 103         ef10_ev_qpost,                          /* eevo_qpost */
 104         siena_ef10_ev_qpoll,                    /* eevo_qpoll */
 105         ef10_ev_qmoderate,                      /* eevo_qmoderate */
 106 #if EFSYS_OPT_QSTATS
 107         ef10_ev_qstats_update,                  /* eevo_qstats_update */
 108 #endif
 109 };
 110 #endif /* EFX_OPTS_EF10() */
 111
 112
 113         __checkReturn   efx_rc_t
 114 efx_ev_init(
 115         __in            efx_nic_t *enp)
 116 {
 117         const efx_ev_ops_t *eevop;
 118         efx_rc_t rc;
 119
 120         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 121         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 122
 123         if (enp->en_mod_flags & EFX_MOD_EV) {
 124                 rc = EINVAL;
 125                 goto fail1;
 126         }
 127
 128         switch (enp->en_family) {
 129 #if EFSYS_OPT_SIENA
 130         case EFX_FAMILY_SIENA:
 131                 eevop = &__efx_ev_siena_ops;
 132                 break;
 133 #endif /* EFSYS_OPT_SIENA */
 134
 135 #if EFSYS_OPT_HUNTINGTON
 136         case EFX_FAMILY_HUNTINGTON:
 137                 eevop = &__efx_ev_ef10_ops;
 138                 break;
 139 #endif /* EFSYS_OPT_HUNTINGTON */
 140
 141 #if EFSYS_OPT_MEDFORD
 142         case EFX_FAMILY_MEDFORD:
 143                 eevop = &__efx_ev_ef10_ops;
 144                 break;
 145 #endif /* EFSYS_OPT_MEDFORD */
 146
 147 #if EFSYS_OPT_MEDFORD2
 148         case EFX_FAMILY_MEDFORD2:
 149                 eevop = &__efx_ev_ef10_ops;
 150                 break;
 151 #endif /* EFSYS_OPT_MEDFORD2 */
 152
 153         default:
 154                 EFSYS_ASSERT(0);
 155                 rc = ENOTSUP;
 156                 goto fail1;
 157         }
 158
 159         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 160
 161         if ((rc = eevop->eevo_init(enp)) != 0)
 162                 goto fail2;
 163
 164         enp->en_eevop = eevop;
 165         enp->en_mod_flags |= EFX_MOD_EV;
 166         return (0);
 167
 168 fail2:
 169         EFSYS_PROBE(fail2);
 170
 171 fail1:
 172         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 173
 174         enp->en_eevop = NULL;
 175         enp->en_mod_flags &= ~EFX_MOD_EV;
 176         return (rc);
 177 }
 178
 179         __checkReturn   size_t
 180 efx_evq_size(
 181         __in    const efx_nic_t *enp,
 182         __in    unsigned int ndescs)
 183 {
 184         const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
 185
 186         return (ndescs * encp->enc_ev_desc_size);
 187 }
 188
 189         __checkReturn   unsigned int
 190 efx_evq_nbufs(
 191         __in    const efx_nic_t *enp,
 192         __in    unsigned int ndescs)
 193 {
 194         return (EFX_DIV_ROUND_UP(efx_evq_size(enp, ndescs), EFX_BUF_SIZE));
 195 }
 196
 197                 void
 198 efx_ev_fini(
 199         __in    efx_nic_t *enp)
 200 {
 201         const efx_ev_ops_t *eevop = enp->en_eevop;
 202
 203         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 204         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
 205         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 206         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
 207         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
 208         EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
 209
 210         eevop->eevo_fini(enp);
 211
 212         enp->en_eevop = NULL;
 213         enp->en_mod_flags &= ~EFX_MOD_EV;
 214 }
 215
 216
 217         __checkReturn   efx_rc_t
 218 efx_ev_qcreate(
 219         __in            efx_nic_t *enp,
 220         __in            unsigned int index,
 221         __in            efsys_mem_t *esmp,
 222         __in            size_t ndescs,
 223         __in            uint32_t id,
 224         __in            uint32_t us,
 225         __in            uint32_t flags,
 226         __deref_out     efx_evq_t **eepp)
 227 {
 228         const efx_ev_ops_t *eevop = enp->en_eevop;
 229         efx_evq_t *eep;
 230         const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
 231         efx_rc_t rc;
 232
 233         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 234         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
 235
 236         EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <,
 237             enp->en_nic_cfg.enc_evq_limit);
 238
 239         if (index >= encp->enc_evq_limit) {
 240                 rc = EINVAL;
 241                 goto fail1;
 242         }
 243
 244         if (us > encp->enc_evq_timer_max_us) {
 245                 rc = EINVAL;
 246                 goto fail2;
 247         }
 248
 249         switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) {
 250         case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT:
 251                 break;
 252         case EFX_EVQ_FLAGS_NOTIFY_DISABLED:
 253                 if (us != 0) {
 254                         rc = EINVAL;
 255                         goto fail3;
 256                 }
 257                 break;
 258         default:
 259                 rc = EINVAL;
 260                 goto fail4;
 261         }
 262
 263         EFSYS_ASSERT(ISP2(encp->enc_evq_max_nevs));
 264         EFSYS_ASSERT(ISP2(encp->enc_evq_min_nevs));
 265
 266         if (!ISP2(ndescs) ||
 267             ndescs < encp->enc_evq_min_nevs ||
 268             ndescs > encp->enc_evq_max_nevs) {
 269                 rc = EINVAL;
 270                 goto fail5;
 271         }
 272
 273         /* Allocate an EVQ object */
 274         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
 275         if (eep == NULL) {
 276                 rc = ENOMEM;
 277                 goto fail6;
 278         }
 279
 280         eep->ee_magic = EFX_EVQ_MAGIC;
 281         eep->ee_enp = enp;
 282         eep->ee_index = index;
 283         eep->ee_mask = ndescs - 1;
 284         eep->ee_flags = flags;
 285         eep->ee_esmp = esmp;
 286
 287         /*
 288          * Set outputs before the queue is created because interrupts may be
 289          * raised for events immediately after the queue is created, before the
 290          * function call below returns. See bug58606.
 291          *
 292          * The eepp pointer passed in by the client must therefore point to data
 293          * shared with the client's event processing context.
 294          */
 295         enp->en_ev_qcount++;
 296         *eepp = eep;
 297
 298         if ((rc = eevop->eevo_qcreate(enp, index, esmp, ndescs, id, us, flags,
 299             eep)) != 0)
 300                 goto fail7;
 301
 302         return (0);
 303
 304 fail7:
 305         EFSYS_PROBE(fail7);
 306
 307         *eepp = NULL;
 308         enp->en_ev_qcount--;
 309         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 310 fail6:
 311         EFSYS_PROBE(fail6);
 312 fail5:
 313         EFSYS_PROBE(fail5);
 314 fail4:
 315         EFSYS_PROBE(fail4);
 316 fail3:
 317         EFSYS_PROBE(fail3);
 318 fail2:
 319         EFSYS_PROBE(fail2);
 320 fail1:
 321         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 322         return (rc);
 323 }
 324
 325                 void
 326 efx_ev_qdestroy(
 327         __in    efx_evq_t *eep)
 328 {
 329         efx_nic_t *enp = eep->ee_enp;
 330         const efx_ev_ops_t *eevop = enp->en_eevop;
 331
 332         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 333
 334         EFSYS_ASSERT(enp->en_ev_qcount != 0);
 335         --enp->en_ev_qcount;
 336
 337         eevop->eevo_qdestroy(eep);
 338
 339         /* Free the EVQ object */
 340         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
 341 }
 342
 343         __checkReturn   efx_rc_t
 344 efx_ev_qprime(
 345         __in            efx_evq_t *eep,
 346         __in            unsigned int count)
 347 {
 348         efx_nic_t *enp = eep->ee_enp;
 349         const efx_ev_ops_t *eevop = enp->en_eevop;
 350         efx_rc_t rc;
 351
 352         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 353
 354         if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
 355                 rc = EINVAL;
 356                 goto fail1;
 357         }
 358
 359         if ((rc = eevop->eevo_qprime(eep, count)) != 0)
 360                 goto fail2;
 361
 362         return (0);
 363
 364 fail2:
 365         EFSYS_PROBE(fail2);
 366 fail1:
 367         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 368         return (rc);
 369 }
 370
 371         __checkReturn   boolean_t
 372 efx_ev_qpending(
 373         __in            efx_evq_t *eep,
 374         __in            unsigned int count)
 375 {
 376         size_t offset;
 377         efx_qword_t qword;
 378
 379         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 380
 381         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 382         EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
 383
 384         return (EFX_EV_PRESENT(qword));
 385 }
 386
 387 #if EFSYS_OPT_EV_PREFETCH
 388
 389                         void
 390 efx_ev_qprefetch(
 391         __in            efx_evq_t *eep,
 392         __in            unsigned int count)
 393 {
 394         unsigned int offset;
 395
 396         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 397
 398         offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
 399         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
 400 }
 401
 402 #endif  /* EFSYS_OPT_EV_PREFETCH */
 403
 404                         void
 405 efx_ev_qpoll(
 406         __in            efx_evq_t *eep,
 407         __inout         unsigned int *countp,
 408         __in            const efx_ev_callbacks_t *eecp,
 409         __in_opt        void *arg)
 410 {
 411         efx_nic_t *enp = eep->ee_enp;
 412         const efx_ev_ops_t *eevop = enp->en_eevop;
 413
 414         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 415
 416         EFSYS_ASSERT(eevop != NULL &&
 417             eevop->eevo_qpoll != NULL);
 418
 419         eevop->eevo_qpoll(eep, countp, eecp, arg);
 420 }
 421
 422                         void
 423 efx_ev_qpost(
 424         __in    efx_evq_t *eep,
 425         __in    uint16_t data)
 426 {
 427         efx_nic_t *enp = eep->ee_enp;
 428         const efx_ev_ops_t *eevop = enp->en_eevop;
 429
 430         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 431
 432         EFSYS_ASSERT(eevop != NULL &&
 433             eevop->eevo_qpost != NULL);
 434
 435         eevop->eevo_qpost(eep, data);
 436 }
 437
 438         __checkReturn   efx_rc_t
 439 efx_ev_usecs_to_ticks(
 440         __in            efx_nic_t *enp,
 441         __in            unsigned int us,
 442         __out           unsigned int *ticksp)
 443 {
 444         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 445         unsigned int ticks;
 446         efx_rc_t rc;
 447
 448         if (encp->enc_evq_timer_quantum_ns == 0) {
 449                 rc = ENOTSUP;
 450                 goto fail1;
 451         }
 452
 453         /* Convert microseconds to a timer tick count */
 454         if (us == 0)
 455                 ticks = 0;
 456         else if (us * 1000 < encp->enc_evq_timer_quantum_ns)
 457                 ticks = 1;      /* Never round down to zero */
 458         else
 459                 ticks = us * 1000 / encp->enc_evq_timer_quantum_ns;
 460
 461         *ticksp = ticks;
 462         return (0);
 463
 464 fail1:
 465         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 466         return (rc);
 467 }
 468
 469         __checkReturn   efx_rc_t
 470 efx_ev_qmoderate(
 471         __in            efx_evq_t *eep,
 472         __in            unsigned int us)
 473 {
 474         efx_nic_t *enp = eep->ee_enp;
 475         const efx_ev_ops_t *eevop = enp->en_eevop;
 476         efx_rc_t rc;
 477
 478         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 479
 480         if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
 481             EFX_EVQ_FLAGS_NOTIFY_DISABLED) {
 482                 rc = EINVAL;
 483                 goto fail1;
 484         }
 485
 486         if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
 487                 goto fail2;
 488
 489         return (0);
 490
 491 fail2:
 492         EFSYS_PROBE(fail2);
 493 fail1:
 494         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 495         return (rc);
 496 }
 497
 498 #if EFSYS_OPT_QSTATS
 499                                         void
 500 efx_ev_qstats_update(
 501         __in                            efx_evq_t *eep,
 502         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
 503
 504 {       efx_nic_t *enp = eep->ee_enp;
 505         const efx_ev_ops_t *eevop = enp->en_eevop;
 506
 507         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 508
 509         eevop->eevo_qstats_update(eep, stat);
 510 }
 511
 512 #endif  /* EFSYS_OPT_QSTATS */
 513
 514 #if EFSYS_OPT_SIENA
 515
 516 static  __checkReturn   efx_rc_t
 517 siena_ev_init(
 518         __in            efx_nic_t *enp)
 519 {
 520         efx_oword_t oword;
 521
 522         /*
 523          * Program the event queue for receive and transmit queue
 524          * flush events.
 525          */
 526         EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
 527         EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
 528         EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
 529
 530         return (0);
 531
 532 }
 533
 534 static  __checkReturn   boolean_t
 535 siena_ev_rx_not_ok(
 536         __in            efx_evq_t *eep,
 537         __in            efx_qword_t *eqp,
 538         __in            uint32_t label,
 539         __in            uint32_t id,
 540         __inout         uint16_t *flagsp)
 541 {
 542         boolean_t ignore = B_FALSE;
 543
 544         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
 545                 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
 546                 EFSYS_PROBE(tobe_disc);
 547                 /*
 548                  * Assume this is a unicast address mismatch, unless below
 549                  * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
 550                  * EV_RX_PAUSE_FRM_ERR is set.
 551                  */
 552                 (*flagsp) |= EFX_ADDR_MISMATCH;
 553         }
 554
 555         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
 556                 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
 557                 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
 558                 (*flagsp) |= EFX_DISCARD;
 559
 560 #if EFSYS_OPT_RX_SCATTER
 561                 /*
 562                  * Lookout for payload queue ran dry errors and ignore them.
 563                  *
 564                  * Sadly for the header/data split cases, the descriptor
 565                  * pointer in this event refers to the header queue and
 566                  * therefore cannot be easily detected as duplicate.
 567                  * So we drop these and rely on the receive processing seeing
 568                  * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
 569                  * the partially received packet.
 570                  */
 571                 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
 572                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
 573                     (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
 574                         ignore = B_TRUE;
 575 #endif  /* EFSYS_OPT_RX_SCATTER */
 576         }
 577
 578         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
 579                 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
 580                 EFSYS_PROBE(crc_err);
 581                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 582                 (*flagsp) |= EFX_DISCARD;
 583         }
 584
 585         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
 586                 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
 587                 EFSYS_PROBE(pause_frm_err);
 588                 (*flagsp) &= ~EFX_ADDR_MISMATCH;
 589                 (*flagsp) |= EFX_DISCARD;
 590         }
 591
 592         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
 593                 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
 594                 EFSYS_PROBE(owner_id_err);
 595                 (*flagsp) |= EFX_DISCARD;
 596         }
 597
 598         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
 599                 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
 600                 EFSYS_PROBE(ipv4_err);
 601                 (*flagsp) &= ~EFX_CKSUM_IPV4;
 602         }
 603
 604         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
 605                 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
 606                 EFSYS_PROBE(udp_chk_err);
 607                 (*flagsp) &= ~EFX_CKSUM_TCPUDP;
 608         }
 609
 610         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
 611                 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
 612
 613                 /*
 614                  * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
 615                  * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
 616                  * condition.
 617                  */
 618                 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
 619         }
 620
 621         return (ignore);
 622 }
 623
 624 static  __checkReturn   boolean_t
 625 siena_ev_rx(
 626         __in            efx_evq_t *eep,
 627         __in            efx_qword_t *eqp,
 628         __in            const efx_ev_callbacks_t *eecp,
 629         __in_opt        void *arg)
 630 {
 631         uint32_t id;
 632         uint32_t size;
 633         uint32_t label;
 634         boolean_t ok;
 635 #if EFSYS_OPT_RX_SCATTER
 636         boolean_t sop;
 637         boolean_t jumbo_cont;
 638 #endif  /* EFSYS_OPT_RX_SCATTER */
 639         uint32_t hdr_type;
 640         boolean_t is_v6;
 641         uint16_t flags;
 642         boolean_t ignore;
 643         boolean_t should_abort;
 644
 645         EFX_EV_QSTAT_INCR(eep, EV_RX);
 646
 647         /* Basic packet information */
 648         id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
 649         size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
 650         label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
 651         ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
 652
 653 #if EFSYS_OPT_RX_SCATTER
 654         sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
 655         jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
 656 #endif  /* EFSYS_OPT_RX_SCATTER */
 657
 658         hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
 659
 660         is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
 661
 662         /*
 663          * If packet is marked as OK and packet type is TCP/IP or
 664          * UDP/IP or other IP, then we can rely on the hardware checksums.
 665          */
 666         switch (hdr_type) {
 667         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
 668                 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
 669                 if (is_v6) {
 670                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
 671                         flags |= EFX_PKT_IPV6;
 672                 } else {
 673                         EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
 674                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 675                 }
 676                 break;
 677
 678         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
 679                 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
 680                 if (is_v6) {
 681                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
 682                         flags |= EFX_PKT_IPV6;
 683                 } else {
 684                         EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
 685                         flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 686                 }
 687                 break;
 688
 689         case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
 690                 if (is_v6) {
 691                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
 692                         flags = EFX_PKT_IPV6;
 693                 } else {
 694                         EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
 695                         flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
 696                 }
 697                 break;
 698
 699         case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
 700                 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
 701                 flags = 0;
 702                 break;
 703
 704         default:
 705                 EFSYS_ASSERT(B_FALSE);
 706                 flags = 0;
 707                 break;
 708         }
 709
 710 #if EFSYS_OPT_RX_SCATTER
 711         /* Report scatter and header/lookahead split buffer flags */
 712         if (sop)
 713                 flags |= EFX_PKT_START;
 714         if (jumbo_cont)
 715                 flags |= EFX_PKT_CONT;
 716 #endif  /* EFSYS_OPT_RX_SCATTER */
 717
 718         /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
 719         if (!ok) {
 720                 ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
 721                 if (ignore) {
 722                         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 723                             uint32_t, size, uint16_t, flags);
 724
 725                         return (B_FALSE);
 726                 }
 727         }
 728
 729         /* If we're not discarding the packet then it is ok */
 730         if (~flags & EFX_DISCARD)
 731                 EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
 732
 733         /* Detect multicast packets that didn't match the filter */
 734         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
 735                 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
 736
 737                 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
 738                         EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
 739                 } else {
 740                         EFSYS_PROBE(mcast_mismatch);
 741                         flags |= EFX_ADDR_MISMATCH;
 742                 }
 743         } else {
 744                 flags |= EFX_PKT_UNICAST;
 745         }
 746
 747         /*
 748          * The packet parser in Siena can abort parsing packets under
 749          * certain error conditions, setting the PKT_NOT_PARSED bit
 750          * (which clears PKT_OK). If this is set, then don't trust
 751          * the PKT_TYPE field.
 752          */
 753         if (!ok) {
 754                 uint32_t parse_err;
 755
 756                 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
 757                 if (parse_err != 0)
 758                         flags |= EFX_CHECK_VLAN;
 759         }
 760
 761         if (~flags & EFX_CHECK_VLAN) {
 762                 uint32_t pkt_type;
 763
 764                 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
 765                 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
 766                         flags |= EFX_PKT_VLAN_TAGGED;
 767         }
 768
 769         EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
 770             uint32_t, size, uint16_t, flags);
 771
 772         EFSYS_ASSERT(eecp->eec_rx != NULL);
 773         should_abort = eecp->eec_rx(arg, label, id, size, flags);
 774
 775         return (should_abort);
 776 }
 777
 778 static  __checkReturn   boolean_t
 779 siena_ev_tx(
 780         __in            efx_evq_t *eep,
 781         __in            efx_qword_t *eqp,
 782         __in            const efx_ev_callbacks_t *eecp,
 783         __in_opt        void *arg)
 784 {
 785         uint32_t id;
 786         uint32_t label;
 787         boolean_t should_abort;
 788
 789         EFX_EV_QSTAT_INCR(eep, EV_TX);
 790
 791         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
 792             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
 793             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
 794             EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
 795
 796                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
 797                 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
 798
 799                 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
 800
 801                 EFSYS_ASSERT(eecp->eec_tx != NULL);
 802                 should_abort = eecp->eec_tx(arg, label, id);
 803
 804                 return (should_abort);
 805         }
 806
 807         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
 808                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 809                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 810                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 811
 812         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
 813                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
 814
 815         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
 816                 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
 817
 818         if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
 819                 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
 820
 821         EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
 822         return (B_FALSE);
 823 }
 824
 825 static  __checkReturn   boolean_t
 826 siena_ev_global(
 827         __in            efx_evq_t *eep,
 828         __in            efx_qword_t *eqp,
 829         __in            const efx_ev_callbacks_t *eecp,
 830         __in_opt        void *arg)
 831 {
 832         _NOTE(ARGUNUSED(eqp, eecp, arg))
 833
 834         EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
 835
 836         return (B_FALSE);
 837 }
 838
 839 static  __checkReturn   boolean_t
 840 siena_ev_driver(
 841         __in            efx_evq_t *eep,
 842         __in            efx_qword_t *eqp,
 843         __in            const efx_ev_callbacks_t *eecp,
 844         __in_opt        void *arg)
 845 {
 846         boolean_t should_abort;
 847
 848         EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
 849         should_abort = B_FALSE;
 850
 851         switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
 852         case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
 853                 uint32_t txq_index;
 854
 855                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
 856
 857                 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 858
 859                 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
 860
 861                 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
 862                 should_abort = eecp->eec_txq_flush_done(arg, txq_index);
 863
 864                 break;
 865         }
 866         case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
 867                 uint32_t rxq_index;
 868                 uint32_t failed;
 869
 870                 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
 871                 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
 872
 873                 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
 874                 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
 875
 876                 if (failed) {
 877                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
 878
 879                         EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
 880
 881                         should_abort = eecp->eec_rxq_flush_failed(arg,
 882                                                                     rxq_index);
 883                 } else {
 884                         EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
 885
 886                         EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
 887
 888                         should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
 889                 }
 890
 891                 break;
 892         }
 893         case FSE_AZ_EVQ_INIT_DONE_EV:
 894                 EFSYS_ASSERT(eecp->eec_initialized != NULL);
 895                 should_abort = eecp->eec_initialized(arg);
 896
 897                 break;
 898
 899         case FSE_AZ_EVQ_NOT_EN_EV:
 900                 EFSYS_PROBE(evq_not_en);
 901                 break;
 902
 903         case FSE_AZ_SRM_UPD_DONE_EV: {
 904                 uint32_t code;
 905
 906                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
 907
 908                 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 909
 910                 EFSYS_ASSERT(eecp->eec_sram != NULL);
 911                 should_abort = eecp->eec_sram(arg, code);
 912
 913                 break;
 914         }
 915         case FSE_AZ_WAKE_UP_EV: {
 916                 uint32_t id;
 917
 918                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 919
 920                 EFSYS_ASSERT(eecp->eec_wake_up != NULL);
 921                 should_abort = eecp->eec_wake_up(arg, id);
 922
 923                 break;
 924         }
 925         case FSE_AZ_TX_PKT_NON_TCP_UDP:
 926                 EFSYS_PROBE(tx_pkt_non_tcp_udp);
 927                 break;
 928
 929         case FSE_AZ_TIMER_EV: {
 930                 uint32_t id;
 931
 932                 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
 933
 934                 EFSYS_ASSERT(eecp->eec_timer != NULL);
 935                 should_abort = eecp->eec_timer(arg, id);
 936
 937                 break;
 938         }
 939         case FSE_AZ_RX_DSC_ERROR_EV:
 940                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
 941
 942                 EFSYS_PROBE(rx_dsc_error);
 943
 944                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 945                 should_abort = eecp->eec_exception(arg,
 946                         EFX_EXCEPTION_RX_DSC_ERROR, 0);
 947
 948                 break;
 949
 950         case FSE_AZ_TX_DSC_ERROR_EV:
 951                 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
 952
 953                 EFSYS_PROBE(tx_dsc_error);
 954
 955                 EFSYS_ASSERT(eecp->eec_exception != NULL);
 956                 should_abort = eecp->eec_exception(arg,
 957                         EFX_EXCEPTION_TX_DSC_ERROR, 0);
 958
 959                 break;
 960
 961         default:
 962                 break;
 963         }
 964
 965         return (should_abort);
 966 }
 967
 968 static  __checkReturn   boolean_t
 969 siena_ev_drv_gen(
 970         __in            efx_evq_t *eep,
 971         __in            efx_qword_t *eqp,
 972         __in            const efx_ev_callbacks_t *eecp,
 973         __in_opt        void *arg)
 974 {
 975         uint32_t data;
 976         boolean_t should_abort;
 977
 978         EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
 979
 980         data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
 981         if (data >= ((uint32_t)1 << 16)) {
 982                 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
 983                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
 984                             uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
 985                 return (B_TRUE);
 986         }
 987
 988         EFSYS_ASSERT(eecp->eec_software != NULL);
 989         should_abort = eecp->eec_software(arg, (uint16_t)data);
 990
 991         return (should_abort);
 992 }
 993
 994 #if EFSYS_OPT_MCDI
 995
 996 static  __checkReturn   boolean_t
 997 siena_ev_mcdi(
 998         __in            efx_evq_t *eep,
 999         __in            efx_qword_t *eqp,
1000         __in            const efx_ev_callbacks_t *eecp,
1001         __in_opt        void *arg)
1002 {
1003         efx_nic_t *enp = eep->ee_enp;
1004         unsigned int code;
1005         boolean_t should_abort = B_FALSE;
1006
1007         EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
1008
1009         if (enp->en_family != EFX_FAMILY_SIENA)
1010                 goto out;
1011
1012         EFSYS_ASSERT(eecp->eec_link_change != NULL);
1013         EFSYS_ASSERT(eecp->eec_exception != NULL);
1014 #if EFSYS_OPT_MON_STATS
1015         EFSYS_ASSERT(eecp->eec_monitor != NULL);
1016 #endif
1017
1018         EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
1019
1020         code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
1021         switch (code) {
1022         case MCDI_EVENT_CODE_BADSSERT:
1023                 efx_mcdi_ev_death(enp, EINTR);
1024                 break;
1025
1026         case MCDI_EVENT_CODE_CMDDONE:
1027                 efx_mcdi_ev_cpl(enp,
1028                     MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
1029                     MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
1030                     MCDI_EV_FIELD(eqp, CMDDONE_ERRNO));
1031                 break;
1032
1033         case MCDI_EVENT_CODE_LINKCHANGE: {
1034                 efx_link_mode_t link_mode;
1035
1036                 siena_phy_link_ev(enp, eqp, &link_mode);
1037                 should_abort = eecp->eec_link_change(arg, link_mode);
1038                 break;
1039         }
1040         case MCDI_EVENT_CODE_SENSOREVT: {
1041 #if EFSYS_OPT_MON_STATS
1042                 efx_mon_stat_t id;
1043                 efx_mon_stat_value_t value;
1044                 efx_rc_t rc;
1045
1046                 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0)
1047                         should_abort = eecp->eec_monitor(arg, id, value);
1048                 else if (rc == ENOTSUP) {
1049                         should_abort = eecp->eec_exception(arg,
1050                                 EFX_EXCEPTION_UNKNOWN_SENSOREVT,
1051                                 MCDI_EV_FIELD(eqp, DATA));
1052                 } else
1053                         EFSYS_ASSERT(rc == ENODEV);     /* Wrong port */
1054 #else
1055                 should_abort = B_FALSE;
1056 #endif
1057                 break;
1058         }
1059         case MCDI_EVENT_CODE_SCHEDERR:
1060                 /* Informational only */
1061                 break;
1062
1063         case MCDI_EVENT_CODE_REBOOT:
1064                 efx_mcdi_ev_death(enp, EIO);
1065                 break;
1066
1067         case MCDI_EVENT_CODE_MAC_STATS_DMA:
1068 #if EFSYS_OPT_MAC_STATS
1069                 if (eecp->eec_mac_stats != NULL) {
1070                         eecp->eec_mac_stats(arg,
1071                             MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION));
1072                 }
1073 #endif
1074                 break;
1075
1076         case MCDI_EVENT_CODE_FWALERT: {
1077                 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON);
1078
1079                 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS)
1080                         should_abort = eecp->eec_exception(arg,
1081                                 EFX_EXCEPTION_FWALERT_SRAM,
1082                                 MCDI_EV_FIELD(eqp, FWALERT_DATA));
1083                 else
1084                         should_abort = eecp->eec_exception(arg,
1085                                 EFX_EXCEPTION_UNKNOWN_FWALERT,
1086                                 MCDI_EV_FIELD(eqp, DATA));
1087                 break;
1088         }
1089
1090         default:
1091                 EFSYS_PROBE1(mc_pcol_error, int, code);
1092                 break;
1093         }
1094
1095 out:
1096         return (should_abort);
1097 }
1098
1099 #endif  /* EFSYS_OPT_MCDI */
1100
1101 static  __checkReturn   efx_rc_t
1102 siena_ev_qprime(
1103         __in            efx_evq_t *eep,
1104         __in            unsigned int count)
1105 {
1106         efx_nic_t *enp = eep->ee_enp;
1107         uint32_t rptr;
1108         efx_dword_t dword;
1109
1110         rptr = count & eep->ee_mask;
1111
1112         EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr);
1113
1114         EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index,
1115                             &dword, B_FALSE);
1116
1117         return (0);
1118 }
1119
1120 static          void
1121 siena_ev_qpost(
1122         __in    efx_evq_t *eep,
1123         __in    uint16_t data)
1124 {
1125         efx_nic_t *enp = eep->ee_enp;
1126         efx_qword_t ev;
1127         efx_oword_t oword;
1128
1129         EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV,
1130             FSF_AZ_EV_DATA_DW0, (uint32_t)data);
1131
1132         EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index,
1133             EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0),
1134             EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1));
1135
1136         EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword);
1137 }
1138
1139 static  __checkReturn   efx_rc_t
1140 siena_ev_qmoderate(
1141         __in            efx_evq_t *eep,
1142         __in            unsigned int us)
1143 {
1144         efx_nic_t *enp = eep->ee_enp;
1145         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1146         unsigned int locked;
1147         efx_dword_t dword;
1148         efx_rc_t rc;
1149
1150         if (us > encp->enc_evq_timer_max_us) {
1151                 rc = EINVAL;
1152                 goto fail1;
1153         }
1154
1155         /* If the value is zero then disable the timer */
1156         if (us == 0) {
1157                 EFX_POPULATE_DWORD_2(dword,
1158                     FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS,
1159                     FRF_CZ_TC_TIMER_VAL, 0);
1160         } else {
1161                 unsigned int ticks;
1162
1163                 if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0)
1164                         goto fail2;
1165
1166                 EFSYS_ASSERT(ticks > 0);
1167                 EFX_POPULATE_DWORD_2(dword,
1168                     FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF,
1169                     FRF_CZ_TC_TIMER_VAL, ticks - 1);
1170         }
1171
1172         locked = (eep->ee_index == 0) ? 1 : 0;
1173
1174         EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0,
1175             eep->ee_index, &dword, locked);
1176
1177         return (0);
1178
1179 fail2:
1180         EFSYS_PROBE(fail2);
1181 fail1:
1182         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1183
1184         return (rc);
1185 }
1186
1187 static  __checkReturn   efx_rc_t
1188 siena_ev_qcreate(
1189         __in            efx_nic_t *enp,
1190         __in            unsigned int index,
1191         __in            efsys_mem_t *esmp,
1192         __in            size_t ndescs,
1193         __in            uint32_t id,
1194         __in            uint32_t us,
1195         __in            uint32_t flags,
1196         __in            efx_evq_t *eep)
1197 {
1198         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1199         uint32_t size;
1200         efx_oword_t oword;
1201         efx_rc_t rc;
1202         boolean_t notify_mode;
1203
1204         _NOTE(ARGUNUSED(esmp))
1205
1206 #if EFSYS_OPT_RX_SCALE
1207         if (enp->en_intr.ei_type == EFX_INTR_LINE &&
1208             index >= EFX_MAXRSS_LEGACY) {
1209                 rc = EINVAL;
1210                 goto fail1;
1211         }
1212 #endif
1213         for (size = 0;
1214             (1U << size) <= encp->enc_evq_max_nevs / encp->enc_evq_min_nevs;
1215             size++)
1216                 if ((1U << size) == (uint32_t)ndescs / encp->enc_evq_min_nevs)
1217                         break;
1218         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1219                 rc = EINVAL;
1220                 goto fail2;
1221         }
1222
1223         /* Set up the handler table */
1224         eep->ee_rx      = siena_ev_rx;
1225         eep->ee_tx      = siena_ev_tx;
1226         eep->ee_driver  = siena_ev_driver;
1227         eep->ee_global  = siena_ev_global;
1228         eep->ee_drv_gen = siena_ev_drv_gen;
1229 #if EFSYS_OPT_MCDI
1230         eep->ee_mcdi    = siena_ev_mcdi;
1231 #endif  /* EFSYS_OPT_MCDI */
1232
1233         notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) !=
1234             EFX_EVQ_FLAGS_NOTIFY_INTERRUPT);
1235
1236         /* Set up the new event queue */
1237         EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1,
1238             FRF_CZ_HOST_NOTIFY_MODE, notify_mode,
1239             FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
1240         EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE);
1241
1242         EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size,
1243             FRF_AZ_EVQ_BUF_BASE_ID, id);
1244
1245         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE);
1246
1247         /* Set initial interrupt moderation */
1248         siena_ev_qmoderate(eep, us);
1249
1250         return (0);
1251
1252 fail2:
1253         EFSYS_PROBE(fail2);
1254 #if EFSYS_OPT_RX_SCALE
1255 fail1:
1256 #endif
1257         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1258
1259         return (rc);
1260 }
1261
1262 #endif /* EFSYS_OPT_SIENA */
1263
1264 #if EFSYS_OPT_QSTATS
1265 #if EFSYS_OPT_NAMES
1266 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock ac223f7134058b4f */
1267 static const char * const __efx_ev_qstat_name[] = {
1268         "all",
1269         "rx",
1270         "rx_ok",
1271         "rx_frm_trunc",
1272         "rx_tobe_disc",
1273         "rx_pause_frm_err",
1274         "rx_buf_owner_id_err",
1275         "rx_ipv4_hdr_chksum_err",
1276         "rx_tcp_udp_chksum_err",
1277         "rx_eth_crc_err",
1278         "rx_ip_frag_err",
1279         "rx_mcast_pkt",
1280         "rx_mcast_hash_match",
1281         "rx_tcp_ipv4",
1282         "rx_tcp_ipv6",
1283         "rx_udp_ipv4",
1284         "rx_udp_ipv6",
1285         "rx_other_ipv4",
1286         "rx_other_ipv6",
1287         "rx_non_ip",
1288         "rx_batch",
1289         "tx",
1290         "tx_wq_ff_full",
1291         "tx_pkt_err",
1292         "tx_pkt_too_big",
1293         "tx_unexpected",
1294         "global",
1295         "global_mnt",
1296         "driver",
1297         "driver_srm_upd_done",
1298         "driver_tx_descq_fls_done",
1299         "driver_rx_descq_fls_done",
1300         "driver_rx_descq_fls_failed",
1301         "driver_rx_dsc_error",
1302         "driver_tx_dsc_error",
1303         "drv_gen",
1304         "mcdi_response",
1305         "rx_parse_incomplete",
1306 };
1307 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */
1308
1309                 const char *
1310 efx_ev_qstat_name(
1311         __in    efx_nic_t *enp,
1312         __in    unsigned int id)
1313 {
1314         _NOTE(ARGUNUSED(enp))
1315
1316         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1317         EFSYS_ASSERT3U(id, <, EV_NQSTATS);
1318
1319         return (__efx_ev_qstat_name[id]);
1320 }
1321 #endif  /* EFSYS_OPT_NAMES */
1322 #endif  /* EFSYS_OPT_QSTATS */
1323
1324 #if EFSYS_OPT_SIENA
1325
1326 #if EFSYS_OPT_QSTATS
1327 static                                  void
1328 siena_ev_qstats_update(
1329         __in                            efx_evq_t *eep,
1330         __inout_ecount(EV_NQSTATS)      efsys_stat_t *stat)
1331 {
1332         unsigned int id;
1333
1334         for (id = 0; id < EV_NQSTATS; id++) {
1335                 efsys_stat_t *essp = &stat[id];
1336
1337                 EFSYS_STAT_INCR(essp, eep->ee_stat[id]);
1338                 eep->ee_stat[id] = 0;
1339         }
1340 }
1341 #endif  /* EFSYS_OPT_QSTATS */
1342
1343 static          void
1344 siena_ev_qdestroy(
1345         __in    efx_evq_t *eep)
1346 {
1347         efx_nic_t *enp = eep->ee_enp;
1348         efx_oword_t oword;
1349
1350         /* Purge event queue */
1351         EFX_ZERO_OWORD(oword);
1352
1353         EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL,
1354             eep->ee_index, &oword, B_TRUE);
1355
1356         EFX_ZERO_OWORD(oword);
1357         EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE);
1358 }
1359
1360 static          void
1361 siena_ev_fini(
1362         __in    efx_nic_t *enp)
1363 {
1364         _NOTE(ARGUNUSED(enp))
1365 }
1366
1367 #endif /* EFSYS_OPT_SIENA */
1368
1369 #if EFX_OPTS_EF10() || EFSYS_OPT_SIENA
1370
1371 #define EFX_EV_BATCH    8
1372
1373 static                  void
1374 siena_ef10_ev_qpoll(
1375         __in            efx_evq_t *eep,
1376         __inout         unsigned int *countp,
1377         __in            const efx_ev_callbacks_t *eecp,
1378         __in_opt        void *arg)
1379 {
1380         efx_qword_t ev[EFX_EV_BATCH];
1381         unsigned int batch;
1382         unsigned int total;
1383         unsigned int count;
1384         unsigned int index;
1385         size_t offset;
1386
1387         /* Ensure events codes match for EF10 (Huntington/Medford) and Siena */
1388         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
1389         EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
1390
1391         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
1392         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
1393         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
1394         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
1395             FSE_AZ_EV_CODE_DRV_GEN_EV);
1396 #if EFSYS_OPT_MCDI
1397         EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
1398             FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
1399 #endif
1400
1401         EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
1402         EFSYS_ASSERT(countp != NULL);
1403         EFSYS_ASSERT(eecp != NULL);
1404
1405         count = *countp;
1406         do {
1407                 /* Read up until the end of the batch period */
1408                 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
1409                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1410                 for (total = 0; total < batch; ++total) {
1411                         EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
1412
1413                         if (!EFX_EV_PRESENT(ev[total]))
1414                                 break;
1415
1416                         EFSYS_PROBE3(event, unsigned int, eep->ee_index,
1417                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
1418                             uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
1419
1420                         offset += sizeof (efx_qword_t);
1421                 }
1422
1423 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1)
1424                 /*
1425                  * Prefetch the next batch when we get within PREFETCH_PERIOD
1426                  * of a completed batch. If the batch is smaller, then prefetch
1427                  * immediately.
1428                  */
1429                 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD)
1430                         EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1431 #endif  /* EFSYS_OPT_EV_PREFETCH */
1432
1433                 /* Process the batch of events */
1434                 for (index = 0; index < total; ++index) {
1435                         boolean_t should_abort;
1436                         uint32_t code;
1437
1438 #if EFSYS_OPT_EV_PREFETCH
1439                         /* Prefetch if we've now reached the batch period */
1440                         if (total == batch &&
1441                             index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) {
1442                                 offset = (count + batch) & eep->ee_mask;
1443                                 offset *= sizeof (efx_qword_t);
1444
1445                                 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
1446                         }
1447 #endif  /* EFSYS_OPT_EV_PREFETCH */
1448
1449                         EFX_EV_QSTAT_INCR(eep, EV_ALL);
1450
1451                         code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
1452                         switch (code) {
1453                         case FSE_AZ_EV_CODE_RX_EV:
1454                                 should_abort = eep->ee_rx(eep,
1455                                     &(ev[index]), eecp, arg);
1456                                 break;
1457                         case FSE_AZ_EV_CODE_TX_EV:
1458                                 should_abort = eep->ee_tx(eep,
1459                                     &(ev[index]), eecp, arg);
1460                                 break;
1461                         case FSE_AZ_EV_CODE_DRIVER_EV:
1462                                 should_abort = eep->ee_driver(eep,
1463                                     &(ev[index]), eecp, arg);
1464                                 break;
1465                         case FSE_AZ_EV_CODE_DRV_GEN_EV:
1466                                 should_abort = eep->ee_drv_gen(eep,
1467                                     &(ev[index]), eecp, arg);
1468                                 break;
1469 #if EFSYS_OPT_MCDI
1470                         case FSE_AZ_EV_CODE_MCDI_EVRESPONSE:
1471                                 should_abort = eep->ee_mcdi(eep,
1472                                     &(ev[index]), eecp, arg);
1473                                 break;
1474 #endif
1475                         case FSE_AZ_EV_CODE_GLOBAL_EV:
1476                                 if (eep->ee_global) {
1477                                         should_abort = eep->ee_global(eep,
1478                                             &(ev[index]), eecp, arg);
1479                                         break;
1480                                 }
1481                                 /* else fallthrough */
1482                         default:
1483                                 EFSYS_PROBE3(bad_event,
1484                                     unsigned int, eep->ee_index,
1485                                     uint32_t,
1486                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
1487                                     uint32_t,
1488                                     EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
1489
1490                                 EFSYS_ASSERT(eecp->eec_exception != NULL);
1491                                 (void) eecp->eec_exception(arg,
1492                                         EFX_EXCEPTION_EV_ERROR, code);
1493                                 should_abort = B_TRUE;
1494                         }
1495                         if (should_abort) {
1496                                 /* Ignore subsequent events */
1497                                 total = index + 1;
1498
1499                                 /*
1500                                  * Poison batch to ensure the outer
1501                                  * loop is broken out of.
1502                                  */
1503                                 EFSYS_ASSERT(batch <= EFX_EV_BATCH);
1504                                 batch += (EFX_EV_BATCH << 1);
1505                                 EFSYS_ASSERT(total != batch);
1506                                 break;
1507                         }
1508                 }
1509
1510                 /*
1511                  * Now that the hardware has most likely moved onto dma'ing
1512                  * into the next cache line, clear the processed events. Take
1513                  * care to only clear out events that we've processed
1514                  */
1515                 EFX_SET_QWORD(ev[0]);
1516                 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
1517                 for (index = 0; index < total; ++index) {
1518                         EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
1519                         offset += sizeof (efx_qword_t);
1520                 }
1521
1522                 count += total;
1523
1524         } while (total == batch);
1525
1526         *countp = count;
1527 }
1528
1529 #endif  /* EFX_OPTS_EF10() || EFSYS_OPT_SIENA */