drivers/net/sfc/base/efx_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2007-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_SIENA
  12
  13 static  __checkReturn   efx_rc_t
  14 siena_rx_init(
  15         __in            efx_nic_t *enp);
  16
  17 static                  void
  18 siena_rx_fini(
  19         __in            efx_nic_t *enp);
  20
  21 #if EFSYS_OPT_RX_SCATTER
  22 static  __checkReturn   efx_rc_t
  23 siena_rx_scatter_enable(
  24         __in            efx_nic_t *enp,
  25         __in            unsigned int buf_size);
  26 #endif /* EFSYS_OPT_RX_SCATTER */
  27
  28 #if EFSYS_OPT_RX_SCALE
  29 static  __checkReturn   efx_rc_t
  30 siena_rx_scale_mode_set(
  31         __in            efx_nic_t *enp,
  32         __in            uint32_t rss_context,
  33         __in            efx_rx_hash_alg_t alg,
  34         __in            efx_rx_hash_type_t type,
  35         __in            boolean_t insert);
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_scale_key_set(
  39         __in            efx_nic_t *enp,
  40         __in            uint32_t rss_context,
  41         __in_ecount(n)  uint8_t *key,
  42         __in            size_t n);
  43
  44 static  __checkReturn   efx_rc_t
  45 siena_rx_scale_tbl_set(
  46         __in            efx_nic_t *enp,
  47         __in            uint32_t rss_context,
  48         __in_ecount(n)  unsigned int *table,
  49         __in            size_t n);
  50
  51 static  __checkReturn   uint32_t
  52 siena_rx_prefix_hash(
  53         __in            efx_nic_t *enp,
  54         __in            efx_rx_hash_alg_t func,
  55         __in            uint8_t *buffer);
  56
  57 #endif /* EFSYS_OPT_RX_SCALE */
  58
  59 static  __checkReturn   efx_rc_t
  60 siena_rx_prefix_pktlen(
  61         __in            efx_nic_t *enp,
  62         __in            uint8_t *buffer,
  63         __out           uint16_t *lengthp);
  64
  65 static                          void
  66 siena_rx_qpost(
  67         __in                    efx_rxq_t *erp,
  68         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
  69         __in                    size_t size,
  70         __in                    unsigned int ndescs,
  71         __in                    unsigned int completed,
  72         __in                    unsigned int added);
  73
  74 static                  void
  75 siena_rx_qpush(
  76         __in            efx_rxq_t *erp,
  77         __in            unsigned int added,
  78         __inout         unsigned int *pushedp);
  79
  80 #if EFSYS_OPT_RX_PACKED_STREAM
  81 static          void
  82 siena_rx_qpush_ps_credits(
  83         __in            efx_rxq_t *erp);
  84
  85 static  __checkReturn   uint8_t *
  86 siena_rx_qps_packet_info(
  87         __in            efx_rxq_t *erp,
  88         __in            uint8_t *buffer,
  89         __in            uint32_t buffer_length,
  90         __in            uint32_t current_offset,
  91         __out           uint16_t *lengthp,
  92         __out           uint32_t *next_offsetp,
  93         __out           uint32_t *timestamp);
  94 #endif
  95
  96 static  __checkReturn   efx_rc_t
  97 siena_rx_qflush(
  98         __in            efx_rxq_t *erp);
  99
 100 static                  void
 101 siena_rx_qenable(
 102         __in            efx_rxq_t *erp);
 103
 104 static  __checkReturn   efx_rc_t
 105 siena_rx_qcreate(
 106         __in            efx_nic_t *enp,
 107         __in            unsigned int index,
 108         __in            unsigned int label,
 109         __in            efx_rxq_type_t type,
 110         __in            uint32_t type_data,
 111         __in            efsys_mem_t *esmp,
 112         __in            size_t ndescs,
 113         __in            uint32_t id,
 114         __in            unsigned int flags,
 115         __in            efx_evq_t *eep,
 116         __in            efx_rxq_t *erp);
 117
 118 static                  void
 119 siena_rx_qdestroy(
 120         __in            efx_rxq_t *erp);
 121
 122 #endif /* EFSYS_OPT_SIENA */
 123
 124
 125 #if EFSYS_OPT_SIENA
 126 static const efx_rx_ops_t __efx_rx_siena_ops = {
 127         siena_rx_init,                          /* erxo_init */
 128         siena_rx_fini,                          /* erxo_fini */
 129 #if EFSYS_OPT_RX_SCATTER
 130         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 131 #endif
 132 #if EFSYS_OPT_RX_SCALE
 133         NULL,                                   /* erxo_scale_context_alloc */
 134         NULL,                                   /* erxo_scale_context_free */
 135         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 136         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 137         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 138         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 139 #endif
 140         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 141         siena_rx_qpost,                         /* erxo_qpost */
 142         siena_rx_qpush,                         /* erxo_qpush */
 143 #if EFSYS_OPT_RX_PACKED_STREAM
 144         siena_rx_qpush_ps_credits,              /* erxo_qpush_ps_credits */
 145         siena_rx_qps_packet_info,               /* erxo_qps_packet_info */
 146 #endif
 147         siena_rx_qflush,                        /* erxo_qflush */
 148         siena_rx_qenable,                       /* erxo_qenable */
 149         siena_rx_qcreate,                       /* erxo_qcreate */
 150         siena_rx_qdestroy,                      /* erxo_qdestroy */
 151 };
 152 #endif  /* EFSYS_OPT_SIENA */
 153
 154 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
 155 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 156         ef10_rx_init,                           /* erxo_init */
 157         ef10_rx_fini,                           /* erxo_fini */
 158 #if EFSYS_OPT_RX_SCATTER
 159         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 160 #endif
 161 #if EFSYS_OPT_RX_SCALE
 162         ef10_rx_scale_context_alloc,            /* erxo_scale_context_alloc */
 163         ef10_rx_scale_context_free,             /* erxo_scale_context_free */
 164         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 165         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 166         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 167         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 168 #endif
 169         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 170         ef10_rx_qpost,                          /* erxo_qpost */
 171         ef10_rx_qpush,                          /* erxo_qpush */
 172 #if EFSYS_OPT_RX_PACKED_STREAM
 173         ef10_rx_qpush_ps_credits,               /* erxo_qpush_ps_credits */
 174         ef10_rx_qps_packet_info,                /* erxo_qps_packet_info */
 175 #endif
 176         ef10_rx_qflush,                         /* erxo_qflush */
 177         ef10_rx_qenable,                        /* erxo_qenable */
 178         ef10_rx_qcreate,                        /* erxo_qcreate */
 179         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 180 };
 181 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
 182
 183
 184         __checkReturn   efx_rc_t
 185 efx_rx_init(
 186         __inout         efx_nic_t *enp)
 187 {
 188         const efx_rx_ops_t *erxop;
 189         efx_rc_t rc;
 190
 191         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 192         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 193
 194         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 195                 rc = EINVAL;
 196                 goto fail1;
 197         }
 198
 199         if (enp->en_mod_flags & EFX_MOD_RX) {
 200                 rc = EINVAL;
 201                 goto fail2;
 202         }
 203
 204         switch (enp->en_family) {
 205 #if EFSYS_OPT_SIENA
 206         case EFX_FAMILY_SIENA:
 207                 erxop = &__efx_rx_siena_ops;
 208                 break;
 209 #endif /* EFSYS_OPT_SIENA */
 210
 211 #if EFSYS_OPT_HUNTINGTON
 212         case EFX_FAMILY_HUNTINGTON:
 213                 erxop = &__efx_rx_ef10_ops;
 214                 break;
 215 #endif /* EFSYS_OPT_HUNTINGTON */
 216
 217 #if EFSYS_OPT_MEDFORD
 218         case EFX_FAMILY_MEDFORD:
 219                 erxop = &__efx_rx_ef10_ops;
 220                 break;
 221 #endif /* EFSYS_OPT_MEDFORD */
 222
 223 #if EFSYS_OPT_MEDFORD2
 224         case EFX_FAMILY_MEDFORD2:
 225                 erxop = &__efx_rx_ef10_ops;
 226                 break;
 227 #endif /* EFSYS_OPT_MEDFORD2 */
 228
 229         default:
 230                 EFSYS_ASSERT(0);
 231                 rc = ENOTSUP;
 232                 goto fail3;
 233         }
 234
 235         if ((rc = erxop->erxo_init(enp)) != 0)
 236                 goto fail4;
 237
 238         enp->en_erxop = erxop;
 239         enp->en_mod_flags |= EFX_MOD_RX;
 240         return (0);
 241
 242 fail4:
 243         EFSYS_PROBE(fail4);
 244 fail3:
 245         EFSYS_PROBE(fail3);
 246 fail2:
 247         EFSYS_PROBE(fail2);
 248 fail1:
 249         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 250
 251         enp->en_erxop = NULL;
 252         enp->en_mod_flags &= ~EFX_MOD_RX;
 253         return (rc);
 254 }
 255
 256                         void
 257 efx_rx_fini(
 258         __in            efx_nic_t *enp)
 259 {
 260         const efx_rx_ops_t *erxop = enp->en_erxop;
 261
 262         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 263         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 264         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 265         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 266
 267         erxop->erxo_fini(enp);
 268
 269         enp->en_erxop = NULL;
 270         enp->en_mod_flags &= ~EFX_MOD_RX;
 271 }
 272
 273 #if EFSYS_OPT_RX_SCATTER
 274         __checkReturn   efx_rc_t
 275 efx_rx_scatter_enable(
 276         __in            efx_nic_t *enp,
 277         __in            unsigned int buf_size)
 278 {
 279         const efx_rx_ops_t *erxop = enp->en_erxop;
 280         efx_rc_t rc;
 281
 282         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 283         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 284
 285         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 286                 goto fail1;
 287
 288         return (0);
 289
 290 fail1:
 291         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 292         return (rc);
 293 }
 294 #endif  /* EFSYS_OPT_RX_SCATTER */
 295
 296 #if EFSYS_OPT_RX_SCALE
 297         __checkReturn                           efx_rc_t
 298 efx_rx_scale_hash_flags_get(
 299         __in                                    efx_nic_t *enp,
 300         __in                                    efx_rx_hash_alg_t hash_alg,
 301         __inout_ecount(EFX_RX_HASH_NFLAGS)      unsigned int *flags,
 302         __out                                   unsigned int *nflagsp)
 303 {
 304         unsigned int *entryp = flags;
 305         efx_rc_t rc;
 306
 307         if (flags == NULL || nflagsp == NULL) {
 308                 rc = EINVAL;
 309                 goto fail1;
 310         }
 311
 312 #define LIST_FLAGS(_entryp, _class, _l4_hashing)                        \
 313         do {                                                            \
 314                 if (_l4_hashing)                                        \
 315                         *(_entryp++) = EFX_RX_HASH(_class, 4TUPLE);     \
 316                                                                         \
 317                 *(_entryp++) = EFX_RX_HASH(_class, 2TUPLE);             \
 318                 *(_entryp++) = EFX_RX_HASH(_class, DISABLE);            \
 319                                                                         \
 320                 _NOTE(CONSTANTCONDITION)                                \
 321         } while (B_FALSE)
 322
 323         switch (hash_alg) {
 324         case EFX_RX_HASHALG_TOEPLITZ:
 325                 LIST_FLAGS(entryp, IPV4_TCP, B_TRUE);
 326                 LIST_FLAGS(entryp, IPV6_TCP, B_TRUE);
 327                 LIST_FLAGS(entryp, IPV4, B_FALSE);
 328                 LIST_FLAGS(entryp, IPV6, B_FALSE);
 329                 break;
 330
 331         default:
 332                 rc = EINVAL;
 333                 goto fail2;
 334         }
 335
 336 #undef LIST_FLAGS
 337
 338         *nflagsp = (unsigned int)(entryp - flags);
 339         EFSYS_ASSERT3U(*nflagsp, <=, EFX_RX_HASH_NFLAGS);
 340
 341         return (0);
 342
 343 fail2:
 344         EFSYS_PROBE(fail2);
 345
 346 fail1:
 347         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 348
 349         return (rc);
 350 }
 351
 352         __checkReturn   efx_rc_t
 353 efx_rx_hash_default_support_get(
 354         __in            efx_nic_t *enp,
 355         __out           efx_rx_hash_support_t *supportp)
 356 {
 357         efx_rc_t rc;
 358
 359         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 360         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 361
 362         if (supportp == NULL) {
 363                 rc = EINVAL;
 364                 goto fail1;
 365         }
 366
 367         /*
 368          * Report the hashing support the client gets by default if it
 369          * does not allocate an RSS context itself.
 370          */
 371         *supportp = enp->en_hash_support;
 372
 373         return (0);
 374
 375 fail1:
 376         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 377
 378         return (rc);
 379 }
 380
 381         __checkReturn   efx_rc_t
 382 efx_rx_scale_default_support_get(
 383         __in            efx_nic_t *enp,
 384         __out           efx_rx_scale_context_type_t *typep)
 385 {
 386         efx_rc_t rc;
 387
 388         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 389         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 390
 391         if (typep == NULL) {
 392                 rc = EINVAL;
 393                 goto fail1;
 394         }
 395
 396         /*
 397          * Report the RSS support the client gets by default if it
 398          * does not allocate an RSS context itself.
 399          */
 400         *typep = enp->en_rss_context_type;
 401
 402         return (0);
 403
 404 fail1:
 405         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 406
 407         return (rc);
 408 }
 409 #endif  /* EFSYS_OPT_RX_SCALE */
 410
 411 #if EFSYS_OPT_RX_SCALE
 412         __checkReturn   efx_rc_t
 413 efx_rx_scale_context_alloc(
 414         __in            efx_nic_t *enp,
 415         __in            efx_rx_scale_context_type_t type,
 416         __in            uint32_t num_queues,
 417         __out           uint32_t *rss_contextp)
 418 {
 419         const efx_rx_ops_t *erxop = enp->en_erxop;
 420         efx_rc_t rc;
 421
 422         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 423         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 424
 425         if (erxop->erxo_scale_context_alloc == NULL) {
 426                 rc = ENOTSUP;
 427                 goto fail1;
 428         }
 429         if ((rc = erxop->erxo_scale_context_alloc(enp, type,
 430                             num_queues, rss_contextp)) != 0) {
 431                 goto fail2;
 432         }
 433
 434         return (0);
 435
 436 fail2:
 437         EFSYS_PROBE(fail2);
 438 fail1:
 439         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 440         return (rc);
 441 }
 442 #endif  /* EFSYS_OPT_RX_SCALE */
 443
 444 #if EFSYS_OPT_RX_SCALE
 445         __checkReturn   efx_rc_t
 446 efx_rx_scale_context_free(
 447         __in            efx_nic_t *enp,
 448         __in            uint32_t rss_context)
 449 {
 450         const efx_rx_ops_t *erxop = enp->en_erxop;
 451         efx_rc_t rc;
 452
 453         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 454         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 455
 456         if (erxop->erxo_scale_context_free == NULL) {
 457                 rc = ENOTSUP;
 458                 goto fail1;
 459         }
 460         if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
 461                 goto fail2;
 462
 463         return (0);
 464
 465 fail2:
 466         EFSYS_PROBE(fail2);
 467 fail1:
 468         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 469         return (rc);
 470 }
 471 #endif  /* EFSYS_OPT_RX_SCALE */
 472
 473 #if EFSYS_OPT_RX_SCALE
 474         __checkReturn   efx_rc_t
 475 efx_rx_scale_mode_set(
 476         __in            efx_nic_t *enp,
 477         __in            uint32_t rss_context,
 478         __in            efx_rx_hash_alg_t alg,
 479         __in            efx_rx_hash_type_t type,
 480         __in            boolean_t insert)
 481 {
 482         const efx_rx_ops_t *erxop = enp->en_erxop;
 483         unsigned int type_flags[EFX_RX_HASH_NFLAGS];
 484         unsigned int type_nflags;
 485         efx_rx_hash_type_t type_check;
 486         unsigned int i;
 487         efx_rc_t rc;
 488
 489         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 490         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 491
 492         /*
 493          * Legacy flags and modern bits cannot be
 494          * used at the same time in the hash type.
 495          */
 496         if ((type & EFX_RX_HASH_LEGACY_MASK) &&
 497             (type & ~EFX_RX_HASH_LEGACY_MASK)) {
 498                 rc = EINVAL;
 499                 goto fail1;
 500         }
 501
 502         /*
 503          * Translate legacy flags to the new representation
 504          * so that chip-specific handlers will consider the
 505          * new flags only.
 506          */
 507         if (type & EFX_RX_HASH_IPV4) {
 508                 type |= EFX_RX_HASH(IPV4, 2TUPLE);
 509                 type |= EFX_RX_HASH(IPV4_TCP, 2TUPLE);
 510         }
 511
 512         if (type & EFX_RX_HASH_TCPIPV4)
 513                 type |= EFX_RX_HASH(IPV4_TCP, 4TUPLE);
 514
 515         if (type & EFX_RX_HASH_IPV6) {
 516                 type |= EFX_RX_HASH(IPV6, 2TUPLE);
 517                 type |= EFX_RX_HASH(IPV6_TCP, 2TUPLE);
 518         }
 519
 520         if (type & EFX_RX_HASH_TCPIPV6)
 521                 type |= EFX_RX_HASH(IPV6_TCP, 4TUPLE);
 522
 523         type &= ~EFX_RX_HASH_LEGACY_MASK;
 524         type_check = type;
 525
 526         /*
 527          * Get the list of supported hash flags and sanitise the input.
 528          */
 529         rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, &type_nflags);
 530         if (rc != 0)
 531                 goto fail2;
 532
 533         for (i = 0; i < type_nflags; ++i) {
 534                 if ((type_check & type_flags[i]) == type_flags[i])
 535                         type_check &= ~(type_flags[i]);
 536         }
 537
 538         if (type_check != 0) {
 539                 rc = EINVAL;
 540                 goto fail3;
 541         }
 542
 543         if (erxop->erxo_scale_mode_set != NULL) {
 544                 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg,
 545                             type, insert)) != 0)
 546                         goto fail4;
 547         }
 548
 549         return (0);
 550
 551 fail4:
 552         EFSYS_PROBE(fail4);
 553 fail3:
 554         EFSYS_PROBE(fail3);
 555 fail2:
 556         EFSYS_PROBE(fail2);
 557 fail1:
 558         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 559         return (rc);
 560 }
 561 #endif  /* EFSYS_OPT_RX_SCALE */
 562
 563 #if EFSYS_OPT_RX_SCALE
 564         __checkReturn   efx_rc_t
 565 efx_rx_scale_key_set(
 566         __in            efx_nic_t *enp,
 567         __in            uint32_t rss_context,
 568         __in_ecount(n)  uint8_t *key,
 569         __in            size_t n)
 570 {
 571         const efx_rx_ops_t *erxop = enp->en_erxop;
 572         efx_rc_t rc;
 573
 574         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 575         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 576
 577         if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0)
 578                 goto fail1;
 579
 580         return (0);
 581
 582 fail1:
 583         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 584
 585         return (rc);
 586 }
 587 #endif  /* EFSYS_OPT_RX_SCALE */
 588
 589 #if EFSYS_OPT_RX_SCALE
 590         __checkReturn   efx_rc_t
 591 efx_rx_scale_tbl_set(
 592         __in            efx_nic_t *enp,
 593         __in            uint32_t rss_context,
 594         __in_ecount(n)  unsigned int *table,
 595         __in            size_t n)
 596 {
 597         const efx_rx_ops_t *erxop = enp->en_erxop;
 598         efx_rc_t rc;
 599
 600         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 601         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 602
 603         if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0)
 604                 goto fail1;
 605
 606         return (0);
 607
 608 fail1:
 609         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 610
 611         return (rc);
 612 }
 613 #endif  /* EFSYS_OPT_RX_SCALE */
 614
 615                                 void
 616 efx_rx_qpost(
 617         __in                    efx_rxq_t *erp,
 618         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 619         __in                    size_t size,
 620         __in                    unsigned int ndescs,
 621         __in                    unsigned int completed,
 622         __in                    unsigned int added)
 623 {
 624         efx_nic_t *enp = erp->er_enp;
 625         const efx_rx_ops_t *erxop = enp->en_erxop;
 626
 627         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 628
 629         erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added);
 630 }
 631
 632 #if EFSYS_OPT_RX_PACKED_STREAM
 633
 634                         void
 635 efx_rx_qpush_ps_credits(
 636         __in            efx_rxq_t *erp)
 637 {
 638         efx_nic_t *enp = erp->er_enp;
 639         const efx_rx_ops_t *erxop = enp->en_erxop;
 640
 641         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 642
 643         erxop->erxo_qpush_ps_credits(erp);
 644 }
 645
 646         __checkReturn   uint8_t *
 647 efx_rx_qps_packet_info(
 648         __in            efx_rxq_t *erp,
 649         __in            uint8_t *buffer,
 650         __in            uint32_t buffer_length,
 651         __in            uint32_t current_offset,
 652         __out           uint16_t *lengthp,
 653         __out           uint32_t *next_offsetp,
 654         __out           uint32_t *timestamp)
 655 {
 656         efx_nic_t *enp = erp->er_enp;
 657         const efx_rx_ops_t *erxop = enp->en_erxop;
 658
 659         return (erxop->erxo_qps_packet_info(erp, buffer,
 660                 buffer_length, current_offset, lengthp,
 661                 next_offsetp, timestamp));
 662 }
 663
 664 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 665
 666                         void
 667 efx_rx_qpush(
 668         __in            efx_rxq_t *erp,
 669         __in            unsigned int added,
 670         __inout         unsigned int *pushedp)
 671 {
 672         efx_nic_t *enp = erp->er_enp;
 673         const efx_rx_ops_t *erxop = enp->en_erxop;
 674
 675         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 676
 677         erxop->erxo_qpush(erp, added, pushedp);
 678 }
 679
 680         __checkReturn   efx_rc_t
 681 efx_rx_qflush(
 682         __in            efx_rxq_t *erp)
 683 {
 684         efx_nic_t *enp = erp->er_enp;
 685         const efx_rx_ops_t *erxop = enp->en_erxop;
 686         efx_rc_t rc;
 687
 688         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 689
 690         if ((rc = erxop->erxo_qflush(erp)) != 0)
 691                 goto fail1;
 692
 693         return (0);
 694
 695 fail1:
 696         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 697
 698         return (rc);
 699 }
 700
 701                         void
 702 efx_rx_qenable(
 703         __in            efx_rxq_t *erp)
 704 {
 705         efx_nic_t *enp = erp->er_enp;
 706         const efx_rx_ops_t *erxop = enp->en_erxop;
 707
 708         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 709
 710         erxop->erxo_qenable(erp);
 711 }
 712
 713 static  __checkReturn   efx_rc_t
 714 efx_rx_qcreate_internal(
 715         __in            efx_nic_t *enp,
 716         __in            unsigned int index,
 717         __in            unsigned int label,
 718         __in            efx_rxq_type_t type,
 719         __in            uint32_t type_data,
 720         __in            efsys_mem_t *esmp,
 721         __in            size_t ndescs,
 722         __in            uint32_t id,
 723         __in            unsigned int flags,
 724         __in            efx_evq_t *eep,
 725         __deref_out     efx_rxq_t **erpp)
 726 {
 727         const efx_rx_ops_t *erxop = enp->en_erxop;
 728         efx_rxq_t *erp;
 729         efx_rc_t rc;
 730
 731         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 732         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 733
 734         /* Allocate an RXQ object */
 735         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 736
 737         if (erp == NULL) {
 738                 rc = ENOMEM;
 739                 goto fail1;
 740         }
 741
 742         erp->er_magic = EFX_RXQ_MAGIC;
 743         erp->er_enp = enp;
 744         erp->er_index = index;
 745         erp->er_mask = ndescs - 1;
 746         erp->er_esmp = esmp;
 747
 748         if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp,
 749             ndescs, id, flags, eep, erp)) != 0)
 750                 goto fail2;
 751
 752         enp->en_rx_qcount++;
 753         *erpp = erp;
 754
 755         return (0);
 756
 757 fail2:
 758         EFSYS_PROBE(fail2);
 759
 760         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 761 fail1:
 762         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 763
 764         return (rc);
 765 }
 766
 767         __checkReturn   efx_rc_t
 768 efx_rx_qcreate(
 769         __in            efx_nic_t *enp,
 770         __in            unsigned int index,
 771         __in            unsigned int label,
 772         __in            efx_rxq_type_t type,
 773         __in            efsys_mem_t *esmp,
 774         __in            size_t ndescs,
 775         __in            uint32_t id,
 776         __in            unsigned int flags,
 777         __in            efx_evq_t *eep,
 778         __deref_out     efx_rxq_t **erpp)
 779 {
 780         return efx_rx_qcreate_internal(enp, index, label, type, 0, esmp, ndescs,
 781             id, flags, eep, erpp);
 782 }
 783
 784 #if EFSYS_OPT_RX_PACKED_STREAM
 785
 786         __checkReturn   efx_rc_t
 787 efx_rx_qcreate_packed_stream(
 788         __in            efx_nic_t *enp,
 789         __in            unsigned int index,
 790         __in            unsigned int label,
 791         __in            uint32_t ps_buf_size,
 792         __in            efsys_mem_t *esmp,
 793         __in            size_t ndescs,
 794         __in            efx_evq_t *eep,
 795         __deref_out     efx_rxq_t **erpp)
 796 {
 797         return efx_rx_qcreate_internal(enp, index, label,
 798             EFX_RXQ_TYPE_PACKED_STREAM, ps_buf_size, esmp, ndescs,
 799             0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp);
 800 }
 801
 802 #endif
 803
 804                         void
 805 efx_rx_qdestroy(
 806         __in            efx_rxq_t *erp)
 807 {
 808         efx_nic_t *enp = erp->er_enp;
 809         const efx_rx_ops_t *erxop = enp->en_erxop;
 810
 811         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 812
 813         erxop->erxo_qdestroy(erp);
 814 }
 815
 816         __checkReturn   efx_rc_t
 817 efx_pseudo_hdr_pkt_length_get(
 818         __in            efx_rxq_t *erp,
 819         __in            uint8_t *buffer,
 820         __out           uint16_t *lengthp)
 821 {
 822         efx_nic_t *enp = erp->er_enp;
 823         const efx_rx_ops_t *erxop = enp->en_erxop;
 824
 825         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 826
 827         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 828 }
 829
 830 #if EFSYS_OPT_RX_SCALE
 831         __checkReturn   uint32_t
 832 efx_pseudo_hdr_hash_get(
 833         __in            efx_rxq_t *erp,
 834         __in            efx_rx_hash_alg_t func,
 835         __in            uint8_t *buffer)
 836 {
 837         efx_nic_t *enp = erp->er_enp;
 838         const efx_rx_ops_t *erxop = enp->en_erxop;
 839
 840         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 841
 842         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 843         return (erxop->erxo_prefix_hash(enp, func, buffer));
 844 }
 845 #endif  /* EFSYS_OPT_RX_SCALE */
 846
 847 #if EFSYS_OPT_SIENA
 848
 849 static  __checkReturn   efx_rc_t
 850 siena_rx_init(
 851         __in            efx_nic_t *enp)
 852 {
 853         efx_oword_t oword;
 854         unsigned int index;
 855
 856         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 857
 858         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 859         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 860         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 861         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 862         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 863         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 864         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 865
 866         /* Zero the RSS table */
 867         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 868             index++) {
 869                 EFX_ZERO_OWORD(oword);
 870                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 871                                     index, &oword, B_TRUE);
 872         }
 873
 874 #if EFSYS_OPT_RX_SCALE
 875         /* The RSS key and indirection table are writable. */
 876         enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 877
 878         /* Hardware can insert RX hash with/without RSS */
 879         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 880 #endif  /* EFSYS_OPT_RX_SCALE */
 881
 882         return (0);
 883 }
 884
 885 #if EFSYS_OPT_RX_SCATTER
 886 static  __checkReturn   efx_rc_t
 887 siena_rx_scatter_enable(
 888         __in            efx_nic_t *enp,
 889         __in            unsigned int buf_size)
 890 {
 891         unsigned int nbuf32;
 892         efx_oword_t oword;
 893         efx_rc_t rc;
 894
 895         nbuf32 = buf_size / 32;
 896         if ((nbuf32 == 0) ||
 897             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 898             ((buf_size % 32) != 0)) {
 899                 rc = EINVAL;
 900                 goto fail1;
 901         }
 902
 903         if (enp->en_rx_qcount > 0) {
 904                 rc = EBUSY;
 905                 goto fail2;
 906         }
 907
 908         /* Set scatter buffer size */
 909         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 910         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 911         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 912
 913         /* Enable scatter for packets not matching a filter */
 914         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 915         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 916         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 917
 918         return (0);
 919
 920 fail2:
 921         EFSYS_PROBE(fail2);
 922 fail1:
 923         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 924
 925         return (rc);
 926 }
 927 #endif  /* EFSYS_OPT_RX_SCATTER */
 928
 929
 930 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 931         do {                                                            \
 932                 efx_oword_t oword;                                      \
 933                                                                         \
 934                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 935                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 936                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 937                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 938                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 939                     (_insert) ? 1 : 0);                                 \
 940                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 941                                                                         \
 942                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 943                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 944                             &oword);                                    \
 945                         EFX_SET_OWORD_FIELD(oword,                      \
 946                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 947                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 948                             &oword);                                    \
 949                 }                                                       \
 950                                                                         \
 951                 _NOTE(CONSTANTCONDITION)                                \
 952         } while (B_FALSE)
 953
 954 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 955         do {                                                            \
 956                 efx_oword_t oword;                                      \
 957                                                                         \
 958                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 959                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 960                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 961                     (_ip) ? 1 : 0);                                     \
 962                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 963                     (_tcp) ? 0 : 1);                                    \
 964                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 965                     (_insert) ? 1 : 0);                                 \
 966                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 967                                                                         \
 968                 _NOTE(CONSTANTCONDITION)                                \
 969         } while (B_FALSE)
 970
 971 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 972         do {                                                            \
 973                 efx_oword_t oword;                                      \
 974                                                                         \
 975                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 976                 EFX_SET_OWORD_FIELD(oword,                              \
 977                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 978                 EFX_SET_OWORD_FIELD(oword,                              \
 979                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 980                 EFX_SET_OWORD_FIELD(oword,                              \
 981                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 982                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 983                                                                         \
 984                 (_rc) = 0;                                              \
 985                                                                         \
 986                 _NOTE(CONSTANTCONDITION)                                \
 987         } while (B_FALSE)
 988
 989
 990 #if EFSYS_OPT_RX_SCALE
 991
 992 static  __checkReturn   efx_rc_t
 993 siena_rx_scale_mode_set(
 994         __in            efx_nic_t *enp,
 995         __in            uint32_t rss_context,
 996         __in            efx_rx_hash_alg_t alg,
 997         __in            efx_rx_hash_type_t type,
 998         __in            boolean_t insert)
 999 {
1000         efx_rx_hash_type_t type_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE);
1001         efx_rx_hash_type_t type_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
1002         efx_rx_hash_type_t type_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE);
1003         efx_rx_hash_type_t type_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
1004         efx_rc_t rc;
1005
1006         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1007                 rc = EINVAL;
1008                 goto fail1;
1009         }
1010
1011         switch (alg) {
1012         case EFX_RX_HASHALG_LFSR:
1013                 EFX_RX_LFSR_HASH(enp, insert);
1014                 break;
1015
1016         case EFX_RX_HASHALG_TOEPLITZ:
1017                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
1018                     (type & type_ipv4) == type_ipv4,
1019                     (type & type_ipv4_tcp) == type_ipv4_tcp);
1020
1021                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
1022                     (type & type_ipv6) == type_ipv6,
1023                     (type & type_ipv6_tcp) == type_ipv6_tcp,
1024                     rc);
1025                 if (rc != 0)
1026                         goto fail2;
1027
1028                 break;
1029
1030         default:
1031                 rc = EINVAL;
1032                 goto fail3;
1033         }
1034
1035         return (0);
1036
1037 fail3:
1038         EFSYS_PROBE(fail3);
1039 fail2:
1040         EFSYS_PROBE(fail2);
1041 fail1:
1042         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1043
1044         EFX_RX_LFSR_HASH(enp, B_FALSE);
1045
1046         return (rc);
1047 }
1048 #endif
1049
1050 #if EFSYS_OPT_RX_SCALE
1051 static  __checkReturn   efx_rc_t
1052 siena_rx_scale_key_set(
1053         __in            efx_nic_t *enp,
1054         __in            uint32_t rss_context,
1055         __in_ecount(n)  uint8_t *key,
1056         __in            size_t n)
1057 {
1058         efx_oword_t oword;
1059         unsigned int byte;
1060         unsigned int offset;
1061         efx_rc_t rc;
1062
1063         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1064                 rc = EINVAL;
1065                 goto fail1;
1066         }
1067
1068         byte = 0;
1069
1070         /* Write Toeplitz IPv4 hash key */
1071         EFX_ZERO_OWORD(oword);
1072         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1073             offset > 0 && byte < n;
1074             --offset)
1075                 oword.eo_u8[offset - 1] = key[byte++];
1076
1077         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1078
1079         byte = 0;
1080
1081         /* Verify Toeplitz IPv4 hash key */
1082         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1083         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1084             offset > 0 && byte < n;
1085             --offset) {
1086                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1087                         rc = EFAULT;
1088                         goto fail2;
1089                 }
1090         }
1091
1092         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
1093                 goto done;
1094
1095         byte = 0;
1096
1097         /* Write Toeplitz IPv6 hash key 3 */
1098         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1099         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1100             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1101             offset > 0 && byte < n;
1102             --offset)
1103                 oword.eo_u8[offset - 1] = key[byte++];
1104
1105         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1106
1107         /* Write Toeplitz IPv6 hash key 2 */
1108         EFX_ZERO_OWORD(oword);
1109         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1110             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1111             offset > 0 && byte < n;
1112             --offset)
1113                 oword.eo_u8[offset - 1] = key[byte++];
1114
1115         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1116
1117         /* Write Toeplitz IPv6 hash key 1 */
1118         EFX_ZERO_OWORD(oword);
1119         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1120             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1121             offset > 0 && byte < n;
1122             --offset)
1123                 oword.eo_u8[offset - 1] = key[byte++];
1124
1125         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1126
1127         byte = 0;
1128
1129         /* Verify Toeplitz IPv6 hash key 3 */
1130         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1131         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1132             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1133             offset > 0 && byte < n;
1134             --offset) {
1135                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1136                         rc = EFAULT;
1137                         goto fail3;
1138                 }
1139         }
1140
1141         /* Verify Toeplitz IPv6 hash key 2 */
1142         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1143         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1144             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1145             offset > 0 && byte < n;
1146             --offset) {
1147                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1148                         rc = EFAULT;
1149                         goto fail4;
1150                 }
1151         }
1152
1153         /* Verify Toeplitz IPv6 hash key 1 */
1154         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1155         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1156             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1157             offset > 0 && byte < n;
1158             --offset) {
1159                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1160                         rc = EFAULT;
1161                         goto fail5;
1162                 }
1163         }
1164
1165 done:
1166         return (0);
1167
1168 fail5:
1169         EFSYS_PROBE(fail5);
1170 fail4:
1171         EFSYS_PROBE(fail4);
1172 fail3:
1173         EFSYS_PROBE(fail3);
1174 fail2:
1175         EFSYS_PROBE(fail2);
1176 fail1:
1177         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1178
1179         return (rc);
1180 }
1181 #endif
1182
1183 #if EFSYS_OPT_RX_SCALE
1184 static  __checkReturn   efx_rc_t
1185 siena_rx_scale_tbl_set(
1186         __in            efx_nic_t *enp,
1187         __in            uint32_t rss_context,
1188         __in_ecount(n)  unsigned int *table,
1189         __in            size_t n)
1190 {
1191         efx_oword_t oword;
1192         int index;
1193         efx_rc_t rc;
1194
1195         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1196         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1197
1198         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1199                 rc = EINVAL;
1200                 goto fail1;
1201         }
1202
1203         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1204                 rc = EINVAL;
1205                 goto fail2;
1206         }
1207
1208         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1209                 uint32_t byte;
1210
1211                 /* Calculate the entry to place in the table */
1212                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1213
1214                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1215
1216                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1217
1218                 /* Write the table */
1219                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1220                                     index, &oword, B_TRUE);
1221         }
1222
1223         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1224                 uint32_t byte;
1225
1226                 /* Determine if we're starting a new batch */
1227                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1228
1229                 /* Read the table */
1230                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1231                                     index, &oword, B_TRUE);
1232
1233                 /* Verify the entry */
1234                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1235                         rc = EFAULT;
1236                         goto fail3;
1237                 }
1238         }
1239
1240         return (0);
1241
1242 fail3:
1243         EFSYS_PROBE(fail3);
1244 fail2:
1245         EFSYS_PROBE(fail2);
1246 fail1:
1247         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1248
1249         return (rc);
1250 }
1251 #endif
1252
1253 /*
1254  * Falcon/Siena pseudo-header
1255  * --------------------------
1256  *
1257  * Receive packets are prefixed by an optional 16 byte pseudo-header.
1258  * The pseudo-header is a byte array of one of the forms:
1259  *
1260  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
1261  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1262  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1263  *
1264  * where:
1265  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
1266  *   LL.LL         LFSR hash     (16-bit big-endian)
1267  */
1268
1269 #if EFSYS_OPT_RX_SCALE
1270 static  __checkReturn   uint32_t
1271 siena_rx_prefix_hash(
1272         __in            efx_nic_t *enp,
1273         __in            efx_rx_hash_alg_t func,
1274         __in            uint8_t *buffer)
1275 {
1276         _NOTE(ARGUNUSED(enp))
1277
1278         switch (func) {
1279         case EFX_RX_HASHALG_TOEPLITZ:
1280                 return ((buffer[12] << 24) |
1281                     (buffer[13] << 16) |
1282                     (buffer[14] <<  8) |
1283                     buffer[15]);
1284
1285         case EFX_RX_HASHALG_LFSR:
1286                 return ((buffer[14] << 8) | buffer[15]);
1287
1288         default:
1289                 EFSYS_ASSERT(0);
1290                 return (0);
1291         }
1292 }
1293 #endif /* EFSYS_OPT_RX_SCALE */
1294
1295 static  __checkReturn   efx_rc_t
1296 siena_rx_prefix_pktlen(
1297         __in            efx_nic_t *enp,
1298         __in            uint8_t *buffer,
1299         __out           uint16_t *lengthp)
1300 {
1301         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1302
1303         /* Not supported by Falcon/Siena hardware */
1304         EFSYS_ASSERT(0);
1305         return (ENOTSUP);
1306 }
1307
1308
1309 static                          void
1310 siena_rx_qpost(
1311         __in                    efx_rxq_t *erp,
1312         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
1313         __in                    size_t size,
1314         __in                    unsigned int ndescs,
1315         __in                    unsigned int completed,
1316         __in                    unsigned int added)
1317 {
1318         efx_qword_t qword;
1319         unsigned int i;
1320         unsigned int offset;
1321         unsigned int id;
1322
1323         /* The client driver must not overfill the queue */
1324         EFSYS_ASSERT3U(added - completed + ndescs, <=,
1325             EFX_RXQ_LIMIT(erp->er_mask + 1));
1326
1327         id = added & (erp->er_mask);
1328         for (i = 0; i < ndescs; i++) {
1329                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1330                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1331                     size_t, size);
1332
1333                 EFX_POPULATE_QWORD_3(qword,
1334                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1335                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1336                     (uint32_t)(addrp[i] & 0xffffffff),
1337                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1338                     (uint32_t)(addrp[i] >> 32));
1339
1340                 offset = id * sizeof (efx_qword_t);
1341                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1342
1343                 id = (id + 1) & (erp->er_mask);
1344         }
1345 }
1346
1347 static                  void
1348 siena_rx_qpush(
1349         __in    efx_rxq_t *erp,
1350         __in    unsigned int added,
1351         __inout unsigned int *pushedp)
1352 {
1353         efx_nic_t *enp = erp->er_enp;
1354         unsigned int pushed = *pushedp;
1355         uint32_t wptr;
1356         efx_oword_t oword;
1357         efx_dword_t dword;
1358
1359         /* All descriptors are pushed */
1360         *pushedp = added;
1361
1362         /* Push the populated descriptors out */
1363         wptr = added & erp->er_mask;
1364
1365         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1366
1367         /* Only write the third DWORD */
1368         EFX_POPULATE_DWORD_1(dword,
1369             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1370
1371         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1372         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1373             wptr, pushed & erp->er_mask);
1374         EFSYS_PIO_WRITE_BARRIER();
1375         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1376                             erp->er_index, &dword, B_FALSE);
1377 }
1378
1379 #if EFSYS_OPT_RX_PACKED_STREAM
1380 static          void
1381 siena_rx_qpush_ps_credits(
1382         __in            efx_rxq_t *erp)
1383 {
1384         /* Not supported by Siena hardware */
1385         EFSYS_ASSERT(0);
1386 }
1387
1388 static          uint8_t *
1389 siena_rx_qps_packet_info(
1390         __in            efx_rxq_t *erp,
1391         __in            uint8_t *buffer,
1392         __in            uint32_t buffer_length,
1393         __in            uint32_t current_offset,
1394         __out           uint16_t *lengthp,
1395         __out           uint32_t *next_offsetp,
1396         __out           uint32_t *timestamp)
1397 {
1398         /* Not supported by Siena hardware */
1399         EFSYS_ASSERT(0);
1400
1401         return (NULL);
1402 }
1403 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1404
1405 static  __checkReturn   efx_rc_t
1406 siena_rx_qflush(
1407         __in    efx_rxq_t *erp)
1408 {
1409         efx_nic_t *enp = erp->er_enp;
1410         efx_oword_t oword;
1411         uint32_t label;
1412
1413         label = erp->er_index;
1414
1415         /* Flush the queue */
1416         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1417             FRF_AZ_RX_FLUSH_DESCQ, label);
1418         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1419
1420         return (0);
1421 }
1422
1423 static          void
1424 siena_rx_qenable(
1425         __in    efx_rxq_t *erp)
1426 {
1427         efx_nic_t *enp = erp->er_enp;
1428         efx_oword_t oword;
1429
1430         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1431
1432         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1433                             erp->er_index, &oword, B_TRUE);
1434
1435         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1436         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1437         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1438
1439         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1440                             erp->er_index, &oword, B_TRUE);
1441 }
1442
1443 static  __checkReturn   efx_rc_t
1444 siena_rx_qcreate(
1445         __in            efx_nic_t *enp,
1446         __in            unsigned int index,
1447         __in            unsigned int label,
1448         __in            efx_rxq_type_t type,
1449         __in            uint32_t type_data,
1450         __in            efsys_mem_t *esmp,
1451         __in            size_t ndescs,
1452         __in            uint32_t id,
1453         __in            unsigned int flags,
1454         __in            efx_evq_t *eep,
1455         __in            efx_rxq_t *erp)
1456 {
1457         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1458         efx_oword_t oword;
1459         uint32_t size;
1460         boolean_t jumbo = B_FALSE;
1461         efx_rc_t rc;
1462
1463         _NOTE(ARGUNUSED(esmp))
1464         _NOTE(ARGUNUSED(type_data))
1465
1466         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1467             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1468         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1469         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1470
1471         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1472         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1473
1474         if (!ISP2(ndescs) ||
1475             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1476                 rc = EINVAL;
1477                 goto fail1;
1478         }
1479         if (index >= encp->enc_rxq_limit) {
1480                 rc = EINVAL;
1481                 goto fail2;
1482         }
1483         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1484             size++)
1485                 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS))
1486                         break;
1487         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1488                 rc = EINVAL;
1489                 goto fail3;
1490         }
1491
1492         switch (type) {
1493         case EFX_RXQ_TYPE_DEFAULT:
1494                 break;
1495
1496         default:
1497                 rc = EINVAL;
1498                 goto fail4;
1499         }
1500
1501         if (flags & EFX_RXQ_FLAG_SCATTER) {
1502 #if EFSYS_OPT_RX_SCATTER
1503                 jumbo = B_TRUE;
1504 #else
1505                 rc = EINVAL;
1506                 goto fail5;
1507 #endif  /* EFSYS_OPT_RX_SCATTER */
1508         }
1509
1510         /* Set up the new descriptor queue */
1511         EFX_POPULATE_OWORD_7(oword,
1512             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1513             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1514             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1515             FRF_AZ_RX_DESCQ_LABEL, label,
1516             FRF_AZ_RX_DESCQ_SIZE, size,
1517             FRF_AZ_RX_DESCQ_TYPE, 0,
1518             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1519
1520         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1521                             erp->er_index, &oword, B_TRUE);
1522
1523         return (0);
1524
1525 #if !EFSYS_OPT_RX_SCATTER
1526 fail5:
1527         EFSYS_PROBE(fail5);
1528 #endif
1529 fail4:
1530         EFSYS_PROBE(fail4);
1531 fail3:
1532         EFSYS_PROBE(fail3);
1533 fail2:
1534         EFSYS_PROBE(fail2);
1535 fail1:
1536         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1537
1538         return (rc);
1539 }
1540
1541 static          void
1542 siena_rx_qdestroy(
1543         __in    efx_rxq_t *erp)
1544 {
1545         efx_nic_t *enp = erp->er_enp;
1546         efx_oword_t oword;
1547
1548         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1549         --enp->en_rx_qcount;
1550
1551         /* Purge descriptor queue */
1552         EFX_ZERO_OWORD(oword);
1553
1554         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1555                             erp->er_index, &oword, B_TRUE);
1556
1557         /* Free the RXQ object */
1558         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1559 }
1560
1561 static          void
1562 siena_rx_fini(
1563         __in    efx_nic_t *enp)
1564 {
1565         _NOTE(ARGUNUSED(enp))
1566 }
1567
1568 #endif /* EFSYS_OPT_SIENA */