drivers/net/sfc/base/efx_rx.c

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_SIENA
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_init(
  39         __in            efx_nic_t *enp);
  40
  41 static                  void
  42 siena_rx_fini(
  43         __in            efx_nic_t *enp);
  44
  45 #if EFSYS_OPT_RX_SCATTER
  46 static  __checkReturn   efx_rc_t
  47 siena_rx_scatter_enable(
  48         __in            efx_nic_t *enp,
  49         __in            unsigned int buf_size);
  50 #endif /* EFSYS_OPT_RX_SCATTER */
  51
  52 #if EFSYS_OPT_RX_SCALE
  53 static  __checkReturn   efx_rc_t
  54 siena_rx_scale_mode_set(
  55         __in            efx_nic_t *enp,
  56         __in            efx_rx_hash_alg_t alg,
  57         __in            efx_rx_hash_type_t type,
  58         __in            boolean_t insert);
  59
  60 static  __checkReturn   efx_rc_t
  61 siena_rx_scale_key_set(
  62         __in            efx_nic_t *enp,
  63         __in_ecount(n)  uint8_t *key,
  64         __in            size_t n);
  65
  66 static  __checkReturn   efx_rc_t
  67 siena_rx_scale_tbl_set(
  68         __in            efx_nic_t *enp,
  69         __in_ecount(n)  unsigned int *table,
  70         __in            size_t n);
  71
  72 static  __checkReturn   uint32_t
  73 siena_rx_prefix_hash(
  74         __in            efx_nic_t *enp,
  75         __in            efx_rx_hash_alg_t func,
  76         __in            uint8_t *buffer);
  77
  78 #endif /* EFSYS_OPT_RX_SCALE */
  79
  80 static  __checkReturn   efx_rc_t
  81 siena_rx_prefix_pktlen(
  82         __in            efx_nic_t *enp,
  83         __in            uint8_t *buffer,
  84         __out           uint16_t *lengthp);
  85
  86 static                  void
  87 siena_rx_qpost(
  88         __in            efx_rxq_t *erp,
  89         __in_ecount(n)  efsys_dma_addr_t *addrp,
  90         __in            size_t size,
  91         __in            unsigned int n,
  92         __in            unsigned int completed,
  93         __in            unsigned int added);
  94
  95 static                  void
  96 siena_rx_qpush(
  97         __in            efx_rxq_t *erp,
  98         __in            unsigned int added,
  99         __inout         unsigned int *pushedp);
 100
 101 #if EFSYS_OPT_RX_PACKED_STREAM
 102 static          void
 103 siena_rx_qps_update_credits(
 104         __in            efx_rxq_t *erp);
 105
 106 static  __checkReturn   uint8_t *
 107 siena_rx_qps_packet_info(
 108         __in            efx_rxq_t *erp,
 109         __in            uint8_t *buffer,
 110         __in            uint32_t buffer_length,
 111         __in            uint32_t current_offset,
 112         __out           uint16_t *lengthp,
 113         __out           uint32_t *next_offsetp,
 114         __out           uint32_t *timestamp);
 115 #endif
 116
 117 static  __checkReturn   efx_rc_t
 118 siena_rx_qflush(
 119         __in            efx_rxq_t *erp);
 120
 121 static                  void
 122 siena_rx_qenable(
 123         __in            efx_rxq_t *erp);
 124
 125 static  __checkReturn   efx_rc_t
 126 siena_rx_qcreate(
 127         __in            efx_nic_t *enp,
 128         __in            unsigned int index,
 129         __in            unsigned int label,
 130         __in            efx_rxq_type_t type,
 131         __in            efsys_mem_t *esmp,
 132         __in            size_t n,
 133         __in            uint32_t id,
 134         __in            efx_evq_t *eep,
 135         __in            efx_rxq_t *erp);
 136
 137 static                  void
 138 siena_rx_qdestroy(
 139         __in            efx_rxq_t *erp);
 140
 141 #endif /* EFSYS_OPT_SIENA */
 142
 143
 144 #if EFSYS_OPT_SIENA
 145 static const efx_rx_ops_t __efx_rx_siena_ops = {
 146         siena_rx_init,                          /* erxo_init */
 147         siena_rx_fini,                          /* erxo_fini */
 148 #if EFSYS_OPT_RX_SCATTER
 149         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 150 #endif
 151 #if EFSYS_OPT_RX_SCALE
 152         NULL,                                   /* erxo_scale_context_alloc */
 153         NULL,                                   /* erxo_scale_context_free */
 154         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 155         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 156         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 157         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 158 #endif
 159         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 160         siena_rx_qpost,                         /* erxo_qpost */
 161         siena_rx_qpush,                         /* erxo_qpush */
 162 #if EFSYS_OPT_RX_PACKED_STREAM
 163         siena_rx_qps_update_credits,            /* erxo_qps_update_credits */
 164         siena_rx_qps_packet_info,               /* erxo_qps_packet_info */
 165 #endif
 166         siena_rx_qflush,                        /* erxo_qflush */
 167         siena_rx_qenable,                       /* erxo_qenable */
 168         siena_rx_qcreate,                       /* erxo_qcreate */
 169         siena_rx_qdestroy,                      /* erxo_qdestroy */
 170 };
 171 #endif  /* EFSYS_OPT_SIENA */
 172
 173 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 174 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 175         ef10_rx_init,                           /* erxo_init */
 176         ef10_rx_fini,                           /* erxo_fini */
 177 #if EFSYS_OPT_RX_SCATTER
 178         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 179 #endif
 180 #if EFSYS_OPT_RX_SCALE
 181         ef10_rx_scale_context_alloc,            /* erxo_scale_context_alloc */
 182         ef10_rx_scale_context_free,             /* erxo_scale_context_free */
 183         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 184         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 185         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 186         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 187 #endif
 188         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 189         ef10_rx_qpost,                          /* erxo_qpost */
 190         ef10_rx_qpush,                          /* erxo_qpush */
 191 #if EFSYS_OPT_RX_PACKED_STREAM
 192         ef10_rx_qps_update_credits,             /* erxo_qps_update_credits */
 193         ef10_rx_qps_packet_info,                /* erxo_qps_packet_info */
 194 #endif
 195         ef10_rx_qflush,                         /* erxo_qflush */
 196         ef10_rx_qenable,                        /* erxo_qenable */
 197         ef10_rx_qcreate,                        /* erxo_qcreate */
 198         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 199 };
 200 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 201
 202
 203         __checkReturn   efx_rc_t
 204 efx_rx_init(
 205         __inout         efx_nic_t *enp)
 206 {
 207         const efx_rx_ops_t *erxop;
 208         efx_rc_t rc;
 209
 210         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 211         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 212
 213         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 214                 rc = EINVAL;
 215                 goto fail1;
 216         }
 217
 218         if (enp->en_mod_flags & EFX_MOD_RX) {
 219                 rc = EINVAL;
 220                 goto fail2;
 221         }
 222
 223         switch (enp->en_family) {
 224 #if EFSYS_OPT_SIENA
 225         case EFX_FAMILY_SIENA:
 226                 erxop = &__efx_rx_siena_ops;
 227                 break;
 228 #endif /* EFSYS_OPT_SIENA */
 229
 230 #if EFSYS_OPT_HUNTINGTON
 231         case EFX_FAMILY_HUNTINGTON:
 232                 erxop = &__efx_rx_ef10_ops;
 233                 break;
 234 #endif /* EFSYS_OPT_HUNTINGTON */
 235
 236 #if EFSYS_OPT_MEDFORD
 237         case EFX_FAMILY_MEDFORD:
 238                 erxop = &__efx_rx_ef10_ops;
 239                 break;
 240 #endif /* EFSYS_OPT_MEDFORD */
 241
 242         default:
 243                 EFSYS_ASSERT(0);
 244                 rc = ENOTSUP;
 245                 goto fail3;
 246         }
 247
 248         if ((rc = erxop->erxo_init(enp)) != 0)
 249                 goto fail4;
 250
 251         enp->en_erxop = erxop;
 252         enp->en_mod_flags |= EFX_MOD_RX;
 253         return (0);
 254
 255 fail4:
 256         EFSYS_PROBE(fail4);
 257 fail3:
 258         EFSYS_PROBE(fail3);
 259 fail2:
 260         EFSYS_PROBE(fail2);
 261 fail1:
 262         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 263
 264         enp->en_erxop = NULL;
 265         enp->en_mod_flags &= ~EFX_MOD_RX;
 266         return (rc);
 267 }
 268
 269                         void
 270 efx_rx_fini(
 271         __in            efx_nic_t *enp)
 272 {
 273         const efx_rx_ops_t *erxop = enp->en_erxop;
 274
 275         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 276         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 277         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 278         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 279
 280         erxop->erxo_fini(enp);
 281
 282         enp->en_erxop = NULL;
 283         enp->en_mod_flags &= ~EFX_MOD_RX;
 284 }
 285
 286 #if EFSYS_OPT_RX_SCATTER
 287         __checkReturn   efx_rc_t
 288 efx_rx_scatter_enable(
 289         __in            efx_nic_t *enp,
 290         __in            unsigned int buf_size)
 291 {
 292         const efx_rx_ops_t *erxop = enp->en_erxop;
 293         efx_rc_t rc;
 294
 295         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 296         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 297
 298         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 299                 goto fail1;
 300
 301         return (0);
 302
 303 fail1:
 304         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 305         return (rc);
 306 }
 307 #endif  /* EFSYS_OPT_RX_SCATTER */
 308
 309 #if EFSYS_OPT_RX_SCALE
 310         __checkReturn   efx_rc_t
 311 efx_rx_hash_default_support_get(
 312         __in            efx_nic_t *enp,
 313         __out           efx_rx_hash_support_t *supportp)
 314 {
 315         efx_rc_t rc;
 316
 317         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 318         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 319
 320         if (supportp == NULL) {
 321                 rc = EINVAL;
 322                 goto fail1;
 323         }
 324
 325         /*
 326          * Report the hashing support the client gets by default if it
 327          * does not allocate an RSS context itself.
 328          */
 329         *supportp = enp->en_hash_support;
 330
 331         return (0);
 332
 333 fail1:
 334         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 335
 336         return (rc);
 337 }
 338
 339         __checkReturn   efx_rc_t
 340 efx_rx_scale_default_support_get(
 341         __in            efx_nic_t *enp,
 342         __out           efx_rx_scale_context_type_t *typep)
 343 {
 344         efx_rc_t rc;
 345
 346         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 347         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 348
 349         if (typep == NULL) {
 350                 rc = EINVAL;
 351                 goto fail1;
 352         }
 353
 354         /*
 355          * Report the RSS support the client gets by default if it
 356          * does not allocate an RSS context itself.
 357          */
 358         *typep = enp->en_rss_context_type;
 359
 360         return (0);
 361
 362 fail1:
 363         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 364
 365         return (rc);
 366 }
 367 #endif  /* EFSYS_OPT_RX_SCALE */
 368
 369 #if EFSYS_OPT_RX_SCALE
 370         __checkReturn   efx_rc_t
 371 efx_rx_scale_context_alloc(
 372         __in            efx_nic_t *enp,
 373         __in            efx_rx_scale_context_type_t type,
 374         __in            uint32_t num_queues,
 375         __out           uint32_t *rss_contextp)
 376 {
 377         const efx_rx_ops_t *erxop = enp->en_erxop;
 378         efx_rc_t rc;
 379
 380         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 381         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 382
 383         if (erxop->erxo_scale_context_alloc == NULL) {
 384                 rc = ENOTSUP;
 385                 goto fail1;
 386         }
 387         if ((rc = erxop->erxo_scale_context_alloc(enp, type,
 388                             num_queues, rss_contextp)) != 0) {
 389                 goto fail2;
 390         }
 391
 392         return (0);
 393
 394 fail2:
 395         EFSYS_PROBE(fail2);
 396 fail1:
 397         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 398         return (rc);
 399 }
 400 #endif  /* EFSYS_OPT_RX_SCALE */
 401
 402 #if EFSYS_OPT_RX_SCALE
 403         __checkReturn   efx_rc_t
 404 efx_rx_scale_context_free(
 405         __in            efx_nic_t *enp,
 406         __in            uint32_t rss_context)
 407 {
 408         const efx_rx_ops_t *erxop = enp->en_erxop;
 409         efx_rc_t rc;
 410
 411         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 412         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 413
 414         if (erxop->erxo_scale_context_free == NULL) {
 415                 rc = ENOTSUP;
 416                 goto fail1;
 417         }
 418         if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
 419                 goto fail2;
 420
 421         return (0);
 422
 423 fail2:
 424         EFSYS_PROBE(fail2);
 425 fail1:
 426         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 427         return (rc);
 428 }
 429 #endif  /* EFSYS_OPT_RX_SCALE */
 430
 431 #if EFSYS_OPT_RX_SCALE
 432         __checkReturn   efx_rc_t
 433 efx_rx_scale_mode_set(
 434         __in            efx_nic_t *enp,
 435         __in            efx_rx_hash_alg_t alg,
 436         __in            efx_rx_hash_type_t type,
 437         __in            boolean_t insert)
 438 {
 439         const efx_rx_ops_t *erxop = enp->en_erxop;
 440         efx_rc_t rc;
 441
 442         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 443         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 444
 445         if (erxop->erxo_scale_mode_set != NULL) {
 446                 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
 447                             type, insert)) != 0)
 448                         goto fail1;
 449         }
 450
 451         return (0);
 452
 453 fail1:
 454         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 455         return (rc);
 456 }
 457 #endif  /* EFSYS_OPT_RX_SCALE */
 458
 459 #if EFSYS_OPT_RX_SCALE
 460         __checkReturn   efx_rc_t
 461 efx_rx_scale_key_set(
 462         __in            efx_nic_t *enp,
 463         __in_ecount(n)  uint8_t *key,
 464         __in            size_t n)
 465 {
 466         const efx_rx_ops_t *erxop = enp->en_erxop;
 467         efx_rc_t rc;
 468
 469         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 470         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 471
 472         if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
 473                 goto fail1;
 474
 475         return (0);
 476
 477 fail1:
 478         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 479
 480         return (rc);
 481 }
 482 #endif  /* EFSYS_OPT_RX_SCALE */
 483
 484 #if EFSYS_OPT_RX_SCALE
 485         __checkReturn   efx_rc_t
 486 efx_rx_scale_tbl_set(
 487         __in            efx_nic_t *enp,
 488         __in_ecount(n)  unsigned int *table,
 489         __in            size_t n)
 490 {
 491         const efx_rx_ops_t *erxop = enp->en_erxop;
 492         efx_rc_t rc;
 493
 494         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 495         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 496
 497         if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
 498                 goto fail1;
 499
 500         return (0);
 501
 502 fail1:
 503         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 504
 505         return (rc);
 506 }
 507 #endif  /* EFSYS_OPT_RX_SCALE */
 508
 509                         void
 510 efx_rx_qpost(
 511         __in            efx_rxq_t *erp,
 512         __in_ecount(n)  efsys_dma_addr_t *addrp,
 513         __in            size_t size,
 514         __in            unsigned int n,
 515         __in            unsigned int completed,
 516         __in            unsigned int added)
 517 {
 518         efx_nic_t *enp = erp->er_enp;
 519         const efx_rx_ops_t *erxop = enp->en_erxop;
 520
 521         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 522
 523         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 524 }
 525
 526 #if EFSYS_OPT_RX_PACKED_STREAM
 527
 528                         void
 529 efx_rx_qps_update_credits(
 530         __in            efx_rxq_t *erp)
 531 {
 532         efx_nic_t *enp = erp->er_enp;
 533         const efx_rx_ops_t *erxop = enp->en_erxop;
 534
 535         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 536
 537         erxop->erxo_qps_update_credits(erp);
 538 }
 539
 540         __checkReturn   uint8_t *
 541 efx_rx_qps_packet_info(
 542         __in            efx_rxq_t *erp,
 543         __in            uint8_t *buffer,
 544         __in            uint32_t buffer_length,
 545         __in            uint32_t current_offset,
 546         __out           uint16_t *lengthp,
 547         __out           uint32_t *next_offsetp,
 548         __out           uint32_t *timestamp)
 549 {
 550         efx_nic_t *enp = erp->er_enp;
 551         const efx_rx_ops_t *erxop = enp->en_erxop;
 552
 553         return (erxop->erxo_qps_packet_info(erp, buffer,
 554                 buffer_length, current_offset, lengthp,
 555                 next_offsetp, timestamp));
 556 }
 557
 558 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 559
 560                         void
 561 efx_rx_qpush(
 562         __in            efx_rxq_t *erp,
 563         __in            unsigned int added,
 564         __inout         unsigned int *pushedp)
 565 {
 566         efx_nic_t *enp = erp->er_enp;
 567         const efx_rx_ops_t *erxop = enp->en_erxop;
 568
 569         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 570
 571         erxop->erxo_qpush(erp, added, pushedp);
 572 }
 573
 574         __checkReturn   efx_rc_t
 575 efx_rx_qflush(
 576         __in            efx_rxq_t *erp)
 577 {
 578         efx_nic_t *enp = erp->er_enp;
 579         const efx_rx_ops_t *erxop = enp->en_erxop;
 580         efx_rc_t rc;
 581
 582         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 583
 584         if ((rc = erxop->erxo_qflush(erp)) != 0)
 585                 goto fail1;
 586
 587         return (0);
 588
 589 fail1:
 590         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 591
 592         return (rc);
 593 }
 594
 595                         void
 596 efx_rx_qenable(
 597         __in            efx_rxq_t *erp)
 598 {
 599         efx_nic_t *enp = erp->er_enp;
 600         const efx_rx_ops_t *erxop = enp->en_erxop;
 601
 602         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 603
 604         erxop->erxo_qenable(erp);
 605 }
 606
 607         __checkReturn   efx_rc_t
 608 efx_rx_qcreate(
 609         __in            efx_nic_t *enp,
 610         __in            unsigned int index,
 611         __in            unsigned int label,
 612         __in            efx_rxq_type_t type,
 613         __in            efsys_mem_t *esmp,
 614         __in            size_t n,
 615         __in            uint32_t id,
 616         __in            efx_evq_t *eep,
 617         __deref_out     efx_rxq_t **erpp)
 618 {
 619         const efx_rx_ops_t *erxop = enp->en_erxop;
 620         efx_rxq_t *erp;
 621         efx_rc_t rc;
 622
 623         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 624         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 625
 626         /* Allocate an RXQ object */
 627         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 628
 629         if (erp == NULL) {
 630                 rc = ENOMEM;
 631                 goto fail1;
 632         }
 633
 634         erp->er_magic = EFX_RXQ_MAGIC;
 635         erp->er_enp = enp;
 636         erp->er_index = index;
 637         erp->er_mask = n - 1;
 638         erp->er_esmp = esmp;
 639
 640         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 641             eep, erp)) != 0)
 642                 goto fail2;
 643
 644         enp->en_rx_qcount++;
 645         *erpp = erp;
 646
 647         return (0);
 648
 649 fail2:
 650         EFSYS_PROBE(fail2);
 651
 652         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 653 fail1:
 654         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 655
 656         return (rc);
 657 }
 658
 659                         void
 660 efx_rx_qdestroy(
 661         __in            efx_rxq_t *erp)
 662 {
 663         efx_nic_t *enp = erp->er_enp;
 664         const efx_rx_ops_t *erxop = enp->en_erxop;
 665
 666         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 667
 668         erxop->erxo_qdestroy(erp);
 669 }
 670
 671         __checkReturn   efx_rc_t
 672 efx_pseudo_hdr_pkt_length_get(
 673         __in            efx_rxq_t *erp,
 674         __in            uint8_t *buffer,
 675         __out           uint16_t *lengthp)
 676 {
 677         efx_nic_t *enp = erp->er_enp;
 678         const efx_rx_ops_t *erxop = enp->en_erxop;
 679
 680         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 681
 682         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 683 }
 684
 685 #if EFSYS_OPT_RX_SCALE
 686         __checkReturn   uint32_t
 687 efx_pseudo_hdr_hash_get(
 688         __in            efx_rxq_t *erp,
 689         __in            efx_rx_hash_alg_t func,
 690         __in            uint8_t *buffer)
 691 {
 692         efx_nic_t *enp = erp->er_enp;
 693         const efx_rx_ops_t *erxop = enp->en_erxop;
 694
 695         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 696
 697         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 698         return (erxop->erxo_prefix_hash(enp, func, buffer));
 699 }
 700 #endif  /* EFSYS_OPT_RX_SCALE */
 701
 702 #if EFSYS_OPT_SIENA
 703
 704 static  __checkReturn   efx_rc_t
 705 siena_rx_init(
 706         __in            efx_nic_t *enp)
 707 {
 708         efx_oword_t oword;
 709         unsigned int index;
 710
 711         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 712
 713         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 714         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 715         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 716         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 717         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 718         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 719         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 720
 721         /* Zero the RSS table */
 722         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 723             index++) {
 724                 EFX_ZERO_OWORD(oword);
 725                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 726                                     index, &oword, B_TRUE);
 727         }
 728
 729 #if EFSYS_OPT_RX_SCALE
 730         /* The RSS key and indirection table are writable. */
 731         enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 732
 733         /* Hardware can insert RX hash with/without RSS */
 734         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 735 #endif  /* EFSYS_OPT_RX_SCALE */
 736
 737         return (0);
 738 }
 739
 740 #if EFSYS_OPT_RX_SCATTER
 741 static  __checkReturn   efx_rc_t
 742 siena_rx_scatter_enable(
 743         __in            efx_nic_t *enp,
 744         __in            unsigned int buf_size)
 745 {
 746         unsigned int nbuf32;
 747         efx_oword_t oword;
 748         efx_rc_t rc;
 749
 750         nbuf32 = buf_size / 32;
 751         if ((nbuf32 == 0) ||
 752             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 753             ((buf_size % 32) != 0)) {
 754                 rc = EINVAL;
 755                 goto fail1;
 756         }
 757
 758         if (enp->en_rx_qcount > 0) {
 759                 rc = EBUSY;
 760                 goto fail2;
 761         }
 762
 763         /* Set scatter buffer size */
 764         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 765         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 766         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 767
 768         /* Enable scatter for packets not matching a filter */
 769         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 770         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 771         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 772
 773         return (0);
 774
 775 fail2:
 776         EFSYS_PROBE(fail2);
 777 fail1:
 778         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 779
 780         return (rc);
 781 }
 782 #endif  /* EFSYS_OPT_RX_SCATTER */
 783
 784
 785 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 786         do {                                                            \
 787                 efx_oword_t oword;                                      \
 788                                                                         \
 789                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 790                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 791                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 792                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 793                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 794                     (_insert) ? 1 : 0);                                 \
 795                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 796                                                                         \
 797                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 798                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 799                             &oword);                                    \
 800                         EFX_SET_OWORD_FIELD(oword,                      \
 801                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 802                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 803                             &oword);                                    \
 804                 }                                                       \
 805                                                                         \
 806                 _NOTE(CONSTANTCONDITION)                                \
 807         } while (B_FALSE)
 808
 809 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 810         do {                                                            \
 811                 efx_oword_t oword;                                      \
 812                                                                         \
 813                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 814                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 815                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 816                     (_ip) ? 1 : 0);                                     \
 817                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 818                     (_tcp) ? 0 : 1);                                    \
 819                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 820                     (_insert) ? 1 : 0);                                 \
 821                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 822                                                                         \
 823                 _NOTE(CONSTANTCONDITION)                                \
 824         } while (B_FALSE)
 825
 826 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 827         do {                                                            \
 828                 efx_oword_t oword;                                      \
 829                                                                         \
 830                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 831                 EFX_SET_OWORD_FIELD(oword,                              \
 832                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 833                 EFX_SET_OWORD_FIELD(oword,                              \
 834                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 835                 EFX_SET_OWORD_FIELD(oword,                              \
 836                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 837                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 838                                                                         \
 839                 (_rc) = 0;                                              \
 840                                                                         \
 841                 _NOTE(CONSTANTCONDITION)                                \
 842         } while (B_FALSE)
 843
 844
 845 #if EFSYS_OPT_RX_SCALE
 846
 847 static  __checkReturn   efx_rc_t
 848 siena_rx_scale_mode_set(
 849         __in            efx_nic_t *enp,
 850         __in            efx_rx_hash_alg_t alg,
 851         __in            efx_rx_hash_type_t type,
 852         __in            boolean_t insert)
 853 {
 854         efx_rc_t rc;
 855
 856         switch (alg) {
 857         case EFX_RX_HASHALG_LFSR:
 858                 EFX_RX_LFSR_HASH(enp, insert);
 859                 break;
 860
 861         case EFX_RX_HASHALG_TOEPLITZ:
 862                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 863                     type & EFX_RX_HASH_IPV4,
 864                     type & EFX_RX_HASH_TCPIPV4);
 865
 866                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 867                     type & EFX_RX_HASH_IPV6,
 868                     type & EFX_RX_HASH_TCPIPV6,
 869                     rc);
 870                 if (rc != 0)
 871                         goto fail1;
 872
 873                 break;
 874
 875         default:
 876                 rc = EINVAL;
 877                 goto fail2;
 878         }
 879
 880         return (0);
 881
 882 fail2:
 883         EFSYS_PROBE(fail2);
 884 fail1:
 885         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 886
 887         EFX_RX_LFSR_HASH(enp, B_FALSE);
 888
 889         return (rc);
 890 }
 891 #endif
 892
 893 #if EFSYS_OPT_RX_SCALE
 894 static  __checkReturn   efx_rc_t
 895 siena_rx_scale_key_set(
 896         __in            efx_nic_t *enp,
 897         __in_ecount(n)  uint8_t *key,
 898         __in            size_t n)
 899 {
 900         efx_oword_t oword;
 901         unsigned int byte;
 902         unsigned int offset;
 903         efx_rc_t rc;
 904
 905         byte = 0;
 906
 907         /* Write Toeplitz IPv4 hash key */
 908         EFX_ZERO_OWORD(oword);
 909         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 910             offset > 0 && byte < n;
 911             --offset)
 912                 oword.eo_u8[offset - 1] = key[byte++];
 913
 914         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 915
 916         byte = 0;
 917
 918         /* Verify Toeplitz IPv4 hash key */
 919         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 920         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 921             offset > 0 && byte < n;
 922             --offset) {
 923                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 924                         rc = EFAULT;
 925                         goto fail1;
 926                 }
 927         }
 928
 929         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 930                 goto done;
 931
 932         byte = 0;
 933
 934         /* Write Toeplitz IPv6 hash key 3 */
 935         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 936         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 937             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 938             offset > 0 && byte < n;
 939             --offset)
 940                 oword.eo_u8[offset - 1] = key[byte++];
 941
 942         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 943
 944         /* Write Toeplitz IPv6 hash key 2 */
 945         EFX_ZERO_OWORD(oword);
 946         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 947             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 948             offset > 0 && byte < n;
 949             --offset)
 950                 oword.eo_u8[offset - 1] = key[byte++];
 951
 952         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 953
 954         /* Write Toeplitz IPv6 hash key 1 */
 955         EFX_ZERO_OWORD(oword);
 956         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 957             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 958             offset > 0 && byte < n;
 959             --offset)
 960                 oword.eo_u8[offset - 1] = key[byte++];
 961
 962         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 963
 964         byte = 0;
 965
 966         /* Verify Toeplitz IPv6 hash key 3 */
 967         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 968         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 969             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 970             offset > 0 && byte < n;
 971             --offset) {
 972                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 973                         rc = EFAULT;
 974                         goto fail2;
 975                 }
 976         }
 977
 978         /* Verify Toeplitz IPv6 hash key 2 */
 979         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 980         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 981             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 982             offset > 0 && byte < n;
 983             --offset) {
 984                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 985                         rc = EFAULT;
 986                         goto fail3;
 987                 }
 988         }
 989
 990         /* Verify Toeplitz IPv6 hash key 1 */
 991         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 992         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 993             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 994             offset > 0 && byte < n;
 995             --offset) {
 996                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 997                         rc = EFAULT;
 998                         goto fail4;
 999                 }
1000         }
1001
1002 done:
1003         return (0);
1004
1005 fail4:
1006         EFSYS_PROBE(fail4);
1007 fail3:
1008         EFSYS_PROBE(fail3);
1009 fail2:
1010         EFSYS_PROBE(fail2);
1011 fail1:
1012         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1013
1014         return (rc);
1015 }
1016 #endif
1017
1018 #if EFSYS_OPT_RX_SCALE
1019 static  __checkReturn   efx_rc_t
1020 siena_rx_scale_tbl_set(
1021         __in            efx_nic_t *enp,
1022         __in_ecount(n)  unsigned int *table,
1023         __in            size_t n)
1024 {
1025         efx_oword_t oword;
1026         int index;
1027         efx_rc_t rc;
1028
1029         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1030         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1031
1032         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1033                 rc = EINVAL;
1034                 goto fail1;
1035         }
1036
1037         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1038                 uint32_t byte;
1039
1040                 /* Calculate the entry to place in the table */
1041                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1042
1043                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1044
1045                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1046
1047                 /* Write the table */
1048                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1049                                     index, &oword, B_TRUE);
1050         }
1051
1052         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1053                 uint32_t byte;
1054
1055                 /* Determine if we're starting a new batch */
1056                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1057
1058                 /* Read the table */
1059                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1060                                     index, &oword, B_TRUE);
1061
1062                 /* Verify the entry */
1063                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1064                         rc = EFAULT;
1065                         goto fail2;
1066                 }
1067         }
1068
1069         return (0);
1070
1071 fail2:
1072         EFSYS_PROBE(fail2);
1073 fail1:
1074         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1075
1076         return (rc);
1077 }
1078 #endif
1079
1080 /*
1081  * Falcon/Siena pseudo-header
1082  * --------------------------
1083  *
1084  * Receive packets are prefixed by an optional 16 byte pseudo-header.
1085  * The pseudo-header is a byte array of one of the forms:
1086  *
1087  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
1088  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1089  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1090  *
1091  * where:
1092  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
1093  *   LL.LL         LFSR hash     (16-bit big-endian)
1094  */
1095
1096 #if EFSYS_OPT_RX_SCALE
1097 static  __checkReturn   uint32_t
1098 siena_rx_prefix_hash(
1099         __in            efx_nic_t *enp,
1100         __in            efx_rx_hash_alg_t func,
1101         __in            uint8_t *buffer)
1102 {
1103         _NOTE(ARGUNUSED(enp))
1104
1105         switch (func) {
1106         case EFX_RX_HASHALG_TOEPLITZ:
1107                 return ((buffer[12] << 24) |
1108                     (buffer[13] << 16) |
1109                     (buffer[14] <<  8) |
1110                     buffer[15]);
1111
1112         case EFX_RX_HASHALG_LFSR:
1113                 return ((buffer[14] << 8) | buffer[15]);
1114
1115         default:
1116                 EFSYS_ASSERT(0);
1117                 return (0);
1118         }
1119 }
1120 #endif /* EFSYS_OPT_RX_SCALE */
1121
1122 static  __checkReturn   efx_rc_t
1123 siena_rx_prefix_pktlen(
1124         __in            efx_nic_t *enp,
1125         __in            uint8_t *buffer,
1126         __out           uint16_t *lengthp)
1127 {
1128         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1129
1130         /* Not supported by Falcon/Siena hardware */
1131         EFSYS_ASSERT(0);
1132         return (ENOTSUP);
1133 }
1134
1135
1136 static                  void
1137 siena_rx_qpost(
1138         __in            efx_rxq_t *erp,
1139         __in_ecount(n)  efsys_dma_addr_t *addrp,
1140         __in            size_t size,
1141         __in            unsigned int n,
1142         __in            unsigned int completed,
1143         __in            unsigned int added)
1144 {
1145         efx_qword_t qword;
1146         unsigned int i;
1147         unsigned int offset;
1148         unsigned int id;
1149
1150         /* The client driver must not overfill the queue */
1151         EFSYS_ASSERT3U(added - completed + n, <=,
1152             EFX_RXQ_LIMIT(erp->er_mask + 1));
1153
1154         id = added & (erp->er_mask);
1155         for (i = 0; i < n; i++) {
1156                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1157                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1158                     size_t, size);
1159
1160                 EFX_POPULATE_QWORD_3(qword,
1161                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1162                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1163                     (uint32_t)(addrp[i] & 0xffffffff),
1164                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1165                     (uint32_t)(addrp[i] >> 32));
1166
1167                 offset = id * sizeof (efx_qword_t);
1168                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1169
1170                 id = (id + 1) & (erp->er_mask);
1171         }
1172 }
1173
1174 static                  void
1175 siena_rx_qpush(
1176         __in    efx_rxq_t *erp,
1177         __in    unsigned int added,
1178         __inout unsigned int *pushedp)
1179 {
1180         efx_nic_t *enp = erp->er_enp;
1181         unsigned int pushed = *pushedp;
1182         uint32_t wptr;
1183         efx_oword_t oword;
1184         efx_dword_t dword;
1185
1186         /* All descriptors are pushed */
1187         *pushedp = added;
1188
1189         /* Push the populated descriptors out */
1190         wptr = added & erp->er_mask;
1191
1192         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1193
1194         /* Only write the third DWORD */
1195         EFX_POPULATE_DWORD_1(dword,
1196             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1197
1198         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1199         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1200             wptr, pushed & erp->er_mask);
1201         EFSYS_PIO_WRITE_BARRIER();
1202         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1203                             erp->er_index, &dword, B_FALSE);
1204 }
1205
1206 #if EFSYS_OPT_RX_PACKED_STREAM
1207 static          void
1208 siena_rx_qps_update_credits(
1209         __in            efx_rxq_t *erp)
1210 {
1211         /* Not supported by Siena hardware */
1212         EFSYS_ASSERT(0);
1213 }
1214
1215 static          uint8_t *
1216 siena_rx_qps_packet_info(
1217         __in            efx_rxq_t *erp,
1218         __in            uint8_t *buffer,
1219         __in            uint32_t buffer_length,
1220         __in            uint32_t current_offset,
1221         __out           uint16_t *lengthp,
1222         __out           uint32_t *next_offsetp,
1223         __out           uint32_t *timestamp)
1224 {
1225         /* Not supported by Siena hardware */
1226         EFSYS_ASSERT(0);
1227
1228         return (NULL);
1229 }
1230 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1231
1232 static  __checkReturn   efx_rc_t
1233 siena_rx_qflush(
1234         __in    efx_rxq_t *erp)
1235 {
1236         efx_nic_t *enp = erp->er_enp;
1237         efx_oword_t oword;
1238         uint32_t label;
1239
1240         label = erp->er_index;
1241
1242         /* Flush the queue */
1243         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1244             FRF_AZ_RX_FLUSH_DESCQ, label);
1245         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1246
1247         return (0);
1248 }
1249
1250 static          void
1251 siena_rx_qenable(
1252         __in    efx_rxq_t *erp)
1253 {
1254         efx_nic_t *enp = erp->er_enp;
1255         efx_oword_t oword;
1256
1257         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1258
1259         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1260                             erp->er_index, &oword, B_TRUE);
1261
1262         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1263         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1264         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1265
1266         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1267                             erp->er_index, &oword, B_TRUE);
1268 }
1269
1270 static  __checkReturn   efx_rc_t
1271 siena_rx_qcreate(
1272         __in            efx_nic_t *enp,
1273         __in            unsigned int index,
1274         __in            unsigned int label,
1275         __in            efx_rxq_type_t type,
1276         __in            efsys_mem_t *esmp,
1277         __in            size_t n,
1278         __in            uint32_t id,
1279         __in            efx_evq_t *eep,
1280         __in            efx_rxq_t *erp)
1281 {
1282         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1283         efx_oword_t oword;
1284         uint32_t size;
1285         boolean_t jumbo;
1286         efx_rc_t rc;
1287
1288         _NOTE(ARGUNUSED(esmp))
1289
1290         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1291             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1292         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1293         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1294
1295         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1296         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1297
1298         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1299                 rc = EINVAL;
1300                 goto fail1;
1301         }
1302         if (index >= encp->enc_rxq_limit) {
1303                 rc = EINVAL;
1304                 goto fail2;
1305         }
1306         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1307             size++)
1308                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1309                         break;
1310         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1311                 rc = EINVAL;
1312                 goto fail3;
1313         }
1314
1315         switch (type) {
1316         case EFX_RXQ_TYPE_DEFAULT:
1317                 jumbo = B_FALSE;
1318                 break;
1319
1320 #if EFSYS_OPT_RX_SCATTER
1321         case EFX_RXQ_TYPE_SCATTER:
1322                 if (enp->en_family < EFX_FAMILY_SIENA) {
1323                         rc = EINVAL;
1324                         goto fail4;
1325                 }
1326                 jumbo = B_TRUE;
1327                 break;
1328 #endif  /* EFSYS_OPT_RX_SCATTER */
1329
1330         default:
1331                 rc = EINVAL;
1332                 goto fail4;
1333         }
1334
1335         /* Set up the new descriptor queue */
1336         EFX_POPULATE_OWORD_7(oword,
1337             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1338             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1339             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1340             FRF_AZ_RX_DESCQ_LABEL, label,
1341             FRF_AZ_RX_DESCQ_SIZE, size,
1342             FRF_AZ_RX_DESCQ_TYPE, 0,
1343             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1344
1345         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1346                             erp->er_index, &oword, B_TRUE);
1347
1348         return (0);
1349
1350 fail4:
1351         EFSYS_PROBE(fail4);
1352 fail3:
1353         EFSYS_PROBE(fail3);
1354 fail2:
1355         EFSYS_PROBE(fail2);
1356 fail1:
1357         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1358
1359         return (rc);
1360 }
1361
1362 static          void
1363 siena_rx_qdestroy(
1364         __in    efx_rxq_t *erp)
1365 {
1366         efx_nic_t *enp = erp->er_enp;
1367         efx_oword_t oword;
1368
1369         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1370         --enp->en_rx_qcount;
1371
1372         /* Purge descriptor queue */
1373         EFX_ZERO_OWORD(oword);
1374
1375         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1376                             erp->er_index, &oword, B_TRUE);
1377
1378         /* Free the RXQ object */
1379         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1380 }
1381
1382 static          void
1383 siena_rx_fini(
1384         __in    efx_nic_t *enp)
1385 {
1386         _NOTE(ARGUNUSED(enp))
1387 }
1388
1389 #endif /* EFSYS_OPT_SIENA */