drivers/net/sfc/base/efx_rx.c

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_SIENA
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_init(
  39         __in            efx_nic_t *enp);
  40
  41 static                  void
  42 siena_rx_fini(
  43         __in            efx_nic_t *enp);
  44
  45 #if EFSYS_OPT_RX_SCATTER
  46 static  __checkReturn   efx_rc_t
  47 siena_rx_scatter_enable(
  48         __in            efx_nic_t *enp,
  49         __in            unsigned int buf_size);
  50 #endif /* EFSYS_OPT_RX_SCATTER */
  51
  52 static  __checkReturn   efx_rc_t
  53 siena_rx_prefix_pktlen(
  54         __in            efx_nic_t *enp,
  55         __in            uint8_t *buffer,
  56         __out           uint16_t *lengthp);
  57
  58 static                  void
  59 siena_rx_qpost(
  60         __in            efx_rxq_t *erp,
  61         __in_ecount(n)  efsys_dma_addr_t *addrp,
  62         __in            size_t size,
  63         __in            unsigned int n,
  64         __in            unsigned int completed,
  65         __in            unsigned int added);
  66
  67 static                  void
  68 siena_rx_qpush(
  69         __in            efx_rxq_t *erp,
  70         __in            unsigned int added,
  71         __inout         unsigned int *pushedp);
  72
  73 static  __checkReturn   efx_rc_t
  74 siena_rx_qflush(
  75         __in            efx_rxq_t *erp);
  76
  77 static                  void
  78 siena_rx_qenable(
  79         __in            efx_rxq_t *erp);
  80
  81 static  __checkReturn   efx_rc_t
  82 siena_rx_qcreate(
  83         __in            efx_nic_t *enp,
  84         __in            unsigned int index,
  85         __in            unsigned int label,
  86         __in            efx_rxq_type_t type,
  87         __in            efsys_mem_t *esmp,
  88         __in            size_t n,
  89         __in            uint32_t id,
  90         __in            efx_evq_t *eep,
  91         __in            efx_rxq_t *erp);
  92
  93 static                  void
  94 siena_rx_qdestroy(
  95         __in            efx_rxq_t *erp);
  96
  97 #endif /* EFSYS_OPT_SIENA */
  98
  99
 100 #if EFSYS_OPT_SIENA
 101 static const efx_rx_ops_t __efx_rx_siena_ops = {
 102         siena_rx_init,                          /* erxo_init */
 103         siena_rx_fini,                          /* erxo_fini */
 104 #if EFSYS_OPT_RX_SCATTER
 105         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 106 #endif
 107         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 108         siena_rx_qpost,                         /* erxo_qpost */
 109         siena_rx_qpush,                         /* erxo_qpush */
 110         siena_rx_qflush,                        /* erxo_qflush */
 111         siena_rx_qenable,                       /* erxo_qenable */
 112         siena_rx_qcreate,                       /* erxo_qcreate */
 113         siena_rx_qdestroy,                      /* erxo_qdestroy */
 114 };
 115 #endif  /* EFSYS_OPT_SIENA */
 116
 117 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 118 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 119         ef10_rx_init,                           /* erxo_init */
 120         ef10_rx_fini,                           /* erxo_fini */
 121 #if EFSYS_OPT_RX_SCATTER
 122         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 123 #endif
 124         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 125         ef10_rx_qpost,                          /* erxo_qpost */
 126         ef10_rx_qpush,                          /* erxo_qpush */
 127         ef10_rx_qflush,                         /* erxo_qflush */
 128         ef10_rx_qenable,                        /* erxo_qenable */
 129         ef10_rx_qcreate,                        /* erxo_qcreate */
 130         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 131 };
 132 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 133
 134
 135         __checkReturn   efx_rc_t
 136 efx_rx_init(
 137         __inout         efx_nic_t *enp)
 138 {
 139         const efx_rx_ops_t *erxop;
 140         efx_rc_t rc;
 141
 142         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 143         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 144
 145         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 146                 rc = EINVAL;
 147                 goto fail1;
 148         }
 149
 150         if (enp->en_mod_flags & EFX_MOD_RX) {
 151                 rc = EINVAL;
 152                 goto fail2;
 153         }
 154
 155         switch (enp->en_family) {
 156 #if EFSYS_OPT_SIENA
 157         case EFX_FAMILY_SIENA:
 158                 erxop = &__efx_rx_siena_ops;
 159                 break;
 160 #endif /* EFSYS_OPT_SIENA */
 161
 162 #if EFSYS_OPT_HUNTINGTON
 163         case EFX_FAMILY_HUNTINGTON:
 164                 erxop = &__efx_rx_ef10_ops;
 165                 break;
 166 #endif /* EFSYS_OPT_HUNTINGTON */
 167
 168 #if EFSYS_OPT_MEDFORD
 169         case EFX_FAMILY_MEDFORD:
 170                 erxop = &__efx_rx_ef10_ops;
 171                 break;
 172 #endif /* EFSYS_OPT_MEDFORD */
 173
 174         default:
 175                 EFSYS_ASSERT(0);
 176                 rc = ENOTSUP;
 177                 goto fail3;
 178         }
 179
 180         if ((rc = erxop->erxo_init(enp)) != 0)
 181                 goto fail4;
 182
 183         enp->en_erxop = erxop;
 184         enp->en_mod_flags |= EFX_MOD_RX;
 185         return (0);
 186
 187 fail4:
 188         EFSYS_PROBE(fail4);
 189 fail3:
 190         EFSYS_PROBE(fail3);
 191 fail2:
 192         EFSYS_PROBE(fail2);
 193 fail1:
 194         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 195
 196         enp->en_erxop = NULL;
 197         enp->en_mod_flags &= ~EFX_MOD_RX;
 198         return (rc);
 199 }
 200
 201                         void
 202 efx_rx_fini(
 203         __in            efx_nic_t *enp)
 204 {
 205         const efx_rx_ops_t *erxop = enp->en_erxop;
 206
 207         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 208         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 209         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 210         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 211
 212         erxop->erxo_fini(enp);
 213
 214         enp->en_erxop = NULL;
 215         enp->en_mod_flags &= ~EFX_MOD_RX;
 216 }
 217
 218 #if EFSYS_OPT_RX_SCATTER
 219         __checkReturn   efx_rc_t
 220 efx_rx_scatter_enable(
 221         __in            efx_nic_t *enp,
 222         __in            unsigned int buf_size)
 223 {
 224         const efx_rx_ops_t *erxop = enp->en_erxop;
 225         efx_rc_t rc;
 226
 227         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 228         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 229
 230         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 231                 goto fail1;
 232
 233         return (0);
 234
 235 fail1:
 236         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 237         return (rc);
 238 }
 239 #endif  /* EFSYS_OPT_RX_SCATTER */
 240
 241                         void
 242 efx_rx_qpost(
 243         __in            efx_rxq_t *erp,
 244         __in_ecount(n)  efsys_dma_addr_t *addrp,
 245         __in            size_t size,
 246         __in            unsigned int n,
 247         __in            unsigned int completed,
 248         __in            unsigned int added)
 249 {
 250         efx_nic_t *enp = erp->er_enp;
 251         const efx_rx_ops_t *erxop = enp->en_erxop;
 252
 253         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 254
 255         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 256 }
 257
 258                         void
 259 efx_rx_qpush(
 260         __in            efx_rxq_t *erp,
 261         __in            unsigned int added,
 262         __inout         unsigned int *pushedp)
 263 {
 264         efx_nic_t *enp = erp->er_enp;
 265         const efx_rx_ops_t *erxop = enp->en_erxop;
 266
 267         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 268
 269         erxop->erxo_qpush(erp, added, pushedp);
 270 }
 271
 272         __checkReturn   efx_rc_t
 273 efx_rx_qflush(
 274         __in            efx_rxq_t *erp)
 275 {
 276         efx_nic_t *enp = erp->er_enp;
 277         const efx_rx_ops_t *erxop = enp->en_erxop;
 278         efx_rc_t rc;
 279
 280         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 281
 282         if ((rc = erxop->erxo_qflush(erp)) != 0)
 283                 goto fail1;
 284
 285         return (0);
 286
 287 fail1:
 288         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 289
 290         return (rc);
 291 }
 292
 293                         void
 294 efx_rx_qenable(
 295         __in            efx_rxq_t *erp)
 296 {
 297         efx_nic_t *enp = erp->er_enp;
 298         const efx_rx_ops_t *erxop = enp->en_erxop;
 299
 300         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 301
 302         erxop->erxo_qenable(erp);
 303 }
 304
 305         __checkReturn   efx_rc_t
 306 efx_rx_qcreate(
 307         __in            efx_nic_t *enp,
 308         __in            unsigned int index,
 309         __in            unsigned int label,
 310         __in            efx_rxq_type_t type,
 311         __in            efsys_mem_t *esmp,
 312         __in            size_t n,
 313         __in            uint32_t id,
 314         __in            efx_evq_t *eep,
 315         __deref_out     efx_rxq_t **erpp)
 316 {
 317         const efx_rx_ops_t *erxop = enp->en_erxop;
 318         efx_rxq_t *erp;
 319         efx_rc_t rc;
 320
 321         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 322         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 323
 324         /* Allocate an RXQ object */
 325         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 326
 327         if (erp == NULL) {
 328                 rc = ENOMEM;
 329                 goto fail1;
 330         }
 331
 332         erp->er_magic = EFX_RXQ_MAGIC;
 333         erp->er_enp = enp;
 334         erp->er_index = index;
 335         erp->er_mask = n - 1;
 336         erp->er_esmp = esmp;
 337
 338         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 339             eep, erp)) != 0)
 340                 goto fail2;
 341
 342         enp->en_rx_qcount++;
 343         *erpp = erp;
 344
 345         return (0);
 346
 347 fail2:
 348         EFSYS_PROBE(fail2);
 349
 350         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 351 fail1:
 352         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 353
 354         return (rc);
 355 }
 356
 357                         void
 358 efx_rx_qdestroy(
 359         __in            efx_rxq_t *erp)
 360 {
 361         efx_nic_t *enp = erp->er_enp;
 362         const efx_rx_ops_t *erxop = enp->en_erxop;
 363
 364         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 365
 366         erxop->erxo_qdestroy(erp);
 367 }
 368
 369         __checkReturn   efx_rc_t
 370 efx_pseudo_hdr_pkt_length_get(
 371         __in            efx_rxq_t *erp,
 372         __in            uint8_t *buffer,
 373         __out           uint16_t *lengthp)
 374 {
 375         efx_nic_t *enp = erp->er_enp;
 376         const efx_rx_ops_t *erxop = enp->en_erxop;
 377
 378         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 379
 380         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 381 }
 382
 383 #if EFSYS_OPT_SIENA
 384
 385 static  __checkReturn   efx_rc_t
 386 siena_rx_init(
 387         __in            efx_nic_t *enp)
 388 {
 389         efx_oword_t oword;
 390         unsigned int index;
 391
 392         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 393
 394         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 395         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 396         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 397         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 398         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 399         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 400         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 401
 402         /* Zero the RSS table */
 403         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 404             index++) {
 405                 EFX_ZERO_OWORD(oword);
 406                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 407                                     index, &oword, B_TRUE);
 408         }
 409
 410         return (0);
 411 }
 412
 413 #if EFSYS_OPT_RX_SCATTER
 414 static  __checkReturn   efx_rc_t
 415 siena_rx_scatter_enable(
 416         __in            efx_nic_t *enp,
 417         __in            unsigned int buf_size)
 418 {
 419         unsigned int nbuf32;
 420         efx_oword_t oword;
 421         efx_rc_t rc;
 422
 423         nbuf32 = buf_size / 32;
 424         if ((nbuf32 == 0) ||
 425             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 426             ((buf_size % 32) != 0)) {
 427                 rc = EINVAL;
 428                 goto fail1;
 429         }
 430
 431         if (enp->en_rx_qcount > 0) {
 432                 rc = EBUSY;
 433                 goto fail2;
 434         }
 435
 436         /* Set scatter buffer size */
 437         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 438         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 439         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 440
 441         /* Enable scatter for packets not matching a filter */
 442         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 443         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 444         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 445
 446         return (0);
 447
 448 fail2:
 449         EFSYS_PROBE(fail2);
 450 fail1:
 451         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 452
 453         return (rc);
 454 }
 455 #endif  /* EFSYS_OPT_RX_SCATTER */
 456
 457
 458 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 459         do {                                                            \
 460                 efx_oword_t oword;                                      \
 461                                                                         \
 462                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 463                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 464                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 465                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 466                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 467                     (_insert) ? 1 : 0);                                 \
 468                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 469                                                                         \
 470                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 471                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 472                             &oword);                                    \
 473                         EFX_SET_OWORD_FIELD(oword,                      \
 474                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 475                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 476                             &oword);                                    \
 477                 }                                                       \
 478                                                                         \
 479                 _NOTE(CONSTANTCONDITION)                                \
 480         } while (B_FALSE)
 481
 482 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 483         do {                                                            \
 484                 efx_oword_t oword;                                      \
 485                                                                         \
 486                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 487                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 488                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 489                     (_ip) ? 1 : 0);                                     \
 490                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 491                     (_tcp) ? 0 : 1);                                    \
 492                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 493                     (_insert) ? 1 : 0);                                 \
 494                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 495                                                                         \
 496                 _NOTE(CONSTANTCONDITION)                                \
 497         } while (B_FALSE)
 498
 499 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 500         do {                                                            \
 501                 efx_oword_t oword;                                      \
 502                                                                         \
 503                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 504                 EFX_SET_OWORD_FIELD(oword,                              \
 505                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 506                 EFX_SET_OWORD_FIELD(oword,                              \
 507                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 508                 EFX_SET_OWORD_FIELD(oword,                              \
 509                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 510                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 511                                                                         \
 512                 (_rc) = 0;                                              \
 513                                                                         \
 514                 _NOTE(CONSTANTCONDITION)                                \
 515         } while (B_FALSE)
 516
 517
 518 /*
 519  * Falcon/Siena pseudo-header
 520  * --------------------------
 521  *
 522  * Receive packets are prefixed by an optional 16 byte pseudo-header.
 523  * The pseudo-header is a byte array of one of the forms:
 524  *
 525  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
 526  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
 527  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
 528  *
 529  * where:
 530  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
 531  *   LL.LL         LFSR hash     (16-bit big-endian)
 532  */
 533
 534 static  __checkReturn   efx_rc_t
 535 siena_rx_prefix_pktlen(
 536         __in            efx_nic_t *enp,
 537         __in            uint8_t *buffer,
 538         __out           uint16_t *lengthp)
 539 {
 540         _NOTE(ARGUNUSED(enp, buffer, lengthp))
 541
 542         /* Not supported by Falcon/Siena hardware */
 543         EFSYS_ASSERT(0);
 544         return (ENOTSUP);
 545 }
 546
 547
 548 static                  void
 549 siena_rx_qpost(
 550         __in            efx_rxq_t *erp,
 551         __in_ecount(n)  efsys_dma_addr_t *addrp,
 552         __in            size_t size,
 553         __in            unsigned int n,
 554         __in            unsigned int completed,
 555         __in            unsigned int added)
 556 {
 557         efx_qword_t qword;
 558         unsigned int i;
 559         unsigned int offset;
 560         unsigned int id;
 561
 562         /* The client driver must not overfill the queue */
 563         EFSYS_ASSERT3U(added - completed + n, <=,
 564             EFX_RXQ_LIMIT(erp->er_mask + 1));
 565
 566         id = added & (erp->er_mask);
 567         for (i = 0; i < n; i++) {
 568                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 569                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 570                     size_t, size);
 571
 572                 EFX_POPULATE_QWORD_3(qword,
 573                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
 574                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
 575                     (uint32_t)(addrp[i] & 0xffffffff),
 576                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
 577                     (uint32_t)(addrp[i] >> 32));
 578
 579                 offset = id * sizeof (efx_qword_t);
 580                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 581
 582                 id = (id + 1) & (erp->er_mask);
 583         }
 584 }
 585
 586 static                  void
 587 siena_rx_qpush(
 588         __in    efx_rxq_t *erp,
 589         __in    unsigned int added,
 590         __inout unsigned int *pushedp)
 591 {
 592         efx_nic_t *enp = erp->er_enp;
 593         unsigned int pushed = *pushedp;
 594         uint32_t wptr;
 595         efx_oword_t oword;
 596         efx_dword_t dword;
 597
 598         /* All descriptors are pushed */
 599         *pushedp = added;
 600
 601         /* Push the populated descriptors out */
 602         wptr = added & erp->er_mask;
 603
 604         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
 605
 606         /* Only write the third DWORD */
 607         EFX_POPULATE_DWORD_1(dword,
 608             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
 609
 610         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 611         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 612             wptr, pushed & erp->er_mask);
 613         EFSYS_PIO_WRITE_BARRIER();
 614         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
 615                             erp->er_index, &dword, B_FALSE);
 616 }
 617
 618 static  __checkReturn   efx_rc_t
 619 siena_rx_qflush(
 620         __in    efx_rxq_t *erp)
 621 {
 622         efx_nic_t *enp = erp->er_enp;
 623         efx_oword_t oword;
 624         uint32_t label;
 625
 626         label = erp->er_index;
 627
 628         /* Flush the queue */
 629         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 630             FRF_AZ_RX_FLUSH_DESCQ, label);
 631         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
 632
 633         return (0);
 634 }
 635
 636 static          void
 637 siena_rx_qenable(
 638         __in    efx_rxq_t *erp)
 639 {
 640         efx_nic_t *enp = erp->er_enp;
 641         efx_oword_t oword;
 642
 643         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 644
 645         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
 646                             erp->er_index, &oword, B_TRUE);
 647
 648         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
 649         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
 650         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
 651
 652         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 653                             erp->er_index, &oword, B_TRUE);
 654 }
 655
 656 static  __checkReturn   efx_rc_t
 657 siena_rx_qcreate(
 658         __in            efx_nic_t *enp,
 659         __in            unsigned int index,
 660         __in            unsigned int label,
 661         __in            efx_rxq_type_t type,
 662         __in            efsys_mem_t *esmp,
 663         __in            size_t n,
 664         __in            uint32_t id,
 665         __in            efx_evq_t *eep,
 666         __in            efx_rxq_t *erp)
 667 {
 668         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 669         efx_oword_t oword;
 670         uint32_t size;
 671         boolean_t jumbo;
 672         efx_rc_t rc;
 673
 674         _NOTE(ARGUNUSED(esmp))
 675
 676         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
 677             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
 678         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 679         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 680
 681         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
 682         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
 683
 684         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
 685                 rc = EINVAL;
 686                 goto fail1;
 687         }
 688         if (index >= encp->enc_rxq_limit) {
 689                 rc = EINVAL;
 690                 goto fail2;
 691         }
 692         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
 693             size++)
 694                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
 695                         break;
 696         if (id + (1 << size) >= encp->enc_buftbl_limit) {
 697                 rc = EINVAL;
 698                 goto fail3;
 699         }
 700
 701         switch (type) {
 702         case EFX_RXQ_TYPE_DEFAULT:
 703                 jumbo = B_FALSE;
 704                 break;
 705
 706 #if EFSYS_OPT_RX_SCATTER
 707         case EFX_RXQ_TYPE_SCATTER:
 708                 if (enp->en_family < EFX_FAMILY_SIENA) {
 709                         rc = EINVAL;
 710                         goto fail4;
 711                 }
 712                 jumbo = B_TRUE;
 713                 break;
 714 #endif  /* EFSYS_OPT_RX_SCATTER */
 715
 716         default:
 717                 rc = EINVAL;
 718                 goto fail4;
 719         }
 720
 721         /* Set up the new descriptor queue */
 722         EFX_POPULATE_OWORD_7(oword,
 723             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
 724             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
 725             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 726             FRF_AZ_RX_DESCQ_LABEL, label,
 727             FRF_AZ_RX_DESCQ_SIZE, size,
 728             FRF_AZ_RX_DESCQ_TYPE, 0,
 729             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
 730
 731         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 732                             erp->er_index, &oword, B_TRUE);
 733
 734         return (0);
 735
 736 fail4:
 737         EFSYS_PROBE(fail4);
 738 fail3:
 739         EFSYS_PROBE(fail3);
 740 fail2:
 741         EFSYS_PROBE(fail2);
 742 fail1:
 743         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 744
 745         return (rc);
 746 }
 747
 748 static          void
 749 siena_rx_qdestroy(
 750         __in    efx_rxq_t *erp)
 751 {
 752         efx_nic_t *enp = erp->er_enp;
 753         efx_oword_t oword;
 754
 755         EFSYS_ASSERT(enp->en_rx_qcount != 0);
 756         --enp->en_rx_qcount;
 757
 758         /* Purge descriptor queue */
 759         EFX_ZERO_OWORD(oword);
 760
 761         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
 762                             erp->er_index, &oword, B_TRUE);
 763
 764         /* Free the RXQ object */
 765         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 766 }
 767
 768 static          void
 769 siena_rx_fini(
 770         __in    efx_nic_t *enp)
 771 {
 772         _NOTE(ARGUNUSED(enp))
 773 }
 774
 775 #endif /* EFSYS_OPT_SIENA */