drivers/net/sfc/base/efx_rx.c

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_SIENA
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_init(
  39         __in            efx_nic_t *enp);
  40
  41 static                  void
  42 siena_rx_fini(
  43         __in            efx_nic_t *enp);
  44
  45 #if EFSYS_OPT_RX_SCATTER
  46 static  __checkReturn   efx_rc_t
  47 siena_rx_scatter_enable(
  48         __in            efx_nic_t *enp,
  49         __in            unsigned int buf_size);
  50 #endif /* EFSYS_OPT_RX_SCATTER */
  51
  52 #if EFSYS_OPT_RX_SCALE
  53 static  __checkReturn   efx_rc_t
  54 siena_rx_scale_mode_set(
  55         __in            efx_nic_t *enp,
  56         __in            efx_rx_hash_alg_t alg,
  57         __in            efx_rx_hash_type_t type,
  58         __in            boolean_t insert);
  59
  60 static  __checkReturn   efx_rc_t
  61 siena_rx_scale_key_set(
  62         __in            efx_nic_t *enp,
  63         __in_ecount(n)  uint8_t *key,
  64         __in            size_t n);
  65
  66 static  __checkReturn   efx_rc_t
  67 siena_rx_scale_tbl_set(
  68         __in            efx_nic_t *enp,
  69         __in_ecount(n)  unsigned int *table,
  70         __in            size_t n);
  71
  72 static  __checkReturn   uint32_t
  73 siena_rx_prefix_hash(
  74         __in            efx_nic_t *enp,
  75         __in            efx_rx_hash_alg_t func,
  76         __in            uint8_t *buffer);
  77
  78 #endif /* EFSYS_OPT_RX_SCALE */
  79
  80 static  __checkReturn   efx_rc_t
  81 siena_rx_prefix_pktlen(
  82         __in            efx_nic_t *enp,
  83         __in            uint8_t *buffer,
  84         __out           uint16_t *lengthp);
  85
  86 static                  void
  87 siena_rx_qpost(
  88         __in            efx_rxq_t *erp,
  89         __in_ecount(n)  efsys_dma_addr_t *addrp,
  90         __in            size_t size,
  91         __in            unsigned int n,
  92         __in            unsigned int completed,
  93         __in            unsigned int added);
  94
  95 static                  void
  96 siena_rx_qpush(
  97         __in            efx_rxq_t *erp,
  98         __in            unsigned int added,
  99         __inout         unsigned int *pushedp);
 100
 101 #if EFSYS_OPT_RX_PACKED_STREAM
 102 static          void
 103 siena_rx_qps_update_credits(
 104         __in            efx_rxq_t *erp);
 105
 106 static  __checkReturn   uint8_t *
 107 siena_rx_qps_packet_info(
 108         __in            efx_rxq_t *erp,
 109         __in            uint8_t *buffer,
 110         __in            uint32_t buffer_length,
 111         __in            uint32_t current_offset,
 112         __out           uint16_t *lengthp,
 113         __out           uint32_t *next_offsetp,
 114         __out           uint32_t *timestamp);
 115 #endif
 116
 117 static  __checkReturn   efx_rc_t
 118 siena_rx_qflush(
 119         __in            efx_rxq_t *erp);
 120
 121 static                  void
 122 siena_rx_qenable(
 123         __in            efx_rxq_t *erp);
 124
 125 static  __checkReturn   efx_rc_t
 126 siena_rx_qcreate(
 127         __in            efx_nic_t *enp,
 128         __in            unsigned int index,
 129         __in            unsigned int label,
 130         __in            efx_rxq_type_t type,
 131         __in            efsys_mem_t *esmp,
 132         __in            size_t n,
 133         __in            uint32_t id,
 134         __in            efx_evq_t *eep,
 135         __in            efx_rxq_t *erp);
 136
 137 static                  void
 138 siena_rx_qdestroy(
 139         __in            efx_rxq_t *erp);
 140
 141 #endif /* EFSYS_OPT_SIENA */
 142
 143
 144 #if EFSYS_OPT_SIENA
 145 static const efx_rx_ops_t __efx_rx_siena_ops = {
 146         siena_rx_init,                          /* erxo_init */
 147         siena_rx_fini,                          /* erxo_fini */
 148 #if EFSYS_OPT_RX_SCATTER
 149         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 150 #endif
 151 #if EFSYS_OPT_RX_SCALE
 152         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 153         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 154         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 155         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 156 #endif
 157         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 158         siena_rx_qpost,                         /* erxo_qpost */
 159         siena_rx_qpush,                         /* erxo_qpush */
 160 #if EFSYS_OPT_RX_PACKED_STREAM
 161         siena_rx_qps_update_credits,            /* erxo_qps_update_credits */
 162         siena_rx_qps_packet_info,               /* erxo_qps_packet_info */
 163 #endif
 164         siena_rx_qflush,                        /* erxo_qflush */
 165         siena_rx_qenable,                       /* erxo_qenable */
 166         siena_rx_qcreate,                       /* erxo_qcreate */
 167         siena_rx_qdestroy,                      /* erxo_qdestroy */
 168 };
 169 #endif  /* EFSYS_OPT_SIENA */
 170
 171 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 172 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 173         ef10_rx_init,                           /* erxo_init */
 174         ef10_rx_fini,                           /* erxo_fini */
 175 #if EFSYS_OPT_RX_SCATTER
 176         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 177 #endif
 178 #if EFSYS_OPT_RX_SCALE
 179         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 180         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 181         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 182         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 183 #endif
 184         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 185         ef10_rx_qpost,                          /* erxo_qpost */
 186         ef10_rx_qpush,                          /* erxo_qpush */
 187 #if EFSYS_OPT_RX_PACKED_STREAM
 188         ef10_rx_qps_update_credits,             /* erxo_qps_update_credits */
 189         ef10_rx_qps_packet_info,                /* erxo_qps_packet_info */
 190 #endif
 191         ef10_rx_qflush,                         /* erxo_qflush */
 192         ef10_rx_qenable,                        /* erxo_qenable */
 193         ef10_rx_qcreate,                        /* erxo_qcreate */
 194         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 195 };
 196 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 197
 198
 199         __checkReturn   efx_rc_t
 200 efx_rx_init(
 201         __inout         efx_nic_t *enp)
 202 {
 203         const efx_rx_ops_t *erxop;
 204         efx_rc_t rc;
 205
 206         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 207         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 208
 209         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 210                 rc = EINVAL;
 211                 goto fail1;
 212         }
 213
 214         if (enp->en_mod_flags & EFX_MOD_RX) {
 215                 rc = EINVAL;
 216                 goto fail2;
 217         }
 218
 219         switch (enp->en_family) {
 220 #if EFSYS_OPT_SIENA
 221         case EFX_FAMILY_SIENA:
 222                 erxop = &__efx_rx_siena_ops;
 223                 break;
 224 #endif /* EFSYS_OPT_SIENA */
 225
 226 #if EFSYS_OPT_HUNTINGTON
 227         case EFX_FAMILY_HUNTINGTON:
 228                 erxop = &__efx_rx_ef10_ops;
 229                 break;
 230 #endif /* EFSYS_OPT_HUNTINGTON */
 231
 232 #if EFSYS_OPT_MEDFORD
 233         case EFX_FAMILY_MEDFORD:
 234                 erxop = &__efx_rx_ef10_ops;
 235                 break;
 236 #endif /* EFSYS_OPT_MEDFORD */
 237
 238         default:
 239                 EFSYS_ASSERT(0);
 240                 rc = ENOTSUP;
 241                 goto fail3;
 242         }
 243
 244         if ((rc = erxop->erxo_init(enp)) != 0)
 245                 goto fail4;
 246
 247         enp->en_erxop = erxop;
 248         enp->en_mod_flags |= EFX_MOD_RX;
 249         return (0);
 250
 251 fail4:
 252         EFSYS_PROBE(fail4);
 253 fail3:
 254         EFSYS_PROBE(fail3);
 255 fail2:
 256         EFSYS_PROBE(fail2);
 257 fail1:
 258         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 259
 260         enp->en_erxop = NULL;
 261         enp->en_mod_flags &= ~EFX_MOD_RX;
 262         return (rc);
 263 }
 264
 265                         void
 266 efx_rx_fini(
 267         __in            efx_nic_t *enp)
 268 {
 269         const efx_rx_ops_t *erxop = enp->en_erxop;
 270
 271         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 272         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 273         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 274         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 275
 276         erxop->erxo_fini(enp);
 277
 278         enp->en_erxop = NULL;
 279         enp->en_mod_flags &= ~EFX_MOD_RX;
 280 }
 281
 282 #if EFSYS_OPT_RX_SCATTER
 283         __checkReturn   efx_rc_t
 284 efx_rx_scatter_enable(
 285         __in            efx_nic_t *enp,
 286         __in            unsigned int buf_size)
 287 {
 288         const efx_rx_ops_t *erxop = enp->en_erxop;
 289         efx_rc_t rc;
 290
 291         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 292         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 293
 294         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 295                 goto fail1;
 296
 297         return (0);
 298
 299 fail1:
 300         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 301         return (rc);
 302 }
 303 #endif  /* EFSYS_OPT_RX_SCATTER */
 304
 305 #if EFSYS_OPT_RX_SCALE
 306         __checkReturn   efx_rc_t
 307 efx_rx_hash_default_support_get(
 308         __in            efx_nic_t *enp,
 309         __out           efx_rx_hash_support_t *supportp)
 310 {
 311         efx_rc_t rc;
 312
 313         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 314         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 315
 316         if (supportp == NULL) {
 317                 rc = EINVAL;
 318                 goto fail1;
 319         }
 320
 321         /*
 322          * Report the hashing support the client gets by default if it
 323          * does not allocate an RSS context itself.
 324          */
 325         *supportp = enp->en_hash_support;
 326
 327         return (0);
 328
 329 fail1:
 330         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 331
 332         return (rc);
 333 }
 334
 335         __checkReturn   efx_rc_t
 336 efx_rx_scale_default_support_get(
 337         __in            efx_nic_t *enp,
 338         __out           efx_rx_scale_context_type_t *typep)
 339 {
 340         efx_rc_t rc;
 341
 342         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 343         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 344
 345         if (typep == NULL) {
 346                 rc = EINVAL;
 347                 goto fail1;
 348         }
 349
 350         /*
 351          * Report the RSS support the client gets by default if it
 352          * does not allocate an RSS context itself.
 353          */
 354         *typep = enp->en_rss_context_type;
 355
 356         return (0);
 357
 358 fail1:
 359         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 360
 361         return (rc);
 362 }
 363
 364         __checkReturn   efx_rc_t
 365 efx_rx_scale_mode_set(
 366         __in            efx_nic_t *enp,
 367         __in            efx_rx_hash_alg_t alg,
 368         __in            efx_rx_hash_type_t type,
 369         __in            boolean_t insert)
 370 {
 371         const efx_rx_ops_t *erxop = enp->en_erxop;
 372         efx_rc_t rc;
 373
 374         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 375         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 376
 377         if (erxop->erxo_scale_mode_set != NULL) {
 378                 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
 379                             type, insert)) != 0)
 380                         goto fail1;
 381         }
 382
 383         return (0);
 384
 385 fail1:
 386         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 387         return (rc);
 388 }
 389 #endif  /* EFSYS_OPT_RX_SCALE */
 390
 391 #if EFSYS_OPT_RX_SCALE
 392         __checkReturn   efx_rc_t
 393 efx_rx_scale_key_set(
 394         __in            efx_nic_t *enp,
 395         __in_ecount(n)  uint8_t *key,
 396         __in            size_t n)
 397 {
 398         const efx_rx_ops_t *erxop = enp->en_erxop;
 399         efx_rc_t rc;
 400
 401         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 402         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 403
 404         if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
 405                 goto fail1;
 406
 407         return (0);
 408
 409 fail1:
 410         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 411
 412         return (rc);
 413 }
 414 #endif  /* EFSYS_OPT_RX_SCALE */
 415
 416 #if EFSYS_OPT_RX_SCALE
 417         __checkReturn   efx_rc_t
 418 efx_rx_scale_tbl_set(
 419         __in            efx_nic_t *enp,
 420         __in_ecount(n)  unsigned int *table,
 421         __in            size_t n)
 422 {
 423         const efx_rx_ops_t *erxop = enp->en_erxop;
 424         efx_rc_t rc;
 425
 426         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 427         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 428
 429         if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
 430                 goto fail1;
 431
 432         return (0);
 433
 434 fail1:
 435         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 436
 437         return (rc);
 438 }
 439 #endif  /* EFSYS_OPT_RX_SCALE */
 440
 441                         void
 442 efx_rx_qpost(
 443         __in            efx_rxq_t *erp,
 444         __in_ecount(n)  efsys_dma_addr_t *addrp,
 445         __in            size_t size,
 446         __in            unsigned int n,
 447         __in            unsigned int completed,
 448         __in            unsigned int added)
 449 {
 450         efx_nic_t *enp = erp->er_enp;
 451         const efx_rx_ops_t *erxop = enp->en_erxop;
 452
 453         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 454
 455         erxop->erxo_qpost(erp, addrp, size, n, completed, added);
 456 }
 457
 458 #if EFSYS_OPT_RX_PACKED_STREAM
 459
 460                         void
 461 efx_rx_qps_update_credits(
 462         __in            efx_rxq_t *erp)
 463 {
 464         efx_nic_t *enp = erp->er_enp;
 465         const efx_rx_ops_t *erxop = enp->en_erxop;
 466
 467         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 468
 469         erxop->erxo_qps_update_credits(erp);
 470 }
 471
 472         __checkReturn   uint8_t *
 473 efx_rx_qps_packet_info(
 474         __in            efx_rxq_t *erp,
 475         __in            uint8_t *buffer,
 476         __in            uint32_t buffer_length,
 477         __in            uint32_t current_offset,
 478         __out           uint16_t *lengthp,
 479         __out           uint32_t *next_offsetp,
 480         __out           uint32_t *timestamp)
 481 {
 482         efx_nic_t *enp = erp->er_enp;
 483         const efx_rx_ops_t *erxop = enp->en_erxop;
 484
 485         return (erxop->erxo_qps_packet_info(erp, buffer,
 486                 buffer_length, current_offset, lengthp,
 487                 next_offsetp, timestamp));
 488 }
 489
 490 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 491
 492                         void
 493 efx_rx_qpush(
 494         __in            efx_rxq_t *erp,
 495         __in            unsigned int added,
 496         __inout         unsigned int *pushedp)
 497 {
 498         efx_nic_t *enp = erp->er_enp;
 499         const efx_rx_ops_t *erxop = enp->en_erxop;
 500
 501         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 502
 503         erxop->erxo_qpush(erp, added, pushedp);
 504 }
 505
 506         __checkReturn   efx_rc_t
 507 efx_rx_qflush(
 508         __in            efx_rxq_t *erp)
 509 {
 510         efx_nic_t *enp = erp->er_enp;
 511         const efx_rx_ops_t *erxop = enp->en_erxop;
 512         efx_rc_t rc;
 513
 514         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 515
 516         if ((rc = erxop->erxo_qflush(erp)) != 0)
 517                 goto fail1;
 518
 519         return (0);
 520
 521 fail1:
 522         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 523
 524         return (rc);
 525 }
 526
 527                         void
 528 efx_rx_qenable(
 529         __in            efx_rxq_t *erp)
 530 {
 531         efx_nic_t *enp = erp->er_enp;
 532         const efx_rx_ops_t *erxop = enp->en_erxop;
 533
 534         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 535
 536         erxop->erxo_qenable(erp);
 537 }
 538
 539         __checkReturn   efx_rc_t
 540 efx_rx_qcreate(
 541         __in            efx_nic_t *enp,
 542         __in            unsigned int index,
 543         __in            unsigned int label,
 544         __in            efx_rxq_type_t type,
 545         __in            efsys_mem_t *esmp,
 546         __in            size_t n,
 547         __in            uint32_t id,
 548         __in            efx_evq_t *eep,
 549         __deref_out     efx_rxq_t **erpp)
 550 {
 551         const efx_rx_ops_t *erxop = enp->en_erxop;
 552         efx_rxq_t *erp;
 553         efx_rc_t rc;
 554
 555         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 556         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 557
 558         /* Allocate an RXQ object */
 559         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 560
 561         if (erp == NULL) {
 562                 rc = ENOMEM;
 563                 goto fail1;
 564         }
 565
 566         erp->er_magic = EFX_RXQ_MAGIC;
 567         erp->er_enp = enp;
 568         erp->er_index = index;
 569         erp->er_mask = n - 1;
 570         erp->er_esmp = esmp;
 571
 572         if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
 573             eep, erp)) != 0)
 574                 goto fail2;
 575
 576         enp->en_rx_qcount++;
 577         *erpp = erp;
 578
 579         return (0);
 580
 581 fail2:
 582         EFSYS_PROBE(fail2);
 583
 584         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 585 fail1:
 586         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 587
 588         return (rc);
 589 }
 590
 591                         void
 592 efx_rx_qdestroy(
 593         __in            efx_rxq_t *erp)
 594 {
 595         efx_nic_t *enp = erp->er_enp;
 596         const efx_rx_ops_t *erxop = enp->en_erxop;
 597
 598         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 599
 600         erxop->erxo_qdestroy(erp);
 601 }
 602
 603         __checkReturn   efx_rc_t
 604 efx_pseudo_hdr_pkt_length_get(
 605         __in            efx_rxq_t *erp,
 606         __in            uint8_t *buffer,
 607         __out           uint16_t *lengthp)
 608 {
 609         efx_nic_t *enp = erp->er_enp;
 610         const efx_rx_ops_t *erxop = enp->en_erxop;
 611
 612         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 613
 614         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 615 }
 616
 617 #if EFSYS_OPT_RX_SCALE
 618         __checkReturn   uint32_t
 619 efx_pseudo_hdr_hash_get(
 620         __in            efx_rxq_t *erp,
 621         __in            efx_rx_hash_alg_t func,
 622         __in            uint8_t *buffer)
 623 {
 624         efx_nic_t *enp = erp->er_enp;
 625         const efx_rx_ops_t *erxop = enp->en_erxop;
 626
 627         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 628
 629         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 630         return (erxop->erxo_prefix_hash(enp, func, buffer));
 631 }
 632 #endif  /* EFSYS_OPT_RX_SCALE */
 633
 634 #if EFSYS_OPT_SIENA
 635
 636 static  __checkReturn   efx_rc_t
 637 siena_rx_init(
 638         __in            efx_nic_t *enp)
 639 {
 640         efx_oword_t oword;
 641         unsigned int index;
 642
 643         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 644
 645         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 646         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 647         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 648         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 649         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 650         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 651         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 652
 653         /* Zero the RSS table */
 654         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 655             index++) {
 656                 EFX_ZERO_OWORD(oword);
 657                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 658                                     index, &oword, B_TRUE);
 659         }
 660
 661 #if EFSYS_OPT_RX_SCALE
 662         /* The RSS key and indirection table are writable. */
 663         enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 664
 665         /* Hardware can insert RX hash with/without RSS */
 666         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 667 #endif  /* EFSYS_OPT_RX_SCALE */
 668
 669         return (0);
 670 }
 671
 672 #if EFSYS_OPT_RX_SCATTER
 673 static  __checkReturn   efx_rc_t
 674 siena_rx_scatter_enable(
 675         __in            efx_nic_t *enp,
 676         __in            unsigned int buf_size)
 677 {
 678         unsigned int nbuf32;
 679         efx_oword_t oword;
 680         efx_rc_t rc;
 681
 682         nbuf32 = buf_size / 32;
 683         if ((nbuf32 == 0) ||
 684             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 685             ((buf_size % 32) != 0)) {
 686                 rc = EINVAL;
 687                 goto fail1;
 688         }
 689
 690         if (enp->en_rx_qcount > 0) {
 691                 rc = EBUSY;
 692                 goto fail2;
 693         }
 694
 695         /* Set scatter buffer size */
 696         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 697         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 698         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 699
 700         /* Enable scatter for packets not matching a filter */
 701         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 702         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 703         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 704
 705         return (0);
 706
 707 fail2:
 708         EFSYS_PROBE(fail2);
 709 fail1:
 710         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 711
 712         return (rc);
 713 }
 714 #endif  /* EFSYS_OPT_RX_SCATTER */
 715
 716
 717 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 718         do {                                                            \
 719                 efx_oword_t oword;                                      \
 720                                                                         \
 721                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 722                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 723                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 724                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 725                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 726                     (_insert) ? 1 : 0);                                 \
 727                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 728                                                                         \
 729                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 730                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 731                             &oword);                                    \
 732                         EFX_SET_OWORD_FIELD(oword,                      \
 733                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 734                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 735                             &oword);                                    \
 736                 }                                                       \
 737                                                                         \
 738                 _NOTE(CONSTANTCONDITION)                                \
 739         } while (B_FALSE)
 740
 741 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 742         do {                                                            \
 743                 efx_oword_t oword;                                      \
 744                                                                         \
 745                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 746                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 747                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 748                     (_ip) ? 1 : 0);                                     \
 749                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 750                     (_tcp) ? 0 : 1);                                    \
 751                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 752                     (_insert) ? 1 : 0);                                 \
 753                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 754                                                                         \
 755                 _NOTE(CONSTANTCONDITION)                                \
 756         } while (B_FALSE)
 757
 758 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 759         do {                                                            \
 760                 efx_oword_t oword;                                      \
 761                                                                         \
 762                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 763                 EFX_SET_OWORD_FIELD(oword,                              \
 764                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 765                 EFX_SET_OWORD_FIELD(oword,                              \
 766                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 767                 EFX_SET_OWORD_FIELD(oword,                              \
 768                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 769                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 770                                                                         \
 771                 (_rc) = 0;                                              \
 772                                                                         \
 773                 _NOTE(CONSTANTCONDITION)                                \
 774         } while (B_FALSE)
 775
 776
 777 #if EFSYS_OPT_RX_SCALE
 778
 779 static  __checkReturn   efx_rc_t
 780 siena_rx_scale_mode_set(
 781         __in            efx_nic_t *enp,
 782         __in            efx_rx_hash_alg_t alg,
 783         __in            efx_rx_hash_type_t type,
 784         __in            boolean_t insert)
 785 {
 786         efx_rc_t rc;
 787
 788         switch (alg) {
 789         case EFX_RX_HASHALG_LFSR:
 790                 EFX_RX_LFSR_HASH(enp, insert);
 791                 break;
 792
 793         case EFX_RX_HASHALG_TOEPLITZ:
 794                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 795                     type & EFX_RX_HASH_IPV4,
 796                     type & EFX_RX_HASH_TCPIPV4);
 797
 798                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 799                     type & EFX_RX_HASH_IPV6,
 800                     type & EFX_RX_HASH_TCPIPV6,
 801                     rc);
 802                 if (rc != 0)
 803                         goto fail1;
 804
 805                 break;
 806
 807         default:
 808                 rc = EINVAL;
 809                 goto fail2;
 810         }
 811
 812         return (0);
 813
 814 fail2:
 815         EFSYS_PROBE(fail2);
 816 fail1:
 817         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 818
 819         EFX_RX_LFSR_HASH(enp, B_FALSE);
 820
 821         return (rc);
 822 }
 823 #endif
 824
 825 #if EFSYS_OPT_RX_SCALE
 826 static  __checkReturn   efx_rc_t
 827 siena_rx_scale_key_set(
 828         __in            efx_nic_t *enp,
 829         __in_ecount(n)  uint8_t *key,
 830         __in            size_t n)
 831 {
 832         efx_oword_t oword;
 833         unsigned int byte;
 834         unsigned int offset;
 835         efx_rc_t rc;
 836
 837         byte = 0;
 838
 839         /* Write Toeplitz IPv4 hash key */
 840         EFX_ZERO_OWORD(oword);
 841         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 842             offset > 0 && byte < n;
 843             --offset)
 844                 oword.eo_u8[offset - 1] = key[byte++];
 845
 846         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 847
 848         byte = 0;
 849
 850         /* Verify Toeplitz IPv4 hash key */
 851         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 852         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 853             offset > 0 && byte < n;
 854             --offset) {
 855                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 856                         rc = EFAULT;
 857                         goto fail1;
 858                 }
 859         }
 860
 861         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 862                 goto done;
 863
 864         byte = 0;
 865
 866         /* Write Toeplitz IPv6 hash key 3 */
 867         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 868         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 869             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 870             offset > 0 && byte < n;
 871             --offset)
 872                 oword.eo_u8[offset - 1] = key[byte++];
 873
 874         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 875
 876         /* Write Toeplitz IPv6 hash key 2 */
 877         EFX_ZERO_OWORD(oword);
 878         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 879             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 880             offset > 0 && byte < n;
 881             --offset)
 882                 oword.eo_u8[offset - 1] = key[byte++];
 883
 884         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 885
 886         /* Write Toeplitz IPv6 hash key 1 */
 887         EFX_ZERO_OWORD(oword);
 888         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 889             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 890             offset > 0 && byte < n;
 891             --offset)
 892                 oword.eo_u8[offset - 1] = key[byte++];
 893
 894         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 895
 896         byte = 0;
 897
 898         /* Verify Toeplitz IPv6 hash key 3 */
 899         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 900         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 901             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 902             offset > 0 && byte < n;
 903             --offset) {
 904                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 905                         rc = EFAULT;
 906                         goto fail2;
 907                 }
 908         }
 909
 910         /* Verify Toeplitz IPv6 hash key 2 */
 911         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
 912         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
 913             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
 914             offset > 0 && byte < n;
 915             --offset) {
 916                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 917                         rc = EFAULT;
 918                         goto fail3;
 919                 }
 920         }
 921
 922         /* Verify Toeplitz IPv6 hash key 1 */
 923         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
 924         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
 925             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
 926             offset > 0 && byte < n;
 927             --offset) {
 928                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 929                         rc = EFAULT;
 930                         goto fail4;
 931                 }
 932         }
 933
 934 done:
 935         return (0);
 936
 937 fail4:
 938         EFSYS_PROBE(fail4);
 939 fail3:
 940         EFSYS_PROBE(fail3);
 941 fail2:
 942         EFSYS_PROBE(fail2);
 943 fail1:
 944         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 945
 946         return (rc);
 947 }
 948 #endif
 949
 950 #if EFSYS_OPT_RX_SCALE
 951 static  __checkReturn   efx_rc_t
 952 siena_rx_scale_tbl_set(
 953         __in            efx_nic_t *enp,
 954         __in_ecount(n)  unsigned int *table,
 955         __in            size_t n)
 956 {
 957         efx_oword_t oword;
 958         int index;
 959         efx_rc_t rc;
 960
 961         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
 962         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
 963
 964         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
 965                 rc = EINVAL;
 966                 goto fail1;
 967         }
 968
 969         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
 970                 uint32_t byte;
 971
 972                 /* Calculate the entry to place in the table */
 973                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 974
 975                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
 976
 977                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
 978
 979                 /* Write the table */
 980                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 981                                     index, &oword, B_TRUE);
 982         }
 983
 984         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
 985                 uint32_t byte;
 986
 987                 /* Determine if we're starting a new batch */
 988                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
 989
 990                 /* Read the table */
 991                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
 992                                     index, &oword, B_TRUE);
 993
 994                 /* Verify the entry */
 995                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
 996                         rc = EFAULT;
 997                         goto fail2;
 998                 }
 999         }
1000
1001         return (0);
1002
1003 fail2:
1004         EFSYS_PROBE(fail2);
1005 fail1:
1006         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1007
1008         return (rc);
1009 }
1010 #endif
1011
1012 /*
1013  * Falcon/Siena pseudo-header
1014  * --------------------------
1015  *
1016  * Receive packets are prefixed by an optional 16 byte pseudo-header.
1017  * The pseudo-header is a byte array of one of the forms:
1018  *
1019  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
1020  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1021  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1022  *
1023  * where:
1024  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
1025  *   LL.LL         LFSR hash     (16-bit big-endian)
1026  */
1027
1028 #if EFSYS_OPT_RX_SCALE
1029 static  __checkReturn   uint32_t
1030 siena_rx_prefix_hash(
1031         __in            efx_nic_t *enp,
1032         __in            efx_rx_hash_alg_t func,
1033         __in            uint8_t *buffer)
1034 {
1035         _NOTE(ARGUNUSED(enp))
1036
1037         switch (func) {
1038         case EFX_RX_HASHALG_TOEPLITZ:
1039                 return ((buffer[12] << 24) |
1040                     (buffer[13] << 16) |
1041                     (buffer[14] <<  8) |
1042                     buffer[15]);
1043
1044         case EFX_RX_HASHALG_LFSR:
1045                 return ((buffer[14] << 8) | buffer[15]);
1046
1047         default:
1048                 EFSYS_ASSERT(0);
1049                 return (0);
1050         }
1051 }
1052 #endif /* EFSYS_OPT_RX_SCALE */
1053
1054 static  __checkReturn   efx_rc_t
1055 siena_rx_prefix_pktlen(
1056         __in            efx_nic_t *enp,
1057         __in            uint8_t *buffer,
1058         __out           uint16_t *lengthp)
1059 {
1060         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1061
1062         /* Not supported by Falcon/Siena hardware */
1063         EFSYS_ASSERT(0);
1064         return (ENOTSUP);
1065 }
1066
1067
1068 static                  void
1069 siena_rx_qpost(
1070         __in            efx_rxq_t *erp,
1071         __in_ecount(n)  efsys_dma_addr_t *addrp,
1072         __in            size_t size,
1073         __in            unsigned int n,
1074         __in            unsigned int completed,
1075         __in            unsigned int added)
1076 {
1077         efx_qword_t qword;
1078         unsigned int i;
1079         unsigned int offset;
1080         unsigned int id;
1081
1082         /* The client driver must not overfill the queue */
1083         EFSYS_ASSERT3U(added - completed + n, <=,
1084             EFX_RXQ_LIMIT(erp->er_mask + 1));
1085
1086         id = added & (erp->er_mask);
1087         for (i = 0; i < n; i++) {
1088                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1089                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1090                     size_t, size);
1091
1092                 EFX_POPULATE_QWORD_3(qword,
1093                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1094                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1095                     (uint32_t)(addrp[i] & 0xffffffff),
1096                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1097                     (uint32_t)(addrp[i] >> 32));
1098
1099                 offset = id * sizeof (efx_qword_t);
1100                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1101
1102                 id = (id + 1) & (erp->er_mask);
1103         }
1104 }
1105
1106 static                  void
1107 siena_rx_qpush(
1108         __in    efx_rxq_t *erp,
1109         __in    unsigned int added,
1110         __inout unsigned int *pushedp)
1111 {
1112         efx_nic_t *enp = erp->er_enp;
1113         unsigned int pushed = *pushedp;
1114         uint32_t wptr;
1115         efx_oword_t oword;
1116         efx_dword_t dword;
1117
1118         /* All descriptors are pushed */
1119         *pushedp = added;
1120
1121         /* Push the populated descriptors out */
1122         wptr = added & erp->er_mask;
1123
1124         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1125
1126         /* Only write the third DWORD */
1127         EFX_POPULATE_DWORD_1(dword,
1128             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1129
1130         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1131         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1132             wptr, pushed & erp->er_mask);
1133         EFSYS_PIO_WRITE_BARRIER();
1134         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1135                             erp->er_index, &dword, B_FALSE);
1136 }
1137
1138 #if EFSYS_OPT_RX_PACKED_STREAM
1139 static          void
1140 siena_rx_qps_update_credits(
1141         __in            efx_rxq_t *erp)
1142 {
1143         /* Not supported by Siena hardware */
1144         EFSYS_ASSERT(0);
1145 }
1146
1147 static          uint8_t *
1148 siena_rx_qps_packet_info(
1149         __in            efx_rxq_t *erp,
1150         __in            uint8_t *buffer,
1151         __in            uint32_t buffer_length,
1152         __in            uint32_t current_offset,
1153         __out           uint16_t *lengthp,
1154         __out           uint32_t *next_offsetp,
1155         __out           uint32_t *timestamp)
1156 {
1157         /* Not supported by Siena hardware */
1158         EFSYS_ASSERT(0);
1159
1160         return (NULL);
1161 }
1162 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1163
1164 static  __checkReturn   efx_rc_t
1165 siena_rx_qflush(
1166         __in    efx_rxq_t *erp)
1167 {
1168         efx_nic_t *enp = erp->er_enp;
1169         efx_oword_t oword;
1170         uint32_t label;
1171
1172         label = erp->er_index;
1173
1174         /* Flush the queue */
1175         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1176             FRF_AZ_RX_FLUSH_DESCQ, label);
1177         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1178
1179         return (0);
1180 }
1181
1182 static          void
1183 siena_rx_qenable(
1184         __in    efx_rxq_t *erp)
1185 {
1186         efx_nic_t *enp = erp->er_enp;
1187         efx_oword_t oword;
1188
1189         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1190
1191         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1192                             erp->er_index, &oword, B_TRUE);
1193
1194         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1195         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1196         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1197
1198         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1199                             erp->er_index, &oword, B_TRUE);
1200 }
1201
1202 static  __checkReturn   efx_rc_t
1203 siena_rx_qcreate(
1204         __in            efx_nic_t *enp,
1205         __in            unsigned int index,
1206         __in            unsigned int label,
1207         __in            efx_rxq_type_t type,
1208         __in            efsys_mem_t *esmp,
1209         __in            size_t n,
1210         __in            uint32_t id,
1211         __in            efx_evq_t *eep,
1212         __in            efx_rxq_t *erp)
1213 {
1214         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1215         efx_oword_t oword;
1216         uint32_t size;
1217         boolean_t jumbo;
1218         efx_rc_t rc;
1219
1220         _NOTE(ARGUNUSED(esmp))
1221
1222         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1223             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1224         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1225         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1226
1227         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1228         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1229
1230         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1231                 rc = EINVAL;
1232                 goto fail1;
1233         }
1234         if (index >= encp->enc_rxq_limit) {
1235                 rc = EINVAL;
1236                 goto fail2;
1237         }
1238         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1239             size++)
1240                 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1241                         break;
1242         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1243                 rc = EINVAL;
1244                 goto fail3;
1245         }
1246
1247         switch (type) {
1248         case EFX_RXQ_TYPE_DEFAULT:
1249                 jumbo = B_FALSE;
1250                 break;
1251
1252 #if EFSYS_OPT_RX_SCATTER
1253         case EFX_RXQ_TYPE_SCATTER:
1254                 if (enp->en_family < EFX_FAMILY_SIENA) {
1255                         rc = EINVAL;
1256                         goto fail4;
1257                 }
1258                 jumbo = B_TRUE;
1259                 break;
1260 #endif  /* EFSYS_OPT_RX_SCATTER */
1261
1262         default:
1263                 rc = EINVAL;
1264                 goto fail4;
1265         }
1266
1267         /* Set up the new descriptor queue */
1268         EFX_POPULATE_OWORD_7(oword,
1269             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1270             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1271             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1272             FRF_AZ_RX_DESCQ_LABEL, label,
1273             FRF_AZ_RX_DESCQ_SIZE, size,
1274             FRF_AZ_RX_DESCQ_TYPE, 0,
1275             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1276
1277         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1278                             erp->er_index, &oword, B_TRUE);
1279
1280         return (0);
1281
1282 fail4:
1283         EFSYS_PROBE(fail4);
1284 fail3:
1285         EFSYS_PROBE(fail3);
1286 fail2:
1287         EFSYS_PROBE(fail2);
1288 fail1:
1289         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1290
1291         return (rc);
1292 }
1293
1294 static          void
1295 siena_rx_qdestroy(
1296         __in    efx_rxq_t *erp)
1297 {
1298         efx_nic_t *enp = erp->er_enp;
1299         efx_oword_t oword;
1300
1301         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1302         --enp->en_rx_qcount;
1303
1304         /* Purge descriptor queue */
1305         EFX_ZERO_OWORD(oword);
1306
1307         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1308                             erp->er_index, &oword, B_TRUE);
1309
1310         /* Free the RXQ object */
1311         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1312 }
1313
1314 static          void
1315 siena_rx_fini(
1316         __in    efx_nic_t *enp)
1317 {
1318         _NOTE(ARGUNUSED(enp))
1319 }
1320
1321 #endif /* EFSYS_OPT_SIENA */