drivers/net/sfc/base/efx_rx.c

   1 /*
   2  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_SIENA
  36
  37 static  __checkReturn   efx_rc_t
  38 siena_rx_init(
  39         __in            efx_nic_t *enp);
  40
  41 static                  void
  42 siena_rx_fini(
  43         __in            efx_nic_t *enp);
  44
  45 #if EFSYS_OPT_RX_SCATTER
  46 static  __checkReturn   efx_rc_t
  47 siena_rx_scatter_enable(
  48         __in            efx_nic_t *enp,
  49         __in            unsigned int buf_size);
  50 #endif /* EFSYS_OPT_RX_SCATTER */
  51
  52 #if EFSYS_OPT_RX_SCALE
  53 static  __checkReturn   efx_rc_t
  54 siena_rx_scale_mode_set(
  55         __in            efx_nic_t *enp,
  56         __in            uint32_t rss_context,
  57         __in            efx_rx_hash_alg_t alg,
  58         __in            efx_rx_hash_type_t type,
  59         __in            boolean_t insert);
  60
  61 static  __checkReturn   efx_rc_t
  62 siena_rx_scale_key_set(
  63         __in            efx_nic_t *enp,
  64         __in            uint32_t rss_context,
  65         __in_ecount(n)  uint8_t *key,
  66         __in            size_t n);
  67
  68 static  __checkReturn   efx_rc_t
  69 siena_rx_scale_tbl_set(
  70         __in            efx_nic_t *enp,
  71         __in            uint32_t rss_context,
  72         __in_ecount(n)  unsigned int *table,
  73         __in            size_t n);
  74
  75 static  __checkReturn   uint32_t
  76 siena_rx_prefix_hash(
  77         __in            efx_nic_t *enp,
  78         __in            efx_rx_hash_alg_t func,
  79         __in            uint8_t *buffer);
  80
  81 #endif /* EFSYS_OPT_RX_SCALE */
  82
  83 static  __checkReturn   efx_rc_t
  84 siena_rx_prefix_pktlen(
  85         __in            efx_nic_t *enp,
  86         __in            uint8_t *buffer,
  87         __out           uint16_t *lengthp);
  88
  89 static                          void
  90 siena_rx_qpost(
  91         __in                    efx_rxq_t *erp,
  92         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
  93         __in                    size_t size,
  94         __in                    unsigned int ndescs,
  95         __in                    unsigned int completed,
  96         __in                    unsigned int added);
  97
  98 static                  void
  99 siena_rx_qpush(
 100         __in            efx_rxq_t *erp,
 101         __in            unsigned int added,
 102         __inout         unsigned int *pushedp);
 103
 104 #if EFSYS_OPT_RX_PACKED_STREAM
 105 static          void
 106 siena_rx_qpush_ps_credits(
 107         __in            efx_rxq_t *erp);
 108
 109 static  __checkReturn   uint8_t *
 110 siena_rx_qps_packet_info(
 111         __in            efx_rxq_t *erp,
 112         __in            uint8_t *buffer,
 113         __in            uint32_t buffer_length,
 114         __in            uint32_t current_offset,
 115         __out           uint16_t *lengthp,
 116         __out           uint32_t *next_offsetp,
 117         __out           uint32_t *timestamp);
 118 #endif
 119
 120 static  __checkReturn   efx_rc_t
 121 siena_rx_qflush(
 122         __in            efx_rxq_t *erp);
 123
 124 static                  void
 125 siena_rx_qenable(
 126         __in            efx_rxq_t *erp);
 127
 128 static  __checkReturn   efx_rc_t
 129 siena_rx_qcreate(
 130         __in            efx_nic_t *enp,
 131         __in            unsigned int index,
 132         __in            unsigned int label,
 133         __in            efx_rxq_type_t type,
 134         __in            uint32_t type_data,
 135         __in            efsys_mem_t *esmp,
 136         __in            size_t ndescs,
 137         __in            uint32_t id,
 138         __in            unsigned int flags,
 139         __in            efx_evq_t *eep,
 140         __in            efx_rxq_t *erp);
 141
 142 static                  void
 143 siena_rx_qdestroy(
 144         __in            efx_rxq_t *erp);
 145
 146 #endif /* EFSYS_OPT_SIENA */
 147
 148
 149 #if EFSYS_OPT_SIENA
 150 static const efx_rx_ops_t __efx_rx_siena_ops = {
 151         siena_rx_init,                          /* erxo_init */
 152         siena_rx_fini,                          /* erxo_fini */
 153 #if EFSYS_OPT_RX_SCATTER
 154         siena_rx_scatter_enable,                /* erxo_scatter_enable */
 155 #endif
 156 #if EFSYS_OPT_RX_SCALE
 157         NULL,                                   /* erxo_scale_context_alloc */
 158         NULL,                                   /* erxo_scale_context_free */
 159         siena_rx_scale_mode_set,                /* erxo_scale_mode_set */
 160         siena_rx_scale_key_set,                 /* erxo_scale_key_set */
 161         siena_rx_scale_tbl_set,                 /* erxo_scale_tbl_set */
 162         siena_rx_prefix_hash,                   /* erxo_prefix_hash */
 163 #endif
 164         siena_rx_prefix_pktlen,                 /* erxo_prefix_pktlen */
 165         siena_rx_qpost,                         /* erxo_qpost */
 166         siena_rx_qpush,                         /* erxo_qpush */
 167 #if EFSYS_OPT_RX_PACKED_STREAM
 168         siena_rx_qpush_ps_credits,              /* erxo_qpush_ps_credits */
 169         siena_rx_qps_packet_info,               /* erxo_qps_packet_info */
 170 #endif
 171         siena_rx_qflush,                        /* erxo_qflush */
 172         siena_rx_qenable,                       /* erxo_qenable */
 173         siena_rx_qcreate,                       /* erxo_qcreate */
 174         siena_rx_qdestroy,                      /* erxo_qdestroy */
 175 };
 176 #endif  /* EFSYS_OPT_SIENA */
 177
 178 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 179 static const efx_rx_ops_t __efx_rx_ef10_ops = {
 180         ef10_rx_init,                           /* erxo_init */
 181         ef10_rx_fini,                           /* erxo_fini */
 182 #if EFSYS_OPT_RX_SCATTER
 183         ef10_rx_scatter_enable,                 /* erxo_scatter_enable */
 184 #endif
 185 #if EFSYS_OPT_RX_SCALE
 186         ef10_rx_scale_context_alloc,            /* erxo_scale_context_alloc */
 187         ef10_rx_scale_context_free,             /* erxo_scale_context_free */
 188         ef10_rx_scale_mode_set,                 /* erxo_scale_mode_set */
 189         ef10_rx_scale_key_set,                  /* erxo_scale_key_set */
 190         ef10_rx_scale_tbl_set,                  /* erxo_scale_tbl_set */
 191         ef10_rx_prefix_hash,                    /* erxo_prefix_hash */
 192 #endif
 193         ef10_rx_prefix_pktlen,                  /* erxo_prefix_pktlen */
 194         ef10_rx_qpost,                          /* erxo_qpost */
 195         ef10_rx_qpush,                          /* erxo_qpush */
 196 #if EFSYS_OPT_RX_PACKED_STREAM
 197         ef10_rx_qpush_ps_credits,               /* erxo_qpush_ps_credits */
 198         ef10_rx_qps_packet_info,                /* erxo_qps_packet_info */
 199 #endif
 200         ef10_rx_qflush,                         /* erxo_qflush */
 201         ef10_rx_qenable,                        /* erxo_qenable */
 202         ef10_rx_qcreate,                        /* erxo_qcreate */
 203         ef10_rx_qdestroy,                       /* erxo_qdestroy */
 204 };
 205 #endif  /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 206
 207
 208         __checkReturn   efx_rc_t
 209 efx_rx_init(
 210         __inout         efx_nic_t *enp)
 211 {
 212         const efx_rx_ops_t *erxop;
 213         efx_rc_t rc;
 214
 215         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 216         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 217
 218         if (!(enp->en_mod_flags & EFX_MOD_EV)) {
 219                 rc = EINVAL;
 220                 goto fail1;
 221         }
 222
 223         if (enp->en_mod_flags & EFX_MOD_RX) {
 224                 rc = EINVAL;
 225                 goto fail2;
 226         }
 227
 228         switch (enp->en_family) {
 229 #if EFSYS_OPT_SIENA
 230         case EFX_FAMILY_SIENA:
 231                 erxop = &__efx_rx_siena_ops;
 232                 break;
 233 #endif /* EFSYS_OPT_SIENA */
 234
 235 #if EFSYS_OPT_HUNTINGTON
 236         case EFX_FAMILY_HUNTINGTON:
 237                 erxop = &__efx_rx_ef10_ops;
 238                 break;
 239 #endif /* EFSYS_OPT_HUNTINGTON */
 240
 241 #if EFSYS_OPT_MEDFORD
 242         case EFX_FAMILY_MEDFORD:
 243                 erxop = &__efx_rx_ef10_ops;
 244                 break;
 245 #endif /* EFSYS_OPT_MEDFORD */
 246
 247         default:
 248                 EFSYS_ASSERT(0);
 249                 rc = ENOTSUP;
 250                 goto fail3;
 251         }
 252
 253         if ((rc = erxop->erxo_init(enp)) != 0)
 254                 goto fail4;
 255
 256         enp->en_erxop = erxop;
 257         enp->en_mod_flags |= EFX_MOD_RX;
 258         return (0);
 259
 260 fail4:
 261         EFSYS_PROBE(fail4);
 262 fail3:
 263         EFSYS_PROBE(fail3);
 264 fail2:
 265         EFSYS_PROBE(fail2);
 266 fail1:
 267         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 268
 269         enp->en_erxop = NULL;
 270         enp->en_mod_flags &= ~EFX_MOD_RX;
 271         return (rc);
 272 }
 273
 274                         void
 275 efx_rx_fini(
 276         __in            efx_nic_t *enp)
 277 {
 278         const efx_rx_ops_t *erxop = enp->en_erxop;
 279
 280         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 281         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
 282         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 283         EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
 284
 285         erxop->erxo_fini(enp);
 286
 287         enp->en_erxop = NULL;
 288         enp->en_mod_flags &= ~EFX_MOD_RX;
 289 }
 290
 291 #if EFSYS_OPT_RX_SCATTER
 292         __checkReturn   efx_rc_t
 293 efx_rx_scatter_enable(
 294         __in            efx_nic_t *enp,
 295         __in            unsigned int buf_size)
 296 {
 297         const efx_rx_ops_t *erxop = enp->en_erxop;
 298         efx_rc_t rc;
 299
 300         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 301         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 302
 303         if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
 304                 goto fail1;
 305
 306         return (0);
 307
 308 fail1:
 309         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 310         return (rc);
 311 }
 312 #endif  /* EFSYS_OPT_RX_SCATTER */
 313
 314 #if EFSYS_OPT_RX_SCALE
 315         __checkReturn   efx_rc_t
 316 efx_rx_hash_default_support_get(
 317         __in            efx_nic_t *enp,
 318         __out           efx_rx_hash_support_t *supportp)
 319 {
 320         efx_rc_t rc;
 321
 322         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 323         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 324
 325         if (supportp == NULL) {
 326                 rc = EINVAL;
 327                 goto fail1;
 328         }
 329
 330         /*
 331          * Report the hashing support the client gets by default if it
 332          * does not allocate an RSS context itself.
 333          */
 334         *supportp = enp->en_hash_support;
 335
 336         return (0);
 337
 338 fail1:
 339         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 340
 341         return (rc);
 342 }
 343
 344         __checkReturn   efx_rc_t
 345 efx_rx_scale_default_support_get(
 346         __in            efx_nic_t *enp,
 347         __out           efx_rx_scale_context_type_t *typep)
 348 {
 349         efx_rc_t rc;
 350
 351         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 352         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 353
 354         if (typep == NULL) {
 355                 rc = EINVAL;
 356                 goto fail1;
 357         }
 358
 359         /*
 360          * Report the RSS support the client gets by default if it
 361          * does not allocate an RSS context itself.
 362          */
 363         *typep = enp->en_rss_context_type;
 364
 365         return (0);
 366
 367 fail1:
 368         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 369
 370         return (rc);
 371 }
 372 #endif  /* EFSYS_OPT_RX_SCALE */
 373
 374 #if EFSYS_OPT_RX_SCALE
 375         __checkReturn   efx_rc_t
 376 efx_rx_scale_context_alloc(
 377         __in            efx_nic_t *enp,
 378         __in            efx_rx_scale_context_type_t type,
 379         __in            uint32_t num_queues,
 380         __out           uint32_t *rss_contextp)
 381 {
 382         const efx_rx_ops_t *erxop = enp->en_erxop;
 383         efx_rc_t rc;
 384
 385         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 386         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 387
 388         if (erxop->erxo_scale_context_alloc == NULL) {
 389                 rc = ENOTSUP;
 390                 goto fail1;
 391         }
 392         if ((rc = erxop->erxo_scale_context_alloc(enp, type,
 393                             num_queues, rss_contextp)) != 0) {
 394                 goto fail2;
 395         }
 396
 397         return (0);
 398
 399 fail2:
 400         EFSYS_PROBE(fail2);
 401 fail1:
 402         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 403         return (rc);
 404 }
 405 #endif  /* EFSYS_OPT_RX_SCALE */
 406
 407 #if EFSYS_OPT_RX_SCALE
 408         __checkReturn   efx_rc_t
 409 efx_rx_scale_context_free(
 410         __in            efx_nic_t *enp,
 411         __in            uint32_t rss_context)
 412 {
 413         const efx_rx_ops_t *erxop = enp->en_erxop;
 414         efx_rc_t rc;
 415
 416         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 417         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 418
 419         if (erxop->erxo_scale_context_free == NULL) {
 420                 rc = ENOTSUP;
 421                 goto fail1;
 422         }
 423         if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
 424                 goto fail2;
 425
 426         return (0);
 427
 428 fail2:
 429         EFSYS_PROBE(fail2);
 430 fail1:
 431         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 432         return (rc);
 433 }
 434 #endif  /* EFSYS_OPT_RX_SCALE */
 435
 436 #if EFSYS_OPT_RX_SCALE
 437         __checkReturn   efx_rc_t
 438 efx_rx_scale_mode_set(
 439         __in            efx_nic_t *enp,
 440         __in            uint32_t rss_context,
 441         __in            efx_rx_hash_alg_t alg,
 442         __in            efx_rx_hash_type_t type,
 443         __in            boolean_t insert)
 444 {
 445         const efx_rx_ops_t *erxop = enp->en_erxop;
 446         efx_rc_t rc;
 447
 448         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 449         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 450
 451         if (erxop->erxo_scale_mode_set != NULL) {
 452                 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg,
 453                             type, insert)) != 0)
 454                         goto fail1;
 455         }
 456
 457         return (0);
 458
 459 fail1:
 460         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 461         return (rc);
 462 }
 463 #endif  /* EFSYS_OPT_RX_SCALE */
 464
 465 #if EFSYS_OPT_RX_SCALE
 466         __checkReturn   efx_rc_t
 467 efx_rx_scale_key_set(
 468         __in            efx_nic_t *enp,
 469         __in            uint32_t rss_context,
 470         __in_ecount(n)  uint8_t *key,
 471         __in            size_t n)
 472 {
 473         const efx_rx_ops_t *erxop = enp->en_erxop;
 474         efx_rc_t rc;
 475
 476         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 477         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 478
 479         if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0)
 480                 goto fail1;
 481
 482         return (0);
 483
 484 fail1:
 485         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 486
 487         return (rc);
 488 }
 489 #endif  /* EFSYS_OPT_RX_SCALE */
 490
 491 #if EFSYS_OPT_RX_SCALE
 492         __checkReturn   efx_rc_t
 493 efx_rx_scale_tbl_set(
 494         __in            efx_nic_t *enp,
 495         __in            uint32_t rss_context,
 496         __in_ecount(n)  unsigned int *table,
 497         __in            size_t n)
 498 {
 499         const efx_rx_ops_t *erxop = enp->en_erxop;
 500         efx_rc_t rc;
 501
 502         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 503         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 504
 505         if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0)
 506                 goto fail1;
 507
 508         return (0);
 509
 510 fail1:
 511         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 512
 513         return (rc);
 514 }
 515 #endif  /* EFSYS_OPT_RX_SCALE */
 516
 517                                 void
 518 efx_rx_qpost(
 519         __in                    efx_rxq_t *erp,
 520         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 521         __in                    size_t size,
 522         __in                    unsigned int ndescs,
 523         __in                    unsigned int completed,
 524         __in                    unsigned int added)
 525 {
 526         efx_nic_t *enp = erp->er_enp;
 527         const efx_rx_ops_t *erxop = enp->en_erxop;
 528
 529         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 530
 531         erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added);
 532 }
 533
 534 #if EFSYS_OPT_RX_PACKED_STREAM
 535
 536                         void
 537 efx_rx_qpush_ps_credits(
 538         __in            efx_rxq_t *erp)
 539 {
 540         efx_nic_t *enp = erp->er_enp;
 541         const efx_rx_ops_t *erxop = enp->en_erxop;
 542
 543         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 544
 545         erxop->erxo_qpush_ps_credits(erp);
 546 }
 547
 548         __checkReturn   uint8_t *
 549 efx_rx_qps_packet_info(
 550         __in            efx_rxq_t *erp,
 551         __in            uint8_t *buffer,
 552         __in            uint32_t buffer_length,
 553         __in            uint32_t current_offset,
 554         __out           uint16_t *lengthp,
 555         __out           uint32_t *next_offsetp,
 556         __out           uint32_t *timestamp)
 557 {
 558         efx_nic_t *enp = erp->er_enp;
 559         const efx_rx_ops_t *erxop = enp->en_erxop;
 560
 561         return (erxop->erxo_qps_packet_info(erp, buffer,
 562                 buffer_length, current_offset, lengthp,
 563                 next_offsetp, timestamp));
 564 }
 565
 566 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 567
 568                         void
 569 efx_rx_qpush(
 570         __in            efx_rxq_t *erp,
 571         __in            unsigned int added,
 572         __inout         unsigned int *pushedp)
 573 {
 574         efx_nic_t *enp = erp->er_enp;
 575         const efx_rx_ops_t *erxop = enp->en_erxop;
 576
 577         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 578
 579         erxop->erxo_qpush(erp, added, pushedp);
 580 }
 581
 582         __checkReturn   efx_rc_t
 583 efx_rx_qflush(
 584         __in            efx_rxq_t *erp)
 585 {
 586         efx_nic_t *enp = erp->er_enp;
 587         const efx_rx_ops_t *erxop = enp->en_erxop;
 588         efx_rc_t rc;
 589
 590         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 591
 592         if ((rc = erxop->erxo_qflush(erp)) != 0)
 593                 goto fail1;
 594
 595         return (0);
 596
 597 fail1:
 598         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 599
 600         return (rc);
 601 }
 602
 603                         void
 604 efx_rx_qenable(
 605         __in            efx_rxq_t *erp)
 606 {
 607         efx_nic_t *enp = erp->er_enp;
 608         const efx_rx_ops_t *erxop = enp->en_erxop;
 609
 610         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 611
 612         erxop->erxo_qenable(erp);
 613 }
 614
 615 static  __checkReturn   efx_rc_t
 616 efx_rx_qcreate_internal(
 617         __in            efx_nic_t *enp,
 618         __in            unsigned int index,
 619         __in            unsigned int label,
 620         __in            efx_rxq_type_t type,
 621         __in            uint32_t type_data,
 622         __in            efsys_mem_t *esmp,
 623         __in            size_t ndescs,
 624         __in            uint32_t id,
 625         __in            unsigned int flags,
 626         __in            efx_evq_t *eep,
 627         __deref_out     efx_rxq_t **erpp)
 628 {
 629         const efx_rx_ops_t *erxop = enp->en_erxop;
 630         efx_rxq_t *erp;
 631         efx_rc_t rc;
 632
 633         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 634         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
 635
 636         /* Allocate an RXQ object */
 637         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
 638
 639         if (erp == NULL) {
 640                 rc = ENOMEM;
 641                 goto fail1;
 642         }
 643
 644         erp->er_magic = EFX_RXQ_MAGIC;
 645         erp->er_enp = enp;
 646         erp->er_index = index;
 647         erp->er_mask = ndescs - 1;
 648         erp->er_esmp = esmp;
 649
 650         if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp,
 651             ndescs, id, flags, eep, erp)) != 0)
 652                 goto fail2;
 653
 654         enp->en_rx_qcount++;
 655         *erpp = erp;
 656
 657         return (0);
 658
 659 fail2:
 660         EFSYS_PROBE(fail2);
 661
 662         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
 663 fail1:
 664         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 665
 666         return (rc);
 667 }
 668
 669         __checkReturn   efx_rc_t
 670 efx_rx_qcreate(
 671         __in            efx_nic_t *enp,
 672         __in            unsigned int index,
 673         __in            unsigned int label,
 674         __in            efx_rxq_type_t type,
 675         __in            efsys_mem_t *esmp,
 676         __in            size_t ndescs,
 677         __in            uint32_t id,
 678         __in            unsigned int flags,
 679         __in            efx_evq_t *eep,
 680         __deref_out     efx_rxq_t **erpp)
 681 {
 682         return efx_rx_qcreate_internal(enp, index, label, type, 0, esmp, ndescs,
 683             id, flags, eep, erpp);
 684 }
 685
 686 #if EFSYS_OPT_RX_PACKED_STREAM
 687
 688         __checkReturn   efx_rc_t
 689 efx_rx_qcreate_packed_stream(
 690         __in            efx_nic_t *enp,
 691         __in            unsigned int index,
 692         __in            unsigned int label,
 693         __in            uint32_t ps_buf_size,
 694         __in            efsys_mem_t *esmp,
 695         __in            size_t ndescs,
 696         __in            efx_evq_t *eep,
 697         __deref_out     efx_rxq_t **erpp)
 698 {
 699         return efx_rx_qcreate_internal(enp, index, label,
 700             EFX_RXQ_TYPE_PACKED_STREAM, ps_buf_size, esmp, ndescs,
 701             0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp);
 702 }
 703
 704 #endif
 705
 706                         void
 707 efx_rx_qdestroy(
 708         __in            efx_rxq_t *erp)
 709 {
 710         efx_nic_t *enp = erp->er_enp;
 711         const efx_rx_ops_t *erxop = enp->en_erxop;
 712
 713         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 714
 715         erxop->erxo_qdestroy(erp);
 716 }
 717
 718         __checkReturn   efx_rc_t
 719 efx_pseudo_hdr_pkt_length_get(
 720         __in            efx_rxq_t *erp,
 721         __in            uint8_t *buffer,
 722         __out           uint16_t *lengthp)
 723 {
 724         efx_nic_t *enp = erp->er_enp;
 725         const efx_rx_ops_t *erxop = enp->en_erxop;
 726
 727         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 728
 729         return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
 730 }
 731
 732 #if EFSYS_OPT_RX_SCALE
 733         __checkReturn   uint32_t
 734 efx_pseudo_hdr_hash_get(
 735         __in            efx_rxq_t *erp,
 736         __in            efx_rx_hash_alg_t func,
 737         __in            uint8_t *buffer)
 738 {
 739         efx_nic_t *enp = erp->er_enp;
 740         const efx_rx_ops_t *erxop = enp->en_erxop;
 741
 742         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
 743
 744         EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
 745         return (erxop->erxo_prefix_hash(enp, func, buffer));
 746 }
 747 #endif  /* EFSYS_OPT_RX_SCALE */
 748
 749 #if EFSYS_OPT_SIENA
 750
 751 static  __checkReturn   efx_rc_t
 752 siena_rx_init(
 753         __in            efx_nic_t *enp)
 754 {
 755         efx_oword_t oword;
 756         unsigned int index;
 757
 758         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 759
 760         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
 761         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
 762         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
 763         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
 764         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
 765         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
 766         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 767
 768         /* Zero the RSS table */
 769         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
 770             index++) {
 771                 EFX_ZERO_OWORD(oword);
 772                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
 773                                     index, &oword, B_TRUE);
 774         }
 775
 776 #if EFSYS_OPT_RX_SCALE
 777         /* The RSS key and indirection table are writable. */
 778         enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 779
 780         /* Hardware can insert RX hash with/without RSS */
 781         enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 782 #endif  /* EFSYS_OPT_RX_SCALE */
 783
 784         return (0);
 785 }
 786
 787 #if EFSYS_OPT_RX_SCATTER
 788 static  __checkReturn   efx_rc_t
 789 siena_rx_scatter_enable(
 790         __in            efx_nic_t *enp,
 791         __in            unsigned int buf_size)
 792 {
 793         unsigned int nbuf32;
 794         efx_oword_t oword;
 795         efx_rc_t rc;
 796
 797         nbuf32 = buf_size / 32;
 798         if ((nbuf32 == 0) ||
 799             (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
 800             ((buf_size % 32) != 0)) {
 801                 rc = EINVAL;
 802                 goto fail1;
 803         }
 804
 805         if (enp->en_rx_qcount > 0) {
 806                 rc = EBUSY;
 807                 goto fail2;
 808         }
 809
 810         /* Set scatter buffer size */
 811         EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
 812         EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
 813         EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
 814
 815         /* Enable scatter for packets not matching a filter */
 816         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 817         EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
 818         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 819
 820         return (0);
 821
 822 fail2:
 823         EFSYS_PROBE(fail2);
 824 fail1:
 825         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 826
 827         return (rc);
 828 }
 829 #endif  /* EFSYS_OPT_RX_SCATTER */
 830
 831
 832 #define EFX_RX_LFSR_HASH(_enp, _insert)                                 \
 833         do {                                                            \
 834                 efx_oword_t oword;                                      \
 835                                                                         \
 836                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 837                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);      \
 838                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);       \
 839                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);       \
 840                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 841                     (_insert) ? 1 : 0);                                 \
 842                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 843                                                                         \
 844                 if ((_enp)->en_family == EFX_FAMILY_SIENA) {            \
 845                         EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3,   \
 846                             &oword);                                    \
 847                         EFX_SET_OWORD_FIELD(oword,                      \
 848                             FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0);        \
 849                         EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3,  \
 850                             &oword);                                    \
 851                 }                                                       \
 852                                                                         \
 853                 _NOTE(CONSTANTCONDITION)                                \
 854         } while (B_FALSE)
 855
 856 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp)             \
 857         do {                                                            \
 858                 efx_oword_t oword;                                      \
 859                                                                         \
 860                 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword);        \
 861                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1);      \
 862                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH,           \
 863                     (_ip) ? 1 : 0);                                     \
 864                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP,           \
 865                     (_tcp) ? 0 : 1);                                    \
 866                 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR,    \
 867                     (_insert) ? 1 : 0);                                 \
 868                 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword);       \
 869                                                                         \
 870                 _NOTE(CONSTANTCONDITION)                                \
 871         } while (B_FALSE)
 872
 873 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc)                 \
 874         do {                                                            \
 875                 efx_oword_t oword;                                      \
 876                                                                         \
 877                 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword);  \
 878                 EFX_SET_OWORD_FIELD(oword,                              \
 879                     FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1);                \
 880                 EFX_SET_OWORD_FIELD(oword,                              \
 881                     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
 882                 EFX_SET_OWORD_FIELD(oword,                              \
 883                     FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1);   \
 884                 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
 885                                                                         \
 886                 (_rc) = 0;                                              \
 887                                                                         \
 888                 _NOTE(CONSTANTCONDITION)                                \
 889         } while (B_FALSE)
 890
 891
 892 #if EFSYS_OPT_RX_SCALE
 893
 894 static  __checkReturn   efx_rc_t
 895 siena_rx_scale_mode_set(
 896         __in            efx_nic_t *enp,
 897         __in            uint32_t rss_context,
 898         __in            efx_rx_hash_alg_t alg,
 899         __in            efx_rx_hash_type_t type,
 900         __in            boolean_t insert)
 901 {
 902         efx_rc_t rc;
 903
 904         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 905                 rc = EINVAL;
 906                 goto fail1;
 907         }
 908
 909         switch (alg) {
 910         case EFX_RX_HASHALG_LFSR:
 911                 EFX_RX_LFSR_HASH(enp, insert);
 912                 break;
 913
 914         case EFX_RX_HASHALG_TOEPLITZ:
 915                 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
 916                     type & EFX_RX_HASH_IPV4,
 917                     type & EFX_RX_HASH_TCPIPV4);
 918
 919                 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
 920                     type & EFX_RX_HASH_IPV6,
 921                     type & EFX_RX_HASH_TCPIPV6,
 922                     rc);
 923                 if (rc != 0)
 924                         goto fail2;
 925
 926                 break;
 927
 928         default:
 929                 rc = EINVAL;
 930                 goto fail3;
 931         }
 932
 933         return (0);
 934
 935 fail3:
 936         EFSYS_PROBE(fail3);
 937 fail2:
 938         EFSYS_PROBE(fail2);
 939 fail1:
 940         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 941
 942         EFX_RX_LFSR_HASH(enp, B_FALSE);
 943
 944         return (rc);
 945 }
 946 #endif
 947
 948 #if EFSYS_OPT_RX_SCALE
 949 static  __checkReturn   efx_rc_t
 950 siena_rx_scale_key_set(
 951         __in            efx_nic_t *enp,
 952         __in            uint32_t rss_context,
 953         __in_ecount(n)  uint8_t *key,
 954         __in            size_t n)
 955 {
 956         efx_oword_t oword;
 957         unsigned int byte;
 958         unsigned int offset;
 959         efx_rc_t rc;
 960
 961         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 962                 rc = EINVAL;
 963                 goto fail1;
 964         }
 965
 966         byte = 0;
 967
 968         /* Write Toeplitz IPv4 hash key */
 969         EFX_ZERO_OWORD(oword);
 970         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 971             offset > 0 && byte < n;
 972             --offset)
 973                 oword.eo_u8[offset - 1] = key[byte++];
 974
 975         EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 976
 977         byte = 0;
 978
 979         /* Verify Toeplitz IPv4 hash key */
 980         EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
 981         for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
 982             offset > 0 && byte < n;
 983             --offset) {
 984                 if (oword.eo_u8[offset - 1] != key[byte++]) {
 985                         rc = EFAULT;
 986                         goto fail2;
 987                 }
 988         }
 989
 990         if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
 991                 goto done;
 992
 993         byte = 0;
 994
 995         /* Write Toeplitz IPv6 hash key 3 */
 996         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
 997         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
 998             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
 999             offset > 0 && byte < n;
1000             --offset)
1001                 oword.eo_u8[offset - 1] = key[byte++];
1002
1003         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1004
1005         /* Write Toeplitz IPv6 hash key 2 */
1006         EFX_ZERO_OWORD(oword);
1007         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1008             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1009             offset > 0 && byte < n;
1010             --offset)
1011                 oword.eo_u8[offset - 1] = key[byte++];
1012
1013         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1014
1015         /* Write Toeplitz IPv6 hash key 1 */
1016         EFX_ZERO_OWORD(oword);
1017         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1018             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1019             offset > 0 && byte < n;
1020             --offset)
1021                 oword.eo_u8[offset - 1] = key[byte++];
1022
1023         EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1024
1025         byte = 0;
1026
1027         /* Verify Toeplitz IPv6 hash key 3 */
1028         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1029         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1030             FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1031             offset > 0 && byte < n;
1032             --offset) {
1033                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1034                         rc = EFAULT;
1035                         goto fail3;
1036                 }
1037         }
1038
1039         /* Verify Toeplitz IPv6 hash key 2 */
1040         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1041         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1042             FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1043             offset > 0 && byte < n;
1044             --offset) {
1045                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1046                         rc = EFAULT;
1047                         goto fail4;
1048                 }
1049         }
1050
1051         /* Verify Toeplitz IPv6 hash key 1 */
1052         EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1053         for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1054             FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1055             offset > 0 && byte < n;
1056             --offset) {
1057                 if (oword.eo_u8[offset - 1] != key[byte++]) {
1058                         rc = EFAULT;
1059                         goto fail5;
1060                 }
1061         }
1062
1063 done:
1064         return (0);
1065
1066 fail5:
1067         EFSYS_PROBE(fail5);
1068 fail4:
1069         EFSYS_PROBE(fail4);
1070 fail3:
1071         EFSYS_PROBE(fail3);
1072 fail2:
1073         EFSYS_PROBE(fail2);
1074 fail1:
1075         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1076
1077         return (rc);
1078 }
1079 #endif
1080
1081 #if EFSYS_OPT_RX_SCALE
1082 static  __checkReturn   efx_rc_t
1083 siena_rx_scale_tbl_set(
1084         __in            efx_nic_t *enp,
1085         __in            uint32_t rss_context,
1086         __in_ecount(n)  unsigned int *table,
1087         __in            size_t n)
1088 {
1089         efx_oword_t oword;
1090         int index;
1091         efx_rc_t rc;
1092
1093         EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1094         EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1095
1096         if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1097                 rc = EINVAL;
1098                 goto fail1;
1099         }
1100
1101         if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1102                 rc = EINVAL;
1103                 goto fail2;
1104         }
1105
1106         for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1107                 uint32_t byte;
1108
1109                 /* Calculate the entry to place in the table */
1110                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1111
1112                 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1113
1114                 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1115
1116                 /* Write the table */
1117                 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1118                                     index, &oword, B_TRUE);
1119         }
1120
1121         for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1122                 uint32_t byte;
1123
1124                 /* Determine if we're starting a new batch */
1125                 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1126
1127                 /* Read the table */
1128                 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1129                                     index, &oword, B_TRUE);
1130
1131                 /* Verify the entry */
1132                 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1133                         rc = EFAULT;
1134                         goto fail3;
1135                 }
1136         }
1137
1138         return (0);
1139
1140 fail3:
1141         EFSYS_PROBE(fail3);
1142 fail2:
1143         EFSYS_PROBE(fail2);
1144 fail1:
1145         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1146
1147         return (rc);
1148 }
1149 #endif
1150
1151 /*
1152  * Falcon/Siena pseudo-header
1153  * --------------------------
1154  *
1155  * Receive packets are prefixed by an optional 16 byte pseudo-header.
1156  * The pseudo-header is a byte array of one of the forms:
1157  *
1158  *  0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15
1159  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1160  * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1161  *
1162  * where:
1163  *   TT.TT.TT.TT   Toeplitz hash (32-bit big-endian)
1164  *   LL.LL         LFSR hash     (16-bit big-endian)
1165  */
1166
1167 #if EFSYS_OPT_RX_SCALE
1168 static  __checkReturn   uint32_t
1169 siena_rx_prefix_hash(
1170         __in            efx_nic_t *enp,
1171         __in            efx_rx_hash_alg_t func,
1172         __in            uint8_t *buffer)
1173 {
1174         _NOTE(ARGUNUSED(enp))
1175
1176         switch (func) {
1177         case EFX_RX_HASHALG_TOEPLITZ:
1178                 return ((buffer[12] << 24) |
1179                     (buffer[13] << 16) |
1180                     (buffer[14] <<  8) |
1181                     buffer[15]);
1182
1183         case EFX_RX_HASHALG_LFSR:
1184                 return ((buffer[14] << 8) | buffer[15]);
1185
1186         default:
1187                 EFSYS_ASSERT(0);
1188                 return (0);
1189         }
1190 }
1191 #endif /* EFSYS_OPT_RX_SCALE */
1192
1193 static  __checkReturn   efx_rc_t
1194 siena_rx_prefix_pktlen(
1195         __in            efx_nic_t *enp,
1196         __in            uint8_t *buffer,
1197         __out           uint16_t *lengthp)
1198 {
1199         _NOTE(ARGUNUSED(enp, buffer, lengthp))
1200
1201         /* Not supported by Falcon/Siena hardware */
1202         EFSYS_ASSERT(0);
1203         return (ENOTSUP);
1204 }
1205
1206
1207 static                          void
1208 siena_rx_qpost(
1209         __in                    efx_rxq_t *erp,
1210         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
1211         __in                    size_t size,
1212         __in                    unsigned int ndescs,
1213         __in                    unsigned int completed,
1214         __in                    unsigned int added)
1215 {
1216         efx_qword_t qword;
1217         unsigned int i;
1218         unsigned int offset;
1219         unsigned int id;
1220
1221         /* The client driver must not overfill the queue */
1222         EFSYS_ASSERT3U(added - completed + ndescs, <=,
1223             EFX_RXQ_LIMIT(erp->er_mask + 1));
1224
1225         id = added & (erp->er_mask);
1226         for (i = 0; i < ndescs; i++) {
1227                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1228                     unsigned int, id, efsys_dma_addr_t, addrp[i],
1229                     size_t, size);
1230
1231                 EFX_POPULATE_QWORD_3(qword,
1232                     FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1233                     FSF_AZ_RX_KER_BUF_ADDR_DW0,
1234                     (uint32_t)(addrp[i] & 0xffffffff),
1235                     FSF_AZ_RX_KER_BUF_ADDR_DW1,
1236                     (uint32_t)(addrp[i] >> 32));
1237
1238                 offset = id * sizeof (efx_qword_t);
1239                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1240
1241                 id = (id + 1) & (erp->er_mask);
1242         }
1243 }
1244
1245 static                  void
1246 siena_rx_qpush(
1247         __in    efx_rxq_t *erp,
1248         __in    unsigned int added,
1249         __inout unsigned int *pushedp)
1250 {
1251         efx_nic_t *enp = erp->er_enp;
1252         unsigned int pushed = *pushedp;
1253         uint32_t wptr;
1254         efx_oword_t oword;
1255         efx_dword_t dword;
1256
1257         /* All descriptors are pushed */
1258         *pushedp = added;
1259
1260         /* Push the populated descriptors out */
1261         wptr = added & erp->er_mask;
1262
1263         EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1264
1265         /* Only write the third DWORD */
1266         EFX_POPULATE_DWORD_1(dword,
1267             EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1268
1269         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1270         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1271             wptr, pushed & erp->er_mask);
1272         EFSYS_PIO_WRITE_BARRIER();
1273         EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1274                             erp->er_index, &dword, B_FALSE);
1275 }
1276
1277 #if EFSYS_OPT_RX_PACKED_STREAM
1278 static          void
1279 siena_rx_qpush_ps_credits(
1280         __in            efx_rxq_t *erp)
1281 {
1282         /* Not supported by Siena hardware */
1283         EFSYS_ASSERT(0);
1284 }
1285
1286 static          uint8_t *
1287 siena_rx_qps_packet_info(
1288         __in            efx_rxq_t *erp,
1289         __in            uint8_t *buffer,
1290         __in            uint32_t buffer_length,
1291         __in            uint32_t current_offset,
1292         __out           uint16_t *lengthp,
1293         __out           uint32_t *next_offsetp,
1294         __out           uint32_t *timestamp)
1295 {
1296         /* Not supported by Siena hardware */
1297         EFSYS_ASSERT(0);
1298
1299         return (NULL);
1300 }
1301 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1302
1303 static  __checkReturn   efx_rc_t
1304 siena_rx_qflush(
1305         __in    efx_rxq_t *erp)
1306 {
1307         efx_nic_t *enp = erp->er_enp;
1308         efx_oword_t oword;
1309         uint32_t label;
1310
1311         label = erp->er_index;
1312
1313         /* Flush the queue */
1314         EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1315             FRF_AZ_RX_FLUSH_DESCQ, label);
1316         EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1317
1318         return (0);
1319 }
1320
1321 static          void
1322 siena_rx_qenable(
1323         __in    efx_rxq_t *erp)
1324 {
1325         efx_nic_t *enp = erp->er_enp;
1326         efx_oword_t oword;
1327
1328         EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1329
1330         EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1331                             erp->er_index, &oword, B_TRUE);
1332
1333         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1334         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1335         EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1336
1337         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1338                             erp->er_index, &oword, B_TRUE);
1339 }
1340
1341 static  __checkReturn   efx_rc_t
1342 siena_rx_qcreate(
1343         __in            efx_nic_t *enp,
1344         __in            unsigned int index,
1345         __in            unsigned int label,
1346         __in            efx_rxq_type_t type,
1347         __in            uint32_t type_data,
1348         __in            efsys_mem_t *esmp,
1349         __in            size_t ndescs,
1350         __in            uint32_t id,
1351         __in            unsigned int flags,
1352         __in            efx_evq_t *eep,
1353         __in            efx_rxq_t *erp)
1354 {
1355         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1356         efx_oword_t oword;
1357         uint32_t size;
1358         boolean_t jumbo = B_FALSE;
1359         efx_rc_t rc;
1360
1361         _NOTE(ARGUNUSED(esmp))
1362         _NOTE(ARGUNUSED(type_data))
1363
1364         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1365             (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1366         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1367         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1368
1369         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1370         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1371
1372         if (!ISP2(ndescs) ||
1373             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1374                 rc = EINVAL;
1375                 goto fail1;
1376         }
1377         if (index >= encp->enc_rxq_limit) {
1378                 rc = EINVAL;
1379                 goto fail2;
1380         }
1381         for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1382             size++)
1383                 if ((1 << size) == (int)(ndescs / EFX_RXQ_MINNDESCS))
1384                         break;
1385         if (id + (1 << size) >= encp->enc_buftbl_limit) {
1386                 rc = EINVAL;
1387                 goto fail3;
1388         }
1389
1390         switch (type) {
1391         case EFX_RXQ_TYPE_DEFAULT:
1392                 break;
1393
1394         default:
1395                 rc = EINVAL;
1396                 goto fail4;
1397         }
1398
1399         if (flags & EFX_RXQ_FLAG_SCATTER) {
1400 #if EFSYS_OPT_RX_SCATTER
1401                 jumbo = B_TRUE;
1402 #else
1403                 rc = EINVAL;
1404                 goto fail5;
1405 #endif  /* EFSYS_OPT_RX_SCATTER */
1406         }
1407
1408         /* Set up the new descriptor queue */
1409         EFX_POPULATE_OWORD_7(oword,
1410             FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1411             FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1412             FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1413             FRF_AZ_RX_DESCQ_LABEL, label,
1414             FRF_AZ_RX_DESCQ_SIZE, size,
1415             FRF_AZ_RX_DESCQ_TYPE, 0,
1416             FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1417
1418         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1419                             erp->er_index, &oword, B_TRUE);
1420
1421         return (0);
1422
1423 #if !EFSYS_OPT_RX_SCATTER
1424 fail5:
1425         EFSYS_PROBE(fail5);
1426 #endif
1427 fail4:
1428         EFSYS_PROBE(fail4);
1429 fail3:
1430         EFSYS_PROBE(fail3);
1431 fail2:
1432         EFSYS_PROBE(fail2);
1433 fail1:
1434         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1435
1436         return (rc);
1437 }
1438
1439 static          void
1440 siena_rx_qdestroy(
1441         __in    efx_rxq_t *erp)
1442 {
1443         efx_nic_t *enp = erp->er_enp;
1444         efx_oword_t oword;
1445
1446         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1447         --enp->en_rx_qcount;
1448
1449         /* Purge descriptor queue */
1450         EFX_ZERO_OWORD(oword);
1451
1452         EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1453                             erp->er_index, &oword, B_TRUE);
1454
1455         /* Free the RXQ object */
1456         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1457 }
1458
1459 static          void
1460 siena_rx_fini(
1461         __in    efx_nic_t *enp)
1462 {
1463         _NOTE(ARGUNUSED(enp))
1464 }
1465
1466 #endif /* EFSYS_OPT_SIENA */