drivers/net/sfc/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2012-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  12
  13
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_init_rxq(
  16         __in            efx_nic_t *enp,
  17         __in            uint32_t ndescs,
  18         __in            uint32_t target_evq,
  19         __in            uint32_t label,
  20         __in            uint32_t instance,
  21         __in            efsys_mem_t *esmp,
  22         __in            boolean_t disable_scatter,
  23         __in            boolean_t want_inner_classes,
  24         __in            uint32_t ps_bufsize,
  25         __in            uint32_t es_bufs_per_desc,
  26         __in            uint32_t es_max_dma_len,
  27         __in            uint32_t es_buf_stride,
  28         __in            uint32_t hol_block_timeout)
  29 {
  30         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  31         efx_mcdi_req_t req;
  32         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_V3_IN_LEN,
  33                 MC_CMD_INIT_RXQ_V3_OUT_LEN);
  34         int npages = EFX_RXQ_NBUFS(ndescs);
  35         int i;
  36         efx_qword_t *dma_addr;
  37         uint64_t addr;
  38         efx_rc_t rc;
  39         uint32_t dma_mode;
  40         boolean_t want_outer_classes;
  41
  42         EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
  43
  44         if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_RXQ_SIZE(ndescs))) {
  45                 rc = EINVAL;
  46                 goto fail1;
  47         }
  48
  49         if (ps_bufsize > 0)
  50                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  51         else if (es_bufs_per_desc > 0)
  52                 dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER;
  53         else
  54                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  55
  56         if (encp->enc_tunnel_encapsulations_supported != 0 &&
  57             !want_inner_classes) {
  58                 /*
  59                  * WANT_OUTER_CLASSES can only be specified on hardware which
  60                  * supports tunnel encapsulation offloads, even though it is
  61                  * effectively the behaviour the hardware gives.
  62                  *
  63                  * Also, on hardware which does support such offloads, older
  64                  * firmware rejects the flag if the offloads are not supported
  65                  * by the current firmware variant, which means this may fail if
  66                  * the capabilities are not updated when the firmware variant
  67                  * changes. This is not an issue on newer firmware, as it was
  68                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
  69                  * specified on all firmware variants.
  70                  */
  71                 want_outer_classes = B_TRUE;
  72         } else {
  73                 want_outer_classes = B_FALSE;
  74         }
  75
  76         req.emr_cmd = MC_CMD_INIT_RXQ;
  77         req.emr_in_buf = payload;
  78         req.emr_in_length = MC_CMD_INIT_RXQ_V3_IN_LEN;
  79         req.emr_out_buf = payload;
  80         req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
  81
  82         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
  83         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  84         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  85         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  86         MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
  87             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  88             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  89             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  90             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  91             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  92             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  93             INIT_RXQ_EXT_IN_DMA_MODE,
  94             dma_mode,
  95             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
  96             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
  97         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  98         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  99
 100         if (es_bufs_per_desc > 0) {
 101                 MCDI_IN_SET_DWORD(req,
 102                     INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET,
 103                     es_bufs_per_desc);
 104                 MCDI_IN_SET_DWORD(req,
 105                     INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len);
 106                 MCDI_IN_SET_DWORD(req,
 107                     INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride);
 108                 MCDI_IN_SET_DWORD(req,
 109                     INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT,
 110                     hol_block_timeout);
 111         }
 112
 113         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
 114         addr = EFSYS_MEM_ADDR(esmp);
 115
 116         for (i = 0; i < npages; i++) {
 117                 EFX_POPULATE_QWORD_2(*dma_addr,
 118                     EFX_DWORD_1, (uint32_t)(addr >> 32),
 119                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
 120
 121                 dma_addr++;
 122                 addr += EFX_BUF_SIZE;
 123         }
 124
 125         efx_mcdi_execute(enp, &req);
 126
 127         if (req.emr_rc != 0) {
 128                 rc = req.emr_rc;
 129                 goto fail2;
 130         }
 131
 132         return (0);
 133
 134 fail2:
 135         EFSYS_PROBE(fail2);
 136 fail1:
 137         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 138
 139         return (rc);
 140 }
 141
 142 static  __checkReturn   efx_rc_t
 143 efx_mcdi_fini_rxq(
 144         __in            efx_nic_t *enp,
 145         __in            uint32_t instance)
 146 {
 147         efx_mcdi_req_t req;
 148         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
 149                 MC_CMD_FINI_RXQ_OUT_LEN);
 150         efx_rc_t rc;
 151
 152         req.emr_cmd = MC_CMD_FINI_RXQ;
 153         req.emr_in_buf = payload;
 154         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 155         req.emr_out_buf = payload;
 156         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 157
 158         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 159
 160         efx_mcdi_execute_quiet(enp, &req);
 161
 162         if (req.emr_rc != 0) {
 163                 rc = req.emr_rc;
 164                 goto fail1;
 165         }
 166
 167         return (0);
 168
 169 fail1:
 170         /*
 171          * EALREADY is not an error, but indicates that the MC has rebooted and
 172          * that the RXQ has already been destroyed.
 173          */
 174         if (rc != EALREADY)
 175                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 176
 177         return (rc);
 178 }
 179
 180 #if EFSYS_OPT_RX_SCALE
 181 static  __checkReturn   efx_rc_t
 182 efx_mcdi_rss_context_alloc(
 183         __in            efx_nic_t *enp,
 184         __in            efx_rx_scale_context_type_t type,
 185         __in            uint32_t num_queues,
 186         __out           uint32_t *rss_contextp)
 187 {
 188         efx_mcdi_req_t req;
 189         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 190                 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
 191         uint32_t rss_context;
 192         uint32_t context_type;
 193         efx_rc_t rc;
 194
 195         if (num_queues > EFX_MAXRSS) {
 196                 rc = EINVAL;
 197                 goto fail1;
 198         }
 199
 200         switch (type) {
 201         case EFX_RX_SCALE_EXCLUSIVE:
 202                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 203                 break;
 204         case EFX_RX_SCALE_SHARED:
 205                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 206                 break;
 207         default:
 208                 rc = EINVAL;
 209                 goto fail2;
 210         }
 211
 212         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 213         req.emr_in_buf = payload;
 214         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 215         req.emr_out_buf = payload;
 216         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 217
 218         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 219             EVB_PORT_ID_ASSIGNED);
 220         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 221
 222         /*
 223          * For exclusive contexts, NUM_QUEUES is only used to validate
 224          * indirection table offsets.
 225          * For shared contexts, the provided context will spread traffic over
 226          * NUM_QUEUES many queues.
 227          */
 228         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 229
 230         efx_mcdi_execute(enp, &req);
 231
 232         if (req.emr_rc != 0) {
 233                 rc = req.emr_rc;
 234                 goto fail3;
 235         }
 236
 237         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 238                 rc = EMSGSIZE;
 239                 goto fail4;
 240         }
 241
 242         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 243         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 244                 rc = ENOENT;
 245                 goto fail5;
 246         }
 247
 248         *rss_contextp = rss_context;
 249
 250         return (0);
 251
 252 fail5:
 253         EFSYS_PROBE(fail5);
 254 fail4:
 255         EFSYS_PROBE(fail4);
 256 fail3:
 257         EFSYS_PROBE(fail3);
 258 fail2:
 259         EFSYS_PROBE(fail2);
 260 fail1:
 261         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 262
 263         return (rc);
 264 }
 265 #endif /* EFSYS_OPT_RX_SCALE */
 266
 267 #if EFSYS_OPT_RX_SCALE
 268 static                  efx_rc_t
 269 efx_mcdi_rss_context_free(
 270         __in            efx_nic_t *enp,
 271         __in            uint32_t rss_context)
 272 {
 273         efx_mcdi_req_t req;
 274         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 275                 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 276         efx_rc_t rc;
 277
 278         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 279                 rc = EINVAL;
 280                 goto fail1;
 281         }
 282
 283         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 284         req.emr_in_buf = payload;
 285         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 286         req.emr_out_buf = payload;
 287         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 288
 289         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 290
 291         efx_mcdi_execute_quiet(enp, &req);
 292
 293         if (req.emr_rc != 0) {
 294                 rc = req.emr_rc;
 295                 goto fail2;
 296         }
 297
 298         return (0);
 299
 300 fail2:
 301         EFSYS_PROBE(fail2);
 302 fail1:
 303         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 304
 305         return (rc);
 306 }
 307 #endif /* EFSYS_OPT_RX_SCALE */
 308
 309 #if EFSYS_OPT_RX_SCALE
 310 static                  efx_rc_t
 311 efx_mcdi_rss_context_set_flags(
 312         __in            efx_nic_t *enp,
 313         __in            uint32_t rss_context,
 314         __in            efx_rx_hash_type_t type)
 315 {
 316         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 317         efx_rx_hash_type_t type_ipv4;
 318         efx_rx_hash_type_t type_ipv4_tcp;
 319         efx_rx_hash_type_t type_ipv6;
 320         efx_rx_hash_type_t type_ipv6_tcp;
 321         efx_rx_hash_type_t modes;
 322         efx_mcdi_req_t req;
 323         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 324                 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 325         efx_rc_t rc;
 326
 327         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
 328                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
 329         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
 330                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
 331         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
 332                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
 333         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
 334                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
 335         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
 336                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
 337         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
 338                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
 339         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
 340                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
 341         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
 342                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
 343
 344         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 345                 rc = EINVAL;
 346                 goto fail1;
 347         }
 348
 349         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 350         req.emr_in_buf = payload;
 351         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 352         req.emr_out_buf = payload;
 353         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 354
 355         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 356             rss_context);
 357
 358         type_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | EFX_RX_HASH(IPV4_TCP, 2TUPLE) |
 359                     EFX_RX_HASH(IPV4_UDP, 2TUPLE);
 360         type_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
 361         type_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | EFX_RX_HASH(IPV6_TCP, 2TUPLE) |
 362                     EFX_RX_HASH(IPV6_UDP, 2TUPLE);
 363         type_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
 364
 365         /*
 366          * Create a copy of the original hash type.
 367          * The copy will be used to fill in RSS_MODE bits and
 368          * may be cleared beforehand. The original variable
 369          * and, thus, EN bits will remain unaffected.
 370          */
 371         modes = type;
 372
 373         /*
 374          * If the firmware lacks support for additional modes, RSS_MODE
 375          * fields must contain zeros, otherwise the operation will fail.
 376          */
 377         if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
 378                 modes = 0;
 379
 380 #define EXTRACT_RSS_MODE(_type, _class)         \
 381         (EFX_EXTRACT_NATIVE(_type, 0, 31,       \
 382         EFX_LOW_BIT(EFX_RX_CLASS_##_class),     \
 383         EFX_HIGH_BIT(EFX_RX_CLASS_##_class)) &  \
 384         EFX_MASK32(EFX_RX_CLASS_##_class))
 385
 386         MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 387             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 388             ((type & type_ipv4) == type_ipv4) ? 1 : 0,
 389             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 390             ((type & type_ipv4_tcp) == type_ipv4_tcp) ? 1 : 0,
 391             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 392             ((type & type_ipv6) == type_ipv6) ? 1 : 0,
 393             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 394             ((type & type_ipv6_tcp) == type_ipv6_tcp) ? 1 : 0,
 395             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
 396             EXTRACT_RSS_MODE(modes, IPV4_TCP),
 397             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
 398             EXTRACT_RSS_MODE(modes, IPV4_UDP),
 399             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
 400             EXTRACT_RSS_MODE(modes, IPV4),
 401             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
 402             EXTRACT_RSS_MODE(modes, IPV6_TCP),
 403             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
 404             EXTRACT_RSS_MODE(modes, IPV6_UDP),
 405             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
 406             EXTRACT_RSS_MODE(modes, IPV6));
 407
 408 #undef EXTRACT_RSS_MODE
 409
 410         efx_mcdi_execute(enp, &req);
 411
 412         if (req.emr_rc != 0) {
 413                 rc = req.emr_rc;
 414                 goto fail2;
 415         }
 416
 417         return (0);
 418
 419 fail2:
 420         EFSYS_PROBE(fail2);
 421 fail1:
 422         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 423
 424         return (rc);
 425 }
 426 #endif /* EFSYS_OPT_RX_SCALE */
 427
 428 #if EFSYS_OPT_RX_SCALE
 429 static                  efx_rc_t
 430 efx_mcdi_rss_context_set_key(
 431         __in            efx_nic_t *enp,
 432         __in            uint32_t rss_context,
 433         __in_ecount(n)  uint8_t *key,
 434         __in            size_t n)
 435 {
 436         efx_mcdi_req_t req;
 437         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 438                 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 439         efx_rc_t rc;
 440
 441         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 442                 rc = EINVAL;
 443                 goto fail1;
 444         }
 445
 446         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 447         req.emr_in_buf = payload;
 448         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 449         req.emr_out_buf = payload;
 450         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 451
 452         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 453             rss_context);
 454
 455         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 456         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 457                 rc = EINVAL;
 458                 goto fail2;
 459         }
 460
 461         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 462             key, n);
 463
 464         efx_mcdi_execute(enp, &req);
 465
 466         if (req.emr_rc != 0) {
 467                 rc = req.emr_rc;
 468                 goto fail3;
 469         }
 470
 471         return (0);
 472
 473 fail3:
 474         EFSYS_PROBE(fail3);
 475 fail2:
 476         EFSYS_PROBE(fail2);
 477 fail1:
 478         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 479
 480         return (rc);
 481 }
 482 #endif /* EFSYS_OPT_RX_SCALE */
 483
 484 #if EFSYS_OPT_RX_SCALE
 485 static                  efx_rc_t
 486 efx_mcdi_rss_context_set_table(
 487         __in            efx_nic_t *enp,
 488         __in            uint32_t rss_context,
 489         __in_ecount(n)  unsigned int *table,
 490         __in            size_t n)
 491 {
 492         efx_mcdi_req_t req;
 493         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 494                 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 495         uint8_t *req_table;
 496         int i, rc;
 497
 498         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 499                 rc = EINVAL;
 500                 goto fail1;
 501         }
 502
 503         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 504         req.emr_in_buf = payload;
 505         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 506         req.emr_out_buf = payload;
 507         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 508
 509         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 510             rss_context);
 511
 512         req_table =
 513             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 514
 515         for (i = 0;
 516             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 517             i++) {
 518                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 519         }
 520
 521         efx_mcdi_execute(enp, &req);
 522
 523         if (req.emr_rc != 0) {
 524                 rc = req.emr_rc;
 525                 goto fail2;
 526         }
 527
 528         return (0);
 529
 530 fail2:
 531         EFSYS_PROBE(fail2);
 532 fail1:
 533         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 534
 535         return (rc);
 536 }
 537 #endif /* EFSYS_OPT_RX_SCALE */
 538
 539
 540         __checkReturn   efx_rc_t
 541 ef10_rx_init(
 542         __in            efx_nic_t *enp)
 543 {
 544 #if EFSYS_OPT_RX_SCALE
 545
 546         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 547                 &enp->en_rss_context) == 0) {
 548                 /*
 549                  * Allocated an exclusive RSS context, which allows both the
 550                  * indirection table and key to be modified.
 551                  */
 552                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 553                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 554         } else {
 555                 /*
 556                  * Failed to allocate an exclusive RSS context. Continue
 557                  * operation without support for RSS. The pseudo-header in
 558                  * received packets will not contain a Toeplitz hash value.
 559                  */
 560                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 561                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 562         }
 563
 564 #endif /* EFSYS_OPT_RX_SCALE */
 565
 566         return (0);
 567 }
 568
 569 #if EFSYS_OPT_RX_SCATTER
 570         __checkReturn   efx_rc_t
 571 ef10_rx_scatter_enable(
 572         __in            efx_nic_t *enp,
 573         __in            unsigned int buf_size)
 574 {
 575         _NOTE(ARGUNUSED(enp, buf_size))
 576         return (0);
 577 }
 578 #endif  /* EFSYS_OPT_RX_SCATTER */
 579
 580 #if EFSYS_OPT_RX_SCALE
 581         __checkReturn   efx_rc_t
 582 ef10_rx_scale_context_alloc(
 583         __in            efx_nic_t *enp,
 584         __in            efx_rx_scale_context_type_t type,
 585         __in            uint32_t num_queues,
 586         __out           uint32_t *rss_contextp)
 587 {
 588         efx_rc_t rc;
 589
 590         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 591         if (rc != 0)
 592                 goto fail1;
 593
 594         return (0);
 595
 596 fail1:
 597         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 598         return (rc);
 599 }
 600 #endif /* EFSYS_OPT_RX_SCALE */
 601
 602 #if EFSYS_OPT_RX_SCALE
 603         __checkReturn   efx_rc_t
 604 ef10_rx_scale_context_free(
 605         __in            efx_nic_t *enp,
 606         __in            uint32_t rss_context)
 607 {
 608         efx_rc_t rc;
 609
 610         rc = efx_mcdi_rss_context_free(enp, rss_context);
 611         if (rc != 0)
 612                 goto fail1;
 613
 614         return (0);
 615
 616 fail1:
 617         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 618         return (rc);
 619 }
 620 #endif /* EFSYS_OPT_RX_SCALE */
 621
 622 #if EFSYS_OPT_RX_SCALE
 623         __checkReturn   efx_rc_t
 624 ef10_rx_scale_mode_set(
 625         __in            efx_nic_t *enp,
 626         __in            uint32_t rss_context,
 627         __in            efx_rx_hash_alg_t alg,
 628         __in            efx_rx_hash_type_t type,
 629         __in            boolean_t insert)
 630 {
 631         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 632         efx_rc_t rc;
 633
 634         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 635
 636         if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
 637             insert == B_FALSE) {
 638                 rc = EINVAL;
 639                 goto fail1;
 640         }
 641
 642         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 643                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 644                         rc = ENOTSUP;
 645                         goto fail2;
 646                 }
 647                 rss_context = enp->en_rss_context;
 648         }
 649
 650         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 651                     rss_context, type)) != 0)
 652                 goto fail3;
 653
 654         return (0);
 655
 656 fail3:
 657         EFSYS_PROBE(fail3);
 658 fail2:
 659         EFSYS_PROBE(fail2);
 660 fail1:
 661         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 662
 663         return (rc);
 664 }
 665 #endif /* EFSYS_OPT_RX_SCALE */
 666
 667 #if EFSYS_OPT_RX_SCALE
 668         __checkReturn   efx_rc_t
 669 ef10_rx_scale_key_set(
 670         __in            efx_nic_t *enp,
 671         __in            uint32_t rss_context,
 672         __in_ecount(n)  uint8_t *key,
 673         __in            size_t n)
 674 {
 675         efx_rc_t rc;
 676
 677         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 678             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 679
 680         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 681                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 682                         rc = ENOTSUP;
 683                         goto fail1;
 684                 }
 685                 rss_context = enp->en_rss_context;
 686         }
 687
 688         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 689                 goto fail2;
 690
 691         return (0);
 692
 693 fail2:
 694         EFSYS_PROBE(fail2);
 695 fail1:
 696         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 697
 698         return (rc);
 699 }
 700 #endif /* EFSYS_OPT_RX_SCALE */
 701
 702 #if EFSYS_OPT_RX_SCALE
 703         __checkReturn   efx_rc_t
 704 ef10_rx_scale_tbl_set(
 705         __in            efx_nic_t *enp,
 706         __in            uint32_t rss_context,
 707         __in_ecount(n)  unsigned int *table,
 708         __in            size_t n)
 709 {
 710         efx_rc_t rc;
 711
 712
 713         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 714                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 715                         rc = ENOTSUP;
 716                         goto fail1;
 717                 }
 718                 rss_context = enp->en_rss_context;
 719         }
 720
 721         if ((rc = efx_mcdi_rss_context_set_table(enp,
 722                     rss_context, table, n)) != 0)
 723                 goto fail2;
 724
 725         return (0);
 726
 727 fail2:
 728         EFSYS_PROBE(fail2);
 729 fail1:
 730         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 731
 732         return (rc);
 733 }
 734 #endif /* EFSYS_OPT_RX_SCALE */
 735
 736
 737 /*
 738  * EF10 RX pseudo-header
 739  * ---------------------
 740  *
 741  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 742  *
 743  *  +00: Toeplitz hash value.
 744  *       (32bit little-endian)
 745  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 746  *       (16bit big-endian)
 747  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 748  *       (16bit big-endian)
 749  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 750  *       (16bit little-endian)
 751  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 752  *       (32bit little-endian)
 753  *
 754  * See "The RX Pseudo-header" in SF-109306-TC.
 755  */
 756
 757         __checkReturn   efx_rc_t
 758 ef10_rx_prefix_pktlen(
 759         __in            efx_nic_t *enp,
 760         __in            uint8_t *buffer,
 761         __out           uint16_t *lengthp)
 762 {
 763         _NOTE(ARGUNUSED(enp))
 764
 765         /*
 766          * The RX pseudo-header contains the packet length, excluding the
 767          * pseudo-header. If the hardware receive datapath was operating in
 768          * cut-through mode then the length in the RX pseudo-header will be
 769          * zero, and the packet length must be obtained from the DMA length
 770          * reported in the RX event.
 771          */
 772         *lengthp = buffer[8] | (buffer[9] << 8);
 773         return (0);
 774 }
 775
 776 #if EFSYS_OPT_RX_SCALE
 777         __checkReturn   uint32_t
 778 ef10_rx_prefix_hash(
 779         __in            efx_nic_t *enp,
 780         __in            efx_rx_hash_alg_t func,
 781         __in            uint8_t *buffer)
 782 {
 783         _NOTE(ARGUNUSED(enp))
 784
 785         switch (func) {
 786         case EFX_RX_HASHALG_PACKED_STREAM:
 787         case EFX_RX_HASHALG_TOEPLITZ:
 788                 return (buffer[0] |
 789                     (buffer[1] << 8) |
 790                     (buffer[2] << 16) |
 791                     (buffer[3] << 24));
 792
 793         default:
 794                 EFSYS_ASSERT(0);
 795                 return (0);
 796         }
 797 }
 798 #endif /* EFSYS_OPT_RX_SCALE */
 799
 800 #if EFSYS_OPT_RX_PACKED_STREAM
 801 /*
 802  * Fake length for RXQ descriptors in packed stream mode
 803  * to make hardware happy
 804  */
 805 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 806 #endif
 807
 808                                 void
 809 ef10_rx_qpost(
 810         __in                    efx_rxq_t *erp,
 811         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 812         __in                    size_t size,
 813         __in                    unsigned int ndescs,
 814         __in                    unsigned int completed,
 815         __in                    unsigned int added)
 816 {
 817         efx_qword_t qword;
 818         unsigned int i;
 819         unsigned int offset;
 820         unsigned int id;
 821
 822         _NOTE(ARGUNUSED(completed))
 823
 824 #if EFSYS_OPT_RX_PACKED_STREAM
 825         /*
 826          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 827          * and equal to 0 after applying mask. Hardware does not like it.
 828          */
 829         if (erp->er_ev_qstate->eers_rx_packed_stream)
 830                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 831 #endif
 832
 833         /* The client driver must not overfill the queue */
 834         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 835             EFX_RXQ_LIMIT(erp->er_mask + 1));
 836
 837         id = added & (erp->er_mask);
 838         for (i = 0; i < ndescs; i++) {
 839                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 840                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 841                     size_t, size);
 842
 843                 EFX_POPULATE_QWORD_3(qword,
 844                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 845                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 846                     (uint32_t)(addrp[i] & 0xffffffff),
 847                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 848                     (uint32_t)(addrp[i] >> 32));
 849
 850                 offset = id * sizeof (efx_qword_t);
 851                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 852
 853                 id = (id + 1) & (erp->er_mask);
 854         }
 855 }
 856
 857                         void
 858 ef10_rx_qpush(
 859         __in    efx_rxq_t *erp,
 860         __in    unsigned int added,
 861         __inout unsigned int *pushedp)
 862 {
 863         efx_nic_t *enp = erp->er_enp;
 864         unsigned int pushed = *pushedp;
 865         uint32_t wptr;
 866         efx_dword_t dword;
 867
 868         /* Hardware has alignment restriction for WPTR */
 869         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 870         if (pushed == wptr)
 871                 return;
 872
 873         *pushedp = wptr;
 874
 875         /* Push the populated descriptors out */
 876         wptr &= erp->er_mask;
 877
 878         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 879
 880         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 881         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 882             wptr, pushed & erp->er_mask);
 883         EFSYS_PIO_WRITE_BARRIER();
 884         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 885             erp->er_index, &dword, B_FALSE);
 886 }
 887
 888 #if EFSYS_OPT_RX_PACKED_STREAM
 889
 890                         void
 891 ef10_rx_qpush_ps_credits(
 892         __in            efx_rxq_t *erp)
 893 {
 894         efx_nic_t *enp = erp->er_enp;
 895         efx_dword_t dword;
 896         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 897         uint32_t credits;
 898
 899         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 900
 901         if (rxq_state->eers_rx_packed_stream_credits == 0)
 902                 return;
 903
 904         /*
 905          * It is a bug if we think that FW has utilized more
 906          * credits than it is allowed to have (maximum). However,
 907          * make sure that we do not credit more than maximum anyway.
 908          */
 909         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 910             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 911         EFX_POPULATE_DWORD_3(dword,
 912             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 913             ERF_DZ_RX_DESC_MAGIC_CMD,
 914             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 915             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 916         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 917             erp->er_index, &dword, B_FALSE);
 918
 919         rxq_state->eers_rx_packed_stream_credits = 0;
 920 }
 921
 922 /*
 923  * In accordance with SF-112241-TC the received data has the following layout:
 924  *  - 8 byte pseudo-header which consist of:
 925  *    - 4 byte little-endian timestamp
 926  *    - 2 byte little-endian captured length in bytes
 927  *    - 2 byte little-endian original packet length in bytes
 928  *  - captured packet bytes
 929  *  - optional padding to align to 64 bytes boundary
 930  *  - 64 bytes scratch space for the host software
 931  */
 932         __checkReturn   uint8_t *
 933 ef10_rx_qps_packet_info(
 934         __in            efx_rxq_t *erp,
 935         __in            uint8_t *buffer,
 936         __in            uint32_t buffer_length,
 937         __in            uint32_t current_offset,
 938         __out           uint16_t *lengthp,
 939         __out           uint32_t *next_offsetp,
 940         __out           uint32_t *timestamp)
 941 {
 942         uint16_t buf_len;
 943         uint8_t *pkt_start;
 944         efx_qword_t *qwordp;
 945         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 946
 947         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 948
 949         buffer += current_offset;
 950         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 951
 952         qwordp = (efx_qword_t *)buffer;
 953         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 954         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 955         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 956
 957         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 958                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 959         *next_offsetp =
 960             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 961
 962         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 963         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 964
 965         if ((*next_offsetp ^ current_offset) &
 966             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 967                 rxq_state->eers_rx_packed_stream_credits++;
 968
 969         return (pkt_start);
 970 }
 971
 972
 973 #endif
 974
 975         __checkReturn   efx_rc_t
 976 ef10_rx_qflush(
 977         __in    efx_rxq_t *erp)
 978 {
 979         efx_nic_t *enp = erp->er_enp;
 980         efx_rc_t rc;
 981
 982         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 983                 goto fail1;
 984
 985         return (0);
 986
 987 fail1:
 988         /*
 989          * EALREADY is not an error, but indicates that the MC has rebooted and
 990          * that the RXQ has already been destroyed. Callers need to know that
 991          * the RXQ flush has completed to avoid waiting until timeout for a
 992          * flush done event that will not be delivered.
 993          */
 994         if (rc != EALREADY)
 995                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 996
 997         return (rc);
 998 }
 999
1000                 void
1001 ef10_rx_qenable(
1002         __in    efx_rxq_t *erp)
1003 {
1004         /* FIXME */
1005         _NOTE(ARGUNUSED(erp))
1006         /* FIXME */
1007 }
1008
1009         __checkReturn   efx_rc_t
1010 ef10_rx_qcreate(
1011         __in            efx_nic_t *enp,
1012         __in            unsigned int index,
1013         __in            unsigned int label,
1014         __in            efx_rxq_type_t type,
1015         __in            const efx_rxq_type_data_t *type_data,
1016         __in            efsys_mem_t *esmp,
1017         __in            size_t ndescs,
1018         __in            uint32_t id,
1019         __in            unsigned int flags,
1020         __in            efx_evq_t *eep,
1021         __in            efx_rxq_t *erp)
1022 {
1023         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1024         efx_rc_t rc;
1025         boolean_t disable_scatter;
1026         boolean_t want_inner_classes;
1027         unsigned int ps_buf_size;
1028         uint32_t es_bufs_per_desc = 0;
1029         uint32_t es_max_dma_len = 0;
1030         uint32_t es_buf_stride = 0;
1031         uint32_t hol_block_timeout = 0;
1032
1033         _NOTE(ARGUNUSED(id, erp, type_data))
1034
1035         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
1036         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1037         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1038
1039         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1040         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1041
1042         if (!ISP2(ndescs) ||
1043             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1044                 rc = EINVAL;
1045                 goto fail1;
1046         }
1047         if (index >= encp->enc_rxq_limit) {
1048                 rc = EINVAL;
1049                 goto fail2;
1050         }
1051
1052         switch (type) {
1053         case EFX_RXQ_TYPE_DEFAULT:
1054                 ps_buf_size = 0;
1055                 break;
1056 #if EFSYS_OPT_RX_PACKED_STREAM
1057         case EFX_RXQ_TYPE_PACKED_STREAM:
1058                 switch (type_data->ertd_packed_stream.eps_buf_size) {
1059                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
1060                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
1061                         break;
1062                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
1063                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
1064                         break;
1065                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1066                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1067                         break;
1068                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1069                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1070                         break;
1071                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1072                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1073                         break;
1074                 default:
1075                         rc = ENOTSUP;
1076                         goto fail3;
1077                 }
1078                 break;
1079 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1080 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1081         case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
1082                 ps_buf_size = 0;
1083                 es_bufs_per_desc =
1084                     type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
1085                 es_max_dma_len =
1086                     type_data->ertd_es_super_buffer.eessb_max_dma_len;
1087                 es_buf_stride =
1088                     type_data->ertd_es_super_buffer.eessb_buf_stride;
1089                 hol_block_timeout =
1090                     type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
1091                 break;
1092 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1093         default:
1094                 rc = ENOTSUP;
1095                 goto fail4;
1096         }
1097
1098 #if EFSYS_OPT_RX_PACKED_STREAM
1099         if (ps_buf_size != 0) {
1100                 /* Check if datapath firmware supports packed stream mode */
1101                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1102                         rc = ENOTSUP;
1103                         goto fail5;
1104                 }
1105                 /* Check if packed stream allows configurable buffer sizes */
1106                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1107                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1108                         rc = ENOTSUP;
1109                         goto fail6;
1110                 }
1111         }
1112 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1113         EFSYS_ASSERT(ps_buf_size == 0);
1114 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1115
1116 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1117         if (es_bufs_per_desc > 0) {
1118                 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
1119                         rc = ENOTSUP;
1120                         goto fail7;
1121                 }
1122                 if (!IS_P2ALIGNED(es_max_dma_len,
1123                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1124                         rc = EINVAL;
1125                         goto fail8;
1126                 }
1127                 if (!IS_P2ALIGNED(es_buf_stride,
1128                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1129                         rc = EINVAL;
1130                         goto fail9;
1131                 }
1132         }
1133 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1134         EFSYS_ASSERT(es_bufs_per_desc == 0);
1135 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1136
1137         /* Scatter can only be disabled if the firmware supports doing so */
1138         if (flags & EFX_RXQ_FLAG_SCATTER)
1139                 disable_scatter = B_FALSE;
1140         else
1141                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1142
1143         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1144                 want_inner_classes = B_TRUE;
1145         else
1146                 want_inner_classes = B_FALSE;
1147
1148         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1149                     esmp, disable_scatter, want_inner_classes,
1150                     ps_buf_size, es_bufs_per_desc, es_max_dma_len,
1151                     es_buf_stride, hol_block_timeout)) != 0)
1152                 goto fail10;
1153
1154         erp->er_eep = eep;
1155         erp->er_label = label;
1156
1157         ef10_ev_rxlabel_init(eep, erp, label, type);
1158
1159         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1160
1161         return (0);
1162
1163 fail10:
1164         EFSYS_PROBE(fail10);
1165 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1166 fail9:
1167         EFSYS_PROBE(fail9);
1168 fail8:
1169         EFSYS_PROBE(fail8);
1170 fail7:
1171         EFSYS_PROBE(fail7);
1172 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1173 #if EFSYS_OPT_RX_PACKED_STREAM
1174 fail6:
1175         EFSYS_PROBE(fail6);
1176 fail5:
1177         EFSYS_PROBE(fail5);
1178 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1179 fail4:
1180         EFSYS_PROBE(fail4);
1181 #if EFSYS_OPT_RX_PACKED_STREAM
1182 fail3:
1183         EFSYS_PROBE(fail3);
1184 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1185 fail2:
1186         EFSYS_PROBE(fail2);
1187 fail1:
1188         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1189
1190         return (rc);
1191 }
1192
1193                 void
1194 ef10_rx_qdestroy(
1195         __in    efx_rxq_t *erp)
1196 {
1197         efx_nic_t *enp = erp->er_enp;
1198         efx_evq_t *eep = erp->er_eep;
1199         unsigned int label = erp->er_label;
1200
1201         ef10_ev_rxlabel_fini(eep, label);
1202
1203         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1204         --enp->en_rx_qcount;
1205
1206         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1207 }
1208
1209                 void
1210 ef10_rx_fini(
1211         __in    efx_nic_t *enp)
1212 {
1213 #if EFSYS_OPT_RX_SCALE
1214         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1215                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1216         enp->en_rss_context = 0;
1217         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1218 #else
1219         _NOTE(ARGUNUSED(enp))
1220 #endif /* EFSYS_OPT_RX_SCALE */
1221 }
1222
1223 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */