drivers/net/sfc/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2012-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  12
  13
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_init_rxq(
  16         __in            efx_nic_t *enp,
  17         __in            uint32_t ndescs,
  18         __in            uint32_t target_evq,
  19         __in            uint32_t label,
  20         __in            uint32_t instance,
  21         __in            efsys_mem_t *esmp,
  22         __in            boolean_t disable_scatter,
  23         __in            boolean_t want_inner_classes,
  24         __in            uint32_t ps_bufsize,
  25         __in            uint32_t es_bufs_per_desc,
  26         __in            uint32_t es_max_dma_len,
  27         __in            uint32_t es_buf_stride,
  28         __in            uint32_t hol_block_timeout)
  29 {
  30         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  31         efx_mcdi_req_t req;
  32         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_V3_IN_LEN,
  33                 MC_CMD_INIT_RXQ_V3_OUT_LEN);
  34         int npages = EFX_RXQ_NBUFS(ndescs);
  35         int i;
  36         efx_qword_t *dma_addr;
  37         uint64_t addr;
  38         efx_rc_t rc;
  39         uint32_t dma_mode;
  40         boolean_t want_outer_classes;
  41
  42         EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
  43
  44         if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_RXQ_SIZE(ndescs))) {
  45                 rc = EINVAL;
  46                 goto fail1;
  47         }
  48
  49         if (ps_bufsize > 0)
  50                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  51         else if (es_bufs_per_desc > 0)
  52                 dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER;
  53         else
  54                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  55
  56         if (encp->enc_tunnel_encapsulations_supported != 0 &&
  57             !want_inner_classes) {
  58                 /*
  59                  * WANT_OUTER_CLASSES can only be specified on hardware which
  60                  * supports tunnel encapsulation offloads, even though it is
  61                  * effectively the behaviour the hardware gives.
  62                  *
  63                  * Also, on hardware which does support such offloads, older
  64                  * firmware rejects the flag if the offloads are not supported
  65                  * by the current firmware variant, which means this may fail if
  66                  * the capabilities are not updated when the firmware variant
  67                  * changes. This is not an issue on newer firmware, as it was
  68                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
  69                  * specified on all firmware variants.
  70                  */
  71                 want_outer_classes = B_TRUE;
  72         } else {
  73                 want_outer_classes = B_FALSE;
  74         }
  75
  76         req.emr_cmd = MC_CMD_INIT_RXQ;
  77         req.emr_in_buf = payload;
  78         req.emr_in_length = MC_CMD_INIT_RXQ_V3_IN_LEN;
  79         req.emr_out_buf = payload;
  80         req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
  81
  82         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
  83         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  84         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  85         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  86         MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
  87             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  88             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  89             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  90             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  91             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  92             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  93             INIT_RXQ_EXT_IN_DMA_MODE,
  94             dma_mode,
  95             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
  96             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
  97         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  98         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  99
 100         if (es_bufs_per_desc > 0) {
 101                 MCDI_IN_SET_DWORD(req,
 102                     INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET,
 103                     es_bufs_per_desc);
 104                 MCDI_IN_SET_DWORD(req,
 105                     INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len);
 106                 MCDI_IN_SET_DWORD(req,
 107                     INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride);
 108                 MCDI_IN_SET_DWORD(req,
 109                     INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT,
 110                     hol_block_timeout);
 111         }
 112
 113         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
 114         addr = EFSYS_MEM_ADDR(esmp);
 115
 116         for (i = 0; i < npages; i++) {
 117                 EFX_POPULATE_QWORD_2(*dma_addr,
 118                     EFX_DWORD_1, (uint32_t)(addr >> 32),
 119                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
 120
 121                 dma_addr++;
 122                 addr += EFX_BUF_SIZE;
 123         }
 124
 125         efx_mcdi_execute(enp, &req);
 126
 127         if (req.emr_rc != 0) {
 128                 rc = req.emr_rc;
 129                 goto fail2;
 130         }
 131
 132         return (0);
 133
 134 fail2:
 135         EFSYS_PROBE(fail2);
 136 fail1:
 137         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 138
 139         return (rc);
 140 }
 141
 142 static  __checkReturn   efx_rc_t
 143 efx_mcdi_fini_rxq(
 144         __in            efx_nic_t *enp,
 145         __in            uint32_t instance)
 146 {
 147         efx_mcdi_req_t req;
 148         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
 149                 MC_CMD_FINI_RXQ_OUT_LEN);
 150         efx_rc_t rc;
 151
 152         req.emr_cmd = MC_CMD_FINI_RXQ;
 153         req.emr_in_buf = payload;
 154         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 155         req.emr_out_buf = payload;
 156         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 157
 158         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 159
 160         efx_mcdi_execute_quiet(enp, &req);
 161
 162         if (req.emr_rc != 0) {
 163                 rc = req.emr_rc;
 164                 goto fail1;
 165         }
 166
 167         return (0);
 168
 169 fail1:
 170         /*
 171          * EALREADY is not an error, but indicates that the MC has rebooted and
 172          * that the RXQ has already been destroyed.
 173          */
 174         if (rc != EALREADY)
 175                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 176
 177         return (rc);
 178 }
 179
 180 #if EFSYS_OPT_RX_SCALE
 181 static  __checkReturn   efx_rc_t
 182 efx_mcdi_rss_context_alloc(
 183         __in            efx_nic_t *enp,
 184         __in            efx_rx_scale_context_type_t type,
 185         __in            uint32_t num_queues,
 186         __out           uint32_t *rss_contextp)
 187 {
 188         efx_mcdi_req_t req;
 189         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 190                 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
 191         uint32_t rss_context;
 192         uint32_t context_type;
 193         efx_rc_t rc;
 194
 195         if (num_queues > EFX_MAXRSS) {
 196                 rc = EINVAL;
 197                 goto fail1;
 198         }
 199
 200         switch (type) {
 201         case EFX_RX_SCALE_EXCLUSIVE:
 202                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 203                 break;
 204         case EFX_RX_SCALE_SHARED:
 205                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 206                 break;
 207         default:
 208                 rc = EINVAL;
 209                 goto fail2;
 210         }
 211
 212         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 213         req.emr_in_buf = payload;
 214         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 215         req.emr_out_buf = payload;
 216         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 217
 218         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 219             EVB_PORT_ID_ASSIGNED);
 220         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 221
 222         /*
 223          * For exclusive contexts, NUM_QUEUES is only used to validate
 224          * indirection table offsets.
 225          * For shared contexts, the provided context will spread traffic over
 226          * NUM_QUEUES many queues.
 227          */
 228         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 229
 230         efx_mcdi_execute(enp, &req);
 231
 232         if (req.emr_rc != 0) {
 233                 rc = req.emr_rc;
 234                 goto fail3;
 235         }
 236
 237         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 238                 rc = EMSGSIZE;
 239                 goto fail4;
 240         }
 241
 242         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 243         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 244                 rc = ENOENT;
 245                 goto fail5;
 246         }
 247
 248         *rss_contextp = rss_context;
 249
 250         return (0);
 251
 252 fail5:
 253         EFSYS_PROBE(fail5);
 254 fail4:
 255         EFSYS_PROBE(fail4);
 256 fail3:
 257         EFSYS_PROBE(fail3);
 258 fail2:
 259         EFSYS_PROBE(fail2);
 260 fail1:
 261         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 262
 263         return (rc);
 264 }
 265 #endif /* EFSYS_OPT_RX_SCALE */
 266
 267 #if EFSYS_OPT_RX_SCALE
 268 static                  efx_rc_t
 269 efx_mcdi_rss_context_free(
 270         __in            efx_nic_t *enp,
 271         __in            uint32_t rss_context)
 272 {
 273         efx_mcdi_req_t req;
 274         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 275                 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 276         efx_rc_t rc;
 277
 278         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 279                 rc = EINVAL;
 280                 goto fail1;
 281         }
 282
 283         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 284         req.emr_in_buf = payload;
 285         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 286         req.emr_out_buf = payload;
 287         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 288
 289         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 290
 291         efx_mcdi_execute_quiet(enp, &req);
 292
 293         if (req.emr_rc != 0) {
 294                 rc = req.emr_rc;
 295                 goto fail2;
 296         }
 297
 298         return (0);
 299
 300 fail2:
 301         EFSYS_PROBE(fail2);
 302 fail1:
 303         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 304
 305         return (rc);
 306 }
 307 #endif /* EFSYS_OPT_RX_SCALE */
 308
 309 #if EFSYS_OPT_RX_SCALE
 310 static                  efx_rc_t
 311 efx_mcdi_rss_context_set_flags(
 312         __in            efx_nic_t *enp,
 313         __in            uint32_t rss_context,
 314         __in            efx_rx_hash_type_t type)
 315 {
 316         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 317         efx_rx_hash_type_t type_ipv4;
 318         efx_rx_hash_type_t type_ipv4_tcp;
 319         efx_rx_hash_type_t type_ipv6;
 320         efx_rx_hash_type_t type_ipv6_tcp;
 321         efx_rx_hash_type_t modes;
 322         efx_mcdi_req_t req;
 323         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 324                 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 325         efx_rc_t rc;
 326
 327         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
 328                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
 329         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
 330                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
 331         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
 332                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
 333         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
 334                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
 335         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
 336                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
 337         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
 338                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
 339         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
 340                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
 341         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
 342                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
 343
 344         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 345                 rc = EINVAL;
 346                 goto fail1;
 347         }
 348
 349         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 350         req.emr_in_buf = payload;
 351         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 352         req.emr_out_buf = payload;
 353         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 354
 355         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 356             rss_context);
 357
 358         type_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | EFX_RX_HASH(IPV4_TCP, 2TUPLE) |
 359                     EFX_RX_HASH(IPV4_UDP, 2TUPLE);
 360         type_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
 361         type_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | EFX_RX_HASH(IPV6_TCP, 2TUPLE) |
 362                     EFX_RX_HASH(IPV6_UDP, 2TUPLE);
 363         type_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
 364
 365         /*
 366          * Create a copy of the original hash type.
 367          * The copy will be used to fill in RSS_MODE bits and
 368          * may be cleared beforehand. The original variable
 369          * and, thus, EN bits will remain unaffected.
 370          */
 371         modes = type;
 372
 373         /*
 374          * If the firmware lacks support for additional modes, RSS_MODE
 375          * fields must contain zeros, otherwise the operation will fail.
 376          */
 377         if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
 378                 modes = 0;
 379
 380         MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 381             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 382             ((type & type_ipv4) == type_ipv4) ? 1 : 0,
 383             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 384             ((type & type_ipv4_tcp) == type_ipv4_tcp) ? 1 : 0,
 385             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 386             ((type & type_ipv6) == type_ipv6) ? 1 : 0,
 387             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 388             ((type & type_ipv6_tcp) == type_ipv6_tcp) ? 1 : 0,
 389             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
 390             (modes >> EFX_RX_CLASS_IPV4_TCP_LBN) &
 391             EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
 392             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
 393             (modes >> EFX_RX_CLASS_IPV4_UDP_LBN) &
 394             EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
 395             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
 396             (modes >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
 397             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
 398             (modes >> EFX_RX_CLASS_IPV6_TCP_LBN) &
 399             EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
 400             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
 401             (modes >> EFX_RX_CLASS_IPV6_UDP_LBN) &
 402             EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
 403             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
 404             (modes >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
 405
 406         efx_mcdi_execute(enp, &req);
 407
 408         if (req.emr_rc != 0) {
 409                 rc = req.emr_rc;
 410                 goto fail2;
 411         }
 412
 413         return (0);
 414
 415 fail2:
 416         EFSYS_PROBE(fail2);
 417 fail1:
 418         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 419
 420         return (rc);
 421 }
 422 #endif /* EFSYS_OPT_RX_SCALE */
 423
 424 #if EFSYS_OPT_RX_SCALE
 425 static                  efx_rc_t
 426 efx_mcdi_rss_context_set_key(
 427         __in            efx_nic_t *enp,
 428         __in            uint32_t rss_context,
 429         __in_ecount(n)  uint8_t *key,
 430         __in            size_t n)
 431 {
 432         efx_mcdi_req_t req;
 433         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 434                 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 435         efx_rc_t rc;
 436
 437         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 438                 rc = EINVAL;
 439                 goto fail1;
 440         }
 441
 442         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 443         req.emr_in_buf = payload;
 444         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 445         req.emr_out_buf = payload;
 446         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 447
 448         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 449             rss_context);
 450
 451         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 452         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 453                 rc = EINVAL;
 454                 goto fail2;
 455         }
 456
 457         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 458             key, n);
 459
 460         efx_mcdi_execute(enp, &req);
 461
 462         if (req.emr_rc != 0) {
 463                 rc = req.emr_rc;
 464                 goto fail3;
 465         }
 466
 467         return (0);
 468
 469 fail3:
 470         EFSYS_PROBE(fail3);
 471 fail2:
 472         EFSYS_PROBE(fail2);
 473 fail1:
 474         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 475
 476         return (rc);
 477 }
 478 #endif /* EFSYS_OPT_RX_SCALE */
 479
 480 #if EFSYS_OPT_RX_SCALE
 481 static                  efx_rc_t
 482 efx_mcdi_rss_context_set_table(
 483         __in            efx_nic_t *enp,
 484         __in            uint32_t rss_context,
 485         __in_ecount(n)  unsigned int *table,
 486         __in            size_t n)
 487 {
 488         efx_mcdi_req_t req;
 489         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 490                 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 491         uint8_t *req_table;
 492         int i, rc;
 493
 494         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 495                 rc = EINVAL;
 496                 goto fail1;
 497         }
 498
 499         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 500         req.emr_in_buf = payload;
 501         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 502         req.emr_out_buf = payload;
 503         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 504
 505         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 506             rss_context);
 507
 508         req_table =
 509             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 510
 511         for (i = 0;
 512             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 513             i++) {
 514                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 515         }
 516
 517         efx_mcdi_execute(enp, &req);
 518
 519         if (req.emr_rc != 0) {
 520                 rc = req.emr_rc;
 521                 goto fail2;
 522         }
 523
 524         return (0);
 525
 526 fail2:
 527         EFSYS_PROBE(fail2);
 528 fail1:
 529         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 530
 531         return (rc);
 532 }
 533 #endif /* EFSYS_OPT_RX_SCALE */
 534
 535
 536         __checkReturn   efx_rc_t
 537 ef10_rx_init(
 538         __in            efx_nic_t *enp)
 539 {
 540 #if EFSYS_OPT_RX_SCALE
 541
 542         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 543                 &enp->en_rss_context) == 0) {
 544                 /*
 545                  * Allocated an exclusive RSS context, which allows both the
 546                  * indirection table and key to be modified.
 547                  */
 548                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 549                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 550         } else {
 551                 /*
 552                  * Failed to allocate an exclusive RSS context. Continue
 553                  * operation without support for RSS. The pseudo-header in
 554                  * received packets will not contain a Toeplitz hash value.
 555                  */
 556                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 557                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 558         }
 559
 560 #endif /* EFSYS_OPT_RX_SCALE */
 561
 562         return (0);
 563 }
 564
 565 #if EFSYS_OPT_RX_SCATTER
 566         __checkReturn   efx_rc_t
 567 ef10_rx_scatter_enable(
 568         __in            efx_nic_t *enp,
 569         __in            unsigned int buf_size)
 570 {
 571         _NOTE(ARGUNUSED(enp, buf_size))
 572         return (0);
 573 }
 574 #endif  /* EFSYS_OPT_RX_SCATTER */
 575
 576 #if EFSYS_OPT_RX_SCALE
 577         __checkReturn   efx_rc_t
 578 ef10_rx_scale_context_alloc(
 579         __in            efx_nic_t *enp,
 580         __in            efx_rx_scale_context_type_t type,
 581         __in            uint32_t num_queues,
 582         __out           uint32_t *rss_contextp)
 583 {
 584         efx_rc_t rc;
 585
 586         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 587         if (rc != 0)
 588                 goto fail1;
 589
 590         return (0);
 591
 592 fail1:
 593         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 594         return (rc);
 595 }
 596 #endif /* EFSYS_OPT_RX_SCALE */
 597
 598 #if EFSYS_OPT_RX_SCALE
 599         __checkReturn   efx_rc_t
 600 ef10_rx_scale_context_free(
 601         __in            efx_nic_t *enp,
 602         __in            uint32_t rss_context)
 603 {
 604         efx_rc_t rc;
 605
 606         rc = efx_mcdi_rss_context_free(enp, rss_context);
 607         if (rc != 0)
 608                 goto fail1;
 609
 610         return (0);
 611
 612 fail1:
 613         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 614         return (rc);
 615 }
 616 #endif /* EFSYS_OPT_RX_SCALE */
 617
 618 #if EFSYS_OPT_RX_SCALE
 619         __checkReturn   efx_rc_t
 620 ef10_rx_scale_mode_set(
 621         __in            efx_nic_t *enp,
 622         __in            uint32_t rss_context,
 623         __in            efx_rx_hash_alg_t alg,
 624         __in            efx_rx_hash_type_t type,
 625         __in            boolean_t insert)
 626 {
 627         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 628         efx_rc_t rc;
 629
 630         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 631
 632         if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
 633             insert == B_FALSE) {
 634                 rc = EINVAL;
 635                 goto fail1;
 636         }
 637
 638         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 639                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 640                         rc = ENOTSUP;
 641                         goto fail2;
 642                 }
 643                 rss_context = enp->en_rss_context;
 644         }
 645
 646         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 647                     rss_context, type)) != 0)
 648                 goto fail3;
 649
 650         return (0);
 651
 652 fail3:
 653         EFSYS_PROBE(fail3);
 654 fail2:
 655         EFSYS_PROBE(fail2);
 656 fail1:
 657         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 658
 659         return (rc);
 660 }
 661 #endif /* EFSYS_OPT_RX_SCALE */
 662
 663 #if EFSYS_OPT_RX_SCALE
 664         __checkReturn   efx_rc_t
 665 ef10_rx_scale_key_set(
 666         __in            efx_nic_t *enp,
 667         __in            uint32_t rss_context,
 668         __in_ecount(n)  uint8_t *key,
 669         __in            size_t n)
 670 {
 671         efx_rc_t rc;
 672
 673         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 674             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 675
 676         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 677                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 678                         rc = ENOTSUP;
 679                         goto fail1;
 680                 }
 681                 rss_context = enp->en_rss_context;
 682         }
 683
 684         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 685                 goto fail2;
 686
 687         return (0);
 688
 689 fail2:
 690         EFSYS_PROBE(fail2);
 691 fail1:
 692         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 693
 694         return (rc);
 695 }
 696 #endif /* EFSYS_OPT_RX_SCALE */
 697
 698 #if EFSYS_OPT_RX_SCALE
 699         __checkReturn   efx_rc_t
 700 ef10_rx_scale_tbl_set(
 701         __in            efx_nic_t *enp,
 702         __in            uint32_t rss_context,
 703         __in_ecount(n)  unsigned int *table,
 704         __in            size_t n)
 705 {
 706         efx_rc_t rc;
 707
 708
 709         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 710                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 711                         rc = ENOTSUP;
 712                         goto fail1;
 713                 }
 714                 rss_context = enp->en_rss_context;
 715         }
 716
 717         if ((rc = efx_mcdi_rss_context_set_table(enp,
 718                     rss_context, table, n)) != 0)
 719                 goto fail2;
 720
 721         return (0);
 722
 723 fail2:
 724         EFSYS_PROBE(fail2);
 725 fail1:
 726         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 727
 728         return (rc);
 729 }
 730 #endif /* EFSYS_OPT_RX_SCALE */
 731
 732
 733 /*
 734  * EF10 RX pseudo-header
 735  * ---------------------
 736  *
 737  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 738  *
 739  *  +00: Toeplitz hash value.
 740  *       (32bit little-endian)
 741  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 742  *       (16bit big-endian)
 743  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 744  *       (16bit big-endian)
 745  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 746  *       (16bit little-endian)
 747  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 748  *       (32bit little-endian)
 749  *
 750  * See "The RX Pseudo-header" in SF-109306-TC.
 751  */
 752
 753         __checkReturn   efx_rc_t
 754 ef10_rx_prefix_pktlen(
 755         __in            efx_nic_t *enp,
 756         __in            uint8_t *buffer,
 757         __out           uint16_t *lengthp)
 758 {
 759         _NOTE(ARGUNUSED(enp))
 760
 761         /*
 762          * The RX pseudo-header contains the packet length, excluding the
 763          * pseudo-header. If the hardware receive datapath was operating in
 764          * cut-through mode then the length in the RX pseudo-header will be
 765          * zero, and the packet length must be obtained from the DMA length
 766          * reported in the RX event.
 767          */
 768         *lengthp = buffer[8] | (buffer[9] << 8);
 769         return (0);
 770 }
 771
 772 #if EFSYS_OPT_RX_SCALE
 773         __checkReturn   uint32_t
 774 ef10_rx_prefix_hash(
 775         __in            efx_nic_t *enp,
 776         __in            efx_rx_hash_alg_t func,
 777         __in            uint8_t *buffer)
 778 {
 779         _NOTE(ARGUNUSED(enp))
 780
 781         switch (func) {
 782         case EFX_RX_HASHALG_PACKED_STREAM:
 783         case EFX_RX_HASHALG_TOEPLITZ:
 784                 return (buffer[0] |
 785                     (buffer[1] << 8) |
 786                     (buffer[2] << 16) |
 787                     (buffer[3] << 24));
 788
 789         default:
 790                 EFSYS_ASSERT(0);
 791                 return (0);
 792         }
 793 }
 794 #endif /* EFSYS_OPT_RX_SCALE */
 795
 796 #if EFSYS_OPT_RX_PACKED_STREAM
 797 /*
 798  * Fake length for RXQ descriptors in packed stream mode
 799  * to make hardware happy
 800  */
 801 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 802 #endif
 803
 804                                 void
 805 ef10_rx_qpost(
 806         __in                    efx_rxq_t *erp,
 807         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 808         __in                    size_t size,
 809         __in                    unsigned int ndescs,
 810         __in                    unsigned int completed,
 811         __in                    unsigned int added)
 812 {
 813         efx_qword_t qword;
 814         unsigned int i;
 815         unsigned int offset;
 816         unsigned int id;
 817
 818         _NOTE(ARGUNUSED(completed))
 819
 820 #if EFSYS_OPT_RX_PACKED_STREAM
 821         /*
 822          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 823          * and equal to 0 after applying mask. Hardware does not like it.
 824          */
 825         if (erp->er_ev_qstate->eers_rx_packed_stream)
 826                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 827 #endif
 828
 829         /* The client driver must not overfill the queue */
 830         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 831             EFX_RXQ_LIMIT(erp->er_mask + 1));
 832
 833         id = added & (erp->er_mask);
 834         for (i = 0; i < ndescs; i++) {
 835                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 836                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 837                     size_t, size);
 838
 839                 EFX_POPULATE_QWORD_3(qword,
 840                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 841                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 842                     (uint32_t)(addrp[i] & 0xffffffff),
 843                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 844                     (uint32_t)(addrp[i] >> 32));
 845
 846                 offset = id * sizeof (efx_qword_t);
 847                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 848
 849                 id = (id + 1) & (erp->er_mask);
 850         }
 851 }
 852
 853                         void
 854 ef10_rx_qpush(
 855         __in    efx_rxq_t *erp,
 856         __in    unsigned int added,
 857         __inout unsigned int *pushedp)
 858 {
 859         efx_nic_t *enp = erp->er_enp;
 860         unsigned int pushed = *pushedp;
 861         uint32_t wptr;
 862         efx_dword_t dword;
 863
 864         /* Hardware has alignment restriction for WPTR */
 865         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 866         if (pushed == wptr)
 867                 return;
 868
 869         *pushedp = wptr;
 870
 871         /* Push the populated descriptors out */
 872         wptr &= erp->er_mask;
 873
 874         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 875
 876         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 877         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 878             wptr, pushed & erp->er_mask);
 879         EFSYS_PIO_WRITE_BARRIER();
 880         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 881             erp->er_index, &dword, B_FALSE);
 882 }
 883
 884 #if EFSYS_OPT_RX_PACKED_STREAM
 885
 886                         void
 887 ef10_rx_qpush_ps_credits(
 888         __in            efx_rxq_t *erp)
 889 {
 890         efx_nic_t *enp = erp->er_enp;
 891         efx_dword_t dword;
 892         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 893         uint32_t credits;
 894
 895         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 896
 897         if (rxq_state->eers_rx_packed_stream_credits == 0)
 898                 return;
 899
 900         /*
 901          * It is a bug if we think that FW has utilized more
 902          * credits than it is allowed to have (maximum). However,
 903          * make sure that we do not credit more than maximum anyway.
 904          */
 905         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 906             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 907         EFX_POPULATE_DWORD_3(dword,
 908             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 909             ERF_DZ_RX_DESC_MAGIC_CMD,
 910             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 911             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 912         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 913             erp->er_index, &dword, B_FALSE);
 914
 915         rxq_state->eers_rx_packed_stream_credits = 0;
 916 }
 917
 918 /*
 919  * In accordance with SF-112241-TC the received data has the following layout:
 920  *  - 8 byte pseudo-header which consist of:
 921  *    - 4 byte little-endian timestamp
 922  *    - 2 byte little-endian captured length in bytes
 923  *    - 2 byte little-endian original packet length in bytes
 924  *  - captured packet bytes
 925  *  - optional padding to align to 64 bytes boundary
 926  *  - 64 bytes scratch space for the host software
 927  */
 928         __checkReturn   uint8_t *
 929 ef10_rx_qps_packet_info(
 930         __in            efx_rxq_t *erp,
 931         __in            uint8_t *buffer,
 932         __in            uint32_t buffer_length,
 933         __in            uint32_t current_offset,
 934         __out           uint16_t *lengthp,
 935         __out           uint32_t *next_offsetp,
 936         __out           uint32_t *timestamp)
 937 {
 938         uint16_t buf_len;
 939         uint8_t *pkt_start;
 940         efx_qword_t *qwordp;
 941         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 942
 943         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 944
 945         buffer += current_offset;
 946         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 947
 948         qwordp = (efx_qword_t *)buffer;
 949         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 950         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 951         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 952
 953         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 954                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 955         *next_offsetp =
 956             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 957
 958         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 959         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 960
 961         if ((*next_offsetp ^ current_offset) &
 962             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 963                 rxq_state->eers_rx_packed_stream_credits++;
 964
 965         return (pkt_start);
 966 }
 967
 968
 969 #endif
 970
 971         __checkReturn   efx_rc_t
 972 ef10_rx_qflush(
 973         __in    efx_rxq_t *erp)
 974 {
 975         efx_nic_t *enp = erp->er_enp;
 976         efx_rc_t rc;
 977
 978         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 979                 goto fail1;
 980
 981         return (0);
 982
 983 fail1:
 984         /*
 985          * EALREADY is not an error, but indicates that the MC has rebooted and
 986          * that the RXQ has already been destroyed. Callers need to know that
 987          * the RXQ flush has completed to avoid waiting until timeout for a
 988          * flush done event that will not be delivered.
 989          */
 990         if (rc != EALREADY)
 991                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 992
 993         return (rc);
 994 }
 995
 996                 void
 997 ef10_rx_qenable(
 998         __in    efx_rxq_t *erp)
 999 {
1000         /* FIXME */
1001         _NOTE(ARGUNUSED(erp))
1002         /* FIXME */
1003 }
1004
1005         __checkReturn   efx_rc_t
1006 ef10_rx_qcreate(
1007         __in            efx_nic_t *enp,
1008         __in            unsigned int index,
1009         __in            unsigned int label,
1010         __in            efx_rxq_type_t type,
1011         __in            const efx_rxq_type_data_t *type_data,
1012         __in            efsys_mem_t *esmp,
1013         __in            size_t ndescs,
1014         __in            uint32_t id,
1015         __in            unsigned int flags,
1016         __in            efx_evq_t *eep,
1017         __in            efx_rxq_t *erp)
1018 {
1019         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1020         efx_rc_t rc;
1021         boolean_t disable_scatter;
1022         boolean_t want_inner_classes;
1023         unsigned int ps_buf_size;
1024         uint32_t es_bufs_per_desc = 0;
1025         uint32_t es_max_dma_len = 0;
1026         uint32_t es_buf_stride = 0;
1027         uint32_t hol_block_timeout = 0;
1028
1029         _NOTE(ARGUNUSED(id, erp, type_data))
1030
1031         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
1032         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1033         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1034
1035         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1036         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1037
1038         if (!ISP2(ndescs) ||
1039             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
1040                 rc = EINVAL;
1041                 goto fail1;
1042         }
1043         if (index >= encp->enc_rxq_limit) {
1044                 rc = EINVAL;
1045                 goto fail2;
1046         }
1047
1048         switch (type) {
1049         case EFX_RXQ_TYPE_DEFAULT:
1050                 ps_buf_size = 0;
1051                 break;
1052 #if EFSYS_OPT_RX_PACKED_STREAM
1053         case EFX_RXQ_TYPE_PACKED_STREAM:
1054                 switch (type_data->ertd_packed_stream.eps_buf_size) {
1055                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
1056                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
1057                         break;
1058                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
1059                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
1060                         break;
1061                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1062                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1063                         break;
1064                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1065                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1066                         break;
1067                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1068                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1069                         break;
1070                 default:
1071                         rc = ENOTSUP;
1072                         goto fail3;
1073                 }
1074                 break;
1075 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1076 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1077         case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
1078                 ps_buf_size = 0;
1079                 es_bufs_per_desc =
1080                     type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
1081                 es_max_dma_len =
1082                     type_data->ertd_es_super_buffer.eessb_max_dma_len;
1083                 es_buf_stride =
1084                     type_data->ertd_es_super_buffer.eessb_buf_stride;
1085                 hol_block_timeout =
1086                     type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
1087                 break;
1088 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1089         default:
1090                 rc = ENOTSUP;
1091                 goto fail4;
1092         }
1093
1094 #if EFSYS_OPT_RX_PACKED_STREAM
1095         if (ps_buf_size != 0) {
1096                 /* Check if datapath firmware supports packed stream mode */
1097                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1098                         rc = ENOTSUP;
1099                         goto fail5;
1100                 }
1101                 /* Check if packed stream allows configurable buffer sizes */
1102                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1103                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1104                         rc = ENOTSUP;
1105                         goto fail6;
1106                 }
1107         }
1108 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1109         EFSYS_ASSERT(ps_buf_size == 0);
1110 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1111
1112 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1113         if (es_bufs_per_desc > 0) {
1114                 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
1115                         rc = ENOTSUP;
1116                         goto fail7;
1117                 }
1118                 if (!IS_P2ALIGNED(es_max_dma_len,
1119                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1120                         rc = EINVAL;
1121                         goto fail8;
1122                 }
1123                 if (!IS_P2ALIGNED(es_buf_stride,
1124                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1125                         rc = EINVAL;
1126                         goto fail9;
1127                 }
1128         }
1129 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1130         EFSYS_ASSERT(es_bufs_per_desc == 0);
1131 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1132
1133         /* Scatter can only be disabled if the firmware supports doing so */
1134         if (flags & EFX_RXQ_FLAG_SCATTER)
1135                 disable_scatter = B_FALSE;
1136         else
1137                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1138
1139         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1140                 want_inner_classes = B_TRUE;
1141         else
1142                 want_inner_classes = B_FALSE;
1143
1144         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1145                     esmp, disable_scatter, want_inner_classes,
1146                     ps_buf_size, es_bufs_per_desc, es_max_dma_len,
1147                     es_buf_stride, hol_block_timeout)) != 0)
1148                 goto fail10;
1149
1150         erp->er_eep = eep;
1151         erp->er_label = label;
1152
1153         ef10_ev_rxlabel_init(eep, erp, label, type);
1154
1155         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1156
1157         return (0);
1158
1159 fail10:
1160         EFSYS_PROBE(fail10);
1161 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1162 fail9:
1163         EFSYS_PROBE(fail9);
1164 fail8:
1165         EFSYS_PROBE(fail8);
1166 fail7:
1167         EFSYS_PROBE(fail7);
1168 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1169 #if EFSYS_OPT_RX_PACKED_STREAM
1170 fail6:
1171         EFSYS_PROBE(fail6);
1172 fail5:
1173         EFSYS_PROBE(fail5);
1174 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1175 fail4:
1176         EFSYS_PROBE(fail4);
1177 #if EFSYS_OPT_RX_PACKED_STREAM
1178 fail3:
1179         EFSYS_PROBE(fail3);
1180 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1181 fail2:
1182         EFSYS_PROBE(fail2);
1183 fail1:
1184         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1185
1186         return (rc);
1187 }
1188
1189                 void
1190 ef10_rx_qdestroy(
1191         __in    efx_rxq_t *erp)
1192 {
1193         efx_nic_t *enp = erp->er_enp;
1194         efx_evq_t *eep = erp->er_eep;
1195         unsigned int label = erp->er_label;
1196
1197         ef10_ev_rxlabel_fini(eep, label);
1198
1199         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1200         --enp->en_rx_qcount;
1201
1202         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1203 }
1204
1205                 void
1206 ef10_rx_fini(
1207         __in    efx_nic_t *enp)
1208 {
1209 #if EFSYS_OPT_RX_SCALE
1210         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1211                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1212         enp->en_rss_context = 0;
1213         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1214 #else
1215         _NOTE(ARGUNUSED(enp))
1216 #endif /* EFSYS_OPT_RX_SCALE */
1217 }
1218
1219 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */