drivers/net/sfc/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2012-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFX_OPTS_EF10()
  12
  13
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_init_rxq(
  16         __in            efx_nic_t *enp,
  17         __in            uint32_t ndescs,
  18         __in            efx_evq_t *eep,
  19         __in            uint32_t label,
  20         __in            uint32_t instance,
  21         __in            efsys_mem_t *esmp,
  22         __in            boolean_t disable_scatter,
  23         __in            boolean_t want_inner_classes,
  24         __in            uint32_t buf_size,
  25         __in            uint32_t ps_bufsize,
  26         __in            uint32_t es_bufs_per_desc,
  27         __in            uint32_t es_max_dma_len,
  28         __in            uint32_t es_buf_stride,
  29         __in            uint32_t hol_block_timeout)
  30 {
  31         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  32         efx_mcdi_req_t req;
  33         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_V4_IN_LEN,
  34                 MC_CMD_INIT_RXQ_V4_OUT_LEN);
  35         int npages = efx_rxq_nbufs(enp, ndescs);
  36         int i;
  37         efx_qword_t *dma_addr;
  38         uint64_t addr;
  39         efx_rc_t rc;
  40         uint32_t dma_mode;
  41         boolean_t want_outer_classes;
  42         boolean_t no_cont_ev;
  43
  44         EFSYS_ASSERT3U(ndescs, <=, encp->enc_rxq_max_ndescs);
  45
  46         if ((esmp == NULL) ||
  47             (EFSYS_MEM_SIZE(esmp) < efx_rxq_size(enp, ndescs))) {
  48                 rc = EINVAL;
  49                 goto fail1;
  50         }
  51
  52         no_cont_ev = (eep->ee_flags & EFX_EVQ_FLAGS_NO_CONT_EV);
  53         if ((no_cont_ev == B_TRUE) && (disable_scatter == B_FALSE)) {
  54                 /* TODO: Support scatter in NO_CONT_EV mode */
  55                 rc = EINVAL;
  56                 goto fail2;
  57         }
  58
  59         if (ps_bufsize > 0)
  60                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  61         else if (es_bufs_per_desc > 0)
  62                 dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER;
  63         else
  64                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  65
  66         if (encp->enc_tunnel_encapsulations_supported != 0 &&
  67             !want_inner_classes) {
  68                 /*
  69                  * WANT_OUTER_CLASSES can only be specified on hardware which
  70                  * supports tunnel encapsulation offloads, even though it is
  71                  * effectively the behaviour the hardware gives.
  72                  *
  73                  * Also, on hardware which does support such offloads, older
  74                  * firmware rejects the flag if the offloads are not supported
  75                  * by the current firmware variant, which means this may fail if
  76                  * the capabilities are not updated when the firmware variant
  77                  * changes. This is not an issue on newer firmware, as it was
  78                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
  79                  * specified on all firmware variants.
  80                  */
  81                 want_outer_classes = B_TRUE;
  82         } else {
  83                 want_outer_classes = B_FALSE;
  84         }
  85
  86         req.emr_cmd = MC_CMD_INIT_RXQ;
  87         req.emr_in_buf = payload;
  88         req.emr_in_length = MC_CMD_INIT_RXQ_V4_IN_LEN;
  89         req.emr_out_buf = payload;
  90         req.emr_out_length = MC_CMD_INIT_RXQ_V4_OUT_LEN;
  91
  92         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
  93         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, eep->ee_index);
  94         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  95         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  96         MCDI_IN_POPULATE_DWORD_10(req, INIT_RXQ_EXT_IN_FLAGS,
  97             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  98             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  99             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
 100             INIT_RXQ_EXT_IN_CRC_MODE, 0,
 101             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
 102             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
 103             INIT_RXQ_EXT_IN_DMA_MODE,
 104             dma_mode,
 105             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
 106             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes,
 107             INIT_RXQ_EXT_IN_FLAG_NO_CONT_EV, no_cont_ev);
 108         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
 109         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, enp->en_vport_id);
 110
 111         if (es_bufs_per_desc > 0) {
 112                 MCDI_IN_SET_DWORD(req,
 113                     INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET,
 114                     es_bufs_per_desc);
 115                 MCDI_IN_SET_DWORD(req,
 116                     INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len);
 117                 MCDI_IN_SET_DWORD(req,
 118                     INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride);
 119                 MCDI_IN_SET_DWORD(req,
 120                     INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT,
 121                     hol_block_timeout);
 122         }
 123
 124         if (encp->enc_init_rxq_with_buffer_size)
 125                 MCDI_IN_SET_DWORD(req, INIT_RXQ_V4_IN_BUFFER_SIZE_BYTES,
 126                     buf_size);
 127
 128         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
 129         addr = EFSYS_MEM_ADDR(esmp);
 130
 131         for (i = 0; i < npages; i++) {
 132                 EFX_POPULATE_QWORD_2(*dma_addr,
 133                     EFX_DWORD_1, (uint32_t)(addr >> 32),
 134                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
 135
 136                 dma_addr++;
 137                 addr += EFX_BUF_SIZE;
 138         }
 139
 140         efx_mcdi_execute(enp, &req);
 141
 142         if (req.emr_rc != 0) {
 143                 rc = req.emr_rc;
 144                 goto fail3;
 145         }
 146
 147         return (0);
 148
 149 fail3:
 150         EFSYS_PROBE(fail3);
 151 fail2:
 152         EFSYS_PROBE(fail2);
 153 fail1:
 154         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 155
 156         return (rc);
 157 }
 158
 159 static  __checkReturn   efx_rc_t
 160 efx_mcdi_fini_rxq(
 161         __in            efx_nic_t *enp,
 162         __in            uint32_t instance)
 163 {
 164         efx_mcdi_req_t req;
 165         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
 166                 MC_CMD_FINI_RXQ_OUT_LEN);
 167         efx_rc_t rc;
 168
 169         req.emr_cmd = MC_CMD_FINI_RXQ;
 170         req.emr_in_buf = payload;
 171         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 172         req.emr_out_buf = payload;
 173         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 174
 175         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 176
 177         efx_mcdi_execute_quiet(enp, &req);
 178
 179         if (req.emr_rc != 0) {
 180                 rc = req.emr_rc;
 181                 goto fail1;
 182         }
 183
 184         return (0);
 185
 186 fail1:
 187         /*
 188          * EALREADY is not an error, but indicates that the MC has rebooted and
 189          * that the RXQ has already been destroyed.
 190          */
 191         if (rc != EALREADY)
 192                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 193
 194         return (rc);
 195 }
 196
 197 #if EFSYS_OPT_RX_SCALE
 198 static  __checkReturn   efx_rc_t
 199 efx_mcdi_rss_context_alloc(
 200         __in            efx_nic_t *enp,
 201         __in            efx_rx_scale_context_type_t type,
 202         __in            uint32_t num_queues,
 203         __out           uint32_t *rss_contextp)
 204 {
 205         efx_mcdi_req_t req;
 206         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 207                 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
 208         uint32_t rss_context;
 209         uint32_t context_type;
 210         efx_rc_t rc;
 211
 212         if (num_queues > EFX_MAXRSS) {
 213                 rc = EINVAL;
 214                 goto fail1;
 215         }
 216
 217         switch (type) {
 218         case EFX_RX_SCALE_EXCLUSIVE:
 219                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 220                 break;
 221         case EFX_RX_SCALE_SHARED:
 222                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 223                 break;
 224         default:
 225                 rc = EINVAL;
 226                 goto fail2;
 227         }
 228
 229         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 230         req.emr_in_buf = payload;
 231         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 232         req.emr_out_buf = payload;
 233         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 234
 235         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 236                 enp->en_vport_id);
 237         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 238
 239         /*
 240          * For exclusive contexts, NUM_QUEUES is only used to validate
 241          * indirection table offsets.
 242          * For shared contexts, the provided context will spread traffic over
 243          * NUM_QUEUES many queues.
 244          */
 245         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 246
 247         efx_mcdi_execute(enp, &req);
 248
 249         if (req.emr_rc != 0) {
 250                 rc = req.emr_rc;
 251                 goto fail3;
 252         }
 253
 254         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 255                 rc = EMSGSIZE;
 256                 goto fail4;
 257         }
 258
 259         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 260         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 261                 rc = ENOENT;
 262                 goto fail5;
 263         }
 264
 265         *rss_contextp = rss_context;
 266
 267         return (0);
 268
 269 fail5:
 270         EFSYS_PROBE(fail5);
 271 fail4:
 272         EFSYS_PROBE(fail4);
 273 fail3:
 274         EFSYS_PROBE(fail3);
 275 fail2:
 276         EFSYS_PROBE(fail2);
 277 fail1:
 278         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 279
 280         return (rc);
 281 }
 282 #endif /* EFSYS_OPT_RX_SCALE */
 283
 284 #if EFSYS_OPT_RX_SCALE
 285 static                  efx_rc_t
 286 efx_mcdi_rss_context_free(
 287         __in            efx_nic_t *enp,
 288         __in            uint32_t rss_context)
 289 {
 290         efx_mcdi_req_t req;
 291         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 292                 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 293         efx_rc_t rc;
 294
 295         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 296                 rc = EINVAL;
 297                 goto fail1;
 298         }
 299
 300         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 301         req.emr_in_buf = payload;
 302         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 303         req.emr_out_buf = payload;
 304         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 305
 306         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 307
 308         efx_mcdi_execute_quiet(enp, &req);
 309
 310         if (req.emr_rc != 0) {
 311                 rc = req.emr_rc;
 312                 goto fail2;
 313         }
 314
 315         return (0);
 316
 317 fail2:
 318         EFSYS_PROBE(fail2);
 319 fail1:
 320         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 321
 322         return (rc);
 323 }
 324 #endif /* EFSYS_OPT_RX_SCALE */
 325
 326 #if EFSYS_OPT_RX_SCALE
 327 static                  efx_rc_t
 328 efx_mcdi_rss_context_set_flags(
 329         __in            efx_nic_t *enp,
 330         __in            uint32_t rss_context,
 331         __in            efx_rx_hash_type_t type)
 332 {
 333         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 334         efx_mcdi_req_t req;
 335         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 336                 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 337         efx_rc_t rc;
 338
 339         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
 340                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
 341         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
 342                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
 343         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
 344                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
 345         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
 346                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
 347         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
 348                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
 349         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
 350                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
 351         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
 352                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
 353         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
 354                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
 355
 356         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 357                 rc = EINVAL;
 358                 goto fail1;
 359         }
 360
 361         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 362         req.emr_in_buf = payload;
 363         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 364         req.emr_out_buf = payload;
 365         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 366
 367         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 368             rss_context);
 369
 370         /*
 371          * If the firmware lacks support for additional modes, RSS_MODE
 372          * fields must contain zeros, otherwise the operation will fail.
 373          */
 374         if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
 375                 type &= EFX_RX_HASH_LEGACY_MASK;
 376
 377         MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 378             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 379             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 380             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 381             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 382             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 383             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 384             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 385             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0,
 386             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
 387             (type >> EFX_RX_CLASS_IPV4_TCP_LBN) &
 388             EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
 389             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
 390             (type >> EFX_RX_CLASS_IPV4_UDP_LBN) &
 391             EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
 392             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
 393             (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
 394             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
 395             (type >> EFX_RX_CLASS_IPV6_TCP_LBN) &
 396             EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
 397             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
 398             (type >> EFX_RX_CLASS_IPV6_UDP_LBN) &
 399             EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
 400             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
 401             (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
 402
 403         efx_mcdi_execute(enp, &req);
 404
 405         if (req.emr_rc != 0) {
 406                 rc = req.emr_rc;
 407                 goto fail2;
 408         }
 409
 410         return (0);
 411
 412 fail2:
 413         EFSYS_PROBE(fail2);
 414 fail1:
 415         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 416
 417         return (rc);
 418 }
 419 #endif /* EFSYS_OPT_RX_SCALE */
 420
 421 #if EFSYS_OPT_RX_SCALE
 422 static                  efx_rc_t
 423 efx_mcdi_rss_context_set_key(
 424         __in            efx_nic_t *enp,
 425         __in            uint32_t rss_context,
 426         __in_ecount(n)  uint8_t *key,
 427         __in            size_t n)
 428 {
 429         efx_mcdi_req_t req;
 430         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 431                 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 432         efx_rc_t rc;
 433
 434         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 435                 rc = EINVAL;
 436                 goto fail1;
 437         }
 438
 439         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 440         req.emr_in_buf = payload;
 441         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 442         req.emr_out_buf = payload;
 443         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 444
 445         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 446             rss_context);
 447
 448         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 449         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 450                 rc = EINVAL;
 451                 goto fail2;
 452         }
 453
 454         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 455             key, n);
 456
 457         efx_mcdi_execute(enp, &req);
 458
 459         if (req.emr_rc != 0) {
 460                 rc = req.emr_rc;
 461                 goto fail3;
 462         }
 463
 464         return (0);
 465
 466 fail3:
 467         EFSYS_PROBE(fail3);
 468 fail2:
 469         EFSYS_PROBE(fail2);
 470 fail1:
 471         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 472
 473         return (rc);
 474 }
 475 #endif /* EFSYS_OPT_RX_SCALE */
 476
 477 #if EFSYS_OPT_RX_SCALE
 478 static                  efx_rc_t
 479 efx_mcdi_rss_context_set_table(
 480         __in            efx_nic_t *enp,
 481         __in            uint32_t rss_context,
 482         __in_ecount(n)  unsigned int *table,
 483         __in            size_t n)
 484 {
 485         efx_mcdi_req_t req;
 486         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 487                 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 488         uint8_t *req_table;
 489         int i, rc;
 490
 491         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 492                 rc = EINVAL;
 493                 goto fail1;
 494         }
 495
 496         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 497         req.emr_in_buf = payload;
 498         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 499         req.emr_out_buf = payload;
 500         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 501
 502         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 503             rss_context);
 504
 505         req_table =
 506             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 507
 508         for (i = 0;
 509             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 510             i++) {
 511                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 512         }
 513
 514         efx_mcdi_execute(enp, &req);
 515
 516         if (req.emr_rc != 0) {
 517                 rc = req.emr_rc;
 518                 goto fail2;
 519         }
 520
 521         return (0);
 522
 523 fail2:
 524         EFSYS_PROBE(fail2);
 525 fail1:
 526         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 527
 528         return (rc);
 529 }
 530 #endif /* EFSYS_OPT_RX_SCALE */
 531
 532
 533         __checkReturn   efx_rc_t
 534 ef10_rx_init(
 535         __in            efx_nic_t *enp)
 536 {
 537 #if EFSYS_OPT_RX_SCALE
 538
 539         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 540                 &enp->en_rss_context) == 0) {
 541                 /*
 542                  * Allocated an exclusive RSS context, which allows both the
 543                  * indirection table and key to be modified.
 544                  */
 545                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 546                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 547         } else {
 548                 /*
 549                  * Failed to allocate an exclusive RSS context. Continue
 550                  * operation without support for RSS. The pseudo-header in
 551                  * received packets will not contain a Toeplitz hash value.
 552                  */
 553                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 554                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 555         }
 556
 557 #endif /* EFSYS_OPT_RX_SCALE */
 558
 559         return (0);
 560 }
 561
 562 #if EFSYS_OPT_RX_SCATTER
 563         __checkReturn   efx_rc_t
 564 ef10_rx_scatter_enable(
 565         __in            efx_nic_t *enp,
 566         __in            unsigned int buf_size)
 567 {
 568         _NOTE(ARGUNUSED(enp, buf_size))
 569         return (0);
 570 }
 571 #endif  /* EFSYS_OPT_RX_SCATTER */
 572
 573 #if EFSYS_OPT_RX_SCALE
 574         __checkReturn   efx_rc_t
 575 ef10_rx_scale_context_alloc(
 576         __in            efx_nic_t *enp,
 577         __in            efx_rx_scale_context_type_t type,
 578         __in            uint32_t num_queues,
 579         __out           uint32_t *rss_contextp)
 580 {
 581         efx_rc_t rc;
 582
 583         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 584         if (rc != 0)
 585                 goto fail1;
 586
 587         return (0);
 588
 589 fail1:
 590         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 591         return (rc);
 592 }
 593 #endif /* EFSYS_OPT_RX_SCALE */
 594
 595 #if EFSYS_OPT_RX_SCALE
 596         __checkReturn   efx_rc_t
 597 ef10_rx_scale_context_free(
 598         __in            efx_nic_t *enp,
 599         __in            uint32_t rss_context)
 600 {
 601         efx_rc_t rc;
 602
 603         rc = efx_mcdi_rss_context_free(enp, rss_context);
 604         if (rc != 0)
 605                 goto fail1;
 606
 607         return (0);
 608
 609 fail1:
 610         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 611         return (rc);
 612 }
 613 #endif /* EFSYS_OPT_RX_SCALE */
 614
 615 #if EFSYS_OPT_RX_SCALE
 616         __checkReturn   efx_rc_t
 617 ef10_rx_scale_mode_set(
 618         __in            efx_nic_t *enp,
 619         __in            uint32_t rss_context,
 620         __in            efx_rx_hash_alg_t alg,
 621         __in            efx_rx_hash_type_t type,
 622         __in            boolean_t insert)
 623 {
 624         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 625         efx_rc_t rc;
 626
 627         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 628
 629         if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
 630             insert == B_FALSE) {
 631                 rc = EINVAL;
 632                 goto fail1;
 633         }
 634
 635         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 636                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 637                         rc = ENOTSUP;
 638                         goto fail2;
 639                 }
 640                 rss_context = enp->en_rss_context;
 641         }
 642
 643         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 644                     rss_context, type)) != 0)
 645                 goto fail3;
 646
 647         return (0);
 648
 649 fail3:
 650         EFSYS_PROBE(fail3);
 651 fail2:
 652         EFSYS_PROBE(fail2);
 653 fail1:
 654         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 655
 656         return (rc);
 657 }
 658 #endif /* EFSYS_OPT_RX_SCALE */
 659
 660 #if EFSYS_OPT_RX_SCALE
 661         __checkReturn   efx_rc_t
 662 ef10_rx_scale_key_set(
 663         __in            efx_nic_t *enp,
 664         __in            uint32_t rss_context,
 665         __in_ecount(n)  uint8_t *key,
 666         __in            size_t n)
 667 {
 668         efx_rc_t rc;
 669
 670         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 671             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 672
 673         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 674                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 675                         rc = ENOTSUP;
 676                         goto fail1;
 677                 }
 678                 rss_context = enp->en_rss_context;
 679         }
 680
 681         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 682                 goto fail2;
 683
 684         return (0);
 685
 686 fail2:
 687         EFSYS_PROBE(fail2);
 688 fail1:
 689         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 690
 691         return (rc);
 692 }
 693 #endif /* EFSYS_OPT_RX_SCALE */
 694
 695 #if EFSYS_OPT_RX_SCALE
 696         __checkReturn   efx_rc_t
 697 ef10_rx_scale_tbl_set(
 698         __in            efx_nic_t *enp,
 699         __in            uint32_t rss_context,
 700         __in_ecount(n)  unsigned int *table,
 701         __in            size_t n)
 702 {
 703         efx_rc_t rc;
 704
 705
 706         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 707                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 708                         rc = ENOTSUP;
 709                         goto fail1;
 710                 }
 711                 rss_context = enp->en_rss_context;
 712         }
 713
 714         if ((rc = efx_mcdi_rss_context_set_table(enp,
 715                     rss_context, table, n)) != 0)
 716                 goto fail2;
 717
 718         return (0);
 719
 720 fail2:
 721         EFSYS_PROBE(fail2);
 722 fail1:
 723         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 724
 725         return (rc);
 726 }
 727 #endif /* EFSYS_OPT_RX_SCALE */
 728
 729
 730 /*
 731  * EF10 RX pseudo-header
 732  * ---------------------
 733  *
 734  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 735  *
 736  *  +00: Toeplitz hash value.
 737  *       (32bit little-endian)
 738  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 739  *       (16bit big-endian)
 740  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 741  *       (16bit big-endian)
 742  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 743  *       (16bit little-endian)
 744  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 745  *       (32bit little-endian)
 746  *
 747  * See "The RX Pseudo-header" in SF-109306-TC.
 748  */
 749
 750         __checkReturn   efx_rc_t
 751 ef10_rx_prefix_pktlen(
 752         __in            efx_nic_t *enp,
 753         __in            uint8_t *buffer,
 754         __out           uint16_t *lengthp)
 755 {
 756         _NOTE(ARGUNUSED(enp))
 757
 758         /*
 759          * The RX pseudo-header contains the packet length, excluding the
 760          * pseudo-header. If the hardware receive datapath was operating in
 761          * cut-through mode then the length in the RX pseudo-header will be
 762          * zero, and the packet length must be obtained from the DMA length
 763          * reported in the RX event.
 764          */
 765         *lengthp = buffer[8] | (buffer[9] << 8);
 766         return (0);
 767 }
 768
 769 #if EFSYS_OPT_RX_SCALE
 770         __checkReturn   uint32_t
 771 ef10_rx_prefix_hash(
 772         __in            efx_nic_t *enp,
 773         __in            efx_rx_hash_alg_t func,
 774         __in            uint8_t *buffer)
 775 {
 776         _NOTE(ARGUNUSED(enp))
 777
 778         switch (func) {
 779         case EFX_RX_HASHALG_PACKED_STREAM:
 780         case EFX_RX_HASHALG_TOEPLITZ:
 781                 return (buffer[0] |
 782                     (buffer[1] << 8) |
 783                     (buffer[2] << 16) |
 784                     (buffer[3] << 24));
 785
 786         default:
 787                 EFSYS_ASSERT(0);
 788                 return (0);
 789         }
 790 }
 791 #endif /* EFSYS_OPT_RX_SCALE */
 792
 793 #if EFSYS_OPT_RX_PACKED_STREAM
 794 /*
 795  * Fake length for RXQ descriptors in packed stream mode
 796  * to make hardware happy
 797  */
 798 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 799 #endif
 800
 801                                 void
 802 ef10_rx_qpost(
 803         __in                    efx_rxq_t *erp,
 804         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 805         __in                    size_t size,
 806         __in                    unsigned int ndescs,
 807         __in                    unsigned int completed,
 808         __in                    unsigned int added)
 809 {
 810         efx_qword_t qword;
 811         unsigned int i;
 812         unsigned int offset;
 813         unsigned int id;
 814
 815         _NOTE(ARGUNUSED(completed))
 816
 817 #if EFSYS_OPT_RX_PACKED_STREAM
 818         /*
 819          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 820          * and equal to 0 after applying mask. Hardware does not like it.
 821          */
 822         if (erp->er_ev_qstate->eers_rx_packed_stream)
 823                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 824 #endif
 825
 826         /* The client driver must not overfill the queue */
 827         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 828             EFX_RXQ_LIMIT(erp->er_mask + 1));
 829
 830         id = added & (erp->er_mask);
 831         for (i = 0; i < ndescs; i++) {
 832                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 833                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 834                     size_t, size);
 835
 836                 EFX_POPULATE_QWORD_3(qword,
 837                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 838                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 839                     (uint32_t)(addrp[i] & 0xffffffff),
 840                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 841                     (uint32_t)(addrp[i] >> 32));
 842
 843                 offset = id * sizeof (efx_qword_t);
 844                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 845
 846                 id = (id + 1) & (erp->er_mask);
 847         }
 848 }
 849
 850                         void
 851 ef10_rx_qpush(
 852         __in    efx_rxq_t *erp,
 853         __in    unsigned int added,
 854         __inout unsigned int *pushedp)
 855 {
 856         efx_nic_t *enp = erp->er_enp;
 857         unsigned int pushed = *pushedp;
 858         uint32_t wptr;
 859         efx_dword_t dword;
 860
 861         /* Hardware has alignment restriction for WPTR */
 862         wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN);
 863         if (pushed == wptr)
 864                 return;
 865
 866         *pushedp = wptr;
 867
 868         /* Push the populated descriptors out */
 869         wptr &= erp->er_mask;
 870
 871         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 872
 873         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 874         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 875             wptr, pushed & erp->er_mask);
 876         EFSYS_PIO_WRITE_BARRIER();
 877         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 878             erp->er_index, &dword, B_FALSE);
 879 }
 880
 881 #if EFSYS_OPT_RX_PACKED_STREAM
 882
 883                         void
 884 ef10_rx_qpush_ps_credits(
 885         __in            efx_rxq_t *erp)
 886 {
 887         efx_nic_t *enp = erp->er_enp;
 888         efx_dword_t dword;
 889         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 890         uint32_t credits;
 891
 892         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 893
 894         if (rxq_state->eers_rx_packed_stream_credits == 0)
 895                 return;
 896
 897         /*
 898          * It is a bug if we think that FW has utilized more
 899          * credits than it is allowed to have (maximum). However,
 900          * make sure that we do not credit more than maximum anyway.
 901          */
 902         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 903             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 904         EFX_POPULATE_DWORD_3(dword,
 905             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 906             ERF_DZ_RX_DESC_MAGIC_CMD,
 907             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 908             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 909         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 910             erp->er_index, &dword, B_FALSE);
 911
 912         rxq_state->eers_rx_packed_stream_credits = 0;
 913 }
 914
 915 /*
 916  * In accordance with SF-112241-TC the received data has the following layout:
 917  *  - 8 byte pseudo-header which consist of:
 918  *    - 4 byte little-endian timestamp
 919  *    - 2 byte little-endian captured length in bytes
 920  *    - 2 byte little-endian original packet length in bytes
 921  *  - captured packet bytes
 922  *  - optional padding to align to 64 bytes boundary
 923  *  - 64 bytes scratch space for the host software
 924  */
 925         __checkReturn   uint8_t *
 926 ef10_rx_qps_packet_info(
 927         __in            efx_rxq_t *erp,
 928         __in            uint8_t *buffer,
 929         __in            uint32_t buffer_length,
 930         __in            uint32_t current_offset,
 931         __out           uint16_t *lengthp,
 932         __out           uint32_t *next_offsetp,
 933         __out           uint32_t *timestamp)
 934 {
 935         uint16_t buf_len;
 936         uint8_t *pkt_start;
 937         efx_qword_t *qwordp;
 938         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 939
 940         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 941
 942         buffer += current_offset;
 943         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 944
 945         qwordp = (efx_qword_t *)buffer;
 946         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 947         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 948         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 949
 950         buf_len = EFX_P2ROUNDUP(uint16_t,
 951             buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 952             EFX_RX_PACKED_STREAM_ALIGNMENT);
 953         *next_offsetp =
 954             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 955
 956         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 957         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 958
 959         if ((*next_offsetp ^ current_offset) &
 960             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 961                 rxq_state->eers_rx_packed_stream_credits++;
 962
 963         return (pkt_start);
 964 }
 965
 966
 967 #endif
 968
 969         __checkReturn   efx_rc_t
 970 ef10_rx_qflush(
 971         __in    efx_rxq_t *erp)
 972 {
 973         efx_nic_t *enp = erp->er_enp;
 974         efx_rc_t rc;
 975
 976         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 977                 goto fail1;
 978
 979         return (0);
 980
 981 fail1:
 982         /*
 983          * EALREADY is not an error, but indicates that the MC has rebooted and
 984          * that the RXQ has already been destroyed. Callers need to know that
 985          * the RXQ flush has completed to avoid waiting until timeout for a
 986          * flush done event that will not be delivered.
 987          */
 988         if (rc != EALREADY)
 989                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 990
 991         return (rc);
 992 }
 993
 994                 void
 995 ef10_rx_qenable(
 996         __in    efx_rxq_t *erp)
 997 {
 998         /* FIXME */
 999         _NOTE(ARGUNUSED(erp))
1000         /* FIXME */
1001 }
1002
1003         __checkReturn   efx_rc_t
1004 ef10_rx_qcreate(
1005         __in            efx_nic_t *enp,
1006         __in            unsigned int index,
1007         __in            unsigned int label,
1008         __in            efx_rxq_type_t type,
1009         __in_opt        const efx_rxq_type_data_t *type_data,
1010         __in            efsys_mem_t *esmp,
1011         __in            size_t ndescs,
1012         __in            uint32_t id,
1013         __in            unsigned int flags,
1014         __in            efx_evq_t *eep,
1015         __in            efx_rxq_t *erp)
1016 {
1017         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1018         efx_rc_t rc;
1019         boolean_t disable_scatter;
1020         boolean_t want_inner_classes;
1021         unsigned int ps_buf_size;
1022         uint32_t es_bufs_per_desc = 0;
1023         uint32_t es_max_dma_len = 0;
1024         uint32_t es_buf_stride = 0;
1025         uint32_t hol_block_timeout = 0;
1026
1027         _NOTE(ARGUNUSED(id, erp))
1028
1029         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
1030         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1031         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1032
1033         if (index >= encp->enc_rxq_limit) {
1034                 rc = EINVAL;
1035                 goto fail1;
1036         }
1037
1038         switch (type) {
1039         case EFX_RXQ_TYPE_DEFAULT:
1040                 if (type_data == NULL) {
1041                         rc = EINVAL;
1042                         goto fail2;
1043                 }
1044                 erp->er_buf_size = type_data->ertd_default.ed_buf_size;
1045                 ps_buf_size = 0;
1046                 break;
1047 #if EFSYS_OPT_RX_PACKED_STREAM
1048         case EFX_RXQ_TYPE_PACKED_STREAM:
1049                 if (type_data == NULL) {
1050                         rc = EINVAL;
1051                         goto fail3;
1052                 }
1053                 switch (type_data->ertd_packed_stream.eps_buf_size) {
1054                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
1055                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
1056                         break;
1057                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
1058                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
1059                         break;
1060                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1061                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1062                         break;
1063                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1064                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1065                         break;
1066                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1067                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1068                         break;
1069                 default:
1070                         rc = ENOTSUP;
1071                         goto fail4;
1072                 }
1073                 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size;
1074                 break;
1075 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1076 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1077         case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
1078                 if (type_data == NULL) {
1079                         rc = EINVAL;
1080                         goto fail5;
1081                 }
1082                 ps_buf_size = 0;
1083                 es_bufs_per_desc =
1084                     type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
1085                 es_max_dma_len =
1086                     type_data->ertd_es_super_buffer.eessb_max_dma_len;
1087                 es_buf_stride =
1088                     type_data->ertd_es_super_buffer.eessb_buf_stride;
1089                 hol_block_timeout =
1090                     type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
1091                 break;
1092 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1093         default:
1094                 rc = ENOTSUP;
1095                 goto fail6;
1096         }
1097
1098 #if EFSYS_OPT_RX_PACKED_STREAM
1099         if (ps_buf_size != 0) {
1100                 /* Check if datapath firmware supports packed stream mode */
1101                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1102                         rc = ENOTSUP;
1103                         goto fail7;
1104                 }
1105                 /* Check if packed stream allows configurable buffer sizes */
1106                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1107                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1108                         rc = ENOTSUP;
1109                         goto fail8;
1110                 }
1111         }
1112 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1113         EFSYS_ASSERT(ps_buf_size == 0);
1114 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1115
1116 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1117         if (es_bufs_per_desc > 0) {
1118                 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
1119                         rc = ENOTSUP;
1120                         goto fail9;
1121                 }
1122                 if (!IS_P2ALIGNED(es_max_dma_len,
1123                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1124                         rc = EINVAL;
1125                         goto fail10;
1126                 }
1127                 if (!IS_P2ALIGNED(es_buf_stride,
1128                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
1129                         rc = EINVAL;
1130                         goto fail11;
1131                 }
1132         }
1133 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1134         EFSYS_ASSERT(es_bufs_per_desc == 0);
1135 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1136
1137         /* Scatter can only be disabled if the firmware supports doing so */
1138         if (flags & EFX_RXQ_FLAG_SCATTER)
1139                 disable_scatter = B_FALSE;
1140         else
1141                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1142
1143         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1144                 want_inner_classes = B_TRUE;
1145         else
1146                 want_inner_classes = B_FALSE;
1147
1148         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index,
1149                     esmp, disable_scatter, want_inner_classes, erp->er_buf_size,
1150                     ps_buf_size, es_bufs_per_desc, es_max_dma_len,
1151                     es_buf_stride, hol_block_timeout)) != 0)
1152                 goto fail12;
1153
1154         erp->er_eep = eep;
1155         erp->er_label = label;
1156
1157         ef10_ev_rxlabel_init(eep, erp, label, type);
1158
1159         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1160
1161         return (0);
1162
1163 fail12:
1164         EFSYS_PROBE(fail12);
1165 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1166 fail11:
1167         EFSYS_PROBE(fail11);
1168 fail10:
1169         EFSYS_PROBE(fail10);
1170 fail9:
1171         EFSYS_PROBE(fail9);
1172 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1173 #if EFSYS_OPT_RX_PACKED_STREAM
1174 fail8:
1175         EFSYS_PROBE(fail8);
1176 fail7:
1177         EFSYS_PROBE(fail7);
1178 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1179 fail6:
1180         EFSYS_PROBE(fail6);
1181 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1182 fail5:
1183         EFSYS_PROBE(fail5);
1184 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1185 #if EFSYS_OPT_RX_PACKED_STREAM
1186 fail4:
1187         EFSYS_PROBE(fail4);
1188 fail3:
1189         EFSYS_PROBE(fail3);
1190 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1191 fail2:
1192         EFSYS_PROBE(fail2);
1193 fail1:
1194         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1195
1196         return (rc);
1197 }
1198
1199                 void
1200 ef10_rx_qdestroy(
1201         __in    efx_rxq_t *erp)
1202 {
1203         efx_nic_t *enp = erp->er_enp;
1204         efx_evq_t *eep = erp->er_eep;
1205         unsigned int label = erp->er_label;
1206
1207         ef10_ev_rxlabel_fini(eep, label);
1208
1209         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1210         --enp->en_rx_qcount;
1211
1212         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1213 }
1214
1215                 void
1216 ef10_rx_fini(
1217         __in    efx_nic_t *enp)
1218 {
1219 #if EFSYS_OPT_RX_SCALE
1220         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1221                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1222         enp->en_rss_context = 0;
1223         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1224 #else
1225         _NOTE(ARGUNUSED(enp))
1226 #endif /* EFSYS_OPT_RX_SCALE */
1227 }
1228
1229 #endif /* EFX_OPTS_EF10() */