drivers/net/sfc/base/ef10_rx.c

   1 /*
   2  * Copyright (c) 2012-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  36
  37
  38 static  __checkReturn   efx_rc_t
  39 efx_mcdi_init_rxq(
  40         __in            efx_nic_t *enp,
  41         __in            uint32_t size,
  42         __in            uint32_t target_evq,
  43         __in            uint32_t label,
  44         __in            uint32_t instance,
  45         __in            efsys_mem_t *esmp,
  46         __in            boolean_t disable_scatter,
  47         __in            uint32_t ps_bufsize)
  48 {
  49         efx_mcdi_req_t req;
  50         uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN,
  51                             MC_CMD_INIT_RXQ_EXT_OUT_LEN)];
  52         int npages = EFX_RXQ_NBUFS(size);
  53         int i;
  54         efx_qword_t *dma_addr;
  55         uint64_t addr;
  56         efx_rc_t rc;
  57         uint32_t dma_mode;
  58
  59         /* If this changes, then the payload size might need to change. */
  60         EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0);
  61         EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
  62
  63         if (ps_bufsize > 0)
  64                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  65         else
  66                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  67
  68         (void) memset(payload, 0, sizeof (payload));
  69         req.emr_cmd = MC_CMD_INIT_RXQ;
  70         req.emr_in_buf = payload;
  71         req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
  72         req.emr_out_buf = payload;
  73         req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
  74
  75         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, size);
  76         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  77         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  78         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
  79         MCDI_IN_POPULATE_DWORD_8(req, INIT_RXQ_EXT_IN_FLAGS,
  80             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
  81             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
  82             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
  83             INIT_RXQ_EXT_IN_CRC_MODE, 0,
  84             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
  85             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
  86             INIT_RXQ_EXT_IN_DMA_MODE,
  87             dma_mode,
  88             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize);
  89         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
  90         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
  91
  92         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
  93         addr = EFSYS_MEM_ADDR(esmp);
  94
  95         for (i = 0; i < npages; i++) {
  96                 EFX_POPULATE_QWORD_2(*dma_addr,
  97                     EFX_DWORD_1, (uint32_t)(addr >> 32),
  98                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
  99
 100                 dma_addr++;
 101                 addr += EFX_BUF_SIZE;
 102         }
 103
 104         efx_mcdi_execute(enp, &req);
 105
 106         if (req.emr_rc != 0) {
 107                 rc = req.emr_rc;
 108                 goto fail1;
 109         }
 110
 111         return (0);
 112
 113 fail1:
 114         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 115
 116         return (rc);
 117 }
 118
 119 static  __checkReturn   efx_rc_t
 120 efx_mcdi_fini_rxq(
 121         __in            efx_nic_t *enp,
 122         __in            uint32_t instance)
 123 {
 124         efx_mcdi_req_t req;
 125         uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
 126                             MC_CMD_FINI_RXQ_OUT_LEN)];
 127         efx_rc_t rc;
 128
 129         (void) memset(payload, 0, sizeof (payload));
 130         req.emr_cmd = MC_CMD_FINI_RXQ;
 131         req.emr_in_buf = payload;
 132         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 133         req.emr_out_buf = payload;
 134         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 135
 136         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 137
 138         efx_mcdi_execute_quiet(enp, &req);
 139
 140         if (req.emr_rc != 0) {
 141                 rc = req.emr_rc;
 142                 goto fail1;
 143         }
 144
 145         return (0);
 146
 147 fail1:
 148         /*
 149          * EALREADY is not an error, but indicates that the MC has rebooted and
 150          * that the RXQ has already been destroyed.
 151          */
 152         if (rc != EALREADY)
 153                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 154
 155         return (rc);
 156 }
 157
 158 #if EFSYS_OPT_RX_SCALE
 159 static  __checkReturn   efx_rc_t
 160 efx_mcdi_rss_context_alloc(
 161         __in            efx_nic_t *enp,
 162         __in            efx_rx_scale_context_type_t type,
 163         __in            uint32_t num_queues,
 164         __out           uint32_t *rss_contextp)
 165 {
 166         efx_mcdi_req_t req;
 167         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 168                             MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
 169         uint32_t rss_context;
 170         uint32_t context_type;
 171         efx_rc_t rc;
 172
 173         if (num_queues > EFX_MAXRSS) {
 174                 rc = EINVAL;
 175                 goto fail1;
 176         }
 177
 178         switch (type) {
 179         case EFX_RX_SCALE_EXCLUSIVE:
 180                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 181                 break;
 182         case EFX_RX_SCALE_SHARED:
 183                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 184                 break;
 185         default:
 186                 rc = EINVAL;
 187                 goto fail2;
 188         }
 189
 190         (void) memset(payload, 0, sizeof (payload));
 191         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 192         req.emr_in_buf = payload;
 193         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 194         req.emr_out_buf = payload;
 195         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 196
 197         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 198             EVB_PORT_ID_ASSIGNED);
 199         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 200         /* NUM_QUEUES is only used to validate indirection table offsets */
 201         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 202
 203         efx_mcdi_execute(enp, &req);
 204
 205         if (req.emr_rc != 0) {
 206                 rc = req.emr_rc;
 207                 goto fail3;
 208         }
 209
 210         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 211                 rc = EMSGSIZE;
 212                 goto fail4;
 213         }
 214
 215         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 216         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 217                 rc = ENOENT;
 218                 goto fail5;
 219         }
 220
 221         *rss_contextp = rss_context;
 222
 223         return (0);
 224
 225 fail5:
 226         EFSYS_PROBE(fail5);
 227 fail4:
 228         EFSYS_PROBE(fail4);
 229 fail3:
 230         EFSYS_PROBE(fail3);
 231 fail2:
 232         EFSYS_PROBE(fail2);
 233 fail1:
 234         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 235
 236         return (rc);
 237 }
 238 #endif /* EFSYS_OPT_RX_SCALE */
 239
 240 #if EFSYS_OPT_RX_SCALE
 241 static                  efx_rc_t
 242 efx_mcdi_rss_context_free(
 243         __in            efx_nic_t *enp,
 244         __in            uint32_t rss_context)
 245 {
 246         efx_mcdi_req_t req;
 247         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 248                             MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
 249         efx_rc_t rc;
 250
 251         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 252                 rc = EINVAL;
 253                 goto fail1;
 254         }
 255
 256         (void) memset(payload, 0, sizeof (payload));
 257         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 258         req.emr_in_buf = payload;
 259         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 260         req.emr_out_buf = payload;
 261         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 262
 263         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 264
 265         efx_mcdi_execute_quiet(enp, &req);
 266
 267         if (req.emr_rc != 0) {
 268                 rc = req.emr_rc;
 269                 goto fail2;
 270         }
 271
 272         return (0);
 273
 274 fail2:
 275         EFSYS_PROBE(fail2);
 276 fail1:
 277         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 278
 279         return (rc);
 280 }
 281 #endif /* EFSYS_OPT_RX_SCALE */
 282
 283 #if EFSYS_OPT_RX_SCALE
 284 static                  efx_rc_t
 285 efx_mcdi_rss_context_set_flags(
 286         __in            efx_nic_t *enp,
 287         __in            uint32_t rss_context,
 288         __in            efx_rx_hash_type_t type)
 289 {
 290         efx_mcdi_req_t req;
 291         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 292                             MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
 293         efx_rc_t rc;
 294
 295         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 296                 rc = EINVAL;
 297                 goto fail1;
 298         }
 299
 300         (void) memset(payload, 0, sizeof (payload));
 301         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 302         req.emr_in_buf = payload;
 303         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 304         req.emr_out_buf = payload;
 305         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 306
 307         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 308             rss_context);
 309
 310         MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 311             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 312             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 313             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 314             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 315             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 316             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 317             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 318             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0);
 319
 320         efx_mcdi_execute(enp, &req);
 321
 322         if (req.emr_rc != 0) {
 323                 rc = req.emr_rc;
 324                 goto fail2;
 325         }
 326
 327         return (0);
 328
 329 fail2:
 330         EFSYS_PROBE(fail2);
 331 fail1:
 332         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 333
 334         return (rc);
 335 }
 336 #endif /* EFSYS_OPT_RX_SCALE */
 337
 338 #if EFSYS_OPT_RX_SCALE
 339 static                  efx_rc_t
 340 efx_mcdi_rss_context_set_key(
 341         __in            efx_nic_t *enp,
 342         __in            uint32_t rss_context,
 343         __in_ecount(n)  uint8_t *key,
 344         __in            size_t n)
 345 {
 346         efx_mcdi_req_t req;
 347         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 348                             MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
 349         efx_rc_t rc;
 350
 351         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 352                 rc = EINVAL;
 353                 goto fail1;
 354         }
 355
 356         (void) memset(payload, 0, sizeof (payload));
 357         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 358         req.emr_in_buf = payload;
 359         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 360         req.emr_out_buf = payload;
 361         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 362
 363         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 364             rss_context);
 365
 366         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 367         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 368                 rc = EINVAL;
 369                 goto fail2;
 370         }
 371
 372         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 373             key, n);
 374
 375         efx_mcdi_execute(enp, &req);
 376
 377         if (req.emr_rc != 0) {
 378                 rc = req.emr_rc;
 379                 goto fail3;
 380         }
 381
 382         return (0);
 383
 384 fail3:
 385         EFSYS_PROBE(fail3);
 386 fail2:
 387         EFSYS_PROBE(fail2);
 388 fail1:
 389         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 390
 391         return (rc);
 392 }
 393 #endif /* EFSYS_OPT_RX_SCALE */
 394
 395 #if EFSYS_OPT_RX_SCALE
 396 static                  efx_rc_t
 397 efx_mcdi_rss_context_set_table(
 398         __in            efx_nic_t *enp,
 399         __in            uint32_t rss_context,
 400         __in_ecount(n)  unsigned int *table,
 401         __in            size_t n)
 402 {
 403         efx_mcdi_req_t req;
 404         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 405                             MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
 406         uint8_t *req_table;
 407         int i, rc;
 408
 409         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 410                 rc = EINVAL;
 411                 goto fail1;
 412         }
 413
 414         (void) memset(payload, 0, sizeof (payload));
 415         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 416         req.emr_in_buf = payload;
 417         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 418         req.emr_out_buf = payload;
 419         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 420
 421         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 422             rss_context);
 423
 424         req_table =
 425             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 426
 427         for (i = 0;
 428             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 429             i++) {
 430                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 431         }
 432
 433         efx_mcdi_execute(enp, &req);
 434
 435         if (req.emr_rc != 0) {
 436                 rc = req.emr_rc;
 437                 goto fail2;
 438         }
 439
 440         return (0);
 441
 442 fail2:
 443         EFSYS_PROBE(fail2);
 444 fail1:
 445         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 446
 447         return (rc);
 448 }
 449 #endif /* EFSYS_OPT_RX_SCALE */
 450
 451
 452         __checkReturn   efx_rc_t
 453 ef10_rx_init(
 454         __in            efx_nic_t *enp)
 455 {
 456 #if EFSYS_OPT_RX_SCALE
 457
 458         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 459                 &enp->en_rss_context) == 0) {
 460                 /*
 461                  * Allocated an exclusive RSS context, which allows both the
 462                  * indirection table and key to be modified.
 463                  */
 464                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 465                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 466         } else {
 467                 /*
 468                  * Failed to allocate an exclusive RSS context. Continue
 469                  * operation without support for RSS. The pseudo-header in
 470                  * received packets will not contain a Toeplitz hash value.
 471                  */
 472                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 473                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 474         }
 475
 476 #endif /* EFSYS_OPT_RX_SCALE */
 477
 478         return (0);
 479 }
 480
 481 #if EFSYS_OPT_RX_SCATTER
 482         __checkReturn   efx_rc_t
 483 ef10_rx_scatter_enable(
 484         __in            efx_nic_t *enp,
 485         __in            unsigned int buf_size)
 486 {
 487         _NOTE(ARGUNUSED(enp, buf_size))
 488         return (0);
 489 }
 490 #endif  /* EFSYS_OPT_RX_SCATTER */
 491
 492 #if EFSYS_OPT_RX_SCALE
 493         __checkReturn   efx_rc_t
 494 ef10_rx_scale_context_alloc(
 495         __in            efx_nic_t *enp,
 496         __in            efx_rx_scale_context_type_t type,
 497         __in            uint32_t num_queues,
 498         __out           uint32_t *rss_contextp)
 499 {
 500         efx_rc_t rc;
 501
 502         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 503         if (rc != 0)
 504                 goto fail1;
 505
 506         return (0);
 507
 508 fail1:
 509         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 510         return (rc);
 511 }
 512 #endif /* EFSYS_OPT_RX_SCALE */
 513
 514 #if EFSYS_OPT_RX_SCALE
 515         __checkReturn   efx_rc_t
 516 ef10_rx_scale_context_free(
 517         __in            efx_nic_t *enp,
 518         __in            uint32_t rss_context)
 519 {
 520         efx_rc_t rc;
 521
 522         rc = efx_mcdi_rss_context_free(enp, rss_context);
 523         if (rc != 0)
 524                 goto fail1;
 525
 526         return (0);
 527
 528 fail1:
 529         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 530         return (rc);
 531 }
 532 #endif /* EFSYS_OPT_RX_SCALE */
 533
 534 #if EFSYS_OPT_RX_SCALE
 535         __checkReturn   efx_rc_t
 536 ef10_rx_scale_mode_set(
 537         __in            efx_nic_t *enp,
 538         __in            uint32_t rss_context,
 539         __in            efx_rx_hash_alg_t alg,
 540         __in            efx_rx_hash_type_t type,
 541         __in            boolean_t insert)
 542 {
 543         efx_rc_t rc;
 544
 545         EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
 546         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 547
 548         if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
 549                 rc = EINVAL;
 550                 goto fail1;
 551         }
 552
 553         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 554                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 555                         rc = ENOTSUP;
 556                         goto fail2;
 557                 }
 558                 rss_context = enp->en_rss_context;
 559         }
 560
 561         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 562                     rss_context, type)) != 0)
 563                 goto fail3;
 564
 565         return (0);
 566
 567 fail3:
 568         EFSYS_PROBE(fail3);
 569 fail2:
 570         EFSYS_PROBE(fail2);
 571 fail1:
 572         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 573
 574         return (rc);
 575 }
 576 #endif /* EFSYS_OPT_RX_SCALE */
 577
 578 #if EFSYS_OPT_RX_SCALE
 579         __checkReturn   efx_rc_t
 580 ef10_rx_scale_key_set(
 581         __in            efx_nic_t *enp,
 582         __in            uint32_t rss_context,
 583         __in_ecount(n)  uint8_t *key,
 584         __in            size_t n)
 585 {
 586         efx_rc_t rc;
 587
 588         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 589             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 590
 591         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 592                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 593                         rc = ENOTSUP;
 594                         goto fail1;
 595                 }
 596                 rss_context = enp->en_rss_context;
 597         }
 598
 599         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 600                 goto fail2;
 601
 602         return (0);
 603
 604 fail2:
 605         EFSYS_PROBE(fail2);
 606 fail1:
 607         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 608
 609         return (rc);
 610 }
 611 #endif /* EFSYS_OPT_RX_SCALE */
 612
 613 #if EFSYS_OPT_RX_SCALE
 614         __checkReturn   efx_rc_t
 615 ef10_rx_scale_tbl_set(
 616         __in            efx_nic_t *enp,
 617         __in            uint32_t rss_context,
 618         __in_ecount(n)  unsigned int *table,
 619         __in            size_t n)
 620 {
 621         efx_rc_t rc;
 622
 623
 624         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 625                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 626                         rc = ENOTSUP;
 627                         goto fail1;
 628                 }
 629                 rss_context = enp->en_rss_context;
 630         }
 631
 632         if ((rc = efx_mcdi_rss_context_set_table(enp,
 633                     rss_context, table, n)) != 0)
 634                 goto fail2;
 635
 636         return (0);
 637
 638 fail2:
 639         EFSYS_PROBE(fail2);
 640 fail1:
 641         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 642
 643         return (rc);
 644 }
 645 #endif /* EFSYS_OPT_RX_SCALE */
 646
 647
 648 /*
 649  * EF10 RX pseudo-header
 650  * ---------------------
 651  *
 652  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 653  *
 654  *  +00: Toeplitz hash value.
 655  *       (32bit little-endian)
 656  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 657  *       (16bit big-endian)
 658  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 659  *       (16bit big-endian)
 660  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 661  *       (16bit little-endian)
 662  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 663  *       (32bit little-endian)
 664  *
 665  * See "The RX Pseudo-header" in SF-109306-TC.
 666  */
 667
 668         __checkReturn   efx_rc_t
 669 ef10_rx_prefix_pktlen(
 670         __in            efx_nic_t *enp,
 671         __in            uint8_t *buffer,
 672         __out           uint16_t *lengthp)
 673 {
 674         _NOTE(ARGUNUSED(enp))
 675
 676         /*
 677          * The RX pseudo-header contains the packet length, excluding the
 678          * pseudo-header. If the hardware receive datapath was operating in
 679          * cut-through mode then the length in the RX pseudo-header will be
 680          * zero, and the packet length must be obtained from the DMA length
 681          * reported in the RX event.
 682          */
 683         *lengthp = buffer[8] | (buffer[9] << 8);
 684         return (0);
 685 }
 686
 687 #if EFSYS_OPT_RX_SCALE
 688         __checkReturn   uint32_t
 689 ef10_rx_prefix_hash(
 690         __in            efx_nic_t *enp,
 691         __in            efx_rx_hash_alg_t func,
 692         __in            uint8_t *buffer)
 693 {
 694         _NOTE(ARGUNUSED(enp))
 695
 696         switch (func) {
 697         case EFX_RX_HASHALG_TOEPLITZ:
 698                 return (buffer[0] |
 699                     (buffer[1] << 8) |
 700                     (buffer[2] << 16) |
 701                     (buffer[3] << 24));
 702
 703         default:
 704                 EFSYS_ASSERT(0);
 705                 return (0);
 706         }
 707 }
 708 #endif /* EFSYS_OPT_RX_SCALE */
 709
 710                         void
 711 ef10_rx_qpost(
 712         __in            efx_rxq_t *erp,
 713         __in_ecount(n)  efsys_dma_addr_t *addrp,
 714         __in            size_t size,
 715         __in            unsigned int n,
 716         __in            unsigned int completed,
 717         __in            unsigned int added)
 718 {
 719         efx_qword_t qword;
 720         unsigned int i;
 721         unsigned int offset;
 722         unsigned int id;
 723
 724         /* The client driver must not overfill the queue */
 725         EFSYS_ASSERT3U(added - completed + n, <=,
 726             EFX_RXQ_LIMIT(erp->er_mask + 1));
 727
 728         id = added & (erp->er_mask);
 729         for (i = 0; i < n; i++) {
 730                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 731                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 732                     size_t, size);
 733
 734                 EFX_POPULATE_QWORD_3(qword,
 735                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 736                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 737                     (uint32_t)(addrp[i] & 0xffffffff),
 738                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 739                     (uint32_t)(addrp[i] >> 32));
 740
 741                 offset = id * sizeof (efx_qword_t);
 742                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 743
 744                 id = (id + 1) & (erp->er_mask);
 745         }
 746 }
 747
 748                         void
 749 ef10_rx_qpush(
 750         __in    efx_rxq_t *erp,
 751         __in    unsigned int added,
 752         __inout unsigned int *pushedp)
 753 {
 754         efx_nic_t *enp = erp->er_enp;
 755         unsigned int pushed = *pushedp;
 756         uint32_t wptr;
 757         efx_dword_t dword;
 758
 759         /* Hardware has alignment restriction for WPTR */
 760         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 761         if (pushed == wptr)
 762                 return;
 763
 764         *pushedp = wptr;
 765
 766         /* Push the populated descriptors out */
 767         wptr &= erp->er_mask;
 768
 769         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 770
 771         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 772         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 773             wptr, pushed & erp->er_mask);
 774         EFSYS_PIO_WRITE_BARRIER();
 775         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 776                             erp->er_index, &dword, B_FALSE);
 777 }
 778
 779 #if EFSYS_OPT_RX_PACKED_STREAM
 780
 781                         void
 782 ef10_rx_qps_update_credits(
 783         __in    efx_rxq_t *erp)
 784 {
 785         efx_nic_t *enp = erp->er_enp;
 786         efx_dword_t dword;
 787         efx_evq_rxq_state_t *rxq_state =
 788                 &erp->er_eep->ee_rxq_state[erp->er_label];
 789
 790         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 791
 792         if (rxq_state->eers_rx_packed_stream_credits == 0)
 793                 return;
 794
 795         EFX_POPULATE_DWORD_3(dword,
 796             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 797             ERF_DZ_RX_DESC_MAGIC_CMD,
 798             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 799             ERF_DZ_RX_DESC_MAGIC_DATA,
 800             rxq_state->eers_rx_packed_stream_credits);
 801         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 802             erp->er_index, &dword, B_FALSE);
 803
 804         rxq_state->eers_rx_packed_stream_credits = 0;
 805 }
 806
 807         __checkReturn   uint8_t *
 808 ef10_rx_qps_packet_info(
 809         __in            efx_rxq_t *erp,
 810         __in            uint8_t *buffer,
 811         __in            uint32_t buffer_length,
 812         __in            uint32_t current_offset,
 813         __out           uint16_t *lengthp,
 814         __out           uint32_t *next_offsetp,
 815         __out           uint32_t *timestamp)
 816 {
 817         uint16_t buf_len;
 818         uint8_t *pkt_start;
 819         efx_qword_t *qwordp;
 820         efx_evq_rxq_state_t *rxq_state =
 821                 &erp->er_eep->ee_rxq_state[erp->er_label];
 822
 823         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 824
 825         buffer += current_offset;
 826         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 827
 828         qwordp = (efx_qword_t *)buffer;
 829         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 830         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 831         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 832
 833         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 834                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 835         *next_offsetp =
 836             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 837
 838         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 839         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 840
 841         if ((*next_offsetp ^ current_offset) &
 842             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) {
 843                 if (rxq_state->eers_rx_packed_stream_credits <
 844                     EFX_RX_PACKED_STREAM_MAX_CREDITS)
 845                         rxq_state->eers_rx_packed_stream_credits++;
 846         }
 847
 848         return (pkt_start);
 849 }
 850
 851
 852 #endif
 853
 854         __checkReturn   efx_rc_t
 855 ef10_rx_qflush(
 856         __in    efx_rxq_t *erp)
 857 {
 858         efx_nic_t *enp = erp->er_enp;
 859         efx_rc_t rc;
 860
 861         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 862                 goto fail1;
 863
 864         return (0);
 865
 866 fail1:
 867         /*
 868          * EALREADY is not an error, but indicates that the MC has rebooted and
 869          * that the RXQ has already been destroyed. Callers need to know that
 870          * the RXQ flush has completed to avoid waiting until timeout for a
 871          * flush done event that will not be delivered.
 872          */
 873         if (rc != EALREADY)
 874                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 875
 876         return (rc);
 877 }
 878
 879                 void
 880 ef10_rx_qenable(
 881         __in    efx_rxq_t *erp)
 882 {
 883         /* FIXME */
 884         _NOTE(ARGUNUSED(erp))
 885         /* FIXME */
 886 }
 887
 888         __checkReturn   efx_rc_t
 889 ef10_rx_qcreate(
 890         __in            efx_nic_t *enp,
 891         __in            unsigned int index,
 892         __in            unsigned int label,
 893         __in            efx_rxq_type_t type,
 894         __in            efsys_mem_t *esmp,
 895         __in            size_t n,
 896         __in            uint32_t id,
 897         __in            efx_evq_t *eep,
 898         __in            efx_rxq_t *erp)
 899 {
 900         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 901         efx_rc_t rc;
 902         boolean_t disable_scatter;
 903         unsigned int ps_buf_size;
 904
 905         _NOTE(ARGUNUSED(id, erp))
 906
 907         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 908         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 909         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 910
 911         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
 912         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
 913
 914         if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
 915                 rc = EINVAL;
 916                 goto fail1;
 917         }
 918         if (index >= encp->enc_rxq_limit) {
 919                 rc = EINVAL;
 920                 goto fail2;
 921         }
 922
 923         switch (type) {
 924         case EFX_RXQ_TYPE_DEFAULT:
 925         case EFX_RXQ_TYPE_SCATTER:
 926                 ps_buf_size = 0;
 927                 break;
 928 #if EFSYS_OPT_RX_PACKED_STREAM
 929         case EFX_RXQ_TYPE_PACKED_STREAM_1M:
 930                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
 931                 break;
 932         case EFX_RXQ_TYPE_PACKED_STREAM_512K:
 933                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
 934                 break;
 935         case EFX_RXQ_TYPE_PACKED_STREAM_256K:
 936                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
 937                 break;
 938         case EFX_RXQ_TYPE_PACKED_STREAM_128K:
 939                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
 940                 break;
 941         case EFX_RXQ_TYPE_PACKED_STREAM_64K:
 942                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
 943                 break;
 944 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 945         default:
 946                 rc = ENOTSUP;
 947                 goto fail3;
 948         }
 949
 950 #if EFSYS_OPT_RX_PACKED_STREAM
 951         if (ps_buf_size != 0) {
 952                 /* Check if datapath firmware supports packed stream mode */
 953                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
 954                         rc = ENOTSUP;
 955                         goto fail4;
 956                 }
 957                 /* Check if packed stream allows configurable buffer sizes */
 958                 if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) &&
 959                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
 960                         rc = ENOTSUP;
 961                         goto fail5;
 962                 }
 963         }
 964 #else /* EFSYS_OPT_RX_PACKED_STREAM */
 965         EFSYS_ASSERT(ps_buf_size == 0);
 966 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 967
 968         /* Scatter can only be disabled if the firmware supports doing so */
 969         if (type == EFX_RXQ_TYPE_SCATTER)
 970                 disable_scatter = B_FALSE;
 971         else
 972                 disable_scatter = encp->enc_rx_disable_scatter_supported;
 973
 974         if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
 975                     esmp, disable_scatter, ps_buf_size)) != 0)
 976                 goto fail6;
 977
 978         erp->er_eep = eep;
 979         erp->er_label = label;
 980
 981         ef10_ev_rxlabel_init(eep, erp, label, ps_buf_size != 0);
 982
 983         return (0);
 984
 985 fail6:
 986         EFSYS_PROBE(fail6);
 987 #if EFSYS_OPT_RX_PACKED_STREAM
 988 fail5:
 989         EFSYS_PROBE(fail5);
 990 fail4:
 991         EFSYS_PROBE(fail4);
 992 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 993 fail3:
 994         EFSYS_PROBE(fail3);
 995 fail2:
 996         EFSYS_PROBE(fail2);
 997 fail1:
 998         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 999
1000         return (rc);
1001 }
1002
1003                 void
1004 ef10_rx_qdestroy(
1005         __in    efx_rxq_t *erp)
1006 {
1007         efx_nic_t *enp = erp->er_enp;
1008         efx_evq_t *eep = erp->er_eep;
1009         unsigned int label = erp->er_label;
1010
1011         ef10_ev_rxlabel_fini(eep, label);
1012
1013         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1014         --enp->en_rx_qcount;
1015
1016         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1017 }
1018
1019                 void
1020 ef10_rx_fini(
1021         __in    efx_nic_t *enp)
1022 {
1023 #if EFSYS_OPT_RX_SCALE
1024         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1025                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1026         enp->en_rss_context = 0;
1027         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1028 #else
1029         _NOTE(ARGUNUSED(enp))
1030 #endif /* EFSYS_OPT_RX_SCALE */
1031 }
1032
1033 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */