drivers/net/sfc/base/ef10_rx.c

   1 /*
   2  * Copyright (c) 2012-2016 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  */
  30
  31 #include "efx.h"
  32 #include "efx_impl.h"
  33
  34
  35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
  36
  37
  38 static  __checkReturn   efx_rc_t
  39 efx_mcdi_init_rxq(
  40         __in            efx_nic_t *enp,
  41         __in            uint32_t ndescs,
  42         __in            uint32_t target_evq,
  43         __in            uint32_t label,
  44         __in            uint32_t instance,
  45         __in            efsys_mem_t *esmp,
  46         __in            boolean_t disable_scatter,
  47         __in            boolean_t want_inner_classes,
  48         __in            uint32_t ps_bufsize)
  49 {
  50         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  51         efx_mcdi_req_t req;
  52         uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN,
  53                             MC_CMD_INIT_RXQ_EXT_OUT_LEN)];
  54         int npages = EFX_RXQ_NBUFS(ndescs);
  55         int i;
  56         efx_qword_t *dma_addr;
  57         uint64_t addr;
  58         efx_rc_t rc;
  59         uint32_t dma_mode;
  60         boolean_t want_outer_classes;
  61
  62         EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
  63
  64         if (ps_bufsize > 0)
  65                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
  66         else
  67                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
  68
  69         if (encp->enc_tunnel_encapsulations_supported != 0 &&
  70             !want_inner_classes) {
  71                 /*
  72                  * WANT_OUTER_CLASSES can only be specified on hardware which
  73                  * supports tunnel encapsulation offloads, even though it is
  74                  * effectively the behaviour the hardware gives.
  75                  *
  76                  * Also, on hardware which does support such offloads, older
  77                  * firmware rejects the flag if the offloads are not supported
  78                  * by the current firmware variant, which means this may fail if
  79                  * the capabilities are not updated when the firmware variant
  80                  * changes. This is not an issue on newer firmware, as it was
  81                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
  82                  * specified on all firmware variants.
  83                  */
  84                 want_outer_classes = B_TRUE;
  85         } else {
  86                 want_outer_classes = B_FALSE;
  87         }
  88
  89         (void) memset(payload, 0, sizeof (payload));
  90         req.emr_cmd = MC_CMD_INIT_RXQ;
  91         req.emr_in_buf = payload;
  92         req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
  93         req.emr_out_buf = payload;
  94         req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
  95
  96         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
  97         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
  98         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
  99         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
 100         MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
 101             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
 102             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
 103             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
 104             INIT_RXQ_EXT_IN_CRC_MODE, 0,
 105             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
 106             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
 107             INIT_RXQ_EXT_IN_DMA_MODE,
 108             dma_mode,
 109             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
 110             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
 111         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
 112         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
 113
 114         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
 115         addr = EFSYS_MEM_ADDR(esmp);
 116
 117         for (i = 0; i < npages; i++) {
 118                 EFX_POPULATE_QWORD_2(*dma_addr,
 119                     EFX_DWORD_1, (uint32_t)(addr >> 32),
 120                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
 121
 122                 dma_addr++;
 123                 addr += EFX_BUF_SIZE;
 124         }
 125
 126         efx_mcdi_execute(enp, &req);
 127
 128         if (req.emr_rc != 0) {
 129                 rc = req.emr_rc;
 130                 goto fail1;
 131         }
 132
 133         return (0);
 134
 135 fail1:
 136         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 137
 138         return (rc);
 139 }
 140
 141 static  __checkReturn   efx_rc_t
 142 efx_mcdi_fini_rxq(
 143         __in            efx_nic_t *enp,
 144         __in            uint32_t instance)
 145 {
 146         efx_mcdi_req_t req;
 147         uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
 148                             MC_CMD_FINI_RXQ_OUT_LEN)];
 149         efx_rc_t rc;
 150
 151         (void) memset(payload, 0, sizeof (payload));
 152         req.emr_cmd = MC_CMD_FINI_RXQ;
 153         req.emr_in_buf = payload;
 154         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
 155         req.emr_out_buf = payload;
 156         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
 157
 158         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
 159
 160         efx_mcdi_execute_quiet(enp, &req);
 161
 162         if (req.emr_rc != 0) {
 163                 rc = req.emr_rc;
 164                 goto fail1;
 165         }
 166
 167         return (0);
 168
 169 fail1:
 170         /*
 171          * EALREADY is not an error, but indicates that the MC has rebooted and
 172          * that the RXQ has already been destroyed.
 173          */
 174         if (rc != EALREADY)
 175                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 176
 177         return (rc);
 178 }
 179
 180 #if EFSYS_OPT_RX_SCALE
 181 static  __checkReturn   efx_rc_t
 182 efx_mcdi_rss_context_alloc(
 183         __in            efx_nic_t *enp,
 184         __in            efx_rx_scale_context_type_t type,
 185         __in            uint32_t num_queues,
 186         __out           uint32_t *rss_contextp)
 187 {
 188         efx_mcdi_req_t req;
 189         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
 190                             MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
 191         uint32_t rss_context;
 192         uint32_t context_type;
 193         efx_rc_t rc;
 194
 195         if (num_queues > EFX_MAXRSS) {
 196                 rc = EINVAL;
 197                 goto fail1;
 198         }
 199
 200         switch (type) {
 201         case EFX_RX_SCALE_EXCLUSIVE:
 202                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
 203                 break;
 204         case EFX_RX_SCALE_SHARED:
 205                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
 206                 break;
 207         default:
 208                 rc = EINVAL;
 209                 goto fail2;
 210         }
 211
 212         (void) memset(payload, 0, sizeof (payload));
 213         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
 214         req.emr_in_buf = payload;
 215         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
 216         req.emr_out_buf = payload;
 217         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
 218
 219         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
 220             EVB_PORT_ID_ASSIGNED);
 221         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
 222
 223         /*
 224          * For exclusive contexts, NUM_QUEUES is only used to validate
 225          * indirection table offsets.
 226          * For shared contexts, the provided context will spread traffic over
 227          * NUM_QUEUES many queues.
 228          */
 229         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
 230
 231         efx_mcdi_execute(enp, &req);
 232
 233         if (req.emr_rc != 0) {
 234                 rc = req.emr_rc;
 235                 goto fail3;
 236         }
 237
 238         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
 239                 rc = EMSGSIZE;
 240                 goto fail4;
 241         }
 242
 243         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
 244         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 245                 rc = ENOENT;
 246                 goto fail5;
 247         }
 248
 249         *rss_contextp = rss_context;
 250
 251         return (0);
 252
 253 fail5:
 254         EFSYS_PROBE(fail5);
 255 fail4:
 256         EFSYS_PROBE(fail4);
 257 fail3:
 258         EFSYS_PROBE(fail3);
 259 fail2:
 260         EFSYS_PROBE(fail2);
 261 fail1:
 262         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 263
 264         return (rc);
 265 }
 266 #endif /* EFSYS_OPT_RX_SCALE */
 267
 268 #if EFSYS_OPT_RX_SCALE
 269 static                  efx_rc_t
 270 efx_mcdi_rss_context_free(
 271         __in            efx_nic_t *enp,
 272         __in            uint32_t rss_context)
 273 {
 274         efx_mcdi_req_t req;
 275         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 276                             MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
 277         efx_rc_t rc;
 278
 279         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 280                 rc = EINVAL;
 281                 goto fail1;
 282         }
 283
 284         (void) memset(payload, 0, sizeof (payload));
 285         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 286         req.emr_in_buf = payload;
 287         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 288         req.emr_out_buf = payload;
 289         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 290
 291         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 292
 293         efx_mcdi_execute_quiet(enp, &req);
 294
 295         if (req.emr_rc != 0) {
 296                 rc = req.emr_rc;
 297                 goto fail2;
 298         }
 299
 300         return (0);
 301
 302 fail2:
 303         EFSYS_PROBE(fail2);
 304 fail1:
 305         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 306
 307         return (rc);
 308 }
 309 #endif /* EFSYS_OPT_RX_SCALE */
 310
 311 #if EFSYS_OPT_RX_SCALE
 312 static                  efx_rc_t
 313 efx_mcdi_rss_context_set_flags(
 314         __in            efx_nic_t *enp,
 315         __in            uint32_t rss_context,
 316         __in            efx_rx_hash_type_t type)
 317 {
 318         efx_mcdi_req_t req;
 319         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 320                             MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
 321         efx_rc_t rc;
 322
 323         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 324                 rc = EINVAL;
 325                 goto fail1;
 326         }
 327
 328         (void) memset(payload, 0, sizeof (payload));
 329         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 330         req.emr_in_buf = payload;
 331         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 332         req.emr_out_buf = payload;
 333         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 334
 335         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 336             rss_context);
 337
 338         MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 339             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 340             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 341             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 342             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 343             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 344             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 345             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 346             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0);
 347
 348         efx_mcdi_execute(enp, &req);
 349
 350         if (req.emr_rc != 0) {
 351                 rc = req.emr_rc;
 352                 goto fail2;
 353         }
 354
 355         return (0);
 356
 357 fail2:
 358         EFSYS_PROBE(fail2);
 359 fail1:
 360         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 361
 362         return (rc);
 363 }
 364 #endif /* EFSYS_OPT_RX_SCALE */
 365
 366 #if EFSYS_OPT_RX_SCALE
 367 static                  efx_rc_t
 368 efx_mcdi_rss_context_set_key(
 369         __in            efx_nic_t *enp,
 370         __in            uint32_t rss_context,
 371         __in_ecount(n)  uint8_t *key,
 372         __in            size_t n)
 373 {
 374         efx_mcdi_req_t req;
 375         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 376                             MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
 377         efx_rc_t rc;
 378
 379         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 380                 rc = EINVAL;
 381                 goto fail1;
 382         }
 383
 384         (void) memset(payload, 0, sizeof (payload));
 385         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 386         req.emr_in_buf = payload;
 387         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 388         req.emr_out_buf = payload;
 389         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 390
 391         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 392             rss_context);
 393
 394         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 395         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 396                 rc = EINVAL;
 397                 goto fail2;
 398         }
 399
 400         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 401             key, n);
 402
 403         efx_mcdi_execute(enp, &req);
 404
 405         if (req.emr_rc != 0) {
 406                 rc = req.emr_rc;
 407                 goto fail3;
 408         }
 409
 410         return (0);
 411
 412 fail3:
 413         EFSYS_PROBE(fail3);
 414 fail2:
 415         EFSYS_PROBE(fail2);
 416 fail1:
 417         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 418
 419         return (rc);
 420 }
 421 #endif /* EFSYS_OPT_RX_SCALE */
 422
 423 #if EFSYS_OPT_RX_SCALE
 424 static                  efx_rc_t
 425 efx_mcdi_rss_context_set_table(
 426         __in            efx_nic_t *enp,
 427         __in            uint32_t rss_context,
 428         __in_ecount(n)  unsigned int *table,
 429         __in            size_t n)
 430 {
 431         efx_mcdi_req_t req;
 432         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 433                             MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
 434         uint8_t *req_table;
 435         int i, rc;
 436
 437         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 438                 rc = EINVAL;
 439                 goto fail1;
 440         }
 441
 442         (void) memset(payload, 0, sizeof (payload));
 443         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 444         req.emr_in_buf = payload;
 445         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 446         req.emr_out_buf = payload;
 447         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 448
 449         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 450             rss_context);
 451
 452         req_table =
 453             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 454
 455         for (i = 0;
 456             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 457             i++) {
 458                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 459         }
 460
 461         efx_mcdi_execute(enp, &req);
 462
 463         if (req.emr_rc != 0) {
 464                 rc = req.emr_rc;
 465                 goto fail2;
 466         }
 467
 468         return (0);
 469
 470 fail2:
 471         EFSYS_PROBE(fail2);
 472 fail1:
 473         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 474
 475         return (rc);
 476 }
 477 #endif /* EFSYS_OPT_RX_SCALE */
 478
 479
 480         __checkReturn   efx_rc_t
 481 ef10_rx_init(
 482         __in            efx_nic_t *enp)
 483 {
 484 #if EFSYS_OPT_RX_SCALE
 485
 486         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 487                 &enp->en_rss_context) == 0) {
 488                 /*
 489                  * Allocated an exclusive RSS context, which allows both the
 490                  * indirection table and key to be modified.
 491                  */
 492                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 493                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 494         } else {
 495                 /*
 496                  * Failed to allocate an exclusive RSS context. Continue
 497                  * operation without support for RSS. The pseudo-header in
 498                  * received packets will not contain a Toeplitz hash value.
 499                  */
 500                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 501                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 502         }
 503
 504 #endif /* EFSYS_OPT_RX_SCALE */
 505
 506         return (0);
 507 }
 508
 509 #if EFSYS_OPT_RX_SCATTER
 510         __checkReturn   efx_rc_t
 511 ef10_rx_scatter_enable(
 512         __in            efx_nic_t *enp,
 513         __in            unsigned int buf_size)
 514 {
 515         _NOTE(ARGUNUSED(enp, buf_size))
 516         return (0);
 517 }
 518 #endif  /* EFSYS_OPT_RX_SCATTER */
 519
 520 #if EFSYS_OPT_RX_SCALE
 521         __checkReturn   efx_rc_t
 522 ef10_rx_scale_context_alloc(
 523         __in            efx_nic_t *enp,
 524         __in            efx_rx_scale_context_type_t type,
 525         __in            uint32_t num_queues,
 526         __out           uint32_t *rss_contextp)
 527 {
 528         efx_rc_t rc;
 529
 530         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 531         if (rc != 0)
 532                 goto fail1;
 533
 534         return (0);
 535
 536 fail1:
 537         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 538         return (rc);
 539 }
 540 #endif /* EFSYS_OPT_RX_SCALE */
 541
 542 #if EFSYS_OPT_RX_SCALE
 543         __checkReturn   efx_rc_t
 544 ef10_rx_scale_context_free(
 545         __in            efx_nic_t *enp,
 546         __in            uint32_t rss_context)
 547 {
 548         efx_rc_t rc;
 549
 550         rc = efx_mcdi_rss_context_free(enp, rss_context);
 551         if (rc != 0)
 552                 goto fail1;
 553
 554         return (0);
 555
 556 fail1:
 557         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 558         return (rc);
 559 }
 560 #endif /* EFSYS_OPT_RX_SCALE */
 561
 562 #if EFSYS_OPT_RX_SCALE
 563         __checkReturn   efx_rc_t
 564 ef10_rx_scale_mode_set(
 565         __in            efx_nic_t *enp,
 566         __in            uint32_t rss_context,
 567         __in            efx_rx_hash_alg_t alg,
 568         __in            efx_rx_hash_type_t type,
 569         __in            boolean_t insert)
 570 {
 571         efx_rc_t rc;
 572
 573         EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
 574         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 575
 576         if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
 577                 rc = EINVAL;
 578                 goto fail1;
 579         }
 580
 581         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 582                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 583                         rc = ENOTSUP;
 584                         goto fail2;
 585                 }
 586                 rss_context = enp->en_rss_context;
 587         }
 588
 589         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 590                     rss_context, type)) != 0)
 591                 goto fail3;
 592
 593         return (0);
 594
 595 fail3:
 596         EFSYS_PROBE(fail3);
 597 fail2:
 598         EFSYS_PROBE(fail2);
 599 fail1:
 600         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 601
 602         return (rc);
 603 }
 604 #endif /* EFSYS_OPT_RX_SCALE */
 605
 606 #if EFSYS_OPT_RX_SCALE
 607         __checkReturn   efx_rc_t
 608 ef10_rx_scale_key_set(
 609         __in            efx_nic_t *enp,
 610         __in            uint32_t rss_context,
 611         __in_ecount(n)  uint8_t *key,
 612         __in            size_t n)
 613 {
 614         efx_rc_t rc;
 615
 616         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 617             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 618
 619         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 620                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 621                         rc = ENOTSUP;
 622                         goto fail1;
 623                 }
 624                 rss_context = enp->en_rss_context;
 625         }
 626
 627         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 628                 goto fail2;
 629
 630         return (0);
 631
 632 fail2:
 633         EFSYS_PROBE(fail2);
 634 fail1:
 635         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 636
 637         return (rc);
 638 }
 639 #endif /* EFSYS_OPT_RX_SCALE */
 640
 641 #if EFSYS_OPT_RX_SCALE
 642         __checkReturn   efx_rc_t
 643 ef10_rx_scale_tbl_set(
 644         __in            efx_nic_t *enp,
 645         __in            uint32_t rss_context,
 646         __in_ecount(n)  unsigned int *table,
 647         __in            size_t n)
 648 {
 649         efx_rc_t rc;
 650
 651
 652         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 653                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 654                         rc = ENOTSUP;
 655                         goto fail1;
 656                 }
 657                 rss_context = enp->en_rss_context;
 658         }
 659
 660         if ((rc = efx_mcdi_rss_context_set_table(enp,
 661                     rss_context, table, n)) != 0)
 662                 goto fail2;
 663
 664         return (0);
 665
 666 fail2:
 667         EFSYS_PROBE(fail2);
 668 fail1:
 669         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 670
 671         return (rc);
 672 }
 673 #endif /* EFSYS_OPT_RX_SCALE */
 674
 675
 676 /*
 677  * EF10 RX pseudo-header
 678  * ---------------------
 679  *
 680  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 681  *
 682  *  +00: Toeplitz hash value.
 683  *       (32bit little-endian)
 684  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 685  *       (16bit big-endian)
 686  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 687  *       (16bit big-endian)
 688  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 689  *       (16bit little-endian)
 690  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 691  *       (32bit little-endian)
 692  *
 693  * See "The RX Pseudo-header" in SF-109306-TC.
 694  */
 695
 696         __checkReturn   efx_rc_t
 697 ef10_rx_prefix_pktlen(
 698         __in            efx_nic_t *enp,
 699         __in            uint8_t *buffer,
 700         __out           uint16_t *lengthp)
 701 {
 702         _NOTE(ARGUNUSED(enp))
 703
 704         /*
 705          * The RX pseudo-header contains the packet length, excluding the
 706          * pseudo-header. If the hardware receive datapath was operating in
 707          * cut-through mode then the length in the RX pseudo-header will be
 708          * zero, and the packet length must be obtained from the DMA length
 709          * reported in the RX event.
 710          */
 711         *lengthp = buffer[8] | (buffer[9] << 8);
 712         return (0);
 713 }
 714
 715 #if EFSYS_OPT_RX_SCALE
 716         __checkReturn   uint32_t
 717 ef10_rx_prefix_hash(
 718         __in            efx_nic_t *enp,
 719         __in            efx_rx_hash_alg_t func,
 720         __in            uint8_t *buffer)
 721 {
 722         _NOTE(ARGUNUSED(enp))
 723
 724         switch (func) {
 725         case EFX_RX_HASHALG_TOEPLITZ:
 726                 return (buffer[0] |
 727                     (buffer[1] << 8) |
 728                     (buffer[2] << 16) |
 729                     (buffer[3] << 24));
 730
 731         default:
 732                 EFSYS_ASSERT(0);
 733                 return (0);
 734         }
 735 }
 736 #endif /* EFSYS_OPT_RX_SCALE */
 737
 738 #if EFSYS_OPT_RX_PACKED_STREAM
 739 /*
 740  * Fake length for RXQ descriptors in packed stream mode
 741  * to make hardware happy
 742  */
 743 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 744 #endif
 745
 746                                 void
 747 ef10_rx_qpost(
 748         __in                    efx_rxq_t *erp,
 749         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 750         __in                    size_t size,
 751         __in                    unsigned int ndescs,
 752         __in                    unsigned int completed,
 753         __in                    unsigned int added)
 754 {
 755         efx_qword_t qword;
 756         unsigned int i;
 757         unsigned int offset;
 758         unsigned int id;
 759
 760         _NOTE(ARGUNUSED(completed))
 761
 762 #if EFSYS_OPT_RX_PACKED_STREAM
 763         /*
 764          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 765          * and equal to 0 after applying mask. Hardware does not like it.
 766          */
 767         if (erp->er_ev_qstate->eers_rx_packed_stream)
 768                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 769 #endif
 770
 771         /* The client driver must not overfill the queue */
 772         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 773             EFX_RXQ_LIMIT(erp->er_mask + 1));
 774
 775         id = added & (erp->er_mask);
 776         for (i = 0; i < ndescs; i++) {
 777                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 778                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 779                     size_t, size);
 780
 781                 EFX_POPULATE_QWORD_3(qword,
 782                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 783                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 784                     (uint32_t)(addrp[i] & 0xffffffff),
 785                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 786                     (uint32_t)(addrp[i] >> 32));
 787
 788                 offset = id * sizeof (efx_qword_t);
 789                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 790
 791                 id = (id + 1) & (erp->er_mask);
 792         }
 793 }
 794
 795                         void
 796 ef10_rx_qpush(
 797         __in    efx_rxq_t *erp,
 798         __in    unsigned int added,
 799         __inout unsigned int *pushedp)
 800 {
 801         efx_nic_t *enp = erp->er_enp;
 802         unsigned int pushed = *pushedp;
 803         uint32_t wptr;
 804         efx_dword_t dword;
 805
 806         /* Hardware has alignment restriction for WPTR */
 807         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
 808         if (pushed == wptr)
 809                 return;
 810
 811         *pushedp = wptr;
 812
 813         /* Push the populated descriptors out */
 814         wptr &= erp->er_mask;
 815
 816         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 817
 818         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 819         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 820             wptr, pushed & erp->er_mask);
 821         EFSYS_PIO_WRITE_BARRIER();
 822         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 823                             erp->er_index, &dword, B_FALSE);
 824 }
 825
 826 #if EFSYS_OPT_RX_PACKED_STREAM
 827
 828                         void
 829 ef10_rx_qpush_ps_credits(
 830         __in            efx_rxq_t *erp)
 831 {
 832         efx_nic_t *enp = erp->er_enp;
 833         efx_dword_t dword;
 834         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 835         uint32_t credits;
 836
 837         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 838
 839         if (rxq_state->eers_rx_packed_stream_credits == 0)
 840                 return;
 841
 842         /*
 843          * It is a bug if we think that FW has utilized more
 844          * credits than it is allowed to have (maximum). However,
 845          * make sure that we do not credit more than maximum anyway.
 846          */
 847         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 848             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 849         EFX_POPULATE_DWORD_3(dword,
 850             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 851             ERF_DZ_RX_DESC_MAGIC_CMD,
 852             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 853             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 854         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 855             erp->er_index, &dword, B_FALSE);
 856
 857         rxq_state->eers_rx_packed_stream_credits = 0;
 858 }
 859
 860 /*
 861  * In accordance with SF-112241-TC the received data has the following layout:
 862  *  - 8 byte pseudo-header which consist of:
 863  *    - 4 byte little-endian timestamp
 864  *    - 2 byte little-endian captured length in bytes
 865  *    - 2 byte little-endian original packet length in bytes
 866  *  - captured packet bytes
 867  *  - optional padding to align to 64 bytes boundary
 868  *  - 64 bytes scratch space for the host software
 869  */
 870         __checkReturn   uint8_t *
 871 ef10_rx_qps_packet_info(
 872         __in            efx_rxq_t *erp,
 873         __in            uint8_t *buffer,
 874         __in            uint32_t buffer_length,
 875         __in            uint32_t current_offset,
 876         __out           uint16_t *lengthp,
 877         __out           uint32_t *next_offsetp,
 878         __out           uint32_t *timestamp)
 879 {
 880         uint16_t buf_len;
 881         uint8_t *pkt_start;
 882         efx_qword_t *qwordp;
 883         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 884
 885         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 886
 887         buffer += current_offset;
 888         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 889
 890         qwordp = (efx_qword_t *)buffer;
 891         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 892         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 893         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 894
 895         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 896                             EFX_RX_PACKED_STREAM_ALIGNMENT);
 897         *next_offsetp =
 898             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 899
 900         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 901         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 902
 903         if ((*next_offsetp ^ current_offset) &
 904             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 905                 rxq_state->eers_rx_packed_stream_credits++;
 906
 907         return (pkt_start);
 908 }
 909
 910
 911 #endif
 912
 913         __checkReturn   efx_rc_t
 914 ef10_rx_qflush(
 915         __in    efx_rxq_t *erp)
 916 {
 917         efx_nic_t *enp = erp->er_enp;
 918         efx_rc_t rc;
 919
 920         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 921                 goto fail1;
 922
 923         return (0);
 924
 925 fail1:
 926         /*
 927          * EALREADY is not an error, but indicates that the MC has rebooted and
 928          * that the RXQ has already been destroyed. Callers need to know that
 929          * the RXQ flush has completed to avoid waiting until timeout for a
 930          * flush done event that will not be delivered.
 931          */
 932         if (rc != EALREADY)
 933                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 934
 935         return (rc);
 936 }
 937
 938                 void
 939 ef10_rx_qenable(
 940         __in    efx_rxq_t *erp)
 941 {
 942         /* FIXME */
 943         _NOTE(ARGUNUSED(erp))
 944         /* FIXME */
 945 }
 946
 947         __checkReturn   efx_rc_t
 948 ef10_rx_qcreate(
 949         __in            efx_nic_t *enp,
 950         __in            unsigned int index,
 951         __in            unsigned int label,
 952         __in            efx_rxq_type_t type,
 953         __in            uint32_t type_data,
 954         __in            efsys_mem_t *esmp,
 955         __in            size_t ndescs,
 956         __in            uint32_t id,
 957         __in            unsigned int flags,
 958         __in            efx_evq_t *eep,
 959         __in            efx_rxq_t *erp)
 960 {
 961         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 962         efx_rc_t rc;
 963         boolean_t disable_scatter;
 964         boolean_t want_inner_classes;
 965         unsigned int ps_buf_size;
 966
 967         _NOTE(ARGUNUSED(id, erp))
 968
 969         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 970         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 971         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 972
 973         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
 974         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
 975
 976         if (!ISP2(ndescs) ||
 977             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
 978                 rc = EINVAL;
 979                 goto fail1;
 980         }
 981         if (index >= encp->enc_rxq_limit) {
 982                 rc = EINVAL;
 983                 goto fail2;
 984         }
 985
 986         switch (type) {
 987         case EFX_RXQ_TYPE_DEFAULT:
 988                 ps_buf_size = 0;
 989                 break;
 990 #if EFSYS_OPT_RX_PACKED_STREAM
 991         case EFX_RXQ_TYPE_PACKED_STREAM:
 992                 switch (type_data) {
 993                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
 994                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
 995                         break;
 996                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
 997                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
 998                         break;
 999                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
1000                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
1001                         break;
1002                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
1003                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
1004                         break;
1005                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
1006                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
1007                         break;
1008                 default:
1009                         rc = ENOTSUP;
1010                         goto fail3;
1011                 }
1012                 break;
1013 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1014         default:
1015                 rc = ENOTSUP;
1016                 goto fail4;
1017         }
1018
1019 #if EFSYS_OPT_RX_PACKED_STREAM
1020         if (ps_buf_size != 0) {
1021                 /* Check if datapath firmware supports packed stream mode */
1022                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1023                         rc = ENOTSUP;
1024                         goto fail5;
1025                 }
1026                 /* Check if packed stream allows configurable buffer sizes */
1027                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
1028                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1029                         rc = ENOTSUP;
1030                         goto fail6;
1031                 }
1032         }
1033 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1034         EFSYS_ASSERT(ps_buf_size == 0);
1035 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1036
1037         /* Scatter can only be disabled if the firmware supports doing so */
1038         if (flags & EFX_RXQ_FLAG_SCATTER)
1039                 disable_scatter = B_FALSE;
1040         else
1041                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1042
1043         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
1044                 want_inner_classes = B_TRUE;
1045         else
1046                 want_inner_classes = B_FALSE;
1047
1048         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1049                     esmp, disable_scatter, want_inner_classes,
1050                     ps_buf_size)) != 0)
1051                 goto fail7;
1052
1053         erp->er_eep = eep;
1054         erp->er_label = label;
1055
1056         ef10_ev_rxlabel_init(eep, erp, label, type);
1057
1058         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1059
1060         return (0);
1061
1062 fail7:
1063         EFSYS_PROBE(fail7);
1064 #if EFSYS_OPT_RX_PACKED_STREAM
1065 fail6:
1066         EFSYS_PROBE(fail6);
1067 fail5:
1068         EFSYS_PROBE(fail5);
1069 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1070 fail4:
1071         EFSYS_PROBE(fail4);
1072 #if EFSYS_OPT_RX_PACKED_STREAM
1073 fail3:
1074         EFSYS_PROBE(fail3);
1075 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1076 fail2:
1077         EFSYS_PROBE(fail2);
1078 fail1:
1079         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1080
1081         return (rc);
1082 }
1083
1084                 void
1085 ef10_rx_qdestroy(
1086         __in    efx_rxq_t *erp)
1087 {
1088         efx_nic_t *enp = erp->er_enp;
1089         efx_evq_t *eep = erp->er_eep;
1090         unsigned int label = erp->er_label;
1091
1092         ef10_ev_rxlabel_fini(eep, label);
1093
1094         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1095         --enp->en_rx_qcount;
1096
1097         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1098 }
1099
1100                 void
1101 ef10_rx_fini(
1102         __in    efx_nic_t *enp)
1103 {
1104 #if EFSYS_OPT_RX_SCALE
1105         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1106                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1107         enp->en_rss_context = 0;
1108         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1109 #else
1110         _NOTE(ARGUNUSED(enp))
1111 #endif /* EFSYS_OPT_RX_SCALE */
1112 }
1113
1114 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */