drivers/common/sfc_efx/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2020 Xilinx, Inc.
   4  * Copyright(c) 2012-2019 Solarflare Communications Inc.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFX_OPTS_EF10()
  12
  13 #if EFSYS_OPT_RX_SCALE
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_rss_context_alloc(
  16         __in            efx_nic_t *enp,
  17         __in            efx_rx_scale_context_type_t type,
  18         __in            uint32_t num_queues,
  19         __out           uint32_t *rss_contextp)
  20 {
  21         efx_mcdi_req_t req;
  22         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
  23                 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
  24         uint32_t rss_context;
  25         uint32_t context_type;
  26         efx_rc_t rc;
  27
  28         if (num_queues > EFX_MAXRSS) {
  29                 rc = EINVAL;
  30                 goto fail1;
  31         }
  32
  33         switch (type) {
  34         case EFX_RX_SCALE_EXCLUSIVE:
  35                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
  36                 break;
  37         case EFX_RX_SCALE_SHARED:
  38                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
  39                 break;
  40         default:
  41                 rc = EINVAL;
  42                 goto fail2;
  43         }
  44
  45         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
  46         req.emr_in_buf = payload;
  47         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
  48         req.emr_out_buf = payload;
  49         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
  50
  51         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
  52                 enp->en_vport_id);
  53         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
  54
  55         /*
  56          * For exclusive contexts, NUM_QUEUES is only used to validate
  57          * indirection table offsets.
  58          * For shared contexts, the provided context will spread traffic over
  59          * NUM_QUEUES many queues.
  60          */
  61         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
  62
  63         efx_mcdi_execute(enp, &req);
  64
  65         if (req.emr_rc != 0) {
  66                 rc = req.emr_rc;
  67                 goto fail3;
  68         }
  69
  70         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
  71                 rc = EMSGSIZE;
  72                 goto fail4;
  73         }
  74
  75         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
  76         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
  77                 rc = ENOENT;
  78                 goto fail5;
  79         }
  80
  81         *rss_contextp = rss_context;
  82
  83         return (0);
  84
  85 fail5:
  86         EFSYS_PROBE(fail5);
  87 fail4:
  88         EFSYS_PROBE(fail4);
  89 fail3:
  90         EFSYS_PROBE(fail3);
  91 fail2:
  92         EFSYS_PROBE(fail2);
  93 fail1:
  94         EFSYS_PROBE1(fail1, efx_rc_t, rc);
  95
  96         return (rc);
  97 }
  98 #endif /* EFSYS_OPT_RX_SCALE */
  99
 100 #if EFSYS_OPT_RX_SCALE
 101 static                  efx_rc_t
 102 efx_mcdi_rss_context_free(
 103         __in            efx_nic_t *enp,
 104         __in            uint32_t rss_context)
 105 {
 106         efx_mcdi_req_t req;
 107         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 108                 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 109         efx_rc_t rc;
 110
 111         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 112                 rc = EINVAL;
 113                 goto fail1;
 114         }
 115
 116         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 117         req.emr_in_buf = payload;
 118         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 119         req.emr_out_buf = payload;
 120         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 121
 122         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 123
 124         efx_mcdi_execute_quiet(enp, &req);
 125
 126         if (req.emr_rc != 0) {
 127                 rc = req.emr_rc;
 128                 goto fail2;
 129         }
 130
 131         return (0);
 132
 133 fail2:
 134         EFSYS_PROBE(fail2);
 135 fail1:
 136         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 137
 138         return (rc);
 139 }
 140 #endif /* EFSYS_OPT_RX_SCALE */
 141
 142 #if EFSYS_OPT_RX_SCALE
 143 static                  efx_rc_t
 144 efx_mcdi_rss_context_set_flags(
 145         __in            efx_nic_t *enp,
 146         __in            uint32_t rss_context,
 147         __in            efx_rx_hash_type_t type)
 148 {
 149         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 150         efx_mcdi_req_t req;
 151         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 152                 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 153         efx_rc_t rc;
 154
 155         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
 156                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
 157         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
 158                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
 159         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
 160                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
 161         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
 162                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
 163         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
 164                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
 165         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
 166                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
 167         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
 168                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
 169         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
 170                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
 171
 172         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 173                 rc = EINVAL;
 174                 goto fail1;
 175         }
 176
 177         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 178         req.emr_in_buf = payload;
 179         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 180         req.emr_out_buf = payload;
 181         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 182
 183         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 184             rss_context);
 185
 186         /*
 187          * If the firmware lacks support for additional modes, RSS_MODE
 188          * fields must contain zeros, otherwise the operation will fail.
 189          */
 190         if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
 191                 type &= EFX_RX_HASH_LEGACY_MASK;
 192
 193         MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 194             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 195             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 196             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 197             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 198             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 199             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 200             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 201             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0,
 202             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
 203             (type >> EFX_RX_CLASS_IPV4_TCP_LBN) &
 204             EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
 205             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
 206             (type >> EFX_RX_CLASS_IPV4_UDP_LBN) &
 207             EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
 208             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
 209             (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
 210             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
 211             (type >> EFX_RX_CLASS_IPV6_TCP_LBN) &
 212             EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
 213             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
 214             (type >> EFX_RX_CLASS_IPV6_UDP_LBN) &
 215             EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
 216             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
 217             (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
 218
 219         efx_mcdi_execute(enp, &req);
 220
 221         if (req.emr_rc != 0) {
 222                 rc = req.emr_rc;
 223                 goto fail2;
 224         }
 225
 226         return (0);
 227
 228 fail2:
 229         EFSYS_PROBE(fail2);
 230 fail1:
 231         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 232
 233         return (rc);
 234 }
 235 #endif /* EFSYS_OPT_RX_SCALE */
 236
 237 #if EFSYS_OPT_RX_SCALE
 238 static                  efx_rc_t
 239 efx_mcdi_rss_context_set_key(
 240         __in            efx_nic_t *enp,
 241         __in            uint32_t rss_context,
 242         __in_ecount(n)  uint8_t *key,
 243         __in            size_t n)
 244 {
 245         efx_mcdi_req_t req;
 246         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 247                 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 248         efx_rc_t rc;
 249
 250         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 251                 rc = EINVAL;
 252                 goto fail1;
 253         }
 254
 255         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 256         req.emr_in_buf = payload;
 257         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 258         req.emr_out_buf = payload;
 259         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 260
 261         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 262             rss_context);
 263
 264         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 265         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 266                 rc = EINVAL;
 267                 goto fail2;
 268         }
 269
 270         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 271             key, n);
 272
 273         efx_mcdi_execute(enp, &req);
 274
 275         if (req.emr_rc != 0) {
 276                 rc = req.emr_rc;
 277                 goto fail3;
 278         }
 279
 280         return (0);
 281
 282 fail3:
 283         EFSYS_PROBE(fail3);
 284 fail2:
 285         EFSYS_PROBE(fail2);
 286 fail1:
 287         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 288
 289         return (rc);
 290 }
 291 #endif /* EFSYS_OPT_RX_SCALE */
 292
 293 #if EFSYS_OPT_RX_SCALE
 294 static                  efx_rc_t
 295 efx_mcdi_rss_context_set_table(
 296         __in            efx_nic_t *enp,
 297         __in            uint32_t rss_context,
 298         __in_ecount(n)  unsigned int *table,
 299         __in            size_t n)
 300 {
 301         efx_mcdi_req_t req;
 302         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 303                 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 304         uint8_t *req_table;
 305         int i, rc;
 306
 307         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 308                 rc = EINVAL;
 309                 goto fail1;
 310         }
 311
 312         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 313         req.emr_in_buf = payload;
 314         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 315         req.emr_out_buf = payload;
 316         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 317
 318         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 319             rss_context);
 320
 321         req_table =
 322             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 323
 324         for (i = 0;
 325             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 326             i++) {
 327                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 328         }
 329
 330         efx_mcdi_execute(enp, &req);
 331
 332         if (req.emr_rc != 0) {
 333                 rc = req.emr_rc;
 334                 goto fail2;
 335         }
 336
 337         return (0);
 338
 339 fail2:
 340         EFSYS_PROBE(fail2);
 341 fail1:
 342         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 343
 344         return (rc);
 345 }
 346 #endif /* EFSYS_OPT_RX_SCALE */
 347
 348
 349         __checkReturn   efx_rc_t
 350 ef10_rx_init(
 351         __in            efx_nic_t *enp)
 352 {
 353 #if EFSYS_OPT_RX_SCALE
 354
 355         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 356                 &enp->en_rss_context) == 0) {
 357                 /*
 358                  * Allocated an exclusive RSS context, which allows both the
 359                  * indirection table and key to be modified.
 360                  */
 361                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 362                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 363         } else {
 364                 /*
 365                  * Failed to allocate an exclusive RSS context. Continue
 366                  * operation without support for RSS. The pseudo-header in
 367                  * received packets will not contain a Toeplitz hash value.
 368                  */
 369                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 370                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 371         }
 372
 373 #endif /* EFSYS_OPT_RX_SCALE */
 374
 375         return (0);
 376 }
 377
 378 #if EFSYS_OPT_RX_SCATTER
 379         __checkReturn   efx_rc_t
 380 ef10_rx_scatter_enable(
 381         __in            efx_nic_t *enp,
 382         __in            unsigned int buf_size)
 383 {
 384         _NOTE(ARGUNUSED(enp, buf_size))
 385         return (0);
 386 }
 387 #endif  /* EFSYS_OPT_RX_SCATTER */
 388
 389 #if EFSYS_OPT_RX_SCALE
 390         __checkReturn   efx_rc_t
 391 ef10_rx_scale_context_alloc(
 392         __in            efx_nic_t *enp,
 393         __in            efx_rx_scale_context_type_t type,
 394         __in            uint32_t num_queues,
 395         __out           uint32_t *rss_contextp)
 396 {
 397         efx_rc_t rc;
 398
 399         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 400         if (rc != 0)
 401                 goto fail1;
 402
 403         return (0);
 404
 405 fail1:
 406         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 407         return (rc);
 408 }
 409 #endif /* EFSYS_OPT_RX_SCALE */
 410
 411 #if EFSYS_OPT_RX_SCALE
 412         __checkReturn   efx_rc_t
 413 ef10_rx_scale_context_free(
 414         __in            efx_nic_t *enp,
 415         __in            uint32_t rss_context)
 416 {
 417         efx_rc_t rc;
 418
 419         rc = efx_mcdi_rss_context_free(enp, rss_context);
 420         if (rc != 0)
 421                 goto fail1;
 422
 423         return (0);
 424
 425 fail1:
 426         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 427         return (rc);
 428 }
 429 #endif /* EFSYS_OPT_RX_SCALE */
 430
 431 #if EFSYS_OPT_RX_SCALE
 432         __checkReturn   efx_rc_t
 433 ef10_rx_scale_mode_set(
 434         __in            efx_nic_t *enp,
 435         __in            uint32_t rss_context,
 436         __in            efx_rx_hash_alg_t alg,
 437         __in            efx_rx_hash_type_t type,
 438         __in            boolean_t insert)
 439 {
 440         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 441         efx_rc_t rc;
 442
 443         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 444
 445         if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
 446             insert == B_FALSE) {
 447                 rc = EINVAL;
 448                 goto fail1;
 449         }
 450
 451         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 452                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 453                         rc = ENOTSUP;
 454                         goto fail2;
 455                 }
 456                 rss_context = enp->en_rss_context;
 457         }
 458
 459         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 460                     rss_context, type)) != 0)
 461                 goto fail3;
 462
 463         return (0);
 464
 465 fail3:
 466         EFSYS_PROBE(fail3);
 467 fail2:
 468         EFSYS_PROBE(fail2);
 469 fail1:
 470         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 471
 472         return (rc);
 473 }
 474 #endif /* EFSYS_OPT_RX_SCALE */
 475
 476 #if EFSYS_OPT_RX_SCALE
 477         __checkReturn   efx_rc_t
 478 ef10_rx_scale_key_set(
 479         __in            efx_nic_t *enp,
 480         __in            uint32_t rss_context,
 481         __in_ecount(n)  uint8_t *key,
 482         __in            size_t n)
 483 {
 484         efx_rc_t rc;
 485
 486         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 487             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 488
 489         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 490                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 491                         rc = ENOTSUP;
 492                         goto fail1;
 493                 }
 494                 rss_context = enp->en_rss_context;
 495         }
 496
 497         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 498                 goto fail2;
 499
 500         return (0);
 501
 502 fail2:
 503         EFSYS_PROBE(fail2);
 504 fail1:
 505         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 506
 507         return (rc);
 508 }
 509 #endif /* EFSYS_OPT_RX_SCALE */
 510
 511 #if EFSYS_OPT_RX_SCALE
 512         __checkReturn   efx_rc_t
 513 ef10_rx_scale_tbl_set(
 514         __in            efx_nic_t *enp,
 515         __in            uint32_t rss_context,
 516         __in_ecount(n)  unsigned int *table,
 517         __in            size_t n)
 518 {
 519         efx_rc_t rc;
 520
 521
 522         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 523                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 524                         rc = ENOTSUP;
 525                         goto fail1;
 526                 }
 527                 rss_context = enp->en_rss_context;
 528         }
 529
 530         if ((rc = efx_mcdi_rss_context_set_table(enp,
 531                     rss_context, table, n)) != 0)
 532                 goto fail2;
 533
 534         return (0);
 535
 536 fail2:
 537         EFSYS_PROBE(fail2);
 538 fail1:
 539         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 540
 541         return (rc);
 542 }
 543 #endif /* EFSYS_OPT_RX_SCALE */
 544
 545
 546 /*
 547  * EF10 RX pseudo-header
 548  * ---------------------
 549  *
 550  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 551  *
 552  *  +00: Toeplitz hash value.
 553  *       (32bit little-endian)
 554  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 555  *       (16bit big-endian)
 556  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 557  *       (16bit big-endian)
 558  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 559  *       (16bit little-endian)
 560  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 561  *       (32bit little-endian)
 562  *
 563  * See "The RX Pseudo-header" in SF-109306-TC.
 564  */
 565
 566         __checkReturn   efx_rc_t
 567 ef10_rx_prefix_pktlen(
 568         __in            efx_nic_t *enp,
 569         __in            uint8_t *buffer,
 570         __out           uint16_t *lengthp)
 571 {
 572         _NOTE(ARGUNUSED(enp))
 573
 574         /*
 575          * The RX pseudo-header contains the packet length, excluding the
 576          * pseudo-header. If the hardware receive datapath was operating in
 577          * cut-through mode then the length in the RX pseudo-header will be
 578          * zero, and the packet length must be obtained from the DMA length
 579          * reported in the RX event.
 580          */
 581         *lengthp = buffer[8] | (buffer[9] << 8);
 582         return (0);
 583 }
 584
 585 #if EFSYS_OPT_RX_SCALE
 586         __checkReturn   uint32_t
 587 ef10_rx_prefix_hash(
 588         __in            efx_nic_t *enp,
 589         __in            efx_rx_hash_alg_t func,
 590         __in            uint8_t *buffer)
 591 {
 592         _NOTE(ARGUNUSED(enp))
 593
 594         switch (func) {
 595         case EFX_RX_HASHALG_PACKED_STREAM:
 596         case EFX_RX_HASHALG_TOEPLITZ:
 597                 return (buffer[0] |
 598                     (buffer[1] << 8) |
 599                     (buffer[2] << 16) |
 600                     (buffer[3] << 24));
 601
 602         default:
 603                 EFSYS_ASSERT(0);
 604                 return (0);
 605         }
 606 }
 607 #endif /* EFSYS_OPT_RX_SCALE */
 608
 609 #if EFSYS_OPT_RX_PACKED_STREAM
 610 /*
 611  * Fake length for RXQ descriptors in packed stream mode
 612  * to make hardware happy
 613  */
 614 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 615 #endif
 616
 617                                 void
 618 ef10_rx_qpost(
 619         __in                    efx_rxq_t *erp,
 620         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 621         __in                    size_t size,
 622         __in                    unsigned int ndescs,
 623         __in                    unsigned int completed,
 624         __in                    unsigned int added)
 625 {
 626         efx_qword_t qword;
 627         unsigned int i;
 628         unsigned int offset;
 629         unsigned int id;
 630
 631         _NOTE(ARGUNUSED(completed))
 632
 633 #if EFSYS_OPT_RX_PACKED_STREAM
 634         /*
 635          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 636          * and equal to 0 after applying mask. Hardware does not like it.
 637          */
 638         if (erp->er_ev_qstate->eers_rx_packed_stream)
 639                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 640 #endif
 641
 642         /* The client driver must not overfill the queue */
 643         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 644             EFX_RXQ_LIMIT(erp->er_mask + 1));
 645
 646         id = added & (erp->er_mask);
 647         for (i = 0; i < ndescs; i++) {
 648                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 649                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 650                     size_t, size);
 651
 652                 EFX_POPULATE_QWORD_3(qword,
 653                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 654                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 655                     (uint32_t)(addrp[i] & 0xffffffff),
 656                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 657                     (uint32_t)(addrp[i] >> 32));
 658
 659                 offset = id * sizeof (efx_qword_t);
 660                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 661
 662                 id = (id + 1) & (erp->er_mask);
 663         }
 664 }
 665
 666                         void
 667 ef10_rx_qpush(
 668         __in    efx_rxq_t *erp,
 669         __in    unsigned int added,
 670         __inout unsigned int *pushedp)
 671 {
 672         efx_nic_t *enp = erp->er_enp;
 673         unsigned int pushed = *pushedp;
 674         uint32_t wptr;
 675         efx_dword_t dword;
 676
 677         /* Hardware has alignment restriction for WPTR */
 678         wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN);
 679         if (pushed == wptr)
 680                 return;
 681
 682         *pushedp = wptr;
 683
 684         /* Push the populated descriptors out */
 685         wptr &= erp->er_mask;
 686
 687         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 688
 689         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 690         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 691             wptr, pushed & erp->er_mask);
 692         EFSYS_PIO_WRITE_BARRIER();
 693         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 694             erp->er_index, &dword, B_FALSE);
 695 }
 696
 697 #if EFSYS_OPT_RX_PACKED_STREAM
 698
 699                         void
 700 ef10_rx_qpush_ps_credits(
 701         __in            efx_rxq_t *erp)
 702 {
 703         efx_nic_t *enp = erp->er_enp;
 704         efx_dword_t dword;
 705         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 706         uint32_t credits;
 707
 708         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 709
 710         if (rxq_state->eers_rx_packed_stream_credits == 0)
 711                 return;
 712
 713         /*
 714          * It is a bug if we think that FW has utilized more
 715          * credits than it is allowed to have (maximum). However,
 716          * make sure that we do not credit more than maximum anyway.
 717          */
 718         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 719             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 720         EFX_POPULATE_DWORD_3(dword,
 721             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 722             ERF_DZ_RX_DESC_MAGIC_CMD,
 723             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 724             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 725         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 726             erp->er_index, &dword, B_FALSE);
 727
 728         rxq_state->eers_rx_packed_stream_credits = 0;
 729 }
 730
 731 /*
 732  * In accordance with SF-112241-TC the received data has the following layout:
 733  *  - 8 byte pseudo-header which consist of:
 734  *    - 4 byte little-endian timestamp
 735  *    - 2 byte little-endian captured length in bytes
 736  *    - 2 byte little-endian original packet length in bytes
 737  *  - captured packet bytes
 738  *  - optional padding to align to 64 bytes boundary
 739  *  - 64 bytes scratch space for the host software
 740  */
 741         __checkReturn   uint8_t *
 742 ef10_rx_qps_packet_info(
 743         __in            efx_rxq_t *erp,
 744         __in            uint8_t *buffer,
 745         __in            uint32_t buffer_length,
 746         __in            uint32_t current_offset,
 747         __out           uint16_t *lengthp,
 748         __out           uint32_t *next_offsetp,
 749         __out           uint32_t *timestamp)
 750 {
 751         uint16_t buf_len;
 752         uint8_t *pkt_start;
 753         efx_qword_t *qwordp;
 754         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 755
 756         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 757
 758         buffer += current_offset;
 759         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 760
 761         qwordp = (efx_qword_t *)buffer;
 762         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 763         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 764         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 765
 766         buf_len = EFX_P2ROUNDUP(uint16_t,
 767             buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 768             EFX_RX_PACKED_STREAM_ALIGNMENT);
 769         *next_offsetp =
 770             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 771
 772         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 773         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 774
 775         if ((*next_offsetp ^ current_offset) &
 776             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 777                 rxq_state->eers_rx_packed_stream_credits++;
 778
 779         return (pkt_start);
 780 }
 781
 782
 783 #endif
 784
 785         __checkReturn   efx_rc_t
 786 ef10_rx_qflush(
 787         __in    efx_rxq_t *erp)
 788 {
 789         efx_nic_t *enp = erp->er_enp;
 790         efx_rc_t rc;
 791
 792         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 793                 goto fail1;
 794
 795         return (0);
 796
 797 fail1:
 798         /*
 799          * EALREADY is not an error, but indicates that the MC has rebooted and
 800          * that the RXQ has already been destroyed. Callers need to know that
 801          * the RXQ flush has completed to avoid waiting until timeout for a
 802          * flush done event that will not be delivered.
 803          */
 804         if (rc != EALREADY)
 805                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 806
 807         return (rc);
 808 }
 809
 810                 void
 811 ef10_rx_qenable(
 812         __in    efx_rxq_t *erp)
 813 {
 814         /* FIXME */
 815         _NOTE(ARGUNUSED(erp))
 816         /* FIXME */
 817 }
 818
 819         __checkReturn   efx_rc_t
 820 ef10_rx_qcreate(
 821         __in            efx_nic_t *enp,
 822         __in            unsigned int index,
 823         __in            unsigned int label,
 824         __in            efx_rxq_type_t type,
 825         __in_opt        const efx_rxq_type_data_t *type_data,
 826         __in            efsys_mem_t *esmp,
 827         __in            size_t ndescs,
 828         __in            uint32_t id,
 829         __in            unsigned int flags,
 830         __in            efx_evq_t *eep,
 831         __in            efx_rxq_t *erp)
 832 {
 833         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 834         efx_rc_t rc;
 835         boolean_t disable_scatter;
 836         boolean_t want_inner_classes;
 837         unsigned int ps_buf_size;
 838         uint32_t es_bufs_per_desc = 0;
 839         uint32_t es_max_dma_len = 0;
 840         uint32_t es_buf_stride = 0;
 841         uint32_t hol_block_timeout = 0;
 842
 843         _NOTE(ARGUNUSED(id, erp))
 844
 845         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 846         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 847         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
 848
 849         if (index >= encp->enc_rxq_limit) {
 850                 rc = EINVAL;
 851                 goto fail1;
 852         }
 853
 854         switch (type) {
 855         case EFX_RXQ_TYPE_DEFAULT:
 856                 if (type_data == NULL) {
 857                         rc = EINVAL;
 858                         goto fail2;
 859                 }
 860                 erp->er_buf_size = type_data->ertd_default.ed_buf_size;
 861                 ps_buf_size = 0;
 862                 break;
 863 #if EFSYS_OPT_RX_PACKED_STREAM
 864         case EFX_RXQ_TYPE_PACKED_STREAM:
 865                 if (type_data == NULL) {
 866                         rc = EINVAL;
 867                         goto fail3;
 868                 }
 869                 switch (type_data->ertd_packed_stream.eps_buf_size) {
 870                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
 871                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
 872                         break;
 873                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
 874                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
 875                         break;
 876                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
 877                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
 878                         break;
 879                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
 880                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
 881                         break;
 882                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
 883                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
 884                         break;
 885                 default:
 886                         rc = ENOTSUP;
 887                         goto fail4;
 888                 }
 889                 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size;
 890                 break;
 891 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 892 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 893         case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
 894                 if (type_data == NULL) {
 895                         rc = EINVAL;
 896                         goto fail5;
 897                 }
 898                 ps_buf_size = 0;
 899                 es_bufs_per_desc =
 900                     type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
 901                 es_max_dma_len =
 902                     type_data->ertd_es_super_buffer.eessb_max_dma_len;
 903                 es_buf_stride =
 904                     type_data->ertd_es_super_buffer.eessb_buf_stride;
 905                 hol_block_timeout =
 906                     type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
 907                 break;
 908 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 909         default:
 910                 rc = ENOTSUP;
 911                 goto fail6;
 912         }
 913
 914 #if EFSYS_OPT_RX_PACKED_STREAM
 915         if (ps_buf_size != 0) {
 916                 /* Check if datapath firmware supports packed stream mode */
 917                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
 918                         rc = ENOTSUP;
 919                         goto fail7;
 920                 }
 921                 /* Check if packed stream allows configurable buffer sizes */
 922                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
 923                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
 924                         rc = ENOTSUP;
 925                         goto fail8;
 926                 }
 927         }
 928 #else /* EFSYS_OPT_RX_PACKED_STREAM */
 929         EFSYS_ASSERT(ps_buf_size == 0);
 930 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 931
 932 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 933         if (es_bufs_per_desc > 0) {
 934                 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
 935                         rc = ENOTSUP;
 936                         goto fail9;
 937                 }
 938                 if (!EFX_IS_P2ALIGNED(uint32_t, es_max_dma_len,
 939                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
 940                         rc = EINVAL;
 941                         goto fail10;
 942                 }
 943                 if (!EFX_IS_P2ALIGNED(uint32_t, es_buf_stride,
 944                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
 945                         rc = EINVAL;
 946                         goto fail11;
 947                 }
 948         }
 949 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 950         EFSYS_ASSERT(es_bufs_per_desc == 0);
 951 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 952
 953         /* Scatter can only be disabled if the firmware supports doing so */
 954         if (flags & EFX_RXQ_FLAG_SCATTER)
 955                 disable_scatter = B_FALSE;
 956         else
 957                 disable_scatter = encp->enc_rx_disable_scatter_supported;
 958
 959         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
 960                 want_inner_classes = B_TRUE;
 961         else
 962                 want_inner_classes = B_FALSE;
 963
 964         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index,
 965                     esmp, disable_scatter, want_inner_classes, erp->er_buf_size,
 966                     ps_buf_size, es_bufs_per_desc, es_max_dma_len,
 967                     es_buf_stride, hol_block_timeout)) != 0)
 968                 goto fail12;
 969
 970         erp->er_eep = eep;
 971         erp->er_label = label;
 972
 973         ef10_ev_rxlabel_init(eep, erp, label, type);
 974
 975         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
 976
 977         return (0);
 978
 979 fail12:
 980         EFSYS_PROBE(fail12);
 981 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 982 fail11:
 983         EFSYS_PROBE(fail11);
 984 fail10:
 985         EFSYS_PROBE(fail10);
 986 fail9:
 987         EFSYS_PROBE(fail9);
 988 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 989 #if EFSYS_OPT_RX_PACKED_STREAM
 990 fail8:
 991         EFSYS_PROBE(fail8);
 992 fail7:
 993         EFSYS_PROBE(fail7);
 994 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 995 fail6:
 996         EFSYS_PROBE(fail6);
 997 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 998 fail5:
 999         EFSYS_PROBE(fail5);
1000 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1001 #if EFSYS_OPT_RX_PACKED_STREAM
1002 fail4:
1003         EFSYS_PROBE(fail4);
1004 fail3:
1005         EFSYS_PROBE(fail3);
1006 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1007 fail2:
1008         EFSYS_PROBE(fail2);
1009 fail1:
1010         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1011
1012         return (rc);
1013 }
1014
1015                 void
1016 ef10_rx_qdestroy(
1017         __in    efx_rxq_t *erp)
1018 {
1019         efx_nic_t *enp = erp->er_enp;
1020         efx_evq_t *eep = erp->er_eep;
1021         unsigned int label = erp->er_label;
1022
1023         ef10_ev_rxlabel_fini(eep, label);
1024
1025         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1026         --enp->en_rx_qcount;
1027
1028         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1029 }
1030
1031                 void
1032 ef10_rx_fini(
1033         __in    efx_nic_t *enp)
1034 {
1035 #if EFSYS_OPT_RX_SCALE
1036         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1037                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1038         enp->en_rss_context = 0;
1039         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1040 #else
1041         _NOTE(ARGUNUSED(enp))
1042 #endif /* EFSYS_OPT_RX_SCALE */
1043 }
1044
1045 #endif /* EFX_OPTS_EF10() */