drivers/common/sfc_efx/base/ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2020 Xilinx, Inc.
   4  * Copyright(c) 2012-2019 Solarflare Communications Inc.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10()
  12
  13 #if EFSYS_OPT_RX_SCALE
  14 static  __checkReturn   efx_rc_t
  15 efx_mcdi_rss_context_alloc(
  16         __in            efx_nic_t *enp,
  17         __in            efx_rx_scale_context_type_t type,
  18         __in            uint32_t num_queues,
  19         __out           uint32_t *rss_contextp)
  20 {
  21         efx_mcdi_req_t req;
  22         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
  23                 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
  24         uint32_t rss_context;
  25         uint32_t context_type;
  26         efx_rc_t rc;
  27
  28         if (num_queues > EFX_MAXRSS) {
  29                 rc = EINVAL;
  30                 goto fail1;
  31         }
  32
  33         switch (type) {
  34         case EFX_RX_SCALE_EXCLUSIVE:
  35                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
  36                 break;
  37         case EFX_RX_SCALE_SHARED:
  38                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
  39                 break;
  40         default:
  41                 rc = EINVAL;
  42                 goto fail2;
  43         }
  44
  45         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
  46         req.emr_in_buf = payload;
  47         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
  48         req.emr_out_buf = payload;
  49         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
  50
  51         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
  52                 enp->en_vport_id);
  53         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
  54
  55         /*
  56          * For exclusive contexts, NUM_QUEUES is only used to validate
  57          * indirection table offsets.
  58          * For shared contexts, the provided context will spread traffic over
  59          * NUM_QUEUES many queues.
  60          */
  61         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
  62
  63         efx_mcdi_execute(enp, &req);
  64
  65         if (req.emr_rc != 0) {
  66                 rc = req.emr_rc;
  67                 goto fail3;
  68         }
  69
  70         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
  71                 rc = EMSGSIZE;
  72                 goto fail4;
  73         }
  74
  75         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
  76         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
  77                 rc = ENOENT;
  78                 goto fail5;
  79         }
  80
  81         *rss_contextp = rss_context;
  82
  83         return (0);
  84
  85 fail5:
  86         EFSYS_PROBE(fail5);
  87 fail4:
  88         EFSYS_PROBE(fail4);
  89 fail3:
  90         EFSYS_PROBE(fail3);
  91 fail2:
  92         EFSYS_PROBE(fail2);
  93 fail1:
  94         EFSYS_PROBE1(fail1, efx_rc_t, rc);
  95
  96         return (rc);
  97 }
  98 #endif /* EFSYS_OPT_RX_SCALE */
  99
 100 #if EFSYS_OPT_RX_SCALE
 101 static                  efx_rc_t
 102 efx_mcdi_rss_context_free(
 103         __in            efx_nic_t *enp,
 104         __in            uint32_t rss_context)
 105 {
 106         efx_mcdi_req_t req;
 107         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
 108                 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
 109         efx_rc_t rc;
 110
 111         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 112                 rc = EINVAL;
 113                 goto fail1;
 114         }
 115
 116         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
 117         req.emr_in_buf = payload;
 118         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
 119         req.emr_out_buf = payload;
 120         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
 121
 122         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
 123
 124         efx_mcdi_execute_quiet(enp, &req);
 125
 126         if (req.emr_rc != 0) {
 127                 rc = req.emr_rc;
 128                 goto fail2;
 129         }
 130
 131         return (0);
 132
 133 fail2:
 134         EFSYS_PROBE(fail2);
 135 fail1:
 136         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 137
 138         return (rc);
 139 }
 140 #endif /* EFSYS_OPT_RX_SCALE */
 141
 142 #if EFSYS_OPT_RX_SCALE
 143 static                  efx_rc_t
 144 efx_mcdi_rss_context_set_flags(
 145         __in            efx_nic_t *enp,
 146         __in            uint32_t rss_context,
 147         __in            efx_rx_hash_type_t type)
 148 {
 149         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 150         efx_mcdi_req_t req;
 151         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
 152                 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
 153         efx_rc_t rc;
 154
 155         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN ==
 156                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN);
 157         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH ==
 158                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH);
 159         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN ==
 160                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN);
 161         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH ==
 162                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH);
 163         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN ==
 164                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN);
 165         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH ==
 166                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH);
 167         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN ==
 168                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN);
 169         EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH ==
 170                     MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH);
 171
 172         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 173                 rc = EINVAL;
 174                 goto fail1;
 175         }
 176
 177         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
 178         req.emr_in_buf = payload;
 179         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
 180         req.emr_out_buf = payload;
 181         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
 182
 183         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
 184             rss_context);
 185
 186         /*
 187          * If the firmware lacks support for additional modes, RSS_MODE
 188          * fields must contain zeros, otherwise the operation will fail.
 189          */
 190         if (encp->enc_rx_scale_additional_modes_supported == B_FALSE)
 191                 type &= EFX_RX_HASH_LEGACY_MASK;
 192
 193         MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
 194             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
 195             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
 196             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
 197             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
 198             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
 199             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
 200             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
 201             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0,
 202             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE,
 203             (type >> EFX_RX_CLASS_IPV4_TCP_LBN) &
 204             EFX_MASK32(EFX_RX_CLASS_IPV4_TCP),
 205             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE,
 206             (type >> EFX_RX_CLASS_IPV4_UDP_LBN) &
 207             EFX_MASK32(EFX_RX_CLASS_IPV4_UDP),
 208             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE,
 209             (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4),
 210             RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE,
 211             (type >> EFX_RX_CLASS_IPV6_TCP_LBN) &
 212             EFX_MASK32(EFX_RX_CLASS_IPV6_TCP),
 213             RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE,
 214             (type >> EFX_RX_CLASS_IPV6_UDP_LBN) &
 215             EFX_MASK32(EFX_RX_CLASS_IPV6_UDP),
 216             RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE,
 217             (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6));
 218
 219         efx_mcdi_execute(enp, &req);
 220
 221         if (req.emr_rc != 0) {
 222                 rc = req.emr_rc;
 223                 goto fail2;
 224         }
 225
 226         return (0);
 227
 228 fail2:
 229         EFSYS_PROBE(fail2);
 230 fail1:
 231         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 232
 233         return (rc);
 234 }
 235 #endif /* EFSYS_OPT_RX_SCALE */
 236
 237 #if EFSYS_OPT_RX_SCALE
 238 static                  efx_rc_t
 239 efx_mcdi_rss_context_set_key(
 240         __in            efx_nic_t *enp,
 241         __in            uint32_t rss_context,
 242         __in_ecount(n)  uint8_t *key,
 243         __in            size_t n)
 244 {
 245         efx_mcdi_req_t req;
 246         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
 247                 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
 248         efx_rc_t rc;
 249
 250         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 251                 rc = EINVAL;
 252                 goto fail1;
 253         }
 254
 255         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
 256         req.emr_in_buf = payload;
 257         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
 258         req.emr_out_buf = payload;
 259         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
 260
 261         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
 262             rss_context);
 263
 264         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 265         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
 266                 rc = EINVAL;
 267                 goto fail2;
 268         }
 269
 270         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
 271             key, n);
 272
 273         efx_mcdi_execute(enp, &req);
 274
 275         if (req.emr_rc != 0) {
 276                 rc = req.emr_rc;
 277                 goto fail3;
 278         }
 279
 280         return (0);
 281
 282 fail3:
 283         EFSYS_PROBE(fail3);
 284 fail2:
 285         EFSYS_PROBE(fail2);
 286 fail1:
 287         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 288
 289         return (rc);
 290 }
 291 #endif /* EFSYS_OPT_RX_SCALE */
 292
 293 #if EFSYS_OPT_RX_SCALE
 294 static                  efx_rc_t
 295 efx_mcdi_rss_context_set_table(
 296         __in            efx_nic_t *enp,
 297         __in            uint32_t rss_context,
 298         __in_ecount(n)  unsigned int *table,
 299         __in            size_t n)
 300 {
 301         efx_mcdi_req_t req;
 302         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
 303                 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
 304         uint8_t *req_table;
 305         int i, rc;
 306
 307         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
 308                 rc = EINVAL;
 309                 goto fail1;
 310         }
 311
 312         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
 313         req.emr_in_buf = payload;
 314         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
 315         req.emr_out_buf = payload;
 316         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
 317
 318         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
 319             rss_context);
 320
 321         req_table =
 322             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
 323
 324         for (i = 0;
 325             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
 326             i++) {
 327                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
 328         }
 329
 330         efx_mcdi_execute(enp, &req);
 331
 332         if (req.emr_rc != 0) {
 333                 rc = req.emr_rc;
 334                 goto fail2;
 335         }
 336
 337         return (0);
 338
 339 fail2:
 340         EFSYS_PROBE(fail2);
 341 fail1:
 342         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 343
 344         return (rc);
 345 }
 346 #endif /* EFSYS_OPT_RX_SCALE */
 347
 348
 349         __checkReturn   efx_rc_t
 350 ef10_rx_init(
 351         __in            efx_nic_t *enp)
 352 {
 353 #if EFSYS_OPT_RX_SCALE
 354
 355         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
 356                 &enp->en_rss_context) == 0) {
 357                 /*
 358                  * Allocated an exclusive RSS context, which allows both the
 359                  * indirection table and key to be modified.
 360                  */
 361                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
 362                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
 363         } else {
 364                 /*
 365                  * Failed to allocate an exclusive RSS context. Continue
 366                  * operation without support for RSS. The pseudo-header in
 367                  * received packets will not contain a Toeplitz hash value.
 368                  */
 369                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
 370                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
 371         }
 372
 373 #endif /* EFSYS_OPT_RX_SCALE */
 374
 375         return (0);
 376 }
 377
 378 #if EFX_OPTS_EF10()
 379
 380 #if EFSYS_OPT_RX_SCATTER
 381         __checkReturn   efx_rc_t
 382 ef10_rx_scatter_enable(
 383         __in            efx_nic_t *enp,
 384         __in            unsigned int buf_size)
 385 {
 386         _NOTE(ARGUNUSED(enp, buf_size))
 387         return (0);
 388 }
 389 #endif  /* EFSYS_OPT_RX_SCATTER */
 390
 391 #endif  /* EFX_OPTS_EF10() */
 392
 393 #if EFSYS_OPT_RX_SCALE
 394         __checkReturn   efx_rc_t
 395 ef10_rx_scale_context_alloc(
 396         __in            efx_nic_t *enp,
 397         __in            efx_rx_scale_context_type_t type,
 398         __in            uint32_t num_queues,
 399         __out           uint32_t *rss_contextp)
 400 {
 401         efx_rc_t rc;
 402
 403         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
 404         if (rc != 0)
 405                 goto fail1;
 406
 407         return (0);
 408
 409 fail1:
 410         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 411         return (rc);
 412 }
 413 #endif /* EFSYS_OPT_RX_SCALE */
 414
 415 #if EFSYS_OPT_RX_SCALE
 416         __checkReturn   efx_rc_t
 417 ef10_rx_scale_context_free(
 418         __in            efx_nic_t *enp,
 419         __in            uint32_t rss_context)
 420 {
 421         efx_rc_t rc;
 422
 423         rc = efx_mcdi_rss_context_free(enp, rss_context);
 424         if (rc != 0)
 425                 goto fail1;
 426
 427         return (0);
 428
 429 fail1:
 430         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 431         return (rc);
 432 }
 433 #endif /* EFSYS_OPT_RX_SCALE */
 434
 435 #if EFSYS_OPT_RX_SCALE
 436         __checkReturn   efx_rc_t
 437 ef10_rx_scale_mode_set(
 438         __in            efx_nic_t *enp,
 439         __in            uint32_t rss_context,
 440         __in            efx_rx_hash_alg_t alg,
 441         __in            efx_rx_hash_type_t type,
 442         __in            boolean_t insert)
 443 {
 444         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 445         efx_rc_t rc;
 446
 447         EFSYS_ASSERT3U(insert, ==, B_TRUE);
 448
 449         if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 ||
 450             insert == B_FALSE) {
 451                 rc = EINVAL;
 452                 goto fail1;
 453         }
 454
 455         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 456                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 457                         rc = ENOTSUP;
 458                         goto fail2;
 459                 }
 460                 rss_context = enp->en_rss_context;
 461         }
 462
 463         if ((rc = efx_mcdi_rss_context_set_flags(enp,
 464                     rss_context, type)) != 0)
 465                 goto fail3;
 466
 467         return (0);
 468
 469 fail3:
 470         EFSYS_PROBE(fail3);
 471 fail2:
 472         EFSYS_PROBE(fail2);
 473 fail1:
 474         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 475
 476         return (rc);
 477 }
 478 #endif /* EFSYS_OPT_RX_SCALE */
 479
 480 #if EFSYS_OPT_RX_SCALE
 481         __checkReturn   efx_rc_t
 482 ef10_rx_scale_key_set(
 483         __in            efx_nic_t *enp,
 484         __in            uint32_t rss_context,
 485         __in_ecount(n)  uint8_t *key,
 486         __in            size_t n)
 487 {
 488         efx_rc_t rc;
 489
 490         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
 491             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
 492
 493         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 494                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 495                         rc = ENOTSUP;
 496                         goto fail1;
 497                 }
 498                 rss_context = enp->en_rss_context;
 499         }
 500
 501         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
 502                 goto fail2;
 503
 504         return (0);
 505
 506 fail2:
 507         EFSYS_PROBE(fail2);
 508 fail1:
 509         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 510
 511         return (rc);
 512 }
 513 #endif /* EFSYS_OPT_RX_SCALE */
 514
 515 #if EFSYS_OPT_RX_SCALE
 516         __checkReturn   efx_rc_t
 517 ef10_rx_scale_tbl_set(
 518         __in            efx_nic_t *enp,
 519         __in            uint32_t rss_context,
 520         __in_ecount(n)  unsigned int *table,
 521         __in            size_t n)
 522 {
 523         efx_rc_t rc;
 524
 525
 526         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
 527                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
 528                         rc = ENOTSUP;
 529                         goto fail1;
 530                 }
 531                 rss_context = enp->en_rss_context;
 532         }
 533
 534         if ((rc = efx_mcdi_rss_context_set_table(enp,
 535                     rss_context, table, n)) != 0)
 536                 goto fail2;
 537
 538         return (0);
 539
 540 fail2:
 541         EFSYS_PROBE(fail2);
 542 fail1:
 543         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 544
 545         return (rc);
 546 }
 547 #endif /* EFSYS_OPT_RX_SCALE */
 548
 549 #if EFX_OPTS_EF10()
 550
 551 /*
 552  * EF10 RX pseudo-header
 553  * ---------------------
 554  *
 555  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
 556  *
 557  *  +00: Toeplitz hash value.
 558  *       (32bit little-endian)
 559  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
 560  *       (16bit big-endian)
 561  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
 562  *       (16bit big-endian)
 563  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
 564  *       (16bit little-endian)
 565  *  +10: MAC timestamp. Zero if timestamping is not enabled.
 566  *       (32bit little-endian)
 567  *
 568  * See "The RX Pseudo-header" in SF-109306-TC.
 569  */
 570
 571         __checkReturn   efx_rc_t
 572 ef10_rx_prefix_pktlen(
 573         __in            efx_nic_t *enp,
 574         __in            uint8_t *buffer,
 575         __out           uint16_t *lengthp)
 576 {
 577         _NOTE(ARGUNUSED(enp))
 578
 579         /*
 580          * The RX pseudo-header contains the packet length, excluding the
 581          * pseudo-header. If the hardware receive datapath was operating in
 582          * cut-through mode then the length in the RX pseudo-header will be
 583          * zero, and the packet length must be obtained from the DMA length
 584          * reported in the RX event.
 585          */
 586         *lengthp = buffer[8] | (buffer[9] << 8);
 587         return (0);
 588 }
 589
 590 #if EFSYS_OPT_RX_SCALE
 591         __checkReturn   uint32_t
 592 ef10_rx_prefix_hash(
 593         __in            efx_nic_t *enp,
 594         __in            efx_rx_hash_alg_t func,
 595         __in            uint8_t *buffer)
 596 {
 597         _NOTE(ARGUNUSED(enp))
 598
 599         switch (func) {
 600         case EFX_RX_HASHALG_PACKED_STREAM:
 601         case EFX_RX_HASHALG_TOEPLITZ:
 602                 return (buffer[0] |
 603                     (buffer[1] << 8) |
 604                     (buffer[2] << 16) |
 605                     (buffer[3] << 24));
 606
 607         default:
 608                 EFSYS_ASSERT(0);
 609                 return (0);
 610         }
 611 }
 612 #endif /* EFSYS_OPT_RX_SCALE */
 613
 614 #if EFSYS_OPT_RX_PACKED_STREAM
 615 /*
 616  * Fake length for RXQ descriptors in packed stream mode
 617  * to make hardware happy
 618  */
 619 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
 620 #endif
 621
 622                                 void
 623 ef10_rx_qpost(
 624         __in                    efx_rxq_t *erp,
 625         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
 626         __in                    size_t size,
 627         __in                    unsigned int ndescs,
 628         __in                    unsigned int completed,
 629         __in                    unsigned int added)
 630 {
 631         efx_qword_t qword;
 632         unsigned int i;
 633         unsigned int offset;
 634         unsigned int id;
 635
 636         _NOTE(ARGUNUSED(completed))
 637
 638 #if EFSYS_OPT_RX_PACKED_STREAM
 639         /*
 640          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
 641          * and equal to 0 after applying mask. Hardware does not like it.
 642          */
 643         if (erp->er_ev_qstate->eers_rx_packed_stream)
 644                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
 645 #endif
 646
 647         /* The client driver must not overfill the queue */
 648         EFSYS_ASSERT3U(added - completed + ndescs, <=,
 649             EFX_RXQ_LIMIT(erp->er_mask + 1));
 650
 651         id = added & (erp->er_mask);
 652         for (i = 0; i < ndescs; i++) {
 653                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
 654                     unsigned int, id, efsys_dma_addr_t, addrp[i],
 655                     size_t, size);
 656
 657                 EFX_POPULATE_QWORD_3(qword,
 658                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
 659                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
 660                     (uint32_t)(addrp[i] & 0xffffffff),
 661                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
 662                     (uint32_t)(addrp[i] >> 32));
 663
 664                 offset = id * sizeof (efx_qword_t);
 665                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
 666
 667                 id = (id + 1) & (erp->er_mask);
 668         }
 669 }
 670
 671                         void
 672 ef10_rx_qpush(
 673         __in    efx_rxq_t *erp,
 674         __in    unsigned int added,
 675         __inout unsigned int *pushedp)
 676 {
 677         efx_nic_t *enp = erp->er_enp;
 678         unsigned int pushed = *pushedp;
 679         uint32_t wptr;
 680         efx_dword_t dword;
 681
 682         /* Hardware has alignment restriction for WPTR */
 683         wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN);
 684         if (pushed == wptr)
 685                 return;
 686
 687         *pushedp = wptr;
 688
 689         /* Push the populated descriptors out */
 690         wptr &= erp->er_mask;
 691
 692         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
 693
 694         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
 695         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
 696             wptr, pushed & erp->er_mask);
 697         EFSYS_PIO_WRITE_BARRIER();
 698         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 699             erp->er_index, &dword, B_FALSE);
 700 }
 701
 702 #if EFSYS_OPT_RX_PACKED_STREAM
 703
 704                         void
 705 ef10_rx_qpush_ps_credits(
 706         __in            efx_rxq_t *erp)
 707 {
 708         efx_nic_t *enp = erp->er_enp;
 709         efx_dword_t dword;
 710         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 711         uint32_t credits;
 712
 713         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 714
 715         if (rxq_state->eers_rx_packed_stream_credits == 0)
 716                 return;
 717
 718         /*
 719          * It is a bug if we think that FW has utilized more
 720          * credits than it is allowed to have (maximum). However,
 721          * make sure that we do not credit more than maximum anyway.
 722          */
 723         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
 724             EFX_RX_PACKED_STREAM_MAX_CREDITS);
 725         EFX_POPULATE_DWORD_3(dword,
 726             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
 727             ERF_DZ_RX_DESC_MAGIC_CMD,
 728             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
 729             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
 730         EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
 731             erp->er_index, &dword, B_FALSE);
 732
 733         rxq_state->eers_rx_packed_stream_credits = 0;
 734 }
 735
 736 /*
 737  * In accordance with SF-112241-TC the received data has the following layout:
 738  *  - 8 byte pseudo-header which consist of:
 739  *    - 4 byte little-endian timestamp
 740  *    - 2 byte little-endian captured length in bytes
 741  *    - 2 byte little-endian original packet length in bytes
 742  *  - captured packet bytes
 743  *  - optional padding to align to 64 bytes boundary
 744  *  - 64 bytes scratch space for the host software
 745  */
 746         __checkReturn   uint8_t *
 747 ef10_rx_qps_packet_info(
 748         __in            efx_rxq_t *erp,
 749         __in            uint8_t *buffer,
 750         __in            uint32_t buffer_length,
 751         __in            uint32_t current_offset,
 752         __out           uint16_t *lengthp,
 753         __out           uint32_t *next_offsetp,
 754         __out           uint32_t *timestamp)
 755 {
 756         uint16_t buf_len;
 757         uint8_t *pkt_start;
 758         efx_qword_t *qwordp;
 759         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
 760
 761         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
 762
 763         buffer += current_offset;
 764         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
 765
 766         qwordp = (efx_qword_t *)buffer;
 767         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
 768         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
 769         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
 770
 771         buf_len = EFX_P2ROUNDUP(uint16_t,
 772             buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
 773             EFX_RX_PACKED_STREAM_ALIGNMENT);
 774         *next_offsetp =
 775             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
 776
 777         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
 778         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
 779
 780         if ((*next_offsetp ^ current_offset) &
 781             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
 782                 rxq_state->eers_rx_packed_stream_credits++;
 783
 784         return (pkt_start);
 785 }
 786
 787
 788 #endif
 789
 790         __checkReturn   efx_rc_t
 791 ef10_rx_qflush(
 792         __in    efx_rxq_t *erp)
 793 {
 794         efx_nic_t *enp = erp->er_enp;
 795         efx_rc_t rc;
 796
 797         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
 798                 goto fail1;
 799
 800         return (0);
 801
 802 fail1:
 803         /*
 804          * EALREADY is not an error, but indicates that the MC has rebooted and
 805          * that the RXQ has already been destroyed. Callers need to know that
 806          * the RXQ flush has completed to avoid waiting until timeout for a
 807          * flush done event that will not be delivered.
 808          */
 809         if (rc != EALREADY)
 810                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
 811
 812         return (rc);
 813 }
 814
 815                 void
 816 ef10_rx_qenable(
 817         __in    efx_rxq_t *erp)
 818 {
 819         /* FIXME */
 820         _NOTE(ARGUNUSED(erp))
 821         /* FIXME */
 822 }
 823
 824         __checkReturn   efx_rc_t
 825 ef10_rx_qcreate(
 826         __in            efx_nic_t *enp,
 827         __in            unsigned int index,
 828         __in            unsigned int label,
 829         __in            efx_rxq_type_t type,
 830         __in_opt        const efx_rxq_type_data_t *type_data,
 831         __in            efsys_mem_t *esmp,
 832         __in            size_t ndescs,
 833         __in            uint32_t id,
 834         __in            unsigned int flags,
 835         __in            efx_evq_t *eep,
 836         __in            efx_rxq_t *erp)
 837 {
 838         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
 839         efx_rc_t rc;
 840         boolean_t disable_scatter;
 841         boolean_t want_inner_classes;
 842         unsigned int ps_buf_size;
 843         uint32_t es_bufs_per_desc = 0;
 844         uint32_t es_max_dma_len = 0;
 845         uint32_t es_buf_stride = 0;
 846         uint32_t hol_block_timeout = 0;
 847
 848         _NOTE(ARGUNUSED(id, erp))
 849
 850         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
 851         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
 852
 853         switch (type) {
 854         case EFX_RXQ_TYPE_DEFAULT:
 855                 if (type_data == NULL) {
 856                         rc = EINVAL;
 857                         goto fail1;
 858                 }
 859                 erp->er_buf_size = type_data->ertd_default.ed_buf_size;
 860                 ps_buf_size = 0;
 861                 break;
 862 #if EFSYS_OPT_RX_PACKED_STREAM
 863         case EFX_RXQ_TYPE_PACKED_STREAM:
 864                 if (type_data == NULL) {
 865                         rc = EINVAL;
 866                         goto fail2;
 867                 }
 868                 switch (type_data->ertd_packed_stream.eps_buf_size) {
 869                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M:
 870                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
 871                         break;
 872                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K:
 873                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
 874                         break;
 875                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K:
 876                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
 877                         break;
 878                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K:
 879                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
 880                         break;
 881                 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K:
 882                         ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
 883                         break;
 884                 default:
 885                         rc = ENOTSUP;
 886                         goto fail3;
 887                 }
 888                 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size;
 889                 break;
 890 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 891 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 892         case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
 893                 if (type_data == NULL) {
 894                         rc = EINVAL;
 895                         goto fail4;
 896                 }
 897                 ps_buf_size = 0;
 898                 es_bufs_per_desc =
 899                     type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
 900                 es_max_dma_len =
 901                     type_data->ertd_es_super_buffer.eessb_max_dma_len;
 902                 es_buf_stride =
 903                     type_data->ertd_es_super_buffer.eessb_buf_stride;
 904                 hol_block_timeout =
 905                     type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
 906                 break;
 907 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 908         default:
 909                 rc = ENOTSUP;
 910                 goto fail5;
 911         }
 912
 913 #if EFSYS_OPT_RX_PACKED_STREAM
 914         if (ps_buf_size != 0) {
 915                 /* Check if datapath firmware supports packed stream mode */
 916                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
 917                         rc = ENOTSUP;
 918                         goto fail6;
 919                 }
 920                 /* Check if packed stream allows configurable buffer sizes */
 921                 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) &&
 922                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
 923                         rc = ENOTSUP;
 924                         goto fail7;
 925                 }
 926         }
 927 #else /* EFSYS_OPT_RX_PACKED_STREAM */
 928         EFSYS_ASSERT(ps_buf_size == 0);
 929 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 930
 931 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 932         if (es_bufs_per_desc > 0) {
 933                 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
 934                         rc = ENOTSUP;
 935                         goto fail8;
 936                 }
 937                 if (!EFX_IS_P2ALIGNED(uint32_t, es_max_dma_len,
 938                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
 939                         rc = EINVAL;
 940                         goto fail9;
 941                 }
 942                 if (!EFX_IS_P2ALIGNED(uint32_t, es_buf_stride,
 943                             EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
 944                         rc = EINVAL;
 945                         goto fail10;
 946                 }
 947         }
 948 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 949         EFSYS_ASSERT(es_bufs_per_desc == 0);
 950 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 951
 952         /* Scatter can only be disabled if the firmware supports doing so */
 953         if (flags & EFX_RXQ_FLAG_SCATTER)
 954                 disable_scatter = B_FALSE;
 955         else
 956                 disable_scatter = encp->enc_rx_disable_scatter_supported;
 957
 958         if (flags & EFX_RXQ_FLAG_INNER_CLASSES)
 959                 want_inner_classes = B_TRUE;
 960         else
 961                 want_inner_classes = B_FALSE;
 962
 963         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index,
 964                     esmp, disable_scatter, want_inner_classes, erp->er_buf_size,
 965                     ps_buf_size, es_bufs_per_desc, es_max_dma_len,
 966                     es_buf_stride, hol_block_timeout)) != 0)
 967                 goto fail11;
 968
 969         erp->er_eep = eep;
 970         erp->er_label = label;
 971
 972         ef10_ev_rxlabel_init(eep, erp, label, type);
 973
 974         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
 975
 976         return (0);
 977
 978 fail11:
 979         EFSYS_PROBE(fail11);
 980 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 981 fail10:
 982         EFSYS_PROBE(fail10);
 983 fail9:
 984         EFSYS_PROBE(fail9);
 985 fail8:
 986         EFSYS_PROBE(fail8);
 987 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 988 #if EFSYS_OPT_RX_PACKED_STREAM
 989 fail7:
 990         EFSYS_PROBE(fail7);
 991 fail6:
 992         EFSYS_PROBE(fail6);
 993 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 994 fail5:
 995         EFSYS_PROBE(fail5);
 996 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
 997 fail4:
 998         EFSYS_PROBE(fail4);
 999 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
1000 #if EFSYS_OPT_RX_PACKED_STREAM
1001 fail3:
1002         EFSYS_PROBE(fail3);
1003 fail2:
1004         EFSYS_PROBE(fail2);
1005 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1006 fail1:
1007         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1008
1009         return (rc);
1010 }
1011
1012                 void
1013 ef10_rx_qdestroy(
1014         __in    efx_rxq_t *erp)
1015 {
1016         efx_evq_t *eep = erp->er_eep;
1017         unsigned int label = erp->er_label;
1018
1019         ef10_ev_rxlabel_fini(eep, label);
1020 }
1021
1022 #endif /* EFX_OPTS_EF10() */
1023
1024                 void
1025 ef10_rx_fini(
1026         __in    efx_nic_t *enp)
1027 {
1028 #if EFSYS_OPT_RX_SCALE
1029         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1030                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1031         enp->en_rss_context = 0;
1032         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1033 #else
1034         _NOTE(ARGUNUSED(enp))
1035 #endif /* EFSYS_OPT_RX_SCALE */
1036 }
1037
1038 #endif /* EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() */