drivers/net/sfc/sfc_flow_rss.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2022 Xilinx, Inc.
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_flow.h>
  11 #include <rte_tailq.h>
  12
  13 #include "efx.h"
  14
  15 #include "sfc.h"
  16 #include "sfc_debug.h"
  17 #include "sfc_flow_rss.h"
  18 #include "sfc_log.h"
  19 #include "sfc_rx.h"
  20
  21 int
  22 sfc_flow_rss_attach(struct sfc_adapter *sa)
  23 {
  24         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
  25         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
  26         int rc;
  27
  28         sfc_log_init(sa, "entry");
  29
  30         flow_rss->qid_span_max = encp->enc_rx_scale_indirection_max_nqueues;
  31         flow_rss->nb_tbl_entries_min = encp->enc_rx_scale_tbl_min_nentries;
  32         flow_rss->nb_tbl_entries_max = encp->enc_rx_scale_tbl_max_nentries;
  33
  34         sfc_log_init(sa, "allocate the bounce buffer for indirection entries");
  35         flow_rss->bounce_tbl = rte_calloc("sfc_flow_rss_bounce_tbl",
  36                                           flow_rss->nb_tbl_entries_max,
  37                                           sizeof(*flow_rss->bounce_tbl), 0);
  38         if (flow_rss->bounce_tbl == NULL) {
  39                 rc = ENOMEM;
  40                 goto fail;
  41         }
  42
  43         TAILQ_INIT(&flow_rss->ctx_list);
  44
  45         sfc_log_init(sa, "done");
  46
  47         return 0;
  48
  49 fail:
  50         sfc_log_init(sa, "failed %d", rc);
  51
  52         return rc;
  53 }
  54
  55 void
  56 sfc_flow_rss_detach(struct sfc_adapter *sa)
  57 {
  58         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
  59
  60         sfc_log_init(sa, "entry");
  61
  62         sfc_log_init(sa, "free the bounce buffer for indirection entries");
  63         rte_free(flow_rss->bounce_tbl);
  64
  65         sfc_log_init(sa, "done");
  66 }
  67
  68 int
  69 sfc_flow_rss_parse_conf(struct sfc_adapter *sa,
  70                         const struct rte_flow_action_rss *in,
  71                         struct sfc_flow_rss_conf *out, uint16_t *sw_qid_minp)
  72 {
  73         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
  74         const struct sfc_flow_rss *flow_rss = &sa->flow_rss;
  75         const struct sfc_rss *ethdev_rss = &sas->rss;
  76         uint16_t sw_qid_min;
  77         uint16_t sw_qid_max;
  78         const uint8_t *key;
  79         unsigned int i;
  80         int rc;
  81
  82         if (in->level) {
  83                 /*
  84                  * The caller demands that RSS hash be computed
  85                  * within the given encapsulation frame / level.
  86                  * Per flow control for that is not implemented.
  87                  */
  88                 sfc_err(sa, "flow-rss: parse: 'level' must be 0");
  89                 return EINVAL;
  90         }
  91
  92         if (in->types != 0) {
  93                 rc = sfc_rx_hf_rte_to_efx(sa, in->types,
  94                                           &out->efx_hash_types);
  95                 if (rc != 0) {
  96                         sfc_err(sa, "flow-rss: parse: failed to process 'types'");
  97                         return rc;
  98                 }
  99         } else {
 100                 sfc_dbg(sa, "flow-rss: parse: 'types' is 0; proceeding with ethdev setting");
 101                 out->efx_hash_types = ethdev_rss->hash_types;
 102         }
 103
 104         if (in->key_len != 0) {
 105                 if (in->key_len != sizeof(out->key)) {
 106                         sfc_err(sa, "flow-rss: parse: 'key_len' must be either %zu or 0",
 107                                 sizeof(out->key));
 108                         return EINVAL;
 109                 }
 110
 111                 if (in->key == NULL) {
 112                         sfc_err(sa, "flow-rss: parse: 'key' is NULL");
 113                         return EINVAL;
 114                 }
 115
 116                 key = in->key;
 117         } else {
 118                 sfc_dbg(sa, "flow-rss: parse: 'key_len' is 0; proceeding with ethdev key");
 119                 key = ethdev_rss->key;
 120         }
 121
 122         rte_memcpy(out->key, key, sizeof(out->key));
 123
 124         switch (in->func) {
 125         case RTE_ETH_HASH_FUNCTION_DEFAULT:
 126                 /*
 127                  * DEFAULT means that conformance to a specific
 128                  * hash algorithm is a don't care to the caller.
 129                  * The driver can pick the one it deems optimal.
 130                  */
 131                 break;
 132         case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
 133                 if (ethdev_rss->hash_alg != EFX_RX_HASHALG_TOEPLITZ) {
 134                         sfc_err(sa, "flow-rss: parse: 'func' TOEPLITZ is unavailable; use DEFAULT");
 135                         return EINVAL;
 136                 }
 137                 break;
 138         default:
 139                 sfc_err(sa, "flow-rss: parse: 'func' #%d is unsupported", in->func);
 140                 return EINVAL;
 141         }
 142
 143         if (in->queue_num == 0) {
 144                 sfc_err(sa, "flow-rss: parse: 'queue_num' is 0; MIN=1");
 145                 return EINVAL;
 146         }
 147
 148         if (in->queue_num > flow_rss->nb_tbl_entries_max) {
 149                 sfc_err(sa, "flow-rss: parse: 'queue_num' is too large; MAX=%u",
 150                         flow_rss->nb_tbl_entries_max);
 151                 return EINVAL;
 152         }
 153
 154         if (in->queue == NULL) {
 155                 sfc_err(sa, "flow-rss: parse: 'queue' is NULL");
 156                 return EINVAL;
 157         }
 158
 159         sw_qid_min = sas->ethdev_rxq_count - 1;
 160         sw_qid_max = 0;
 161
 162         out->nb_qid_offsets = 0;
 163
 164         for (i = 0; i < in->queue_num; ++i) {
 165                 uint16_t sw_qid = in->queue[i];
 166
 167                 if (sw_qid >= sas->ethdev_rxq_count) {
 168                         sfc_err(sa, "flow-rss: parse: queue=%u does not exist",
 169                                 sw_qid);
 170                         return EINVAL;
 171                 }
 172
 173                 if (sw_qid < sw_qid_min)
 174                         sw_qid_min = sw_qid;
 175
 176                 if (sw_qid > sw_qid_max)
 177                         sw_qid_max = sw_qid;
 178
 179                 if (sw_qid != in->queue[0] + i)
 180                         out->nb_qid_offsets = in->queue_num;
 181         }
 182
 183         out->qid_span = sw_qid_max - sw_qid_min + 1;
 184
 185         if (out->qid_span > flow_rss->qid_span_max) {
 186                 sfc_err(sa, "flow-rss: parse: queue ID span %u is too large; MAX=%u",
 187                         out->qid_span, flow_rss->qid_span_max);
 188                 return EINVAL;
 189         }
 190
 191         if (sw_qid_minp != NULL)
 192                 *sw_qid_minp = sw_qid_min;
 193
 194         return 0;
 195 }
 196
 197 struct sfc_flow_rss_ctx *
 198 sfc_flow_rss_ctx_reuse(struct sfc_adapter *sa,
 199                        const struct sfc_flow_rss_conf *conf,
 200                        uint16_t sw_qid_min, const uint16_t *sw_qids)
 201 {
 202         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 203         static struct sfc_flow_rss_ctx *ctx;
 204
 205         SFC_ASSERT(sfc_adapter_is_locked(sa));
 206
 207         TAILQ_FOREACH(ctx, &flow_rss->ctx_list, entries) {
 208                 if (memcmp(&ctx->conf, conf, sizeof(*conf)) != 0)
 209                         continue;
 210
 211                 if (conf->nb_qid_offsets != 0) {
 212                         bool match_confirmed = true;
 213                         unsigned int i;
 214
 215                         for (i = 0; i < conf->nb_qid_offsets; ++i) {
 216                                 uint16_t qid_offset = sw_qids[i] - sw_qid_min;
 217
 218                                 if (ctx->qid_offsets[i] != qid_offset) {
 219                                         match_confirmed = false;
 220                                         break;
 221                                 }
 222                         }
 223
 224                         if (!match_confirmed)
 225                                 continue;
 226                 }
 227
 228                 sfc_dbg(sa, "flow-rss: reusing ctx=%p", ctx);
 229                 ++(ctx->refcnt);
 230                 return ctx;
 231         }
 232
 233         return NULL;
 234 }
 235
 236 int
 237 sfc_flow_rss_ctx_add(struct sfc_adapter *sa,
 238                      const struct sfc_flow_rss_conf *conf, uint16_t sw_qid_min,
 239                      const uint16_t *sw_qids, struct sfc_flow_rss_ctx **ctxp)
 240 {
 241         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 242         struct sfc_flow_rss_ctx *ctx;
 243
 244         SFC_ASSERT(sfc_adapter_is_locked(sa));
 245
 246         ctx = rte_zmalloc("sfc_flow_rss_ctx", sizeof(*ctx), 0);
 247         if (ctx == NULL)
 248                 return ENOMEM;
 249
 250         if (conf->nb_qid_offsets != 0) {
 251                 unsigned int i;
 252
 253                 ctx->qid_offsets = rte_calloc("sfc_flow_rss_ctx_qid_offsets",
 254                                               conf->nb_qid_offsets,
 255                                               sizeof(*ctx->qid_offsets), 0);
 256                 if (ctx->qid_offsets == NULL) {
 257                         rte_free(ctx);
 258                         return ENOMEM;
 259                 }
 260
 261                 for (i = 0; i < conf->nb_qid_offsets; ++i)
 262                         ctx->qid_offsets[i] = sw_qids[i] - sw_qid_min;
 263         }
 264
 265         ctx->conf = *conf;
 266         ctx->refcnt = 1;
 267
 268         TAILQ_INSERT_TAIL(&flow_rss->ctx_list, ctx, entries);
 269
 270         *ctxp = ctx;
 271
 272         sfc_dbg(sa, "flow-rss: added ctx=%p", ctx);
 273
 274         return 0;
 275 }
 276
 277 void
 278 sfc_flow_rss_ctx_del(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 279 {
 280         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 281
 282         if (ctx == NULL)
 283                 return;
 284
 285         SFC_ASSERT(sfc_adapter_is_locked(sa));
 286
 287         if (ctx->dummy)
 288                 return;
 289
 290         SFC_ASSERT(ctx->refcnt != 0);
 291
 292         --(ctx->refcnt);
 293
 294         if (ctx->refcnt != 0)
 295                 return;
 296
 297         if (ctx->nic_handle_refcnt != 0) {
 298                 sfc_err(sa, "flow-rss: deleting ctx=%p abandons its NIC resource: handle=0x%08x, refcnt=%u",
 299                         ctx, ctx->nic_handle, ctx->nic_handle_refcnt);
 300         }
 301
 302         TAILQ_REMOVE(&flow_rss->ctx_list, ctx, entries);
 303         rte_free(ctx->qid_offsets);
 304         rte_free(ctx);
 305
 306         sfc_dbg(sa, "flow-rss: deleted ctx=%p", ctx);
 307 }
 308
 309 static int
 310 sfc_flow_rss_ctx_program_tbl(struct sfc_adapter *sa,
 311                              unsigned int nb_tbl_entries,
 312                              const struct sfc_flow_rss_ctx *ctx)
 313 {
 314         const struct sfc_flow_rss_conf *conf = &ctx->conf;
 315         unsigned int *tbl = sa->flow_rss.bounce_tbl;
 316         unsigned int i;
 317
 318         SFC_ASSERT(sfc_adapter_is_locked(sa));
 319
 320         if (conf->nb_qid_offsets != 0) {
 321                 SFC_ASSERT(ctx->qid_offsets != NULL);
 322
 323                 for (i = 0; i < nb_tbl_entries; ++i)
 324                         tbl[i] = ctx->qid_offsets[i % conf->nb_qid_offsets];
 325         } else {
 326                 for (i = 0; i < nb_tbl_entries; ++i)
 327                         tbl[i] = i % conf->qid_span;
 328         }
 329
 330         return efx_rx_scale_tbl_set(sa->nic, ctx->nic_handle,
 331                                     tbl, nb_tbl_entries);
 332 }
 333
 334 int
 335 sfc_flow_rss_ctx_program(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 336 {
 337         efx_rx_scale_context_type_t ctx_type = EFX_RX_SCALE_EXCLUSIVE;
 338         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 339         const struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 340         struct sfc_rss *ethdev_rss = &sas->rss;
 341         struct sfc_flow_rss_conf *conf;
 342         bool allocation_done = B_FALSE;
 343         unsigned int nb_qid_offsets;
 344         unsigned int nb_tbl_entries;
 345         int rc;
 346
 347         if (ctx == NULL)
 348                 return 0;
 349
 350         conf = &ctx->conf;
 351
 352         SFC_ASSERT(sfc_adapter_is_locked(sa));
 353
 354         if (conf->nb_qid_offsets != 0)
 355                 nb_qid_offsets = conf->nb_qid_offsets;
 356         else
 357                 nb_qid_offsets = conf->qid_span;
 358
 359         if (!RTE_IS_POWER_OF_2(nb_qid_offsets)) {
 360                 /*
 361                  * Most likely, it pays to enlarge the indirection
 362                  * table to facilitate better distribution quality.
 363                  */
 364                 nb_qid_offsets = flow_rss->nb_tbl_entries_max;
 365         }
 366
 367         nb_tbl_entries = RTE_MAX(flow_rss->nb_tbl_entries_min, nb_qid_offsets);
 368
 369         if (ctx->nic_handle_refcnt == 0) {
 370                 rc = efx_rx_scale_context_alloc_v2(sa->nic, ctx_type,
 371                                                    conf->qid_span,
 372                                                    nb_tbl_entries,
 373                                                    &ctx->nic_handle);
 374                 if (rc != 0) {
 375                         sfc_err(sa, "flow-rss: failed to allocate NIC resource for ctx=%p: type=%d, qid_span=%u, nb_tbl_entries=%u; rc=%d",
 376                                 ctx, ctx_type, conf->qid_span, nb_tbl_entries, rc);
 377                         goto fail;
 378                 }
 379
 380                 sfc_dbg(sa, "flow-rss: allocated NIC resource for ctx=%p: type=%d, qid_span=%u, nb_tbl_entries=%u; handle=0x%08x",
 381                         ctx, ctx_type, conf->qid_span, nb_tbl_entries,
 382                         ctx->nic_handle);
 383
 384                 ++(ctx->nic_handle_refcnt);
 385                 allocation_done = B_TRUE;
 386         } else {
 387                 ++(ctx->nic_handle_refcnt);
 388                 return 0;
 389         }
 390
 391         rc = efx_rx_scale_mode_set(sa->nic, ctx->nic_handle,
 392                                    ethdev_rss->hash_alg,
 393                                    (ctx->dummy) ? ethdev_rss->hash_types :
 394                                                   conf->efx_hash_types,
 395                                    B_TRUE);
 396         if (rc != 0) {
 397                 sfc_err(sa, "flow-rss: failed to configure hash for ctx=%p: efx_hash_alg=%d, efx_hash_types=0x%08x; rc=%d",
 398                         ctx, ethdev_rss->hash_alg,
 399                         (ctx->dummy) ? ethdev_rss->hash_types :
 400                                        conf->efx_hash_types,
 401                         rc);
 402                 goto fail;
 403         }
 404
 405         rc = efx_rx_scale_key_set(sa->nic, ctx->nic_handle,
 406                                   (ctx->dummy) ? ethdev_rss->key : conf->key,
 407                                   RTE_DIM(conf->key));
 408         if (rc != 0) {
 409                 sfc_err(sa, "flow-rss: failed to set key for ctx=%p; rc=%d",
 410                         ctx, rc);
 411                 goto fail;
 412         }
 413
 414         rc = sfc_flow_rss_ctx_program_tbl(sa, nb_tbl_entries, ctx);
 415         if (rc != 0) {
 416                 sfc_err(sa, "flow-rss: failed to program table for ctx=%p: nb_tbl_entries=%u; rc=%d",
 417                         ctx, nb_tbl_entries, rc);
 418                 goto fail;
 419         }
 420
 421         return 0;
 422
 423 fail:
 424         if (allocation_done)
 425                 sfc_flow_rss_ctx_terminate(sa, ctx);
 426
 427         return rc;
 428 }
 429
 430 void
 431 sfc_flow_rss_ctx_terminate(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 432 {
 433         if (ctx == NULL)
 434                 return;
 435
 436         SFC_ASSERT(sfc_adapter_is_locked(sa));
 437
 438         SFC_ASSERT(ctx->nic_handle_refcnt != 0);
 439         --(ctx->nic_handle_refcnt);
 440
 441         if (ctx->nic_handle_refcnt == 0) {
 442                 int rc;
 443
 444                 rc = efx_rx_scale_context_free(sa->nic, ctx->nic_handle);
 445                 if (rc != 0) {
 446                         sfc_err(sa, "flow-rss: failed to release NIC resource for ctx=%p: handle=0x%08x; rc=%d",
 447                                 ctx, ctx->nic_handle, rc);
 448
 449                         sfc_warn(sa, "flow-rss: proceeding despite the prior error");
 450                 }
 451
 452                 sfc_dbg(sa, "flow-rss: released NIC resource for ctx=%p; rc=%d",
 453                         ctx, rc);
 454         }
 455 }