drivers/net/sfc/sfc_flow_rss.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2022 Xilinx, Inc.
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_flow.h>
  11 #include <rte_tailq.h>
  12
  13 #include "efx.h"
  14
  15 #include "sfc.h"
  16 #include "sfc_debug.h"
  17 #include "sfc_flow_rss.h"
  18 #include "sfc_log.h"
  19 #include "sfc_rx.h"
  20
  21 int
  22 sfc_flow_rss_attach(struct sfc_adapter *sa)
  23 {
  24         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
  25
  26         sfc_log_init(sa, "entry");
  27
  28         TAILQ_INIT(&flow_rss->ctx_list);
  29
  30         sfc_log_init(sa, "done");
  31
  32         return 0;
  33 }
  34
  35 void
  36 sfc_flow_rss_detach(struct sfc_adapter *sa)
  37 {
  38         sfc_log_init(sa, "entry");
  39
  40         sfc_log_init(sa, "done");
  41 }
  42
  43 int
  44 sfc_flow_rss_parse_conf(struct sfc_adapter *sa,
  45                         const struct rte_flow_action_rss *in,
  46                         struct sfc_flow_rss_conf *out, uint16_t *sw_qid_minp)
  47 {
  48         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
  49         const struct sfc_rss *ethdev_rss = &sas->rss;
  50         uint16_t sw_qid_min;
  51         uint16_t sw_qid_max;
  52         const uint8_t *key;
  53         unsigned int i;
  54         int rc;
  55
  56         if (in->level) {
  57                 /*
  58                  * The caller demands that RSS hash be computed
  59                  * within the given encapsulation frame / level.
  60                  * Per flow control for that is not implemented.
  61                  */
  62                 sfc_err(sa, "flow-rss: parse: 'level' must be 0");
  63                 return EINVAL;
  64         }
  65
  66         if (in->types != 0) {
  67                 rc = sfc_rx_hf_rte_to_efx(sa, in->types,
  68                                           &out->efx_hash_types);
  69                 if (rc != 0) {
  70                         sfc_err(sa, "flow-rss: parse: failed to process 'types'");
  71                         return rc;
  72                 }
  73         } else {
  74                 sfc_dbg(sa, "flow-rss: parse: 'types' is 0; proceeding with ethdev setting");
  75                 out->efx_hash_types = ethdev_rss->hash_types;
  76         }
  77
  78         if (in->key_len != 0) {
  79                 if (in->key_len != sizeof(out->key)) {
  80                         sfc_err(sa, "flow-rss: parse: 'key_len' must be either %zu or 0",
  81                                 sizeof(out->key));
  82                         return EINVAL;
  83                 }
  84
  85                 if (in->key == NULL) {
  86                         sfc_err(sa, "flow-rss: parse: 'key' is NULL");
  87                         return EINVAL;
  88                 }
  89
  90                 key = in->key;
  91         } else {
  92                 sfc_dbg(sa, "flow-rss: parse: 'key_len' is 0; proceeding with ethdev key");
  93                 key = ethdev_rss->key;
  94         }
  95
  96         rte_memcpy(out->key, key, sizeof(out->key));
  97
  98         switch (in->func) {
  99         case RTE_ETH_HASH_FUNCTION_DEFAULT:
 100                 /*
 101                  * DEFAULT means that conformance to a specific
 102                  * hash algorithm is a don't care to the caller.
 103                  * The driver can pick the one it deems optimal.
 104                  */
 105                 break;
 106         case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
 107                 if (ethdev_rss->hash_alg != EFX_RX_HASHALG_TOEPLITZ) {
 108                         sfc_err(sa, "flow-rss: parse: 'func' TOEPLITZ is unavailable; use DEFAULT");
 109                         return EINVAL;
 110                 }
 111                 break;
 112         default:
 113                 sfc_err(sa, "flow-rss: parse: 'func' #%d is unsupported", in->func);
 114                 return EINVAL;
 115         }
 116
 117         if (in->queue_num == 0) {
 118                 sfc_err(sa, "flow-rss: parse: 'queue_num' is 0; MIN=1");
 119                 return EINVAL;
 120         }
 121
 122         if (in->queue_num > EFX_RSS_TBL_SIZE) {
 123                 sfc_err(sa, "flow-rss: parse: 'queue_num' is too large; MAX=%u",
 124                         EFX_RSS_TBL_SIZE);
 125                 return EINVAL;
 126         }
 127
 128         if (in->queue == NULL) {
 129                 sfc_err(sa, "flow-rss: parse: 'queue' is NULL");
 130                 return EINVAL;
 131         }
 132
 133         sw_qid_min = sas->ethdev_rxq_count - 1;
 134         sw_qid_max = 0;
 135
 136         out->nb_qid_offsets = 0;
 137
 138         for (i = 0; i < in->queue_num; ++i) {
 139                 uint16_t sw_qid = in->queue[i];
 140
 141                 if (sw_qid >= sas->ethdev_rxq_count) {
 142                         sfc_err(sa, "flow-rss: parse: queue=%u does not exist",
 143                                 sw_qid);
 144                         return EINVAL;
 145                 }
 146
 147                 if (sw_qid < sw_qid_min)
 148                         sw_qid_min = sw_qid;
 149
 150                 if (sw_qid > sw_qid_max)
 151                         sw_qid_max = sw_qid;
 152
 153                 if (sw_qid != in->queue[0] + i)
 154                         out->nb_qid_offsets = in->queue_num;
 155         }
 156
 157         out->qid_span = sw_qid_max - sw_qid_min + 1;
 158
 159         if (out->qid_span > EFX_MAXRSS) {
 160                 sfc_err(sa, "flow-rss: parse: queue ID span %u is too large; MAX=%u",
 161                         out->qid_span, EFX_MAXRSS);
 162                 return EINVAL;
 163         }
 164
 165         if (sw_qid_minp != NULL)
 166                 *sw_qid_minp = sw_qid_min;
 167
 168         return 0;
 169 }
 170
 171 struct sfc_flow_rss_ctx *
 172 sfc_flow_rss_ctx_reuse(struct sfc_adapter *sa,
 173                        const struct sfc_flow_rss_conf *conf,
 174                        uint16_t sw_qid_min, const uint16_t *sw_qids)
 175 {
 176         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 177         static struct sfc_flow_rss_ctx *ctx;
 178
 179         SFC_ASSERT(sfc_adapter_is_locked(sa));
 180
 181         TAILQ_FOREACH(ctx, &flow_rss->ctx_list, entries) {
 182                 if (memcmp(&ctx->conf, conf, sizeof(*conf)) != 0)
 183                         continue;
 184
 185                 if (conf->nb_qid_offsets != 0) {
 186                         bool match_confirmed = true;
 187                         unsigned int i;
 188
 189                         for (i = 0; i < conf->nb_qid_offsets; ++i) {
 190                                 uint16_t qid_offset = sw_qids[i] - sw_qid_min;
 191
 192                                 if (ctx->qid_offsets[i] != qid_offset) {
 193                                         match_confirmed = false;
 194                                         break;
 195                                 }
 196                         }
 197
 198                         if (!match_confirmed)
 199                                 continue;
 200                 }
 201
 202                 sfc_dbg(sa, "flow-rss: reusing ctx=%p", ctx);
 203                 ++(ctx->refcnt);
 204                 return ctx;
 205         }
 206
 207         return NULL;
 208 }
 209
 210 int
 211 sfc_flow_rss_ctx_add(struct sfc_adapter *sa,
 212                      const struct sfc_flow_rss_conf *conf, uint16_t sw_qid_min,
 213                      const uint16_t *sw_qids, struct sfc_flow_rss_ctx **ctxp)
 214 {
 215         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 216         struct sfc_flow_rss_ctx *ctx;
 217
 218         SFC_ASSERT(sfc_adapter_is_locked(sa));
 219
 220         ctx = rte_zmalloc("sfc_flow_rss_ctx", sizeof(*ctx), 0);
 221         if (ctx == NULL)
 222                 return ENOMEM;
 223
 224         if (conf->nb_qid_offsets != 0) {
 225                 unsigned int i;
 226
 227                 ctx->qid_offsets = rte_calloc("sfc_flow_rss_ctx_qid_offsets",
 228                                               conf->nb_qid_offsets,
 229                                               sizeof(*ctx->qid_offsets), 0);
 230                 if (ctx->qid_offsets == NULL) {
 231                         rte_free(ctx);
 232                         return ENOMEM;
 233                 }
 234
 235                 for (i = 0; i < conf->nb_qid_offsets; ++i)
 236                         ctx->qid_offsets[i] = sw_qids[i] - sw_qid_min;
 237         }
 238
 239         ctx->conf = *conf;
 240         ctx->refcnt = 1;
 241
 242         TAILQ_INSERT_TAIL(&flow_rss->ctx_list, ctx, entries);
 243
 244         *ctxp = ctx;
 245
 246         sfc_dbg(sa, "flow-rss: added ctx=%p", ctx);
 247
 248         return 0;
 249 }
 250
 251 void
 252 sfc_flow_rss_ctx_del(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 253 {
 254         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 255
 256         if (ctx == NULL)
 257                 return;
 258
 259         SFC_ASSERT(sfc_adapter_is_locked(sa));
 260
 261         if (ctx->dummy)
 262                 return;
 263
 264         SFC_ASSERT(ctx->refcnt != 0);
 265
 266         --(ctx->refcnt);
 267
 268         if (ctx->refcnt != 0)
 269                 return;
 270
 271         if (ctx->nic_handle_refcnt != 0) {
 272                 sfc_err(sa, "flow-rss: deleting ctx=%p abandons its NIC resource: handle=0x%08x, refcnt=%u",
 273                         ctx, ctx->nic_handle, ctx->nic_handle_refcnt);
 274         }
 275
 276         TAILQ_REMOVE(&flow_rss->ctx_list, ctx, entries);
 277         rte_free(ctx->qid_offsets);
 278         rte_free(ctx);
 279
 280         sfc_dbg(sa, "flow-rss: deleted ctx=%p", ctx);
 281 }
 282
 283 static int
 284 sfc_flow_rss_ctx_program_tbl(struct sfc_adapter *sa,
 285                              const struct sfc_flow_rss_ctx *ctx)
 286 {
 287         const struct sfc_flow_rss_conf *conf = &ctx->conf;
 288         unsigned int *tbl = sa->flow_rss.bounce_tbl;
 289         unsigned int i;
 290
 291         SFC_ASSERT(sfc_adapter_is_locked(sa));
 292
 293         if (conf->nb_qid_offsets != 0) {
 294                 SFC_ASSERT(ctx->qid_offsets != NULL);
 295
 296                 for (i = 0; i < EFX_RSS_TBL_SIZE; ++i)
 297                         tbl[i] = ctx->qid_offsets[i % conf->nb_qid_offsets];
 298         } else {
 299                 for (i = 0; i < EFX_RSS_TBL_SIZE; ++i)
 300                         tbl[i] = i % conf->qid_span;
 301         }
 302
 303         return efx_rx_scale_tbl_set(sa->nic, ctx->nic_handle,
 304                                     tbl, EFX_RSS_TBL_SIZE);
 305 }
 306
 307 int
 308 sfc_flow_rss_ctx_program(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 309 {
 310         efx_rx_scale_context_type_t ctx_type = EFX_RX_SCALE_EXCLUSIVE;
 311         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 312         struct sfc_rss *ethdev_rss = &sas->rss;
 313         struct sfc_flow_rss_conf *conf;
 314         bool allocation_done = B_FALSE;
 315         int rc;
 316
 317         if (ctx == NULL)
 318                 return 0;
 319
 320         conf = &ctx->conf;
 321
 322         SFC_ASSERT(sfc_adapter_is_locked(sa));
 323
 324         if (ctx->nic_handle_refcnt == 0) {
 325                 rc = efx_rx_scale_context_alloc(sa->nic, ctx_type,
 326                                                 conf->qid_span,
 327                                                 &ctx->nic_handle);
 328                 if (rc != 0) {
 329                         sfc_err(sa, "flow-rss: failed to allocate NIC resource for ctx=%p: type=%d, qid_span=%u, rc=%d",
 330                                 ctx, ctx_type, conf->qid_span, rc);
 331                         goto fail;
 332                 }
 333
 334                 sfc_dbg(sa, "flow-rss: allocated NIC resource for ctx=%p: type=%d, qid_span=%u; handle=0x%08x",
 335                         ctx, ctx_type, conf->qid_span,
 336                         ctx->nic_handle);
 337
 338                 ++(ctx->nic_handle_refcnt);
 339                 allocation_done = B_TRUE;
 340         } else {
 341                 ++(ctx->nic_handle_refcnt);
 342                 return 0;
 343         }
 344
 345         rc = efx_rx_scale_mode_set(sa->nic, ctx->nic_handle,
 346                                    ethdev_rss->hash_alg,
 347                                    (ctx->dummy) ? ethdev_rss->hash_types :
 348                                                   conf->efx_hash_types,
 349                                    B_TRUE);
 350         if (rc != 0) {
 351                 sfc_err(sa, "flow-rss: failed to configure hash for ctx=%p: efx_hash_alg=%d, efx_hash_types=0x%08x; rc=%d",
 352                         ctx, ethdev_rss->hash_alg,
 353                         (ctx->dummy) ? ethdev_rss->hash_types :
 354                                        conf->efx_hash_types,
 355                         rc);
 356                 goto fail;
 357         }
 358
 359         rc = efx_rx_scale_key_set(sa->nic, ctx->nic_handle,
 360                                   (ctx->dummy) ? ethdev_rss->key : conf->key,
 361                                   RTE_DIM(conf->key));
 362         if (rc != 0) {
 363                 sfc_err(sa, "flow-rss: failed to set key for ctx=%p; rc=%d",
 364                         ctx, rc);
 365                 goto fail;
 366         }
 367
 368         rc = sfc_flow_rss_ctx_program_tbl(sa, ctx);
 369         if (rc != 0) {
 370                 sfc_err(sa, "flow-rss: failed to program table for ctx=%p; rc=%d",
 371                         ctx, rc);
 372                 goto fail;
 373         }
 374
 375         return 0;
 376
 377 fail:
 378         if (allocation_done)
 379                 sfc_flow_rss_ctx_terminate(sa, ctx);
 380
 381         return rc;
 382 }
 383
 384 void
 385 sfc_flow_rss_ctx_terminate(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 386 {
 387         if (ctx == NULL)
 388                 return;
 389
 390         SFC_ASSERT(sfc_adapter_is_locked(sa));
 391
 392         SFC_ASSERT(ctx->nic_handle_refcnt != 0);
 393         --(ctx->nic_handle_refcnt);
 394
 395         if (ctx->nic_handle_refcnt == 0) {
 396                 int rc;
 397
 398                 rc = efx_rx_scale_context_free(sa->nic, ctx->nic_handle);
 399                 if (rc != 0) {
 400                         sfc_err(sa, "flow-rss: failed to release NIC resource for ctx=%p: handle=0x%08x; rc=%d",
 401                                 ctx, ctx->nic_handle, rc);
 402
 403                         sfc_warn(sa, "flow-rss: proceeding despite the prior error");
 404                 }
 405
 406                 sfc_dbg(sa, "flow-rss: released NIC resource for ctx=%p; rc=%d",
 407                         ctx, rc);
 408         }
 409 }