drivers/net/sfc/sfc_flow_rss.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2022 Xilinx, Inc.
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_flow.h>
  11 #include <rte_tailq.h>
  12
  13 #include "efx.h"
  14
  15 #include "sfc.h"
  16 #include "sfc_debug.h"
  17 #include "sfc_flow_rss.h"
  18 #include "sfc_log.h"
  19 #include "sfc_rx.h"
  20
  21 int
  22 sfc_flow_rss_attach(struct sfc_adapter *sa)
  23 {
  24         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
  25         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
  26
  27         sfc_log_init(sa, "entry");
  28
  29         flow_rss->qid_span_max = encp->enc_rx_scale_indirection_max_nqueues;
  30
  31         TAILQ_INIT(&flow_rss->ctx_list);
  32
  33         sfc_log_init(sa, "done");
  34
  35         return 0;
  36 }
  37
  38 void
  39 sfc_flow_rss_detach(struct sfc_adapter *sa)
  40 {
  41         sfc_log_init(sa, "entry");
  42
  43         sfc_log_init(sa, "done");
  44 }
  45
  46 int
  47 sfc_flow_rss_parse_conf(struct sfc_adapter *sa,
  48                         const struct rte_flow_action_rss *in,
  49                         struct sfc_flow_rss_conf *out, uint16_t *sw_qid_minp)
  50 {
  51         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
  52         const struct sfc_flow_rss *flow_rss = &sa->flow_rss;
  53         const struct sfc_rss *ethdev_rss = &sas->rss;
  54         uint16_t sw_qid_min;
  55         uint16_t sw_qid_max;
  56         const uint8_t *key;
  57         unsigned int i;
  58         int rc;
  59
  60         if (in->level) {
  61                 /*
  62                  * The caller demands that RSS hash be computed
  63                  * within the given encapsulation frame / level.
  64                  * Per flow control for that is not implemented.
  65                  */
  66                 sfc_err(sa, "flow-rss: parse: 'level' must be 0");
  67                 return EINVAL;
  68         }
  69
  70         if (in->types != 0) {
  71                 rc = sfc_rx_hf_rte_to_efx(sa, in->types,
  72                                           &out->efx_hash_types);
  73                 if (rc != 0) {
  74                         sfc_err(sa, "flow-rss: parse: failed to process 'types'");
  75                         return rc;
  76                 }
  77         } else {
  78                 sfc_dbg(sa, "flow-rss: parse: 'types' is 0; proceeding with ethdev setting");
  79                 out->efx_hash_types = ethdev_rss->hash_types;
  80         }
  81
  82         if (in->key_len != 0) {
  83                 if (in->key_len != sizeof(out->key)) {
  84                         sfc_err(sa, "flow-rss: parse: 'key_len' must be either %zu or 0",
  85                                 sizeof(out->key));
  86                         return EINVAL;
  87                 }
  88
  89                 if (in->key == NULL) {
  90                         sfc_err(sa, "flow-rss: parse: 'key' is NULL");
  91                         return EINVAL;
  92                 }
  93
  94                 key = in->key;
  95         } else {
  96                 sfc_dbg(sa, "flow-rss: parse: 'key_len' is 0; proceeding with ethdev key");
  97                 key = ethdev_rss->key;
  98         }
  99
 100         rte_memcpy(out->key, key, sizeof(out->key));
 101
 102         switch (in->func) {
 103         case RTE_ETH_HASH_FUNCTION_DEFAULT:
 104                 /*
 105                  * DEFAULT means that conformance to a specific
 106                  * hash algorithm is a don't care to the caller.
 107                  * The driver can pick the one it deems optimal.
 108                  */
 109                 break;
 110         case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
 111                 if (ethdev_rss->hash_alg != EFX_RX_HASHALG_TOEPLITZ) {
 112                         sfc_err(sa, "flow-rss: parse: 'func' TOEPLITZ is unavailable; use DEFAULT");
 113                         return EINVAL;
 114                 }
 115                 break;
 116         default:
 117                 sfc_err(sa, "flow-rss: parse: 'func' #%d is unsupported", in->func);
 118                 return EINVAL;
 119         }
 120
 121         if (in->queue_num == 0) {
 122                 sfc_err(sa, "flow-rss: parse: 'queue_num' is 0; MIN=1");
 123                 return EINVAL;
 124         }
 125
 126         if (in->queue_num > EFX_RSS_TBL_SIZE) {
 127                 sfc_err(sa, "flow-rss: parse: 'queue_num' is too large; MAX=%u",
 128                         EFX_RSS_TBL_SIZE);
 129                 return EINVAL;
 130         }
 131
 132         if (in->queue == NULL) {
 133                 sfc_err(sa, "flow-rss: parse: 'queue' is NULL");
 134                 return EINVAL;
 135         }
 136
 137         sw_qid_min = sas->ethdev_rxq_count - 1;
 138         sw_qid_max = 0;
 139
 140         out->nb_qid_offsets = 0;
 141
 142         for (i = 0; i < in->queue_num; ++i) {
 143                 uint16_t sw_qid = in->queue[i];
 144
 145                 if (sw_qid >= sas->ethdev_rxq_count) {
 146                         sfc_err(sa, "flow-rss: parse: queue=%u does not exist",
 147                                 sw_qid);
 148                         return EINVAL;
 149                 }
 150
 151                 if (sw_qid < sw_qid_min)
 152                         sw_qid_min = sw_qid;
 153
 154                 if (sw_qid > sw_qid_max)
 155                         sw_qid_max = sw_qid;
 156
 157                 if (sw_qid != in->queue[0] + i)
 158                         out->nb_qid_offsets = in->queue_num;
 159         }
 160
 161         out->qid_span = sw_qid_max - sw_qid_min + 1;
 162
 163         if (out->qid_span > flow_rss->qid_span_max) {
 164                 sfc_err(sa, "flow-rss: parse: queue ID span %u is too large; MAX=%u",
 165                         out->qid_span, flow_rss->qid_span_max);
 166                 return EINVAL;
 167         }
 168
 169         if (sw_qid_minp != NULL)
 170                 *sw_qid_minp = sw_qid_min;
 171
 172         return 0;
 173 }
 174
 175 struct sfc_flow_rss_ctx *
 176 sfc_flow_rss_ctx_reuse(struct sfc_adapter *sa,
 177                        const struct sfc_flow_rss_conf *conf,
 178                        uint16_t sw_qid_min, const uint16_t *sw_qids)
 179 {
 180         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 181         static struct sfc_flow_rss_ctx *ctx;
 182
 183         SFC_ASSERT(sfc_adapter_is_locked(sa));
 184
 185         TAILQ_FOREACH(ctx, &flow_rss->ctx_list, entries) {
 186                 if (memcmp(&ctx->conf, conf, sizeof(*conf)) != 0)
 187                         continue;
 188
 189                 if (conf->nb_qid_offsets != 0) {
 190                         bool match_confirmed = true;
 191                         unsigned int i;
 192
 193                         for (i = 0; i < conf->nb_qid_offsets; ++i) {
 194                                 uint16_t qid_offset = sw_qids[i] - sw_qid_min;
 195
 196                                 if (ctx->qid_offsets[i] != qid_offset) {
 197                                         match_confirmed = false;
 198                                         break;
 199                                 }
 200                         }
 201
 202                         if (!match_confirmed)
 203                                 continue;
 204                 }
 205
 206                 sfc_dbg(sa, "flow-rss: reusing ctx=%p", ctx);
 207                 ++(ctx->refcnt);
 208                 return ctx;
 209         }
 210
 211         return NULL;
 212 }
 213
 214 int
 215 sfc_flow_rss_ctx_add(struct sfc_adapter *sa,
 216                      const struct sfc_flow_rss_conf *conf, uint16_t sw_qid_min,
 217                      const uint16_t *sw_qids, struct sfc_flow_rss_ctx **ctxp)
 218 {
 219         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 220         struct sfc_flow_rss_ctx *ctx;
 221
 222         SFC_ASSERT(sfc_adapter_is_locked(sa));
 223
 224         ctx = rte_zmalloc("sfc_flow_rss_ctx", sizeof(*ctx), 0);
 225         if (ctx == NULL)
 226                 return ENOMEM;
 227
 228         if (conf->nb_qid_offsets != 0) {
 229                 unsigned int i;
 230
 231                 ctx->qid_offsets = rte_calloc("sfc_flow_rss_ctx_qid_offsets",
 232                                               conf->nb_qid_offsets,
 233                                               sizeof(*ctx->qid_offsets), 0);
 234                 if (ctx->qid_offsets == NULL) {
 235                         rte_free(ctx);
 236                         return ENOMEM;
 237                 }
 238
 239                 for (i = 0; i < conf->nb_qid_offsets; ++i)
 240                         ctx->qid_offsets[i] = sw_qids[i] - sw_qid_min;
 241         }
 242
 243         ctx->conf = *conf;
 244         ctx->refcnt = 1;
 245
 246         TAILQ_INSERT_TAIL(&flow_rss->ctx_list, ctx, entries);
 247
 248         *ctxp = ctx;
 249
 250         sfc_dbg(sa, "flow-rss: added ctx=%p", ctx);
 251
 252         return 0;
 253 }
 254
 255 void
 256 sfc_flow_rss_ctx_del(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 257 {
 258         struct sfc_flow_rss *flow_rss = &sa->flow_rss;
 259
 260         if (ctx == NULL)
 261                 return;
 262
 263         SFC_ASSERT(sfc_adapter_is_locked(sa));
 264
 265         if (ctx->dummy)
 266                 return;
 267
 268         SFC_ASSERT(ctx->refcnt != 0);
 269
 270         --(ctx->refcnt);
 271
 272         if (ctx->refcnt != 0)
 273                 return;
 274
 275         if (ctx->nic_handle_refcnt != 0) {
 276                 sfc_err(sa, "flow-rss: deleting ctx=%p abandons its NIC resource: handle=0x%08x, refcnt=%u",
 277                         ctx, ctx->nic_handle, ctx->nic_handle_refcnt);
 278         }
 279
 280         TAILQ_REMOVE(&flow_rss->ctx_list, ctx, entries);
 281         rte_free(ctx->qid_offsets);
 282         rte_free(ctx);
 283
 284         sfc_dbg(sa, "flow-rss: deleted ctx=%p", ctx);
 285 }
 286
 287 static int
 288 sfc_flow_rss_ctx_program_tbl(struct sfc_adapter *sa,
 289                              const struct sfc_flow_rss_ctx *ctx)
 290 {
 291         const struct sfc_flow_rss_conf *conf = &ctx->conf;
 292         unsigned int *tbl = sa->flow_rss.bounce_tbl;
 293         unsigned int i;
 294
 295         SFC_ASSERT(sfc_adapter_is_locked(sa));
 296
 297         if (conf->nb_qid_offsets != 0) {
 298                 SFC_ASSERT(ctx->qid_offsets != NULL);
 299
 300                 for (i = 0; i < EFX_RSS_TBL_SIZE; ++i)
 301                         tbl[i] = ctx->qid_offsets[i % conf->nb_qid_offsets];
 302         } else {
 303                 for (i = 0; i < EFX_RSS_TBL_SIZE; ++i)
 304                         tbl[i] = i % conf->qid_span;
 305         }
 306
 307         return efx_rx_scale_tbl_set(sa->nic, ctx->nic_handle,
 308                                     tbl, EFX_RSS_TBL_SIZE);
 309 }
 310
 311 int
 312 sfc_flow_rss_ctx_program(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 313 {
 314         efx_rx_scale_context_type_t ctx_type = EFX_RX_SCALE_EXCLUSIVE;
 315         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 316         struct sfc_rss *ethdev_rss = &sas->rss;
 317         struct sfc_flow_rss_conf *conf;
 318         bool allocation_done = B_FALSE;
 319         int rc;
 320
 321         if (ctx == NULL)
 322                 return 0;
 323
 324         conf = &ctx->conf;
 325
 326         SFC_ASSERT(sfc_adapter_is_locked(sa));
 327
 328         if (ctx->nic_handle_refcnt == 0) {
 329                 rc = efx_rx_scale_context_alloc(sa->nic, ctx_type,
 330                                                 conf->qid_span,
 331                                                 &ctx->nic_handle);
 332                 if (rc != 0) {
 333                         sfc_err(sa, "flow-rss: failed to allocate NIC resource for ctx=%p: type=%d, qid_span=%u, rc=%d",
 334                                 ctx, ctx_type, conf->qid_span, rc);
 335                         goto fail;
 336                 }
 337
 338                 sfc_dbg(sa, "flow-rss: allocated NIC resource for ctx=%p: type=%d, qid_span=%u; handle=0x%08x",
 339                         ctx, ctx_type, conf->qid_span,
 340                         ctx->nic_handle);
 341
 342                 ++(ctx->nic_handle_refcnt);
 343                 allocation_done = B_TRUE;
 344         } else {
 345                 ++(ctx->nic_handle_refcnt);
 346                 return 0;
 347         }
 348
 349         rc = efx_rx_scale_mode_set(sa->nic, ctx->nic_handle,
 350                                    ethdev_rss->hash_alg,
 351                                    (ctx->dummy) ? ethdev_rss->hash_types :
 352                                                   conf->efx_hash_types,
 353                                    B_TRUE);
 354         if (rc != 0) {
 355                 sfc_err(sa, "flow-rss: failed to configure hash for ctx=%p: efx_hash_alg=%d, efx_hash_types=0x%08x; rc=%d",
 356                         ctx, ethdev_rss->hash_alg,
 357                         (ctx->dummy) ? ethdev_rss->hash_types :
 358                                        conf->efx_hash_types,
 359                         rc);
 360                 goto fail;
 361         }
 362
 363         rc = efx_rx_scale_key_set(sa->nic, ctx->nic_handle,
 364                                   (ctx->dummy) ? ethdev_rss->key : conf->key,
 365                                   RTE_DIM(conf->key));
 366         if (rc != 0) {
 367                 sfc_err(sa, "flow-rss: failed to set key for ctx=%p; rc=%d",
 368                         ctx, rc);
 369                 goto fail;
 370         }
 371
 372         rc = sfc_flow_rss_ctx_program_tbl(sa, ctx);
 373         if (rc != 0) {
 374                 sfc_err(sa, "flow-rss: failed to program table for ctx=%p; rc=%d",
 375                         ctx, rc);
 376                 goto fail;
 377         }
 378
 379         return 0;
 380
 381 fail:
 382         if (allocation_done)
 383                 sfc_flow_rss_ctx_terminate(sa, ctx);
 384
 385         return rc;
 386 }
 387
 388 void
 389 sfc_flow_rss_ctx_terminate(struct sfc_adapter *sa, struct sfc_flow_rss_ctx *ctx)
 390 {
 391         if (ctx == NULL)
 392                 return;
 393
 394         SFC_ASSERT(sfc_adapter_is_locked(sa));
 395
 396         SFC_ASSERT(ctx->nic_handle_refcnt != 0);
 397         --(ctx->nic_handle_refcnt);
 398
 399         if (ctx->nic_handle_refcnt == 0) {
 400                 int rc;
 401
 402                 rc = efx_rx_scale_context_free(sa->nic, ctx->nic_handle);
 403                 if (rc != 0) {
 404                         sfc_err(sa, "flow-rss: failed to release NIC resource for ctx=%p: handle=0x%08x; rc=%d",
 405                                 ctx, ctx->nic_handle, rc);
 406
 407                         sfc_warn(sa, "flow-rss: proceeding despite the prior error");
 408                 }
 409
 410                 sfc_dbg(sa, "flow-rss: released NIC resource for ctx=%p; rc=%d",
 411                         ctx, rc);
 412         }
 413 }