drivers/net/mlx5/mlx5_flow_flex.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4 #include <rte_malloc.h>
   5 #include <mlx5_devx_cmds.h>
   6 #include <mlx5_malloc.h>
   7 #include "mlx5.h"
   8 #include "mlx5_flow.h"
   9
  10 static_assert(sizeof(uint32_t) * CHAR_BIT >= MLX5_PORT_FLEX_ITEM_NUM,
  11               "Flex item maximal number exceeds uint32_t bit width");
  12
  13 /**
  14  *  Routine called once on port initialization to init flex item
  15  *  related infrastructure initialization
  16  *
  17  * @param dev
  18  *   Ethernet device to perform flex item initialization
  19  *
  20  * @return
  21  *   0 on success, a negative errno value otherwise and rte_errno is set.
  22  */
  23 int
  24 mlx5_flex_item_port_init(struct rte_eth_dev *dev)
  25 {
  26         struct mlx5_priv *priv = dev->data->dev_private;
  27
  28         rte_spinlock_init(&priv->flex_item_sl);
  29         MLX5_ASSERT(!priv->flex_item_map);
  30         return 0;
  31 }
  32
  33 /**
  34  *  Routine called once on port close to perform flex item
  35  *  related infrastructure cleanup.
  36  *
  37  * @param dev
  38  *   Ethernet device to perform cleanup
  39  */
  40 void
  41 mlx5_flex_item_port_cleanup(struct rte_eth_dev *dev)
  42 {
  43         struct mlx5_priv *priv = dev->data->dev_private;
  44         uint32_t i;
  45
  46         for (i = 0; i < MLX5_PORT_FLEX_ITEM_NUM && priv->flex_item_map ; i++) {
  47                 if (priv->flex_item_map & (1 << i)) {
  48                         struct mlx5_flex_item *flex = &priv->flex_item[i];
  49
  50                         claim_zero(mlx5_list_unregister
  51                                         (priv->sh->flex_parsers_dv,
  52                                          &flex->devx_fp->entry));
  53                         flex->devx_fp = NULL;
  54                         flex->refcnt = 0;
  55                         priv->flex_item_map &= ~(1 << i);
  56                 }
  57         }
  58 }
  59
  60 static int
  61 mlx5_flex_index(struct mlx5_priv *priv, struct mlx5_flex_item *item)
  62 {
  63         uintptr_t start = (uintptr_t)&priv->flex_item[0];
  64         uintptr_t entry = (uintptr_t)item;
  65         uintptr_t idx = (entry - start) / sizeof(struct mlx5_flex_item);
  66
  67         if (entry < start ||
  68             idx >= MLX5_PORT_FLEX_ITEM_NUM ||
  69             (entry - start) % sizeof(struct mlx5_flex_item) ||
  70             !(priv->flex_item_map & (1u << idx)))
  71                 return -1;
  72         return (int)idx;
  73 }
  74
  75 static struct mlx5_flex_item *
  76 mlx5_flex_alloc(struct mlx5_priv *priv)
  77 {
  78         struct mlx5_flex_item *item = NULL;
  79
  80         rte_spinlock_lock(&priv->flex_item_sl);
  81         if (~priv->flex_item_map) {
  82                 uint32_t idx = rte_bsf32(~priv->flex_item_map);
  83
  84                 if (idx < MLX5_PORT_FLEX_ITEM_NUM) {
  85                         item = &priv->flex_item[idx];
  86                         MLX5_ASSERT(!item->refcnt);
  87                         MLX5_ASSERT(!item->devx_fp);
  88                         item->devx_fp = NULL;
  89                         __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE);
  90                         priv->flex_item_map |= 1u << idx;
  91                 }
  92         }
  93         rte_spinlock_unlock(&priv->flex_item_sl);
  94         return item;
  95 }
  96
  97 static void
  98 mlx5_flex_free(struct mlx5_priv *priv, struct mlx5_flex_item *item)
  99 {
 100         int idx = mlx5_flex_index(priv, item);
 101
 102         MLX5_ASSERT(idx >= 0 &&
 103                     idx < MLX5_PORT_FLEX_ITEM_NUM &&
 104                     (priv->flex_item_map & (1u << idx)));
 105         if (idx >= 0) {
 106                 rte_spinlock_lock(&priv->flex_item_sl);
 107                 MLX5_ASSERT(!item->refcnt);
 108                 MLX5_ASSERT(!item->devx_fp);
 109                 item->devx_fp = NULL;
 110                 __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE);
 111                 priv->flex_item_map &= ~(1u << idx);
 112                 rte_spinlock_unlock(&priv->flex_item_sl);
 113         }
 114 }
 115
 116 static uint32_t
 117 mlx5_flex_get_bitfield(const struct rte_flow_item_flex *item,
 118                        uint32_t pos, uint32_t width, uint32_t shift)
 119 {
 120         const uint8_t *ptr = item->pattern + pos / CHAR_BIT;
 121         uint32_t val, vbits;
 122
 123         /* Proceed the bitfield start byte. */
 124         MLX5_ASSERT(width <= sizeof(uint32_t) * CHAR_BIT && width);
 125         MLX5_ASSERT(width + shift <= sizeof(uint32_t) * CHAR_BIT);
 126         if (item->length <= pos / CHAR_BIT)
 127                 return 0;
 128         val = *ptr++ >> (pos % CHAR_BIT);
 129         vbits = CHAR_BIT - pos % CHAR_BIT;
 130         pos = (pos + vbits) / CHAR_BIT;
 131         vbits = RTE_MIN(vbits, width);
 132         val &= RTE_BIT32(vbits) - 1;
 133         while (vbits < width && pos < item->length) {
 134                 uint32_t part = RTE_MIN(width - vbits, (uint32_t)CHAR_BIT);
 135                 uint32_t tmp = *ptr++;
 136
 137                 pos++;
 138                 tmp &= RTE_BIT32(part) - 1;
 139                 val |= tmp << vbits;
 140                 vbits += part;
 141         }
 142         return rte_bswap32(val <<= shift);
 143 }
 144
 145 #define SET_FP_MATCH_SAMPLE_ID(x, def, msk, val, sid) \
 146         do { \
 147                 uint32_t tmp, out = (def); \
 148                 tmp = MLX5_GET(fte_match_set_misc4, misc4_v, \
 149                                prog_sample_field_value_##x); \
 150                 tmp = (tmp & ~out) | (val); \
 151                 MLX5_SET(fte_match_set_misc4, misc4_v, \
 152                          prog_sample_field_value_##x, tmp); \
 153                 tmp = MLX5_GET(fte_match_set_misc4, misc4_m, \
 154                                prog_sample_field_value_##x); \
 155                 tmp = (tmp & ~out) | (msk); \
 156                 MLX5_SET(fte_match_set_misc4, misc4_m, \
 157                          prog_sample_field_value_##x, tmp); \
 158                 tmp = tmp ? (sid) : 0; \
 159                 MLX5_SET(fte_match_set_misc4, misc4_v, \
 160                          prog_sample_field_id_##x, tmp);\
 161                 MLX5_SET(fte_match_set_misc4, misc4_m, \
 162                          prog_sample_field_id_##x, tmp); \
 163         } while (0)
 164
 165 __rte_always_inline static void
 166 mlx5_flex_set_match_sample(void *misc4_m, void *misc4_v,
 167                            uint32_t def, uint32_t mask, uint32_t value,
 168                            uint32_t sample_id, uint32_t id)
 169 {
 170         switch (id) {
 171         case 0:
 172                 SET_FP_MATCH_SAMPLE_ID(0, def, mask, value, sample_id);
 173                 break;
 174         case 1:
 175                 SET_FP_MATCH_SAMPLE_ID(1, def, mask, value, sample_id);
 176                 break;
 177         case 2:
 178                 SET_FP_MATCH_SAMPLE_ID(2, def, mask, value, sample_id);
 179                 break;
 180         case 3:
 181                 SET_FP_MATCH_SAMPLE_ID(3, def, mask, value, sample_id);
 182                 break;
 183         case 4:
 184                 SET_FP_MATCH_SAMPLE_ID(4, def, mask, value, sample_id);
 185                 break;
 186         case 5:
 187                 SET_FP_MATCH_SAMPLE_ID(5, def, mask, value, sample_id);
 188                 break;
 189         case 6:
 190                 SET_FP_MATCH_SAMPLE_ID(6, def, mask, value, sample_id);
 191                 break;
 192         case 7:
 193                 SET_FP_MATCH_SAMPLE_ID(7, def, mask, value, sample_id);
 194                 break;
 195         default:
 196                 MLX5_ASSERT(false);
 197                 break;
 198         }
 199 #undef SET_FP_MATCH_SAMPLE_ID
 200 }
 201 /**
 202  * Translate item pattern into matcher fields according to translation
 203  * array.
 204  *
 205  * @param dev
 206  *   Ethernet device to translate flex item on.
 207  * @param[in, out] matcher
 208  *   Flow matcher to configure
 209  * @param[in, out] key
 210  *   Flow matcher value.
 211  * @param[in] item
 212  *   Flow pattern to translate.
 213  * @param[in] is_inner
 214  *   Inner Flex Item (follows after tunnel header).
 215  *
 216  * @return
 217  *   0 on success, a negative errno value otherwise and rte_errno is set.
 218  */
 219 void
 220 mlx5_flex_flow_translate_item(struct rte_eth_dev *dev,
 221                               void *matcher, void *key,
 222                               const struct rte_flow_item *item,
 223                               bool is_inner)
 224 {
 225         const struct rte_flow_item_flex *spec, *mask;
 226         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
 227                                      misc_parameters_4);
 228         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
 229         struct mlx5_flex_item *tp;
 230         uint32_t i, pos = 0;
 231
 232         RTE_SET_USED(dev);
 233         MLX5_ASSERT(item->spec && item->mask);
 234         spec = item->spec;
 235         mask = item->mask;
 236         tp = (struct mlx5_flex_item *)spec->handle;
 237         MLX5_ASSERT(mlx5_flex_index(dev->data->dev_private, tp) >= 0);
 238         for (i = 0; i < tp->mapnum; i++) {
 239                 struct mlx5_flex_pattern_field *map = tp->map + i;
 240                 uint32_t id = map->reg_id;
 241                 uint32_t def = (RTE_BIT64(map->width) - 1) << map->shift;
 242                 uint32_t val, msk;
 243
 244                 /* Skip placeholders for DUMMY fields. */
 245                 if (id == MLX5_INVALID_SAMPLE_REG_ID) {
 246                         pos += map->width;
 247                         continue;
 248                 }
 249                 val = mlx5_flex_get_bitfield(spec, pos, map->width, map->shift);
 250                 msk = mlx5_flex_get_bitfield(mask, pos, map->width, map->shift);
 251                 MLX5_ASSERT(map->width);
 252                 MLX5_ASSERT(id < tp->devx_fp->num_samples);
 253                 if (tp->tunnel_mode == FLEX_TUNNEL_MODE_MULTI && is_inner) {
 254                         uint32_t num_samples = tp->devx_fp->num_samples / 2;
 255
 256                         MLX5_ASSERT(tp->devx_fp->num_samples % 2 == 0);
 257                         MLX5_ASSERT(id < num_samples);
 258                         id += num_samples;
 259                 }
 260                 mlx5_flex_set_match_sample(misc4_m, misc4_v,
 261                                            def, msk & def, val & msk & def,
 262                                            tp->devx_fp->sample_ids[id], id);
 263                 pos += map->width;
 264         }
 265 }
 266
 267 /**
 268  * Convert flex item handle (from the RTE flow) to flex item index on port.
 269  * Optionally can increment flex item object reference count.
 270  *
 271  * @param dev
 272  *   Ethernet device to acquire flex item on.
 273  * @param[in] handle
 274  *   Flow item handle from item spec.
 275  * @param[in] acquire
 276  *   If set - increment reference counter.
 277  *
 278  * @return
 279  *   >=0 - index on success, a negative errno value otherwise
 280  *         and rte_errno is set.
 281  */
 282 int
 283 mlx5_flex_acquire_index(struct rte_eth_dev *dev,
 284                         struct rte_flow_item_flex_handle *handle,
 285                         bool acquire)
 286 {
 287         struct mlx5_priv *priv = dev->data->dev_private;
 288         struct mlx5_flex_item *flex = (struct mlx5_flex_item *)handle;
 289         int ret = mlx5_flex_index(priv, flex);
 290
 291         if (ret < 0) {
 292                 errno = -EINVAL;
 293                 rte_errno = EINVAL;
 294                 return ret;
 295         }
 296         if (acquire)
 297                 __atomic_add_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE);
 298         return ret;
 299 }
 300
 301 /**
 302  * Release flex item index on port - decrements reference counter by index.
 303  *
 304  * @param dev
 305  *   Ethernet device to acquire flex item on.
 306  * @param[in] index
 307  *   Flow item index.
 308  *
 309  * @return
 310  *   0 - on success, a negative errno value otherwise and rte_errno is set.
 311  */
 312 int
 313 mlx5_flex_release_index(struct rte_eth_dev *dev,
 314                         int index)
 315 {
 316         struct mlx5_priv *priv = dev->data->dev_private;
 317         struct mlx5_flex_item *flex;
 318
 319         if (index >= MLX5_PORT_FLEX_ITEM_NUM ||
 320             !(priv->flex_item_map & (1u << index))) {
 321                 errno = EINVAL;
 322                 rte_errno = -EINVAL;
 323                 return -EINVAL;
 324         }
 325         flex = priv->flex_item + index;
 326         if (flex->refcnt <= 1) {
 327                 MLX5_ASSERT(false);
 328                 errno = EINVAL;
 329                 rte_errno = -EINVAL;
 330                 return -EINVAL;
 331         }
 332         __atomic_sub_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE);
 333         return 0;
 334 }
 335
 336 /*
 337  * Calculate largest mask value for a given shift.
 338  *
 339  *   shift      mask
 340  * ------- ---------------
 341  *    0     b111100  0x3C
 342  *    1     b111110  0x3E
 343  *    2     b111111  0x3F
 344  *    3     b011111  0x1F
 345  *    4     b001111  0x0F
 346  *    5     b000111  0x07
 347  */
 348 static uint8_t
 349 mlx5_flex_hdr_len_mask(uint8_t shift,
 350                        const struct mlx5_hca_flex_attr *attr)
 351 {
 352         uint32_t base_mask;
 353         int diff = shift - MLX5_PARSE_GRAPH_NODE_HDR_LEN_SHIFT_DWORD;
 354
 355         base_mask = mlx5_hca_parse_graph_node_base_hdr_len_mask(attr);
 356         return diff == 0 ? base_mask :
 357                diff < 0 ? (base_mask << -diff) & base_mask : base_mask >> diff;
 358 }
 359
 360 static int
 361 mlx5_flex_translate_length(struct mlx5_hca_flex_attr *attr,
 362                            const struct rte_flow_item_flex_conf *conf,
 363                            struct mlx5_flex_parser_devx *devx,
 364                            struct rte_flow_error *error)
 365 {
 366         const struct rte_flow_item_flex_field *field = &conf->next_header;
 367         struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
 368         uint32_t len_width, mask;
 369
 370         if (field->field_base % CHAR_BIT)
 371                 return rte_flow_error_set
 372                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 373                          "not byte aligned header length field");
 374         switch (field->field_mode) {
 375         case FIELD_MODE_DUMMY:
 376                 return rte_flow_error_set
 377                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 378                          "invalid header length field mode (DUMMY)");
 379         case FIELD_MODE_FIXED:
 380                 if (!(attr->header_length_mode &
 381                     RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIXED)))
 382                         return rte_flow_error_set
 383                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 384                                  "unsupported header length field mode (FIXED)");
 385                 if (attr->header_length_mask_width < field->field_size)
 386                         return rte_flow_error_set
 387                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 388                                  "header length field width exceeds limit");
 389                 if (field->offset_shift < 0 ||
 390                     field->offset_shift > attr->header_length_mask_width)
 391                         return rte_flow_error_set
 392                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 393                                  "invalid header length field shift (FIXED");
 394                 if (field->field_base < 0)
 395                         return rte_flow_error_set
 396                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 397                                  "negative header length field base (FIXED)");
 398                 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED;
 399                 break;
 400         case FIELD_MODE_OFFSET:
 401                 if (!(attr->header_length_mode &
 402                     RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIELD)))
 403                         return rte_flow_error_set
 404                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 405                                  "unsupported header length field mode (OFFSET)");
 406                 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIELD;
 407                 if (field->offset_mask == 0 ||
 408                     !rte_is_power_of_2(field->offset_mask + 1))
 409                         return rte_flow_error_set
 410                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 411                                  "invalid length field offset mask (OFFSET)");
 412                 len_width = rte_fls_u32(field->offset_mask);
 413                 if (len_width > attr->header_length_mask_width)
 414                         return rte_flow_error_set
 415                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 416                                  "length field offset mask too wide (OFFSET)");
 417                 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr);
 418                 if (mask < field->offset_mask)
 419                         return rte_flow_error_set
 420                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 421                                  "length field shift too big (OFFSET)");
 422                 node->header_length_field_mask = RTE_MIN(mask,
 423                                                          field->offset_mask);
 424                 break;
 425         case FIELD_MODE_BITMASK:
 426                 if (!(attr->header_length_mode &
 427                     RTE_BIT32(MLX5_GRAPH_NODE_LEN_BITMASK)))
 428                         return rte_flow_error_set
 429                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 430                                  "unsupported header length field mode (BITMASK)");
 431                 if (attr->header_length_mask_width < field->field_size)
 432                         return rte_flow_error_set
 433                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 434                                  "header length field width exceeds limit");
 435                 node->header_length_mode = MLX5_GRAPH_NODE_LEN_BITMASK;
 436                 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr);
 437                 if (mask < field->offset_mask)
 438                         return rte_flow_error_set
 439                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 440                                  "length field shift too big (BITMASK)");
 441                 node->header_length_field_mask = RTE_MIN(mask,
 442                                                          field->offset_mask);
 443                 break;
 444         default:
 445                 return rte_flow_error_set
 446                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 447                          "unknown header length field mode");
 448         }
 449         if (field->field_base / CHAR_BIT >= 0 &&
 450             field->field_base / CHAR_BIT > attr->max_base_header_length)
 451                 return rte_flow_error_set
 452                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 453                          "header length field base exceeds limit");
 454         node->header_length_base_value = field->field_base / CHAR_BIT;
 455         if (field->field_mode == FIELD_MODE_OFFSET ||
 456             field->field_mode == FIELD_MODE_BITMASK) {
 457                 if (field->offset_shift > 15 || field->offset_shift < 0)
 458                         return rte_flow_error_set
 459                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 460                                  "header length field shift exceeds limit");
 461                 node->header_length_field_shift = field->offset_shift;
 462                 node->header_length_field_offset = field->offset_base;
 463         }
 464         return 0;
 465 }
 466
 467 static int
 468 mlx5_flex_translate_next(struct mlx5_hca_flex_attr *attr,
 469                          const struct rte_flow_item_flex_conf *conf,
 470                          struct mlx5_flex_parser_devx *devx,
 471                          struct rte_flow_error *error)
 472 {
 473         const struct rte_flow_item_flex_field *field = &conf->next_protocol;
 474         struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
 475
 476         switch (field->field_mode) {
 477         case FIELD_MODE_DUMMY:
 478                 if (conf->nb_outputs)
 479                         return rte_flow_error_set
 480                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 481                                  "next protocol field is required (DUMMY)");
 482                 return 0;
 483         case FIELD_MODE_FIXED:
 484                 break;
 485         case FIELD_MODE_OFFSET:
 486                 return rte_flow_error_set
 487                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 488                          "unsupported next protocol field mode (OFFSET)");
 489                 break;
 490         case FIELD_MODE_BITMASK:
 491                 return rte_flow_error_set
 492                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 493                          "unsupported next protocol field mode (BITMASK)");
 494         default:
 495                 return rte_flow_error_set
 496                         (error, EINVAL,
 497                          RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 498                          "unknown next protocol field mode");
 499         }
 500         MLX5_ASSERT(field->field_mode == FIELD_MODE_FIXED);
 501         if (!conf->nb_outputs)
 502                 return rte_flow_error_set
 503                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 504                          "out link(s) is required if next field present");
 505         if (attr->max_next_header_offset < field->field_base)
 506                 return rte_flow_error_set
 507                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 508                          "next protocol field base exceeds limit");
 509         if (field->offset_shift)
 510                 return rte_flow_error_set
 511                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 512                          "unsupported next protocol field shift");
 513         node->next_header_field_offset = field->field_base;
 514         node->next_header_field_size = field->field_size;
 515         return 0;
 516 }
 517
 518 /* Helper structure to handle field bit intervals. */
 519 struct mlx5_flex_field_cover {
 520         uint16_t num;
 521         int32_t start[MLX5_FLEX_ITEM_MAPPING_NUM];
 522         int32_t end[MLX5_FLEX_ITEM_MAPPING_NUM];
 523         uint8_t mapped[MLX5_FLEX_ITEM_MAPPING_NUM / CHAR_BIT + 1];
 524 };
 525
 526 static void
 527 mlx5_flex_insert_field(struct mlx5_flex_field_cover *cover,
 528                        uint16_t num, int32_t start, int32_t end)
 529 {
 530         MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM);
 531         MLX5_ASSERT(num <= cover->num);
 532         if (num < cover->num) {
 533                 memmove(&cover->start[num + 1], &cover->start[num],
 534                         (cover->num - num) * sizeof(int32_t));
 535                 memmove(&cover->end[num + 1],   &cover->end[num],
 536                         (cover->num - num) * sizeof(int32_t));
 537         }
 538         cover->start[num] = start;
 539         cover->end[num] = end;
 540         cover->num++;
 541 }
 542
 543 static void
 544 mlx5_flex_merge_field(struct mlx5_flex_field_cover *cover, uint16_t num)
 545 {
 546         uint32_t i, del = 0;
 547         int32_t end;
 548
 549         MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM);
 550         MLX5_ASSERT(num < (cover->num - 1));
 551         end = cover->end[num];
 552         for (i = num + 1; i < cover->num; i++) {
 553                 if (end < cover->start[i])
 554                         break;
 555                 del++;
 556                 if (end <= cover->end[i]) {
 557                         cover->end[num] = cover->end[i];
 558                         break;
 559                 }
 560         }
 561         if (del) {
 562                 MLX5_ASSERT(del < (cover->num - 1u - num));
 563                 cover->num -= del;
 564                 MLX5_ASSERT(cover->num > num);
 565                 if ((cover->num - num) > 1) {
 566                         memmove(&cover->start[num + 1],
 567                                 &cover->start[num + 1 + del],
 568                                 (cover->num - num - 1) * sizeof(int32_t));
 569                         memmove(&cover->end[num + 1],
 570                                 &cover->end[num + 1 + del],
 571                                 (cover->num - num - 1) * sizeof(int32_t));
 572                 }
 573         }
 574 }
 575
 576 /*
 577  * Validate the sample field and update interval array
 578  * if parameters match with the 'match" field.
 579  * Returns:
 580  *    < 0  - error
 581  *    == 0 - no match, interval array not updated
 582  *    > 0  - match, interval array updated
 583  */
 584 static int
 585 mlx5_flex_cover_sample(struct mlx5_flex_field_cover *cover,
 586                        struct rte_flow_item_flex_field *field,
 587                        struct rte_flow_item_flex_field *match,
 588                        struct mlx5_hca_flex_attr *attr,
 589                        struct rte_flow_error *error)
 590 {
 591         int32_t start, end;
 592         uint32_t i;
 593
 594         switch (field->field_mode) {
 595         case FIELD_MODE_DUMMY:
 596                 return 0;
 597         case FIELD_MODE_FIXED:
 598                 if (!(attr->sample_offset_mode &
 599                     RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIXED)))
 600                         return rte_flow_error_set
 601                                 (error, EINVAL,
 602                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 603                                  "unsupported sample field mode (FIXED)");
 604                 if (field->offset_shift)
 605                         return rte_flow_error_set
 606                                 (error, EINVAL,
 607                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 608                                  "invalid sample field shift (FIXED");
 609                 if (field->field_base < 0)
 610                         return rte_flow_error_set
 611                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 612                                  "invalid sample field base (FIXED)");
 613                 if (field->field_base / CHAR_BIT > attr->max_sample_base_offset)
 614                         return rte_flow_error_set
 615                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 616                                  "sample field base exceeds limit (FIXED)");
 617                 break;
 618         case FIELD_MODE_OFFSET:
 619                 if (!(attr->sample_offset_mode &
 620                     RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIELD)))
 621                         return rte_flow_error_set
 622                                 (error, EINVAL,
 623                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 624                                  "unsupported sample field mode (OFFSET)");
 625                 if (field->field_base / CHAR_BIT >= 0 &&
 626                     field->field_base / CHAR_BIT > attr->max_sample_base_offset)
 627                         return rte_flow_error_set
 628                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 629                                 "sample field base exceeds limit");
 630                 break;
 631         case FIELD_MODE_BITMASK:
 632                 if (!(attr->sample_offset_mode &
 633                     RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_BITMASK)))
 634                         return rte_flow_error_set
 635                                 (error, EINVAL,
 636                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 637                                  "unsupported sample field mode (BITMASK)");
 638                 if (field->field_base / CHAR_BIT >= 0 &&
 639                     field->field_base / CHAR_BIT > attr->max_sample_base_offset)
 640                         return rte_flow_error_set
 641                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 642                                 "sample field base exceeds limit");
 643                 break;
 644         default:
 645                 return rte_flow_error_set
 646                         (error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 648                          "unknown data sample field mode");
 649         }
 650         if (!match) {
 651                 if (!field->field_size)
 652                         return rte_flow_error_set
 653                                 (error, EINVAL,
 654                                 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 655                                 "zero sample field width");
 656                 if (field->field_id)
 657                         DRV_LOG(DEBUG, "sample field id hint ignored");
 658         } else {
 659                 if (field->field_mode != match->field_mode ||
 660                     field->offset_base | match->offset_base ||
 661                     field->offset_mask | match->offset_mask ||
 662                     field->offset_shift | match->offset_shift)
 663                         return 0;
 664         }
 665         start = field->field_base;
 666         end = start + field->field_size;
 667         /* Add the new or similar field to interval array. */
 668         if (!cover->num) {
 669                 cover->start[cover->num] = start;
 670                 cover->end[cover->num] = end;
 671                 cover->num = 1;
 672                 return 1;
 673         }
 674         for (i = 0; i < cover->num; i++) {
 675                 if (start > cover->end[i]) {
 676                         if (i >= (cover->num - 1u)) {
 677                                 mlx5_flex_insert_field(cover, cover->num,
 678                                                        start, end);
 679                                 break;
 680                         }
 681                         continue;
 682                 }
 683                 if (end < cover->start[i]) {
 684                         mlx5_flex_insert_field(cover, i, start, end);
 685                         break;
 686                 }
 687                 if (start < cover->start[i])
 688                         cover->start[i] = start;
 689                 if (end > cover->end[i]) {
 690                         cover->end[i] = end;
 691                         if (i < (cover->num - 1u))
 692                                 mlx5_flex_merge_field(cover, i);
 693                 }
 694                 break;
 695         }
 696         return 1;
 697 }
 698
 699 static void
 700 mlx5_flex_config_sample(struct mlx5_devx_match_sample_attr *na,
 701                         struct rte_flow_item_flex_field *field,
 702                         enum rte_flow_item_flex_tunnel_mode tunnel_mode)
 703 {
 704         memset(na, 0, sizeof(struct mlx5_devx_match_sample_attr));
 705         na->flow_match_sample_en = 1;
 706         switch (field->field_mode) {
 707         case FIELD_MODE_FIXED:
 708                 na->flow_match_sample_offset_mode =
 709                         MLX5_GRAPH_SAMPLE_OFFSET_FIXED;
 710                 break;
 711         case FIELD_MODE_OFFSET:
 712                 na->flow_match_sample_offset_mode =
 713                         MLX5_GRAPH_SAMPLE_OFFSET_FIELD;
 714                 na->flow_match_sample_field_offset = field->offset_base;
 715                 na->flow_match_sample_field_offset_mask = field->offset_mask;
 716                 na->flow_match_sample_field_offset_shift = field->offset_shift;
 717                 break;
 718         case FIELD_MODE_BITMASK:
 719                 na->flow_match_sample_offset_mode =
 720                         MLX5_GRAPH_SAMPLE_OFFSET_BITMASK;
 721                 na->flow_match_sample_field_offset = field->offset_base;
 722                 na->flow_match_sample_field_offset_mask = field->offset_mask;
 723                 na->flow_match_sample_field_offset_shift = field->offset_shift;
 724                 break;
 725         default:
 726                 MLX5_ASSERT(false);
 727                 break;
 728         }
 729         switch (tunnel_mode) {
 730         case FLEX_TUNNEL_MODE_SINGLE:
 731                 /* Fallthrough */
 732         case FLEX_TUNNEL_MODE_TUNNEL:
 733                 na->flow_match_sample_tunnel_mode =
 734                         MLX5_GRAPH_SAMPLE_TUNNEL_FIRST;
 735                 break;
 736         case FLEX_TUNNEL_MODE_MULTI:
 737                 /* Fallthrough */
 738         case FLEX_TUNNEL_MODE_OUTER:
 739                 na->flow_match_sample_tunnel_mode =
 740                         MLX5_GRAPH_SAMPLE_TUNNEL_OUTER;
 741                 break;
 742         case FLEX_TUNNEL_MODE_INNER:
 743                 na->flow_match_sample_tunnel_mode =
 744                         MLX5_GRAPH_SAMPLE_TUNNEL_INNER;
 745                 break;
 746         default:
 747                 MLX5_ASSERT(false);
 748                 break;
 749         }
 750 }
 751
 752 /* Map specified field to set/subset of allocated sample registers. */
 753 static int
 754 mlx5_flex_map_sample(struct rte_flow_item_flex_field *field,
 755                      struct mlx5_flex_parser_devx *parser,
 756                      struct mlx5_flex_item *item,
 757                      struct rte_flow_error *error)
 758 {
 759         struct mlx5_devx_match_sample_attr node;
 760         int32_t start = field->field_base;
 761         int32_t end = start + field->field_size;
 762         struct mlx5_flex_pattern_field *trans;
 763         uint32_t i, done_bits = 0;
 764
 765         if (field->field_mode == FIELD_MODE_DUMMY) {
 766                 done_bits = field->field_size;
 767                 while (done_bits) {
 768                         uint32_t part = RTE_MIN(done_bits,
 769                                                 sizeof(uint32_t) * CHAR_BIT);
 770                         if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM)
 771                                 return rte_flow_error_set
 772                                         (error,
 773                                          EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 774                                          "too many flex item pattern translations");
 775                         trans = &item->map[item->mapnum];
 776                         trans->reg_id = MLX5_INVALID_SAMPLE_REG_ID;
 777                         trans->shift = 0;
 778                         trans->width = part;
 779                         item->mapnum++;
 780                         done_bits -= part;
 781                 }
 782                 return 0;
 783         }
 784         mlx5_flex_config_sample(&node, field, item->tunnel_mode);
 785         for (i = 0; i < parser->num_samples; i++) {
 786                 struct mlx5_devx_match_sample_attr *sample =
 787                         &parser->devx_conf.sample[i];
 788                 int32_t reg_start, reg_end;
 789                 int32_t cov_start, cov_end;
 790
 791                 MLX5_ASSERT(sample->flow_match_sample_en);
 792                 if (!sample->flow_match_sample_en)
 793                         break;
 794                 node.flow_match_sample_field_base_offset =
 795                         sample->flow_match_sample_field_base_offset;
 796                 if (memcmp(&node, sample, sizeof(node)))
 797                         continue;
 798                 reg_start = (int8_t)sample->flow_match_sample_field_base_offset;
 799                 reg_start *= CHAR_BIT;
 800                 reg_end = reg_start + 32;
 801                 if (end <= reg_start || start >= reg_end)
 802                         continue;
 803                 cov_start = RTE_MAX(reg_start, start);
 804                 cov_end = RTE_MIN(reg_end, end);
 805                 MLX5_ASSERT(cov_end > cov_start);
 806                 done_bits += cov_end - cov_start;
 807                 if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM)
 808                         return rte_flow_error_set
 809                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 810                                  "too many flex item pattern translations");
 811                 trans = &item->map[item->mapnum];
 812                 item->mapnum++;
 813                 trans->reg_id = i;
 814                 trans->shift = cov_start - reg_start;
 815                 trans->width = cov_end - cov_start;
 816         }
 817         if (done_bits != field->field_size) {
 818                 MLX5_ASSERT(false);
 819                 return rte_flow_error_set
 820                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 821                          "failed to map field to sample register");
 822         }
 823         return 0;
 824 }
 825
 826 /* Allocate sample registers for the specified field type and interval array. */
 827 static int
 828 mlx5_flex_alloc_sample(struct mlx5_flex_field_cover *cover,
 829                        struct mlx5_flex_parser_devx *parser,
 830                        struct mlx5_flex_item *item,
 831                        struct rte_flow_item_flex_field *field,
 832                        struct mlx5_hca_flex_attr *attr,
 833                        struct rte_flow_error *error)
 834 {
 835         struct mlx5_devx_match_sample_attr node;
 836         uint32_t idx = 0;
 837
 838         mlx5_flex_config_sample(&node, field, item->tunnel_mode);
 839         while (idx < cover->num) {
 840                 int32_t start, end;
 841
 842                 /*
 843                  * Sample base offsets are in bytes, should be aligned
 844                  * to 32-bit as required by firmware for samples.
 845                  */
 846                 start = RTE_ALIGN_FLOOR(cover->start[idx],
 847                                         sizeof(uint32_t) * CHAR_BIT);
 848                 node.flow_match_sample_field_base_offset =
 849                                                 (start / CHAR_BIT) & 0xFF;
 850                 /* Allocate sample register. */
 851                 if (parser->num_samples >= MLX5_GRAPH_NODE_SAMPLE_NUM ||
 852                     parser->num_samples >= attr->max_num_sample ||
 853                     parser->num_samples >= attr->max_num_prog_sample)
 854                         return rte_flow_error_set
 855                                 (error, EINVAL,
 856                                  RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 857                                  "no sample registers to handle all flex item fields");
 858                 parser->devx_conf.sample[parser->num_samples] = node;
 859                 parser->num_samples++;
 860                 /* Remove or update covered intervals. */
 861                 end = start + 32;
 862                 while (idx < cover->num) {
 863                         if (end >= cover->end[idx]) {
 864                                 idx++;
 865                                 continue;
 866                         }
 867                         if (end > cover->start[idx])
 868                                 cover->start[idx] = end;
 869                         break;
 870                 }
 871         }
 872         return 0;
 873 }
 874
 875 static int
 876 mlx5_flex_translate_sample(struct mlx5_hca_flex_attr *attr,
 877                            const struct rte_flow_item_flex_conf *conf,
 878                            struct mlx5_flex_parser_devx *parser,
 879                            struct mlx5_flex_item *item,
 880                            struct rte_flow_error *error)
 881 {
 882         struct mlx5_flex_field_cover cover;
 883         uint32_t i, j;
 884         int ret;
 885
 886         switch (conf->tunnel) {
 887         case FLEX_TUNNEL_MODE_SINGLE:
 888                 /* Fallthrough */
 889         case FLEX_TUNNEL_MODE_OUTER:
 890                 /* Fallthrough */
 891         case FLEX_TUNNEL_MODE_INNER:
 892                 /* Fallthrough */
 893         case FLEX_TUNNEL_MODE_MULTI:
 894                 /* Fallthrough */
 895         case FLEX_TUNNEL_MODE_TUNNEL:
 896                 break;
 897         default:
 898                 return rte_flow_error_set
 899                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 900                          "unrecognized tunnel mode");
 901         }
 902         item->tunnel_mode = conf->tunnel;
 903         if (conf->nb_samples > MLX5_FLEX_ITEM_MAPPING_NUM)
 904                 return rte_flow_error_set
 905                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 906                          "sample field number exceeds limit");
 907         /*
 908          * The application can specify fields smaller or bigger than 32 bits
 909          * covered with single sample register and it can specify field
 910          * offsets in any order.
 911          *
 912          * Gather all similar fields together, build array of bit intervals
 913          * in asсending order and try to cover with the smallest set of sample
 914          * registers.
 915          */
 916         memset(&cover, 0, sizeof(cover));
 917         for (i = 0; i < conf->nb_samples; i++) {
 918                 struct rte_flow_item_flex_field *fl = conf->sample_data + i;
 919
 920                 /* Check whether field was covered in the previous iteration. */
 921                 if (cover.mapped[i / CHAR_BIT] & (1u << (i % CHAR_BIT)))
 922                         continue;
 923                 if (fl->field_mode == FIELD_MODE_DUMMY)
 924                         continue;
 925                 /* Build an interval array for the field and similar ones */
 926                 cover.num = 0;
 927                 /* Add the first field to array unconditionally. */
 928                 ret = mlx5_flex_cover_sample(&cover, fl, NULL, attr, error);
 929                 if (ret < 0)
 930                         return ret;
 931                 MLX5_ASSERT(ret > 0);
 932                 cover.mapped[i / CHAR_BIT] |= 1u << (i % CHAR_BIT);
 933                 for (j = i + 1; j < conf->nb_samples; j++) {
 934                         struct rte_flow_item_flex_field *ft;
 935
 936                         /* Add field to array if its type matches. */
 937                         ft = conf->sample_data + j;
 938                         ret = mlx5_flex_cover_sample(&cover, ft, fl,
 939                                                      attr, error);
 940                         if (ret < 0)
 941                                 return ret;
 942                         if (!ret)
 943                                 continue;
 944                         cover.mapped[j / CHAR_BIT] |= 1u << (j % CHAR_BIT);
 945                 }
 946                 /* Allocate sample registers to cover array of intervals. */
 947                 ret = mlx5_flex_alloc_sample(&cover, parser, item,
 948                                              fl, attr, error);
 949                 if (ret)
 950                         return ret;
 951         }
 952         /* Build the item pattern translating data on flow creation. */
 953         item->mapnum = 0;
 954         memset(&item->map, 0, sizeof(item->map));
 955         for (i = 0; i < conf->nb_samples; i++) {
 956                 struct rte_flow_item_flex_field *fl = conf->sample_data + i;
 957
 958                 ret = mlx5_flex_map_sample(fl, parser, item, error);
 959                 if (ret) {
 960                         MLX5_ASSERT(false);
 961                         return ret;
 962                 }
 963         }
 964         if (conf->tunnel == FLEX_TUNNEL_MODE_MULTI) {
 965                 /*
 966                  * In FLEX_TUNNEL_MODE_MULTI tunnel mode PMD creates 2 sets
 967                  * of samples. The first set is for outer and the second set
 968                  * for inner flex flow item. Outer and inner samples differ
 969                  * only in tunnel_mode.
 970                  */
 971                 if (parser->num_samples > MLX5_GRAPH_NODE_SAMPLE_NUM / 2)
 972                         return rte_flow_error_set
 973                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
 974                                  "no sample registers for inner");
 975                 rte_memcpy(parser->devx_conf.sample + parser->num_samples,
 976                            parser->devx_conf.sample,
 977                            parser->num_samples *
 978                                         sizeof(parser->devx_conf.sample[0]));
 979                 for (i = 0; i < parser->num_samples; i++) {
 980                         struct mlx5_devx_match_sample_attr *sm = i +
 981                                 parser->devx_conf.sample + parser->num_samples;
 982
 983                         sm->flow_match_sample_tunnel_mode =
 984                                                 MLX5_GRAPH_SAMPLE_TUNNEL_INNER;
 985                 }
 986                 parser->num_samples *= 2;
 987         }
 988         return 0;
 989 }
 990
 991 static int
 992 mlx5_flex_arc_type(enum rte_flow_item_type type, int in)
 993 {
 994         switch (type) {
 995         case RTE_FLOW_ITEM_TYPE_ETH:
 996                 return  MLX5_GRAPH_ARC_NODE_MAC;
 997         case RTE_FLOW_ITEM_TYPE_IPV4:
 998                 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV4;
 999         case RTE_FLOW_ITEM_TYPE_IPV6:
1000                 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV6;
1001         case RTE_FLOW_ITEM_TYPE_UDP:
1002                 return MLX5_GRAPH_ARC_NODE_UDP;
1003         case RTE_FLOW_ITEM_TYPE_TCP:
1004                 return MLX5_GRAPH_ARC_NODE_TCP;
1005         case RTE_FLOW_ITEM_TYPE_MPLS:
1006                 return MLX5_GRAPH_ARC_NODE_MPLS;
1007         case RTE_FLOW_ITEM_TYPE_GRE:
1008                 return MLX5_GRAPH_ARC_NODE_GRE;
1009         case RTE_FLOW_ITEM_TYPE_GENEVE:
1010                 return MLX5_GRAPH_ARC_NODE_GENEVE;
1011         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1012                 return MLX5_GRAPH_ARC_NODE_VXLAN_GPE;
1013         default:
1014                 return -EINVAL;
1015         }
1016 }
1017
1018 static int
1019 mlx5_flex_arc_in_eth(const struct rte_flow_item *item,
1020                      struct rte_flow_error *error)
1021 {
1022         const struct rte_flow_item_eth *spec = item->spec;
1023         const struct rte_flow_item_eth *mask = item->mask;
1024         struct rte_flow_item_eth eth = { .hdr.ether_type = RTE_BE16(0xFFFF) };
1025
1026         if (memcmp(mask, &eth, sizeof(struct rte_flow_item_eth))) {
1027                 return rte_flow_error_set
1028                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1029                          "invalid eth item mask");
1030         }
1031         return rte_be_to_cpu_16(spec->hdr.ether_type);
1032 }
1033
1034 static int
1035 mlx5_flex_arc_in_udp(const struct rte_flow_item *item,
1036                      struct rte_flow_error *error)
1037 {
1038         const struct rte_flow_item_udp *spec = item->spec;
1039         const struct rte_flow_item_udp *mask = item->mask;
1040         struct rte_flow_item_udp udp = { .hdr.dst_port = RTE_BE16(0xFFFF) };
1041
1042         if (memcmp(mask, &udp, sizeof(struct rte_flow_item_udp))) {
1043                 return rte_flow_error_set
1044                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1045                          "invalid eth item mask");
1046         }
1047         return rte_be_to_cpu_16(spec->hdr.dst_port);
1048 }
1049
1050 static int
1051 mlx5_flex_translate_arc_in(struct mlx5_hca_flex_attr *attr,
1052                            const struct rte_flow_item_flex_conf *conf,
1053                            struct mlx5_flex_parser_devx *devx,
1054                            struct mlx5_flex_item *item,
1055                            struct rte_flow_error *error)
1056 {
1057         struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
1058         uint32_t i;
1059
1060         RTE_SET_USED(item);
1061         if (conf->nb_inputs > attr->max_num_arc_in)
1062                 return rte_flow_error_set
1063                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1064                          "too many input links");
1065         for (i = 0; i < conf->nb_inputs; i++) {
1066                 struct mlx5_devx_graph_arc_attr *arc = node->in + i;
1067                 struct rte_flow_item_flex_link *link = conf->input_link + i;
1068                 const struct rte_flow_item *rte_item = &link->item;
1069                 int arc_type;
1070                 int ret;
1071
1072                 if (!rte_item->spec || !rte_item->mask || rte_item->last)
1073                         return rte_flow_error_set
1074                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1075                                  "invalid flex item IN arc format");
1076                 arc_type = mlx5_flex_arc_type(rte_item->type, true);
1077                 if (arc_type < 0 || !(attr->node_in & RTE_BIT32(arc_type)))
1078                         return rte_flow_error_set
1079                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1080                                  "unsupported flex item IN arc type");
1081                 arc->arc_parse_graph_node = arc_type;
1082                 arc->start_inner_tunnel = 0;
1083                 /*
1084                  * Configure arc IN condition value. The value location depends
1085                  * on protocol. Current FW version supports IP & UDP for IN
1086                  * arcs only, and locations for these protocols are defined.
1087                  * Add more protocols when available.
1088                  */
1089                 switch (rte_item->type) {
1090                 case RTE_FLOW_ITEM_TYPE_ETH:
1091                         ret = mlx5_flex_arc_in_eth(rte_item, error);
1092                         break;
1093                 case RTE_FLOW_ITEM_TYPE_UDP:
1094                         ret = mlx5_flex_arc_in_udp(rte_item, error);
1095                         break;
1096                 default:
1097                         MLX5_ASSERT(false);
1098                         return rte_flow_error_set
1099                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1100                                  "unsupported flex item IN arc type");
1101                 }
1102                 if (ret < 0)
1103                         return ret;
1104                 arc->compare_condition_value = (uint16_t)ret;
1105         }
1106         return 0;
1107 }
1108
1109 static int
1110 mlx5_flex_translate_arc_out(struct mlx5_hca_flex_attr *attr,
1111                             const struct rte_flow_item_flex_conf *conf,
1112                             struct mlx5_flex_parser_devx *devx,
1113                             struct mlx5_flex_item *item,
1114                             struct rte_flow_error *error)
1115 {
1116         struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
1117         bool is_tunnel = conf->tunnel == FLEX_TUNNEL_MODE_TUNNEL;
1118         uint32_t i;
1119
1120         RTE_SET_USED(item);
1121         if (conf->nb_outputs > attr->max_num_arc_out)
1122                 return rte_flow_error_set
1123                         (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1124                          "too many output links");
1125         for (i = 0; i < conf->nb_outputs; i++) {
1126                 struct mlx5_devx_graph_arc_attr *arc = node->out + i;
1127                 struct rte_flow_item_flex_link *link = conf->output_link + i;
1128                 const struct rte_flow_item *rte_item = &link->item;
1129                 int arc_type;
1130
1131                 if (rte_item->spec || rte_item->mask || rte_item->last)
1132                         return rte_flow_error_set
1133                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1134                                  "flex node: invalid OUT arc format");
1135                 arc_type = mlx5_flex_arc_type(rte_item->type, false);
1136                 if (arc_type < 0 || !(attr->node_out & RTE_BIT32(arc_type)))
1137                         return rte_flow_error_set
1138                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1139                                  "unsupported flex item OUT arc type");
1140                 arc->arc_parse_graph_node = arc_type;
1141                 arc->start_inner_tunnel = !!is_tunnel;
1142                 arc->compare_condition_value = link->next;
1143         }
1144         return 0;
1145 }
1146
1147 /* Translate RTE flex item API configuration into flaex parser settings. */
1148 static int
1149 mlx5_flex_translate_conf(struct rte_eth_dev *dev,
1150                          const struct rte_flow_item_flex_conf *conf,
1151                          struct mlx5_flex_parser_devx *devx,
1152                          struct mlx5_flex_item *item,
1153                          struct rte_flow_error *error)
1154 {
1155         struct mlx5_priv *priv = dev->data->dev_private;
1156         struct mlx5_hca_flex_attr *attr = &priv->config.hca_attr.flex;
1157         int ret;
1158
1159         ret = mlx5_flex_translate_length(attr, conf, devx, error);
1160         if (ret)
1161                 return ret;
1162         ret = mlx5_flex_translate_next(attr, conf, devx, error);
1163         if (ret)
1164                 return ret;
1165         ret = mlx5_flex_translate_sample(attr, conf, devx, item, error);
1166         if (ret)
1167                 return ret;
1168         ret = mlx5_flex_translate_arc_in(attr, conf, devx, item, error);
1169         if (ret)
1170                 return ret;
1171         ret = mlx5_flex_translate_arc_out(attr, conf, devx, item, error);
1172         if (ret)
1173                 return ret;
1174         return 0;
1175 }
1176
1177 /**
1178  * Create the flex item with specified configuration over the Ethernet device.
1179  *
1180  * @param dev
1181  *   Ethernet device to create flex item on.
1182  * @param[in] conf
1183  *   Flex item configuration.
1184  * @param[out] error
1185  *   Perform verbose error reporting if not NULL. PMDs initialize this
1186  *   structure in case of error only.
1187  *
1188  * @return
1189  *   Non-NULL opaque pointer on success, NULL otherwise and rte_errno is set.
1190  */
1191 struct rte_flow_item_flex_handle *
1192 flow_dv_item_create(struct rte_eth_dev *dev,
1193                     const struct rte_flow_item_flex_conf *conf,
1194                     struct rte_flow_error *error)
1195 {
1196         struct mlx5_priv *priv = dev->data->dev_private;
1197         struct mlx5_flex_parser_devx devx_config = { .devx_obj = NULL };
1198         struct mlx5_flex_item *flex;
1199         struct mlx5_list_entry *ent;
1200
1201         MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
1202         flex = mlx5_flex_alloc(priv);
1203         if (!flex) {
1204                 rte_flow_error_set(error, ENOMEM,
1205                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1206                                    "too many flex items created on the port");
1207                 return NULL;
1208         }
1209         if (mlx5_flex_translate_conf(dev, conf, &devx_config, flex, error))
1210                 goto error;
1211         ent = mlx5_list_register(priv->sh->flex_parsers_dv, &devx_config);
1212         if (!ent) {
1213                 rte_flow_error_set(error, ENOMEM,
1214                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1215                                    "flex item creation failure");
1216                 goto error;
1217         }
1218         flex->devx_fp = container_of(ent, struct mlx5_flex_parser_devx, entry);
1219         /* Mark initialized flex item valid. */
1220         __atomic_add_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE);
1221         return (struct rte_flow_item_flex_handle *)flex;
1222
1223 error:
1224         mlx5_flex_free(priv, flex);
1225         return NULL;
1226 }
1227
1228 /**
1229  * Release the flex item on the specified Ethernet device.
1230  *
1231  * @param dev
1232  *   Ethernet device to destroy flex item on.
1233  * @param[in] handle
1234  *   Handle of the item existing on the specified device.
1235  * @param[out] error
1236  *   Perform verbose error reporting if not NULL. PMDs initialize this
1237  *   structure in case of error only.
1238  *
1239  * @return
1240  *   0 on success, a negative errno value otherwise and rte_errno is set.
1241  */
1242 int
1243 flow_dv_item_release(struct rte_eth_dev *dev,
1244                      const struct rte_flow_item_flex_handle *handle,
1245                      struct rte_flow_error *error)
1246 {
1247         struct mlx5_priv *priv = dev->data->dev_private;
1248         struct mlx5_flex_item *flex =
1249                 (struct mlx5_flex_item *)(uintptr_t)handle;
1250         uint32_t old_refcnt = 1;
1251         int rc;
1252
1253         MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
1254         rte_spinlock_lock(&priv->flex_item_sl);
1255         if (mlx5_flex_index(priv, flex) < 0) {
1256                 rte_spinlock_unlock(&priv->flex_item_sl);
1257                 return rte_flow_error_set(error, EINVAL,
1258                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1259                                           "invalid flex item handle value");
1260         }
1261         if (!__atomic_compare_exchange_n(&flex->refcnt, &old_refcnt, 0, 0,
1262                                          __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
1263                 rte_spinlock_unlock(&priv->flex_item_sl);
1264                 return rte_flow_error_set(error, EBUSY,
1265                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1266                                           "flex item has flow references");
1267         }
1268         /* Flex item is marked as invalid, we can leave locked section. */
1269         rte_spinlock_unlock(&priv->flex_item_sl);
1270         MLX5_ASSERT(flex->devx_fp);
1271         rc = mlx5_list_unregister(priv->sh->flex_parsers_dv,
1272                                   &flex->devx_fp->entry);
1273         flex->devx_fp = NULL;
1274         mlx5_flex_free(priv, flex);
1275         if (rc < 0)
1276                 return rte_flow_error_set(error, EBUSY,
1277                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1278                                           "flex item release failure");
1279         return 0;
1280 }
1281
1282 /* DevX flex parser list callbacks. */
1283 struct mlx5_list_entry *
1284 mlx5_flex_parser_create_cb(void *list_ctx, void *ctx)
1285 {
1286         struct mlx5_dev_ctx_shared *sh = list_ctx;
1287         struct mlx5_flex_parser_devx *fp, *conf = ctx;
1288         int ret;
1289
1290         fp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flex_parser_devx),
1291                          0, SOCKET_ID_ANY);
1292         if (!fp)
1293                 return NULL;
1294         /* Copy the requested configurations. */
1295         fp->num_samples = conf->num_samples;
1296         memcpy(&fp->devx_conf, &conf->devx_conf, sizeof(fp->devx_conf));
1297         /* Create DevX flex parser. */
1298         fp->devx_obj = mlx5_devx_cmd_create_flex_parser(sh->cdev->ctx,
1299                                                         &fp->devx_conf);
1300         if (!fp->devx_obj)
1301                 goto error;
1302         /* Query the firmware assigned sample ids. */
1303         ret = mlx5_devx_cmd_query_parse_samples(fp->devx_obj,
1304                                                 fp->sample_ids,
1305                                                 fp->num_samples);
1306         if (ret)
1307                 goto error;
1308         DRV_LOG(DEBUG, "DEVx flex parser %p created, samples num: %u",
1309                 (const void *)fp, fp->num_samples);
1310         return &fp->entry;
1311 error:
1312         if (fp->devx_obj)
1313                 mlx5_devx_cmd_destroy((void *)(uintptr_t)fp->devx_obj);
1314         if (fp)
1315                 mlx5_free(fp);
1316         return NULL;
1317 }
1318
1319 int
1320 mlx5_flex_parser_match_cb(void *list_ctx,
1321                           struct mlx5_list_entry *iter, void *ctx)
1322 {
1323         struct mlx5_flex_parser_devx *fp =
1324                 container_of(iter, struct mlx5_flex_parser_devx, entry);
1325         struct mlx5_flex_parser_devx *org =
1326                 container_of(ctx, struct mlx5_flex_parser_devx, entry);
1327
1328         RTE_SET_USED(list_ctx);
1329         return !iter || !ctx || memcmp(&fp->devx_conf,
1330                                        &org->devx_conf,
1331                                        sizeof(fp->devx_conf));
1332 }
1333
1334 void
1335 mlx5_flex_parser_remove_cb(void *list_ctx, struct mlx5_list_entry *entry)
1336 {
1337         struct mlx5_flex_parser_devx *fp =
1338                 container_of(entry, struct mlx5_flex_parser_devx, entry);
1339
1340         RTE_SET_USED(list_ctx);
1341         MLX5_ASSERT(fp->devx_obj);
1342         claim_zero(mlx5_devx_cmd_destroy(fp->devx_obj));
1343         DRV_LOG(DEBUG, "DEVx flex parser %p destroyed", (const void *)fp);
1344         mlx5_free(entry);
1345 }
1346
1347 struct mlx5_list_entry *
1348 mlx5_flex_parser_clone_cb(void *list_ctx,
1349                           struct mlx5_list_entry *entry, void *ctx)
1350 {
1351         struct mlx5_flex_parser_devx *fp;
1352
1353         RTE_SET_USED(list_ctx);
1354         RTE_SET_USED(entry);
1355         fp = mlx5_malloc(0, sizeof(struct mlx5_flex_parser_devx),
1356                          0, SOCKET_ID_ANY);
1357         if (!fp)
1358                 return NULL;
1359         memcpy(fp, ctx, sizeof(struct mlx5_flex_parser_devx));
1360         return &fp->entry;
1361 }
1362
1363 void
1364 mlx5_flex_parser_clone_free_cb(void *list_ctx, struct mlx5_list_entry *entry)
1365 {
1366         struct mlx5_flex_parser_devx *fp =
1367                 container_of(entry, struct mlx5_flex_parser_devx, entry);
1368         RTE_SET_USED(list_ctx);
1369         mlx5_free(fp);
1370 }