1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
4 #include <rte_malloc.h>
5 #include <mlx5_devx_cmds.h>
6 #include <mlx5_malloc.h>
10 static_assert(sizeof(uint32_t) * CHAR_BIT >= MLX5_PORT_FLEX_ITEM_NUM,
11 "Flex item maximal number exceeds uint32_t bit width");
14 * Routine called once on port initialization to init flex item
15 * related infrastructure initialization
18 * Ethernet device to perform flex item initialization
21 * 0 on success, a negative errno value otherwise and rte_errno is set.
24 mlx5_flex_item_port_init(struct rte_eth_dev *dev)
26 struct mlx5_priv *priv = dev->data->dev_private;
28 rte_spinlock_init(&priv->flex_item_sl);
29 MLX5_ASSERT(!priv->flex_item_map);
34 * Routine called once on port close to perform flex item
35 * related infrastructure cleanup.
38 * Ethernet device to perform cleanup
41 mlx5_flex_item_port_cleanup(struct rte_eth_dev *dev)
43 struct mlx5_priv *priv = dev->data->dev_private;
46 for (i = 0; i < MLX5_PORT_FLEX_ITEM_NUM && priv->flex_item_map ; i++) {
47 if (priv->flex_item_map & (1 << i)) {
48 struct mlx5_flex_item *flex = &priv->flex_item[i];
50 claim_zero(mlx5_list_unregister
51 (priv->sh->flex_parsers_dv,
52 &flex->devx_fp->entry));
55 priv->flex_item_map &= ~(1 << i);
61 mlx5_flex_index(struct mlx5_priv *priv, struct mlx5_flex_item *item)
63 uintptr_t start = (uintptr_t)&priv->flex_item[0];
64 uintptr_t entry = (uintptr_t)item;
65 uintptr_t idx = (entry - start) / sizeof(struct mlx5_flex_item);
68 idx >= MLX5_PORT_FLEX_ITEM_NUM ||
69 (entry - start) % sizeof(struct mlx5_flex_item) ||
70 !(priv->flex_item_map & (1u << idx)))
75 static struct mlx5_flex_item *
76 mlx5_flex_alloc(struct mlx5_priv *priv)
78 struct mlx5_flex_item *item = NULL;
80 rte_spinlock_lock(&priv->flex_item_sl);
81 if (~priv->flex_item_map) {
82 uint32_t idx = rte_bsf32(~priv->flex_item_map);
84 if (idx < MLX5_PORT_FLEX_ITEM_NUM) {
85 item = &priv->flex_item[idx];
86 MLX5_ASSERT(!item->refcnt);
87 MLX5_ASSERT(!item->devx_fp);
89 __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE);
90 priv->flex_item_map |= 1u << idx;
93 rte_spinlock_unlock(&priv->flex_item_sl);
98 mlx5_flex_free(struct mlx5_priv *priv, struct mlx5_flex_item *item)
100 int idx = mlx5_flex_index(priv, item);
102 MLX5_ASSERT(idx >= 0 &&
103 idx < MLX5_PORT_FLEX_ITEM_NUM &&
104 (priv->flex_item_map & (1u << idx)));
106 rte_spinlock_lock(&priv->flex_item_sl);
107 MLX5_ASSERT(!item->refcnt);
108 MLX5_ASSERT(!item->devx_fp);
109 item->devx_fp = NULL;
110 __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE);
111 priv->flex_item_map &= ~(1u << idx);
112 rte_spinlock_unlock(&priv->flex_item_sl);
117 mlx5_flex_get_bitfield(const struct rte_flow_item_flex *item,
118 uint32_t pos, uint32_t width, uint32_t shift)
120 const uint8_t *ptr = item->pattern + pos / CHAR_BIT;
123 /* Proceed the bitfield start byte. */
124 MLX5_ASSERT(width <= sizeof(uint32_t) * CHAR_BIT && width);
125 MLX5_ASSERT(width + shift <= sizeof(uint32_t) * CHAR_BIT);
126 if (item->length <= pos / CHAR_BIT)
128 val = *ptr++ >> (pos % CHAR_BIT);
129 vbits = CHAR_BIT - pos % CHAR_BIT;
130 pos = (pos + vbits) / CHAR_BIT;
131 vbits = RTE_MIN(vbits, width);
132 val &= RTE_BIT32(vbits) - 1;
133 while (vbits < width && pos < item->length) {
134 uint32_t part = RTE_MIN(width - vbits, (uint32_t)CHAR_BIT);
135 uint32_t tmp = *ptr++;
138 tmp &= RTE_BIT32(part) - 1;
142 return rte_bswap32(val <<= shift);
145 #define SET_FP_MATCH_SAMPLE_ID(x, def, msk, val, sid) \
147 uint32_t tmp, out = (def); \
148 tmp = MLX5_GET(fte_match_set_misc4, misc4_v, \
149 prog_sample_field_value_##x); \
150 tmp = (tmp & ~out) | (val); \
151 MLX5_SET(fte_match_set_misc4, misc4_v, \
152 prog_sample_field_value_##x, tmp); \
153 tmp = MLX5_GET(fte_match_set_misc4, misc4_m, \
154 prog_sample_field_value_##x); \
155 tmp = (tmp & ~out) | (msk); \
156 MLX5_SET(fte_match_set_misc4, misc4_m, \
157 prog_sample_field_value_##x, tmp); \
158 tmp = tmp ? (sid) : 0; \
159 MLX5_SET(fte_match_set_misc4, misc4_v, \
160 prog_sample_field_id_##x, tmp);\
161 MLX5_SET(fte_match_set_misc4, misc4_m, \
162 prog_sample_field_id_##x, tmp); \
165 __rte_always_inline static void
166 mlx5_flex_set_match_sample(void *misc4_m, void *misc4_v,
167 uint32_t def, uint32_t mask, uint32_t value,
168 uint32_t sample_id, uint32_t id)
172 SET_FP_MATCH_SAMPLE_ID(0, def, mask, value, sample_id);
175 SET_FP_MATCH_SAMPLE_ID(1, def, mask, value, sample_id);
178 SET_FP_MATCH_SAMPLE_ID(2, def, mask, value, sample_id);
181 SET_FP_MATCH_SAMPLE_ID(3, def, mask, value, sample_id);
184 SET_FP_MATCH_SAMPLE_ID(4, def, mask, value, sample_id);
187 SET_FP_MATCH_SAMPLE_ID(5, def, mask, value, sample_id);
190 SET_FP_MATCH_SAMPLE_ID(6, def, mask, value, sample_id);
193 SET_FP_MATCH_SAMPLE_ID(7, def, mask, value, sample_id);
199 #undef SET_FP_MATCH_SAMPLE_ID
202 * Translate item pattern into matcher fields according to translation
206 * Ethernet device to translate flex item on.
207 * @param[in, out] matcher
208 * Flow matcher to configure
209 * @param[in, out] key
210 * Flow matcher value.
212 * Flow pattern to translate.
213 * @param[in] is_inner
214 * Inner Flex Item (follows after tunnel header).
217 * 0 on success, a negative errno value otherwise and rte_errno is set.
220 mlx5_flex_flow_translate_item(struct rte_eth_dev *dev,
221 void *matcher, void *key,
222 const struct rte_flow_item *item,
225 const struct rte_flow_item_flex *spec, *mask;
226 void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
228 void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
229 struct mlx5_flex_item *tp;
233 MLX5_ASSERT(item->spec && item->mask);
236 tp = (struct mlx5_flex_item *)spec->handle;
237 MLX5_ASSERT(mlx5_flex_index(dev->data->dev_private, tp) >= 0);
238 for (i = 0; i < tp->mapnum; i++) {
239 struct mlx5_flex_pattern_field *map = tp->map + i;
240 uint32_t id = map->reg_id;
241 uint32_t def = (RTE_BIT64(map->width) - 1) << map->shift;
244 /* Skip placeholders for DUMMY fields. */
245 if (id == MLX5_INVALID_SAMPLE_REG_ID) {
249 val = mlx5_flex_get_bitfield(spec, pos, map->width, map->shift);
250 msk = mlx5_flex_get_bitfield(mask, pos, map->width, map->shift);
251 MLX5_ASSERT(map->width);
252 MLX5_ASSERT(id < tp->devx_fp->num_samples);
253 if (tp->tunnel_mode == FLEX_TUNNEL_MODE_MULTI && is_inner) {
254 uint32_t num_samples = tp->devx_fp->num_samples / 2;
256 MLX5_ASSERT(tp->devx_fp->num_samples % 2 == 0);
257 MLX5_ASSERT(id < num_samples);
260 mlx5_flex_set_match_sample(misc4_m, misc4_v,
261 def, msk & def, val & msk & def,
262 tp->devx_fp->sample_ids[id], id);
268 * Convert flex item handle (from the RTE flow) to flex item index on port.
269 * Optionally can increment flex item object reference count.
272 * Ethernet device to acquire flex item on.
274 * Flow item handle from item spec.
276 * If set - increment reference counter.
279 * >=0 - index on success, a negative errno value otherwise
280 * and rte_errno is set.
283 mlx5_flex_acquire_index(struct rte_eth_dev *dev,
284 struct rte_flow_item_flex_handle *handle,
287 struct mlx5_priv *priv = dev->data->dev_private;
288 struct mlx5_flex_item *flex = (struct mlx5_flex_item *)handle;
289 int ret = mlx5_flex_index(priv, flex);
297 __atomic_add_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE);
302 * Release flex item index on port - decrements reference counter by index.
305 * Ethernet device to acquire flex item on.
310 * 0 - on success, a negative errno value otherwise and rte_errno is set.
313 mlx5_flex_release_index(struct rte_eth_dev *dev,
316 struct mlx5_priv *priv = dev->data->dev_private;
317 struct mlx5_flex_item *flex;
319 if (index >= MLX5_PORT_FLEX_ITEM_NUM ||
320 !(priv->flex_item_map & (1u << index))) {
325 flex = priv->flex_item + index;
326 if (flex->refcnt <= 1) {
332 __atomic_sub_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE);
337 * Calculate largest mask value for a given shift.
340 * ------- ---------------
349 mlx5_flex_hdr_len_mask(uint8_t shift,
350 const struct mlx5_hca_flex_attr *attr)
353 int diff = shift - MLX5_PARSE_GRAPH_NODE_HDR_LEN_SHIFT_DWORD;
355 base_mask = mlx5_hca_parse_graph_node_base_hdr_len_mask(attr);
356 return diff == 0 ? base_mask :
357 diff < 0 ? (base_mask << -diff) & base_mask : base_mask >> diff;
361 mlx5_flex_translate_length(struct mlx5_hca_flex_attr *attr,
362 const struct rte_flow_item_flex_conf *conf,
363 struct mlx5_flex_parser_devx *devx,
364 struct rte_flow_error *error)
366 const struct rte_flow_item_flex_field *field = &conf->next_header;
367 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
368 uint32_t len_width, mask;
370 if (field->field_base % CHAR_BIT)
371 return rte_flow_error_set
372 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
373 "not byte aligned header length field");
374 switch (field->field_mode) {
375 case FIELD_MODE_DUMMY:
376 return rte_flow_error_set
377 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
378 "invalid header length field mode (DUMMY)");
379 case FIELD_MODE_FIXED:
380 if (!(attr->header_length_mode &
381 RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIXED)))
382 return rte_flow_error_set
383 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
384 "unsupported header length field mode (FIXED)");
385 if (attr->header_length_mask_width < field->field_size)
386 return rte_flow_error_set
387 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
388 "header length field width exceeds limit");
389 if (field->offset_shift < 0 ||
390 field->offset_shift > attr->header_length_mask_width)
391 return rte_flow_error_set
392 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
393 "invalid header length field shift (FIXED");
394 if (field->field_base < 0)
395 return rte_flow_error_set
396 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
397 "negative header length field base (FIXED)");
398 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED;
400 case FIELD_MODE_OFFSET:
401 if (!(attr->header_length_mode &
402 RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIELD)))
403 return rte_flow_error_set
404 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
405 "unsupported header length field mode (OFFSET)");
406 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIELD;
407 if (field->offset_mask == 0 ||
408 !rte_is_power_of_2(field->offset_mask + 1))
409 return rte_flow_error_set
410 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
411 "invalid length field offset mask (OFFSET)");
412 len_width = rte_fls_u32(field->offset_mask);
413 if (len_width > attr->header_length_mask_width)
414 return rte_flow_error_set
415 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
416 "length field offset mask too wide (OFFSET)");
417 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr);
418 if (mask < field->offset_mask)
419 return rte_flow_error_set
420 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
421 "length field shift too big (OFFSET)");
422 node->header_length_field_mask = RTE_MIN(mask,
425 case FIELD_MODE_BITMASK:
426 if (!(attr->header_length_mode &
427 RTE_BIT32(MLX5_GRAPH_NODE_LEN_BITMASK)))
428 return rte_flow_error_set
429 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
430 "unsupported header length field mode (BITMASK)");
431 if (attr->header_length_mask_width < field->field_size)
432 return rte_flow_error_set
433 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
434 "header length field width exceeds limit");
435 node->header_length_mode = MLX5_GRAPH_NODE_LEN_BITMASK;
436 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr);
437 if (mask < field->offset_mask)
438 return rte_flow_error_set
439 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
440 "length field shift too big (BITMASK)");
441 node->header_length_field_mask = RTE_MIN(mask,
445 return rte_flow_error_set
446 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
447 "unknown header length field mode");
449 if (field->field_base / CHAR_BIT >= 0 &&
450 field->field_base / CHAR_BIT > attr->max_base_header_length)
451 return rte_flow_error_set
452 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
453 "header length field base exceeds limit");
454 node->header_length_base_value = field->field_base / CHAR_BIT;
455 if (field->field_mode == FIELD_MODE_OFFSET ||
456 field->field_mode == FIELD_MODE_BITMASK) {
457 if (field->offset_shift > 15 || field->offset_shift < 0)
458 return rte_flow_error_set
459 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
460 "header length field shift exceeds limit");
461 node->header_length_field_shift = field->offset_shift;
462 node->header_length_field_offset = field->offset_base;
468 mlx5_flex_translate_next(struct mlx5_hca_flex_attr *attr,
469 const struct rte_flow_item_flex_conf *conf,
470 struct mlx5_flex_parser_devx *devx,
471 struct rte_flow_error *error)
473 const struct rte_flow_item_flex_field *field = &conf->next_protocol;
474 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
476 switch (field->field_mode) {
477 case FIELD_MODE_DUMMY:
478 if (conf->nb_outputs)
479 return rte_flow_error_set
480 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
481 "next protocol field is required (DUMMY)");
483 case FIELD_MODE_FIXED:
485 case FIELD_MODE_OFFSET:
486 return rte_flow_error_set
487 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
488 "unsupported next protocol field mode (OFFSET)");
490 case FIELD_MODE_BITMASK:
491 return rte_flow_error_set
492 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
493 "unsupported next protocol field mode (BITMASK)");
495 return rte_flow_error_set
497 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
498 "unknown next protocol field mode");
500 MLX5_ASSERT(field->field_mode == FIELD_MODE_FIXED);
501 if (!conf->nb_outputs)
502 return rte_flow_error_set
503 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
504 "out link(s) is required if next field present");
505 if (attr->max_next_header_offset < field->field_base)
506 return rte_flow_error_set
507 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
508 "next protocol field base exceeds limit");
509 if (field->offset_shift)
510 return rte_flow_error_set
511 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
512 "unsupported next protocol field shift");
513 node->next_header_field_offset = field->field_base;
514 node->next_header_field_size = field->field_size;
518 /* Helper structure to handle field bit intervals. */
519 struct mlx5_flex_field_cover {
521 int32_t start[MLX5_FLEX_ITEM_MAPPING_NUM];
522 int32_t end[MLX5_FLEX_ITEM_MAPPING_NUM];
523 uint8_t mapped[MLX5_FLEX_ITEM_MAPPING_NUM / CHAR_BIT + 1];
527 mlx5_flex_insert_field(struct mlx5_flex_field_cover *cover,
528 uint16_t num, int32_t start, int32_t end)
530 MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM);
531 MLX5_ASSERT(num <= cover->num);
532 if (num < cover->num) {
533 memmove(&cover->start[num + 1], &cover->start[num],
534 (cover->num - num) * sizeof(int32_t));
535 memmove(&cover->end[num + 1], &cover->end[num],
536 (cover->num - num) * sizeof(int32_t));
538 cover->start[num] = start;
539 cover->end[num] = end;
544 mlx5_flex_merge_field(struct mlx5_flex_field_cover *cover, uint16_t num)
549 MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM);
550 MLX5_ASSERT(num < (cover->num - 1));
551 end = cover->end[num];
552 for (i = num + 1; i < cover->num; i++) {
553 if (end < cover->start[i])
556 if (end <= cover->end[i]) {
557 cover->end[num] = cover->end[i];
562 MLX5_ASSERT(del < (cover->num - 1u - num));
564 MLX5_ASSERT(cover->num > num);
565 if ((cover->num - num) > 1) {
566 memmove(&cover->start[num + 1],
567 &cover->start[num + 1 + del],
568 (cover->num - num - 1) * sizeof(int32_t));
569 memmove(&cover->end[num + 1],
570 &cover->end[num + 1 + del],
571 (cover->num - num - 1) * sizeof(int32_t));
577 * Validate the sample field and update interval array
578 * if parameters match with the 'match" field.
581 * == 0 - no match, interval array not updated
582 * > 0 - match, interval array updated
585 mlx5_flex_cover_sample(struct mlx5_flex_field_cover *cover,
586 struct rte_flow_item_flex_field *field,
587 struct rte_flow_item_flex_field *match,
588 struct mlx5_hca_flex_attr *attr,
589 struct rte_flow_error *error)
594 switch (field->field_mode) {
595 case FIELD_MODE_DUMMY:
597 case FIELD_MODE_FIXED:
598 if (!(attr->sample_offset_mode &
599 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIXED)))
600 return rte_flow_error_set
602 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
603 "unsupported sample field mode (FIXED)");
604 if (field->offset_shift)
605 return rte_flow_error_set
607 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
608 "invalid sample field shift (FIXED");
609 if (field->field_base < 0)
610 return rte_flow_error_set
611 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
612 "invalid sample field base (FIXED)");
613 if (field->field_base / CHAR_BIT > attr->max_sample_base_offset)
614 return rte_flow_error_set
615 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
616 "sample field base exceeds limit (FIXED)");
618 case FIELD_MODE_OFFSET:
619 if (!(attr->sample_offset_mode &
620 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIELD)))
621 return rte_flow_error_set
623 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
624 "unsupported sample field mode (OFFSET)");
625 if (field->field_base / CHAR_BIT >= 0 &&
626 field->field_base / CHAR_BIT > attr->max_sample_base_offset)
627 return rte_flow_error_set
628 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
629 "sample field base exceeds limit");
631 case FIELD_MODE_BITMASK:
632 if (!(attr->sample_offset_mode &
633 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_BITMASK)))
634 return rte_flow_error_set
636 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
637 "unsupported sample field mode (BITMASK)");
638 if (field->field_base / CHAR_BIT >= 0 &&
639 field->field_base / CHAR_BIT > attr->max_sample_base_offset)
640 return rte_flow_error_set
641 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
642 "sample field base exceeds limit");
645 return rte_flow_error_set
647 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
648 "unknown data sample field mode");
651 if (!field->field_size)
652 return rte_flow_error_set
654 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
655 "zero sample field width");
657 DRV_LOG(DEBUG, "sample field id hint ignored");
659 if (field->field_mode != match->field_mode ||
660 field->offset_base | match->offset_base ||
661 field->offset_mask | match->offset_mask ||
662 field->offset_shift | match->offset_shift)
665 start = field->field_base;
666 end = start + field->field_size;
667 /* Add the new or similar field to interval array. */
669 cover->start[cover->num] = start;
670 cover->end[cover->num] = end;
674 for (i = 0; i < cover->num; i++) {
675 if (start > cover->end[i]) {
676 if (i >= (cover->num - 1u)) {
677 mlx5_flex_insert_field(cover, cover->num,
683 if (end < cover->start[i]) {
684 mlx5_flex_insert_field(cover, i, start, end);
687 if (start < cover->start[i])
688 cover->start[i] = start;
689 if (end > cover->end[i]) {
691 if (i < (cover->num - 1u))
692 mlx5_flex_merge_field(cover, i);
700 mlx5_flex_config_sample(struct mlx5_devx_match_sample_attr *na,
701 struct rte_flow_item_flex_field *field,
702 enum rte_flow_item_flex_tunnel_mode tunnel_mode)
704 memset(na, 0, sizeof(struct mlx5_devx_match_sample_attr));
705 na->flow_match_sample_en = 1;
706 switch (field->field_mode) {
707 case FIELD_MODE_FIXED:
708 na->flow_match_sample_offset_mode =
709 MLX5_GRAPH_SAMPLE_OFFSET_FIXED;
711 case FIELD_MODE_OFFSET:
712 na->flow_match_sample_offset_mode =
713 MLX5_GRAPH_SAMPLE_OFFSET_FIELD;
714 na->flow_match_sample_field_offset = field->offset_base;
715 na->flow_match_sample_field_offset_mask = field->offset_mask;
716 na->flow_match_sample_field_offset_shift = field->offset_shift;
718 case FIELD_MODE_BITMASK:
719 na->flow_match_sample_offset_mode =
720 MLX5_GRAPH_SAMPLE_OFFSET_BITMASK;
721 na->flow_match_sample_field_offset = field->offset_base;
722 na->flow_match_sample_field_offset_mask = field->offset_mask;
723 na->flow_match_sample_field_offset_shift = field->offset_shift;
729 switch (tunnel_mode) {
730 case FLEX_TUNNEL_MODE_SINGLE:
732 case FLEX_TUNNEL_MODE_TUNNEL:
733 na->flow_match_sample_tunnel_mode =
734 MLX5_GRAPH_SAMPLE_TUNNEL_FIRST;
736 case FLEX_TUNNEL_MODE_MULTI:
738 case FLEX_TUNNEL_MODE_OUTER:
739 na->flow_match_sample_tunnel_mode =
740 MLX5_GRAPH_SAMPLE_TUNNEL_OUTER;
742 case FLEX_TUNNEL_MODE_INNER:
743 na->flow_match_sample_tunnel_mode =
744 MLX5_GRAPH_SAMPLE_TUNNEL_INNER;
752 /* Map specified field to set/subset of allocated sample registers. */
754 mlx5_flex_map_sample(struct rte_flow_item_flex_field *field,
755 struct mlx5_flex_parser_devx *parser,
756 struct mlx5_flex_item *item,
757 struct rte_flow_error *error)
759 struct mlx5_devx_match_sample_attr node;
760 int32_t start = field->field_base;
761 int32_t end = start + field->field_size;
762 struct mlx5_flex_pattern_field *trans;
763 uint32_t i, done_bits = 0;
765 if (field->field_mode == FIELD_MODE_DUMMY) {
766 done_bits = field->field_size;
768 uint32_t part = RTE_MIN(done_bits,
769 sizeof(uint32_t) * CHAR_BIT);
770 if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM)
771 return rte_flow_error_set
773 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
774 "too many flex item pattern translations");
775 trans = &item->map[item->mapnum];
776 trans->reg_id = MLX5_INVALID_SAMPLE_REG_ID;
784 mlx5_flex_config_sample(&node, field, item->tunnel_mode);
785 for (i = 0; i < parser->num_samples; i++) {
786 struct mlx5_devx_match_sample_attr *sample =
787 &parser->devx_conf.sample[i];
788 int32_t reg_start, reg_end;
789 int32_t cov_start, cov_end;
791 MLX5_ASSERT(sample->flow_match_sample_en);
792 if (!sample->flow_match_sample_en)
794 node.flow_match_sample_field_base_offset =
795 sample->flow_match_sample_field_base_offset;
796 if (memcmp(&node, sample, sizeof(node)))
798 reg_start = (int8_t)sample->flow_match_sample_field_base_offset;
799 reg_start *= CHAR_BIT;
800 reg_end = reg_start + 32;
801 if (end <= reg_start || start >= reg_end)
803 cov_start = RTE_MAX(reg_start, start);
804 cov_end = RTE_MIN(reg_end, end);
805 MLX5_ASSERT(cov_end > cov_start);
806 done_bits += cov_end - cov_start;
807 if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM)
808 return rte_flow_error_set
809 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
810 "too many flex item pattern translations");
811 trans = &item->map[item->mapnum];
814 trans->shift = cov_start - reg_start;
815 trans->width = cov_end - cov_start;
817 if (done_bits != field->field_size) {
819 return rte_flow_error_set
820 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
821 "failed to map field to sample register");
826 /* Allocate sample registers for the specified field type and interval array. */
828 mlx5_flex_alloc_sample(struct mlx5_flex_field_cover *cover,
829 struct mlx5_flex_parser_devx *parser,
830 struct mlx5_flex_item *item,
831 struct rte_flow_item_flex_field *field,
832 struct mlx5_hca_flex_attr *attr,
833 struct rte_flow_error *error)
835 struct mlx5_devx_match_sample_attr node;
838 mlx5_flex_config_sample(&node, field, item->tunnel_mode);
839 while (idx < cover->num) {
843 * Sample base offsets are in bytes, should be aligned
844 * to 32-bit as required by firmware for samples.
846 start = RTE_ALIGN_FLOOR(cover->start[idx],
847 sizeof(uint32_t) * CHAR_BIT);
848 node.flow_match_sample_field_base_offset =
849 (start / CHAR_BIT) & 0xFF;
850 /* Allocate sample register. */
851 if (parser->num_samples >= MLX5_GRAPH_NODE_SAMPLE_NUM ||
852 parser->num_samples >= attr->max_num_sample ||
853 parser->num_samples >= attr->max_num_prog_sample)
854 return rte_flow_error_set
856 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
857 "no sample registers to handle all flex item fields");
858 parser->devx_conf.sample[parser->num_samples] = node;
859 parser->num_samples++;
860 /* Remove or update covered intervals. */
862 while (idx < cover->num) {
863 if (end >= cover->end[idx]) {
867 if (end > cover->start[idx])
868 cover->start[idx] = end;
876 mlx5_flex_translate_sample(struct mlx5_hca_flex_attr *attr,
877 const struct rte_flow_item_flex_conf *conf,
878 struct mlx5_flex_parser_devx *parser,
879 struct mlx5_flex_item *item,
880 struct rte_flow_error *error)
882 struct mlx5_flex_field_cover cover;
886 switch (conf->tunnel) {
887 case FLEX_TUNNEL_MODE_SINGLE:
889 case FLEX_TUNNEL_MODE_OUTER:
891 case FLEX_TUNNEL_MODE_INNER:
893 case FLEX_TUNNEL_MODE_MULTI:
895 case FLEX_TUNNEL_MODE_TUNNEL:
898 return rte_flow_error_set
899 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
900 "unrecognized tunnel mode");
902 item->tunnel_mode = conf->tunnel;
903 if (conf->nb_samples > MLX5_FLEX_ITEM_MAPPING_NUM)
904 return rte_flow_error_set
905 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
906 "sample field number exceeds limit");
908 * The application can specify fields smaller or bigger than 32 bits
909 * covered with single sample register and it can specify field
910 * offsets in any order.
912 * Gather all similar fields together, build array of bit intervals
913 * in asсending order and try to cover with the smallest set of sample
916 memset(&cover, 0, sizeof(cover));
917 for (i = 0; i < conf->nb_samples; i++) {
918 struct rte_flow_item_flex_field *fl = conf->sample_data + i;
920 /* Check whether field was covered in the previous iteration. */
921 if (cover.mapped[i / CHAR_BIT] & (1u << (i % CHAR_BIT)))
923 if (fl->field_mode == FIELD_MODE_DUMMY)
925 /* Build an interval array for the field and similar ones */
927 /* Add the first field to array unconditionally. */
928 ret = mlx5_flex_cover_sample(&cover, fl, NULL, attr, error);
931 MLX5_ASSERT(ret > 0);
932 cover.mapped[i / CHAR_BIT] |= 1u << (i % CHAR_BIT);
933 for (j = i + 1; j < conf->nb_samples; j++) {
934 struct rte_flow_item_flex_field *ft;
936 /* Add field to array if its type matches. */
937 ft = conf->sample_data + j;
938 ret = mlx5_flex_cover_sample(&cover, ft, fl,
944 cover.mapped[j / CHAR_BIT] |= 1u << (j % CHAR_BIT);
946 /* Allocate sample registers to cover array of intervals. */
947 ret = mlx5_flex_alloc_sample(&cover, parser, item,
952 /* Build the item pattern translating data on flow creation. */
954 memset(&item->map, 0, sizeof(item->map));
955 for (i = 0; i < conf->nb_samples; i++) {
956 struct rte_flow_item_flex_field *fl = conf->sample_data + i;
958 ret = mlx5_flex_map_sample(fl, parser, item, error);
964 if (conf->tunnel == FLEX_TUNNEL_MODE_MULTI) {
966 * In FLEX_TUNNEL_MODE_MULTI tunnel mode PMD creates 2 sets
967 * of samples. The first set is for outer and the second set
968 * for inner flex flow item. Outer and inner samples differ
969 * only in tunnel_mode.
971 if (parser->num_samples > MLX5_GRAPH_NODE_SAMPLE_NUM / 2)
972 return rte_flow_error_set
973 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
974 "no sample registers for inner");
975 rte_memcpy(parser->devx_conf.sample + parser->num_samples,
976 parser->devx_conf.sample,
977 parser->num_samples *
978 sizeof(parser->devx_conf.sample[0]));
979 for (i = 0; i < parser->num_samples; i++) {
980 struct mlx5_devx_match_sample_attr *sm = i +
981 parser->devx_conf.sample + parser->num_samples;
983 sm->flow_match_sample_tunnel_mode =
984 MLX5_GRAPH_SAMPLE_TUNNEL_INNER;
986 parser->num_samples *= 2;
992 mlx5_flex_arc_type(enum rte_flow_item_type type, int in)
995 case RTE_FLOW_ITEM_TYPE_ETH:
996 return MLX5_GRAPH_ARC_NODE_MAC;
997 case RTE_FLOW_ITEM_TYPE_IPV4:
998 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV4;
999 case RTE_FLOW_ITEM_TYPE_IPV6:
1000 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV6;
1001 case RTE_FLOW_ITEM_TYPE_UDP:
1002 return MLX5_GRAPH_ARC_NODE_UDP;
1003 case RTE_FLOW_ITEM_TYPE_TCP:
1004 return MLX5_GRAPH_ARC_NODE_TCP;
1005 case RTE_FLOW_ITEM_TYPE_MPLS:
1006 return MLX5_GRAPH_ARC_NODE_MPLS;
1007 case RTE_FLOW_ITEM_TYPE_GRE:
1008 return MLX5_GRAPH_ARC_NODE_GRE;
1009 case RTE_FLOW_ITEM_TYPE_GENEVE:
1010 return MLX5_GRAPH_ARC_NODE_GENEVE;
1011 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1012 return MLX5_GRAPH_ARC_NODE_VXLAN_GPE;
1019 mlx5_flex_arc_in_eth(const struct rte_flow_item *item,
1020 struct rte_flow_error *error)
1022 const struct rte_flow_item_eth *spec = item->spec;
1023 const struct rte_flow_item_eth *mask = item->mask;
1024 struct rte_flow_item_eth eth = { .hdr.ether_type = RTE_BE16(0xFFFF) };
1026 if (memcmp(mask, ð, sizeof(struct rte_flow_item_eth))) {
1027 return rte_flow_error_set
1028 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1029 "invalid eth item mask");
1031 return rte_be_to_cpu_16(spec->hdr.ether_type);
1035 mlx5_flex_arc_in_udp(const struct rte_flow_item *item,
1036 struct rte_flow_error *error)
1038 const struct rte_flow_item_udp *spec = item->spec;
1039 const struct rte_flow_item_udp *mask = item->mask;
1040 struct rte_flow_item_udp udp = { .hdr.dst_port = RTE_BE16(0xFFFF) };
1042 if (memcmp(mask, &udp, sizeof(struct rte_flow_item_udp))) {
1043 return rte_flow_error_set
1044 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1045 "invalid eth item mask");
1047 return rte_be_to_cpu_16(spec->hdr.dst_port);
1051 mlx5_flex_translate_arc_in(struct mlx5_hca_flex_attr *attr,
1052 const struct rte_flow_item_flex_conf *conf,
1053 struct mlx5_flex_parser_devx *devx,
1054 struct mlx5_flex_item *item,
1055 struct rte_flow_error *error)
1057 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
1061 if (conf->nb_inputs > attr->max_num_arc_in)
1062 return rte_flow_error_set
1063 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1064 "too many input links");
1065 for (i = 0; i < conf->nb_inputs; i++) {
1066 struct mlx5_devx_graph_arc_attr *arc = node->in + i;
1067 struct rte_flow_item_flex_link *link = conf->input_link + i;
1068 const struct rte_flow_item *rte_item = &link->item;
1072 if (!rte_item->spec || !rte_item->mask || rte_item->last)
1073 return rte_flow_error_set
1074 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1075 "invalid flex item IN arc format");
1076 arc_type = mlx5_flex_arc_type(rte_item->type, true);
1077 if (arc_type < 0 || !(attr->node_in & RTE_BIT32(arc_type)))
1078 return rte_flow_error_set
1079 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1080 "unsupported flex item IN arc type");
1081 arc->arc_parse_graph_node = arc_type;
1082 arc->start_inner_tunnel = 0;
1084 * Configure arc IN condition value. The value location depends
1085 * on protocol. Current FW version supports IP & UDP for IN
1086 * arcs only, and locations for these protocols are defined.
1087 * Add more protocols when available.
1089 switch (rte_item->type) {
1090 case RTE_FLOW_ITEM_TYPE_ETH:
1091 ret = mlx5_flex_arc_in_eth(rte_item, error);
1093 case RTE_FLOW_ITEM_TYPE_UDP:
1094 ret = mlx5_flex_arc_in_udp(rte_item, error);
1098 return rte_flow_error_set
1099 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1100 "unsupported flex item IN arc type");
1104 arc->compare_condition_value = (uint16_t)ret;
1110 mlx5_flex_translate_arc_out(struct mlx5_hca_flex_attr *attr,
1111 const struct rte_flow_item_flex_conf *conf,
1112 struct mlx5_flex_parser_devx *devx,
1113 struct mlx5_flex_item *item,
1114 struct rte_flow_error *error)
1116 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
1117 bool is_tunnel = conf->tunnel == FLEX_TUNNEL_MODE_TUNNEL;
1121 if (conf->nb_outputs > attr->max_num_arc_out)
1122 return rte_flow_error_set
1123 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1124 "too many output links");
1125 for (i = 0; i < conf->nb_outputs; i++) {
1126 struct mlx5_devx_graph_arc_attr *arc = node->out + i;
1127 struct rte_flow_item_flex_link *link = conf->output_link + i;
1128 const struct rte_flow_item *rte_item = &link->item;
1131 if (rte_item->spec || rte_item->mask || rte_item->last)
1132 return rte_flow_error_set
1133 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1134 "flex node: invalid OUT arc format");
1135 arc_type = mlx5_flex_arc_type(rte_item->type, false);
1136 if (arc_type < 0 || !(attr->node_out & RTE_BIT32(arc_type)))
1137 return rte_flow_error_set
1138 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1139 "unsupported flex item OUT arc type");
1140 arc->arc_parse_graph_node = arc_type;
1141 arc->start_inner_tunnel = !!is_tunnel;
1142 arc->compare_condition_value = link->next;
1147 /* Translate RTE flex item API configuration into flaex parser settings. */
1149 mlx5_flex_translate_conf(struct rte_eth_dev *dev,
1150 const struct rte_flow_item_flex_conf *conf,
1151 struct mlx5_flex_parser_devx *devx,
1152 struct mlx5_flex_item *item,
1153 struct rte_flow_error *error)
1155 struct mlx5_priv *priv = dev->data->dev_private;
1156 struct mlx5_hca_flex_attr *attr = &priv->config.hca_attr.flex;
1159 ret = mlx5_flex_translate_length(attr, conf, devx, error);
1162 ret = mlx5_flex_translate_next(attr, conf, devx, error);
1165 ret = mlx5_flex_translate_sample(attr, conf, devx, item, error);
1168 ret = mlx5_flex_translate_arc_in(attr, conf, devx, item, error);
1171 ret = mlx5_flex_translate_arc_out(attr, conf, devx, item, error);
1178 * Create the flex item with specified configuration over the Ethernet device.
1181 * Ethernet device to create flex item on.
1183 * Flex item configuration.
1185 * Perform verbose error reporting if not NULL. PMDs initialize this
1186 * structure in case of error only.
1189 * Non-NULL opaque pointer on success, NULL otherwise and rte_errno is set.
1191 struct rte_flow_item_flex_handle *
1192 flow_dv_item_create(struct rte_eth_dev *dev,
1193 const struct rte_flow_item_flex_conf *conf,
1194 struct rte_flow_error *error)
1196 struct mlx5_priv *priv = dev->data->dev_private;
1197 struct mlx5_flex_parser_devx devx_config = { .devx_obj = NULL };
1198 struct mlx5_flex_item *flex;
1199 struct mlx5_list_entry *ent;
1201 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
1202 flex = mlx5_flex_alloc(priv);
1204 rte_flow_error_set(error, ENOMEM,
1205 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1206 "too many flex items created on the port");
1209 if (mlx5_flex_translate_conf(dev, conf, &devx_config, flex, error))
1211 ent = mlx5_list_register(priv->sh->flex_parsers_dv, &devx_config);
1213 rte_flow_error_set(error, ENOMEM,
1214 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1215 "flex item creation failure");
1218 flex->devx_fp = container_of(ent, struct mlx5_flex_parser_devx, entry);
1219 /* Mark initialized flex item valid. */
1220 __atomic_add_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE);
1221 return (struct rte_flow_item_flex_handle *)flex;
1224 mlx5_flex_free(priv, flex);
1229 * Release the flex item on the specified Ethernet device.
1232 * Ethernet device to destroy flex item on.
1234 * Handle of the item existing on the specified device.
1236 * Perform verbose error reporting if not NULL. PMDs initialize this
1237 * structure in case of error only.
1240 * 0 on success, a negative errno value otherwise and rte_errno is set.
1243 flow_dv_item_release(struct rte_eth_dev *dev,
1244 const struct rte_flow_item_flex_handle *handle,
1245 struct rte_flow_error *error)
1247 struct mlx5_priv *priv = dev->data->dev_private;
1248 struct mlx5_flex_item *flex =
1249 (struct mlx5_flex_item *)(uintptr_t)handle;
1250 uint32_t old_refcnt = 1;
1253 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
1254 rte_spinlock_lock(&priv->flex_item_sl);
1255 if (mlx5_flex_index(priv, flex) < 0) {
1256 rte_spinlock_unlock(&priv->flex_item_sl);
1257 return rte_flow_error_set(error, EINVAL,
1258 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1259 "invalid flex item handle value");
1261 if (!__atomic_compare_exchange_n(&flex->refcnt, &old_refcnt, 0, 0,
1262 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
1263 rte_spinlock_unlock(&priv->flex_item_sl);
1264 return rte_flow_error_set(error, EBUSY,
1265 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1266 "flex item has flow references");
1268 /* Flex item is marked as invalid, we can leave locked section. */
1269 rte_spinlock_unlock(&priv->flex_item_sl);
1270 MLX5_ASSERT(flex->devx_fp);
1271 rc = mlx5_list_unregister(priv->sh->flex_parsers_dv,
1272 &flex->devx_fp->entry);
1273 flex->devx_fp = NULL;
1274 mlx5_flex_free(priv, flex);
1276 return rte_flow_error_set(error, EBUSY,
1277 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1278 "flex item release failure");
1282 /* DevX flex parser list callbacks. */
1283 struct mlx5_list_entry *
1284 mlx5_flex_parser_create_cb(void *list_ctx, void *ctx)
1286 struct mlx5_dev_ctx_shared *sh = list_ctx;
1287 struct mlx5_flex_parser_devx *fp, *conf = ctx;
1290 fp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flex_parser_devx),
1294 /* Copy the requested configurations. */
1295 fp->num_samples = conf->num_samples;
1296 memcpy(&fp->devx_conf, &conf->devx_conf, sizeof(fp->devx_conf));
1297 /* Create DevX flex parser. */
1298 fp->devx_obj = mlx5_devx_cmd_create_flex_parser(sh->cdev->ctx,
1302 /* Query the firmware assigned sample ids. */
1303 ret = mlx5_devx_cmd_query_parse_samples(fp->devx_obj,
1308 DRV_LOG(DEBUG, "DEVx flex parser %p created, samples num: %u",
1309 (const void *)fp, fp->num_samples);
1313 mlx5_devx_cmd_destroy((void *)(uintptr_t)fp->devx_obj);
1320 mlx5_flex_parser_match_cb(void *list_ctx,
1321 struct mlx5_list_entry *iter, void *ctx)
1323 struct mlx5_flex_parser_devx *fp =
1324 container_of(iter, struct mlx5_flex_parser_devx, entry);
1325 struct mlx5_flex_parser_devx *org =
1326 container_of(ctx, struct mlx5_flex_parser_devx, entry);
1328 RTE_SET_USED(list_ctx);
1329 return !iter || !ctx || memcmp(&fp->devx_conf,
1331 sizeof(fp->devx_conf));
1335 mlx5_flex_parser_remove_cb(void *list_ctx, struct mlx5_list_entry *entry)
1337 struct mlx5_flex_parser_devx *fp =
1338 container_of(entry, struct mlx5_flex_parser_devx, entry);
1340 RTE_SET_USED(list_ctx);
1341 MLX5_ASSERT(fp->devx_obj);
1342 claim_zero(mlx5_devx_cmd_destroy(fp->devx_obj));
1343 DRV_LOG(DEBUG, "DEVx flex parser %p destroyed", (const void *)fp);
1347 struct mlx5_list_entry *
1348 mlx5_flex_parser_clone_cb(void *list_ctx,
1349 struct mlx5_list_entry *entry, void *ctx)
1351 struct mlx5_flex_parser_devx *fp;
1353 RTE_SET_USED(list_ctx);
1354 RTE_SET_USED(entry);
1355 fp = mlx5_malloc(0, sizeof(struct mlx5_flex_parser_devx),
1359 memcpy(fp, ctx, sizeof(struct mlx5_flex_parser_devx));
1364 mlx5_flex_parser_clone_free_cb(void *list_ctx, struct mlx5_list_entry *entry)
1366 struct mlx5_flex_parser_devx *fp =
1367 container_of(entry, struct mlx5_flex_parser_devx, entry);
1368 RTE_SET_USED(list_ctx);