1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
4 #include <rte_malloc.h>
5 #include <mlx5_devx_cmds.h>
6 #include <mlx5_malloc.h>
10 static_assert(sizeof(uint32_t) * CHAR_BIT >= MLX5_PORT_FLEX_ITEM_NUM,
11 "Flex item maximal number exceeds uint32_t bit width");
14 * Routine called once on port initialization to init flex item
15 * related infrastructure initialization
18 * Ethernet device to perform flex item initialization
21 * 0 on success, a negative errno value otherwise and rte_errno is set.
24 mlx5_flex_item_port_init(struct rte_eth_dev *dev)
26 struct mlx5_priv *priv = dev->data->dev_private;
28 rte_spinlock_init(&priv->flex_item_sl);
29 MLX5_ASSERT(!priv->flex_item_map);
34 * Routine called once on port close to perform flex item
35 * related infrastructure cleanup.
38 * Ethernet device to perform cleanup
41 mlx5_flex_item_port_cleanup(struct rte_eth_dev *dev)
43 struct mlx5_priv *priv = dev->data->dev_private;
46 for (i = 0; i < MLX5_PORT_FLEX_ITEM_NUM && priv->flex_item_map ; i++) {
47 if (priv->flex_item_map & (1 << i)) {
48 struct mlx5_flex_item *flex = &priv->flex_item[i];
50 claim_zero(mlx5_list_unregister
51 (priv->sh->flex_parsers_dv,
52 &flex->devx_fp->entry));
55 priv->flex_item_map &= ~(1 << i);
61 mlx5_flex_index(struct mlx5_priv *priv, struct mlx5_flex_item *item)
63 uintptr_t start = (uintptr_t)&priv->flex_item[0];
64 uintptr_t entry = (uintptr_t)item;
65 uintptr_t idx = (entry - start) / sizeof(struct mlx5_flex_item);
68 idx >= MLX5_PORT_FLEX_ITEM_NUM ||
69 (entry - start) % sizeof(struct mlx5_flex_item) ||
70 !(priv->flex_item_map & (1u << idx)))
75 static struct mlx5_flex_item *
76 mlx5_flex_alloc(struct mlx5_priv *priv)
78 struct mlx5_flex_item *item = NULL;
80 rte_spinlock_lock(&priv->flex_item_sl);
81 if (~priv->flex_item_map) {
82 uint32_t idx = rte_bsf32(~priv->flex_item_map);
84 if (idx < MLX5_PORT_FLEX_ITEM_NUM) {
85 item = &priv->flex_item[idx];
86 MLX5_ASSERT(!item->refcnt);
87 MLX5_ASSERT(!item->devx_fp);
89 __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE);
90 priv->flex_item_map |= 1u << idx;
93 rte_spinlock_unlock(&priv->flex_item_sl);
98 mlx5_flex_free(struct mlx5_priv *priv, struct mlx5_flex_item *item)
100 int idx = mlx5_flex_index(priv, item);
102 MLX5_ASSERT(idx >= 0 &&
103 idx < MLX5_PORT_FLEX_ITEM_NUM &&
104 (priv->flex_item_map & (1u << idx)));
106 rte_spinlock_lock(&priv->flex_item_sl);
107 MLX5_ASSERT(!item->refcnt);
108 MLX5_ASSERT(!item->devx_fp);
109 item->devx_fp = NULL;
110 __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE);
111 priv->flex_item_map &= ~(1u << idx);
112 rte_spinlock_unlock(&priv->flex_item_sl);
117 * Calculate largest mask value for a given shift.
120 * ------- ---------------
129 mlx5_flex_hdr_len_mask(uint8_t shift,
130 const struct mlx5_hca_flex_attr *attr)
133 int diff = shift - MLX5_PARSE_GRAPH_NODE_HDR_LEN_SHIFT_DWORD;
135 base_mask = mlx5_hca_parse_graph_node_base_hdr_len_mask(attr);
136 return diff == 0 ? base_mask :
137 diff < 0 ? (base_mask << -diff) & base_mask : base_mask >> diff;
141 mlx5_flex_translate_length(struct mlx5_hca_flex_attr *attr,
142 const struct rte_flow_item_flex_conf *conf,
143 struct mlx5_flex_parser_devx *devx,
144 struct rte_flow_error *error)
146 const struct rte_flow_item_flex_field *field = &conf->next_header;
147 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
148 uint32_t len_width, mask;
150 if (field->field_base % CHAR_BIT)
151 return rte_flow_error_set
152 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
153 "not byte aligned header length field");
154 switch (field->field_mode) {
155 case FIELD_MODE_DUMMY:
156 return rte_flow_error_set
157 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
158 "invalid header length field mode (DUMMY)");
159 case FIELD_MODE_FIXED:
160 if (!(attr->header_length_mode &
161 RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIXED)))
162 return rte_flow_error_set
163 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
164 "unsupported header length field mode (FIXED)");
165 if (attr->header_length_mask_width < field->field_size)
166 return rte_flow_error_set
167 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
168 "header length field width exceeds limit");
169 if (field->offset_shift < 0 ||
170 field->offset_shift > attr->header_length_mask_width)
171 return rte_flow_error_set
172 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
173 "invalid header length field shift (FIXED");
174 if (field->field_base < 0)
175 return rte_flow_error_set
176 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
177 "negative header length field base (FIXED)");
178 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED;
180 case FIELD_MODE_OFFSET:
181 if (!(attr->header_length_mode &
182 RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIELD)))
183 return rte_flow_error_set
184 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
185 "unsupported header length field mode (OFFSET)");
186 node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIELD;
187 if (field->offset_mask == 0 ||
188 !rte_is_power_of_2(field->offset_mask + 1))
189 return rte_flow_error_set
190 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
191 "invalid length field offset mask (OFFSET)");
192 len_width = rte_fls_u32(field->offset_mask);
193 if (len_width > attr->header_length_mask_width)
194 return rte_flow_error_set
195 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
196 "length field offset mask too wide (OFFSET)");
197 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr);
198 if (mask < field->offset_mask)
199 return rte_flow_error_set
200 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
201 "length field shift too big (OFFSET)");
202 node->header_length_field_mask = RTE_MIN(mask,
205 case FIELD_MODE_BITMASK:
206 if (!(attr->header_length_mode &
207 RTE_BIT32(MLX5_GRAPH_NODE_LEN_BITMASK)))
208 return rte_flow_error_set
209 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
210 "unsupported header length field mode (BITMASK)");
211 if (attr->header_length_mask_width < field->field_size)
212 return rte_flow_error_set
213 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
214 "header length field width exceeds limit");
215 node->header_length_mode = MLX5_GRAPH_NODE_LEN_BITMASK;
216 mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr);
217 if (mask < field->offset_mask)
218 return rte_flow_error_set
219 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
220 "length field shift too big (BITMASK)");
221 node->header_length_field_mask = RTE_MIN(mask,
225 return rte_flow_error_set
226 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
227 "unknown header length field mode");
229 if (field->field_base / CHAR_BIT >= 0 &&
230 field->field_base / CHAR_BIT > attr->max_base_header_length)
231 return rte_flow_error_set
232 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
233 "header length field base exceeds limit");
234 node->header_length_base_value = field->field_base / CHAR_BIT;
235 if (field->field_mode == FIELD_MODE_OFFSET ||
236 field->field_mode == FIELD_MODE_BITMASK) {
237 if (field->offset_shift > 15 || field->offset_shift < 0)
238 return rte_flow_error_set
239 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
240 "header length field shift exceeeds limit");
241 node->header_length_field_shift = field->offset_shift;
242 node->header_length_field_offset = field->offset_base;
248 mlx5_flex_translate_next(struct mlx5_hca_flex_attr *attr,
249 const struct rte_flow_item_flex_conf *conf,
250 struct mlx5_flex_parser_devx *devx,
251 struct rte_flow_error *error)
253 const struct rte_flow_item_flex_field *field = &conf->next_protocol;
254 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
256 switch (field->field_mode) {
257 case FIELD_MODE_DUMMY:
258 if (conf->nb_outputs)
259 return rte_flow_error_set
260 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
261 "next protocol field is required (DUMMY)");
263 case FIELD_MODE_FIXED:
265 case FIELD_MODE_OFFSET:
266 return rte_flow_error_set
267 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
268 "unsupported next protocol field mode (OFFSET)");
270 case FIELD_MODE_BITMASK:
271 return rte_flow_error_set
272 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
273 "unsupported next protocol field mode (BITMASK)");
275 return rte_flow_error_set
277 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
278 "unknown next protocol field mode");
280 MLX5_ASSERT(field->field_mode == FIELD_MODE_FIXED);
281 if (!conf->nb_outputs)
282 return rte_flow_error_set
283 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
284 "out link(s) is required if next field present");
285 if (attr->max_next_header_offset < field->field_base)
286 return rte_flow_error_set
287 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
288 "next protocol field base exceeds limit");
289 if (field->offset_shift)
290 return rte_flow_error_set
291 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
292 "unsupported next protocol field shift");
293 node->next_header_field_offset = field->field_base;
294 node->next_header_field_size = field->field_size;
298 /* Helper structure to handle field bit intervals. */
299 struct mlx5_flex_field_cover {
301 int32_t start[MLX5_FLEX_ITEM_MAPPING_NUM];
302 int32_t end[MLX5_FLEX_ITEM_MAPPING_NUM];
303 uint8_t mapped[MLX5_FLEX_ITEM_MAPPING_NUM / CHAR_BIT + 1];
307 mlx5_flex_insert_field(struct mlx5_flex_field_cover *cover,
308 uint16_t num, int32_t start, int32_t end)
310 MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM);
311 MLX5_ASSERT(num <= cover->num);
312 if (num < cover->num) {
313 memmove(&cover->start[num + 1], &cover->start[num],
314 (cover->num - num) * sizeof(int32_t));
315 memmove(&cover->end[num + 1], &cover->end[num],
316 (cover->num - num) * sizeof(int32_t));
318 cover->start[num] = start;
319 cover->end[num] = end;
324 mlx5_flex_merge_field(struct mlx5_flex_field_cover *cover, uint16_t num)
329 MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM);
330 MLX5_ASSERT(num < (cover->num - 1));
331 end = cover->end[num];
332 for (i = num + 1; i < cover->num; i++) {
333 if (end < cover->start[i])
336 if (end <= cover->end[i]) {
337 cover->end[num] = cover->end[i];
342 MLX5_ASSERT(del < (cover->num - 1u - num));
344 MLX5_ASSERT(cover->num > num);
345 if ((cover->num - num) > 1) {
346 memmove(&cover->start[num + 1],
347 &cover->start[num + 1 + del],
348 (cover->num - num - 1) * sizeof(int32_t));
349 memmove(&cover->end[num + 1],
350 &cover->end[num + 1 + del],
351 (cover->num - num - 1) * sizeof(int32_t));
357 * Validate the sample field and update interval array
358 * if parameters match with the 'match" field.
361 * == 0 - no match, interval array not updated
362 * > 0 - match, interval array updated
365 mlx5_flex_cover_sample(struct mlx5_flex_field_cover *cover,
366 struct rte_flow_item_flex_field *field,
367 struct rte_flow_item_flex_field *match,
368 struct mlx5_hca_flex_attr *attr,
369 struct rte_flow_error *error)
374 switch (field->field_mode) {
375 case FIELD_MODE_DUMMY:
377 case FIELD_MODE_FIXED:
378 if (!(attr->sample_offset_mode &
379 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIXED)))
380 return rte_flow_error_set
382 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
383 "unsupported sample field mode (FIXED)");
384 if (field->offset_shift)
385 return rte_flow_error_set
387 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
388 "invalid sample field shift (FIXED");
389 if (field->field_base < 0)
390 return rte_flow_error_set
391 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
392 "invalid sample field base (FIXED)");
393 if (field->field_base / CHAR_BIT > attr->max_sample_base_offset)
394 return rte_flow_error_set
395 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
396 "sample field base exceeds limit (FIXED)");
398 case FIELD_MODE_OFFSET:
399 if (!(attr->sample_offset_mode &
400 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIELD)))
401 return rte_flow_error_set
403 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
404 "unsupported sample field mode (OFFSET)");
405 if (field->field_base / CHAR_BIT >= 0 &&
406 field->field_base / CHAR_BIT > attr->max_sample_base_offset)
407 return rte_flow_error_set
408 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
409 "sample field base exceeds limit");
411 case FIELD_MODE_BITMASK:
412 if (!(attr->sample_offset_mode &
413 RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_BITMASK)))
414 return rte_flow_error_set
416 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
417 "unsupported sample field mode (BITMASK)");
418 if (field->field_base / CHAR_BIT >= 0 &&
419 field->field_base / CHAR_BIT > attr->max_sample_base_offset)
420 return rte_flow_error_set
421 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
422 "sample field base exceeds limit");
425 return rte_flow_error_set
427 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
428 "unknown data sample field mode");
431 if (!field->field_size)
432 return rte_flow_error_set
434 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
435 "zero sample field width");
437 DRV_LOG(DEBUG, "sample field id hint ignored");
439 if (field->field_mode != match->field_mode ||
440 field->offset_base | match->offset_base ||
441 field->offset_mask | match->offset_mask ||
442 field->offset_shift | match->offset_shift)
445 start = field->field_base;
446 end = start + field->field_size;
447 /* Add the new or similar field to interval array. */
449 cover->start[cover->num] = start;
450 cover->end[cover->num] = end;
454 for (i = 0; i < cover->num; i++) {
455 if (start > cover->end[i]) {
456 if (i >= (cover->num - 1u)) {
457 mlx5_flex_insert_field(cover, cover->num,
463 if (end < cover->start[i]) {
464 mlx5_flex_insert_field(cover, i, start, end);
467 if (start < cover->start[i])
468 cover->start[i] = start;
469 if (end > cover->end[i]) {
471 if (i < (cover->num - 1u))
472 mlx5_flex_merge_field(cover, i);
480 mlx5_flex_config_sample(struct mlx5_devx_match_sample_attr *na,
481 struct rte_flow_item_flex_field *field,
482 enum rte_flow_item_flex_tunnel_mode tunnel_mode)
484 memset(na, 0, sizeof(struct mlx5_devx_match_sample_attr));
485 na->flow_match_sample_en = 1;
486 switch (field->field_mode) {
487 case FIELD_MODE_FIXED:
488 na->flow_match_sample_offset_mode =
489 MLX5_GRAPH_SAMPLE_OFFSET_FIXED;
491 case FIELD_MODE_OFFSET:
492 na->flow_match_sample_offset_mode =
493 MLX5_GRAPH_SAMPLE_OFFSET_FIELD;
494 na->flow_match_sample_field_offset = field->offset_base;
495 na->flow_match_sample_field_offset_mask = field->offset_mask;
496 na->flow_match_sample_field_offset_shift = field->offset_shift;
498 case FIELD_MODE_BITMASK:
499 na->flow_match_sample_offset_mode =
500 MLX5_GRAPH_SAMPLE_OFFSET_BITMASK;
501 na->flow_match_sample_field_offset = field->offset_base;
502 na->flow_match_sample_field_offset_mask = field->offset_mask;
503 na->flow_match_sample_field_offset_shift = field->offset_shift;
509 switch (tunnel_mode) {
510 case FLEX_TUNNEL_MODE_SINGLE:
512 case FLEX_TUNNEL_MODE_TUNNEL:
513 na->flow_match_sample_tunnel_mode =
514 MLX5_GRAPH_SAMPLE_TUNNEL_FIRST;
516 case FLEX_TUNNEL_MODE_MULTI:
518 case FLEX_TUNNEL_MODE_OUTER:
519 na->flow_match_sample_tunnel_mode =
520 MLX5_GRAPH_SAMPLE_TUNNEL_OUTER;
522 case FLEX_TUNNEL_MODE_INNER:
523 na->flow_match_sample_tunnel_mode =
524 MLX5_GRAPH_SAMPLE_TUNNEL_INNER;
532 /* Map specified field to set/subset of allocated sample registers. */
534 mlx5_flex_map_sample(struct rte_flow_item_flex_field *field,
535 struct mlx5_flex_parser_devx *parser,
536 struct mlx5_flex_item *item,
537 struct rte_flow_error *error)
539 struct mlx5_devx_match_sample_attr node;
540 int32_t start = field->field_base;
541 int32_t end = start + field->field_size;
542 struct mlx5_flex_pattern_field *trans;
543 uint32_t i, done_bits = 0;
545 if (field->field_mode == FIELD_MODE_DUMMY) {
546 done_bits = field->field_size;
548 uint32_t part = RTE_MIN(done_bits,
549 sizeof(uint32_t) * CHAR_BIT);
550 if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM)
551 return rte_flow_error_set
553 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
554 "too many flex item pattern translations");
555 trans = &item->map[item->mapnum];
556 trans->reg_id = MLX5_INVALID_SAMPLE_REG_ID;
564 mlx5_flex_config_sample(&node, field, item->tunnel_mode);
565 for (i = 0; i < parser->num_samples; i++) {
566 struct mlx5_devx_match_sample_attr *sample =
567 &parser->devx_conf.sample[i];
568 int32_t reg_start, reg_end;
569 int32_t cov_start, cov_end;
571 MLX5_ASSERT(sample->flow_match_sample_en);
572 if (!sample->flow_match_sample_en)
574 node.flow_match_sample_field_base_offset =
575 sample->flow_match_sample_field_base_offset;
576 if (memcmp(&node, sample, sizeof(node)))
578 reg_start = (int8_t)sample->flow_match_sample_field_base_offset;
579 reg_start *= CHAR_BIT;
580 reg_end = reg_start + 32;
581 if (end <= reg_start || start >= reg_end)
583 cov_start = RTE_MAX(reg_start, start);
584 cov_end = RTE_MIN(reg_end, end);
585 MLX5_ASSERT(cov_end > cov_start);
586 done_bits += cov_end - cov_start;
587 if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM)
588 return rte_flow_error_set
589 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
590 "too many flex item pattern translations");
591 trans = &item->map[item->mapnum];
594 trans->shift = cov_start - reg_start;
595 trans->width = cov_end - cov_start;
597 if (done_bits != field->field_size) {
599 return rte_flow_error_set
600 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
601 "failed to map field to sample register");
606 /* Allocate sample registers for the specified field type and interval array. */
608 mlx5_flex_alloc_sample(struct mlx5_flex_field_cover *cover,
609 struct mlx5_flex_parser_devx *parser,
610 struct mlx5_flex_item *item,
611 struct rte_flow_item_flex_field *field,
612 struct mlx5_hca_flex_attr *attr,
613 struct rte_flow_error *error)
615 struct mlx5_devx_match_sample_attr node;
618 mlx5_flex_config_sample(&node, field, item->tunnel_mode);
619 while (idx < cover->num) {
623 * Sample base offsets are in bytes, should be aligned
624 * to 32-bit as required by firmware for samples.
626 start = RTE_ALIGN_FLOOR(cover->start[idx],
627 sizeof(uint32_t) * CHAR_BIT);
628 node.flow_match_sample_field_base_offset =
629 (start / CHAR_BIT) & 0xFF;
630 /* Allocate sample register. */
631 if (parser->num_samples >= MLX5_GRAPH_NODE_SAMPLE_NUM ||
632 parser->num_samples >= attr->max_num_sample ||
633 parser->num_samples >= attr->max_num_prog_sample)
634 return rte_flow_error_set
636 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
637 "no sample registers to handle all flex item fields");
638 parser->devx_conf.sample[parser->num_samples] = node;
639 parser->num_samples++;
640 /* Remove or update covered intervals. */
642 while (idx < cover->num) {
643 if (end >= cover->end[idx]) {
647 if (end > cover->start[idx])
648 cover->start[idx] = end;
656 mlx5_flex_translate_sample(struct mlx5_hca_flex_attr *attr,
657 const struct rte_flow_item_flex_conf *conf,
658 struct mlx5_flex_parser_devx *parser,
659 struct mlx5_flex_item *item,
660 struct rte_flow_error *error)
662 struct mlx5_flex_field_cover cover;
666 switch (conf->tunnel) {
667 case FLEX_TUNNEL_MODE_SINGLE:
669 case FLEX_TUNNEL_MODE_OUTER:
671 case FLEX_TUNNEL_MODE_INNER:
673 case FLEX_TUNNEL_MODE_MULTI:
675 case FLEX_TUNNEL_MODE_TUNNEL:
678 return rte_flow_error_set
679 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
680 "unrecognized tunnel mode");
682 item->tunnel_mode = conf->tunnel;
683 if (conf->nb_samples > MLX5_FLEX_ITEM_MAPPING_NUM)
684 return rte_flow_error_set
685 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
686 "sample field number exceeds limit");
688 * The application can specify fields smaller or bigger than 32 bits
689 * covered with single sample register and it can specify field
690 * offsets in any order.
692 * Gather all similar fields together, build array of bit intervals
693 * in asсending order and try to cover with the smallest set of sample
696 memset(&cover, 0, sizeof(cover));
697 for (i = 0; i < conf->nb_samples; i++) {
698 struct rte_flow_item_flex_field *fl = conf->sample_data + i;
700 /* Check whether field was covered in the previous iteration. */
701 if (cover.mapped[i / CHAR_BIT] & (1u << (i % CHAR_BIT)))
703 if (fl->field_mode == FIELD_MODE_DUMMY)
705 /* Build an interval array for the field and similar ones */
707 /* Add the first field to array unconditionally. */
708 ret = mlx5_flex_cover_sample(&cover, fl, NULL, attr, error);
711 MLX5_ASSERT(ret > 0);
712 cover.mapped[i / CHAR_BIT] |= 1u << (i % CHAR_BIT);
713 for (j = i + 1; j < conf->nb_samples; j++) {
714 struct rte_flow_item_flex_field *ft;
716 /* Add field to array if its type matches. */
717 ft = conf->sample_data + j;
718 ret = mlx5_flex_cover_sample(&cover, ft, fl,
724 cover.mapped[j / CHAR_BIT] |= 1u << (j % CHAR_BIT);
726 /* Allocate sample registers to cover array of intervals. */
727 ret = mlx5_flex_alloc_sample(&cover, parser, item,
732 /* Build the item pattern translating data on flow creation. */
734 memset(&item->map, 0, sizeof(item->map));
735 for (i = 0; i < conf->nb_samples; i++) {
736 struct rte_flow_item_flex_field *fl = conf->sample_data + i;
738 ret = mlx5_flex_map_sample(fl, parser, item, error);
744 if (conf->tunnel == FLEX_TUNNEL_MODE_MULTI) {
746 * In FLEX_TUNNEL_MODE_MULTI tunnel mode PMD creates 2 sets
747 * of samples. The first set is for outer and the second set
748 * for inner flex flow item. Outer and inner samples differ
749 * only in tunnel_mode.
751 if (parser->num_samples > MLX5_GRAPH_NODE_SAMPLE_NUM / 2)
752 return rte_flow_error_set
753 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
754 "no sample registers for inner");
755 rte_memcpy(parser->devx_conf.sample + parser->num_samples,
756 parser->devx_conf.sample,
757 parser->num_samples *
758 sizeof(parser->devx_conf.sample[0]));
759 for (i = 0; i < parser->num_samples; i++) {
760 struct mlx5_devx_match_sample_attr *sm = i +
761 parser->devx_conf.sample + parser->num_samples;
763 sm->flow_match_sample_tunnel_mode =
764 MLX5_GRAPH_SAMPLE_TUNNEL_INNER;
766 parser->num_samples *= 2;
772 mlx5_flex_arc_type(enum rte_flow_item_type type, int in)
775 case RTE_FLOW_ITEM_TYPE_ETH:
776 return MLX5_GRAPH_ARC_NODE_MAC;
777 case RTE_FLOW_ITEM_TYPE_IPV4:
778 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV4;
779 case RTE_FLOW_ITEM_TYPE_IPV6:
780 return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV6;
781 case RTE_FLOW_ITEM_TYPE_UDP:
782 return MLX5_GRAPH_ARC_NODE_UDP;
783 case RTE_FLOW_ITEM_TYPE_TCP:
784 return MLX5_GRAPH_ARC_NODE_TCP;
785 case RTE_FLOW_ITEM_TYPE_MPLS:
786 return MLX5_GRAPH_ARC_NODE_MPLS;
787 case RTE_FLOW_ITEM_TYPE_GRE:
788 return MLX5_GRAPH_ARC_NODE_GRE;
789 case RTE_FLOW_ITEM_TYPE_GENEVE:
790 return MLX5_GRAPH_ARC_NODE_GENEVE;
791 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
792 return MLX5_GRAPH_ARC_NODE_VXLAN_GPE;
799 mlx5_flex_arc_in_eth(const struct rte_flow_item *item,
800 struct rte_flow_error *error)
802 const struct rte_flow_item_eth *spec = item->spec;
803 const struct rte_flow_item_eth *mask = item->mask;
804 struct rte_flow_item_eth eth = { .hdr.ether_type = RTE_BE16(0xFFFF) };
806 if (memcmp(mask, ð, sizeof(struct rte_flow_item_eth))) {
807 return rte_flow_error_set
808 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
809 "invalid eth item mask");
811 return rte_be_to_cpu_16(spec->hdr.ether_type);
815 mlx5_flex_arc_in_udp(const struct rte_flow_item *item,
816 struct rte_flow_error *error)
818 const struct rte_flow_item_udp *spec = item->spec;
819 const struct rte_flow_item_udp *mask = item->mask;
820 struct rte_flow_item_udp udp = { .hdr.dst_port = RTE_BE16(0xFFFF) };
822 if (memcmp(mask, &udp, sizeof(struct rte_flow_item_udp))) {
823 return rte_flow_error_set
824 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
825 "invalid eth item mask");
827 return rte_be_to_cpu_16(spec->hdr.dst_port);
831 mlx5_flex_translate_arc_in(struct mlx5_hca_flex_attr *attr,
832 const struct rte_flow_item_flex_conf *conf,
833 struct mlx5_flex_parser_devx *devx,
834 struct mlx5_flex_item *item,
835 struct rte_flow_error *error)
837 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
841 if (conf->nb_inputs > attr->max_num_arc_in)
842 return rte_flow_error_set
843 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
844 "too many input links");
845 for (i = 0; i < conf->nb_inputs; i++) {
846 struct mlx5_devx_graph_arc_attr *arc = node->in + i;
847 struct rte_flow_item_flex_link *link = conf->input_link + i;
848 const struct rte_flow_item *rte_item = &link->item;
852 if (!rte_item->spec || !rte_item->mask || rte_item->last)
853 return rte_flow_error_set
854 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
855 "invalid flex item IN arc format");
856 arc_type = mlx5_flex_arc_type(rte_item->type, true);
857 if (arc_type < 0 || !(attr->node_in & RTE_BIT32(arc_type)))
858 return rte_flow_error_set
859 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
860 "unsupported flex item IN arc type");
861 arc->arc_parse_graph_node = arc_type;
862 arc->start_inner_tunnel = 0;
864 * Configure arc IN condition value. The value location depends
865 * on protocol. Current FW version supports IP & UDP for IN
866 * arcs only, and locations for these protocols are defined.
867 * Add more protocols when available.
869 switch (rte_item->type) {
870 case RTE_FLOW_ITEM_TYPE_ETH:
871 ret = mlx5_flex_arc_in_eth(rte_item, error);
873 case RTE_FLOW_ITEM_TYPE_UDP:
874 ret = mlx5_flex_arc_in_udp(rte_item, error);
878 return rte_flow_error_set
879 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
880 "unsupported flex item IN arc type");
884 arc->compare_condition_value = (uint16_t)ret;
890 mlx5_flex_translate_arc_out(struct mlx5_hca_flex_attr *attr,
891 const struct rte_flow_item_flex_conf *conf,
892 struct mlx5_flex_parser_devx *devx,
893 struct mlx5_flex_item *item,
894 struct rte_flow_error *error)
896 struct mlx5_devx_graph_node_attr *node = &devx->devx_conf;
897 bool is_tunnel = conf->tunnel == FLEX_TUNNEL_MODE_TUNNEL;
901 if (conf->nb_outputs > attr->max_num_arc_out)
902 return rte_flow_error_set
903 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
904 "too many output links");
905 for (i = 0; i < conf->nb_outputs; i++) {
906 struct mlx5_devx_graph_arc_attr *arc = node->out + i;
907 struct rte_flow_item_flex_link *link = conf->output_link + i;
908 const struct rte_flow_item *rte_item = &link->item;
911 if (rte_item->spec || rte_item->mask || rte_item->last)
912 return rte_flow_error_set
913 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
914 "flex node: invalid OUT arc format");
915 arc_type = mlx5_flex_arc_type(rte_item->type, false);
916 if (arc_type < 0 || !(attr->node_out & RTE_BIT32(arc_type)))
917 return rte_flow_error_set
918 (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL,
919 "unsupported flex item OUT arc type");
920 arc->arc_parse_graph_node = arc_type;
921 arc->start_inner_tunnel = !!is_tunnel;
922 arc->compare_condition_value = link->next;
927 /* Translate RTE flex item API configuration into flaex parser settings. */
929 mlx5_flex_translate_conf(struct rte_eth_dev *dev,
930 const struct rte_flow_item_flex_conf *conf,
931 struct mlx5_flex_parser_devx *devx,
932 struct mlx5_flex_item *item,
933 struct rte_flow_error *error)
935 struct mlx5_priv *priv = dev->data->dev_private;
936 struct mlx5_hca_flex_attr *attr = &priv->config.hca_attr.flex;
939 ret = mlx5_flex_translate_length(attr, conf, devx, error);
942 ret = mlx5_flex_translate_next(attr, conf, devx, error);
945 ret = mlx5_flex_translate_sample(attr, conf, devx, item, error);
948 ret = mlx5_flex_translate_arc_in(attr, conf, devx, item, error);
951 ret = mlx5_flex_translate_arc_out(attr, conf, devx, item, error);
958 * Create the flex item with specified configuration over the Ethernet device.
961 * Ethernet device to create flex item on.
963 * Flex item configuration.
965 * Perform verbose error reporting if not NULL. PMDs initialize this
966 * structure in case of error only.
969 * Non-NULL opaque pointer on success, NULL otherwise and rte_errno is set.
971 struct rte_flow_item_flex_handle *
972 flow_dv_item_create(struct rte_eth_dev *dev,
973 const struct rte_flow_item_flex_conf *conf,
974 struct rte_flow_error *error)
976 struct mlx5_priv *priv = dev->data->dev_private;
977 struct mlx5_flex_parser_devx devx_config = { .devx_obj = NULL };
978 struct mlx5_flex_item *flex;
979 struct mlx5_list_entry *ent;
981 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
982 flex = mlx5_flex_alloc(priv);
984 rte_flow_error_set(error, ENOMEM,
985 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
986 "too many flex items created on the port");
989 if (mlx5_flex_translate_conf(dev, conf, &devx_config, flex, error))
991 ent = mlx5_list_register(priv->sh->flex_parsers_dv, &devx_config);
993 rte_flow_error_set(error, ENOMEM,
994 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
995 "flex item creation failure");
998 flex->devx_fp = container_of(ent, struct mlx5_flex_parser_devx, entry);
999 /* Mark initialized flex item valid. */
1000 __atomic_add_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE);
1001 return (struct rte_flow_item_flex_handle *)flex;
1004 mlx5_flex_free(priv, flex);
1009 * Release the flex item on the specified Ethernet device.
1012 * Ethernet device to destroy flex item on.
1014 * Handle of the item existing on the specified device.
1016 * Perform verbose error reporting if not NULL. PMDs initialize this
1017 * structure in case of error only.
1020 * 0 on success, a negative errno value otherwise and rte_errno is set.
1023 flow_dv_item_release(struct rte_eth_dev *dev,
1024 const struct rte_flow_item_flex_handle *handle,
1025 struct rte_flow_error *error)
1027 struct mlx5_priv *priv = dev->data->dev_private;
1028 struct mlx5_flex_item *flex =
1029 (struct mlx5_flex_item *)(uintptr_t)handle;
1030 uint32_t old_refcnt = 1;
1033 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
1034 rte_spinlock_lock(&priv->flex_item_sl);
1035 if (mlx5_flex_index(priv, flex) < 0) {
1036 rte_spinlock_unlock(&priv->flex_item_sl);
1037 return rte_flow_error_set(error, EINVAL,
1038 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1039 "invalid flex item handle value");
1041 if (!__atomic_compare_exchange_n(&flex->refcnt, &old_refcnt, 0, 0,
1042 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
1043 rte_spinlock_unlock(&priv->flex_item_sl);
1044 return rte_flow_error_set(error, EBUSY,
1045 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1046 "flex item has flow references");
1048 /* Flex item is marked as invalid, we can leave locked section. */
1049 rte_spinlock_unlock(&priv->flex_item_sl);
1050 MLX5_ASSERT(flex->devx_fp);
1051 rc = mlx5_list_unregister(priv->sh->flex_parsers_dv,
1052 &flex->devx_fp->entry);
1053 flex->devx_fp = NULL;
1054 mlx5_flex_free(priv, flex);
1056 return rte_flow_error_set(error, EBUSY,
1057 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1058 "flex item release failure");
1062 /* DevX flex parser list callbacks. */
1063 struct mlx5_list_entry *
1064 mlx5_flex_parser_create_cb(void *list_ctx, void *ctx)
1066 struct mlx5_dev_ctx_shared *sh = list_ctx;
1067 struct mlx5_flex_parser_devx *fp, *conf = ctx;
1070 fp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flex_parser_devx),
1074 /* Copy the requested configurations. */
1075 fp->num_samples = conf->num_samples;
1076 memcpy(&fp->devx_conf, &conf->devx_conf, sizeof(fp->devx_conf));
1077 /* Create DevX flex parser. */
1078 fp->devx_obj = mlx5_devx_cmd_create_flex_parser(sh->cdev->ctx,
1082 /* Query the firmware assigned sample ids. */
1083 ret = mlx5_devx_cmd_query_parse_samples(fp->devx_obj,
1088 DRV_LOG(DEBUG, "DEVx flex parser %p created, samples num: %u",
1089 (const void *)fp, fp->num_samples);
1093 mlx5_devx_cmd_destroy((void *)(uintptr_t)fp->devx_obj);
1100 mlx5_flex_parser_match_cb(void *list_ctx,
1101 struct mlx5_list_entry *iter, void *ctx)
1103 struct mlx5_flex_parser_devx *fp =
1104 container_of(iter, struct mlx5_flex_parser_devx, entry);
1105 struct mlx5_flex_parser_devx *org =
1106 container_of(ctx, struct mlx5_flex_parser_devx, entry);
1108 RTE_SET_USED(list_ctx);
1109 return !iter || !ctx || memcmp(&fp->devx_conf,
1111 sizeof(fp->devx_conf));
1115 mlx5_flex_parser_remove_cb(void *list_ctx, struct mlx5_list_entry *entry)
1117 struct mlx5_flex_parser_devx *fp =
1118 container_of(entry, struct mlx5_flex_parser_devx, entry);
1120 RTE_SET_USED(list_ctx);
1121 MLX5_ASSERT(fp->devx_obj);
1122 claim_zero(mlx5_devx_cmd_destroy(fp->devx_obj));
1123 DRV_LOG(DEBUG, "DEVx flex parser %p destroyed", (const void *)fp);
1127 struct mlx5_list_entry *
1128 mlx5_flex_parser_clone_cb(void *list_ctx,
1129 struct mlx5_list_entry *entry, void *ctx)
1131 struct mlx5_flex_parser_devx *fp;
1133 RTE_SET_USED(list_ctx);
1134 RTE_SET_USED(entry);
1135 fp = mlx5_malloc(0, sizeof(struct mlx5_flex_parser_devx),
1139 memcpy(fp, ctx, sizeof(struct mlx5_flex_parser_devx));
1144 mlx5_flex_parser_clone_free_cb(void *list_ctx, struct mlx5_list_entry *entry)
1146 struct mlx5_flex_parser_devx *fp =
1147 container_of(entry, struct mlx5_flex_parser_devx, entry);
1148 RTE_SET_USED(list_ctx);