1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
13 #pragma GCC diagnostic ignored "-Wpedantic"
15 #include <infiniband/verbs.h>
17 #pragma GCC diagnostic error "-Wpedantic"
20 #include <rte_common.h>
21 #include <rte_ether.h>
22 #include <rte_ethdev_driver.h>
24 #include <rte_flow_driver.h>
25 #include <rte_malloc.h>
30 #include "mlx5_defs.h"
31 #include "mlx5_glue.h"
32 #include "mlx5_flow.h"
34 #include "mlx5_rxtx.h"
36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
38 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
39 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
42 #ifndef HAVE_MLX5DV_DR_ESWITCH
43 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
44 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
48 #ifndef HAVE_MLX5DV_DR
49 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
65 * Initialize flow attributes structure according to flow items' types.
68 * Pointer to item specification.
70 * Pointer to flow attributes structure.
73 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
75 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
77 case RTE_FLOW_ITEM_TYPE_IPV4:
80 case RTE_FLOW_ITEM_TYPE_IPV6:
83 case RTE_FLOW_ITEM_TYPE_UDP:
86 case RTE_FLOW_ITEM_TYPE_TCP:
96 struct field_modify_info {
97 uint32_t size; /* Size of field in protocol header, in bytes. */
98 uint32_t offset; /* Offset of field in protocol header, in bytes. */
99 enum mlx5_modification_field id;
102 struct field_modify_info modify_eth[] = {
103 {4, 0, MLX5_MODI_OUT_DMAC_47_16},
104 {2, 4, MLX5_MODI_OUT_DMAC_15_0},
105 {4, 6, MLX5_MODI_OUT_SMAC_47_16},
106 {2, 10, MLX5_MODI_OUT_SMAC_15_0},
110 struct field_modify_info modify_ipv4[] = {
111 {1, 8, MLX5_MODI_OUT_IPV4_TTL},
112 {4, 12, MLX5_MODI_OUT_SIPV4},
113 {4, 16, MLX5_MODI_OUT_DIPV4},
117 struct field_modify_info modify_ipv6[] = {
118 {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
119 {4, 8, MLX5_MODI_OUT_SIPV6_127_96},
120 {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
121 {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
122 {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
123 {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
124 {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
125 {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
126 {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
130 struct field_modify_info modify_udp[] = {
131 {2, 0, MLX5_MODI_OUT_UDP_SPORT},
132 {2, 2, MLX5_MODI_OUT_UDP_DPORT},
136 struct field_modify_info modify_tcp[] = {
137 {2, 0, MLX5_MODI_OUT_TCP_SPORT},
138 {2, 2, MLX5_MODI_OUT_TCP_DPORT},
139 {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
140 {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
145 * Acquire the synchronizing object to protect multithreaded access
146 * to shared dv context. Lock occurs only if context is actually
147 * shared, i.e. we have multiport IB device and representors are
151 * Pointer to the rte_eth_dev structure.
154 flow_d_shared_lock(struct rte_eth_dev *dev)
156 struct mlx5_priv *priv = dev->data->dev_private;
157 struct mlx5_ibv_shared *sh = priv->sh;
159 if (sh->dv_refcnt > 1) {
162 ret = pthread_mutex_lock(&sh->dv_mutex);
169 flow_d_shared_unlock(struct rte_eth_dev *dev)
171 struct mlx5_priv *priv = dev->data->dev_private;
172 struct mlx5_ibv_shared *sh = priv->sh;
174 if (sh->dv_refcnt > 1) {
177 ret = pthread_mutex_unlock(&sh->dv_mutex);
184 * Convert modify-header action to DV specification.
187 * Pointer to item specification.
189 * Pointer to field modification information.
190 * @param[in,out] resource
191 * Pointer to the modify-header resource.
193 * Type of modification.
195 * Pointer to the error structure.
198 * 0 on success, a negative errno value otherwise and rte_errno is set.
201 flow_dv_convert_modify_action(struct rte_flow_item *item,
202 struct field_modify_info *field,
203 struct mlx5_flow_dv_modify_hdr_resource *resource,
205 struct rte_flow_error *error)
207 uint32_t i = resource->actions_num;
208 struct mlx5_modification_cmd *actions = resource->actions;
209 const uint8_t *spec = item->spec;
210 const uint8_t *mask = item->mask;
213 while (field->size) {
215 /* Generate modify command for each mask segment. */
216 memcpy(&set, &mask[field->offset], field->size);
218 if (i >= MLX5_MODIFY_NUM)
219 return rte_flow_error_set(error, EINVAL,
220 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
221 "too many items to modify");
222 actions[i].action_type = type;
223 actions[i].field = field->id;
224 actions[i].length = field->size ==
225 4 ? 0 : field->size * 8;
226 rte_memcpy(&actions[i].data[4 - field->size],
227 &spec[field->offset], field->size);
228 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
231 if (resource->actions_num != i)
232 resource->actions_num = i;
235 if (!resource->actions_num)
236 return rte_flow_error_set(error, EINVAL,
237 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
238 "invalid modification flow item");
243 * Convert modify-header set IPv4 address action to DV specification.
245 * @param[in,out] resource
246 * Pointer to the modify-header resource.
248 * Pointer to action specification.
250 * Pointer to the error structure.
253 * 0 on success, a negative errno value otherwise and rte_errno is set.
256 flow_dv_convert_action_modify_ipv4
257 (struct mlx5_flow_dv_modify_hdr_resource *resource,
258 const struct rte_flow_action *action,
259 struct rte_flow_error *error)
261 const struct rte_flow_action_set_ipv4 *conf =
262 (const struct rte_flow_action_set_ipv4 *)(action->conf);
263 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
264 struct rte_flow_item_ipv4 ipv4;
265 struct rte_flow_item_ipv4 ipv4_mask;
267 memset(&ipv4, 0, sizeof(ipv4));
268 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
269 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
270 ipv4.hdr.src_addr = conf->ipv4_addr;
271 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
273 ipv4.hdr.dst_addr = conf->ipv4_addr;
274 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
277 item.mask = &ipv4_mask;
278 return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
279 MLX5_MODIFICATION_TYPE_SET, error);
283 * Convert modify-header set IPv6 address action to DV specification.
285 * @param[in,out] resource
286 * Pointer to the modify-header resource.
288 * Pointer to action specification.
290 * Pointer to the error structure.
293 * 0 on success, a negative errno value otherwise and rte_errno is set.
296 flow_dv_convert_action_modify_ipv6
297 (struct mlx5_flow_dv_modify_hdr_resource *resource,
298 const struct rte_flow_action *action,
299 struct rte_flow_error *error)
301 const struct rte_flow_action_set_ipv6 *conf =
302 (const struct rte_flow_action_set_ipv6 *)(action->conf);
303 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
304 struct rte_flow_item_ipv6 ipv6;
305 struct rte_flow_item_ipv6 ipv6_mask;
307 memset(&ipv6, 0, sizeof(ipv6));
308 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
309 if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
310 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
311 sizeof(ipv6.hdr.src_addr));
312 memcpy(&ipv6_mask.hdr.src_addr,
313 &rte_flow_item_ipv6_mask.hdr.src_addr,
314 sizeof(ipv6.hdr.src_addr));
316 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
317 sizeof(ipv6.hdr.dst_addr));
318 memcpy(&ipv6_mask.hdr.dst_addr,
319 &rte_flow_item_ipv6_mask.hdr.dst_addr,
320 sizeof(ipv6.hdr.dst_addr));
323 item.mask = &ipv6_mask;
324 return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
325 MLX5_MODIFICATION_TYPE_SET, error);
329 * Convert modify-header set MAC address action to DV specification.
331 * @param[in,out] resource
332 * Pointer to the modify-header resource.
334 * Pointer to action specification.
336 * Pointer to the error structure.
339 * 0 on success, a negative errno value otherwise and rte_errno is set.
342 flow_dv_convert_action_modify_mac
343 (struct mlx5_flow_dv_modify_hdr_resource *resource,
344 const struct rte_flow_action *action,
345 struct rte_flow_error *error)
347 const struct rte_flow_action_set_mac *conf =
348 (const struct rte_flow_action_set_mac *)(action->conf);
349 struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
350 struct rte_flow_item_eth eth;
351 struct rte_flow_item_eth eth_mask;
353 memset(ð, 0, sizeof(eth));
354 memset(ð_mask, 0, sizeof(eth_mask));
355 if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
356 memcpy(ð.src.addr_bytes, &conf->mac_addr,
357 sizeof(eth.src.addr_bytes));
358 memcpy(ð_mask.src.addr_bytes,
359 &rte_flow_item_eth_mask.src.addr_bytes,
360 sizeof(eth_mask.src.addr_bytes));
362 memcpy(ð.dst.addr_bytes, &conf->mac_addr,
363 sizeof(eth.dst.addr_bytes));
364 memcpy(ð_mask.dst.addr_bytes,
365 &rte_flow_item_eth_mask.dst.addr_bytes,
366 sizeof(eth_mask.dst.addr_bytes));
369 item.mask = ð_mask;
370 return flow_dv_convert_modify_action(&item, modify_eth, resource,
371 MLX5_MODIFICATION_TYPE_SET, error);
375 * Convert modify-header set TP action to DV specification.
377 * @param[in,out] resource
378 * Pointer to the modify-header resource.
380 * Pointer to action specification.
382 * Pointer to rte_flow_item objects list.
384 * Pointer to flow attributes structure.
386 * Pointer to the error structure.
389 * 0 on success, a negative errno value otherwise and rte_errno is set.
392 flow_dv_convert_action_modify_tp
393 (struct mlx5_flow_dv_modify_hdr_resource *resource,
394 const struct rte_flow_action *action,
395 const struct rte_flow_item *items,
396 union flow_dv_attr *attr,
397 struct rte_flow_error *error)
399 const struct rte_flow_action_set_tp *conf =
400 (const struct rte_flow_action_set_tp *)(action->conf);
401 struct rte_flow_item item;
402 struct rte_flow_item_udp udp;
403 struct rte_flow_item_udp udp_mask;
404 struct rte_flow_item_tcp tcp;
405 struct rte_flow_item_tcp tcp_mask;
406 struct field_modify_info *field;
409 flow_dv_attr_init(items, attr);
411 memset(&udp, 0, sizeof(udp));
412 memset(&udp_mask, 0, sizeof(udp_mask));
413 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
414 udp.hdr.src_port = conf->port;
415 udp_mask.hdr.src_port =
416 rte_flow_item_udp_mask.hdr.src_port;
418 udp.hdr.dst_port = conf->port;
419 udp_mask.hdr.dst_port =
420 rte_flow_item_udp_mask.hdr.dst_port;
422 item.type = RTE_FLOW_ITEM_TYPE_UDP;
424 item.mask = &udp_mask;
428 memset(&tcp, 0, sizeof(tcp));
429 memset(&tcp_mask, 0, sizeof(tcp_mask));
430 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
431 tcp.hdr.src_port = conf->port;
432 tcp_mask.hdr.src_port =
433 rte_flow_item_tcp_mask.hdr.src_port;
435 tcp.hdr.dst_port = conf->port;
436 tcp_mask.hdr.dst_port =
437 rte_flow_item_tcp_mask.hdr.dst_port;
439 item.type = RTE_FLOW_ITEM_TYPE_TCP;
441 item.mask = &tcp_mask;
444 return flow_dv_convert_modify_action(&item, field, resource,
445 MLX5_MODIFICATION_TYPE_SET, error);
449 * Convert modify-header set TTL action to DV specification.
451 * @param[in,out] resource
452 * Pointer to the modify-header resource.
454 * Pointer to action specification.
456 * Pointer to rte_flow_item objects list.
458 * Pointer to flow attributes structure.
460 * Pointer to the error structure.
463 * 0 on success, a negative errno value otherwise and rte_errno is set.
466 flow_dv_convert_action_modify_ttl
467 (struct mlx5_flow_dv_modify_hdr_resource *resource,
468 const struct rte_flow_action *action,
469 const struct rte_flow_item *items,
470 union flow_dv_attr *attr,
471 struct rte_flow_error *error)
473 const struct rte_flow_action_set_ttl *conf =
474 (const struct rte_flow_action_set_ttl *)(action->conf);
475 struct rte_flow_item item;
476 struct rte_flow_item_ipv4 ipv4;
477 struct rte_flow_item_ipv4 ipv4_mask;
478 struct rte_flow_item_ipv6 ipv6;
479 struct rte_flow_item_ipv6 ipv6_mask;
480 struct field_modify_info *field;
483 flow_dv_attr_init(items, attr);
485 memset(&ipv4, 0, sizeof(ipv4));
486 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
487 ipv4.hdr.time_to_live = conf->ttl_value;
488 ipv4_mask.hdr.time_to_live = 0xFF;
489 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
491 item.mask = &ipv4_mask;
495 memset(&ipv6, 0, sizeof(ipv6));
496 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
497 ipv6.hdr.hop_limits = conf->ttl_value;
498 ipv6_mask.hdr.hop_limits = 0xFF;
499 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
501 item.mask = &ipv6_mask;
504 return flow_dv_convert_modify_action(&item, field, resource,
505 MLX5_MODIFICATION_TYPE_SET, error);
509 * Convert modify-header decrement TTL action to DV specification.
511 * @param[in,out] resource
512 * Pointer to the modify-header resource.
514 * Pointer to action specification.
516 * Pointer to rte_flow_item objects list.
518 * Pointer to flow attributes structure.
520 * Pointer to the error structure.
523 * 0 on success, a negative errno value otherwise and rte_errno is set.
526 flow_dv_convert_action_modify_dec_ttl
527 (struct mlx5_flow_dv_modify_hdr_resource *resource,
528 const struct rte_flow_item *items,
529 union flow_dv_attr *attr,
530 struct rte_flow_error *error)
532 struct rte_flow_item item;
533 struct rte_flow_item_ipv4 ipv4;
534 struct rte_flow_item_ipv4 ipv4_mask;
535 struct rte_flow_item_ipv6 ipv6;
536 struct rte_flow_item_ipv6 ipv6_mask;
537 struct field_modify_info *field;
540 flow_dv_attr_init(items, attr);
542 memset(&ipv4, 0, sizeof(ipv4));
543 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
544 ipv4.hdr.time_to_live = 0xFF;
545 ipv4_mask.hdr.time_to_live = 0xFF;
546 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
548 item.mask = &ipv4_mask;
552 memset(&ipv6, 0, sizeof(ipv6));
553 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
554 ipv6.hdr.hop_limits = 0xFF;
555 ipv6_mask.hdr.hop_limits = 0xFF;
556 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
558 item.mask = &ipv6_mask;
561 return flow_dv_convert_modify_action(&item, field, resource,
562 MLX5_MODIFICATION_TYPE_ADD, error);
566 * Convert modify-header increment/decrement TCP Sequence number
567 * to DV specification.
569 * @param[in,out] resource
570 * Pointer to the modify-header resource.
572 * Pointer to action specification.
574 * Pointer to the error structure.
577 * 0 on success, a negative errno value otherwise and rte_errno is set.
580 flow_dv_convert_action_modify_tcp_seq
581 (struct mlx5_flow_dv_modify_hdr_resource *resource,
582 const struct rte_flow_action *action,
583 struct rte_flow_error *error)
585 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
586 uint64_t value = rte_be_to_cpu_32(*conf);
587 struct rte_flow_item item;
588 struct rte_flow_item_tcp tcp;
589 struct rte_flow_item_tcp tcp_mask;
591 memset(&tcp, 0, sizeof(tcp));
592 memset(&tcp_mask, 0, sizeof(tcp_mask));
593 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
595 * The HW has no decrement operation, only increment operation.
596 * To simulate decrement X from Y using increment operation
597 * we need to add UINT32_MAX X times to Y.
598 * Each adding of UINT32_MAX decrements Y by 1.
601 tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
602 tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
603 item.type = RTE_FLOW_ITEM_TYPE_TCP;
605 item.mask = &tcp_mask;
606 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
607 MLX5_MODIFICATION_TYPE_ADD, error);
611 * Convert modify-header increment/decrement TCP Acknowledgment number
612 * to DV specification.
614 * @param[in,out] resource
615 * Pointer to the modify-header resource.
617 * Pointer to action specification.
619 * Pointer to the error structure.
622 * 0 on success, a negative errno value otherwise and rte_errno is set.
625 flow_dv_convert_action_modify_tcp_ack
626 (struct mlx5_flow_dv_modify_hdr_resource *resource,
627 const struct rte_flow_action *action,
628 struct rte_flow_error *error)
630 const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
631 uint64_t value = rte_be_to_cpu_32(*conf);
632 struct rte_flow_item item;
633 struct rte_flow_item_tcp tcp;
634 struct rte_flow_item_tcp tcp_mask;
636 memset(&tcp, 0, sizeof(tcp));
637 memset(&tcp_mask, 0, sizeof(tcp_mask));
638 if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
640 * The HW has no decrement operation, only increment operation.
641 * To simulate decrement X from Y using increment operation
642 * we need to add UINT32_MAX X times to Y.
643 * Each adding of UINT32_MAX decrements Y by 1.
646 tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
647 tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
648 item.type = RTE_FLOW_ITEM_TYPE_TCP;
650 item.mask = &tcp_mask;
651 return flow_dv_convert_modify_action(&item, modify_tcp, resource,
652 MLX5_MODIFICATION_TYPE_ADD, error);
656 * Validate META item.
659 * Pointer to the rte_eth_dev structure.
661 * Item specification.
663 * Attributes of flow that includes this item.
665 * Pointer to error structure.
668 * 0 on success, a negative errno value otherwise and rte_errno is set.
671 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
672 const struct rte_flow_item *item,
673 const struct rte_flow_attr *attr,
674 struct rte_flow_error *error)
676 const struct rte_flow_item_meta *spec = item->spec;
677 const struct rte_flow_item_meta *mask = item->mask;
678 const struct rte_flow_item_meta nic_mask = {
679 .data = RTE_BE32(UINT32_MAX)
682 uint64_t offloads = dev->data->dev_conf.txmode.offloads;
684 if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
685 return rte_flow_error_set(error, EPERM,
686 RTE_FLOW_ERROR_TYPE_ITEM,
688 "match on metadata offload "
689 "configuration is off for this port");
691 return rte_flow_error_set(error, EINVAL,
692 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
694 "data cannot be empty");
696 return rte_flow_error_set(error, EINVAL,
697 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
699 "data cannot be zero");
701 mask = &rte_flow_item_meta_mask;
702 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
703 (const uint8_t *)&nic_mask,
704 sizeof(struct rte_flow_item_meta),
709 return rte_flow_error_set(error, ENOTSUP,
710 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
712 "pattern not supported for ingress");
717 * Validate vport item.
720 * Pointer to the rte_eth_dev structure.
722 * Item specification.
724 * Attributes of flow that includes this item.
725 * @param[in] item_flags
726 * Bit-fields that holds the items detected until now.
728 * Pointer to error structure.
731 * 0 on success, a negative errno value otherwise and rte_errno is set.
734 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
735 const struct rte_flow_item *item,
736 const struct rte_flow_attr *attr,
738 struct rte_flow_error *error)
740 const struct rte_flow_item_port_id *spec = item->spec;
741 const struct rte_flow_item_port_id *mask = item->mask;
742 const struct rte_flow_item_port_id switch_mask = {
745 uint16_t esw_domain_id;
746 uint16_t item_port_esw_domain_id;
750 return rte_flow_error_set(error, EINVAL,
751 RTE_FLOW_ERROR_TYPE_ITEM,
753 "match on port id is valid only"
754 " when transfer flag is enabled");
755 if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
756 return rte_flow_error_set(error, ENOTSUP,
757 RTE_FLOW_ERROR_TYPE_ITEM, item,
758 "multiple source ports are not"
762 if (mask->id != 0xffffffff)
763 return rte_flow_error_set(error, ENOTSUP,
764 RTE_FLOW_ERROR_TYPE_ITEM_MASK,
766 "no support for partial mask on"
768 ret = mlx5_flow_item_acceptable
769 (item, (const uint8_t *)mask,
770 (const uint8_t *)&rte_flow_item_port_id_mask,
771 sizeof(struct rte_flow_item_port_id),
777 ret = mlx5_port_to_eswitch_info(spec->id, &item_port_esw_domain_id,
780 return rte_flow_error_set(error, -ret,
781 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
782 "failed to obtain E-Switch info for"
784 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
785 &esw_domain_id, NULL);
787 return rte_flow_error_set(error, -ret,
788 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
790 "failed to obtain E-Switch info");
791 if (item_port_esw_domain_id != esw_domain_id)
792 return rte_flow_error_set(error, -ret,
793 RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
794 "cannot match on a port from a"
795 " different E-Switch");
800 * Validate count action.
805 * Pointer to error structure.
808 * 0 on success, a negative errno value otherwise and rte_errno is set.
811 flow_dv_validate_action_count(struct rte_eth_dev *dev,
812 struct rte_flow_error *error)
814 struct mlx5_priv *priv = dev->data->dev_private;
816 if (!priv->config.devx)
818 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
822 return rte_flow_error_set
824 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
826 "count action not supported");
830 * Validate the L2 encap action.
832 * @param[in] action_flags
833 * Holds the actions detected until now.
835 * Pointer to the encap action.
837 * Pointer to flow attributes
839 * Pointer to error structure.
842 * 0 on success, a negative errno value otherwise and rte_errno is set.
845 flow_dv_validate_action_l2_encap(uint64_t action_flags,
846 const struct rte_flow_action *action,
847 const struct rte_flow_attr *attr,
848 struct rte_flow_error *error)
851 return rte_flow_error_set(error, EINVAL,
852 RTE_FLOW_ERROR_TYPE_ACTION, action,
853 "configuration cannot be null");
854 if (action_flags & MLX5_FLOW_ACTION_DROP)
855 return rte_flow_error_set(error, EINVAL,
856 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
857 "can't drop and encap in same flow");
858 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
859 return rte_flow_error_set(error, EINVAL,
860 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
861 "can only have a single encap or"
862 " decap action in a flow");
863 if (!attr->transfer && attr->ingress)
864 return rte_flow_error_set(error, ENOTSUP,
865 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
867 "encap action not supported for "
873 * Validate the L2 decap action.
875 * @param[in] action_flags
876 * Holds the actions detected until now.
878 * Pointer to flow attributes
880 * Pointer to error structure.
883 * 0 on success, a negative errno value otherwise and rte_errno is set.
886 flow_dv_validate_action_l2_decap(uint64_t action_flags,
887 const struct rte_flow_attr *attr,
888 struct rte_flow_error *error)
890 if (action_flags & MLX5_FLOW_ACTION_DROP)
891 return rte_flow_error_set(error, EINVAL,
892 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
893 "can't drop and decap in same flow");
894 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS | MLX5_FLOW_DECAP_ACTIONS))
895 return rte_flow_error_set(error, EINVAL,
896 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
897 "can only have a single encap or"
898 " decap action in a flow");
899 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
900 return rte_flow_error_set(error, EINVAL,
901 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
902 "can't have decap action after"
905 return rte_flow_error_set(error, ENOTSUP,
906 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
908 "decap action not supported for "
914 * Validate the raw encap action.
916 * @param[in] action_flags
917 * Holds the actions detected until now.
919 * Pointer to the encap action.
921 * Pointer to flow attributes
923 * Pointer to error structure.
926 * 0 on success, a negative errno value otherwise and rte_errno is set.
929 flow_dv_validate_action_raw_encap(uint64_t action_flags,
930 const struct rte_flow_action *action,
931 const struct rte_flow_attr *attr,
932 struct rte_flow_error *error)
935 return rte_flow_error_set(error, EINVAL,
936 RTE_FLOW_ERROR_TYPE_ACTION, action,
937 "configuration cannot be null");
938 if (action_flags & MLX5_FLOW_ACTION_DROP)
939 return rte_flow_error_set(error, EINVAL,
940 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
941 "can't drop and encap in same flow");
942 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
943 return rte_flow_error_set(error, EINVAL,
944 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
945 "can only have a single encap"
946 " action in a flow");
947 /* encap without preceding decap is not supported for ingress */
948 if (!attr->transfer && attr->ingress &&
949 !(action_flags & MLX5_FLOW_ACTION_RAW_DECAP))
950 return rte_flow_error_set(error, ENOTSUP,
951 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
953 "encap action not supported for "
959 * Validate the raw decap action.
961 * @param[in] action_flags
962 * Holds the actions detected until now.
964 * Pointer to the encap action.
966 * Pointer to flow attributes
968 * Pointer to error structure.
971 * 0 on success, a negative errno value otherwise and rte_errno is set.
974 flow_dv_validate_action_raw_decap(uint64_t action_flags,
975 const struct rte_flow_action *action,
976 const struct rte_flow_attr *attr,
977 struct rte_flow_error *error)
979 if (action_flags & MLX5_FLOW_ACTION_DROP)
980 return rte_flow_error_set(error, EINVAL,
981 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
982 "can't drop and decap in same flow");
983 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
984 return rte_flow_error_set(error, EINVAL,
985 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
986 "can't have encap action before"
988 if (action_flags & MLX5_FLOW_DECAP_ACTIONS)
989 return rte_flow_error_set(error, EINVAL,
990 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
991 "can only have a single decap"
992 " action in a flow");
993 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
994 return rte_flow_error_set(error, EINVAL,
995 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
996 "can't have decap action after"
998 /* decap action is valid on egress only if it is followed by encap */
1000 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
1001 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
1004 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP)
1005 return rte_flow_error_set
1007 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1008 NULL, "decap action not supported"
1015 * Find existing encap/decap resource or create and register a new one.
1017 * @param dev[in, out]
1018 * Pointer to rte_eth_dev structure.
1019 * @param[in, out] resource
1020 * Pointer to encap/decap resource.
1021 * @parm[in, out] dev_flow
1022 * Pointer to the dev_flow.
1024 * pointer to error structure.
1027 * 0 on success otherwise -errno and errno is set.
1030 flow_dv_encap_decap_resource_register
1031 (struct rte_eth_dev *dev,
1032 struct mlx5_flow_dv_encap_decap_resource *resource,
1033 struct mlx5_flow *dev_flow,
1034 struct rte_flow_error *error)
1036 struct mlx5_priv *priv = dev->data->dev_private;
1037 struct mlx5_ibv_shared *sh = priv->sh;
1038 struct mlx5_flow_dv_encap_decap_resource *cache_resource;
1039 struct rte_flow *flow = dev_flow->flow;
1040 struct mlx5dv_dr_domain *domain;
1042 resource->flags = flow->group ? 0 : 1;
1043 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
1044 domain = sh->fdb_domain;
1045 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
1046 domain = sh->rx_domain;
1048 domain = sh->tx_domain;
1050 /* Lookup a matching resource from cache. */
1051 LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
1052 if (resource->reformat_type == cache_resource->reformat_type &&
1053 resource->ft_type == cache_resource->ft_type &&
1054 resource->flags == cache_resource->flags &&
1055 resource->size == cache_resource->size &&
1056 !memcmp((const void *)resource->buf,
1057 (const void *)cache_resource->buf,
1059 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
1060 (void *)cache_resource,
1061 rte_atomic32_read(&cache_resource->refcnt));
1062 rte_atomic32_inc(&cache_resource->refcnt);
1063 dev_flow->dv.encap_decap = cache_resource;
1067 /* Register new encap/decap resource. */
1068 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1069 if (!cache_resource)
1070 return rte_flow_error_set(error, ENOMEM,
1071 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1072 "cannot allocate resource memory");
1073 *cache_resource = *resource;
1074 cache_resource->verbs_action =
1075 mlx5_glue->dv_create_flow_action_packet_reformat
1076 (sh->ctx, cache_resource->reformat_type,
1077 cache_resource->ft_type, domain, cache_resource->flags,
1078 cache_resource->size,
1079 (cache_resource->size ? cache_resource->buf : NULL));
1080 if (!cache_resource->verbs_action) {
1081 rte_free(cache_resource);
1082 return rte_flow_error_set(error, ENOMEM,
1083 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1084 NULL, "cannot create action");
1086 rte_atomic32_init(&cache_resource->refcnt);
1087 rte_atomic32_inc(&cache_resource->refcnt);
1088 LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
1089 dev_flow->dv.encap_decap = cache_resource;
1090 DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
1091 (void *)cache_resource,
1092 rte_atomic32_read(&cache_resource->refcnt));
1097 * Find existing table jump resource or create and register a new one.
1099 * @param dev[in, out]
1100 * Pointer to rte_eth_dev structure.
1101 * @param[in, out] resource
1102 * Pointer to jump table resource.
1103 * @parm[in, out] dev_flow
1104 * Pointer to the dev_flow.
1106 * pointer to error structure.
1109 * 0 on success otherwise -errno and errno is set.
1112 flow_dv_jump_tbl_resource_register
1113 (struct rte_eth_dev *dev,
1114 struct mlx5_flow_dv_jump_tbl_resource *resource,
1115 struct mlx5_flow *dev_flow,
1116 struct rte_flow_error *error)
1118 struct mlx5_priv *priv = dev->data->dev_private;
1119 struct mlx5_ibv_shared *sh = priv->sh;
1120 struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
1122 /* Lookup a matching resource from cache. */
1123 LIST_FOREACH(cache_resource, &sh->jump_tbl, next) {
1124 if (resource->tbl == cache_resource->tbl) {
1125 DRV_LOG(DEBUG, "jump table resource resource %p: refcnt %d++",
1126 (void *)cache_resource,
1127 rte_atomic32_read(&cache_resource->refcnt));
1128 rte_atomic32_inc(&cache_resource->refcnt);
1129 dev_flow->dv.jump = cache_resource;
1133 /* Register new jump table resource. */
1134 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1135 if (!cache_resource)
1136 return rte_flow_error_set(error, ENOMEM,
1137 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1138 "cannot allocate resource memory");
1139 *cache_resource = *resource;
1140 cache_resource->action =
1141 mlx5_glue->dr_create_flow_action_dest_flow_tbl
1142 (resource->tbl->obj);
1143 if (!cache_resource->action) {
1144 rte_free(cache_resource);
1145 return rte_flow_error_set(error, ENOMEM,
1146 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1147 NULL, "cannot create action");
1149 rte_atomic32_init(&cache_resource->refcnt);
1150 rte_atomic32_inc(&cache_resource->refcnt);
1151 LIST_INSERT_HEAD(&sh->jump_tbl, cache_resource, next);
1152 dev_flow->dv.jump = cache_resource;
1153 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
1154 (void *)cache_resource,
1155 rte_atomic32_read(&cache_resource->refcnt));
1160 * Find existing table port ID resource or create and register a new one.
1162 * @param dev[in, out]
1163 * Pointer to rte_eth_dev structure.
1164 * @param[in, out] resource
1165 * Pointer to port ID action resource.
1166 * @parm[in, out] dev_flow
1167 * Pointer to the dev_flow.
1169 * pointer to error structure.
1172 * 0 on success otherwise -errno and errno is set.
1175 flow_dv_port_id_action_resource_register
1176 (struct rte_eth_dev *dev,
1177 struct mlx5_flow_dv_port_id_action_resource *resource,
1178 struct mlx5_flow *dev_flow,
1179 struct rte_flow_error *error)
1181 struct mlx5_priv *priv = dev->data->dev_private;
1182 struct mlx5_ibv_shared *sh = priv->sh;
1183 struct mlx5_flow_dv_port_id_action_resource *cache_resource;
1185 /* Lookup a matching resource from cache. */
1186 LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
1187 if (resource->port_id == cache_resource->port_id) {
1188 DRV_LOG(DEBUG, "port id action resource resource %p: "
1190 (void *)cache_resource,
1191 rte_atomic32_read(&cache_resource->refcnt));
1192 rte_atomic32_inc(&cache_resource->refcnt);
1193 dev_flow->dv.port_id_action = cache_resource;
1197 /* Register new port id action resource. */
1198 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
1199 if (!cache_resource)
1200 return rte_flow_error_set(error, ENOMEM,
1201 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1202 "cannot allocate resource memory");
1203 *cache_resource = *resource;
1204 cache_resource->action =
1205 mlx5_glue->dr_create_flow_action_dest_vport
1206 (priv->sh->fdb_domain, resource->port_id);
1207 if (!cache_resource->action) {
1208 rte_free(cache_resource);
1209 return rte_flow_error_set(error, ENOMEM,
1210 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1211 NULL, "cannot create action");
1213 rte_atomic32_init(&cache_resource->refcnt);
1214 rte_atomic32_inc(&cache_resource->refcnt);
1215 LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
1216 dev_flow->dv.port_id_action = cache_resource;
1217 DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
1218 (void *)cache_resource,
1219 rte_atomic32_read(&cache_resource->refcnt));
1224 * Get the size of specific rte_flow_item_type
1226 * @param[in] item_type
1227 * Tested rte_flow_item_type.
1230 * sizeof struct item_type, 0 if void or irrelevant.
1233 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
1237 switch (item_type) {
1238 case RTE_FLOW_ITEM_TYPE_ETH:
1239 retval = sizeof(struct rte_flow_item_eth);
1241 case RTE_FLOW_ITEM_TYPE_VLAN:
1242 retval = sizeof(struct rte_flow_item_vlan);
1244 case RTE_FLOW_ITEM_TYPE_IPV4:
1245 retval = sizeof(struct rte_flow_item_ipv4);
1247 case RTE_FLOW_ITEM_TYPE_IPV6:
1248 retval = sizeof(struct rte_flow_item_ipv6);
1250 case RTE_FLOW_ITEM_TYPE_UDP:
1251 retval = sizeof(struct rte_flow_item_udp);
1253 case RTE_FLOW_ITEM_TYPE_TCP:
1254 retval = sizeof(struct rte_flow_item_tcp);
1256 case RTE_FLOW_ITEM_TYPE_VXLAN:
1257 retval = sizeof(struct rte_flow_item_vxlan);
1259 case RTE_FLOW_ITEM_TYPE_GRE:
1260 retval = sizeof(struct rte_flow_item_gre);
1262 case RTE_FLOW_ITEM_TYPE_NVGRE:
1263 retval = sizeof(struct rte_flow_item_nvgre);
1265 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1266 retval = sizeof(struct rte_flow_item_vxlan_gpe);
1268 case RTE_FLOW_ITEM_TYPE_MPLS:
1269 retval = sizeof(struct rte_flow_item_mpls);
1271 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
1279 #define MLX5_ENCAP_IPV4_VERSION 0x40
1280 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05
1281 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40
1282 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000
1283 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff
1284 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000
1285 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04
1288 * Convert the encap action data from list of rte_flow_item to raw buffer
1291 * Pointer to rte_flow_item objects list.
1293 * Pointer to the output buffer.
1295 * Pointer to the output buffer size.
1297 * Pointer to the error structure.
1300 * 0 on success, a negative errno value otherwise and rte_errno is set.
1303 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
1304 size_t *size, struct rte_flow_error *error)
1306 struct rte_ether_hdr *eth = NULL;
1307 struct rte_vlan_hdr *vlan = NULL;
1308 struct rte_ipv4_hdr *ipv4 = NULL;
1309 struct rte_ipv6_hdr *ipv6 = NULL;
1310 struct rte_udp_hdr *udp = NULL;
1311 struct rte_vxlan_hdr *vxlan = NULL;
1312 struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
1313 struct rte_gre_hdr *gre = NULL;
1315 size_t temp_size = 0;
1318 return rte_flow_error_set(error, EINVAL,
1319 RTE_FLOW_ERROR_TYPE_ACTION,
1320 NULL, "invalid empty data");
1321 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1322 len = flow_dv_get_item_len(items->type);
1323 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
1324 return rte_flow_error_set(error, EINVAL,
1325 RTE_FLOW_ERROR_TYPE_ACTION,
1326 (void *)items->type,
1327 "items total size is too big"
1328 " for encap action");
1329 rte_memcpy((void *)&buf[temp_size], items->spec, len);
1330 switch (items->type) {
1331 case RTE_FLOW_ITEM_TYPE_ETH:
1332 eth = (struct rte_ether_hdr *)&buf[temp_size];
1334 case RTE_FLOW_ITEM_TYPE_VLAN:
1335 vlan = (struct rte_vlan_hdr *)&buf[temp_size];
1337 return rte_flow_error_set(error, EINVAL,
1338 RTE_FLOW_ERROR_TYPE_ACTION,
1339 (void *)items->type,
1340 "eth header not found");
1341 if (!eth->ether_type)
1342 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
1344 case RTE_FLOW_ITEM_TYPE_IPV4:
1345 ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
1347 return rte_flow_error_set(error, EINVAL,
1348 RTE_FLOW_ERROR_TYPE_ACTION,
1349 (void *)items->type,
1350 "neither eth nor vlan"
1352 if (vlan && !vlan->eth_proto)
1353 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1354 else if (eth && !eth->ether_type)
1355 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
1356 if (!ipv4->version_ihl)
1357 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
1358 MLX5_ENCAP_IPV4_IHL_MIN;
1359 if (!ipv4->time_to_live)
1360 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
1362 case RTE_FLOW_ITEM_TYPE_IPV6:
1363 ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
1365 return rte_flow_error_set(error, EINVAL,
1366 RTE_FLOW_ERROR_TYPE_ACTION,
1367 (void *)items->type,
1368 "neither eth nor vlan"
1370 if (vlan && !vlan->eth_proto)
1371 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1372 else if (eth && !eth->ether_type)
1373 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
1374 if (!ipv6->vtc_flow)
1376 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
1377 if (!ipv6->hop_limits)
1378 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
1380 case RTE_FLOW_ITEM_TYPE_UDP:
1381 udp = (struct rte_udp_hdr *)&buf[temp_size];
1383 return rte_flow_error_set(error, EINVAL,
1384 RTE_FLOW_ERROR_TYPE_ACTION,
1385 (void *)items->type,
1386 "ip header not found");
1387 if (ipv4 && !ipv4->next_proto_id)
1388 ipv4->next_proto_id = IPPROTO_UDP;
1389 else if (ipv6 && !ipv6->proto)
1390 ipv6->proto = IPPROTO_UDP;
1392 case RTE_FLOW_ITEM_TYPE_VXLAN:
1393 vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
1395 return rte_flow_error_set(error, EINVAL,
1396 RTE_FLOW_ERROR_TYPE_ACTION,
1397 (void *)items->type,
1398 "udp header not found");
1400 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
1401 if (!vxlan->vx_flags)
1403 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
1405 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1406 vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
1408 return rte_flow_error_set(error, EINVAL,
1409 RTE_FLOW_ERROR_TYPE_ACTION,
1410 (void *)items->type,
1411 "udp header not found");
1412 if (!vxlan_gpe->proto)
1413 return rte_flow_error_set(error, EINVAL,
1414 RTE_FLOW_ERROR_TYPE_ACTION,
1415 (void *)items->type,
1416 "next protocol not found");
1419 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
1420 if (!vxlan_gpe->vx_flags)
1421 vxlan_gpe->vx_flags =
1422 MLX5_ENCAP_VXLAN_GPE_FLAGS;
1424 case RTE_FLOW_ITEM_TYPE_GRE:
1425 case RTE_FLOW_ITEM_TYPE_NVGRE:
1426 gre = (struct rte_gre_hdr *)&buf[temp_size];
1428 return rte_flow_error_set(error, EINVAL,
1429 RTE_FLOW_ERROR_TYPE_ACTION,
1430 (void *)items->type,
1431 "next protocol not found");
1433 return rte_flow_error_set(error, EINVAL,
1434 RTE_FLOW_ERROR_TYPE_ACTION,
1435 (void *)items->type,
1436 "ip header not found");
1437 if (ipv4 && !ipv4->next_proto_id)
1438 ipv4->next_proto_id = IPPROTO_GRE;
1439 else if (ipv6 && !ipv6->proto)
1440 ipv6->proto = IPPROTO_GRE;
1442 case RTE_FLOW_ITEM_TYPE_VOID:
1445 return rte_flow_error_set(error, EINVAL,
1446 RTE_FLOW_ERROR_TYPE_ACTION,
1447 (void *)items->type,
1448 "unsupported item type");
1458 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
1460 struct rte_ether_hdr *eth = NULL;
1461 struct rte_vlan_hdr *vlan = NULL;
1462 struct rte_ipv6_hdr *ipv6 = NULL;
1463 struct rte_udp_hdr *udp = NULL;
1467 eth = (struct rte_ether_hdr *)data;
1468 next_hdr = (char *)(eth + 1);
1469 proto = RTE_BE16(eth->ether_type);
1472 while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
1473 next_hdr += sizeof(struct rte_vlan_hdr);
1474 vlan = (struct rte_vlan_hdr *)next_hdr;
1475 proto = RTE_BE16(vlan->eth_proto);
1478 /* HW calculates IPv4 csum. no need to proceed */
1479 if (proto == RTE_ETHER_TYPE_IPV4)
1482 /* non IPv4/IPv6 header. not supported */
1483 if (proto != RTE_ETHER_TYPE_IPV6) {
1484 return rte_flow_error_set(error, ENOTSUP,
1485 RTE_FLOW_ERROR_TYPE_ACTION,
1486 NULL, "Cannot offload non IPv4/IPv6");
1489 ipv6 = (struct rte_ipv6_hdr *)next_hdr;
1491 /* ignore non UDP */
1492 if (ipv6->proto != IPPROTO_UDP)
1495 udp = (struct rte_udp_hdr *)(ipv6 + 1);
1496 udp->dgram_cksum = 0;
1502 * Convert L2 encap action to DV specification.
1505 * Pointer to rte_eth_dev structure.
1507 * Pointer to action structure.
1508 * @param[in, out] dev_flow
1509 * Pointer to the mlx5_flow.
1510 * @param[in] transfer
1511 * Mark if the flow is E-Switch flow.
1513 * Pointer to the error structure.
1516 * 0 on success, a negative errno value otherwise and rte_errno is set.
1519 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
1520 const struct rte_flow_action *action,
1521 struct mlx5_flow *dev_flow,
1523 struct rte_flow_error *error)
1525 const struct rte_flow_item *encap_data;
1526 const struct rte_flow_action_raw_encap *raw_encap_data;
1527 struct mlx5_flow_dv_encap_decap_resource res = {
1529 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
1530 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1531 MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
1534 if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
1536 (const struct rte_flow_action_raw_encap *)action->conf;
1537 res.size = raw_encap_data->size;
1538 memcpy(res.buf, raw_encap_data->data, res.size);
1539 if (flow_dv_zero_encap_udp_csum(res.buf, error))
1542 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
1544 ((const struct rte_flow_action_vxlan_encap *)
1545 action->conf)->definition;
1548 ((const struct rte_flow_action_nvgre_encap *)
1549 action->conf)->definition;
1550 if (flow_dv_convert_encap_data(encap_data, res.buf,
1554 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1555 return rte_flow_error_set(error, EINVAL,
1556 RTE_FLOW_ERROR_TYPE_ACTION,
1557 NULL, "can't create L2 encap action");
1562 * Convert L2 decap action to DV specification.
1565 * Pointer to rte_eth_dev structure.
1566 * @param[in, out] dev_flow
1567 * Pointer to the mlx5_flow.
1568 * @param[in] transfer
1569 * Mark if the flow is E-Switch flow.
1571 * Pointer to the error structure.
1574 * 0 on success, a negative errno value otherwise and rte_errno is set.
1577 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
1578 struct mlx5_flow *dev_flow,
1580 struct rte_flow_error *error)
1582 struct mlx5_flow_dv_encap_decap_resource res = {
1585 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
1586 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
1587 MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
1590 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1591 return rte_flow_error_set(error, EINVAL,
1592 RTE_FLOW_ERROR_TYPE_ACTION,
1593 NULL, "can't create L2 decap action");
1598 * Convert raw decap/encap (L3 tunnel) action to DV specification.
1601 * Pointer to rte_eth_dev structure.
1603 * Pointer to action structure.
1604 * @param[in, out] dev_flow
1605 * Pointer to the mlx5_flow.
1607 * Pointer to the flow attributes.
1609 * Pointer to the error structure.
1612 * 0 on success, a negative errno value otherwise and rte_errno is set.
1615 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
1616 const struct rte_flow_action *action,
1617 struct mlx5_flow *dev_flow,
1618 const struct rte_flow_attr *attr,
1619 struct rte_flow_error *error)
1621 const struct rte_flow_action_raw_encap *encap_data;
1622 struct mlx5_flow_dv_encap_decap_resource res;
1624 encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
1625 res.size = encap_data->size;
1626 memcpy(res.buf, encap_data->data, res.size);
1627 res.reformat_type = attr->egress ?
1628 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL :
1629 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
1631 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
1633 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
1634 MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
1635 if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
1636 return rte_flow_error_set(error, EINVAL,
1637 RTE_FLOW_ERROR_TYPE_ACTION,
1638 NULL, "can't create encap action");
1643 * Validate the modify-header actions.
1645 * @param[in] action_flags
1646 * Holds the actions detected until now.
1648 * Pointer to the modify action.
1650 * Pointer to error structure.
1653 * 0 on success, a negative errno value otherwise and rte_errno is set.
1656 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
1657 const struct rte_flow_action *action,
1658 struct rte_flow_error *error)
1660 if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
1661 return rte_flow_error_set(error, EINVAL,
1662 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1663 NULL, "action configuration not set");
1664 if (action_flags & MLX5_FLOW_ENCAP_ACTIONS)
1665 return rte_flow_error_set(error, EINVAL,
1666 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1667 "can't have encap action before"
1673 * Validate the modify-header MAC address actions.
1675 * @param[in] action_flags
1676 * Holds the actions detected until now.
1678 * Pointer to the modify action.
1679 * @param[in] item_flags
1680 * Holds the items detected.
1682 * Pointer to error structure.
1685 * 0 on success, a negative errno value otherwise and rte_errno is set.
1688 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
1689 const struct rte_flow_action *action,
1690 const uint64_t item_flags,
1691 struct rte_flow_error *error)
1695 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1697 if (!(item_flags & MLX5_FLOW_LAYER_L2))
1698 return rte_flow_error_set(error, EINVAL,
1699 RTE_FLOW_ERROR_TYPE_ACTION,
1701 "no L2 item in pattern");
1707 * Validate the modify-header IPv4 address actions.
1709 * @param[in] action_flags
1710 * Holds the actions detected until now.
1712 * Pointer to the modify action.
1713 * @param[in] item_flags
1714 * Holds the items detected.
1716 * Pointer to error structure.
1719 * 0 on success, a negative errno value otherwise and rte_errno is set.
1722 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
1723 const struct rte_flow_action *action,
1724 const uint64_t item_flags,
1725 struct rte_flow_error *error)
1729 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1731 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
1732 return rte_flow_error_set(error, EINVAL,
1733 RTE_FLOW_ERROR_TYPE_ACTION,
1735 "no ipv4 item in pattern");
1741 * Validate the modify-header IPv6 address actions.
1743 * @param[in] action_flags
1744 * Holds the actions detected until now.
1746 * Pointer to the modify action.
1747 * @param[in] item_flags
1748 * Holds the items detected.
1750 * Pointer to error structure.
1753 * 0 on success, a negative errno value otherwise and rte_errno is set.
1756 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
1757 const struct rte_flow_action *action,
1758 const uint64_t item_flags,
1759 struct rte_flow_error *error)
1763 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1765 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
1766 return rte_flow_error_set(error, EINVAL,
1767 RTE_FLOW_ERROR_TYPE_ACTION,
1769 "no ipv6 item in pattern");
1775 * Validate the modify-header TP actions.
1777 * @param[in] action_flags
1778 * Holds the actions detected until now.
1780 * Pointer to the modify action.
1781 * @param[in] item_flags
1782 * Holds the items detected.
1784 * Pointer to error structure.
1787 * 0 on success, a negative errno value otherwise and rte_errno is set.
1790 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
1791 const struct rte_flow_action *action,
1792 const uint64_t item_flags,
1793 struct rte_flow_error *error)
1797 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1799 if (!(item_flags & MLX5_FLOW_LAYER_L4))
1800 return rte_flow_error_set(error, EINVAL,
1801 RTE_FLOW_ERROR_TYPE_ACTION,
1802 NULL, "no transport layer "
1809 * Validate the modify-header actions of increment/decrement
1810 * TCP Sequence-number.
1812 * @param[in] action_flags
1813 * Holds the actions detected until now.
1815 * Pointer to the modify action.
1816 * @param[in] item_flags
1817 * Holds the items detected.
1819 * Pointer to error structure.
1822 * 0 on success, a negative errno value otherwise and rte_errno is set.
1825 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
1826 const struct rte_flow_action *action,
1827 const uint64_t item_flags,
1828 struct rte_flow_error *error)
1832 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1834 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1835 return rte_flow_error_set(error, EINVAL,
1836 RTE_FLOW_ERROR_TYPE_ACTION,
1837 NULL, "no TCP item in"
1839 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
1840 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
1841 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
1842 (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
1843 return rte_flow_error_set(error, EINVAL,
1844 RTE_FLOW_ERROR_TYPE_ACTION,
1846 "cannot decrease and increase"
1847 " TCP sequence number"
1848 " at the same time");
1854 * Validate the modify-header actions of increment/decrement
1855 * TCP Acknowledgment number.
1857 * @param[in] action_flags
1858 * Holds the actions detected until now.
1860 * Pointer to the modify action.
1861 * @param[in] item_flags
1862 * Holds the items detected.
1864 * Pointer to error structure.
1867 * 0 on success, a negative errno value otherwise and rte_errno is set.
1870 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
1871 const struct rte_flow_action *action,
1872 const uint64_t item_flags,
1873 struct rte_flow_error *error)
1877 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1879 if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
1880 return rte_flow_error_set(error, EINVAL,
1881 RTE_FLOW_ERROR_TYPE_ACTION,
1882 NULL, "no TCP item in"
1884 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
1885 (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
1886 (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
1887 (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
1888 return rte_flow_error_set(error, EINVAL,
1889 RTE_FLOW_ERROR_TYPE_ACTION,
1891 "cannot decrease and increase"
1892 " TCP acknowledgment number"
1893 " at the same time");
1899 * Validate the modify-header TTL actions.
1901 * @param[in] action_flags
1902 * Holds the actions detected until now.
1904 * Pointer to the modify action.
1905 * @param[in] item_flags
1906 * Holds the items detected.
1908 * Pointer to error structure.
1911 * 0 on success, a negative errno value otherwise and rte_errno is set.
1914 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
1915 const struct rte_flow_action *action,
1916 const uint64_t item_flags,
1917 struct rte_flow_error *error)
1921 ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
1923 if (!(item_flags & MLX5_FLOW_LAYER_L3))
1924 return rte_flow_error_set(error, EINVAL,
1925 RTE_FLOW_ERROR_TYPE_ACTION,
1927 "no IP protocol in pattern");
1933 * Validate jump action.
1936 * Pointer to the modify action.
1938 * The group of the current flow.
1940 * Pointer to error structure.
1943 * 0 on success, a negative errno value otherwise and rte_errno is set.
1946 flow_dv_validate_action_jump(const struct rte_flow_action *action,
1948 struct rte_flow_error *error)
1950 if (action->type != RTE_FLOW_ACTION_TYPE_JUMP && !action->conf)
1951 return rte_flow_error_set(error, EINVAL,
1952 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1953 NULL, "action configuration not set");
1954 if (group >= ((const struct rte_flow_action_jump *)action->conf)->group)
1955 return rte_flow_error_set(error, EINVAL,
1956 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1957 "target group must be higher then"
1958 " the current flow group");
1963 * Validate the port_id action.
1966 * Pointer to rte_eth_dev structure.
1967 * @param[in] action_flags
1968 * Bit-fields that holds the actions detected until now.
1970 * Port_id RTE action structure.
1972 * Attributes of flow that includes this action.
1974 * Pointer to error structure.
1977 * 0 on success, a negative errno value otherwise and rte_errno is set.
1980 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
1981 uint64_t action_flags,
1982 const struct rte_flow_action *action,
1983 const struct rte_flow_attr *attr,
1984 struct rte_flow_error *error)
1986 const struct rte_flow_action_port_id *port_id;
1988 uint16_t esw_domain_id;
1989 uint16_t act_port_domain_id;
1992 if (!attr->transfer)
1993 return rte_flow_error_set(error, ENOTSUP,
1994 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1996 "port id action is valid in transfer"
1998 if (!action || !action->conf)
1999 return rte_flow_error_set(error, ENOTSUP,
2000 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2002 "port id action parameters must be"
2004 if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
2005 MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2006 return rte_flow_error_set(error, EINVAL,
2007 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2008 "can have only one fate actions in"
2010 ret = mlx5_port_to_eswitch_info(dev->data->port_id,
2011 &esw_domain_id, NULL);
2013 return rte_flow_error_set(error, -ret,
2014 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2016 "failed to obtain E-Switch info");
2017 port_id = action->conf;
2018 port = port_id->original ? dev->data->port_id : port_id->id;
2019 ret = mlx5_port_to_eswitch_info(port, &act_port_domain_id, NULL);
2021 return rte_flow_error_set
2023 RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
2024 "failed to obtain E-Switch port id for port");
2025 if (act_port_domain_id != esw_domain_id)
2026 return rte_flow_error_set
2028 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2029 "port does not belong to"
2030 " E-Switch being configured");
2035 * Find existing modify-header resource or create and register a new one.
2037 * @param dev[in, out]
2038 * Pointer to rte_eth_dev structure.
2039 * @param[in, out] resource
2040 * Pointer to modify-header resource.
2041 * @parm[in, out] dev_flow
2042 * Pointer to the dev_flow.
2044 * pointer to error structure.
2047 * 0 on success otherwise -errno and errno is set.
2050 flow_dv_modify_hdr_resource_register
2051 (struct rte_eth_dev *dev,
2052 struct mlx5_flow_dv_modify_hdr_resource *resource,
2053 struct mlx5_flow *dev_flow,
2054 struct rte_flow_error *error)
2056 struct mlx5_priv *priv = dev->data->dev_private;
2057 struct mlx5_ibv_shared *sh = priv->sh;
2058 struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
2059 struct mlx5dv_dr_domain *ns;
2061 if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2062 ns = sh->fdb_domain;
2063 else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
2068 dev_flow->flow->group ? 0 : MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
2069 /* Lookup a matching resource from cache. */
2070 LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
2071 if (resource->ft_type == cache_resource->ft_type &&
2072 resource->actions_num == cache_resource->actions_num &&
2073 resource->flags == cache_resource->flags &&
2074 !memcmp((const void *)resource->actions,
2075 (const void *)cache_resource->actions,
2076 (resource->actions_num *
2077 sizeof(resource->actions[0])))) {
2078 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
2079 (void *)cache_resource,
2080 rte_atomic32_read(&cache_resource->refcnt));
2081 rte_atomic32_inc(&cache_resource->refcnt);
2082 dev_flow->dv.modify_hdr = cache_resource;
2086 /* Register new modify-header resource. */
2087 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
2088 if (!cache_resource)
2089 return rte_flow_error_set(error, ENOMEM,
2090 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2091 "cannot allocate resource memory");
2092 *cache_resource = *resource;
2093 cache_resource->verbs_action =
2094 mlx5_glue->dv_create_flow_action_modify_header
2095 (sh->ctx, cache_resource->ft_type,
2096 ns, cache_resource->flags,
2097 cache_resource->actions_num *
2098 sizeof(cache_resource->actions[0]),
2099 (uint64_t *)cache_resource->actions);
2100 if (!cache_resource->verbs_action) {
2101 rte_free(cache_resource);
2102 return rte_flow_error_set(error, ENOMEM,
2103 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2104 NULL, "cannot create action");
2106 rte_atomic32_init(&cache_resource->refcnt);
2107 rte_atomic32_inc(&cache_resource->refcnt);
2108 LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
2109 dev_flow->dv.modify_hdr = cache_resource;
2110 DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
2111 (void *)cache_resource,
2112 rte_atomic32_read(&cache_resource->refcnt));
2117 * Get or create a flow counter.
2120 * Pointer to the Ethernet device structure.
2122 * Indicate if this counter is shared with other flows.
2124 * Counter identifier.
2127 * pointer to flow counter on success, NULL otherwise and rte_errno is set.
2129 static struct mlx5_flow_counter *
2130 flow_dv_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
2132 struct mlx5_priv *priv = dev->data->dev_private;
2133 struct mlx5_flow_counter *cnt = NULL;
2134 struct mlx5_devx_counter_set *dcs = NULL;
2137 if (!priv->config.devx) {
2142 LIST_FOREACH(cnt, &priv->flow_counters, next) {
2143 if (cnt->shared && cnt->id == id) {
2149 cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
2150 dcs = rte_calloc(__func__, 1, sizeof(*dcs), 0);
2155 ret = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, dcs);
2158 struct mlx5_flow_counter tmpl = {
2164 tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
2170 LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
2180 * Release a flow counter.
2182 * @param[in] counter
2183 * Pointer to the counter handler.
2186 flow_dv_counter_release(struct mlx5_flow_counter *counter)
2192 if (--counter->ref_cnt == 0) {
2193 ret = mlx5_devx_cmd_flow_counter_free(counter->dcs->obj);
2195 DRV_LOG(ERR, "Failed to free devx counters, %d", ret);
2196 LIST_REMOVE(counter, next);
2197 rte_free(counter->dcs);
2203 * Verify the @p attributes will be correctly understood by the NIC and store
2204 * them in the @p flow if everything is correct.
2207 * Pointer to dev struct.
2208 * @param[in] attributes
2209 * Pointer to flow attributes
2211 * Pointer to error structure.
2214 * 0 on success, a negative errno value otherwise and rte_errno is set.
2217 flow_dv_validate_attributes(struct rte_eth_dev *dev,
2218 const struct rte_flow_attr *attributes,
2219 struct rte_flow_error *error)
2221 struct mlx5_priv *priv = dev->data->dev_private;
2222 uint32_t priority_max = priv->config.flow_prio - 1;
2224 #ifndef HAVE_MLX5DV_DR
2225 if (attributes->group)
2226 return rte_flow_error_set(error, ENOTSUP,
2227 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
2229 "groups is not supported");
2231 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
2232 attributes->priority >= priority_max)
2233 return rte_flow_error_set(error, ENOTSUP,
2234 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2236 "priority out of range");
2237 if (attributes->transfer) {
2238 if (!priv->config.dv_esw_en)
2239 return rte_flow_error_set
2241 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2242 "E-Switch dr is not supported");
2243 if (!(priv->representor || priv->master))
2244 return rte_flow_error_set
2245 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2246 NULL, "E-Switch configurationd can only be"
2247 " done by a master or a representor device");
2248 if (attributes->egress)
2249 return rte_flow_error_set
2251 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
2252 "egress is not supported");
2253 if (attributes->group >= MLX5_MAX_TABLES_FDB)
2254 return rte_flow_error_set
2256 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2257 NULL, "group must be smaller than "
2258 RTE_STR(MLX5_MAX_FDB_TABLES));
2260 if (!(attributes->egress ^ attributes->ingress))
2261 return rte_flow_error_set(error, ENOTSUP,
2262 RTE_FLOW_ERROR_TYPE_ATTR, NULL,
2263 "must specify exactly one of "
2264 "ingress or egress");
2269 * Internal validation function. For validating both actions and items.
2272 * Pointer to the rte_eth_dev structure.
2274 * Pointer to the flow attributes.
2276 * Pointer to the list of items.
2277 * @param[in] actions
2278 * Pointer to the list of actions.
2280 * Pointer to the error structure.
2283 * 0 on success, a negative errno value otherwise and rte_errno is set.
2286 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
2287 const struct rte_flow_item items[],
2288 const struct rte_flow_action actions[],
2289 struct rte_flow_error *error)
2292 uint64_t action_flags = 0;
2293 uint64_t item_flags = 0;
2294 uint64_t last_item = 0;
2295 uint8_t next_protocol = 0xff;
2297 struct rte_flow_item_tcp nic_tcp_mask = {
2300 .src_port = RTE_BE16(UINT16_MAX),
2301 .dst_port = RTE_BE16(UINT16_MAX),
2307 ret = flow_dv_validate_attributes(dev, attr, error);
2310 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2311 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2312 switch (items->type) {
2313 case RTE_FLOW_ITEM_TYPE_VOID:
2315 case RTE_FLOW_ITEM_TYPE_PORT_ID:
2316 ret = flow_dv_validate_item_port_id
2317 (dev, items, attr, item_flags, error);
2320 last_item |= MLX5_FLOW_ITEM_PORT_ID;
2322 case RTE_FLOW_ITEM_TYPE_ETH:
2323 ret = mlx5_flow_validate_item_eth(items, item_flags,
2327 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
2328 MLX5_FLOW_LAYER_OUTER_L2;
2330 case RTE_FLOW_ITEM_TYPE_VLAN:
2331 ret = mlx5_flow_validate_item_vlan(items, item_flags,
2335 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2336 MLX5_FLOW_LAYER_OUTER_VLAN;
2338 case RTE_FLOW_ITEM_TYPE_IPV4:
2339 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
2343 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
2344 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
2345 if (items->mask != NULL &&
2346 ((const struct rte_flow_item_ipv4 *)
2347 items->mask)->hdr.next_proto_id) {
2349 ((const struct rte_flow_item_ipv4 *)
2350 (items->spec))->hdr.next_proto_id;
2352 ((const struct rte_flow_item_ipv4 *)
2353 (items->mask))->hdr.next_proto_id;
2355 /* Reset for inner layer. */
2356 next_protocol = 0xff;
2359 case RTE_FLOW_ITEM_TYPE_IPV6:
2360 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
2364 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
2365 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
2366 if (items->mask != NULL &&
2367 ((const struct rte_flow_item_ipv6 *)
2368 items->mask)->hdr.proto) {
2370 ((const struct rte_flow_item_ipv6 *)
2371 items->spec)->hdr.proto;
2373 ((const struct rte_flow_item_ipv6 *)
2374 items->mask)->hdr.proto;
2376 /* Reset for inner layer. */
2377 next_protocol = 0xff;
2380 case RTE_FLOW_ITEM_TYPE_TCP:
2381 ret = mlx5_flow_validate_item_tcp
2388 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
2389 MLX5_FLOW_LAYER_OUTER_L4_TCP;
2391 case RTE_FLOW_ITEM_TYPE_UDP:
2392 ret = mlx5_flow_validate_item_udp(items, item_flags,
2397 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
2398 MLX5_FLOW_LAYER_OUTER_L4_UDP;
2400 case RTE_FLOW_ITEM_TYPE_GRE:
2401 case RTE_FLOW_ITEM_TYPE_NVGRE:
2402 ret = mlx5_flow_validate_item_gre(items, item_flags,
2403 next_protocol, error);
2406 last_item = MLX5_FLOW_LAYER_GRE;
2408 case RTE_FLOW_ITEM_TYPE_VXLAN:
2409 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
2413 last_item = MLX5_FLOW_LAYER_VXLAN;
2415 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2416 ret = mlx5_flow_validate_item_vxlan_gpe(items,
2421 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
2423 case RTE_FLOW_ITEM_TYPE_MPLS:
2424 ret = mlx5_flow_validate_item_mpls(dev, items,
2429 last_item = MLX5_FLOW_LAYER_MPLS;
2431 case RTE_FLOW_ITEM_TYPE_META:
2432 ret = flow_dv_validate_item_meta(dev, items, attr,
2436 last_item = MLX5_FLOW_ITEM_METADATA;
2438 case RTE_FLOW_ITEM_TYPE_ICMP:
2439 ret = mlx5_flow_validate_item_icmp(items, item_flags,
2444 item_flags |= MLX5_FLOW_LAYER_ICMP;
2446 case RTE_FLOW_ITEM_TYPE_ICMP6:
2447 ret = mlx5_flow_validate_item_icmp6(items, item_flags,
2452 item_flags |= MLX5_FLOW_LAYER_ICMP6;
2455 return rte_flow_error_set(error, ENOTSUP,
2456 RTE_FLOW_ERROR_TYPE_ITEM,
2457 NULL, "item not supported");
2459 item_flags |= last_item;
2461 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2462 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
2463 return rte_flow_error_set(error, ENOTSUP,
2464 RTE_FLOW_ERROR_TYPE_ACTION,
2465 actions, "too many actions");
2466 switch (actions->type) {
2467 case RTE_FLOW_ACTION_TYPE_VOID:
2469 case RTE_FLOW_ACTION_TYPE_PORT_ID:
2470 ret = flow_dv_validate_action_port_id(dev,
2477 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
2480 case RTE_FLOW_ACTION_TYPE_FLAG:
2481 ret = mlx5_flow_validate_action_flag(action_flags,
2485 action_flags |= MLX5_FLOW_ACTION_FLAG;
2488 case RTE_FLOW_ACTION_TYPE_MARK:
2489 ret = mlx5_flow_validate_action_mark(actions,
2494 action_flags |= MLX5_FLOW_ACTION_MARK;
2497 case RTE_FLOW_ACTION_TYPE_DROP:
2498 ret = mlx5_flow_validate_action_drop(action_flags,
2502 action_flags |= MLX5_FLOW_ACTION_DROP;
2505 case RTE_FLOW_ACTION_TYPE_QUEUE:
2506 ret = mlx5_flow_validate_action_queue(actions,
2511 action_flags |= MLX5_FLOW_ACTION_QUEUE;
2514 case RTE_FLOW_ACTION_TYPE_RSS:
2515 ret = mlx5_flow_validate_action_rss(actions,
2521 action_flags |= MLX5_FLOW_ACTION_RSS;
2524 case RTE_FLOW_ACTION_TYPE_COUNT:
2525 ret = flow_dv_validate_action_count(dev, error);
2528 action_flags |= MLX5_FLOW_ACTION_COUNT;
2531 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
2532 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
2533 ret = flow_dv_validate_action_l2_encap(action_flags,
2538 action_flags |= actions->type ==
2539 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
2540 MLX5_FLOW_ACTION_VXLAN_ENCAP :
2541 MLX5_FLOW_ACTION_NVGRE_ENCAP;
2544 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
2545 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
2546 ret = flow_dv_validate_action_l2_decap(action_flags,
2550 action_flags |= actions->type ==
2551 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
2552 MLX5_FLOW_ACTION_VXLAN_DECAP :
2553 MLX5_FLOW_ACTION_NVGRE_DECAP;
2556 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
2557 ret = flow_dv_validate_action_raw_encap(action_flags,
2562 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
2565 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
2566 ret = flow_dv_validate_action_raw_decap(action_flags,
2571 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
2574 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
2575 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
2576 ret = flow_dv_validate_action_modify_mac(action_flags,
2582 /* Count all modify-header actions as one action. */
2583 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
2585 action_flags |= actions->type ==
2586 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
2587 MLX5_FLOW_ACTION_SET_MAC_SRC :
2588 MLX5_FLOW_ACTION_SET_MAC_DST;
2591 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
2592 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
2593 ret = flow_dv_validate_action_modify_ipv4(action_flags,
2599 /* Count all modify-header actions as one action. */
2600 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
2602 action_flags |= actions->type ==
2603 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
2604 MLX5_FLOW_ACTION_SET_IPV4_SRC :
2605 MLX5_FLOW_ACTION_SET_IPV4_DST;
2607 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
2608 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
2609 ret = flow_dv_validate_action_modify_ipv6(action_flags,
2615 /* Count all modify-header actions as one action. */
2616 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
2618 action_flags |= actions->type ==
2619 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
2620 MLX5_FLOW_ACTION_SET_IPV6_SRC :
2621 MLX5_FLOW_ACTION_SET_IPV6_DST;
2623 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
2624 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
2625 ret = flow_dv_validate_action_modify_tp(action_flags,
2631 /* Count all modify-header actions as one action. */
2632 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
2634 action_flags |= actions->type ==
2635 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
2636 MLX5_FLOW_ACTION_SET_TP_SRC :
2637 MLX5_FLOW_ACTION_SET_TP_DST;
2639 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
2640 case RTE_FLOW_ACTION_TYPE_SET_TTL:
2641 ret = flow_dv_validate_action_modify_ttl(action_flags,
2647 /* Count all modify-header actions as one action. */
2648 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
2650 action_flags |= actions->type ==
2651 RTE_FLOW_ACTION_TYPE_SET_TTL ?
2652 MLX5_FLOW_ACTION_SET_TTL :
2653 MLX5_FLOW_ACTION_DEC_TTL;
2655 case RTE_FLOW_ACTION_TYPE_JUMP:
2656 ret = flow_dv_validate_action_jump(actions,
2657 attr->group, error);
2661 action_flags |= MLX5_FLOW_ACTION_JUMP;
2663 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
2664 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
2665 ret = flow_dv_validate_action_modify_tcp_seq
2672 /* Count all modify-header actions as one action. */
2673 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
2675 action_flags |= actions->type ==
2676 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
2677 MLX5_FLOW_ACTION_INC_TCP_SEQ :
2678 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
2680 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
2681 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
2682 ret = flow_dv_validate_action_modify_tcp_ack
2689 /* Count all modify-header actions as one action. */
2690 if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
2692 action_flags |= actions->type ==
2693 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
2694 MLX5_FLOW_ACTION_INC_TCP_ACK :
2695 MLX5_FLOW_ACTION_DEC_TCP_ACK;
2698 return rte_flow_error_set(error, ENOTSUP,
2699 RTE_FLOW_ERROR_TYPE_ACTION,
2701 "action not supported");
2704 /* Eswitch has few restrictions on using items and actions */
2705 if (attr->transfer) {
2706 if (action_flags & MLX5_FLOW_ACTION_FLAG)
2707 return rte_flow_error_set(error, ENOTSUP,
2708 RTE_FLOW_ERROR_TYPE_ACTION,
2710 "unsupported action FLAG");
2711 if (action_flags & MLX5_FLOW_ACTION_MARK)
2712 return rte_flow_error_set(error, ENOTSUP,
2713 RTE_FLOW_ERROR_TYPE_ACTION,
2715 "unsupported action MARK");
2716 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
2717 return rte_flow_error_set(error, ENOTSUP,
2718 RTE_FLOW_ERROR_TYPE_ACTION,
2720 "unsupported action QUEUE");
2721 if (action_flags & MLX5_FLOW_ACTION_RSS)
2722 return rte_flow_error_set(error, ENOTSUP,
2723 RTE_FLOW_ERROR_TYPE_ACTION,
2725 "unsupported action RSS");
2726 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
2727 return rte_flow_error_set(error, EINVAL,
2728 RTE_FLOW_ERROR_TYPE_ACTION,
2730 "no fate action is found");
2732 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
2733 return rte_flow_error_set(error, EINVAL,
2734 RTE_FLOW_ERROR_TYPE_ACTION,
2736 "no fate action is found");
2742 * Internal preparation function. Allocates the DV flow size,
2743 * this size is constant.
2746 * Pointer to the flow attributes.
2748 * Pointer to the list of items.
2749 * @param[in] actions
2750 * Pointer to the list of actions.
2752 * Pointer to the error structure.
2755 * Pointer to mlx5_flow object on success,
2756 * otherwise NULL and rte_errno is set.
2758 static struct mlx5_flow *
2759 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
2760 const struct rte_flow_item items[] __rte_unused,
2761 const struct rte_flow_action actions[] __rte_unused,
2762 struct rte_flow_error *error)
2764 uint32_t size = sizeof(struct mlx5_flow);
2765 struct mlx5_flow *flow;
2767 flow = rte_calloc(__func__, 1, size, 0);
2769 rte_flow_error_set(error, ENOMEM,
2770 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2771 "not enough memory to create flow");
2774 flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
2780 * Sanity check for match mask and value. Similar to check_valid_spec() in
2781 * kernel driver. If unmasked bit is present in value, it returns failure.
2784 * pointer to match mask buffer.
2785 * @param match_value
2786 * pointer to match value buffer.
2789 * 0 if valid, -EINVAL otherwise.
2792 flow_dv_check_valid_spec(void *match_mask, void *match_value)
2794 uint8_t *m = match_mask;
2795 uint8_t *v = match_value;
2798 for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
2801 "match_value differs from match_criteria"
2802 " %p[%u] != %p[%u]",
2803 match_value, i, match_mask, i);
2812 * Add Ethernet item to matcher and to the value.
2814 * @param[in, out] matcher
2816 * @param[in, out] key
2817 * Flow matcher value.
2819 * Flow pattern to translate.
2821 * Item is inner pattern.
2824 flow_dv_translate_item_eth(void *matcher, void *key,
2825 const struct rte_flow_item *item, int inner)
2827 const struct rte_flow_item_eth *eth_m = item->mask;
2828 const struct rte_flow_item_eth *eth_v = item->spec;
2829 const struct rte_flow_item_eth nic_mask = {
2830 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2831 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
2832 .type = RTE_BE16(0xffff),
2844 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2846 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2848 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2850 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2852 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
2853 ð_m->dst, sizeof(eth_m->dst));
2854 /* The value must be in the range of the mask. */
2855 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
2856 for (i = 0; i < sizeof(eth_m->dst); ++i)
2857 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
2858 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
2859 ð_m->src, sizeof(eth_m->src));
2860 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
2861 /* The value must be in the range of the mask. */
2862 for (i = 0; i < sizeof(eth_m->dst); ++i)
2863 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
2864 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
2865 rte_be_to_cpu_16(eth_m->type));
2866 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
2867 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
2871 * Add VLAN item to matcher and to the value.
2873 * @param[in, out] matcher
2875 * @param[in, out] key
2876 * Flow matcher value.
2878 * Flow pattern to translate.
2880 * Item is inner pattern.
2883 flow_dv_translate_item_vlan(void *matcher, void *key,
2884 const struct rte_flow_item *item,
2887 const struct rte_flow_item_vlan *vlan_m = item->mask;
2888 const struct rte_flow_item_vlan *vlan_v = item->spec;
2889 const struct rte_flow_item_vlan nic_mask = {
2890 .tci = RTE_BE16(0x0fff),
2891 .inner_type = RTE_BE16(0xffff),
2903 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2905 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2907 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2909 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2911 tci_m = rte_be_to_cpu_16(vlan_m->tci);
2912 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
2913 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
2914 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
2915 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
2916 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
2917 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
2918 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
2919 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
2920 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
2924 * Add IPV4 item to matcher and to the value.
2926 * @param[in, out] matcher
2928 * @param[in, out] key
2929 * Flow matcher value.
2931 * Flow pattern to translate.
2933 * Item is inner pattern.
2935 * The group to insert the rule.
2938 flow_dv_translate_item_ipv4(void *matcher, void *key,
2939 const struct rte_flow_item *item,
2940 int inner, uint32_t group)
2942 const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
2943 const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
2944 const struct rte_flow_item_ipv4 nic_mask = {
2946 .src_addr = RTE_BE32(0xffffffff),
2947 .dst_addr = RTE_BE32(0xffffffff),
2948 .type_of_service = 0xff,
2949 .next_proto_id = 0xff,
2959 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2961 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
2963 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
2965 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
2968 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
2970 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x4);
2971 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
2976 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2977 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2978 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2979 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2980 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
2981 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
2982 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
2983 src_ipv4_src_ipv6.ipv4_layout.ipv4);
2984 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2985 src_ipv4_src_ipv6.ipv4_layout.ipv4);
2986 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
2987 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
2988 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
2989 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
2990 ipv4_m->hdr.type_of_service);
2991 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
2992 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
2993 ipv4_m->hdr.type_of_service >> 2);
2994 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
2995 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
2996 ipv4_m->hdr.next_proto_id);
2997 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2998 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
3002 * Add IPV6 item to matcher and to the value.
3004 * @param[in, out] matcher
3006 * @param[in, out] key
3007 * Flow matcher value.
3009 * Flow pattern to translate.
3011 * Item is inner pattern.
3013 * The group to insert the rule.
3016 flow_dv_translate_item_ipv6(void *matcher, void *key,
3017 const struct rte_flow_item *item,
3018 int inner, uint32_t group)
3020 const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
3021 const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
3022 const struct rte_flow_item_ipv6 nic_mask = {
3025 "\xff\xff\xff\xff\xff\xff\xff\xff"
3026 "\xff\xff\xff\xff\xff\xff\xff\xff",
3028 "\xff\xff\xff\xff\xff\xff\xff\xff"
3029 "\xff\xff\xff\xff\xff\xff\xff\xff",
3030 .vtc_flow = RTE_BE32(0xffffffff),
3037 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3038 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3047 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3049 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3051 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3053 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3056 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
3058 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0x6);
3059 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
3064 size = sizeof(ipv6_m->hdr.dst_addr);
3065 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3066 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3067 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3068 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
3069 memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
3070 for (i = 0; i < size; ++i)
3071 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
3072 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
3073 src_ipv4_src_ipv6.ipv6_layout.ipv6);
3074 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
3075 src_ipv4_src_ipv6.ipv6_layout.ipv6);
3076 memcpy(l24_m, ipv6_m->hdr.src_addr, size);
3077 for (i = 0; i < size; ++i)
3078 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
3080 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
3081 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
3082 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
3083 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
3084 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
3085 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
3088 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
3090 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
3093 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
3095 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
3099 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
3101 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3102 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
3106 * Add TCP item to matcher and to the value.
3108 * @param[in, out] matcher
3110 * @param[in, out] key
3111 * Flow matcher value.
3113 * Flow pattern to translate.
3115 * Item is inner pattern.
3118 flow_dv_translate_item_tcp(void *matcher, void *key,
3119 const struct rte_flow_item *item,
3122 const struct rte_flow_item_tcp *tcp_m = item->mask;
3123 const struct rte_flow_item_tcp *tcp_v = item->spec;
3128 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3130 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3132 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3134 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3136 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3137 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
3141 tcp_m = &rte_flow_item_tcp_mask;
3142 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
3143 rte_be_to_cpu_16(tcp_m->hdr.src_port));
3144 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
3145 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
3146 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
3147 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
3148 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
3149 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
3150 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
3151 tcp_m->hdr.tcp_flags);
3152 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
3153 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
3157 * Add UDP item to matcher and to the value.
3159 * @param[in, out] matcher
3161 * @param[in, out] key
3162 * Flow matcher value.
3164 * Flow pattern to translate.
3166 * Item is inner pattern.
3169 flow_dv_translate_item_udp(void *matcher, void *key,
3170 const struct rte_flow_item *item,
3173 const struct rte_flow_item_udp *udp_m = item->mask;
3174 const struct rte_flow_item_udp *udp_v = item->spec;
3179 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3181 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3183 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3185 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3187 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3188 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
3192 udp_m = &rte_flow_item_udp_mask;
3193 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
3194 rte_be_to_cpu_16(udp_m->hdr.src_port));
3195 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
3196 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
3197 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
3198 rte_be_to_cpu_16(udp_m->hdr.dst_port));
3199 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
3200 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
3204 * Add GRE item to matcher and to the value.
3206 * @param[in, out] matcher
3208 * @param[in, out] key
3209 * Flow matcher value.
3211 * Flow pattern to translate.
3213 * Item is inner pattern.
3216 flow_dv_translate_item_gre(void *matcher, void *key,
3217 const struct rte_flow_item *item,
3220 const struct rte_flow_item_gre *gre_m = item->mask;
3221 const struct rte_flow_item_gre *gre_v = item->spec;
3224 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3225 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3228 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3230 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3232 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3234 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3236 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3237 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
3241 gre_m = &rte_flow_item_gre_mask;
3242 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
3243 rte_be_to_cpu_16(gre_m->protocol));
3244 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
3245 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
3249 * Add NVGRE item to matcher and to the value.
3251 * @param[in, out] matcher
3253 * @param[in, out] key
3254 * Flow matcher value.
3256 * Flow pattern to translate.
3258 * Item is inner pattern.
3261 flow_dv_translate_item_nvgre(void *matcher, void *key,
3262 const struct rte_flow_item *item,
3265 const struct rte_flow_item_nvgre *nvgre_m = item->mask;
3266 const struct rte_flow_item_nvgre *nvgre_v = item->spec;
3267 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3268 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3269 const char *tni_flow_id_m = (const char *)nvgre_m->tni;
3270 const char *tni_flow_id_v = (const char *)nvgre_v->tni;
3276 flow_dv_translate_item_gre(matcher, key, item, inner);
3280 nvgre_m = &rte_flow_item_nvgre_mask;
3281 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
3282 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
3283 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
3284 memcpy(gre_key_m, tni_flow_id_m, size);
3285 for (i = 0; i < size; ++i)
3286 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
3290 * Add VXLAN item to matcher and to the value.
3292 * @param[in, out] matcher
3294 * @param[in, out] key
3295 * Flow matcher value.
3297 * Flow pattern to translate.
3299 * Item is inner pattern.
3302 flow_dv_translate_item_vxlan(void *matcher, void *key,
3303 const struct rte_flow_item *item,
3306 const struct rte_flow_item_vxlan *vxlan_m = item->mask;
3307 const struct rte_flow_item_vxlan *vxlan_v = item->spec;
3310 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3311 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3319 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3321 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3323 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3325 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3327 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
3328 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
3329 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
3330 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
3331 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
3336 vxlan_m = &rte_flow_item_vxlan_mask;
3337 size = sizeof(vxlan_m->vni);
3338 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
3339 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
3340 memcpy(vni_m, vxlan_m->vni, size);
3341 for (i = 0; i < size; ++i)
3342 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
3346 * Add MPLS item to matcher and to the value.
3348 * @param[in, out] matcher
3350 * @param[in, out] key
3351 * Flow matcher value.
3353 * Flow pattern to translate.
3354 * @param[in] prev_layer
3355 * The protocol layer indicated in previous item.
3357 * Item is inner pattern.
3360 flow_dv_translate_item_mpls(void *matcher, void *key,
3361 const struct rte_flow_item *item,
3362 uint64_t prev_layer,
3365 const uint32_t *in_mpls_m = item->mask;
3366 const uint32_t *in_mpls_v = item->spec;
3367 uint32_t *out_mpls_m = 0;
3368 uint32_t *out_mpls_v = 0;
3369 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3370 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3371 void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
3373 void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
3374 void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
3375 void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3377 switch (prev_layer) {
3378 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
3379 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
3380 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
3381 MLX5_UDP_PORT_MPLS);
3383 case MLX5_FLOW_LAYER_GRE:
3384 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
3385 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
3386 RTE_ETHER_TYPE_MPLS);
3389 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
3390 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
3397 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
3398 switch (prev_layer) {
3399 case MLX5_FLOW_LAYER_OUTER_L4_UDP:
3401 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
3402 outer_first_mpls_over_udp);
3404 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
3405 outer_first_mpls_over_udp);
3407 case MLX5_FLOW_LAYER_GRE:
3409 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
3410 outer_first_mpls_over_gre);
3412 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
3413 outer_first_mpls_over_gre);
3416 /* Inner MPLS not over GRE is not supported. */
3419 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
3423 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
3429 if (out_mpls_m && out_mpls_v) {
3430 *out_mpls_m = *in_mpls_m;
3431 *out_mpls_v = *in_mpls_v & *in_mpls_m;
3436 * Add META item to matcher
3438 * @param[in, out] matcher
3440 * @param[in, out] key
3441 * Flow matcher value.
3443 * Flow pattern to translate.
3445 * Item is inner pattern.
3448 flow_dv_translate_item_meta(void *matcher, void *key,
3449 const struct rte_flow_item *item)
3451 const struct rte_flow_item_meta *meta_m;
3452 const struct rte_flow_item_meta *meta_v;
3454 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
3456 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
3458 meta_m = (const void *)item->mask;
3460 meta_m = &rte_flow_item_meta_mask;
3461 meta_v = (const void *)item->spec;
3463 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
3464 rte_be_to_cpu_32(meta_m->data));
3465 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
3466 rte_be_to_cpu_32(meta_v->data & meta_m->data));
3471 * Add source vport match to the specified matcher.
3473 * @param[in, out] matcher
3475 * @param[in, out] key
3476 * Flow matcher value.
3478 * Source vport value to match
3483 flow_dv_translate_item_source_vport(void *matcher, void *key,
3484 int16_t port, uint16_t mask)
3486 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
3487 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
3489 MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
3490 MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
3494 * Translate port-id item to eswitch match on port-id.
3497 * The devich to configure through.
3498 * @param[in, out] matcher
3500 * @param[in, out] key
3501 * Flow matcher value.
3503 * Flow pattern to translate.
3506 * 0 on success, a negative errno value otherwise.
3509 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
3510 void *key, const struct rte_flow_item *item)
3512 const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
3513 const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
3514 uint16_t mask, val, id;
3517 mask = pid_m ? pid_m->id : 0xffff;
3518 id = pid_v ? pid_v->id : dev->data->port_id;
3519 ret = mlx5_port_to_eswitch_info(id, NULL, &val);
3522 flow_dv_translate_item_source_vport(matcher, key, val, mask);
3527 * Add ICMP6 item to matcher and to the value.
3529 * @param[in, out] matcher
3531 * @param[in, out] key
3532 * Flow matcher value.
3534 * Flow pattern to translate.
3536 * Item is inner pattern.
3539 flow_dv_translate_item_icmp6(void *matcher, void *key,
3540 const struct rte_flow_item *item,
3543 const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
3544 const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
3547 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
3549 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
3551 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3553 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3555 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3557 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3559 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
3560 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
3564 icmp6_m = &rte_flow_item_icmp6_mask;
3565 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
3566 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
3567 icmp6_v->type & icmp6_m->type);
3568 MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
3569 MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
3570 icmp6_v->code & icmp6_m->code);
3574 * Add ICMP item to matcher and to the value.
3576 * @param[in, out] matcher
3578 * @param[in, out] key
3579 * Flow matcher value.
3581 * Flow pattern to translate.
3583 * Item is inner pattern.
3586 flow_dv_translate_item_icmp(void *matcher, void *key,
3587 const struct rte_flow_item *item,
3590 const struct rte_flow_item_icmp *icmp_m = item->mask;
3591 const struct rte_flow_item_icmp *icmp_v = item->spec;
3594 void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
3596 void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
3598 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3600 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
3602 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
3604 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
3606 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
3607 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
3611 icmp_m = &rte_flow_item_icmp_mask;
3612 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
3613 icmp_m->hdr.icmp_type);
3614 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
3615 icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
3616 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
3617 icmp_m->hdr.icmp_code);
3618 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
3619 icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
3622 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
3624 #define HEADER_IS_ZERO(match_criteria, headers) \
3625 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
3626 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
3629 * Calculate flow matcher enable bitmap.
3631 * @param match_criteria
3632 * Pointer to flow matcher criteria.
3635 * Bitmap of enabled fields.
3638 flow_dv_matcher_enable(uint32_t *match_criteria)
3640 uint8_t match_criteria_enable;
3642 match_criteria_enable =
3643 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
3644 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
3645 match_criteria_enable |=
3646 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
3647 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
3648 match_criteria_enable |=
3649 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
3650 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
3651 match_criteria_enable |=
3652 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
3653 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
3654 #ifdef HAVE_MLX5DV_DR
3655 match_criteria_enable |=
3656 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
3657 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
3659 return match_criteria_enable;
3666 * @param dev[in, out]
3667 * Pointer to rte_eth_dev structure.
3668 * @param[in] table_id
3671 * Direction of the table.
3672 * @param[in] transfer
3673 * E-Switch or NIC flow.
3675 * pointer to error structure.
3678 * Returns tables resource based on the index, NULL in case of failed.
3680 static struct mlx5_flow_tbl_resource *
3681 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
3682 uint32_t table_id, uint8_t egress,
3684 struct rte_flow_error *error)
3686 struct mlx5_priv *priv = dev->data->dev_private;
3687 struct mlx5_ibv_shared *sh = priv->sh;
3688 struct mlx5_flow_tbl_resource *tbl;
3690 #ifdef HAVE_MLX5DV_DR
3692 tbl = &sh->fdb_tbl[table_id];
3694 tbl->obj = mlx5_glue->dr_create_flow_tbl
3695 (sh->fdb_domain, table_id);
3696 } else if (egress) {
3697 tbl = &sh->tx_tbl[table_id];
3699 tbl->obj = mlx5_glue->dr_create_flow_tbl
3700 (sh->tx_domain, table_id);
3702 tbl = &sh->rx_tbl[table_id];
3704 tbl->obj = mlx5_glue->dr_create_flow_tbl
3705 (sh->rx_domain, table_id);
3708 rte_flow_error_set(error, ENOMEM,
3709 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3710 NULL, "cannot create table");
3713 rte_atomic32_inc(&tbl->refcnt);
3719 return &sh->fdb_tbl[table_id];
3721 return &sh->tx_tbl[table_id];
3723 return &sh->rx_tbl[table_id];
3728 * Release a flow table.
3731 * Table resource to be released.
3734 * Returns 0 if table was released, else return 1;
3737 flow_dv_tbl_resource_release(struct mlx5_flow_tbl_resource *tbl)
3741 if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
3742 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
3750 * Register the flow matcher.
3752 * @param dev[in, out]
3753 * Pointer to rte_eth_dev structure.
3754 * @param[in, out] matcher
3755 * Pointer to flow matcher.
3756 * @parm[in, out] dev_flow
3757 * Pointer to the dev_flow.
3759 * pointer to error structure.
3762 * 0 on success otherwise -errno and errno is set.
3765 flow_dv_matcher_register(struct rte_eth_dev *dev,
3766 struct mlx5_flow_dv_matcher *matcher,
3767 struct mlx5_flow *dev_flow,
3768 struct rte_flow_error *error)
3770 struct mlx5_priv *priv = dev->data->dev_private;
3771 struct mlx5_ibv_shared *sh = priv->sh;
3772 struct mlx5_flow_dv_matcher *cache_matcher;
3773 struct mlx5dv_flow_matcher_attr dv_attr = {
3774 .type = IBV_FLOW_ATTR_NORMAL,
3775 .match_mask = (void *)&matcher->mask,
3777 struct mlx5_flow_tbl_resource *tbl = NULL;
3779 /* Lookup from cache. */
3780 LIST_FOREACH(cache_matcher, &sh->matchers, next) {
3781 if (matcher->crc == cache_matcher->crc &&
3782 matcher->priority == cache_matcher->priority &&
3783 matcher->egress == cache_matcher->egress &&
3784 matcher->group == cache_matcher->group &&
3785 matcher->transfer == cache_matcher->transfer &&
3786 !memcmp((const void *)matcher->mask.buf,
3787 (const void *)cache_matcher->mask.buf,
3788 cache_matcher->mask.size)) {
3790 "priority %hd use %s matcher %p: refcnt %d++",
3791 cache_matcher->priority,
3792 cache_matcher->egress ? "tx" : "rx",
3793 (void *)cache_matcher,
3794 rte_atomic32_read(&cache_matcher->refcnt));
3795 rte_atomic32_inc(&cache_matcher->refcnt);
3796 dev_flow->dv.matcher = cache_matcher;
3800 /* Register new matcher. */
3801 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
3803 return rte_flow_error_set(error, ENOMEM,
3804 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3805 "cannot allocate matcher memory");
3806 tbl = flow_dv_tbl_resource_get(dev, matcher->group * MLX5_GROUP_FACTOR,
3807 matcher->egress, matcher->transfer,
3810 rte_free(cache_matcher);
3811 return rte_flow_error_set(error, ENOMEM,
3812 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3813 NULL, "cannot create table");
3815 *cache_matcher = *matcher;
3816 dv_attr.match_criteria_enable =
3817 flow_dv_matcher_enable(cache_matcher->mask.buf);
3818 dv_attr.priority = matcher->priority;
3819 if (matcher->egress)
3820 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
3821 cache_matcher->matcher_object =
3822 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
3823 if (!cache_matcher->matcher_object) {
3824 rte_free(cache_matcher);
3825 #ifdef HAVE_MLX5DV_DR
3826 flow_dv_tbl_resource_release(tbl);
3828 return rte_flow_error_set(error, ENOMEM,
3829 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3830 NULL, "cannot create matcher");
3832 rte_atomic32_inc(&cache_matcher->refcnt);
3833 LIST_INSERT_HEAD(&sh->matchers, cache_matcher, next);
3834 dev_flow->dv.matcher = cache_matcher;
3835 DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
3836 cache_matcher->priority,
3837 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
3838 rte_atomic32_read(&cache_matcher->refcnt));
3839 rte_atomic32_inc(&tbl->refcnt);
3844 * Find existing tag resource or create and register a new one.
3846 * @param dev[in, out]
3847 * Pointer to rte_eth_dev structure.
3848 * @param[in, out] resource
3849 * Pointer to tag resource.
3850 * @parm[in, out] dev_flow
3851 * Pointer to the dev_flow.
3853 * pointer to error structure.
3856 * 0 on success otherwise -errno and errno is set.
3859 flow_dv_tag_resource_register
3860 (struct rte_eth_dev *dev,
3861 struct mlx5_flow_dv_tag_resource *resource,
3862 struct mlx5_flow *dev_flow,
3863 struct rte_flow_error *error)
3865 struct mlx5_priv *priv = dev->data->dev_private;
3866 struct mlx5_ibv_shared *sh = priv->sh;
3867 struct mlx5_flow_dv_tag_resource *cache_resource;
3869 /* Lookup a matching resource from cache. */
3870 LIST_FOREACH(cache_resource, &sh->tags, next) {
3871 if (resource->tag == cache_resource->tag) {
3872 DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
3873 (void *)cache_resource,
3874 rte_atomic32_read(&cache_resource->refcnt));
3875 rte_atomic32_inc(&cache_resource->refcnt);
3876 dev_flow->flow->tag_resource = cache_resource;
3880 /* Register new resource. */
3881 cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
3882 if (!cache_resource)
3883 return rte_flow_error_set(error, ENOMEM,
3884 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3885 "cannot allocate resource memory");
3886 *cache_resource = *resource;
3887 cache_resource->action = mlx5_glue->dv_create_flow_action_tag
3889 if (!cache_resource->action) {
3890 rte_free(cache_resource);
3891 return rte_flow_error_set(error, ENOMEM,
3892 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3893 NULL, "cannot create action");
3895 rte_atomic32_init(&cache_resource->refcnt);
3896 rte_atomic32_inc(&cache_resource->refcnt);
3897 LIST_INSERT_HEAD(&sh->tags, cache_resource, next);
3898 dev_flow->flow->tag_resource = cache_resource;
3899 DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
3900 (void *)cache_resource,
3901 rte_atomic32_read(&cache_resource->refcnt));
3909 * Pointer to Ethernet device.
3911 * Pointer to mlx5_flow.
3914 * 1 while a reference on it exists, 0 when freed.
3917 flow_dv_tag_release(struct rte_eth_dev *dev,
3918 struct mlx5_flow_dv_tag_resource *tag)
3921 DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
3922 dev->data->port_id, (void *)tag,
3923 rte_atomic32_read(&tag->refcnt));
3924 if (rte_atomic32_dec_and_test(&tag->refcnt)) {
3925 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
3926 LIST_REMOVE(tag, next);
3927 DRV_LOG(DEBUG, "port %u tag %p: removed",
3928 dev->data->port_id, (void *)tag);
3936 * Translate port ID action to vport.
3939 * Pointer to rte_eth_dev structure.
3941 * Pointer to the port ID action.
3942 * @param[out] dst_port_id
3943 * The target port ID.
3945 * Pointer to the error structure.
3948 * 0 on success, a negative errno value otherwise and rte_errno is set.
3951 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
3952 const struct rte_flow_action *action,
3953 uint32_t *dst_port_id,
3954 struct rte_flow_error *error)
3959 const struct rte_flow_action_port_id *conf =
3960 (const struct rte_flow_action_port_id *)action->conf;
3962 port = conf->original ? dev->data->port_id : conf->id;
3963 ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
3965 return rte_flow_error_set(error, -ret,
3966 RTE_FLOW_ERROR_TYPE_ACTION,
3968 "No eswitch info was found for port");
3969 *dst_port_id = port_id;
3974 * Fill the flow with DV spec.
3977 * Pointer to rte_eth_dev structure.
3978 * @param[in, out] dev_flow
3979 * Pointer to the sub flow.
3981 * Pointer to the flow attributes.
3983 * Pointer to the list of items.
3984 * @param[in] actions
3985 * Pointer to the list of actions.
3987 * Pointer to the error structure.
3990 * 0 on success, a negative errno value otherwise and rte_errno is set.
3993 flow_dv_translate(struct rte_eth_dev *dev,
3994 struct mlx5_flow *dev_flow,
3995 const struct rte_flow_attr *attr,
3996 const struct rte_flow_item items[],
3997 const struct rte_flow_action actions[],
3998 struct rte_flow_error *error)
4000 struct mlx5_priv *priv = dev->data->dev_private;
4001 struct rte_flow *flow = dev_flow->flow;
4002 uint64_t item_flags = 0;
4003 uint64_t last_item = 0;
4004 uint64_t action_flags = 0;
4005 uint64_t priority = attr->priority;
4006 struct mlx5_flow_dv_matcher matcher = {
4008 .size = sizeof(matcher.mask.buf),
4012 bool actions_end = false;
4013 struct mlx5_flow_dv_modify_hdr_resource res = {
4014 .ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4015 MLX5DV_FLOW_TABLE_TYPE_NIC_RX
4017 union flow_dv_attr flow_attr = { .attr = 0 };
4018 struct mlx5_flow_dv_tag_resource tag_resource;
4019 uint32_t modify_action_position = UINT32_MAX;
4020 void *match_mask = matcher.mask.buf;
4021 void *match_value = dev_flow->dv.value.buf;
4023 flow->group = attr->group;
4025 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4026 if (priority == MLX5_FLOW_PRIO_RSVD)
4027 priority = priv->config.flow_prio - 1;
4028 for (; !actions_end ; actions++) {
4029 const struct rte_flow_action_queue *queue;
4030 const struct rte_flow_action_rss *rss;
4031 const struct rte_flow_action *action = actions;
4032 const struct rte_flow_action_count *count = action->conf;
4033 const uint8_t *rss_key;
4034 const struct rte_flow_action_jump *jump_data;
4035 struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
4036 struct mlx5_flow_tbl_resource *tbl;
4037 uint32_t port_id = 0;
4038 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
4040 switch (actions->type) {
4041 case RTE_FLOW_ACTION_TYPE_VOID:
4043 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4044 if (flow_dv_translate_action_port_id(dev, action,
4047 port_id_resource.port_id = port_id;
4048 if (flow_dv_port_id_action_resource_register
4049 (dev, &port_id_resource, dev_flow, error))
4051 dev_flow->dv.actions[actions_n++] =
4052 dev_flow->dv.port_id_action->action;
4053 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4055 case RTE_FLOW_ACTION_TYPE_FLAG:
4057 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
4058 if (!flow->tag_resource)
4059 if (flow_dv_tag_resource_register
4060 (dev, &tag_resource, dev_flow, error))
4062 dev_flow->dv.actions[actions_n++] =
4063 flow->tag_resource->action;
4064 action_flags |= MLX5_FLOW_ACTION_FLAG;
4066 case RTE_FLOW_ACTION_TYPE_MARK:
4067 tag_resource.tag = mlx5_flow_mark_set
4068 (((const struct rte_flow_action_mark *)
4069 (actions->conf))->id);
4070 if (!flow->tag_resource)
4071 if (flow_dv_tag_resource_register
4072 (dev, &tag_resource, dev_flow, error))
4074 dev_flow->dv.actions[actions_n++] =
4075 flow->tag_resource->action;
4076 action_flags |= MLX5_FLOW_ACTION_MARK;
4078 case RTE_FLOW_ACTION_TYPE_DROP:
4079 action_flags |= MLX5_FLOW_ACTION_DROP;
4081 case RTE_FLOW_ACTION_TYPE_QUEUE:
4082 queue = actions->conf;
4083 flow->rss.queue_num = 1;
4084 (*flow->queue)[0] = queue->index;
4085 action_flags |= MLX5_FLOW_ACTION_QUEUE;
4087 case RTE_FLOW_ACTION_TYPE_RSS:
4088 rss = actions->conf;
4090 memcpy((*flow->queue), rss->queue,
4091 rss->queue_num * sizeof(uint16_t));
4092 flow->rss.queue_num = rss->queue_num;
4093 /* NULL RSS key indicates default RSS key. */
4094 rss_key = !rss->key ? rss_hash_default_key : rss->key;
4095 memcpy(flow->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
4096 /* RSS type 0 indicates default RSS type ETH_RSS_IP. */
4097 flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
4098 flow->rss.level = rss->level;
4099 action_flags |= MLX5_FLOW_ACTION_RSS;
4101 case RTE_FLOW_ACTION_TYPE_COUNT:
4102 if (!priv->config.devx) {
4103 rte_errno = ENOTSUP;
4106 flow->counter = flow_dv_counter_new(dev, count->shared,
4108 if (flow->counter == NULL)
4110 dev_flow->dv.actions[actions_n++] =
4111 flow->counter->action;
4112 action_flags |= MLX5_FLOW_ACTION_COUNT;
4115 if (rte_errno == ENOTSUP)
4116 return rte_flow_error_set
4118 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4120 "count action not supported");
4122 return rte_flow_error_set
4124 RTE_FLOW_ERROR_TYPE_ACTION,
4126 "cannot create counter"
4128 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
4129 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
4130 if (flow_dv_create_action_l2_encap(dev, actions,
4135 dev_flow->dv.actions[actions_n++] =
4136 dev_flow->dv.encap_decap->verbs_action;
4137 action_flags |= actions->type ==
4138 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
4139 MLX5_FLOW_ACTION_VXLAN_ENCAP :
4140 MLX5_FLOW_ACTION_NVGRE_ENCAP;
4142 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
4143 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
4144 if (flow_dv_create_action_l2_decap(dev, dev_flow,
4148 dev_flow->dv.actions[actions_n++] =
4149 dev_flow->dv.encap_decap->verbs_action;
4150 action_flags |= actions->type ==
4151 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
4152 MLX5_FLOW_ACTION_VXLAN_DECAP :
4153 MLX5_FLOW_ACTION_NVGRE_DECAP;
4155 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4156 /* Handle encap with preceding decap. */
4157 if (action_flags & MLX5_FLOW_ACTION_RAW_DECAP) {
4158 if (flow_dv_create_action_raw_encap
4159 (dev, actions, dev_flow, attr, error))
4161 dev_flow->dv.actions[actions_n++] =
4162 dev_flow->dv.encap_decap->verbs_action;
4164 /* Handle encap without preceding decap. */
4165 if (flow_dv_create_action_l2_encap
4166 (dev, actions, dev_flow, attr->transfer,
4169 dev_flow->dv.actions[actions_n++] =
4170 dev_flow->dv.encap_decap->verbs_action;
4172 action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
4174 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
4175 /* Check if this decap is followed by encap. */
4176 for (; action->type != RTE_FLOW_ACTION_TYPE_END &&
4177 action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP;
4180 /* Handle decap only if it isn't followed by encap. */
4181 if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4182 if (flow_dv_create_action_l2_decap
4183 (dev, dev_flow, attr->transfer, error))
4185 dev_flow->dv.actions[actions_n++] =
4186 dev_flow->dv.encap_decap->verbs_action;
4188 /* If decap is followed by encap, handle it at encap. */
4189 action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
4191 case RTE_FLOW_ACTION_TYPE_JUMP:
4192 jump_data = action->conf;
4193 tbl = flow_dv_tbl_resource_get(dev, jump_data->group *
4196 attr->transfer, error);
4198 return rte_flow_error_set
4200 RTE_FLOW_ERROR_TYPE_ACTION,
4202 "cannot create jump action.");
4203 jump_tbl_resource.tbl = tbl;
4204 if (flow_dv_jump_tbl_resource_register
4205 (dev, &jump_tbl_resource, dev_flow, error)) {
4206 flow_dv_tbl_resource_release(tbl);
4207 return rte_flow_error_set
4209 RTE_FLOW_ERROR_TYPE_ACTION,
4211 "cannot create jump action.");
4213 dev_flow->dv.actions[actions_n++] =
4214 dev_flow->dv.jump->action;
4215 action_flags |= MLX5_FLOW_ACTION_JUMP;
4217 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
4218 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
4219 if (flow_dv_convert_action_modify_mac(&res, actions,
4222 action_flags |= actions->type ==
4223 RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
4224 MLX5_FLOW_ACTION_SET_MAC_SRC :
4225 MLX5_FLOW_ACTION_SET_MAC_DST;
4227 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
4228 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
4229 if (flow_dv_convert_action_modify_ipv4(&res, actions,
4232 action_flags |= actions->type ==
4233 RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
4234 MLX5_FLOW_ACTION_SET_IPV4_SRC :
4235 MLX5_FLOW_ACTION_SET_IPV4_DST;
4237 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
4238 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
4239 if (flow_dv_convert_action_modify_ipv6(&res, actions,
4242 action_flags |= actions->type ==
4243 RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
4244 MLX5_FLOW_ACTION_SET_IPV6_SRC :
4245 MLX5_FLOW_ACTION_SET_IPV6_DST;
4247 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
4248 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
4249 if (flow_dv_convert_action_modify_tp(&res, actions,
4253 action_flags |= actions->type ==
4254 RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
4255 MLX5_FLOW_ACTION_SET_TP_SRC :
4256 MLX5_FLOW_ACTION_SET_TP_DST;
4258 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
4259 if (flow_dv_convert_action_modify_dec_ttl(&res, items,
4263 action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
4265 case RTE_FLOW_ACTION_TYPE_SET_TTL:
4266 if (flow_dv_convert_action_modify_ttl(&res, actions,
4270 action_flags |= MLX5_FLOW_ACTION_SET_TTL;
4272 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
4273 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
4274 if (flow_dv_convert_action_modify_tcp_seq(&res, actions,
4277 action_flags |= actions->type ==
4278 RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
4279 MLX5_FLOW_ACTION_INC_TCP_SEQ :
4280 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
4283 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
4284 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
4285 if (flow_dv_convert_action_modify_tcp_ack(&res, actions,
4288 action_flags |= actions->type ==
4289 RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
4290 MLX5_FLOW_ACTION_INC_TCP_ACK :
4291 MLX5_FLOW_ACTION_DEC_TCP_ACK;
4293 case RTE_FLOW_ACTION_TYPE_END:
4295 if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) {
4296 /* create modify action if needed. */
4297 if (flow_dv_modify_hdr_resource_register
4302 dev_flow->dv.actions[modify_action_position] =
4303 dev_flow->dv.modify_hdr->verbs_action;
4309 if ((action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS) &&
4310 modify_action_position == UINT32_MAX)
4311 modify_action_position = actions_n++;
4313 dev_flow->dv.actions_n = actions_n;
4314 flow->actions = action_flags;
4315 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4316 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4318 switch (items->type) {
4319 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4320 flow_dv_translate_item_port_id(dev, match_mask,
4321 match_value, items);
4322 last_item = MLX5_FLOW_ITEM_PORT_ID;
4324 case RTE_FLOW_ITEM_TYPE_ETH:
4325 flow_dv_translate_item_eth(match_mask, match_value,
4327 matcher.priority = MLX5_PRIORITY_MAP_L2;
4328 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4329 MLX5_FLOW_LAYER_OUTER_L2;
4331 case RTE_FLOW_ITEM_TYPE_VLAN:
4332 flow_dv_translate_item_vlan(match_mask, match_value,
4334 matcher.priority = MLX5_PRIORITY_MAP_L2;
4335 last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
4336 MLX5_FLOW_LAYER_INNER_VLAN) :
4337 (MLX5_FLOW_LAYER_OUTER_L2 |
4338 MLX5_FLOW_LAYER_OUTER_VLAN);
4340 case RTE_FLOW_ITEM_TYPE_IPV4:
4341 flow_dv_translate_item_ipv4(match_mask, match_value,
4342 items, tunnel, attr->group);
4343 matcher.priority = MLX5_PRIORITY_MAP_L3;
4344 dev_flow->dv.hash_fields |=
4345 mlx5_flow_hashfields_adjust
4347 MLX5_IPV4_LAYER_TYPES,
4348 MLX5_IPV4_IBV_RX_HASH);
4349 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4350 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4352 case RTE_FLOW_ITEM_TYPE_IPV6:
4353 flow_dv_translate_item_ipv6(match_mask, match_value,
4354 items, tunnel, attr->group);
4355 matcher.priority = MLX5_PRIORITY_MAP_L3;
4356 dev_flow->dv.hash_fields |=
4357 mlx5_flow_hashfields_adjust
4359 MLX5_IPV6_LAYER_TYPES,
4360 MLX5_IPV6_IBV_RX_HASH);
4361 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4362 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4364 case RTE_FLOW_ITEM_TYPE_TCP:
4365 flow_dv_translate_item_tcp(match_mask, match_value,
4367 matcher.priority = MLX5_PRIORITY_MAP_L4;
4368 dev_flow->dv.hash_fields |=
4369 mlx5_flow_hashfields_adjust
4370 (dev_flow, tunnel, ETH_RSS_TCP,
4371 IBV_RX_HASH_SRC_PORT_TCP |
4372 IBV_RX_HASH_DST_PORT_TCP);
4373 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
4374 MLX5_FLOW_LAYER_OUTER_L4_TCP;
4376 case RTE_FLOW_ITEM_TYPE_UDP:
4377 flow_dv_translate_item_udp(match_mask, match_value,
4379 matcher.priority = MLX5_PRIORITY_MAP_L4;
4380 dev_flow->dv.hash_fields |=
4381 mlx5_flow_hashfields_adjust
4382 (dev_flow, tunnel, ETH_RSS_UDP,
4383 IBV_RX_HASH_SRC_PORT_UDP |
4384 IBV_RX_HASH_DST_PORT_UDP);
4385 last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
4386 MLX5_FLOW_LAYER_OUTER_L4_UDP;
4388 case RTE_FLOW_ITEM_TYPE_GRE:
4389 flow_dv_translate_item_gre(match_mask, match_value,
4391 last_item = MLX5_FLOW_LAYER_GRE;
4393 case RTE_FLOW_ITEM_TYPE_NVGRE:
4394 flow_dv_translate_item_nvgre(match_mask, match_value,
4396 last_item = MLX5_FLOW_LAYER_GRE;
4398 case RTE_FLOW_ITEM_TYPE_VXLAN:
4399 flow_dv_translate_item_vxlan(match_mask, match_value,
4401 last_item = MLX5_FLOW_LAYER_VXLAN;
4403 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4404 flow_dv_translate_item_vxlan(match_mask, match_value,
4406 last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
4408 case RTE_FLOW_ITEM_TYPE_MPLS:
4409 flow_dv_translate_item_mpls(match_mask, match_value,
4410 items, last_item, tunnel);
4411 last_item = MLX5_FLOW_LAYER_MPLS;
4413 case RTE_FLOW_ITEM_TYPE_META:
4414 flow_dv_translate_item_meta(match_mask, match_value,
4416 last_item = MLX5_FLOW_ITEM_METADATA;
4418 case RTE_FLOW_ITEM_TYPE_ICMP:
4419 flow_dv_translate_item_icmp(match_mask, match_value,
4421 item_flags |= MLX5_FLOW_LAYER_ICMP;
4423 case RTE_FLOW_ITEM_TYPE_ICMP6:
4424 flow_dv_translate_item_icmp6(match_mask, match_value,
4426 item_flags |= MLX5_FLOW_LAYER_ICMP6;
4431 item_flags |= last_item;
4434 * In case of ingress traffic when E-Switch mode is enabled,
4435 * we have two cases where we need to set the source port manually.
4436 * The first one, is in case of Nic steering rule, and the second is
4437 * E-Switch rule where no port_id item was found. In both cases
4438 * the source port is set according the current port in use.
4440 if ((attr->ingress && !(item_flags & MLX5_FLOW_ITEM_PORT_ID)) &&
4441 (priv->representor || priv->master)) {
4442 if (flow_dv_translate_item_port_id(dev, match_mask,
4446 assert(!flow_dv_check_valid_spec(matcher.mask.buf,
4447 dev_flow->dv.value.buf));
4448 dev_flow->layers = item_flags;
4449 /* Register matcher. */
4450 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
4452 matcher.priority = mlx5_flow_adjust_priority(dev, priority,
4454 matcher.egress = attr->egress;
4455 matcher.group = attr->group;
4456 matcher.transfer = attr->transfer;
4457 if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
4463 * Apply the flow to the NIC.
4466 * Pointer to the Ethernet device structure.
4467 * @param[in, out] flow
4468 * Pointer to flow structure.
4470 * Pointer to error structure.
4473 * 0 on success, a negative errno value otherwise and rte_errno is set.
4476 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
4477 struct rte_flow_error *error)
4479 struct mlx5_flow_dv *dv;
4480 struct mlx5_flow *dev_flow;
4481 struct mlx5_priv *priv = dev->data->dev_private;
4485 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
4488 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
4489 if (flow->transfer) {
4490 dv->actions[n++] = priv->sh->esw_drop_action;
4492 dv->hrxq = mlx5_hrxq_drop_new(dev);
4496 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4498 "cannot get drop hash queue");
4501 dv->actions[n++] = dv->hrxq->action;
4503 } else if (flow->actions &
4504 (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
4505 struct mlx5_hrxq *hrxq;
4507 hrxq = mlx5_hrxq_get(dev, flow->key,
4508 MLX5_RSS_HASH_KEY_LEN,
4511 flow->rss.queue_num);
4513 hrxq = mlx5_hrxq_new
4514 (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
4515 dv->hash_fields, (*flow->queue),
4516 flow->rss.queue_num,
4517 !!(dev_flow->layers &
4518 MLX5_FLOW_LAYER_TUNNEL));
4522 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4523 "cannot get hash queue");
4527 dv->actions[n++] = dv->hrxq->action;
4530 mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
4531 (void *)&dv->value, n,
4534 rte_flow_error_set(error, errno,
4535 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4537 "hardware refuses to create flow");
4543 err = rte_errno; /* Save rte_errno before cleanup. */
4544 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
4545 struct mlx5_flow_dv *dv = &dev_flow->dv;
4547 if (flow->actions & MLX5_FLOW_ACTION_DROP)
4548 mlx5_hrxq_drop_release(dev);
4550 mlx5_hrxq_release(dev, dv->hrxq);
4554 rte_errno = err; /* Restore rte_errno. */
4559 * Release the flow matcher.
4562 * Pointer to Ethernet device.
4564 * Pointer to mlx5_flow.
4567 * 1 while a reference on it exists, 0 when freed.
4570 flow_dv_matcher_release(struct rte_eth_dev *dev,
4571 struct mlx5_flow *flow)
4573 struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
4574 struct mlx5_priv *priv = dev->data->dev_private;
4575 struct mlx5_ibv_shared *sh = priv->sh;
4576 struct mlx5_flow_tbl_resource *tbl;
4578 assert(matcher->matcher_object);
4579 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
4580 dev->data->port_id, (void *)matcher,
4581 rte_atomic32_read(&matcher->refcnt));
4582 if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
4583 claim_zero(mlx5_glue->dv_destroy_flow_matcher
4584 (matcher->matcher_object));
4585 LIST_REMOVE(matcher, next);
4586 if (matcher->egress)
4587 tbl = &sh->tx_tbl[matcher->group];
4589 tbl = &sh->rx_tbl[matcher->group];
4590 flow_dv_tbl_resource_release(tbl);
4592 DRV_LOG(DEBUG, "port %u matcher %p: removed",
4593 dev->data->port_id, (void *)matcher);
4600 * Release an encap/decap resource.
4603 * Pointer to mlx5_flow.
4606 * 1 while a reference on it exists, 0 when freed.
4609 flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
4611 struct mlx5_flow_dv_encap_decap_resource *cache_resource =
4612 flow->dv.encap_decap;
4614 assert(cache_resource->verbs_action);
4615 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
4616 (void *)cache_resource,
4617 rte_atomic32_read(&cache_resource->refcnt));
4618 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
4619 claim_zero(mlx5_glue->destroy_flow_action
4620 (cache_resource->verbs_action));
4621 LIST_REMOVE(cache_resource, next);
4622 rte_free(cache_resource);
4623 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
4624 (void *)cache_resource);
4631 * Release an jump to table action resource.
4634 * Pointer to mlx5_flow.
4637 * 1 while a reference on it exists, 0 when freed.
4640 flow_dv_jump_tbl_resource_release(struct mlx5_flow *flow)
4642 struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
4645 assert(cache_resource->action);
4646 DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
4647 (void *)cache_resource,
4648 rte_atomic32_read(&cache_resource->refcnt));
4649 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
4650 claim_zero(mlx5_glue->destroy_flow_action
4651 (cache_resource->action));
4652 LIST_REMOVE(cache_resource, next);
4653 flow_dv_tbl_resource_release(cache_resource->tbl);
4654 rte_free(cache_resource);
4655 DRV_LOG(DEBUG, "jump table resource %p: removed",
4656 (void *)cache_resource);
4663 * Release a modify-header resource.
4666 * Pointer to mlx5_flow.
4669 * 1 while a reference on it exists, 0 when freed.
4672 flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
4674 struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
4675 flow->dv.modify_hdr;
4677 assert(cache_resource->verbs_action);
4678 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
4679 (void *)cache_resource,
4680 rte_atomic32_read(&cache_resource->refcnt));
4681 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
4682 claim_zero(mlx5_glue->destroy_flow_action
4683 (cache_resource->verbs_action));
4684 LIST_REMOVE(cache_resource, next);
4685 rte_free(cache_resource);
4686 DRV_LOG(DEBUG, "modify-header resource %p: removed",
4687 (void *)cache_resource);
4694 * Release port ID action resource.
4697 * Pointer to mlx5_flow.
4700 * 1 while a reference on it exists, 0 when freed.
4703 flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
4705 struct mlx5_flow_dv_port_id_action_resource *cache_resource =
4706 flow->dv.port_id_action;
4708 assert(cache_resource->action);
4709 DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
4710 (void *)cache_resource,
4711 rte_atomic32_read(&cache_resource->refcnt));
4712 if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
4713 claim_zero(mlx5_glue->destroy_flow_action
4714 (cache_resource->action));
4715 LIST_REMOVE(cache_resource, next);
4716 rte_free(cache_resource);
4717 DRV_LOG(DEBUG, "port id action resource %p: removed",
4718 (void *)cache_resource);
4725 * Remove the flow from the NIC but keeps it in memory.
4728 * Pointer to Ethernet device.
4729 * @param[in, out] flow
4730 * Pointer to flow structure.
4733 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
4735 struct mlx5_flow_dv *dv;
4736 struct mlx5_flow *dev_flow;
4740 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
4743 claim_zero(mlx5_glue->dv_destroy_flow(dv->flow));
4747 if (flow->actions & MLX5_FLOW_ACTION_DROP)
4748 mlx5_hrxq_drop_release(dev);
4750 mlx5_hrxq_release(dev, dv->hrxq);
4757 * Remove the flow from the NIC and the memory.
4760 * Pointer to the Ethernet device structure.
4761 * @param[in, out] flow
4762 * Pointer to flow structure.
4765 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
4767 struct mlx5_flow *dev_flow;
4771 flow_dv_remove(dev, flow);
4772 if (flow->counter) {
4773 flow_dv_counter_release(flow->counter);
4774 flow->counter = NULL;
4776 if (flow->tag_resource) {
4777 flow_dv_tag_release(dev, flow->tag_resource);
4778 flow->tag_resource = NULL;
4780 while (!LIST_EMPTY(&flow->dev_flows)) {
4781 dev_flow = LIST_FIRST(&flow->dev_flows);
4782 LIST_REMOVE(dev_flow, next);
4783 if (dev_flow->dv.matcher)
4784 flow_dv_matcher_release(dev, dev_flow);
4785 if (dev_flow->dv.encap_decap)
4786 flow_dv_encap_decap_resource_release(dev_flow);
4787 if (dev_flow->dv.modify_hdr)
4788 flow_dv_modify_hdr_resource_release(dev_flow);
4789 if (dev_flow->dv.jump)
4790 flow_dv_jump_tbl_resource_release(dev_flow);
4791 if (dev_flow->dv.port_id_action)
4792 flow_dv_port_id_action_resource_release(dev_flow);
4798 * Query a dv flow rule for its statistics via devx.
4801 * Pointer to Ethernet device.
4803 * Pointer to the sub flow.
4805 * data retrieved by the query.
4807 * Perform verbose error reporting if not NULL.
4810 * 0 on success, a negative errno value otherwise and rte_errno is set.
4813 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
4814 void *data, struct rte_flow_error *error)
4816 struct mlx5_priv *priv = dev->data->dev_private;
4817 struct rte_flow_query_count *qc = data;
4822 if (!priv->config.devx)
4823 return rte_flow_error_set(error, ENOTSUP,
4824 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4826 "counters are not supported");
4827 if (flow->counter) {
4828 err = mlx5_devx_cmd_flow_counter_query
4829 (flow->counter->dcs,
4830 qc->reset, &pkts, &bytes);
4832 return rte_flow_error_set
4834 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4836 "cannot read counters");
4839 qc->hits = pkts - flow->counter->hits;
4840 qc->bytes = bytes - flow->counter->bytes;
4842 flow->counter->hits = pkts;
4843 flow->counter->bytes = bytes;
4847 return rte_flow_error_set(error, EINVAL,
4848 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4850 "counters are not available");
4856 * @see rte_flow_query()
4860 flow_dv_query(struct rte_eth_dev *dev,
4861 struct rte_flow *flow __rte_unused,
4862 const struct rte_flow_action *actions __rte_unused,
4863 void *data __rte_unused,
4864 struct rte_flow_error *error __rte_unused)
4868 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
4869 switch (actions->type) {
4870 case RTE_FLOW_ACTION_TYPE_VOID:
4872 case RTE_FLOW_ACTION_TYPE_COUNT:
4873 ret = flow_dv_query_count(dev, flow, data, error);
4876 return rte_flow_error_set(error, ENOTSUP,
4877 RTE_FLOW_ERROR_TYPE_ACTION,
4879 "action not supported");
4886 * Mutex-protected thunk to flow_dv_translate().
4889 flow_d_translate(struct rte_eth_dev *dev,
4890 struct mlx5_flow *dev_flow,
4891 const struct rte_flow_attr *attr,
4892 const struct rte_flow_item items[],
4893 const struct rte_flow_action actions[],
4894 struct rte_flow_error *error)
4898 flow_d_shared_lock(dev);
4899 ret = flow_dv_translate(dev, dev_flow, attr, items, actions, error);
4900 flow_d_shared_unlock(dev);
4905 * Mutex-protected thunk to flow_dv_apply().
4908 flow_d_apply(struct rte_eth_dev *dev,
4909 struct rte_flow *flow,
4910 struct rte_flow_error *error)
4914 flow_d_shared_lock(dev);
4915 ret = flow_dv_apply(dev, flow, error);
4916 flow_d_shared_unlock(dev);
4921 * Mutex-protected thunk to flow_dv_remove().
4924 flow_d_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
4926 flow_d_shared_lock(dev);
4927 flow_dv_remove(dev, flow);
4928 flow_d_shared_unlock(dev);
4932 * Mutex-protected thunk to flow_dv_destroy().
4935 flow_d_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
4937 flow_d_shared_lock(dev);
4938 flow_dv_destroy(dev, flow);
4939 flow_d_shared_unlock(dev);
4942 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
4943 .validate = flow_dv_validate,
4944 .prepare = flow_dv_prepare,
4945 .translate = flow_d_translate,
4946 .apply = flow_d_apply,
4947 .remove = flow_d_remove,
4948 .destroy = flow_d_destroy,
4949 .query = flow_dv_query,
4952 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */