1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
13 #pragma GCC diagnostic ignored "-Wpedantic"
15 #include <infiniband/verbs.h>
17 #pragma GCC diagnostic error "-Wpedantic"
20 #include <rte_common.h>
21 #include <rte_ether.h>
22 #include <rte_eth_ctrl.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
31 #include "mlx5_defs.h"
33 #include "mlx5_glue.h"
34 #include "mlx5_flow.h"
36 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
38 #define MLX5_ENCAP_MAX_LEN 132
44 * Pointer to the rte_eth_dev structure.
48 * Attributes of flow that includes this item.
50 * Pointer to error structure.
53 * 0 on success, a negative errno value otherwise and rte_errno is set.
56 flow_dv_validate_item_meta(struct rte_eth_dev *dev,
57 const struct rte_flow_item *item,
58 const struct rte_flow_attr *attr,
59 struct rte_flow_error *error)
61 const struct rte_flow_item_meta *spec = item->spec;
62 const struct rte_flow_item_meta *mask = item->mask;
63 const struct rte_flow_item_meta nic_mask = {
64 .data = RTE_BE32(UINT32_MAX)
67 uint64_t offloads = dev->data->dev_conf.txmode.offloads;
69 if (!(offloads & DEV_TX_OFFLOAD_MATCH_METADATA))
70 return rte_flow_error_set(error, EPERM,
71 RTE_FLOW_ERROR_TYPE_ITEM,
73 "match on metadata offload "
74 "configuration is off for this port");
76 return rte_flow_error_set(error, EINVAL,
77 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
79 "data cannot be empty");
81 return rte_flow_error_set(error, EINVAL,
82 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
84 "data cannot be zero");
86 mask = &rte_flow_item_meta_mask;
87 ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
88 (const uint8_t *)&nic_mask,
89 sizeof(struct rte_flow_item_meta),
94 return rte_flow_error_set(error, ENOTSUP,
95 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
97 "pattern not supported for ingress");
102 * Validate the L2 encap action.
104 * @param[in] action_flags
105 * Holds the actions detected until now.
107 * Pointer to the encap action.
109 * Pointer to flow attributes
111 * Pointer to error structure.
114 * 0 on success, a negative errno value otherwise and rte_errno is set.
117 flow_dv_validate_action_l2_encap(uint64_t action_flags,
118 const struct rte_flow_action *action,
119 const struct rte_flow_attr *attr,
120 struct rte_flow_error *error)
123 return rte_flow_error_set(error, EINVAL,
124 RTE_FLOW_ERROR_TYPE_ACTION, action,
125 "configuration cannot be null");
126 if (action_flags & MLX5_FLOW_ACTION_DROP)
127 return rte_flow_error_set(error, EINVAL,
128 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
129 "can't drop and encap in same flow");
130 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS |
131 MLX5_FLOW_ACTION_VXLAN_DECAP))
132 return rte_flow_error_set(error, EINVAL,
133 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
134 "can only have a single encap or"
135 " decap action in a flow");
137 return rte_flow_error_set(error, ENOTSUP,
138 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
140 "encap action not supported for "
146 * Validate the L2 decap action.
148 * @param[in] action_flags
149 * Holds the actions detected until now.
151 * Pointer to flow attributes
153 * Pointer to error structure.
156 * 0 on success, a negative errno value otherwise and rte_errno is set.
159 flow_dv_validate_action_l2_decap(uint64_t action_flags,
160 const struct rte_flow_attr *attr,
161 struct rte_flow_error *error)
163 if (action_flags & MLX5_FLOW_ACTION_DROP)
164 return rte_flow_error_set(error, EINVAL,
165 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
166 "can't drop and decap in same flow");
167 if (action_flags & (MLX5_FLOW_ENCAP_ACTIONS |
168 MLX5_FLOW_ACTION_VXLAN_DECAP))
169 return rte_flow_error_set(error, EINVAL,
170 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
171 "can only have a single encap or"
172 " decap action in a flow");
174 return rte_flow_error_set(error, ENOTSUP,
175 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
177 "decap action not supported for "
183 * Get the size of specific rte_flow_item_type
185 * @param[in] item_type
186 * Tested rte_flow_item_type.
189 * sizeof struct item_type, 0 if void or irrelevant.
192 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
197 case RTE_FLOW_ITEM_TYPE_ETH:
198 retval = sizeof(struct rte_flow_item_eth);
200 case RTE_FLOW_ITEM_TYPE_VLAN:
201 retval = sizeof(struct rte_flow_item_vlan);
203 case RTE_FLOW_ITEM_TYPE_IPV4:
204 retval = sizeof(struct rte_flow_item_ipv4);
206 case RTE_FLOW_ITEM_TYPE_IPV6:
207 retval = sizeof(struct rte_flow_item_ipv6);
209 case RTE_FLOW_ITEM_TYPE_UDP:
210 retval = sizeof(struct rte_flow_item_udp);
212 case RTE_FLOW_ITEM_TYPE_TCP:
213 retval = sizeof(struct rte_flow_item_tcp);
215 case RTE_FLOW_ITEM_TYPE_VXLAN:
216 retval = sizeof(struct rte_flow_item_vxlan);
218 case RTE_FLOW_ITEM_TYPE_GRE:
219 retval = sizeof(struct rte_flow_item_gre);
221 case RTE_FLOW_ITEM_TYPE_NVGRE:
222 retval = sizeof(struct rte_flow_item_nvgre);
224 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
225 retval = sizeof(struct rte_flow_item_vxlan_gpe);
227 case RTE_FLOW_ITEM_TYPE_MPLS:
228 retval = sizeof(struct rte_flow_item_mpls);
230 case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
238 #define MLX5_ENCAP_IPV4_VERSION 0x40
239 #define MLX5_ENCAP_IPV4_IHL_MIN 0x05
240 #define MLX5_ENCAP_IPV4_TTL_DEF 0x40
241 #define MLX5_ENCAP_IPV6_VTC_FLOW 0x60000000
242 #define MLX5_ENCAP_IPV6_HOP_LIMIT 0xff
243 #define MLX5_ENCAP_VXLAN_FLAGS 0x08000000
244 #define MLX5_ENCAP_VXLAN_GPE_FLAGS 0x04
247 * Convert the encap action data from list of rte_flow_item to raw buffer
250 * Pointer to rte_flow_item objects list.
252 * Pointer to the output buffer.
254 * Pointer to the output buffer size.
256 * Pointer to the error structure.
259 * 0 on success, a negative errno value otherwise and rte_errno is set.
262 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
263 size_t *size, struct rte_flow_error *error)
265 struct ether_hdr *eth = NULL;
266 struct vlan_hdr *vlan = NULL;
267 struct ipv4_hdr *ipv4 = NULL;
268 struct ipv6_hdr *ipv6 = NULL;
269 struct udp_hdr *udp = NULL;
270 struct vxlan_hdr *vxlan = NULL;
271 struct vxlan_gpe_hdr *vxlan_gpe = NULL;
272 struct gre_hdr *gre = NULL;
274 size_t temp_size = 0;
277 return rte_flow_error_set(error, EINVAL,
278 RTE_FLOW_ERROR_TYPE_ACTION,
279 NULL, "invalid empty data");
280 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
281 len = flow_dv_get_item_len(items->type);
282 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
283 return rte_flow_error_set(error, EINVAL,
284 RTE_FLOW_ERROR_TYPE_ACTION,
286 "items total size is too big"
287 " for encap action");
288 rte_memcpy((void *)&buf[temp_size], items->spec, len);
289 switch (items->type) {
290 case RTE_FLOW_ITEM_TYPE_ETH:
291 eth = (struct ether_hdr *)&buf[temp_size];
293 case RTE_FLOW_ITEM_TYPE_VLAN:
294 vlan = (struct vlan_hdr *)&buf[temp_size];
296 return rte_flow_error_set(error, EINVAL,
297 RTE_FLOW_ERROR_TYPE_ACTION,
299 "eth header not found");
300 if (!eth->ether_type)
301 eth->ether_type = RTE_BE16(ETHER_TYPE_VLAN);
303 case RTE_FLOW_ITEM_TYPE_IPV4:
304 ipv4 = (struct ipv4_hdr *)&buf[temp_size];
306 return rte_flow_error_set(error, EINVAL,
307 RTE_FLOW_ERROR_TYPE_ACTION,
309 "neither eth nor vlan"
311 if (vlan && !vlan->eth_proto)
312 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv4);
313 else if (eth && !eth->ether_type)
314 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv4);
315 if (!ipv4->version_ihl)
316 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
317 MLX5_ENCAP_IPV4_IHL_MIN;
318 if (!ipv4->time_to_live)
319 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
321 case RTE_FLOW_ITEM_TYPE_IPV6:
322 ipv6 = (struct ipv6_hdr *)&buf[temp_size];
324 return rte_flow_error_set(error, EINVAL,
325 RTE_FLOW_ERROR_TYPE_ACTION,
327 "neither eth nor vlan"
329 if (vlan && !vlan->eth_proto)
330 vlan->eth_proto = RTE_BE16(ETHER_TYPE_IPv6);
331 else if (eth && !eth->ether_type)
332 eth->ether_type = RTE_BE16(ETHER_TYPE_IPv6);
335 RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
336 if (!ipv6->hop_limits)
337 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
339 case RTE_FLOW_ITEM_TYPE_UDP:
340 udp = (struct udp_hdr *)&buf[temp_size];
342 return rte_flow_error_set(error, EINVAL,
343 RTE_FLOW_ERROR_TYPE_ACTION,
345 "ip header not found");
346 if (ipv4 && !ipv4->next_proto_id)
347 ipv4->next_proto_id = IPPROTO_UDP;
348 else if (ipv6 && !ipv6->proto)
349 ipv6->proto = IPPROTO_UDP;
351 case RTE_FLOW_ITEM_TYPE_VXLAN:
352 vxlan = (struct vxlan_hdr *)&buf[temp_size];
354 return rte_flow_error_set(error, EINVAL,
355 RTE_FLOW_ERROR_TYPE_ACTION,
357 "udp header not found");
359 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
360 if (!vxlan->vx_flags)
362 RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
364 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
365 vxlan_gpe = (struct vxlan_gpe_hdr *)&buf[temp_size];
367 return rte_flow_error_set(error, EINVAL,
368 RTE_FLOW_ERROR_TYPE_ACTION,
370 "udp header not found");
371 if (!vxlan_gpe->proto)
372 return rte_flow_error_set(error, EINVAL,
373 RTE_FLOW_ERROR_TYPE_ACTION,
375 "next protocol not found");
378 RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
379 if (!vxlan_gpe->vx_flags)
380 vxlan_gpe->vx_flags =
381 MLX5_ENCAP_VXLAN_GPE_FLAGS;
383 case RTE_FLOW_ITEM_TYPE_GRE:
384 case RTE_FLOW_ITEM_TYPE_NVGRE:
385 gre = (struct gre_hdr *)&buf[temp_size];
387 return rte_flow_error_set(error, EINVAL,
388 RTE_FLOW_ERROR_TYPE_ACTION,
390 "next protocol not found");
392 return rte_flow_error_set(error, EINVAL,
393 RTE_FLOW_ERROR_TYPE_ACTION,
395 "ip header not found");
396 if (ipv4 && !ipv4->next_proto_id)
397 ipv4->next_proto_id = IPPROTO_GRE;
398 else if (ipv6 && !ipv6->proto)
399 ipv6->proto = IPPROTO_GRE;
401 case RTE_FLOW_ITEM_TYPE_VOID:
404 return rte_flow_error_set(error, EINVAL,
405 RTE_FLOW_ERROR_TYPE_ACTION,
407 "unsupported item type");
417 * Convert L2 encap action to DV specification.
420 * Pointer to rte_eth_dev structure.
422 * Pointer to action structure.
424 * Pointer to the error structure.
427 * Pointer to action on success, NULL otherwise and rte_errno is set.
429 static struct ibv_flow_action *
430 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
431 const struct rte_flow_action *action,
432 struct rte_flow_error *error)
434 struct ibv_flow_action *verbs_action = NULL;
435 const struct rte_flow_item *encap_data;
436 struct priv *priv = dev->data->dev_private;
437 uint8_t buf[MLX5_ENCAP_MAX_LEN];
439 int convert_result = 0;
441 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
442 encap_data = ((const struct rte_flow_action_vxlan_encap *)
443 action->conf)->definition;
445 encap_data = ((const struct rte_flow_action_nvgre_encap *)
446 action->conf)->definition;
447 convert_result = flow_dv_convert_encap_data(encap_data, buf,
451 verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
452 (priv->ctx, size, buf,
453 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
454 MLX5DV_FLOW_TABLE_TYPE_NIC_TX);
456 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
457 NULL, "cannot create L2 encap action");
462 * Convert L2 decap action to DV specification.
465 * Pointer to rte_eth_dev structure.
467 * Pointer to the error structure.
470 * Pointer to action on success, NULL otherwise and rte_errno is set.
472 static struct ibv_flow_action *
473 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
474 struct rte_flow_error *error)
476 struct ibv_flow_action *verbs_action = NULL;
477 struct priv *priv = dev->data->dev_private;
479 verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat
481 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
482 MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
484 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
485 NULL, "cannot create L2 decap action");
490 * Verify the @p attributes will be correctly understood by the NIC and store
491 * them in the @p flow if everything is correct.
494 * Pointer to dev struct.
495 * @param[in] attributes
496 * Pointer to flow attributes
498 * Pointer to error structure.
501 * 0 on success, a negative errno value otherwise and rte_errno is set.
504 flow_dv_validate_attributes(struct rte_eth_dev *dev,
505 const struct rte_flow_attr *attributes,
506 struct rte_flow_error *error)
508 struct priv *priv = dev->data->dev_private;
509 uint32_t priority_max = priv->config.flow_prio - 1;
511 if (attributes->group)
512 return rte_flow_error_set(error, ENOTSUP,
513 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
515 "groups is not supported");
516 if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
517 attributes->priority >= priority_max)
518 return rte_flow_error_set(error, ENOTSUP,
519 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
521 "priority out of range");
522 if (attributes->transfer)
523 return rte_flow_error_set(error, ENOTSUP,
524 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
526 "transfer is not supported");
527 if (!(attributes->egress ^ attributes->ingress))
528 return rte_flow_error_set(error, ENOTSUP,
529 RTE_FLOW_ERROR_TYPE_ATTR, NULL,
530 "must specify exactly one of "
531 "ingress or egress");
536 * Internal validation function. For validating both actions and items.
539 * Pointer to the rte_eth_dev structure.
541 * Pointer to the flow attributes.
543 * Pointer to the list of items.
545 * Pointer to the list of actions.
547 * Pointer to the error structure.
550 * 0 on success, a negative errno value otherwise and rte_ernno is set.
553 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
554 const struct rte_flow_item items[],
555 const struct rte_flow_action actions[],
556 struct rte_flow_error *error)
559 uint64_t action_flags = 0;
560 uint64_t item_flags = 0;
562 uint8_t next_protocol = 0xff;
567 ret = flow_dv_validate_attributes(dev, attr, error);
570 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
571 tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
572 switch (items->type) {
573 case RTE_FLOW_ITEM_TYPE_VOID:
575 case RTE_FLOW_ITEM_TYPE_ETH:
576 ret = mlx5_flow_validate_item_eth(items, item_flags,
580 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
581 MLX5_FLOW_LAYER_OUTER_L2;
583 case RTE_FLOW_ITEM_TYPE_VLAN:
584 ret = mlx5_flow_validate_item_vlan(items, item_flags,
588 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
589 MLX5_FLOW_LAYER_OUTER_VLAN;
591 case RTE_FLOW_ITEM_TYPE_IPV4:
592 ret = mlx5_flow_validate_item_ipv4(items, item_flags,
596 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
597 MLX5_FLOW_LAYER_OUTER_L3_IPV4;
598 if (items->mask != NULL &&
599 ((const struct rte_flow_item_ipv4 *)
600 items->mask)->hdr.next_proto_id)
602 ((const struct rte_flow_item_ipv4 *)
603 (items->spec))->hdr.next_proto_id;
605 case RTE_FLOW_ITEM_TYPE_IPV6:
606 ret = mlx5_flow_validate_item_ipv6(items, item_flags,
610 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
611 MLX5_FLOW_LAYER_OUTER_L3_IPV6;
612 if (items->mask != NULL &&
613 ((const struct rte_flow_item_ipv6 *)
614 items->mask)->hdr.proto)
616 ((const struct rte_flow_item_ipv6 *)
617 items->spec)->hdr.proto;
619 case RTE_FLOW_ITEM_TYPE_TCP:
620 ret = mlx5_flow_validate_item_tcp
623 &rte_flow_item_tcp_mask,
627 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
628 MLX5_FLOW_LAYER_OUTER_L4_TCP;
630 case RTE_FLOW_ITEM_TYPE_UDP:
631 ret = mlx5_flow_validate_item_udp(items, item_flags,
636 item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
637 MLX5_FLOW_LAYER_OUTER_L4_UDP;
639 case RTE_FLOW_ITEM_TYPE_GRE:
640 case RTE_FLOW_ITEM_TYPE_NVGRE:
641 ret = mlx5_flow_validate_item_gre(items, item_flags,
642 next_protocol, error);
645 item_flags |= MLX5_FLOW_LAYER_GRE;
647 case RTE_FLOW_ITEM_TYPE_VXLAN:
648 ret = mlx5_flow_validate_item_vxlan(items, item_flags,
652 item_flags |= MLX5_FLOW_LAYER_VXLAN;
654 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
655 ret = mlx5_flow_validate_item_vxlan_gpe(items,
660 item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE;
662 case RTE_FLOW_ITEM_TYPE_META:
663 ret = flow_dv_validate_item_meta(dev, items, attr,
667 item_flags |= MLX5_FLOW_ITEM_METADATA;
670 return rte_flow_error_set(error, ENOTSUP,
671 RTE_FLOW_ERROR_TYPE_ITEM,
672 NULL, "item not supported");
675 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
676 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
677 return rte_flow_error_set(error, ENOTSUP,
678 RTE_FLOW_ERROR_TYPE_ACTION,
679 actions, "too many actions");
680 switch (actions->type) {
681 case RTE_FLOW_ACTION_TYPE_VOID:
683 case RTE_FLOW_ACTION_TYPE_FLAG:
684 ret = mlx5_flow_validate_action_flag(action_flags,
688 action_flags |= MLX5_FLOW_ACTION_FLAG;
691 case RTE_FLOW_ACTION_TYPE_MARK:
692 ret = mlx5_flow_validate_action_mark(actions,
697 action_flags |= MLX5_FLOW_ACTION_MARK;
700 case RTE_FLOW_ACTION_TYPE_DROP:
701 ret = mlx5_flow_validate_action_drop(action_flags,
705 action_flags |= MLX5_FLOW_ACTION_DROP;
708 case RTE_FLOW_ACTION_TYPE_QUEUE:
709 ret = mlx5_flow_validate_action_queue(actions,
714 action_flags |= MLX5_FLOW_ACTION_QUEUE;
717 case RTE_FLOW_ACTION_TYPE_RSS:
718 ret = mlx5_flow_validate_action_rss(actions,
723 action_flags |= MLX5_FLOW_ACTION_RSS;
726 case RTE_FLOW_ACTION_TYPE_COUNT:
727 ret = mlx5_flow_validate_action_count(dev, attr, error);
730 action_flags |= MLX5_FLOW_ACTION_COUNT;
733 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
734 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
735 ret = flow_dv_validate_action_l2_encap(action_flags,
740 action_flags |= actions->type ==
741 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
742 MLX5_FLOW_ACTION_VXLAN_ENCAP :
743 MLX5_FLOW_ACTION_NVGRE_ENCAP;
747 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
748 ret = flow_dv_validate_action_l2_decap(action_flags,
752 action_flags |= MLX5_FLOW_ACTION_VXLAN_DECAP;
756 return rte_flow_error_set(error, ENOTSUP,
757 RTE_FLOW_ERROR_TYPE_ACTION,
759 "action not supported");
762 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
763 return rte_flow_error_set(error, EINVAL,
764 RTE_FLOW_ERROR_TYPE_ACTION, actions,
765 "no fate action is found");
770 * Internal preparation function. Allocates the DV flow size,
771 * this size is constant.
774 * Pointer to the flow attributes.
776 * Pointer to the list of items.
778 * Pointer to the list of actions.
779 * @param[out] item_flags
780 * Pointer to bit mask of all items detected.
781 * @param[out] action_flags
782 * Pointer to bit mask of all actions detected.
784 * Pointer to the error structure.
787 * Pointer to mlx5_flow object on success,
788 * otherwise NULL and rte_ernno is set.
790 static struct mlx5_flow *
791 flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
792 const struct rte_flow_item items[] __rte_unused,
793 const struct rte_flow_action actions[] __rte_unused,
794 uint64_t *item_flags __rte_unused,
795 uint64_t *action_flags __rte_unused,
796 struct rte_flow_error *error)
798 uint32_t size = sizeof(struct mlx5_flow);
799 struct mlx5_flow *flow;
801 flow = rte_calloc(__func__, 1, size, 0);
803 rte_flow_error_set(error, ENOMEM,
804 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
805 "not enough memory to create flow");
808 flow->dv.value.size = MLX5_ST_SZ_DB(fte_match_param);
813 * Add Ethernet item to matcher and to the value.
815 * @param[in, out] matcher
817 * @param[in, out] key
818 * Flow matcher value.
820 * Flow pattern to translate.
822 * Item is inner pattern.
825 flow_dv_translate_item_eth(void *matcher, void *key,
826 const struct rte_flow_item *item, int inner)
828 const struct rte_flow_item_eth *eth_m = item->mask;
829 const struct rte_flow_item_eth *eth_v = item->spec;
830 const struct rte_flow_item_eth nic_mask = {
831 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
832 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
833 .type = RTE_BE16(0xffff),
845 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
847 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
849 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
851 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
853 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
854 ð_m->dst, sizeof(eth_m->dst));
855 /* The value must be in the range of the mask. */
856 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
857 for (i = 0; i < sizeof(eth_m->dst); ++i)
858 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
859 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
860 ð_m->src, sizeof(eth_m->src));
861 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
862 /* The value must be in the range of the mask. */
863 for (i = 0; i < sizeof(eth_m->dst); ++i)
864 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
865 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
866 rte_be_to_cpu_16(eth_m->type));
867 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, ethertype);
868 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
872 * Add VLAN item to matcher and to the value.
874 * @param[in, out] matcher
876 * @param[in, out] key
877 * Flow matcher value.
879 * Flow pattern to translate.
881 * Item is inner pattern.
884 flow_dv_translate_item_vlan(void *matcher, void *key,
885 const struct rte_flow_item *item,
888 const struct rte_flow_item_vlan *vlan_m = item->mask;
889 const struct rte_flow_item_vlan *vlan_v = item->spec;
890 const struct rte_flow_item_vlan nic_mask = {
891 .tci = RTE_BE16(0x0fff),
892 .inner_type = RTE_BE16(0xffff),
904 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
906 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
908 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
910 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
912 tci_m = rte_be_to_cpu_16(vlan_m->tci);
913 tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
914 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
915 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
916 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
917 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
918 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
919 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
920 MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
921 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
925 * Add IPV4 item to matcher and to the value.
927 * @param[in, out] matcher
929 * @param[in, out] key
930 * Flow matcher value.
932 * Flow pattern to translate.
934 * Item is inner pattern.
937 flow_dv_translate_item_ipv4(void *matcher, void *key,
938 const struct rte_flow_item *item,
941 const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
942 const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
943 const struct rte_flow_item_ipv4 nic_mask = {
945 .src_addr = RTE_BE32(0xffffffff),
946 .dst_addr = RTE_BE32(0xffffffff),
947 .type_of_service = 0xff,
948 .next_proto_id = 0xff,
958 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
960 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
962 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
964 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
966 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
967 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 4);
972 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
973 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
974 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
975 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
976 *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
977 *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
978 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
979 src_ipv4_src_ipv6.ipv4_layout.ipv4);
980 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
981 src_ipv4_src_ipv6.ipv4_layout.ipv4);
982 *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
983 *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
984 tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
985 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
986 ipv4_m->hdr.type_of_service);
987 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
988 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
989 ipv4_m->hdr.type_of_service >> 2);
990 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
991 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
992 ipv4_m->hdr.next_proto_id);
993 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
994 ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
998 * Add IPV6 item to matcher and to the value.
1000 * @param[in, out] matcher
1002 * @param[in, out] key
1003 * Flow matcher value.
1005 * Flow pattern to translate.
1007 * Item is inner pattern.
1010 flow_dv_translate_item_ipv6(void *matcher, void *key,
1011 const struct rte_flow_item *item,
1014 const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
1015 const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
1016 const struct rte_flow_item_ipv6 nic_mask = {
1019 "\xff\xff\xff\xff\xff\xff\xff\xff"
1020 "\xff\xff\xff\xff\xff\xff\xff\xff",
1022 "\xff\xff\xff\xff\xff\xff\xff\xff"
1023 "\xff\xff\xff\xff\xff\xff\xff\xff",
1024 .vtc_flow = RTE_BE32(0xffffffff),
1031 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1032 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1041 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1043 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1045 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1047 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1049 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
1050 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 6);
1055 size = sizeof(ipv6_m->hdr.dst_addr);
1056 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1057 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1058 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1059 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
1060 memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
1061 for (i = 0; i < size; ++i)
1062 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
1063 l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
1064 src_ipv4_src_ipv6.ipv6_layout.ipv6);
1065 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1066 src_ipv4_src_ipv6.ipv6_layout.ipv6);
1067 memcpy(l24_m, ipv6_m->hdr.src_addr, size);
1068 for (i = 0; i < size; ++i)
1069 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
1071 vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
1072 vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
1073 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
1074 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
1075 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
1076 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
1079 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
1081 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
1084 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
1086 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
1090 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
1092 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
1093 ipv6_v->hdr.proto & ipv6_m->hdr.proto);
1097 * Add TCP item to matcher and to the value.
1099 * @param[in, out] matcher
1101 * @param[in, out] key
1102 * Flow matcher value.
1104 * Flow pattern to translate.
1106 * Item is inner pattern.
1109 flow_dv_translate_item_tcp(void *matcher, void *key,
1110 const struct rte_flow_item *item,
1113 const struct rte_flow_item_tcp *tcp_m = item->mask;
1114 const struct rte_flow_item_tcp *tcp_v = item->spec;
1119 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1121 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1123 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1125 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1127 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1128 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
1132 tcp_m = &rte_flow_item_tcp_mask;
1133 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
1134 rte_be_to_cpu_16(tcp_m->hdr.src_port));
1135 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
1136 rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
1137 MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
1138 rte_be_to_cpu_16(tcp_m->hdr.dst_port));
1139 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
1140 rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
1144 * Add UDP item to matcher and to the value.
1146 * @param[in, out] matcher
1148 * @param[in, out] key
1149 * Flow matcher value.
1151 * Flow pattern to translate.
1153 * Item is inner pattern.
1156 flow_dv_translate_item_udp(void *matcher, void *key,
1157 const struct rte_flow_item *item,
1160 const struct rte_flow_item_udp *udp_m = item->mask;
1161 const struct rte_flow_item_udp *udp_v = item->spec;
1166 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1168 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1170 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1172 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1174 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1175 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
1179 udp_m = &rte_flow_item_udp_mask;
1180 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
1181 rte_be_to_cpu_16(udp_m->hdr.src_port));
1182 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
1183 rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
1184 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
1185 rte_be_to_cpu_16(udp_m->hdr.dst_port));
1186 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
1187 rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
1191 * Add GRE item to matcher and to the value.
1193 * @param[in, out] matcher
1195 * @param[in, out] key
1196 * Flow matcher value.
1198 * Flow pattern to translate.
1200 * Item is inner pattern.
1203 flow_dv_translate_item_gre(void *matcher, void *key,
1204 const struct rte_flow_item *item,
1207 const struct rte_flow_item_gre *gre_m = item->mask;
1208 const struct rte_flow_item_gre *gre_v = item->spec;
1211 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1212 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1215 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1217 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1219 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1221 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1223 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
1224 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
1228 gre_m = &rte_flow_item_gre_mask;
1229 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
1230 rte_be_to_cpu_16(gre_m->protocol));
1231 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
1232 rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
1236 * Add NVGRE item to matcher and to the value.
1238 * @param[in, out] matcher
1240 * @param[in, out] key
1241 * Flow matcher value.
1243 * Flow pattern to translate.
1245 * Item is inner pattern.
1248 flow_dv_translate_item_nvgre(void *matcher, void *key,
1249 const struct rte_flow_item *item,
1252 const struct rte_flow_item_nvgre *nvgre_m = item->mask;
1253 const struct rte_flow_item_nvgre *nvgre_v = item->spec;
1254 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1255 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1256 const char *tni_flow_id_m = (const char *)nvgre_m->tni;
1257 const char *tni_flow_id_v = (const char *)nvgre_v->tni;
1263 flow_dv_translate_item_gre(matcher, key, item, inner);
1267 nvgre_m = &rte_flow_item_nvgre_mask;
1268 size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
1269 gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
1270 gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
1271 memcpy(gre_key_m, tni_flow_id_m, size);
1272 for (i = 0; i < size; ++i)
1273 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
1277 * Add VXLAN item to matcher and to the value.
1279 * @param[in, out] matcher
1281 * @param[in, out] key
1282 * Flow matcher value.
1284 * Flow pattern to translate.
1286 * Item is inner pattern.
1289 flow_dv_translate_item_vxlan(void *matcher, void *key,
1290 const struct rte_flow_item *item,
1293 const struct rte_flow_item_vxlan *vxlan_m = item->mask;
1294 const struct rte_flow_item_vxlan *vxlan_v = item->spec;
1297 void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
1298 void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
1306 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1308 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
1310 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
1312 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
1314 dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
1315 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
1316 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
1317 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
1318 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
1323 vxlan_m = &rte_flow_item_vxlan_mask;
1324 size = sizeof(vxlan_m->vni);
1325 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
1326 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
1327 memcpy(vni_m, vxlan_m->vni, size);
1328 for (i = 0; i < size; ++i)
1329 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
1333 * Add META item to matcher
1335 * @param[in, out] matcher
1337 * @param[in, out] key
1338 * Flow matcher value.
1340 * Flow pattern to translate.
1342 * Item is inner pattern.
1345 flow_dv_translate_item_meta(void *matcher, void *key,
1346 const struct rte_flow_item *item)
1348 const struct rte_flow_item_meta *meta_m;
1349 const struct rte_flow_item_meta *meta_v;
1351 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
1353 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
1355 meta_m = (const void *)item->mask;
1357 meta_m = &rte_flow_item_meta_mask;
1358 meta_v = (const void *)item->spec;
1360 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a,
1361 rte_be_to_cpu_32(meta_m->data));
1362 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a,
1363 rte_be_to_cpu_32(meta_v->data & meta_m->data));
1368 * Update the matcher and the value based the selected item.
1370 * @param[in, out] matcher
1372 * @param[in, out] key
1373 * Flow matcher value.
1375 * Flow pattern to translate.
1376 * @param[in, out] dev_flow
1377 * Pointer to the mlx5_flow.
1379 * Item is inner pattern.
1382 flow_dv_create_item(void *matcher, void *key,
1383 const struct rte_flow_item *item,
1384 struct mlx5_flow *dev_flow,
1387 struct mlx5_flow_dv_matcher *tmatcher = matcher;
1389 switch (item->type) {
1390 case RTE_FLOW_ITEM_TYPE_ETH:
1391 flow_dv_translate_item_eth(tmatcher->mask.buf, key, item,
1393 tmatcher->priority = MLX5_PRIORITY_MAP_L2;
1395 case RTE_FLOW_ITEM_TYPE_VLAN:
1396 flow_dv_translate_item_vlan(tmatcher->mask.buf, key, item,
1399 case RTE_FLOW_ITEM_TYPE_IPV4:
1400 flow_dv_translate_item_ipv4(tmatcher->mask.buf, key, item,
1402 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1403 dev_flow->dv.hash_fields |=
1404 mlx5_flow_hashfields_adjust(dev_flow, inner,
1405 MLX5_IPV4_LAYER_TYPES,
1406 MLX5_IPV4_IBV_RX_HASH);
1408 case RTE_FLOW_ITEM_TYPE_IPV6:
1409 flow_dv_translate_item_ipv6(tmatcher->mask.buf, key, item,
1411 tmatcher->priority = MLX5_PRIORITY_MAP_L3;
1412 dev_flow->dv.hash_fields |=
1413 mlx5_flow_hashfields_adjust(dev_flow, inner,
1414 MLX5_IPV6_LAYER_TYPES,
1415 MLX5_IPV6_IBV_RX_HASH);
1417 case RTE_FLOW_ITEM_TYPE_TCP:
1418 flow_dv_translate_item_tcp(tmatcher->mask.buf, key, item,
1420 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1421 dev_flow->dv.hash_fields |=
1422 mlx5_flow_hashfields_adjust(dev_flow, inner,
1424 (IBV_RX_HASH_SRC_PORT_TCP |
1425 IBV_RX_HASH_DST_PORT_TCP));
1427 case RTE_FLOW_ITEM_TYPE_UDP:
1428 flow_dv_translate_item_udp(tmatcher->mask.buf, key, item,
1430 tmatcher->priority = MLX5_PRIORITY_MAP_L4;
1431 dev_flow->verbs.hash_fields |=
1432 mlx5_flow_hashfields_adjust(dev_flow, inner,
1434 (IBV_RX_HASH_SRC_PORT_UDP |
1435 IBV_RX_HASH_DST_PORT_UDP));
1437 case RTE_FLOW_ITEM_TYPE_GRE:
1438 flow_dv_translate_item_gre(tmatcher->mask.buf, key, item,
1441 case RTE_FLOW_ITEM_TYPE_NVGRE:
1442 flow_dv_translate_item_nvgre(tmatcher->mask.buf, key, item,
1445 case RTE_FLOW_ITEM_TYPE_VXLAN:
1446 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
1447 flow_dv_translate_item_vxlan(tmatcher->mask.buf, key, item,
1450 case RTE_FLOW_ITEM_TYPE_META:
1451 flow_dv_translate_item_meta(tmatcher->mask.buf, key, item);
1459 * Store the requested actions in an array.
1462 * Pointer to rte_eth_dev structure.
1464 * Flow action to translate.
1465 * @param[in, out] dev_flow
1466 * Pointer to the mlx5_flow.
1468 * Pointer to the error structure.
1471 * 0 on success, a negative errno value otherwise and rte_errno is set.
1474 flow_dv_create_action(struct rte_eth_dev *dev,
1475 const struct rte_flow_action *action,
1476 struct mlx5_flow *dev_flow,
1477 struct rte_flow_error *error)
1479 const struct rte_flow_action_queue *queue;
1480 const struct rte_flow_action_rss *rss;
1481 int actions_n = dev_flow->dv.actions_n;
1482 struct rte_flow *flow = dev_flow->flow;
1484 switch (action->type) {
1485 case RTE_FLOW_ACTION_TYPE_VOID:
1487 case RTE_FLOW_ACTION_TYPE_FLAG:
1488 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1489 dev_flow->dv.actions[actions_n].tag_value =
1490 mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
1492 flow->actions |= MLX5_FLOW_ACTION_FLAG;
1494 case RTE_FLOW_ACTION_TYPE_MARK:
1495 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_TAG;
1496 dev_flow->dv.actions[actions_n].tag_value =
1498 (((const struct rte_flow_action_mark *)
1499 (action->conf))->id);
1500 flow->actions |= MLX5_FLOW_ACTION_MARK;
1503 case RTE_FLOW_ACTION_TYPE_DROP:
1504 dev_flow->dv.actions[actions_n].type = MLX5DV_FLOW_ACTION_DROP;
1505 flow->actions |= MLX5_FLOW_ACTION_DROP;
1507 case RTE_FLOW_ACTION_TYPE_QUEUE:
1508 queue = action->conf;
1509 flow->rss.queue_num = 1;
1510 (*flow->queue)[0] = queue->index;
1511 flow->actions |= MLX5_FLOW_ACTION_QUEUE;
1513 case RTE_FLOW_ACTION_TYPE_RSS:
1516 memcpy((*flow->queue), rss->queue,
1517 rss->queue_num * sizeof(uint16_t));
1518 flow->rss.queue_num = rss->queue_num;
1519 memcpy(flow->key, rss->key, MLX5_RSS_HASH_KEY_LEN);
1520 flow->rss.types = rss->types;
1521 flow->rss.level = rss->level;
1522 /* Added to array only in apply since we need the QP */
1523 flow->actions |= MLX5_FLOW_ACTION_RSS;
1525 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
1526 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
1527 dev_flow->dv.actions[actions_n].type =
1528 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1529 dev_flow->dv.actions[actions_n].action =
1530 flow_dv_create_action_l2_encap(dev, action,
1532 if (!(dev_flow->dv.actions[actions_n].action))
1534 dev_flow->dv.encap_decap_verbs_action =
1535 dev_flow->dv.actions[actions_n].action;
1536 flow->actions |= action->type ==
1537 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
1538 MLX5_FLOW_ACTION_VXLAN_ENCAP :
1539 MLX5_FLOW_ACTION_NVGRE_ENCAP;
1542 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
1543 dev_flow->dv.actions[actions_n].type =
1544 MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
1545 dev_flow->dv.actions[actions_n].action =
1546 flow_dv_create_action_l2_decap(dev, error);
1547 if (!(dev_flow->dv.actions[actions_n].action))
1549 dev_flow->dv.encap_decap_verbs_action =
1550 dev_flow->dv.actions[actions_n].action;
1551 flow->actions |= MLX5_FLOW_ACTION_VXLAN_DECAP;
1557 dev_flow->dv.actions_n = actions_n;
1561 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
1563 #define HEADER_IS_ZERO(match_criteria, headers) \
1564 !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
1565 matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
1568 * Calculate flow matcher enable bitmap.
1570 * @param match_criteria
1571 * Pointer to flow matcher criteria.
1574 * Bitmap of enabled fields.
1577 flow_dv_matcher_enable(uint32_t *match_criteria)
1579 uint8_t match_criteria_enable;
1581 match_criteria_enable =
1582 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
1583 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
1584 match_criteria_enable |=
1585 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
1586 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
1587 match_criteria_enable |=
1588 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
1589 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
1590 match_criteria_enable |=
1591 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
1592 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
1594 return match_criteria_enable;
1598 * Register the flow matcher.
1600 * @param dev[in, out]
1601 * Pointer to rte_eth_dev structure.
1602 * @param[in, out] matcher
1603 * Pointer to flow matcher.
1604 * @parm[in, out] dev_flow
1605 * Pointer to the dev_flow.
1607 * pointer to error structure.
1610 * 0 on success otherwise -errno and errno is set.
1613 flow_dv_matcher_register(struct rte_eth_dev *dev,
1614 struct mlx5_flow_dv_matcher *matcher,
1615 struct mlx5_flow *dev_flow,
1616 struct rte_flow_error *error)
1618 struct priv *priv = dev->data->dev_private;
1619 struct mlx5_flow_dv_matcher *cache_matcher;
1620 struct mlx5dv_flow_matcher_attr dv_attr = {
1621 .type = IBV_FLOW_ATTR_NORMAL,
1622 .match_mask = (void *)&matcher->mask,
1625 /* Lookup from cache. */
1626 LIST_FOREACH(cache_matcher, &priv->matchers, next) {
1627 if (matcher->crc == cache_matcher->crc &&
1628 matcher->priority == cache_matcher->priority &&
1629 matcher->egress == cache_matcher->egress &&
1630 !memcmp((const void *)matcher->mask.buf,
1631 (const void *)cache_matcher->mask.buf,
1632 cache_matcher->mask.size)) {
1634 "priority %hd use %s matcher %p: refcnt %d++",
1635 cache_matcher->priority,
1636 cache_matcher->egress ? "tx" : "rx",
1637 (void *)cache_matcher,
1638 rte_atomic32_read(&cache_matcher->refcnt));
1639 rte_atomic32_inc(&cache_matcher->refcnt);
1640 dev_flow->dv.matcher = cache_matcher;
1644 /* Register new matcher. */
1645 cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
1647 return rte_flow_error_set(error, ENOMEM,
1648 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1649 "cannot allocate matcher memory");
1650 *cache_matcher = *matcher;
1651 dv_attr.match_criteria_enable =
1652 flow_dv_matcher_enable(cache_matcher->mask.buf);
1653 dv_attr.priority = matcher->priority;
1654 if (matcher->egress)
1655 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
1656 cache_matcher->matcher_object =
1657 mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr);
1658 if (!cache_matcher->matcher_object) {
1659 rte_free(cache_matcher);
1660 return rte_flow_error_set(error, ENOMEM,
1661 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1662 NULL, "cannot create matcher");
1664 rte_atomic32_inc(&cache_matcher->refcnt);
1665 LIST_INSERT_HEAD(&priv->matchers, cache_matcher, next);
1666 dev_flow->dv.matcher = cache_matcher;
1667 DRV_LOG(DEBUG, "priority %hd new %s matcher %p: refcnt %d",
1668 cache_matcher->priority,
1669 cache_matcher->egress ? "tx" : "rx", (void *)cache_matcher,
1670 rte_atomic32_read(&cache_matcher->refcnt));
1675 * Fill the flow with DV spec.
1678 * Pointer to rte_eth_dev structure.
1679 * @param[in, out] dev_flow
1680 * Pointer to the sub flow.
1682 * Pointer to the flow attributes.
1684 * Pointer to the list of items.
1685 * @param[in] actions
1686 * Pointer to the list of actions.
1688 * Pointer to the error structure.
1691 * 0 on success, a negative errno value otherwise and rte_ernno is set.
1694 flow_dv_translate(struct rte_eth_dev *dev,
1695 struct mlx5_flow *dev_flow,
1696 const struct rte_flow_attr *attr,
1697 const struct rte_flow_item items[],
1698 const struct rte_flow_action actions[] __rte_unused,
1699 struct rte_flow_error *error)
1701 struct priv *priv = dev->data->dev_private;
1702 uint64_t priority = attr->priority;
1703 struct mlx5_flow_dv_matcher matcher = {
1705 .size = sizeof(matcher.mask.buf),
1708 void *match_value = dev_flow->dv.value.buf;
1711 if (priority == MLX5_FLOW_PRIO_RSVD)
1712 priority = priv->config.flow_prio - 1;
1713 for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1714 tunnel = !!(dev_flow->layers & MLX5_FLOW_LAYER_TUNNEL);
1715 flow_dv_create_item(&matcher, match_value, items, dev_flow,
1718 matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
1720 if (priority == MLX5_FLOW_PRIO_RSVD)
1721 priority = priv->config.flow_prio - 1;
1722 matcher.priority = mlx5_flow_adjust_priority(dev, priority,
1724 matcher.egress = attr->egress;
1725 if (flow_dv_matcher_register(dev, &matcher, dev_flow, error))
1727 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++)
1728 if (flow_dv_create_action(dev, actions, dev_flow, error))
1734 * Apply the flow to the NIC.
1737 * Pointer to the Ethernet device structure.
1738 * @param[in, out] flow
1739 * Pointer to flow structure.
1741 * Pointer to error structure.
1744 * 0 on success, a negative errno value otherwise and rte_errno is set.
1747 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
1748 struct rte_flow_error *error)
1750 struct mlx5_flow_dv *dv;
1751 struct mlx5_flow *dev_flow;
1755 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1758 if (flow->actions & MLX5_FLOW_ACTION_DROP) {
1759 dv->hrxq = mlx5_hrxq_drop_new(dev);
1763 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1764 "cannot get drop hash queue");
1767 dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1768 dv->actions[n].qp = dv->hrxq->qp;
1770 } else if (flow->actions &
1771 (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
1772 struct mlx5_hrxq *hrxq;
1773 hrxq = mlx5_hrxq_get(dev, flow->key,
1774 MLX5_RSS_HASH_KEY_LEN,
1777 flow->rss.queue_num);
1779 hrxq = mlx5_hrxq_new
1780 (dev, flow->key, MLX5_RSS_HASH_KEY_LEN,
1781 dv->hash_fields, (*flow->queue),
1782 flow->rss.queue_num,
1783 !!(dev_flow->layers &
1784 MLX5_FLOW_LAYER_TUNNEL));
1788 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1789 "cannot get hash queue");
1793 dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
1794 dv->actions[n].qp = hrxq->qp;
1798 mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
1799 (void *)&dv->value, n,
1802 rte_flow_error_set(error, errno,
1803 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1805 "hardware refuses to create flow");
1811 err = rte_errno; /* Save rte_errno before cleanup. */
1812 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1813 struct mlx5_flow_dv *dv = &dev_flow->dv;
1815 if (flow->actions & MLX5_FLOW_ACTION_DROP)
1816 mlx5_hrxq_drop_release(dev);
1818 mlx5_hrxq_release(dev, dv->hrxq);
1822 rte_errno = err; /* Restore rte_errno. */
1827 * Release the flow matcher.
1830 * Pointer to Ethernet device.
1832 * Pointer to mlx5_flow.
1835 * 1 while a reference on it exists, 0 when freed.
1838 flow_dv_matcher_release(struct rte_eth_dev *dev,
1839 struct mlx5_flow *flow)
1841 struct mlx5_flow_dv_matcher *matcher = flow->dv.matcher;
1843 assert(matcher->matcher_object);
1844 DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
1845 dev->data->port_id, (void *)matcher,
1846 rte_atomic32_read(&matcher->refcnt));
1847 if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
1848 claim_zero(mlx5_glue->dv_destroy_flow_matcher
1849 (matcher->matcher_object));
1850 LIST_REMOVE(matcher, next);
1852 DRV_LOG(DEBUG, "port %u matcher %p: removed",
1853 dev->data->port_id, (void *)matcher);
1860 * Remove the flow from the NIC but keeps it in memory.
1863 * Pointer to Ethernet device.
1864 * @param[in, out] flow
1865 * Pointer to flow structure.
1868 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
1870 struct mlx5_flow_dv *dv;
1871 struct mlx5_flow *dev_flow;
1875 LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
1878 claim_zero(mlx5_glue->destroy_flow(dv->flow));
1882 if (flow->actions & MLX5_FLOW_ACTION_DROP)
1883 mlx5_hrxq_drop_release(dev);
1885 mlx5_hrxq_release(dev, dv->hrxq);
1890 flow->counter = NULL;
1894 * Remove the flow from the NIC and the memory.
1897 * Pointer to the Ethernet device structure.
1898 * @param[in, out] flow
1899 * Pointer to flow structure.
1902 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
1904 struct mlx5_flow *dev_flow;
1908 flow_dv_remove(dev, flow);
1909 while (!LIST_EMPTY(&flow->dev_flows)) {
1910 dev_flow = LIST_FIRST(&flow->dev_flows);
1911 LIST_REMOVE(dev_flow, next);
1912 if (dev_flow->dv.matcher)
1913 flow_dv_matcher_release(dev, dev_flow);
1914 if (dev_flow->dv.encap_decap_verbs_action) {
1915 claim_zero(mlx5_glue->destroy_flow_action
1916 (dev_flow->dv.encap_decap_verbs_action));
1917 dev_flow->dv.encap_decap_verbs_action = NULL;
1926 * @see rte_flow_query()
1930 flow_dv_query(struct rte_eth_dev *dev __rte_unused,
1931 struct rte_flow *flow __rte_unused,
1932 const struct rte_flow_action *actions __rte_unused,
1933 void *data __rte_unused,
1934 struct rte_flow_error *error __rte_unused)
1936 rte_errno = ENOTSUP;
1941 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
1942 .validate = flow_dv_validate,
1943 .prepare = flow_dv_prepare,
1944 .translate = flow_dv_translate,
1945 .apply = flow_dv_apply,
1946 .remove = flow_dv_remove,
1947 .destroy = flow_dv_destroy,
1948 .query = flow_dv_query,
1951 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */