1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox Technologies, Ltd
11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
13 #pragma GCC diagnostic ignored "-Wpedantic"
15 #include <infiniband/verbs.h>
17 #pragma GCC diagnostic error "-Wpedantic"
20 #include <rte_common.h>
21 #include <rte_ether.h>
22 #include <rte_eth_ctrl.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
30 #include "mlx5_defs.h"
32 #include "mlx5_glue.h"
34 /* Flow priority for control plane flows. */
35 #define MLX5_CTRL_FLOW_PRIORITY 1
37 /* Internet Protocol versions. */
42 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
43 struct ibv_flow_spec_counter_action {
48 /* Dev ops structure defined in mlx5.c */
49 extern const struct eth_dev_ops mlx5_dev_ops;
50 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
52 /** Structure give to the conversion functions. */
53 struct mlx5_flow_data {
54 struct mlx5_flow_parse *parser; /** Parser context. */
55 struct rte_flow_error *error; /** Error context. */
59 mlx5_flow_create_eth(const struct rte_flow_item *item,
60 const void *default_mask,
61 struct mlx5_flow_data *data);
64 mlx5_flow_create_vlan(const struct rte_flow_item *item,
65 const void *default_mask,
66 struct mlx5_flow_data *data);
69 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
70 const void *default_mask,
71 struct mlx5_flow_data *data);
74 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
75 const void *default_mask,
76 struct mlx5_flow_data *data);
79 mlx5_flow_create_udp(const struct rte_flow_item *item,
80 const void *default_mask,
81 struct mlx5_flow_data *data);
84 mlx5_flow_create_tcp(const struct rte_flow_item *item,
85 const void *default_mask,
86 struct mlx5_flow_data *data);
89 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
90 const void *default_mask,
91 struct mlx5_flow_data *data);
94 mlx5_flow_create_gre(const struct rte_flow_item *item,
95 const void *default_mask,
96 struct mlx5_flow_data *data);
98 struct mlx5_flow_parse;
101 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
105 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
108 mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
110 /* Hash RX queue types. */
121 /* Initialization data for hash RX queue. */
122 struct hash_rxq_init {
123 uint64_t hash_fields; /* Fields that participate in the hash. */
124 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
125 unsigned int flow_priority; /* Flow priority to use. */
126 unsigned int ip_version; /* Internet protocol. */
129 /* Initialization data for hash RX queues. */
130 const struct hash_rxq_init hash_rxq_init[] = {
132 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
133 IBV_RX_HASH_DST_IPV4 |
134 IBV_RX_HASH_SRC_PORT_TCP |
135 IBV_RX_HASH_DST_PORT_TCP),
136 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
138 .ip_version = MLX5_IPV4,
141 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
142 IBV_RX_HASH_DST_IPV4 |
143 IBV_RX_HASH_SRC_PORT_UDP |
144 IBV_RX_HASH_DST_PORT_UDP),
145 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
147 .ip_version = MLX5_IPV4,
150 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
151 IBV_RX_HASH_DST_IPV4),
152 .dpdk_rss_hf = (ETH_RSS_IPV4 |
155 .ip_version = MLX5_IPV4,
158 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
159 IBV_RX_HASH_DST_IPV6 |
160 IBV_RX_HASH_SRC_PORT_TCP |
161 IBV_RX_HASH_DST_PORT_TCP),
162 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
164 .ip_version = MLX5_IPV6,
167 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
168 IBV_RX_HASH_DST_IPV6 |
169 IBV_RX_HASH_SRC_PORT_UDP |
170 IBV_RX_HASH_DST_PORT_UDP),
171 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
173 .ip_version = MLX5_IPV6,
176 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
177 IBV_RX_HASH_DST_IPV6),
178 .dpdk_rss_hf = (ETH_RSS_IPV6 |
181 .ip_version = MLX5_IPV6,
190 /* Number of entries in hash_rxq_init[]. */
191 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
193 /** Structure for holding counter stats. */
194 struct mlx5_flow_counter_stats {
195 uint64_t hits; /**< Number of packets matched by the rule. */
196 uint64_t bytes; /**< Number of bytes matched by the rule. */
199 /** Structure for Drop queue. */
200 struct mlx5_hrxq_drop {
201 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
202 struct ibv_qp *qp; /**< Verbs queue pair. */
203 struct ibv_wq *wq; /**< Verbs work queue. */
204 struct ibv_cq *cq; /**< Verbs completion queue. */
207 /* Flows structures. */
209 uint64_t hash_fields; /**< Fields that participate in the hash. */
210 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
211 struct ibv_flow *ibv_flow; /**< Verbs flow. */
212 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
215 /* Drop flows structures. */
216 struct mlx5_flow_drop {
217 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
218 struct ibv_flow *ibv_flow; /**< Verbs flow. */
222 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
223 uint32_t mark:1; /**< Set if the flow is marked. */
224 uint32_t drop:1; /**< Drop queue. */
225 struct rte_flow_action_rss rss_conf; /**< RSS configuration */
226 uint16_t (*queues)[]; /**< Queues indexes to use. */
227 uint8_t rss_key[40]; /**< copy of the RSS key. */
228 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
229 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
230 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
231 /**< Flow with Rx queue. */
234 /** Static initializer for items. */
236 (const enum rte_flow_item_type []){ \
237 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
240 #define IS_TUNNEL(type) ( \
241 (type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
242 (type) == RTE_FLOW_ITEM_TYPE_GRE)
244 /** Structure to generate a simple graph of layers supported by the NIC. */
245 struct mlx5_flow_items {
246 /** List of possible actions for these items. */
247 const enum rte_flow_action_type *const actions;
248 /** Bit-masks corresponding to the possibilities for the item. */
251 * Default bit-masks to use when item->mask is not provided. When
252 * \default_mask is also NULL, the full supported bit-mask (\mask) is
255 const void *default_mask;
256 /** Bit-masks size in bytes. */
257 const unsigned int mask_sz;
259 * Conversion function from rte_flow to NIC specific flow.
262 * rte_flow item to convert.
263 * @param default_mask
264 * Default bit-masks to use when item->mask is not provided.
266 * Internal structure to store the conversion.
269 * 0 on success, a negative errno value otherwise and rte_errno is
272 int (*convert)(const struct rte_flow_item *item,
273 const void *default_mask,
274 struct mlx5_flow_data *data);
275 /** Size in bytes of the destination structure. */
276 const unsigned int dst_sz;
277 /** List of possible following items. */
278 const enum rte_flow_item_type *const items;
281 /** Valid action for this PMD. */
282 static const enum rte_flow_action_type valid_actions[] = {
283 RTE_FLOW_ACTION_TYPE_DROP,
284 RTE_FLOW_ACTION_TYPE_QUEUE,
285 RTE_FLOW_ACTION_TYPE_MARK,
286 RTE_FLOW_ACTION_TYPE_FLAG,
287 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
288 RTE_FLOW_ACTION_TYPE_COUNT,
290 RTE_FLOW_ACTION_TYPE_END,
293 /** Graph of supported items and associated actions. */
294 static const struct mlx5_flow_items mlx5_flow_items[] = {
295 [RTE_FLOW_ITEM_TYPE_END] = {
296 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
297 RTE_FLOW_ITEM_TYPE_VXLAN,
298 RTE_FLOW_ITEM_TYPE_GRE),
300 [RTE_FLOW_ITEM_TYPE_ETH] = {
301 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
302 RTE_FLOW_ITEM_TYPE_IPV4,
303 RTE_FLOW_ITEM_TYPE_IPV6),
304 .actions = valid_actions,
305 .mask = &(const struct rte_flow_item_eth){
306 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
307 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
310 .default_mask = &rte_flow_item_eth_mask,
311 .mask_sz = sizeof(struct rte_flow_item_eth),
312 .convert = mlx5_flow_create_eth,
313 .dst_sz = sizeof(struct ibv_flow_spec_eth),
315 [RTE_FLOW_ITEM_TYPE_VLAN] = {
316 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
317 RTE_FLOW_ITEM_TYPE_IPV6),
318 .actions = valid_actions,
319 .mask = &(const struct rte_flow_item_vlan){
323 .default_mask = &rte_flow_item_vlan_mask,
324 .mask_sz = sizeof(struct rte_flow_item_vlan),
325 .convert = mlx5_flow_create_vlan,
328 [RTE_FLOW_ITEM_TYPE_IPV4] = {
329 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
330 RTE_FLOW_ITEM_TYPE_TCP,
331 RTE_FLOW_ITEM_TYPE_GRE),
332 .actions = valid_actions,
333 .mask = &(const struct rte_flow_item_ipv4){
337 .type_of_service = -1,
341 .default_mask = &rte_flow_item_ipv4_mask,
342 .mask_sz = sizeof(struct rte_flow_item_ipv4),
343 .convert = mlx5_flow_create_ipv4,
344 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
346 [RTE_FLOW_ITEM_TYPE_IPV6] = {
347 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
348 RTE_FLOW_ITEM_TYPE_TCP,
349 RTE_FLOW_ITEM_TYPE_GRE),
350 .actions = valid_actions,
351 .mask = &(const struct rte_flow_item_ipv6){
354 0xff, 0xff, 0xff, 0xff,
355 0xff, 0xff, 0xff, 0xff,
356 0xff, 0xff, 0xff, 0xff,
357 0xff, 0xff, 0xff, 0xff,
360 0xff, 0xff, 0xff, 0xff,
361 0xff, 0xff, 0xff, 0xff,
362 0xff, 0xff, 0xff, 0xff,
363 0xff, 0xff, 0xff, 0xff,
370 .default_mask = &rte_flow_item_ipv6_mask,
371 .mask_sz = sizeof(struct rte_flow_item_ipv6),
372 .convert = mlx5_flow_create_ipv6,
373 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
375 [RTE_FLOW_ITEM_TYPE_UDP] = {
376 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
377 .actions = valid_actions,
378 .mask = &(const struct rte_flow_item_udp){
384 .default_mask = &rte_flow_item_udp_mask,
385 .mask_sz = sizeof(struct rte_flow_item_udp),
386 .convert = mlx5_flow_create_udp,
387 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
389 [RTE_FLOW_ITEM_TYPE_TCP] = {
390 .actions = valid_actions,
391 .mask = &(const struct rte_flow_item_tcp){
397 .default_mask = &rte_flow_item_tcp_mask,
398 .mask_sz = sizeof(struct rte_flow_item_tcp),
399 .convert = mlx5_flow_create_tcp,
400 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
402 [RTE_FLOW_ITEM_TYPE_GRE] = {
403 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
404 RTE_FLOW_ITEM_TYPE_IPV4,
405 RTE_FLOW_ITEM_TYPE_IPV6),
406 .actions = valid_actions,
407 .mask = &(const struct rte_flow_item_gre){
410 .default_mask = &rte_flow_item_gre_mask,
411 .mask_sz = sizeof(struct rte_flow_item_gre),
412 .convert = mlx5_flow_create_gre,
413 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
415 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
416 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
417 .actions = valid_actions,
418 .mask = &(const struct rte_flow_item_vxlan){
419 .vni = "\xff\xff\xff",
421 .default_mask = &rte_flow_item_vxlan_mask,
422 .mask_sz = sizeof(struct rte_flow_item_vxlan),
423 .convert = mlx5_flow_create_vxlan,
424 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
428 /** Structure to pass to the conversion function. */
429 struct mlx5_flow_parse {
430 uint32_t inner; /**< Verbs value, set once tunnel is encountered. */
432 /**< Whether resources should remain after a validate. */
433 uint32_t drop:1; /**< Target is a drop queue. */
434 uint32_t mark:1; /**< Mark is present in the flow. */
435 uint32_t count:1; /**< Count is present in the flow. */
436 uint32_t mark_id; /**< Mark identifier. */
437 struct rte_flow_action_rss rss_conf; /**< RSS configuration */
438 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
439 uint8_t rss_key[40]; /**< copy of the RSS key. */
440 enum hash_rxq_type layer; /**< Last pattern layer detected. */
441 enum hash_rxq_type out_layer; /**< Last outer pattern layer detected. */
442 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
444 struct ibv_flow_attr *ibv_attr;
445 /**< Pointer to Verbs attributes. */
447 /**< Current position or total size of the attribute. */
448 } queue[RTE_DIM(hash_rxq_init)];
451 static const struct rte_flow_ops mlx5_flow_ops = {
452 .validate = mlx5_flow_validate,
453 .create = mlx5_flow_create,
454 .destroy = mlx5_flow_destroy,
455 .flush = mlx5_flow_flush,
456 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
457 .query = mlx5_flow_query,
461 .isolate = mlx5_flow_isolate,
464 /* Convert FDIR request to Generic flow. */
466 struct rte_flow_attr attr;
467 struct rte_flow_action actions[2];
468 struct rte_flow_item items[4];
469 struct rte_flow_item_eth l2;
470 struct rte_flow_item_eth l2_mask;
472 struct rte_flow_item_ipv4 ipv4;
473 struct rte_flow_item_ipv6 ipv6;
476 struct rte_flow_item_ipv4 ipv4;
477 struct rte_flow_item_ipv6 ipv6;
480 struct rte_flow_item_udp udp;
481 struct rte_flow_item_tcp tcp;
484 struct rte_flow_item_udp udp;
485 struct rte_flow_item_tcp tcp;
487 struct rte_flow_action_queue queue;
490 /* Verbs specification header. */
491 struct ibv_spec_header {
492 enum ibv_flow_spec_type type;
497 * Check support for a given item.
500 * Item specification.
502 * Bit-masks covering supported fields to compare with spec, last and mask in
505 * Bit-Mask size in bytes.
508 * 0 on success, a negative errno value otherwise and rte_errno is set.
511 mlx5_flow_item_validate(const struct rte_flow_item *item,
512 const uint8_t *mask, unsigned int size)
514 if (!item->spec && (item->mask || item->last)) {
518 if (item->spec && !item->mask) {
520 const uint8_t *spec = item->spec;
522 for (i = 0; i < size; ++i)
523 if ((spec[i] | mask[i]) != mask[i]) {
528 if (item->last && !item->mask) {
530 const uint8_t *spec = item->last;
532 for (i = 0; i < size; ++i)
533 if ((spec[i] | mask[i]) != mask[i]) {
540 const uint8_t *spec = item->spec;
542 for (i = 0; i < size; ++i)
543 if ((spec[i] | mask[i]) != mask[i]) {
548 if (item->spec && item->last) {
551 const uint8_t *apply = mask;
557 for (i = 0; i < size; ++i) {
558 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
559 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
561 ret = memcmp(spec, last, size);
571 * Extract attribute to the parser.
574 * Flow rule attributes.
576 * Perform verbose error reporting if not NULL.
579 * 0 on success, a negative errno value otherwise and rte_errno is set.
582 mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
583 struct rte_flow_error *error)
586 rte_flow_error_set(error, ENOTSUP,
587 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
589 "groups are not supported");
592 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
593 rte_flow_error_set(error, ENOTSUP,
594 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
596 "priorities are not supported");
600 rte_flow_error_set(error, ENOTSUP,
601 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
603 "egress is not supported");
606 if (attr->transfer) {
607 rte_flow_error_set(error, ENOTSUP,
608 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
610 "transfer is not supported");
613 if (!attr->ingress) {
614 rte_flow_error_set(error, ENOTSUP,
615 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
617 "only ingress is supported");
624 * Extract actions request to the parser.
627 * Pointer to Ethernet device.
629 * Associated actions (list terminated by the END action).
631 * Perform verbose error reporting if not NULL.
632 * @param[in, out] parser
633 * Internal parser structure.
636 * 0 on success, a negative errno value otherwise and rte_errno is set.
639 mlx5_flow_convert_actions(struct rte_eth_dev *dev,
640 const struct rte_flow_action actions[],
641 struct rte_flow_error *error,
642 struct mlx5_flow_parse *parser)
644 enum { FATE = 1, MARK = 2, COUNT = 4, };
645 uint32_t overlap = 0;
646 struct priv *priv = dev->data->dev_private;
648 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
649 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
651 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
653 goto exit_action_overlap;
656 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
657 const struct rte_flow_action_queue *queue =
658 (const struct rte_flow_action_queue *)
662 goto exit_action_overlap;
664 if (!queue || (queue->index > (priv->rxqs_n - 1)))
665 goto exit_action_not_supported;
666 parser->queues[0] = queue->index;
667 parser->rss_conf = (struct rte_flow_action_rss){
669 .queue = parser->queues,
671 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
672 const struct rte_flow_action_rss *rss =
673 (const struct rte_flow_action_rss *)
675 const uint8_t *rss_key;
676 uint32_t rss_key_len;
680 goto exit_action_overlap;
683 rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
684 rte_flow_error_set(error, EINVAL,
685 RTE_FLOW_ERROR_TYPE_ACTION,
687 "the only supported RSS hash"
688 " function is Toeplitz");
692 rte_flow_error_set(error, EINVAL,
693 RTE_FLOW_ERROR_TYPE_ACTION,
695 "a nonzero RSS encapsulation"
696 " level is not supported");
699 if (rss->types & MLX5_RSS_HF_MASK) {
700 rte_flow_error_set(error, EINVAL,
701 RTE_FLOW_ERROR_TYPE_ACTION,
703 "unsupported RSS type"
708 rss_key_len = rss->key_len;
711 rss_key_len = rss_hash_default_key_len;
712 rss_key = rss_hash_default_key;
714 if (rss_key_len != RTE_DIM(parser->rss_key)) {
715 rte_flow_error_set(error, EINVAL,
716 RTE_FLOW_ERROR_TYPE_ACTION,
718 "RSS hash key must be"
719 " exactly 40 bytes long");
722 if (!rss->queue_num) {
723 rte_flow_error_set(error, EINVAL,
724 RTE_FLOW_ERROR_TYPE_ACTION,
729 if (rss->queue_num > RTE_DIM(parser->queues)) {
730 rte_flow_error_set(error, EINVAL,
731 RTE_FLOW_ERROR_TYPE_ACTION,
733 "too many queues for RSS"
737 for (n = 0; n < rss->queue_num; ++n) {
738 if (rss->queue[n] >= priv->rxqs_n) {
739 rte_flow_error_set(error, EINVAL,
740 RTE_FLOW_ERROR_TYPE_ACTION,
742 "queue id > number of"
747 parser->rss_conf = (struct rte_flow_action_rss){
748 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
751 .key_len = rss_key_len,
752 .queue_num = rss->queue_num,
753 .key = memcpy(parser->rss_key, rss_key,
754 sizeof(*rss_key) * rss_key_len),
755 .queue = memcpy(parser->queues, rss->queue,
756 sizeof(*rss->queue) *
759 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
760 const struct rte_flow_action_mark *mark =
761 (const struct rte_flow_action_mark *)
765 goto exit_action_overlap;
768 rte_flow_error_set(error, EINVAL,
769 RTE_FLOW_ERROR_TYPE_ACTION,
771 "mark must be defined");
773 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
774 rte_flow_error_set(error, ENOTSUP,
775 RTE_FLOW_ERROR_TYPE_ACTION,
777 "mark must be between 0"
782 parser->mark_id = mark->id;
783 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
785 goto exit_action_overlap;
788 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
789 priv->config.flow_counter_en) {
791 goto exit_action_overlap;
795 goto exit_action_not_supported;
798 /* When fate is unknown, drop traffic. */
799 if (!(overlap & FATE))
801 if (parser->drop && parser->mark)
803 if (!parser->rss_conf.queue_num && !parser->drop) {
804 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
805 NULL, "no valid action");
809 exit_action_not_supported:
810 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
811 actions, "action not supported");
814 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
815 actions, "overlapping actions are not supported");
823 * Pattern specification (list terminated by the END pattern item).
825 * Perform verbose error reporting if not NULL.
826 * @param[in, out] parser
827 * Internal parser structure.
830 * 0 on success, a negative errno value otherwise and rte_errno is set.
833 mlx5_flow_convert_items_validate(const struct rte_flow_item items[],
834 struct rte_flow_error *error,
835 struct mlx5_flow_parse *parser)
837 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
841 /* Initialise the offsets to start after verbs attribute. */
842 for (i = 0; i != hash_rxq_init_n; ++i)
843 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
844 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
845 const struct mlx5_flow_items *token = NULL;
848 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
852 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
854 if (cur_item->items[i] == items->type) {
855 token = &mlx5_flow_items[items->type];
861 goto exit_item_not_supported;
864 ret = mlx5_flow_item_validate(items,
865 (const uint8_t *)cur_item->mask,
868 goto exit_item_not_supported;
869 if (IS_TUNNEL(items->type)) {
871 rte_flow_error_set(error, ENOTSUP,
872 RTE_FLOW_ERROR_TYPE_ITEM,
874 "Cannot recognize multiple"
875 " tunnel encapsulations.");
878 parser->inner = IBV_FLOW_SPEC_INNER;
881 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
883 for (n = 0; n != hash_rxq_init_n; ++n)
884 parser->queue[n].offset += cur_item->dst_sz;
888 parser->queue[HASH_RXQ_ETH].offset +=
889 sizeof(struct ibv_flow_spec_action_drop);
892 for (i = 0; i != hash_rxq_init_n; ++i)
893 parser->queue[i].offset +=
894 sizeof(struct ibv_flow_spec_action_tag);
897 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
899 for (i = 0; i != hash_rxq_init_n; ++i)
900 parser->queue[i].offset += size;
903 exit_item_not_supported:
904 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
905 items, "item not supported");
909 * Allocate memory space to store verbs flow attributes.
912 * Amount of byte to allocate.
914 * Perform verbose error reporting if not NULL.
917 * A verbs flow attribute on success, NULL otherwise and rte_errno is set.
919 static struct ibv_flow_attr *
920 mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error)
922 struct ibv_flow_attr *ibv_attr;
924 ibv_attr = rte_calloc(__func__, 1, size, 0);
926 rte_flow_error_set(error, ENOMEM,
927 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
929 "cannot allocate verbs spec attributes");
936 * Make inner packet matching with an higher priority from the non Inner
940 * Pointer to Ethernet device.
941 * @param[in, out] parser
942 * Internal parser structure.
944 * User flow attribute.
947 mlx5_flow_update_priority(struct rte_eth_dev *dev,
948 struct mlx5_flow_parse *parser,
949 const struct rte_flow_attr *attr)
951 struct priv *priv = dev->data->dev_private;
955 /* 8 priorities >= 16 priorities
956 * Control flow: 4-7 8-15
957 * User normal flow: 1-3 4-7
958 * User tunnel flow: 0-2 0-3
960 priority = attr->priority * MLX5_VERBS_FLOW_PRIO_8;
961 if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
964 * Lower non-tunnel flow Verbs priority 1 if only support 8 Verbs
965 * priorities, lower 4 otherwise.
967 if (!parser->inner) {
968 if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
971 priority += MLX5_VERBS_FLOW_PRIO_8 / 2;
974 parser->queue[HASH_RXQ_ETH].ibv_attr->priority = priority +
975 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
978 for (i = 0; i != hash_rxq_init_n; ++i) {
979 if (!parser->queue[i].ibv_attr)
981 parser->queue[i].ibv_attr->priority = priority +
982 hash_rxq_init[i].flow_priority;
987 * Finalise verbs flow attributes.
989 * @param[in, out] parser
990 * Internal parser structure.
993 mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
995 const unsigned int ipv4 =
996 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
997 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
998 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
999 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
1000 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
1001 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
1004 /* Remove any other flow not matching the pattern. */
1005 if (parser->rss_conf.queue_num == 1 && !parser->rss_conf.types) {
1006 for (i = 0; i != hash_rxq_init_n; ++i) {
1007 if (i == HASH_RXQ_ETH)
1009 rte_free(parser->queue[i].ibv_attr);
1010 parser->queue[i].ibv_attr = NULL;
1014 if (parser->layer == HASH_RXQ_ETH) {
1018 * This layer becomes useless as the pattern define under
1021 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
1022 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
1024 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
1025 for (i = ohmin; i != (ohmax + 1); ++i) {
1026 if (!parser->queue[i].ibv_attr)
1028 rte_free(parser->queue[i].ibv_attr);
1029 parser->queue[i].ibv_attr = NULL;
1031 /* Remove impossible flow according to the RSS configuration. */
1032 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
1033 parser->rss_conf.types) {
1034 /* Remove any other flow. */
1035 for (i = hmin; i != (hmax + 1); ++i) {
1036 if ((i == parser->layer) ||
1037 (!parser->queue[i].ibv_attr))
1039 rte_free(parser->queue[i].ibv_attr);
1040 parser->queue[i].ibv_attr = NULL;
1042 } else if (!parser->queue[ip].ibv_attr) {
1043 /* no RSS possible with the current configuration. */
1044 parser->rss_conf.queue_num = 1;
1049 * Fill missing layers in verbs specifications, or compute the correct
1050 * offset to allocate the memory space for the attributes and
1053 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
1055 struct ibv_flow_spec_ipv4_ext ipv4;
1056 struct ibv_flow_spec_ipv6 ipv6;
1057 struct ibv_flow_spec_tcp_udp udp_tcp;
1062 if (i == parser->layer)
1064 if (parser->layer == HASH_RXQ_ETH) {
1065 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
1066 size = sizeof(struct ibv_flow_spec_ipv4_ext);
1067 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
1068 .type = IBV_FLOW_SPEC_IPV4_EXT,
1072 size = sizeof(struct ibv_flow_spec_ipv6);
1073 specs.ipv6 = (struct ibv_flow_spec_ipv6){
1074 .type = IBV_FLOW_SPEC_IPV6,
1078 if (parser->queue[i].ibv_attr) {
1079 dst = (void *)((uintptr_t)
1080 parser->queue[i].ibv_attr +
1081 parser->queue[i].offset);
1082 memcpy(dst, &specs, size);
1083 ++parser->queue[i].ibv_attr->num_of_specs;
1085 parser->queue[i].offset += size;
1087 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1088 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1089 size = sizeof(struct ibv_flow_spec_tcp_udp);
1090 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1091 .type = ((i == HASH_RXQ_UDPV4 ||
1092 i == HASH_RXQ_UDPV6) ?
1097 if (parser->queue[i].ibv_attr) {
1098 dst = (void *)((uintptr_t)
1099 parser->queue[i].ibv_attr +
1100 parser->queue[i].offset);
1101 memcpy(dst, &specs, size);
1102 ++parser->queue[i].ibv_attr->num_of_specs;
1104 parser->queue[i].offset += size;
1110 * Validate and convert a flow supported by the NIC.
1113 * Pointer to Ethernet device.
1115 * Flow rule attributes.
1116 * @param[in] pattern
1117 * Pattern specification (list terminated by the END pattern item).
1118 * @param[in] actions
1119 * Associated actions (list terminated by the END action).
1121 * Perform verbose error reporting if not NULL.
1122 * @param[in, out] parser
1123 * Internal parser structure.
1126 * 0 on success, a negative errno value otherwise and rte_errno is set.
1129 mlx5_flow_convert(struct rte_eth_dev *dev,
1130 const struct rte_flow_attr *attr,
1131 const struct rte_flow_item items[],
1132 const struct rte_flow_action actions[],
1133 struct rte_flow_error *error,
1134 struct mlx5_flow_parse *parser)
1136 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1140 /* First step. Validate the attributes, items and actions. */
1141 *parser = (struct mlx5_flow_parse){
1142 .create = parser->create,
1143 .layer = HASH_RXQ_ETH,
1144 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1146 ret = mlx5_flow_convert_attributes(attr, error);
1149 ret = mlx5_flow_convert_actions(dev, actions, error, parser);
1152 ret = mlx5_flow_convert_items_validate(items, error, parser);
1155 mlx5_flow_convert_finalise(parser);
1158 * Allocate the memory space to store verbs specifications.
1161 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1163 parser->queue[HASH_RXQ_ETH].ibv_attr =
1164 mlx5_flow_convert_allocate(offset, error);
1165 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1167 parser->queue[HASH_RXQ_ETH].offset =
1168 sizeof(struct ibv_flow_attr);
1170 for (i = 0; i != hash_rxq_init_n; ++i) {
1171 unsigned int offset;
1173 if (!(parser->rss_conf.types &
1174 hash_rxq_init[i].dpdk_rss_hf) &&
1175 (i != HASH_RXQ_ETH))
1177 offset = parser->queue[i].offset;
1178 parser->queue[i].ibv_attr =
1179 mlx5_flow_convert_allocate(offset, error);
1180 if (!parser->queue[i].ibv_attr)
1182 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1185 /* Third step. Conversion parse, fill the specifications. */
1187 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1188 struct mlx5_flow_data data = {
1193 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1195 cur_item = &mlx5_flow_items[items->type];
1196 ret = cur_item->convert(items,
1197 (cur_item->default_mask ?
1198 cur_item->default_mask :
1205 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1206 if (parser->count && parser->create) {
1207 mlx5_flow_create_count(dev, parser);
1209 goto exit_count_error;
1212 * Last step. Complete missing specification to reach the RSS
1216 mlx5_flow_convert_finalise(parser);
1217 mlx5_flow_update_priority(dev, parser, attr);
1219 /* Only verification is expected, all resources should be released. */
1220 if (!parser->create) {
1221 for (i = 0; i != hash_rxq_init_n; ++i) {
1222 if (parser->queue[i].ibv_attr) {
1223 rte_free(parser->queue[i].ibv_attr);
1224 parser->queue[i].ibv_attr = NULL;
1230 for (i = 0; i != hash_rxq_init_n; ++i) {
1231 if (parser->queue[i].ibv_attr) {
1232 rte_free(parser->queue[i].ibv_attr);
1233 parser->queue[i].ibv_attr = NULL;
1236 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1237 NULL, "cannot allocate verbs spec attributes");
1240 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1241 NULL, "cannot create counter");
1246 * Copy the specification created into the flow.
1249 * Internal parser structure.
1251 * Create specification.
1253 * Size in bytes of the specification to copy.
1256 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1262 for (i = 0; i != hash_rxq_init_n; ++i) {
1263 if (!parser->queue[i].ibv_attr)
1265 /* Specification must be the same l3 type or none. */
1266 if (parser->layer == HASH_RXQ_ETH ||
1267 (hash_rxq_init[parser->layer].ip_version ==
1268 hash_rxq_init[i].ip_version) ||
1269 (hash_rxq_init[i].ip_version == 0)) {
1270 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1271 parser->queue[i].offset);
1272 memcpy(dst, src, size);
1273 ++parser->queue[i].ibv_attr->num_of_specs;
1274 parser->queue[i].offset += size;
1280 * Convert Ethernet item to Verbs specification.
1283 * Item specification.
1284 * @param default_mask[in]
1285 * Default bit-masks to use when item->mask is not provided.
1286 * @param data[in, out]
1290 * 0 on success, a negative errno value otherwise and rte_errno is set.
1293 mlx5_flow_create_eth(const struct rte_flow_item *item,
1294 const void *default_mask,
1295 struct mlx5_flow_data *data)
1297 const struct rte_flow_item_eth *spec = item->spec;
1298 const struct rte_flow_item_eth *mask = item->mask;
1299 struct mlx5_flow_parse *parser = data->parser;
1300 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1301 struct ibv_flow_spec_eth eth = {
1302 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1306 /* Don't update layer for the inner pattern. */
1308 parser->layer = HASH_RXQ_ETH;
1313 mask = default_mask;
1314 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1315 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1316 eth.val.ether_type = spec->type;
1317 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1318 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1319 eth.mask.ether_type = mask->type;
1320 /* Remove unwanted bits from values. */
1321 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1322 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1323 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1325 eth.val.ether_type &= eth.mask.ether_type;
1327 mlx5_flow_create_copy(parser, ð, eth_size);
1332 * Convert VLAN item to Verbs specification.
1335 * Item specification.
1336 * @param default_mask[in]
1337 * Default bit-masks to use when item->mask is not provided.
1338 * @param data[in, out]
1342 * 0 on success, a negative errno value otherwise and rte_errno is set.
1345 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1346 const void *default_mask,
1347 struct mlx5_flow_data *data)
1349 const struct rte_flow_item_vlan *spec = item->spec;
1350 const struct rte_flow_item_vlan *mask = item->mask;
1351 struct mlx5_flow_parse *parser = data->parser;
1352 struct ibv_flow_spec_eth *eth;
1353 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1354 const char *msg = "VLAN cannot be empty";
1359 mask = default_mask;
1361 for (i = 0; i != hash_rxq_init_n; ++i) {
1362 if (!parser->queue[i].ibv_attr)
1365 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1366 parser->queue[i].offset - eth_size);
1367 eth->val.vlan_tag = spec->tci;
1368 eth->mask.vlan_tag = mask->tci;
1369 eth->val.vlan_tag &= eth->mask.vlan_tag;
1371 * From verbs perspective an empty VLAN is equivalent
1372 * to a packet without VLAN layer.
1374 if (!eth->mask.vlan_tag)
1376 /* Outer TPID cannot be matched. */
1377 if (eth->mask.ether_type) {
1378 msg = "VLAN TPID matching is not supported";
1381 eth->val.ether_type = spec->inner_type;
1382 eth->mask.ether_type = mask->inner_type;
1383 eth->val.ether_type &= eth->mask.ether_type;
1388 return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
1393 * Convert IPv4 item to Verbs specification.
1396 * Item specification.
1397 * @param default_mask[in]
1398 * Default bit-masks to use when item->mask is not provided.
1399 * @param data[in, out]
1403 * 0 on success, a negative errno value otherwise and rte_errno is set.
1406 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1407 const void *default_mask,
1408 struct mlx5_flow_data *data)
1410 const struct rte_flow_item_ipv4 *spec = item->spec;
1411 const struct rte_flow_item_ipv4 *mask = item->mask;
1412 struct mlx5_flow_parse *parser = data->parser;
1413 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1414 struct ibv_flow_spec_ipv4_ext ipv4 = {
1415 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1419 /* Don't update layer for the inner pattern. */
1421 parser->layer = HASH_RXQ_IPV4;
1424 mask = default_mask;
1425 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1426 .src_ip = spec->hdr.src_addr,
1427 .dst_ip = spec->hdr.dst_addr,
1428 .proto = spec->hdr.next_proto_id,
1429 .tos = spec->hdr.type_of_service,
1431 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1432 .src_ip = mask->hdr.src_addr,
1433 .dst_ip = mask->hdr.dst_addr,
1434 .proto = mask->hdr.next_proto_id,
1435 .tos = mask->hdr.type_of_service,
1437 /* Remove unwanted bits from values. */
1438 ipv4.val.src_ip &= ipv4.mask.src_ip;
1439 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1440 ipv4.val.proto &= ipv4.mask.proto;
1441 ipv4.val.tos &= ipv4.mask.tos;
1443 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1448 * Convert IPv6 item to Verbs specification.
1451 * Item specification.
1452 * @param default_mask[in]
1453 * Default bit-masks to use when item->mask is not provided.
1454 * @param data[in, out]
1458 * 0 on success, a negative errno value otherwise and rte_errno is set.
1461 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1462 const void *default_mask,
1463 struct mlx5_flow_data *data)
1465 const struct rte_flow_item_ipv6 *spec = item->spec;
1466 const struct rte_flow_item_ipv6 *mask = item->mask;
1467 struct mlx5_flow_parse *parser = data->parser;
1468 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1469 struct ibv_flow_spec_ipv6 ipv6 = {
1470 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1474 /* Don't update layer for the inner pattern. */
1476 parser->layer = HASH_RXQ_IPV6;
1479 uint32_t vtc_flow_val;
1480 uint32_t vtc_flow_mask;
1483 mask = default_mask;
1484 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1485 RTE_DIM(ipv6.val.src_ip));
1486 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1487 RTE_DIM(ipv6.val.dst_ip));
1488 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1489 RTE_DIM(ipv6.mask.src_ip));
1490 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1491 RTE_DIM(ipv6.mask.dst_ip));
1492 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
1493 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
1494 ipv6.val.flow_label =
1495 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
1497 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
1499 ipv6.val.next_hdr = spec->hdr.proto;
1500 ipv6.val.hop_limit = spec->hdr.hop_limits;
1501 ipv6.mask.flow_label =
1502 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
1504 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
1506 ipv6.mask.next_hdr = mask->hdr.proto;
1507 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1508 /* Remove unwanted bits from values. */
1509 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1510 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1511 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1513 ipv6.val.flow_label &= ipv6.mask.flow_label;
1514 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
1515 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1516 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1518 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1523 * Convert UDP item to Verbs specification.
1526 * Item specification.
1527 * @param default_mask[in]
1528 * Default bit-masks to use when item->mask is not provided.
1529 * @param data[in, out]
1533 * 0 on success, a negative errno value otherwise and rte_errno is set.
1536 mlx5_flow_create_udp(const struct rte_flow_item *item,
1537 const void *default_mask,
1538 struct mlx5_flow_data *data)
1540 const struct rte_flow_item_udp *spec = item->spec;
1541 const struct rte_flow_item_udp *mask = item->mask;
1542 struct mlx5_flow_parse *parser = data->parser;
1543 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1544 struct ibv_flow_spec_tcp_udp udp = {
1545 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1549 /* Don't update layer for the inner pattern. */
1550 if (!parser->inner) {
1551 if (parser->layer == HASH_RXQ_IPV4)
1552 parser->layer = HASH_RXQ_UDPV4;
1554 parser->layer = HASH_RXQ_UDPV6;
1558 mask = default_mask;
1559 udp.val.dst_port = spec->hdr.dst_port;
1560 udp.val.src_port = spec->hdr.src_port;
1561 udp.mask.dst_port = mask->hdr.dst_port;
1562 udp.mask.src_port = mask->hdr.src_port;
1563 /* Remove unwanted bits from values. */
1564 udp.val.src_port &= udp.mask.src_port;
1565 udp.val.dst_port &= udp.mask.dst_port;
1567 mlx5_flow_create_copy(parser, &udp, udp_size);
1572 * Convert TCP item to Verbs specification.
1575 * Item specification.
1576 * @param default_mask[in]
1577 * Default bit-masks to use when item->mask is not provided.
1578 * @param data[in, out]
1582 * 0 on success, a negative errno value otherwise and rte_errno is set.
1585 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1586 const void *default_mask,
1587 struct mlx5_flow_data *data)
1589 const struct rte_flow_item_tcp *spec = item->spec;
1590 const struct rte_flow_item_tcp *mask = item->mask;
1591 struct mlx5_flow_parse *parser = data->parser;
1592 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1593 struct ibv_flow_spec_tcp_udp tcp = {
1594 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1598 /* Don't update layer for the inner pattern. */
1599 if (!parser->inner) {
1600 if (parser->layer == HASH_RXQ_IPV4)
1601 parser->layer = HASH_RXQ_TCPV4;
1603 parser->layer = HASH_RXQ_TCPV6;
1607 mask = default_mask;
1608 tcp.val.dst_port = spec->hdr.dst_port;
1609 tcp.val.src_port = spec->hdr.src_port;
1610 tcp.mask.dst_port = mask->hdr.dst_port;
1611 tcp.mask.src_port = mask->hdr.src_port;
1612 /* Remove unwanted bits from values. */
1613 tcp.val.src_port &= tcp.mask.src_port;
1614 tcp.val.dst_port &= tcp.mask.dst_port;
1616 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1621 * Convert VXLAN item to Verbs specification.
1624 * Item specification.
1625 * @param default_mask[in]
1626 * Default bit-masks to use when item->mask is not provided.
1627 * @param data[in, out]
1631 * 0 on success, a negative errno value otherwise and rte_errno is set.
1634 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1635 const void *default_mask,
1636 struct mlx5_flow_data *data)
1638 const struct rte_flow_item_vxlan *spec = item->spec;
1639 const struct rte_flow_item_vxlan *mask = item->mask;
1640 struct mlx5_flow_parse *parser = data->parser;
1641 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1642 struct ibv_flow_spec_tunnel vxlan = {
1643 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1652 parser->inner = IBV_FLOW_SPEC_INNER;
1655 mask = default_mask;
1656 memcpy(&id.vni[1], spec->vni, 3);
1657 vxlan.val.tunnel_id = id.vlan_id;
1658 memcpy(&id.vni[1], mask->vni, 3);
1659 vxlan.mask.tunnel_id = id.vlan_id;
1660 /* Remove unwanted bits from values. */
1661 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1664 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1665 * layer is defined in the Verbs specification it is interpreted as
1666 * wildcard and all packets will match this rule, if it follows a full
1667 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1668 * before will also match this rule.
1669 * To avoid such situation, VNI 0 is currently refused.
1671 if (!vxlan.val.tunnel_id)
1672 return rte_flow_error_set(data->error, EINVAL,
1673 RTE_FLOW_ERROR_TYPE_ITEM,
1675 "VxLAN vni cannot be 0");
1676 mlx5_flow_create_copy(parser, &vxlan, size);
1681 * Convert GRE item to Verbs specification.
1684 * Item specification.
1685 * @param default_mask[in]
1686 * Default bit-masks to use when item->mask is not provided.
1687 * @param data[in, out]
1691 * 0 on success, a negative errno value otherwise and rte_errno is set.
1694 mlx5_flow_create_gre(const struct rte_flow_item *item __rte_unused,
1695 const void *default_mask __rte_unused,
1696 struct mlx5_flow_data *data)
1698 struct mlx5_flow_parse *parser = data->parser;
1699 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1700 struct ibv_flow_spec_tunnel tunnel = {
1701 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1704 struct ibv_flow_spec_ipv4_ext *ipv4;
1705 struct ibv_flow_spec_ipv6 *ipv6;
1708 parser->inner = IBV_FLOW_SPEC_INNER;
1709 /* Update encapsulation IP layer protocol. */
1710 for (i = 0; i != hash_rxq_init_n; ++i) {
1711 if (!parser->queue[i].ibv_attr)
1713 if (parser->out_layer == HASH_RXQ_IPV4) {
1714 ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1715 parser->queue[i].offset -
1716 sizeof(struct ibv_flow_spec_ipv4_ext));
1717 if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
1719 ipv4->val.proto = MLX5_GRE;
1720 ipv4->mask.proto = 0xff;
1721 } else if (parser->out_layer == HASH_RXQ_IPV6) {
1722 ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1723 parser->queue[i].offset -
1724 sizeof(struct ibv_flow_spec_ipv6));
1725 if (ipv6->mask.next_hdr &&
1726 ipv6->val.next_hdr != MLX5_GRE)
1728 ipv6->val.next_hdr = MLX5_GRE;
1729 ipv6->mask.next_hdr = 0xff;
1732 if (i != hash_rxq_init_n)
1733 return rte_flow_error_set(data->error, EINVAL,
1734 RTE_FLOW_ERROR_TYPE_ITEM,
1736 "IP protocol of GRE must be 47");
1737 mlx5_flow_create_copy(parser, &tunnel, size);
1742 * Convert mark/flag action to Verbs specification.
1745 * Internal parser structure.
1750 * 0 on success, a negative errno value otherwise and rte_errno is set.
1753 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1755 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1756 struct ibv_flow_spec_action_tag tag = {
1757 .type = IBV_FLOW_SPEC_ACTION_TAG,
1759 .tag_id = mlx5_flow_mark_set(mark_id),
1762 assert(parser->mark);
1763 mlx5_flow_create_copy(parser, &tag, size);
1768 * Convert count action to Verbs specification.
1771 * Pointer to Ethernet device.
1773 * Pointer to MLX5 flow parser structure.
1776 * 0 on success, a negative errno value otherwise and rte_errno is set.
1779 mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
1780 struct mlx5_flow_parse *parser __rte_unused)
1782 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1783 struct priv *priv = dev->data->dev_private;
1784 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1785 struct ibv_counter_set_init_attr init_attr = {0};
1786 struct ibv_flow_spec_counter_action counter = {
1787 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1789 .counter_set_handle = 0,
1792 init_attr.counter_set_id = 0;
1793 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
1798 counter.counter_set_handle = parser->cs->handle;
1799 mlx5_flow_create_copy(parser, &counter, size);
1805 * Complete flow rule creation with a drop queue.
1808 * Pointer to Ethernet device.
1810 * Internal parser structure.
1812 * Pointer to the rte_flow.
1814 * Perform verbose error reporting if not NULL.
1817 * 0 on success, a negative errno value otherwise and rte_errno is set.
1820 mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
1821 struct mlx5_flow_parse *parser,
1822 struct rte_flow *flow,
1823 struct rte_flow_error *error)
1825 struct priv *priv = dev->data->dev_private;
1826 struct ibv_flow_spec_action_drop *drop;
1827 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1832 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
1833 parser->queue[HASH_RXQ_ETH].offset);
1834 *drop = (struct ibv_flow_spec_action_drop){
1835 .type = IBV_FLOW_SPEC_ACTION_DROP,
1838 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
1839 parser->queue[HASH_RXQ_ETH].offset += size;
1840 flow->frxq[HASH_RXQ_ETH].ibv_attr =
1841 parser->queue[HASH_RXQ_ETH].ibv_attr;
1843 flow->cs = parser->cs;
1844 if (!priv->dev->data->dev_started)
1846 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
1847 flow->frxq[HASH_RXQ_ETH].ibv_flow =
1848 mlx5_glue->create_flow(priv->flow_drop_queue->qp,
1849 flow->frxq[HASH_RXQ_ETH].ibv_attr);
1850 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1851 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1852 NULL, "flow rule creation failure");
1858 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1859 claim_zero(mlx5_glue->destroy_flow
1860 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
1861 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
1863 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
1864 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
1865 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
1868 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1876 * Create hash Rx queues when RSS is enabled.
1879 * Pointer to Ethernet device.
1881 * Internal parser structure.
1883 * Pointer to the rte_flow.
1885 * Perform verbose error reporting if not NULL.
1888 * 0 on success, a negative errno value otherwise and rte_errno is set.
1891 mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
1892 struct mlx5_flow_parse *parser,
1893 struct rte_flow *flow,
1894 struct rte_flow_error *error)
1896 struct priv *priv = dev->data->dev_private;
1899 for (i = 0; i != hash_rxq_init_n; ++i) {
1900 uint64_t hash_fields;
1902 if (!parser->queue[i].ibv_attr)
1904 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1905 parser->queue[i].ibv_attr = NULL;
1906 hash_fields = hash_rxq_init[i].hash_fields;
1907 if (!priv->dev->data->dev_started)
1909 flow->frxq[i].hrxq =
1911 parser->rss_conf.key,
1912 parser->rss_conf.key_len,
1914 parser->rss_conf.queue,
1915 parser->rss_conf.queue_num);
1916 if (flow->frxq[i].hrxq)
1918 flow->frxq[i].hrxq =
1920 parser->rss_conf.key,
1921 parser->rss_conf.key_len,
1923 parser->rss_conf.queue,
1924 parser->rss_conf.queue_num);
1925 if (!flow->frxq[i].hrxq) {
1926 return rte_flow_error_set(error, ENOMEM,
1927 RTE_FLOW_ERROR_TYPE_HANDLE,
1929 "cannot create hash rxq");
1936 * Complete flow rule creation.
1939 * Pointer to Ethernet device.
1941 * Internal parser structure.
1943 * Pointer to the rte_flow.
1945 * Perform verbose error reporting if not NULL.
1948 * 0 on success, a negative errno value otherwise and rte_errno is set.
1951 mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
1952 struct mlx5_flow_parse *parser,
1953 struct rte_flow *flow,
1954 struct rte_flow_error *error)
1956 struct priv *priv = dev->data->dev_private;
1959 unsigned int flows_n = 0;
1963 assert(!parser->drop);
1964 ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
1968 flow->cs = parser->cs;
1969 if (!priv->dev->data->dev_started)
1971 for (i = 0; i != hash_rxq_init_n; ++i) {
1972 if (!flow->frxq[i].hrxq)
1974 flow->frxq[i].ibv_flow =
1975 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
1976 flow->frxq[i].ibv_attr);
1977 if (!flow->frxq[i].ibv_flow) {
1978 rte_flow_error_set(error, ENOMEM,
1979 RTE_FLOW_ERROR_TYPE_HANDLE,
1980 NULL, "flow rule creation failure");
1984 DRV_LOG(DEBUG, "port %u %p type %d QP %p ibv_flow %p",
1987 (void *)flow->frxq[i].hrxq,
1988 (void *)flow->frxq[i].ibv_flow);
1991 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
1992 NULL, "internal error in flow creation");
1995 for (i = 0; i != parser->rss_conf.queue_num; ++i) {
1996 struct mlx5_rxq_data *q =
1997 (*priv->rxqs)[parser->rss_conf.queue[i]];
1999 q->mark |= parser->mark;
2003 ret = rte_errno; /* Save rte_errno before cleanup. */
2005 for (i = 0; i != hash_rxq_init_n; ++i) {
2006 if (flow->frxq[i].ibv_flow) {
2007 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
2009 claim_zero(mlx5_glue->destroy_flow(ibv_flow));
2011 if (flow->frxq[i].hrxq)
2012 mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
2013 if (flow->frxq[i].ibv_attr)
2014 rte_free(flow->frxq[i].ibv_attr);
2017 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
2021 rte_errno = ret; /* Restore rte_errno. */
2029 * Pointer to Ethernet device.
2031 * Pointer to a TAILQ flow list.
2033 * Flow rule attributes.
2034 * @param[in] pattern
2035 * Pattern specification (list terminated by the END pattern item).
2036 * @param[in] actions
2037 * Associated actions (list terminated by the END action).
2039 * Perform verbose error reporting if not NULL.
2042 * A flow on success, NULL otherwise and rte_errno is set.
2044 static struct rte_flow *
2045 mlx5_flow_list_create(struct rte_eth_dev *dev,
2046 struct mlx5_flows *list,
2047 const struct rte_flow_attr *attr,
2048 const struct rte_flow_item items[],
2049 const struct rte_flow_action actions[],
2050 struct rte_flow_error *error)
2052 struct mlx5_flow_parse parser = { .create = 1, };
2053 struct rte_flow *flow = NULL;
2057 ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
2060 flow = rte_calloc(__func__, 1,
2062 parser.rss_conf.queue_num * sizeof(uint16_t),
2065 rte_flow_error_set(error, ENOMEM,
2066 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2068 "cannot allocate flow memory");
2071 /* Copy configuration. */
2072 flow->queues = (uint16_t (*)[])(flow + 1);
2073 flow->rss_conf = (struct rte_flow_action_rss){
2074 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
2076 .types = parser.rss_conf.types,
2077 .key_len = parser.rss_conf.key_len,
2078 .queue_num = parser.rss_conf.queue_num,
2079 .key = memcpy(flow->rss_key, parser.rss_conf.key,
2080 sizeof(*parser.rss_conf.key) *
2081 parser.rss_conf.key_len),
2082 .queue = memcpy(flow->queues, parser.rss_conf.queue,
2083 sizeof(*parser.rss_conf.queue) *
2084 parser.rss_conf.queue_num),
2086 flow->mark = parser.mark;
2087 /* finalise the flow. */
2089 ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
2092 ret = mlx5_flow_create_action_queue(dev, &parser, flow, error);
2095 TAILQ_INSERT_TAIL(list, flow, next);
2096 DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id,
2100 DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id,
2102 for (i = 0; i != hash_rxq_init_n; ++i) {
2103 if (parser.queue[i].ibv_attr)
2104 rte_free(parser.queue[i].ibv_attr);
2111 * Validate a flow supported by the NIC.
2113 * @see rte_flow_validate()
2117 mlx5_flow_validate(struct rte_eth_dev *dev,
2118 const struct rte_flow_attr *attr,
2119 const struct rte_flow_item items[],
2120 const struct rte_flow_action actions[],
2121 struct rte_flow_error *error)
2123 struct mlx5_flow_parse parser = { .create = 0, };
2125 return mlx5_flow_convert(dev, attr, items, actions, error, &parser);
2131 * @see rte_flow_create()
2135 mlx5_flow_create(struct rte_eth_dev *dev,
2136 const struct rte_flow_attr *attr,
2137 const struct rte_flow_item items[],
2138 const struct rte_flow_action actions[],
2139 struct rte_flow_error *error)
2141 struct priv *priv = dev->data->dev_private;
2143 return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
2148 * Destroy a flow in a list.
2151 * Pointer to Ethernet device.
2153 * Pointer to a TAILQ flow list.
2158 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
2159 struct rte_flow *flow)
2161 struct priv *priv = dev->data->dev_private;
2164 if (flow->drop || !flow->mark)
2166 for (i = 0; i != flow->rss_conf.queue_num; ++i) {
2167 struct rte_flow *tmp;
2171 * To remove the mark from the queue, the queue must not be
2172 * present in any other marked flow (RSS or not).
2174 TAILQ_FOREACH(tmp, list, next) {
2176 uint16_t *tqs = NULL;
2181 for (j = 0; j != hash_rxq_init_n; ++j) {
2182 if (!tmp->frxq[j].hrxq)
2184 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2185 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2189 for (j = 0; (j != tq_n) && !mark; j++)
2190 if (tqs[j] == (*flow->queues)[i])
2193 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2197 if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
2198 claim_zero(mlx5_glue->destroy_flow
2199 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2200 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
2202 for (i = 0; i != hash_rxq_init_n; ++i) {
2203 struct mlx5_flow *frxq = &flow->frxq[i];
2206 claim_zero(mlx5_glue->destroy_flow
2209 mlx5_hrxq_release(dev, frxq->hrxq);
2211 rte_free(frxq->ibv_attr);
2215 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
2218 TAILQ_REMOVE(list, flow, next);
2219 DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id,
2225 * Destroy all flows.
2228 * Pointer to Ethernet device.
2230 * Pointer to a TAILQ flow list.
2233 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
2235 while (!TAILQ_EMPTY(list)) {
2236 struct rte_flow *flow;
2238 flow = TAILQ_FIRST(list);
2239 mlx5_flow_list_destroy(dev, list, flow);
2244 * Create drop queue.
2247 * Pointer to Ethernet device.
2250 * 0 on success, a negative errno value otherwise and rte_errno is set.
2253 mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
2255 struct priv *priv = dev->data->dev_private;
2256 struct mlx5_hrxq_drop *fdq = NULL;
2260 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2263 "port %u cannot allocate memory for drop queue",
2264 dev->data->port_id);
2268 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
2270 DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue",
2271 dev->data->port_id);
2275 fdq->wq = mlx5_glue->create_wq
2277 &(struct ibv_wq_init_attr){
2278 .wq_type = IBV_WQT_RQ,
2285 DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue",
2286 dev->data->port_id);
2290 fdq->ind_table = mlx5_glue->create_rwq_ind_table
2292 &(struct ibv_rwq_ind_table_init_attr){
2293 .log_ind_tbl_size = 0,
2294 .ind_tbl = &fdq->wq,
2297 if (!fdq->ind_table) {
2299 "port %u cannot allocate indirection table for drop"
2301 dev->data->port_id);
2305 fdq->qp = mlx5_glue->create_qp_ex
2307 &(struct ibv_qp_init_attr_ex){
2308 .qp_type = IBV_QPT_RAW_PACKET,
2310 IBV_QP_INIT_ATTR_PD |
2311 IBV_QP_INIT_ATTR_IND_TABLE |
2312 IBV_QP_INIT_ATTR_RX_HASH,
2313 .rx_hash_conf = (struct ibv_rx_hash_conf){
2315 IBV_RX_HASH_FUNC_TOEPLITZ,
2316 .rx_hash_key_len = rss_hash_default_key_len,
2317 .rx_hash_key = rss_hash_default_key,
2318 .rx_hash_fields_mask = 0,
2320 .rwq_ind_tbl = fdq->ind_table,
2324 DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue",
2325 dev->data->port_id);
2329 priv->flow_drop_queue = fdq;
2333 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2335 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2337 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2339 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2342 priv->flow_drop_queue = NULL;
2347 * Delete drop queue.
2350 * Pointer to Ethernet device.
2353 mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
2355 struct priv *priv = dev->data->dev_private;
2356 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2361 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2363 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2365 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2367 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2369 priv->flow_drop_queue = NULL;
2376 * Pointer to Ethernet device.
2378 * Pointer to a TAILQ flow list.
2381 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
2383 struct priv *priv = dev->data->dev_private;
2384 struct rte_flow *flow;
2386 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2388 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2391 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
2393 claim_zero(mlx5_glue->destroy_flow
2394 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2395 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
2396 DRV_LOG(DEBUG, "port %u flow %p removed",
2397 dev->data->port_id, (void *)flow);
2401 /* Verify the flow has not already been cleaned. */
2402 for (i = 0; i != hash_rxq_init_n; ++i) {
2403 if (!flow->frxq[i].ibv_flow)
2406 * Indirection table may be necessary to remove the
2407 * flags in the Rx queues.
2408 * This helps to speed-up the process by avoiding
2411 ind_tbl = flow->frxq[i].hrxq->ind_table;
2414 if (i == hash_rxq_init_n)
2418 for (i = 0; i != ind_tbl->queues_n; ++i)
2419 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2421 for (i = 0; i != hash_rxq_init_n; ++i) {
2422 if (!flow->frxq[i].ibv_flow)
2424 claim_zero(mlx5_glue->destroy_flow
2425 (flow->frxq[i].ibv_flow));
2426 flow->frxq[i].ibv_flow = NULL;
2427 mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
2428 flow->frxq[i].hrxq = NULL;
2430 DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
2439 * Pointer to Ethernet device.
2441 * Pointer to a TAILQ flow list.
2444 * 0 on success, a negative errno value otherwise and rte_errno is set.
2447 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
2449 struct priv *priv = dev->data->dev_private;
2450 struct rte_flow *flow;
2452 TAILQ_FOREACH(flow, list, next) {
2456 flow->frxq[HASH_RXQ_ETH].ibv_flow =
2457 mlx5_glue->create_flow
2458 (priv->flow_drop_queue->qp,
2459 flow->frxq[HASH_RXQ_ETH].ibv_attr);
2460 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
2462 "port %u flow %p cannot be applied",
2463 dev->data->port_id, (void *)flow);
2467 DRV_LOG(DEBUG, "port %u flow %p applied",
2468 dev->data->port_id, (void *)flow);
2472 for (i = 0; i != hash_rxq_init_n; ++i) {
2473 if (!flow->frxq[i].ibv_attr)
2475 flow->frxq[i].hrxq =
2476 mlx5_hrxq_get(dev, flow->rss_conf.key,
2477 flow->rss_conf.key_len,
2478 hash_rxq_init[i].hash_fields,
2479 flow->rss_conf.queue,
2480 flow->rss_conf.queue_num);
2481 if (flow->frxq[i].hrxq)
2483 flow->frxq[i].hrxq =
2484 mlx5_hrxq_new(dev, flow->rss_conf.key,
2485 flow->rss_conf.key_len,
2486 hash_rxq_init[i].hash_fields,
2487 flow->rss_conf.queue,
2488 flow->rss_conf.queue_num);
2489 if (!flow->frxq[i].hrxq) {
2491 "port %u flow %p cannot be applied",
2492 dev->data->port_id, (void *)flow);
2497 flow->frxq[i].ibv_flow =
2498 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
2499 flow->frxq[i].ibv_attr);
2500 if (!flow->frxq[i].ibv_flow) {
2502 "port %u flow %p cannot be applied",
2503 dev->data->port_id, (void *)flow);
2507 DRV_LOG(DEBUG, "port %u flow %p applied",
2508 dev->data->port_id, (void *)flow);
2512 for (i = 0; i != flow->rss_conf.queue_num; ++i)
2513 (*priv->rxqs)[flow->rss_conf.queue[i]]->mark = 1;
2519 * Verify the flow list is empty
2522 * Pointer to Ethernet device.
2524 * @return the number of flows not released.
2527 mlx5_flow_verify(struct rte_eth_dev *dev)
2529 struct priv *priv = dev->data->dev_private;
2530 struct rte_flow *flow;
2533 TAILQ_FOREACH(flow, &priv->flows, next) {
2534 DRV_LOG(DEBUG, "port %u flow %p still referenced",
2535 dev->data->port_id, (void *)flow);
2542 * Enable a control flow configured from the control plane.
2545 * Pointer to Ethernet device.
2547 * An Ethernet flow spec to apply.
2549 * An Ethernet flow mask to apply.
2551 * A VLAN flow spec to apply.
2553 * A VLAN flow mask to apply.
2556 * 0 on success, a negative errno value otherwise and rte_errno is set.
2559 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2560 struct rte_flow_item_eth *eth_spec,
2561 struct rte_flow_item_eth *eth_mask,
2562 struct rte_flow_item_vlan *vlan_spec,
2563 struct rte_flow_item_vlan *vlan_mask)
2565 struct priv *priv = dev->data->dev_private;
2566 const struct rte_flow_attr attr = {
2568 .priority = MLX5_CTRL_FLOW_PRIORITY,
2570 struct rte_flow_item items[] = {
2572 .type = RTE_FLOW_ITEM_TYPE_ETH,
2578 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2579 RTE_FLOW_ITEM_TYPE_END,
2585 .type = RTE_FLOW_ITEM_TYPE_END,
2588 uint16_t queue[priv->reta_idx_n];
2589 struct rte_flow_action_rss action_rss = {
2590 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
2592 .types = priv->rss_conf.rss_hf,
2593 .key_len = priv->rss_conf.rss_key_len,
2594 .queue_num = priv->reta_idx_n,
2595 .key = priv->rss_conf.rss_key,
2598 struct rte_flow_action actions[] = {
2600 .type = RTE_FLOW_ACTION_TYPE_RSS,
2601 .conf = &action_rss,
2604 .type = RTE_FLOW_ACTION_TYPE_END,
2607 struct rte_flow *flow;
2608 struct rte_flow_error error;
2611 if (!priv->reta_idx_n) {
2615 for (i = 0; i != priv->reta_idx_n; ++i)
2616 queue[i] = (*priv->reta_idx)[i];
2617 flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
2625 * Enable a flow control configured from the control plane.
2628 * Pointer to Ethernet device.
2630 * An Ethernet flow spec to apply.
2632 * An Ethernet flow mask to apply.
2635 * 0 on success, a negative errno value otherwise and rte_errno is set.
2638 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2639 struct rte_flow_item_eth *eth_spec,
2640 struct rte_flow_item_eth *eth_mask)
2642 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2648 * @see rte_flow_destroy()
2652 mlx5_flow_destroy(struct rte_eth_dev *dev,
2653 struct rte_flow *flow,
2654 struct rte_flow_error *error __rte_unused)
2656 struct priv *priv = dev->data->dev_private;
2658 mlx5_flow_list_destroy(dev, &priv->flows, flow);
2663 * Destroy all flows.
2665 * @see rte_flow_flush()
2669 mlx5_flow_flush(struct rte_eth_dev *dev,
2670 struct rte_flow_error *error __rte_unused)
2672 struct priv *priv = dev->data->dev_private;
2674 mlx5_flow_list_flush(dev, &priv->flows);
2678 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2680 * Query flow counter.
2684 * @param counter_value
2685 * returned data from the counter.
2688 * 0 on success, a negative errno value otherwise and rte_errno is set.
2691 mlx5_flow_query_count(struct ibv_counter_set *cs,
2692 struct mlx5_flow_counter_stats *counter_stats,
2693 struct rte_flow_query_count *query_count,
2694 struct rte_flow_error *error)
2696 uint64_t counters[2];
2697 struct ibv_query_counter_set_attr query_cs_attr = {
2699 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2701 struct ibv_counter_set_data query_out = {
2703 .outlen = 2 * sizeof(uint64_t),
2705 int err = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
2708 return rte_flow_error_set(error, err,
2709 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2711 "cannot read counter");
2712 query_count->hits_set = 1;
2713 query_count->bytes_set = 1;
2714 query_count->hits = counters[0] - counter_stats->hits;
2715 query_count->bytes = counters[1] - counter_stats->bytes;
2716 if (query_count->reset) {
2717 counter_stats->hits = counters[0];
2718 counter_stats->bytes = counters[1];
2726 * @see rte_flow_query()
2730 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
2731 struct rte_flow *flow,
2732 enum rte_flow_action_type action __rte_unused,
2734 struct rte_flow_error *error)
2739 ret = mlx5_flow_query_count(flow->cs,
2740 &flow->counter_stats,
2741 (struct rte_flow_query_count *)data,
2746 return rte_flow_error_set(error, EINVAL,
2747 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2749 "no counter found for flow");
2758 * @see rte_flow_isolate()
2762 mlx5_flow_isolate(struct rte_eth_dev *dev,
2764 struct rte_flow_error *error)
2766 struct priv *priv = dev->data->dev_private;
2768 if (dev->data->dev_started) {
2769 rte_flow_error_set(error, EBUSY,
2770 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2772 "port must be stopped first");
2775 priv->isolated = !!enable;
2777 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2779 priv->dev->dev_ops = &mlx5_dev_ops;
2784 * Convert a flow director filter to a generic flow.
2787 * Pointer to Ethernet device.
2788 * @param fdir_filter
2789 * Flow director filter to add.
2791 * Generic flow parameters structure.
2794 * 0 on success, a negative errno value otherwise and rte_errno is set.
2797 mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
2798 const struct rte_eth_fdir_filter *fdir_filter,
2799 struct mlx5_fdir *attributes)
2801 struct priv *priv = dev->data->dev_private;
2802 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2803 const struct rte_eth_fdir_masks *mask =
2804 &dev->data->dev_conf.fdir_conf.mask;
2806 /* Validate queue number. */
2807 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2808 DRV_LOG(ERR, "port %u invalid queue number %d",
2809 dev->data->port_id, fdir_filter->action.rx_queue);
2813 attributes->attr.ingress = 1;
2814 attributes->items[0] = (struct rte_flow_item) {
2815 .type = RTE_FLOW_ITEM_TYPE_ETH,
2816 .spec = &attributes->l2,
2817 .mask = &attributes->l2_mask,
2819 switch (fdir_filter->action.behavior) {
2820 case RTE_ETH_FDIR_ACCEPT:
2821 attributes->actions[0] = (struct rte_flow_action){
2822 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2823 .conf = &attributes->queue,
2826 case RTE_ETH_FDIR_REJECT:
2827 attributes->actions[0] = (struct rte_flow_action){
2828 .type = RTE_FLOW_ACTION_TYPE_DROP,
2832 DRV_LOG(ERR, "port %u invalid behavior %d",
2834 fdir_filter->action.behavior);
2835 rte_errno = ENOTSUP;
2838 attributes->queue.index = fdir_filter->action.rx_queue;
2840 switch (fdir_filter->input.flow_type) {
2841 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2842 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2843 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2844 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2845 .src_addr = input->flow.ip4_flow.src_ip,
2846 .dst_addr = input->flow.ip4_flow.dst_ip,
2847 .time_to_live = input->flow.ip4_flow.ttl,
2848 .type_of_service = input->flow.ip4_flow.tos,
2849 .next_proto_id = input->flow.ip4_flow.proto,
2851 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
2852 .src_addr = mask->ipv4_mask.src_ip,
2853 .dst_addr = mask->ipv4_mask.dst_ip,
2854 .time_to_live = mask->ipv4_mask.ttl,
2855 .type_of_service = mask->ipv4_mask.tos,
2856 .next_proto_id = mask->ipv4_mask.proto,
2858 attributes->items[1] = (struct rte_flow_item){
2859 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2860 .spec = &attributes->l3,
2861 .mask = &attributes->l3_mask,
2864 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2865 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2866 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2867 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2868 .hop_limits = input->flow.ipv6_flow.hop_limits,
2869 .proto = input->flow.ipv6_flow.proto,
2872 memcpy(attributes->l3.ipv6.hdr.src_addr,
2873 input->flow.ipv6_flow.src_ip,
2874 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2875 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2876 input->flow.ipv6_flow.dst_ip,
2877 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2878 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
2879 mask->ipv6_mask.src_ip,
2880 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2881 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
2882 mask->ipv6_mask.dst_ip,
2883 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
2884 attributes->items[1] = (struct rte_flow_item){
2885 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2886 .spec = &attributes->l3,
2887 .mask = &attributes->l3_mask,
2891 DRV_LOG(ERR, "port %u invalid flow type%d",
2892 dev->data->port_id, fdir_filter->input.flow_type);
2893 rte_errno = ENOTSUP;
2897 switch (fdir_filter->input.flow_type) {
2898 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2899 attributes->l4.udp.hdr = (struct udp_hdr){
2900 .src_port = input->flow.udp4_flow.src_port,
2901 .dst_port = input->flow.udp4_flow.dst_port,
2903 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2904 .src_port = mask->src_port_mask,
2905 .dst_port = mask->dst_port_mask,
2907 attributes->items[2] = (struct rte_flow_item){
2908 .type = RTE_FLOW_ITEM_TYPE_UDP,
2909 .spec = &attributes->l4,
2910 .mask = &attributes->l4_mask,
2913 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2914 attributes->l4.tcp.hdr = (struct tcp_hdr){
2915 .src_port = input->flow.tcp4_flow.src_port,
2916 .dst_port = input->flow.tcp4_flow.dst_port,
2918 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2919 .src_port = mask->src_port_mask,
2920 .dst_port = mask->dst_port_mask,
2922 attributes->items[2] = (struct rte_flow_item){
2923 .type = RTE_FLOW_ITEM_TYPE_TCP,
2924 .spec = &attributes->l4,
2925 .mask = &attributes->l4_mask,
2928 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2929 attributes->l4.udp.hdr = (struct udp_hdr){
2930 .src_port = input->flow.udp6_flow.src_port,
2931 .dst_port = input->flow.udp6_flow.dst_port,
2933 attributes->l4_mask.udp.hdr = (struct udp_hdr){
2934 .src_port = mask->src_port_mask,
2935 .dst_port = mask->dst_port_mask,
2937 attributes->items[2] = (struct rte_flow_item){
2938 .type = RTE_FLOW_ITEM_TYPE_UDP,
2939 .spec = &attributes->l4,
2940 .mask = &attributes->l4_mask,
2943 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2944 attributes->l4.tcp.hdr = (struct tcp_hdr){
2945 .src_port = input->flow.tcp6_flow.src_port,
2946 .dst_port = input->flow.tcp6_flow.dst_port,
2948 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
2949 .src_port = mask->src_port_mask,
2950 .dst_port = mask->dst_port_mask,
2952 attributes->items[2] = (struct rte_flow_item){
2953 .type = RTE_FLOW_ITEM_TYPE_TCP,
2954 .spec = &attributes->l4,
2955 .mask = &attributes->l4_mask,
2958 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2959 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2962 DRV_LOG(ERR, "port %u invalid flow type%d",
2963 dev->data->port_id, fdir_filter->input.flow_type);
2964 rte_errno = ENOTSUP;
2971 * Add new flow director filter and store it in list.
2974 * Pointer to Ethernet device.
2975 * @param fdir_filter
2976 * Flow director filter to add.
2979 * 0 on success, a negative errno value otherwise and rte_errno is set.
2982 mlx5_fdir_filter_add(struct rte_eth_dev *dev,
2983 const struct rte_eth_fdir_filter *fdir_filter)
2985 struct priv *priv = dev->data->dev_private;
2986 struct mlx5_fdir attributes = {
2989 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2990 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2994 struct mlx5_flow_parse parser = {
2995 .layer = HASH_RXQ_ETH,
2997 struct rte_flow_error error;
2998 struct rte_flow *flow;
3001 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
3004 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
3005 attributes.actions, &error, &parser);
3008 flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
3009 attributes.items, attributes.actions,
3012 DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id,
3020 * Delete specific filter.
3023 * Pointer to Ethernet device.
3024 * @param fdir_filter
3025 * Filter to be deleted.
3028 * 0 on success, a negative errno value otherwise and rte_errno is set.
3031 mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
3032 const struct rte_eth_fdir_filter *fdir_filter)
3034 struct priv *priv = dev->data->dev_private;
3035 struct mlx5_fdir attributes = {
3038 struct mlx5_flow_parse parser = {
3040 .layer = HASH_RXQ_ETH,
3042 struct rte_flow_error error;
3043 struct rte_flow *flow;
3047 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
3050 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
3051 attributes.actions, &error, &parser);
3055 * Special case for drop action which is only set in the
3056 * specifications when the flow is created. In this situation the
3057 * drop specification is missing.
3060 struct ibv_flow_spec_action_drop *drop;
3062 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
3063 parser.queue[HASH_RXQ_ETH].offset);
3064 *drop = (struct ibv_flow_spec_action_drop){
3065 .type = IBV_FLOW_SPEC_ACTION_DROP,
3066 .size = sizeof(struct ibv_flow_spec_action_drop),
3068 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
3070 TAILQ_FOREACH(flow, &priv->flows, next) {
3071 struct ibv_flow_attr *attr;
3072 struct ibv_spec_header *attr_h;
3074 struct ibv_flow_attr *flow_attr;
3075 struct ibv_spec_header *flow_h;
3077 unsigned int specs_n;
3079 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
3080 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
3081 /* Compare first the attributes. */
3082 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
3084 if (attr->num_of_specs == 0)
3086 spec = (void *)((uintptr_t)attr +
3087 sizeof(struct ibv_flow_attr));
3088 flow_spec = (void *)((uintptr_t)flow_attr +
3089 sizeof(struct ibv_flow_attr));
3090 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
3091 for (i = 0; i != specs_n; ++i) {
3094 if (memcmp(spec, flow_spec,
3095 RTE_MIN(attr_h->size, flow_h->size)))
3097 spec = (void *)((uintptr_t)spec + attr_h->size);
3098 flow_spec = (void *)((uintptr_t)flow_spec +
3101 /* At this point, the flow match. */
3104 /* The flow does not match. */
3107 ret = rte_errno; /* Save rte_errno before cleanup. */
3109 mlx5_flow_list_destroy(dev, &priv->flows, flow);
3111 for (i = 0; i != hash_rxq_init_n; ++i) {
3112 if (parser.queue[i].ibv_attr)
3113 rte_free(parser.queue[i].ibv_attr);
3115 rte_errno = ret; /* Restore rte_errno. */
3120 * Update queue for specific filter.
3123 * Pointer to Ethernet device.
3124 * @param fdir_filter
3125 * Filter to be updated.
3128 * 0 on success, a negative errno value otherwise and rte_errno is set.
3131 mlx5_fdir_filter_update(struct rte_eth_dev *dev,
3132 const struct rte_eth_fdir_filter *fdir_filter)
3136 ret = mlx5_fdir_filter_delete(dev, fdir_filter);
3139 return mlx5_fdir_filter_add(dev, fdir_filter);
3143 * Flush all filters.
3146 * Pointer to Ethernet device.
3149 mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
3151 struct priv *priv = dev->data->dev_private;
3153 mlx5_flow_list_flush(dev, &priv->flows);
3157 * Get flow director information.
3160 * Pointer to Ethernet device.
3161 * @param[out] fdir_info
3162 * Resulting flow director information.
3165 mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
3167 struct priv *priv = dev->data->dev_private;
3168 struct rte_eth_fdir_masks *mask =
3169 &priv->dev->data->dev_conf.fdir_conf.mask;
3171 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
3172 fdir_info->guarant_spc = 0;
3173 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
3174 fdir_info->max_flexpayload = 0;
3175 fdir_info->flow_types_mask[0] = 0;
3176 fdir_info->flex_payload_unit = 0;
3177 fdir_info->max_flex_payload_segment_num = 0;
3178 fdir_info->flex_payload_limit = 0;
3179 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
3183 * Deal with flow director operations.
3186 * Pointer to Ethernet device.
3188 * Operation to perform.
3190 * Pointer to operation-specific structure.
3193 * 0 on success, a negative errno value otherwise and rte_errno is set.
3196 mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
3199 struct priv *priv = dev->data->dev_private;
3200 enum rte_fdir_mode fdir_mode =
3201 priv->dev->data->dev_conf.fdir_conf.mode;
3203 if (filter_op == RTE_ETH_FILTER_NOP)
3205 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
3206 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
3207 DRV_LOG(ERR, "port %u flow director mode %d not supported",
3208 dev->data->port_id, fdir_mode);
3212 switch (filter_op) {
3213 case RTE_ETH_FILTER_ADD:
3214 return mlx5_fdir_filter_add(dev, arg);
3215 case RTE_ETH_FILTER_UPDATE:
3216 return mlx5_fdir_filter_update(dev, arg);
3217 case RTE_ETH_FILTER_DELETE:
3218 return mlx5_fdir_filter_delete(dev, arg);
3219 case RTE_ETH_FILTER_FLUSH:
3220 mlx5_fdir_filter_flush(dev);
3222 case RTE_ETH_FILTER_INFO:
3223 mlx5_fdir_info_get(dev, arg);
3226 DRV_LOG(DEBUG, "port %u unknown operation %u",
3227 dev->data->port_id, filter_op);
3235 * Manage filter operations.
3238 * Pointer to Ethernet device structure.
3239 * @param filter_type
3242 * Operation to perform.
3244 * Pointer to operation-specific structure.
3247 * 0 on success, a negative errno value otherwise and rte_errno is set.
3250 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3251 enum rte_filter_type filter_type,
3252 enum rte_filter_op filter_op,
3255 switch (filter_type) {
3256 case RTE_ETH_FILTER_GENERIC:
3257 if (filter_op != RTE_ETH_FILTER_GET) {
3261 *(const void **)arg = &mlx5_flow_ops;
3263 case RTE_ETH_FILTER_FDIR:
3264 return mlx5_fdir_ctrl_func(dev, filter_op, arg);
3266 DRV_LOG(ERR, "port %u filter type (%d) not supported",
3267 dev->data->port_id, filter_type);
3268 rte_errno = ENOTSUP;
3275 * Detect number of Verbs flow priorities supported.
3278 * Pointer to Ethernet device.
3281 * number of supported Verbs flow priority.
3284 mlx5_get_max_verbs_prio(struct rte_eth_dev *dev)
3286 struct priv *priv = dev->data->dev_private;
3287 unsigned int verb_priorities = MLX5_VERBS_FLOW_PRIO_8;
3289 struct ibv_flow_attr attr;
3290 struct ibv_flow_spec_eth eth;
3291 struct ibv_flow_spec_action_drop drop;
3297 .type = IBV_FLOW_SPEC_ETH,
3298 .size = sizeof(struct ibv_flow_spec_eth),
3301 .size = sizeof(struct ibv_flow_spec_action_drop),
3302 .type = IBV_FLOW_SPEC_ACTION_DROP,
3305 struct ibv_flow *flow;
3308 flow_attr.attr.priority = verb_priorities - 1;
3309 flow = mlx5_glue->create_flow(priv->flow_drop_queue->qp,
3312 claim_zero(mlx5_glue->destroy_flow(flow));
3313 /* Try more priorities. */
3314 verb_priorities *= 2;
3316 /* Failed, restore last right number. */
3317 verb_priorities /= 2;
3321 DRV_LOG(DEBUG, "port %u Verbs flow priorities: %d,"
3322 " user flow priorities: %d",
3323 dev->data->port_id, verb_priorities, MLX5_CTRL_FLOW_PRIORITY);
3324 return verb_priorities;