1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox.
10 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
12 #pragma GCC diagnostic ignored "-Wpedantic"
14 #include <infiniband/verbs.h>
16 #pragma GCC diagnostic error "-Wpedantic"
19 #include <rte_ethdev_driver.h>
21 #include <rte_flow_driver.h>
22 #include <rte_malloc.h>
26 #include "mlx5_defs.h"
28 #include "mlx5_glue.h"
30 /* Define minimal priority for control plane flows. */
31 #define MLX5_CTRL_FLOW_PRIORITY 4
33 /* Internet Protocol versions. */
37 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
38 struct ibv_flow_spec_counter_action {
43 /* Dev ops structure defined in mlx5.c */
44 extern const struct eth_dev_ops mlx5_dev_ops;
45 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
48 mlx5_flow_create_eth(const struct rte_flow_item *item,
49 const void *default_mask,
53 mlx5_flow_create_vlan(const struct rte_flow_item *item,
54 const void *default_mask,
58 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
59 const void *default_mask,
63 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
64 const void *default_mask,
68 mlx5_flow_create_udp(const struct rte_flow_item *item,
69 const void *default_mask,
73 mlx5_flow_create_tcp(const struct rte_flow_item *item,
74 const void *default_mask,
78 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
79 const void *default_mask,
82 struct mlx5_flow_parse;
85 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
89 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
92 mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
94 /* Hash RX queue types. */
105 /* Initialization data for hash RX queue. */
106 struct hash_rxq_init {
107 uint64_t hash_fields; /* Fields that participate in the hash. */
108 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
109 unsigned int flow_priority; /* Flow priority to use. */
110 unsigned int ip_version; /* Internet protocol. */
113 /* Initialization data for hash RX queues. */
114 const struct hash_rxq_init hash_rxq_init[] = {
116 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
117 IBV_RX_HASH_DST_IPV4 |
118 IBV_RX_HASH_SRC_PORT_TCP |
119 IBV_RX_HASH_DST_PORT_TCP),
120 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
122 .ip_version = MLX5_IPV4,
125 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
126 IBV_RX_HASH_DST_IPV4 |
127 IBV_RX_HASH_SRC_PORT_UDP |
128 IBV_RX_HASH_DST_PORT_UDP),
129 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
131 .ip_version = MLX5_IPV4,
134 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
135 IBV_RX_HASH_DST_IPV4),
136 .dpdk_rss_hf = (ETH_RSS_IPV4 |
139 .ip_version = MLX5_IPV4,
142 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
143 IBV_RX_HASH_DST_IPV6 |
144 IBV_RX_HASH_SRC_PORT_TCP |
145 IBV_RX_HASH_DST_PORT_TCP),
146 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
148 .ip_version = MLX5_IPV6,
151 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
152 IBV_RX_HASH_DST_IPV6 |
153 IBV_RX_HASH_SRC_PORT_UDP |
154 IBV_RX_HASH_DST_PORT_UDP),
155 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
157 .ip_version = MLX5_IPV6,
160 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
161 IBV_RX_HASH_DST_IPV6),
162 .dpdk_rss_hf = (ETH_RSS_IPV6 |
165 .ip_version = MLX5_IPV6,
174 /* Number of entries in hash_rxq_init[]. */
175 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
177 /** Structure for holding counter stats. */
178 struct mlx5_flow_counter_stats {
179 uint64_t hits; /**< Number of packets matched by the rule. */
180 uint64_t bytes; /**< Number of bytes matched by the rule. */
183 /** Structure for Drop queue. */
184 struct mlx5_hrxq_drop {
185 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
186 struct ibv_qp *qp; /**< Verbs queue pair. */
187 struct ibv_wq *wq; /**< Verbs work queue. */
188 struct ibv_cq *cq; /**< Verbs completion queue. */
191 /* Flows structures. */
193 uint64_t hash_fields; /**< Fields that participate in the hash. */
194 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
195 struct ibv_flow *ibv_flow; /**< Verbs flow. */
196 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
199 /* Drop flows structures. */
200 struct mlx5_flow_drop {
201 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
202 struct ibv_flow *ibv_flow; /**< Verbs flow. */
206 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
207 uint32_t mark:1; /**< Set if the flow is marked. */
208 uint32_t drop:1; /**< Drop queue. */
209 uint16_t queues_n; /**< Number of entries in queue[]. */
210 uint16_t (*queues)[]; /**< Queues indexes to use. */
211 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
212 uint8_t rss_key[40]; /**< copy of the RSS key. */
213 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
214 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
215 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
216 /**< Flow with Rx queue. */
219 /** Static initializer for items. */
221 (const enum rte_flow_item_type []){ \
222 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
225 /** Structure to generate a simple graph of layers supported by the NIC. */
226 struct mlx5_flow_items {
227 /** List of possible actions for these items. */
228 const enum rte_flow_action_type *const actions;
229 /** Bit-masks corresponding to the possibilities for the item. */
232 * Default bit-masks to use when item->mask is not provided. When
233 * \default_mask is also NULL, the full supported bit-mask (\mask) is
236 const void *default_mask;
237 /** Bit-masks size in bytes. */
238 const unsigned int mask_sz;
240 * Conversion function from rte_flow to NIC specific flow.
243 * rte_flow item to convert.
244 * @param default_mask
245 * Default bit-masks to use when item->mask is not provided.
247 * Internal structure to store the conversion.
250 * 0 on success, negative value otherwise.
252 int (*convert)(const struct rte_flow_item *item,
253 const void *default_mask,
255 /** Size in bytes of the destination structure. */
256 const unsigned int dst_sz;
257 /** List of possible following items. */
258 const enum rte_flow_item_type *const items;
261 /** Valid action for this PMD. */
262 static const enum rte_flow_action_type valid_actions[] = {
263 RTE_FLOW_ACTION_TYPE_DROP,
264 RTE_FLOW_ACTION_TYPE_QUEUE,
265 RTE_FLOW_ACTION_TYPE_MARK,
266 RTE_FLOW_ACTION_TYPE_FLAG,
267 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
268 RTE_FLOW_ACTION_TYPE_COUNT,
270 RTE_FLOW_ACTION_TYPE_END,
273 /** Graph of supported items and associated actions. */
274 static const struct mlx5_flow_items mlx5_flow_items[] = {
275 [RTE_FLOW_ITEM_TYPE_END] = {
276 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
277 RTE_FLOW_ITEM_TYPE_VXLAN),
279 [RTE_FLOW_ITEM_TYPE_ETH] = {
280 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
281 RTE_FLOW_ITEM_TYPE_IPV4,
282 RTE_FLOW_ITEM_TYPE_IPV6),
283 .actions = valid_actions,
284 .mask = &(const struct rte_flow_item_eth){
285 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
286 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
289 .default_mask = &rte_flow_item_eth_mask,
290 .mask_sz = sizeof(struct rte_flow_item_eth),
291 .convert = mlx5_flow_create_eth,
292 .dst_sz = sizeof(struct ibv_flow_spec_eth),
294 [RTE_FLOW_ITEM_TYPE_VLAN] = {
295 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
296 RTE_FLOW_ITEM_TYPE_IPV6),
297 .actions = valid_actions,
298 .mask = &(const struct rte_flow_item_vlan){
301 .default_mask = &rte_flow_item_vlan_mask,
302 .mask_sz = sizeof(struct rte_flow_item_vlan),
303 .convert = mlx5_flow_create_vlan,
306 [RTE_FLOW_ITEM_TYPE_IPV4] = {
307 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
308 RTE_FLOW_ITEM_TYPE_TCP),
309 .actions = valid_actions,
310 .mask = &(const struct rte_flow_item_ipv4){
314 .type_of_service = -1,
318 .default_mask = &rte_flow_item_ipv4_mask,
319 .mask_sz = sizeof(struct rte_flow_item_ipv4),
320 .convert = mlx5_flow_create_ipv4,
321 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
323 [RTE_FLOW_ITEM_TYPE_IPV6] = {
324 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
325 RTE_FLOW_ITEM_TYPE_TCP),
326 .actions = valid_actions,
327 .mask = &(const struct rte_flow_item_ipv6){
330 0xff, 0xff, 0xff, 0xff,
331 0xff, 0xff, 0xff, 0xff,
332 0xff, 0xff, 0xff, 0xff,
333 0xff, 0xff, 0xff, 0xff,
336 0xff, 0xff, 0xff, 0xff,
337 0xff, 0xff, 0xff, 0xff,
338 0xff, 0xff, 0xff, 0xff,
339 0xff, 0xff, 0xff, 0xff,
346 .default_mask = &rte_flow_item_ipv6_mask,
347 .mask_sz = sizeof(struct rte_flow_item_ipv6),
348 .convert = mlx5_flow_create_ipv6,
349 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
351 [RTE_FLOW_ITEM_TYPE_UDP] = {
352 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
353 .actions = valid_actions,
354 .mask = &(const struct rte_flow_item_udp){
360 .default_mask = &rte_flow_item_udp_mask,
361 .mask_sz = sizeof(struct rte_flow_item_udp),
362 .convert = mlx5_flow_create_udp,
363 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
365 [RTE_FLOW_ITEM_TYPE_TCP] = {
366 .actions = valid_actions,
367 .mask = &(const struct rte_flow_item_tcp){
373 .default_mask = &rte_flow_item_tcp_mask,
374 .mask_sz = sizeof(struct rte_flow_item_tcp),
375 .convert = mlx5_flow_create_tcp,
376 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
378 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
379 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
380 .actions = valid_actions,
381 .mask = &(const struct rte_flow_item_vxlan){
382 .vni = "\xff\xff\xff",
384 .default_mask = &rte_flow_item_vxlan_mask,
385 .mask_sz = sizeof(struct rte_flow_item_vxlan),
386 .convert = mlx5_flow_create_vxlan,
387 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
391 /** Structure to pass to the conversion function. */
392 struct mlx5_flow_parse {
393 uint32_t inner; /**< Set once VXLAN is encountered. */
395 /**< Whether resources should remain after a validate. */
396 uint32_t drop:1; /**< Target is a drop queue. */
397 uint32_t mark:1; /**< Mark is present in the flow. */
398 uint32_t count:1; /**< Count is present in the flow. */
399 uint32_t mark_id; /**< Mark identifier. */
400 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
401 uint16_t queues_n; /**< Number of entries in queue[]. */
402 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
403 uint8_t rss_key[40]; /**< copy of the RSS key. */
404 enum hash_rxq_type layer; /**< Last pattern layer detected. */
405 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
407 struct ibv_flow_attr *ibv_attr;
408 /**< Pointer to Verbs attributes. */
410 /**< Current position or total size of the attribute. */
411 } queue[RTE_DIM(hash_rxq_init)];
414 static const struct rte_flow_ops mlx5_flow_ops = {
415 .validate = mlx5_flow_validate,
416 .create = mlx5_flow_create,
417 .destroy = mlx5_flow_destroy,
418 .flush = mlx5_flow_flush,
419 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
420 .query = mlx5_flow_query,
424 .isolate = mlx5_flow_isolate,
427 /* Convert FDIR request to Generic flow. */
429 struct rte_flow_attr attr;
430 struct rte_flow_action actions[2];
431 struct rte_flow_item items[4];
432 struct rte_flow_item_eth l2;
433 struct rte_flow_item_eth l2_mask;
435 struct rte_flow_item_ipv4 ipv4;
436 struct rte_flow_item_ipv6 ipv6;
439 struct rte_flow_item_udp udp;
440 struct rte_flow_item_tcp tcp;
442 struct rte_flow_action_queue queue;
445 /* Verbs specification header. */
446 struct ibv_spec_header {
447 enum ibv_flow_spec_type type;
452 * Check support for a given item.
455 * Item specification.
457 * Bit-masks covering supported fields to compare with spec, last and mask in
460 * Bit-Mask size in bytes.
466 mlx5_flow_item_validate(const struct rte_flow_item *item,
467 const uint8_t *mask, unsigned int size)
471 if (!item->spec && (item->mask || item->last))
473 if (item->spec && !item->mask) {
475 const uint8_t *spec = item->spec;
477 for (i = 0; i < size; ++i)
478 if ((spec[i] | mask[i]) != mask[i])
481 if (item->last && !item->mask) {
483 const uint8_t *spec = item->last;
485 for (i = 0; i < size; ++i)
486 if ((spec[i] | mask[i]) != mask[i])
491 const uint8_t *spec = item->spec;
493 for (i = 0; i < size; ++i)
494 if ((spec[i] | mask[i]) != mask[i])
497 if (item->spec && item->last) {
500 const uint8_t *apply = mask;
505 for (i = 0; i < size; ++i) {
506 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
507 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
509 ret = memcmp(spec, last, size);
515 * Copy the RSS configuration from the user ones, of the rss_conf is null,
516 * uses the driver one.
519 * Internal parser structure.
521 * User RSS configuration to save.
524 * 0 on success, errno value on failure.
527 mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser,
528 const struct rte_eth_rss_conf *rss_conf)
531 * This function is also called at the beginning of
532 * mlx5_flow_convert_actions() to initialize the parser with the
533 * device default RSS configuration.
536 if (rss_conf->rss_hf & MLX5_RSS_HF_MASK)
538 if (rss_conf->rss_key_len != 40)
540 if (rss_conf->rss_key_len && rss_conf->rss_key) {
541 parser->rss_conf.rss_key_len = rss_conf->rss_key_len;
542 memcpy(parser->rss_key, rss_conf->rss_key,
543 rss_conf->rss_key_len);
544 parser->rss_conf.rss_key = parser->rss_key;
546 parser->rss_conf.rss_hf = rss_conf->rss_hf;
552 * Extract attribute to the parser.
555 * Flow rule attributes.
557 * Perform verbose error reporting if not NULL.
560 * 0 on success, a negative errno value otherwise and rte_errno is set.
563 mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
564 struct rte_flow_error *error)
567 rte_flow_error_set(error, ENOTSUP,
568 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
570 "groups are not supported");
573 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
574 rte_flow_error_set(error, ENOTSUP,
575 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
577 "priorities are not supported");
581 rte_flow_error_set(error, ENOTSUP,
582 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
584 "egress is not supported");
587 if (!attr->ingress) {
588 rte_flow_error_set(error, ENOTSUP,
589 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
591 "only ingress is supported");
598 * Extract actions request to the parser.
601 * Pointer to Ethernet device.
603 * Associated actions (list terminated by the END action).
605 * Perform verbose error reporting if not NULL.
606 * @param[in, out] parser
607 * Internal parser structure.
610 * 0 on success, a negative errno value otherwise and rte_errno is set.
613 mlx5_flow_convert_actions(struct rte_eth_dev *dev,
614 const struct rte_flow_action actions[],
615 struct rte_flow_error *error,
616 struct mlx5_flow_parse *parser)
618 struct priv *priv = dev->data->dev_private;
621 * Add default RSS configuration necessary for Verbs to create QP even
622 * if no RSS is necessary.
624 mlx5_flow_convert_rss_conf(parser,
625 (const struct rte_eth_rss_conf *)
627 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
628 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
630 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
632 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
633 const struct rte_flow_action_queue *queue =
634 (const struct rte_flow_action_queue *)
639 if (!queue || (queue->index > (priv->rxqs_n - 1)))
640 goto exit_action_not_supported;
641 for (n = 0; n < parser->queues_n; ++n) {
642 if (parser->queues[n] == queue->index) {
647 if (parser->queues_n > 1 && !found) {
648 rte_flow_error_set(error, ENOTSUP,
649 RTE_FLOW_ERROR_TYPE_ACTION,
651 "queue action not in RSS queues");
655 parser->queues_n = 1;
656 parser->queues[0] = queue->index;
658 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
659 const struct rte_flow_action_rss *rss =
660 (const struct rte_flow_action_rss *)
664 if (!rss || !rss->num) {
665 rte_flow_error_set(error, EINVAL,
666 RTE_FLOW_ERROR_TYPE_ACTION,
671 if (parser->queues_n == 1) {
674 assert(parser->queues_n);
675 for (n = 0; n < rss->num; ++n) {
676 if (parser->queues[0] ==
683 rte_flow_error_set(error, ENOTSUP,
684 RTE_FLOW_ERROR_TYPE_ACTION,
686 "queue action not in RSS"
691 for (n = 0; n < rss->num; ++n) {
692 if (rss->queue[n] >= priv->rxqs_n) {
693 rte_flow_error_set(error, EINVAL,
694 RTE_FLOW_ERROR_TYPE_ACTION,
696 "queue id > number of"
701 for (n = 0; n < rss->num; ++n)
702 parser->queues[n] = rss->queue[n];
703 parser->queues_n = rss->num;
704 if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) {
705 rte_flow_error_set(error, EINVAL,
706 RTE_FLOW_ERROR_TYPE_ACTION,
708 "wrong RSS configuration");
711 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
712 const struct rte_flow_action_mark *mark =
713 (const struct rte_flow_action_mark *)
717 rte_flow_error_set(error, EINVAL,
718 RTE_FLOW_ERROR_TYPE_ACTION,
720 "mark must be defined");
722 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
723 rte_flow_error_set(error, ENOTSUP,
724 RTE_FLOW_ERROR_TYPE_ACTION,
726 "mark must be between 0"
731 parser->mark_id = mark->id;
732 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
734 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
735 priv->config.flow_counter_en) {
738 goto exit_action_not_supported;
741 if (parser->drop && parser->mark)
743 if (!parser->queues_n && !parser->drop) {
744 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
745 NULL, "no valid action");
749 exit_action_not_supported:
750 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
751 actions, "action not supported");
759 * Pattern specification (list terminated by the END pattern item).
761 * Perform verbose error reporting if not NULL.
762 * @param[in, out] parser
763 * Internal parser structure.
766 * 0 on success, a negative errno value otherwise and rte_errno is set.
769 mlx5_flow_convert_items_validate(const struct rte_flow_item items[],
770 struct rte_flow_error *error,
771 struct mlx5_flow_parse *parser)
773 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
776 /* Initialise the offsets to start after verbs attribute. */
777 for (i = 0; i != hash_rxq_init_n; ++i)
778 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
779 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
780 const struct mlx5_flow_items *token = NULL;
784 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
788 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
790 if (cur_item->items[i] == items->type) {
791 token = &mlx5_flow_items[items->type];
796 goto exit_item_not_supported;
798 err = mlx5_flow_item_validate(items,
799 (const uint8_t *)cur_item->mask,
802 goto exit_item_not_supported;
803 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
805 rte_flow_error_set(error, ENOTSUP,
806 RTE_FLOW_ERROR_TYPE_ITEM,
808 "cannot recognize multiple"
809 " VXLAN encapsulations");
812 parser->inner = IBV_FLOW_SPEC_INNER;
815 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
817 for (n = 0; n != hash_rxq_init_n; ++n)
818 parser->queue[n].offset += cur_item->dst_sz;
822 parser->queue[HASH_RXQ_ETH].offset +=
823 sizeof(struct ibv_flow_spec_action_drop);
826 for (i = 0; i != hash_rxq_init_n; ++i)
827 parser->queue[i].offset +=
828 sizeof(struct ibv_flow_spec_action_tag);
831 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
833 for (i = 0; i != hash_rxq_init_n; ++i)
834 parser->queue[i].offset += size;
837 exit_item_not_supported:
838 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
839 items, "item not supported");
844 * Allocate memory space to store verbs flow attributes.
846 * @param[in] priority
849 * Amount of byte to allocate.
851 * Perform verbose error reporting if not NULL.
854 * A verbs flow attribute on success, NULL otherwise.
856 static struct ibv_flow_attr *
857 mlx5_flow_convert_allocate(unsigned int priority,
859 struct rte_flow_error *error)
861 struct ibv_flow_attr *ibv_attr;
863 ibv_attr = rte_calloc(__func__, 1, size, 0);
865 rte_flow_error_set(error, ENOMEM,
866 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
868 "cannot allocate verbs spec attributes.");
871 ibv_attr->priority = priority;
876 * Finalise verbs flow attributes.
878 * @param[in, out] parser
879 * Internal parser structure.
882 mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
884 const unsigned int ipv4 =
885 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
886 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
887 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
888 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
889 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
890 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
893 /* Remove any other flow not matching the pattern. */
894 if (parser->queues_n == 1) {
895 for (i = 0; i != hash_rxq_init_n; ++i) {
896 if (i == HASH_RXQ_ETH)
898 rte_free(parser->queue[i].ibv_attr);
899 parser->queue[i].ibv_attr = NULL;
903 if (parser->layer == HASH_RXQ_ETH) {
907 * This layer becomes useless as the pattern define under
910 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
911 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
913 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
914 for (i = ohmin; i != (ohmax + 1); ++i) {
915 if (!parser->queue[i].ibv_attr)
917 rte_free(parser->queue[i].ibv_attr);
918 parser->queue[i].ibv_attr = NULL;
920 /* Remove impossible flow according to the RSS configuration. */
921 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
922 parser->rss_conf.rss_hf) {
923 /* Remove any other flow. */
924 for (i = hmin; i != (hmax + 1); ++i) {
925 if ((i == parser->layer) ||
926 (!parser->queue[i].ibv_attr))
928 rte_free(parser->queue[i].ibv_attr);
929 parser->queue[i].ibv_attr = NULL;
931 } else if (!parser->queue[ip].ibv_attr) {
932 /* no RSS possible with the current configuration. */
933 parser->queues_n = 1;
938 * Fill missing layers in verbs specifications, or compute the correct
939 * offset to allocate the memory space for the attributes and
942 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
944 struct ibv_flow_spec_ipv4_ext ipv4;
945 struct ibv_flow_spec_ipv6 ipv6;
946 struct ibv_flow_spec_tcp_udp udp_tcp;
951 if (i == parser->layer)
953 if (parser->layer == HASH_RXQ_ETH) {
954 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
955 size = sizeof(struct ibv_flow_spec_ipv4_ext);
956 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
957 .type = IBV_FLOW_SPEC_IPV4_EXT,
961 size = sizeof(struct ibv_flow_spec_ipv6);
962 specs.ipv6 = (struct ibv_flow_spec_ipv6){
963 .type = IBV_FLOW_SPEC_IPV6,
967 if (parser->queue[i].ibv_attr) {
968 dst = (void *)((uintptr_t)
969 parser->queue[i].ibv_attr +
970 parser->queue[i].offset);
971 memcpy(dst, &specs, size);
972 ++parser->queue[i].ibv_attr->num_of_specs;
974 parser->queue[i].offset += size;
976 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
977 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
978 size = sizeof(struct ibv_flow_spec_tcp_udp);
979 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
980 .type = ((i == HASH_RXQ_UDPV4 ||
981 i == HASH_RXQ_UDPV6) ?
986 if (parser->queue[i].ibv_attr) {
987 dst = (void *)((uintptr_t)
988 parser->queue[i].ibv_attr +
989 parser->queue[i].offset);
990 memcpy(dst, &specs, size);
991 ++parser->queue[i].ibv_attr->num_of_specs;
993 parser->queue[i].offset += size;
999 * Validate and convert a flow supported by the NIC.
1002 * Pointer to Ethernet device.
1004 * Flow rule attributes.
1005 * @param[in] pattern
1006 * Pattern specification (list terminated by the END pattern item).
1007 * @param[in] actions
1008 * Associated actions (list terminated by the END action).
1010 * Perform verbose error reporting if not NULL.
1011 * @param[in, out] parser
1012 * Internal parser structure.
1015 * 0 on success, a negative errno value otherwise and rte_errno is set.
1018 mlx5_flow_convert(struct rte_eth_dev *dev,
1019 const struct rte_flow_attr *attr,
1020 const struct rte_flow_item items[],
1021 const struct rte_flow_action actions[],
1022 struct rte_flow_error *error,
1023 struct mlx5_flow_parse *parser)
1025 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1029 /* First step. Validate the attributes, items and actions. */
1030 *parser = (struct mlx5_flow_parse){
1031 .create = parser->create,
1032 .layer = HASH_RXQ_ETH,
1033 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1035 ret = mlx5_flow_convert_attributes(attr, error);
1038 ret = mlx5_flow_convert_actions(dev, actions, error, parser);
1041 ret = mlx5_flow_convert_items_validate(items, error, parser);
1044 mlx5_flow_convert_finalise(parser);
1047 * Allocate the memory space to store verbs specifications.
1050 unsigned int priority =
1052 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1053 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1055 parser->queue[HASH_RXQ_ETH].ibv_attr =
1056 mlx5_flow_convert_allocate(priority, offset, error);
1057 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1059 parser->queue[HASH_RXQ_ETH].offset =
1060 sizeof(struct ibv_flow_attr);
1062 for (i = 0; i != hash_rxq_init_n; ++i) {
1063 unsigned int priority =
1065 hash_rxq_init[i].flow_priority;
1066 unsigned int offset;
1068 if (!(parser->rss_conf.rss_hf &
1069 hash_rxq_init[i].dpdk_rss_hf) &&
1070 (i != HASH_RXQ_ETH))
1072 offset = parser->queue[i].offset;
1073 parser->queue[i].ibv_attr =
1074 mlx5_flow_convert_allocate(priority,
1076 if (!parser->queue[i].ibv_attr)
1078 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1081 /* Third step. Conversion parse, fill the specifications. */
1083 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1084 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1086 cur_item = &mlx5_flow_items[items->type];
1087 ret = cur_item->convert(items,
1088 (cur_item->default_mask ?
1089 cur_item->default_mask :
1093 rte_flow_error_set(error, ret,
1094 RTE_FLOW_ERROR_TYPE_ITEM,
1095 items, "item not supported");
1100 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1101 if (parser->count && parser->create) {
1102 mlx5_flow_create_count(dev, parser);
1104 goto exit_count_error;
1107 * Last step. Complete missing specification to reach the RSS
1110 if (!parser->drop) {
1111 mlx5_flow_convert_finalise(parser);
1113 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1115 hash_rxq_init[parser->layer].flow_priority;
1118 /* Only verification is expected, all resources should be released. */
1119 if (!parser->create) {
1120 for (i = 0; i != hash_rxq_init_n; ++i) {
1121 if (parser->queue[i].ibv_attr) {
1122 rte_free(parser->queue[i].ibv_attr);
1123 parser->queue[i].ibv_attr = NULL;
1129 for (i = 0; i != hash_rxq_init_n; ++i) {
1130 if (parser->queue[i].ibv_attr) {
1131 rte_free(parser->queue[i].ibv_attr);
1132 parser->queue[i].ibv_attr = NULL;
1135 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1136 NULL, "cannot allocate verbs spec attributes.");
1139 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1140 NULL, "cannot create counter.");
1145 * Copy the specification created into the flow.
1148 * Internal parser structure.
1150 * Create specification.
1152 * Size in bytes of the specification to copy.
1155 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1161 for (i = 0; i != hash_rxq_init_n; ++i) {
1162 if (!parser->queue[i].ibv_attr)
1164 /* Specification must be the same l3 type or none. */
1165 if (parser->layer == HASH_RXQ_ETH ||
1166 (hash_rxq_init[parser->layer].ip_version ==
1167 hash_rxq_init[i].ip_version) ||
1168 (hash_rxq_init[i].ip_version == 0)) {
1169 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1170 parser->queue[i].offset);
1171 memcpy(dst, src, size);
1172 ++parser->queue[i].ibv_attr->num_of_specs;
1173 parser->queue[i].offset += size;
1179 * Convert Ethernet item to Verbs specification.
1182 * Item specification.
1183 * @param default_mask[in]
1184 * Default bit-masks to use when item->mask is not provided.
1185 * @param data[in, out]
1189 mlx5_flow_create_eth(const struct rte_flow_item *item,
1190 const void *default_mask,
1193 const struct rte_flow_item_eth *spec = item->spec;
1194 const struct rte_flow_item_eth *mask = item->mask;
1195 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1196 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1197 struct ibv_flow_spec_eth eth = {
1198 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1202 /* Don't update layer for the inner pattern. */
1204 parser->layer = HASH_RXQ_ETH;
1209 mask = default_mask;
1210 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1211 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1212 eth.val.ether_type = spec->type;
1213 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1214 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1215 eth.mask.ether_type = mask->type;
1216 /* Remove unwanted bits from values. */
1217 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1218 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1219 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1221 eth.val.ether_type &= eth.mask.ether_type;
1223 mlx5_flow_create_copy(parser, ð, eth_size);
1228 * Convert VLAN item to Verbs specification.
1231 * Item specification.
1232 * @param default_mask[in]
1233 * Default bit-masks to use when item->mask is not provided.
1234 * @param data[in, out]
1238 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1239 const void *default_mask,
1242 const struct rte_flow_item_vlan *spec = item->spec;
1243 const struct rte_flow_item_vlan *mask = item->mask;
1244 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1245 struct ibv_flow_spec_eth *eth;
1246 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1251 mask = default_mask;
1253 for (i = 0; i != hash_rxq_init_n; ++i) {
1254 if (!parser->queue[i].ibv_attr)
1257 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1258 parser->queue[i].offset - eth_size);
1259 eth->val.vlan_tag = spec->tci;
1260 eth->mask.vlan_tag = mask->tci;
1261 eth->val.vlan_tag &= eth->mask.vlan_tag;
1268 * Convert IPv4 item to Verbs specification.
1271 * Item specification.
1272 * @param default_mask[in]
1273 * Default bit-masks to use when item->mask is not provided.
1274 * @param data[in, out]
1278 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1279 const void *default_mask,
1282 const struct rte_flow_item_ipv4 *spec = item->spec;
1283 const struct rte_flow_item_ipv4 *mask = item->mask;
1284 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1285 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1286 struct ibv_flow_spec_ipv4_ext ipv4 = {
1287 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1291 /* Don't update layer for the inner pattern. */
1293 parser->layer = HASH_RXQ_IPV4;
1296 mask = default_mask;
1297 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1298 .src_ip = spec->hdr.src_addr,
1299 .dst_ip = spec->hdr.dst_addr,
1300 .proto = spec->hdr.next_proto_id,
1301 .tos = spec->hdr.type_of_service,
1303 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1304 .src_ip = mask->hdr.src_addr,
1305 .dst_ip = mask->hdr.dst_addr,
1306 .proto = mask->hdr.next_proto_id,
1307 .tos = mask->hdr.type_of_service,
1309 /* Remove unwanted bits from values. */
1310 ipv4.val.src_ip &= ipv4.mask.src_ip;
1311 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1312 ipv4.val.proto &= ipv4.mask.proto;
1313 ipv4.val.tos &= ipv4.mask.tos;
1315 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1320 * Convert IPv6 item to Verbs specification.
1323 * Item specification.
1324 * @param default_mask[in]
1325 * Default bit-masks to use when item->mask is not provided.
1326 * @param data[in, out]
1330 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1331 const void *default_mask,
1334 const struct rte_flow_item_ipv6 *spec = item->spec;
1335 const struct rte_flow_item_ipv6 *mask = item->mask;
1336 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1337 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1338 struct ibv_flow_spec_ipv6 ipv6 = {
1339 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1343 /* Don't update layer for the inner pattern. */
1345 parser->layer = HASH_RXQ_IPV6;
1348 uint32_t vtc_flow_val;
1349 uint32_t vtc_flow_mask;
1352 mask = default_mask;
1353 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1354 RTE_DIM(ipv6.val.src_ip));
1355 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1356 RTE_DIM(ipv6.val.dst_ip));
1357 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1358 RTE_DIM(ipv6.mask.src_ip));
1359 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1360 RTE_DIM(ipv6.mask.dst_ip));
1361 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
1362 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
1363 ipv6.val.flow_label =
1364 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
1366 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
1368 ipv6.val.next_hdr = spec->hdr.proto;
1369 ipv6.val.hop_limit = spec->hdr.hop_limits;
1370 ipv6.mask.flow_label =
1371 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
1373 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
1375 ipv6.mask.next_hdr = mask->hdr.proto;
1376 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1377 /* Remove unwanted bits from values. */
1378 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1379 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1380 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1382 ipv6.val.flow_label &= ipv6.mask.flow_label;
1383 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
1384 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1385 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1387 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1392 * Convert UDP item to Verbs specification.
1395 * Item specification.
1396 * @param default_mask[in]
1397 * Default bit-masks to use when item->mask is not provided.
1398 * @param data[in, out]
1402 mlx5_flow_create_udp(const struct rte_flow_item *item,
1403 const void *default_mask,
1406 const struct rte_flow_item_udp *spec = item->spec;
1407 const struct rte_flow_item_udp *mask = item->mask;
1408 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1409 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1410 struct ibv_flow_spec_tcp_udp udp = {
1411 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1415 /* Don't update layer for the inner pattern. */
1416 if (!parser->inner) {
1417 if (parser->layer == HASH_RXQ_IPV4)
1418 parser->layer = HASH_RXQ_UDPV4;
1420 parser->layer = HASH_RXQ_UDPV6;
1424 mask = default_mask;
1425 udp.val.dst_port = spec->hdr.dst_port;
1426 udp.val.src_port = spec->hdr.src_port;
1427 udp.mask.dst_port = mask->hdr.dst_port;
1428 udp.mask.src_port = mask->hdr.src_port;
1429 /* Remove unwanted bits from values. */
1430 udp.val.src_port &= udp.mask.src_port;
1431 udp.val.dst_port &= udp.mask.dst_port;
1433 mlx5_flow_create_copy(parser, &udp, udp_size);
1438 * Convert TCP item to Verbs specification.
1441 * Item specification.
1442 * @param default_mask[in]
1443 * Default bit-masks to use when item->mask is not provided.
1444 * @param data[in, out]
1448 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1449 const void *default_mask,
1452 const struct rte_flow_item_tcp *spec = item->spec;
1453 const struct rte_flow_item_tcp *mask = item->mask;
1454 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1455 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1456 struct ibv_flow_spec_tcp_udp tcp = {
1457 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1461 /* Don't update layer for the inner pattern. */
1462 if (!parser->inner) {
1463 if (parser->layer == HASH_RXQ_IPV4)
1464 parser->layer = HASH_RXQ_TCPV4;
1466 parser->layer = HASH_RXQ_TCPV6;
1470 mask = default_mask;
1471 tcp.val.dst_port = spec->hdr.dst_port;
1472 tcp.val.src_port = spec->hdr.src_port;
1473 tcp.mask.dst_port = mask->hdr.dst_port;
1474 tcp.mask.src_port = mask->hdr.src_port;
1475 /* Remove unwanted bits from values. */
1476 tcp.val.src_port &= tcp.mask.src_port;
1477 tcp.val.dst_port &= tcp.mask.dst_port;
1479 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1484 * Convert VXLAN item to Verbs specification.
1487 * Item specification.
1488 * @param default_mask[in]
1489 * Default bit-masks to use when item->mask is not provided.
1490 * @param data[in, out]
1494 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1495 const void *default_mask,
1498 const struct rte_flow_item_vxlan *spec = item->spec;
1499 const struct rte_flow_item_vxlan *mask = item->mask;
1500 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1501 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1502 struct ibv_flow_spec_tunnel vxlan = {
1503 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1512 parser->inner = IBV_FLOW_SPEC_INNER;
1515 mask = default_mask;
1516 memcpy(&id.vni[1], spec->vni, 3);
1517 vxlan.val.tunnel_id = id.vlan_id;
1518 memcpy(&id.vni[1], mask->vni, 3);
1519 vxlan.mask.tunnel_id = id.vlan_id;
1520 /* Remove unwanted bits from values. */
1521 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1524 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1525 * layer is defined in the Verbs specification it is interpreted as
1526 * wildcard and all packets will match this rule, if it follows a full
1527 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1528 * before will also match this rule.
1529 * To avoid such situation, VNI 0 is currently refused.
1531 if (!vxlan.val.tunnel_id)
1533 mlx5_flow_create_copy(parser, &vxlan, size);
1538 * Convert mark/flag action to Verbs specification.
1541 * Internal parser structure.
1546 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1548 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1549 struct ibv_flow_spec_action_tag tag = {
1550 .type = IBV_FLOW_SPEC_ACTION_TAG,
1552 .tag_id = mlx5_flow_mark_set(mark_id),
1555 assert(parser->mark);
1556 mlx5_flow_create_copy(parser, &tag, size);
1561 * Convert count action to Verbs specification.
1564 * Pointer to Ethernet device.
1566 * Pointer to MLX5 flow parser structure.
1569 * 0 on success, errno value on failure.
1572 mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
1573 struct mlx5_flow_parse *parser __rte_unused)
1575 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1576 struct priv *priv = dev->data->dev_private;
1577 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1578 struct ibv_counter_set_init_attr init_attr = {0};
1579 struct ibv_flow_spec_counter_action counter = {
1580 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1582 .counter_set_handle = 0,
1585 init_attr.counter_set_id = 0;
1586 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
1589 counter.counter_set_handle = parser->cs->handle;
1590 mlx5_flow_create_copy(parser, &counter, size);
1596 * Complete flow rule creation with a drop queue.
1599 * Pointer to Ethernet device.
1601 * Internal parser structure.
1603 * Pointer to the rte_flow.
1605 * Perform verbose error reporting if not NULL.
1608 * 0 on success, errno value on failure.
1611 mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
1612 struct mlx5_flow_parse *parser,
1613 struct rte_flow *flow,
1614 struct rte_flow_error *error)
1616 struct priv *priv = dev->data->dev_private;
1617 struct ibv_flow_spec_action_drop *drop;
1618 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1624 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
1625 parser->queue[HASH_RXQ_ETH].offset);
1626 *drop = (struct ibv_flow_spec_action_drop){
1627 .type = IBV_FLOW_SPEC_ACTION_DROP,
1630 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
1631 parser->queue[HASH_RXQ_ETH].offset += size;
1632 flow->frxq[HASH_RXQ_ETH].ibv_attr =
1633 parser->queue[HASH_RXQ_ETH].ibv_attr;
1635 flow->cs = parser->cs;
1636 if (!priv->dev->data->dev_started)
1638 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
1639 flow->frxq[HASH_RXQ_ETH].ibv_flow =
1640 mlx5_glue->create_flow(priv->flow_drop_queue->qp,
1641 flow->frxq[HASH_RXQ_ETH].ibv_attr);
1642 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1643 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1644 NULL, "flow rule creation failure");
1651 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1652 claim_zero(mlx5_glue->destroy_flow
1653 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
1654 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
1656 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
1657 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
1658 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
1661 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1669 * Create hash Rx queues when RSS is enabled.
1672 * Pointer to Ethernet device.
1674 * Internal parser structure.
1676 * Pointer to the rte_flow.
1678 * Perform verbose error reporting if not NULL.
1681 * 0 on success, a errno value otherwise and rte_errno is set.
1684 mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
1685 struct mlx5_flow_parse *parser,
1686 struct rte_flow *flow,
1687 struct rte_flow_error *error)
1689 struct priv *priv = dev->data->dev_private;
1692 for (i = 0; i != hash_rxq_init_n; ++i) {
1693 uint64_t hash_fields;
1695 if (!parser->queue[i].ibv_attr)
1697 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1698 parser->queue[i].ibv_attr = NULL;
1699 hash_fields = hash_rxq_init[i].hash_fields;
1700 if (!priv->dev->data->dev_started)
1702 flow->frxq[i].hrxq =
1704 parser->rss_conf.rss_key,
1705 parser->rss_conf.rss_key_len,
1709 if (flow->frxq[i].hrxq)
1711 flow->frxq[i].hrxq =
1713 parser->rss_conf.rss_key,
1714 parser->rss_conf.rss_key_len,
1718 if (!flow->frxq[i].hrxq) {
1719 rte_flow_error_set(error, ENOMEM,
1720 RTE_FLOW_ERROR_TYPE_HANDLE,
1721 NULL, "cannot create hash rxq");
1729 * Complete flow rule creation.
1732 * Pointer to Ethernet device.
1734 * Internal parser structure.
1736 * Pointer to the rte_flow.
1738 * Perform verbose error reporting if not NULL.
1741 * 0 on success, a errno value otherwise and rte_errno is set.
1744 mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
1745 struct mlx5_flow_parse *parser,
1746 struct rte_flow *flow,
1747 struct rte_flow_error *error)
1749 struct priv *priv = dev->data->dev_private;
1752 unsigned int flows_n = 0;
1756 assert(!parser->drop);
1757 err = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
1761 flow->cs = parser->cs;
1762 if (!priv->dev->data->dev_started)
1764 for (i = 0; i != hash_rxq_init_n; ++i) {
1765 if (!flow->frxq[i].hrxq)
1767 flow->frxq[i].ibv_flow =
1768 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
1769 flow->frxq[i].ibv_attr);
1770 if (!flow->frxq[i].ibv_flow) {
1771 rte_flow_error_set(error, ENOMEM,
1772 RTE_FLOW_ERROR_TYPE_HANDLE,
1773 NULL, "flow rule creation failure");
1778 DEBUG("%p type %d QP %p ibv_flow %p",
1780 (void *)flow->frxq[i].hrxq,
1781 (void *)flow->frxq[i].ibv_flow);
1784 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
1785 NULL, "internal error in flow creation");
1788 for (i = 0; i != parser->queues_n; ++i) {
1789 struct mlx5_rxq_data *q =
1790 (*priv->rxqs)[parser->queues[i]];
1792 q->mark |= parser->mark;
1797 for (i = 0; i != hash_rxq_init_n; ++i) {
1798 if (flow->frxq[i].ibv_flow) {
1799 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1801 claim_zero(mlx5_glue->destroy_flow(ibv_flow));
1803 if (flow->frxq[i].hrxq)
1804 mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
1805 if (flow->frxq[i].ibv_attr)
1806 rte_free(flow->frxq[i].ibv_attr);
1809 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1820 * Pointer to Ethernet device.
1822 * Pointer to a TAILQ flow list.
1824 * Flow rule attributes.
1825 * @param[in] pattern
1826 * Pattern specification (list terminated by the END pattern item).
1827 * @param[in] actions
1828 * Associated actions (list terminated by the END action).
1830 * Perform verbose error reporting if not NULL.
1833 * A flow on success, NULL otherwise.
1835 static struct rte_flow *
1836 mlx5_flow_list_create(struct rte_eth_dev *dev,
1837 struct mlx5_flows *list,
1838 const struct rte_flow_attr *attr,
1839 const struct rte_flow_item items[],
1840 const struct rte_flow_action actions[],
1841 struct rte_flow_error *error)
1843 struct mlx5_flow_parse parser = { .create = 1, };
1844 struct rte_flow *flow = NULL;
1848 err = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
1851 flow = rte_calloc(__func__, 1,
1852 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1855 rte_flow_error_set(error, ENOMEM,
1856 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1858 "cannot allocate flow memory");
1861 /* Copy queues configuration. */
1862 flow->queues = (uint16_t (*)[])(flow + 1);
1863 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1864 flow->queues_n = parser.queues_n;
1865 flow->mark = parser.mark;
1866 /* Copy RSS configuration. */
1867 flow->rss_conf = parser.rss_conf;
1868 flow->rss_conf.rss_key = flow->rss_key;
1869 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1870 /* finalise the flow. */
1872 err = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
1875 err = mlx5_flow_create_action_queue(dev, &parser, flow, error);
1878 TAILQ_INSERT_TAIL(list, flow, next);
1879 DEBUG("Flow created %p", (void *)flow);
1882 ERROR("flow creation error: %s", error->message);
1883 for (i = 0; i != hash_rxq_init_n; ++i) {
1884 if (parser.queue[i].ibv_attr)
1885 rte_free(parser.queue[i].ibv_attr);
1892 * Validate a flow supported by the NIC.
1894 * @see rte_flow_validate()
1898 mlx5_flow_validate(struct rte_eth_dev *dev,
1899 const struct rte_flow_attr *attr,
1900 const struct rte_flow_item items[],
1901 const struct rte_flow_action actions[],
1902 struct rte_flow_error *error)
1905 struct mlx5_flow_parse parser = { .create = 0, };
1907 ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
1914 * @see rte_flow_create()
1918 mlx5_flow_create(struct rte_eth_dev *dev,
1919 const struct rte_flow_attr *attr,
1920 const struct rte_flow_item items[],
1921 const struct rte_flow_action actions[],
1922 struct rte_flow_error *error)
1924 struct priv *priv = dev->data->dev_private;
1926 return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
1931 * Destroy a flow in a list.
1934 * Pointer to Ethernet device.
1936 * Pointer to a TAILQ flow list.
1941 mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
1942 struct rte_flow *flow)
1944 struct priv *priv = dev->data->dev_private;
1947 if (flow->drop || !flow->mark)
1949 for (i = 0; i != flow->queues_n; ++i) {
1950 struct rte_flow *tmp;
1954 * To remove the mark from the queue, the queue must not be
1955 * present in any other marked flow (RSS or not).
1957 TAILQ_FOREACH(tmp, list, next) {
1959 uint16_t *tqs = NULL;
1964 for (j = 0; j != hash_rxq_init_n; ++j) {
1965 if (!tmp->frxq[j].hrxq)
1967 tqs = tmp->frxq[j].hrxq->ind_table->queues;
1968 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
1972 for (j = 0; (j != tq_n) && !mark; j++)
1973 if (tqs[j] == (*flow->queues)[i])
1976 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
1980 if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
1981 claim_zero(mlx5_glue->destroy_flow
1982 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
1983 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
1985 for (i = 0; i != hash_rxq_init_n; ++i) {
1986 struct mlx5_flow *frxq = &flow->frxq[i];
1989 claim_zero(mlx5_glue->destroy_flow
1992 mlx5_hrxq_release(dev, frxq->hrxq);
1994 rte_free(frxq->ibv_attr);
1998 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
2001 TAILQ_REMOVE(list, flow, next);
2002 DEBUG("Flow destroyed %p", (void *)flow);
2007 * Destroy all flows.
2010 * Pointer to Ethernet device.
2012 * Pointer to a TAILQ flow list.
2015 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
2017 while (!TAILQ_EMPTY(list)) {
2018 struct rte_flow *flow;
2020 flow = TAILQ_FIRST(list);
2021 mlx5_flow_list_destroy(dev, list, flow);
2026 * Create drop queue.
2029 * Pointer to Ethernet device.
2035 mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
2037 struct priv *priv = dev->data->dev_private;
2038 struct mlx5_hrxq_drop *fdq = NULL;
2042 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2044 WARN("cannot allocate memory for drop queue");
2047 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
2049 WARN("cannot allocate CQ for drop queue");
2052 fdq->wq = mlx5_glue->create_wq
2054 &(struct ibv_wq_init_attr){
2055 .wq_type = IBV_WQT_RQ,
2062 WARN("cannot allocate WQ for drop queue");
2065 fdq->ind_table = mlx5_glue->create_rwq_ind_table
2067 &(struct ibv_rwq_ind_table_init_attr){
2068 .log_ind_tbl_size = 0,
2069 .ind_tbl = &fdq->wq,
2072 if (!fdq->ind_table) {
2073 WARN("cannot allocate indirection table for drop queue");
2076 fdq->qp = mlx5_glue->create_qp_ex
2078 &(struct ibv_qp_init_attr_ex){
2079 .qp_type = IBV_QPT_RAW_PACKET,
2081 IBV_QP_INIT_ATTR_PD |
2082 IBV_QP_INIT_ATTR_IND_TABLE |
2083 IBV_QP_INIT_ATTR_RX_HASH,
2084 .rx_hash_conf = (struct ibv_rx_hash_conf){
2086 IBV_RX_HASH_FUNC_TOEPLITZ,
2087 .rx_hash_key_len = rss_hash_default_key_len,
2088 .rx_hash_key = rss_hash_default_key,
2089 .rx_hash_fields_mask = 0,
2091 .rwq_ind_tbl = fdq->ind_table,
2095 WARN("cannot allocate QP for drop queue");
2098 priv->flow_drop_queue = fdq;
2102 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2104 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2106 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2108 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2111 priv->flow_drop_queue = NULL;
2116 * Delete drop queue.
2119 * Pointer to Ethernet device.
2122 mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
2124 struct priv *priv = dev->data->dev_private;
2125 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2130 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2132 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2134 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2136 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2138 priv->flow_drop_queue = NULL;
2145 * Pointer to Ethernet device.
2147 * Pointer to a TAILQ flow list.
2150 mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
2152 struct priv *priv = dev->data->dev_private;
2153 struct rte_flow *flow;
2155 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2157 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2160 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
2162 claim_zero(mlx5_glue->destroy_flow
2163 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2164 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
2165 DEBUG("Flow %p removed", (void *)flow);
2169 /* Verify the flow has not already been cleaned. */
2170 for (i = 0; i != hash_rxq_init_n; ++i) {
2171 if (!flow->frxq[i].ibv_flow)
2174 * Indirection table may be necessary to remove the
2175 * flags in the Rx queues.
2176 * This helps to speed-up the process by avoiding
2179 ind_tbl = flow->frxq[i].hrxq->ind_table;
2182 if (i == hash_rxq_init_n)
2186 for (i = 0; i != ind_tbl->queues_n; ++i)
2187 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2189 for (i = 0; i != hash_rxq_init_n; ++i) {
2190 if (!flow->frxq[i].ibv_flow)
2192 claim_zero(mlx5_glue->destroy_flow
2193 (flow->frxq[i].ibv_flow));
2194 flow->frxq[i].ibv_flow = NULL;
2195 mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
2196 flow->frxq[i].hrxq = NULL;
2198 DEBUG("Flow %p removed", (void *)flow);
2206 * Pointer to Ethernet device.
2208 * Pointer to a TAILQ flow list.
2211 * 0 on success, a errno value otherwise and rte_errno is set.
2214 mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
2216 struct priv *priv = dev->data->dev_private;
2217 struct rte_flow *flow;
2219 TAILQ_FOREACH(flow, list, next) {
2223 flow->frxq[HASH_RXQ_ETH].ibv_flow =
2224 mlx5_glue->create_flow
2225 (priv->flow_drop_queue->qp,
2226 flow->frxq[HASH_RXQ_ETH].ibv_attr);
2227 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
2228 DEBUG("Flow %p cannot be applied",
2233 DEBUG("Flow %p applied", (void *)flow);
2237 for (i = 0; i != hash_rxq_init_n; ++i) {
2238 if (!flow->frxq[i].ibv_attr)
2240 flow->frxq[i].hrxq =
2241 mlx5_hrxq_get(dev, flow->rss_conf.rss_key,
2242 flow->rss_conf.rss_key_len,
2243 hash_rxq_init[i].hash_fields,
2246 if (flow->frxq[i].hrxq)
2248 flow->frxq[i].hrxq =
2249 mlx5_hrxq_new(dev, flow->rss_conf.rss_key,
2250 flow->rss_conf.rss_key_len,
2251 hash_rxq_init[i].hash_fields,
2254 if (!flow->frxq[i].hrxq) {
2255 DEBUG("Flow %p cannot be applied",
2261 flow->frxq[i].ibv_flow =
2262 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
2263 flow->frxq[i].ibv_attr);
2264 if (!flow->frxq[i].ibv_flow) {
2265 DEBUG("Flow %p cannot be applied",
2270 DEBUG("Flow %p applied", (void *)flow);
2274 for (i = 0; i != flow->queues_n; ++i)
2275 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2281 * Verify the flow list is empty
2284 * Pointer to Ethernet device.
2286 * @return the number of flows not released.
2289 mlx5_flow_verify(struct rte_eth_dev *dev)
2291 struct priv *priv = dev->data->dev_private;
2292 struct rte_flow *flow;
2295 TAILQ_FOREACH(flow, &priv->flows, next) {
2296 DEBUG("%p: flow %p still referenced", (void *)dev,
2304 * Enable a control flow configured from the control plane.
2307 * Pointer to Ethernet device.
2309 * An Ethernet flow spec to apply.
2311 * An Ethernet flow mask to apply.
2313 * A VLAN flow spec to apply.
2315 * A VLAN flow mask to apply.
2321 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2322 struct rte_flow_item_eth *eth_spec,
2323 struct rte_flow_item_eth *eth_mask,
2324 struct rte_flow_item_vlan *vlan_spec,
2325 struct rte_flow_item_vlan *vlan_mask)
2327 struct priv *priv = dev->data->dev_private;
2328 const struct rte_flow_attr attr = {
2330 .priority = MLX5_CTRL_FLOW_PRIORITY,
2332 struct rte_flow_item items[] = {
2334 .type = RTE_FLOW_ITEM_TYPE_ETH,
2340 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2341 RTE_FLOW_ITEM_TYPE_END,
2347 .type = RTE_FLOW_ITEM_TYPE_END,
2350 struct rte_flow_action actions[] = {
2352 .type = RTE_FLOW_ACTION_TYPE_RSS,
2355 .type = RTE_FLOW_ACTION_TYPE_END,
2358 struct rte_flow *flow;
2359 struct rte_flow_error error;
2362 struct rte_flow_action_rss rss;
2364 const struct rte_eth_rss_conf *rss_conf;
2366 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2370 if (!priv->reta_idx_n)
2372 for (i = 0; i != priv->reta_idx_n; ++i)
2373 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2374 action_rss.local.rss_conf = &priv->rss_conf;
2375 action_rss.local.num = priv->reta_idx_n;
2376 actions[0].conf = (const void *)&action_rss.rss;
2377 flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
2385 * Enable a flow control configured from the control plane.
2388 * Pointer to Ethernet device.
2390 * An Ethernet flow spec to apply.
2392 * An Ethernet flow mask to apply.
2398 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2399 struct rte_flow_item_eth *eth_spec,
2400 struct rte_flow_item_eth *eth_mask)
2402 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2408 * @see rte_flow_destroy()
2412 mlx5_flow_destroy(struct rte_eth_dev *dev,
2413 struct rte_flow *flow,
2414 struct rte_flow_error *error __rte_unused)
2416 struct priv *priv = dev->data->dev_private;
2418 mlx5_flow_list_destroy(dev, &priv->flows, flow);
2423 * Destroy all flows.
2425 * @see rte_flow_flush()
2429 mlx5_flow_flush(struct rte_eth_dev *dev,
2430 struct rte_flow_error *error __rte_unused)
2432 struct priv *priv = dev->data->dev_private;
2434 mlx5_flow_list_flush(dev, &priv->flows);
2438 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2440 * Query flow counter.
2444 * @param counter_value
2445 * returned data from the counter.
2448 * 0 on success, a errno value otherwise and rte_errno is set.
2451 mlx5_flow_query_count(struct ibv_counter_set *cs,
2452 struct mlx5_flow_counter_stats *counter_stats,
2453 struct rte_flow_query_count *query_count,
2454 struct rte_flow_error *error)
2456 uint64_t counters[2];
2457 struct ibv_query_counter_set_attr query_cs_attr = {
2459 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2461 struct ibv_counter_set_data query_out = {
2463 .outlen = 2 * sizeof(uint64_t),
2465 int res = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
2468 rte_flow_error_set(error, -res,
2469 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2471 "cannot read counter");
2474 query_count->hits_set = 1;
2475 query_count->bytes_set = 1;
2476 query_count->hits = counters[0] - counter_stats->hits;
2477 query_count->bytes = counters[1] - counter_stats->bytes;
2478 if (query_count->reset) {
2479 counter_stats->hits = counters[0];
2480 counter_stats->bytes = counters[1];
2488 * @see rte_flow_query()
2492 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
2493 struct rte_flow *flow,
2494 enum rte_flow_action_type action __rte_unused,
2496 struct rte_flow_error *error)
2501 res = mlx5_flow_query_count(flow->cs,
2502 &flow->counter_stats,
2503 (struct rte_flow_query_count *)data,
2506 rte_flow_error_set(error, res,
2507 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2509 "no counter found for flow");
2518 * @see rte_flow_isolate()
2522 mlx5_flow_isolate(struct rte_eth_dev *dev,
2524 struct rte_flow_error *error)
2526 struct priv *priv = dev->data->dev_private;
2528 if (dev->data->dev_started) {
2529 rte_flow_error_set(error, EBUSY,
2530 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2532 "port must be stopped first");
2535 priv->isolated = !!enable;
2537 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2539 priv->dev->dev_ops = &mlx5_dev_ops;
2544 * Convert a flow director filter to a generic flow.
2547 * Pointer to Ethernet device.
2548 * @param fdir_filter
2549 * Flow director filter to add.
2551 * Generic flow parameters structure.
2554 * 0 on success, errno value on error.
2557 mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
2558 const struct rte_eth_fdir_filter *fdir_filter,
2559 struct mlx5_fdir *attributes)
2561 struct priv *priv = dev->data->dev_private;
2562 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2564 /* Validate queue number. */
2565 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2566 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2569 attributes->attr.ingress = 1;
2570 attributes->items[0] = (struct rte_flow_item) {
2571 .type = RTE_FLOW_ITEM_TYPE_ETH,
2572 .spec = &attributes->l2,
2573 .mask = &attributes->l2_mask,
2575 switch (fdir_filter->action.behavior) {
2576 case RTE_ETH_FDIR_ACCEPT:
2577 attributes->actions[0] = (struct rte_flow_action){
2578 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2579 .conf = &attributes->queue,
2582 case RTE_ETH_FDIR_REJECT:
2583 attributes->actions[0] = (struct rte_flow_action){
2584 .type = RTE_FLOW_ACTION_TYPE_DROP,
2588 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2591 attributes->queue.index = fdir_filter->action.rx_queue;
2592 switch (fdir_filter->input.flow_type) {
2593 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2594 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2595 .src_addr = input->flow.udp4_flow.ip.src_ip,
2596 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2597 .time_to_live = input->flow.udp4_flow.ip.ttl,
2598 .type_of_service = input->flow.udp4_flow.ip.tos,
2599 .next_proto_id = input->flow.udp4_flow.ip.proto,
2601 attributes->l4.udp.hdr = (struct udp_hdr){
2602 .src_port = input->flow.udp4_flow.src_port,
2603 .dst_port = input->flow.udp4_flow.dst_port,
2605 attributes->items[1] = (struct rte_flow_item){
2606 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2607 .spec = &attributes->l3,
2608 .mask = &attributes->l3,
2610 attributes->items[2] = (struct rte_flow_item){
2611 .type = RTE_FLOW_ITEM_TYPE_UDP,
2612 .spec = &attributes->l4,
2613 .mask = &attributes->l4,
2616 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2617 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2618 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2619 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2620 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2621 .type_of_service = input->flow.tcp4_flow.ip.tos,
2622 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2624 attributes->l4.tcp.hdr = (struct tcp_hdr){
2625 .src_port = input->flow.tcp4_flow.src_port,
2626 .dst_port = input->flow.tcp4_flow.dst_port,
2628 attributes->items[1] = (struct rte_flow_item){
2629 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2630 .spec = &attributes->l3,
2631 .mask = &attributes->l3,
2633 attributes->items[2] = (struct rte_flow_item){
2634 .type = RTE_FLOW_ITEM_TYPE_TCP,
2635 .spec = &attributes->l4,
2636 .mask = &attributes->l4,
2639 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2640 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2641 .src_addr = input->flow.ip4_flow.src_ip,
2642 .dst_addr = input->flow.ip4_flow.dst_ip,
2643 .time_to_live = input->flow.ip4_flow.ttl,
2644 .type_of_service = input->flow.ip4_flow.tos,
2645 .next_proto_id = input->flow.ip4_flow.proto,
2647 attributes->items[1] = (struct rte_flow_item){
2648 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2649 .spec = &attributes->l3,
2650 .mask = &attributes->l3,
2653 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2654 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2655 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2656 .proto = input->flow.udp6_flow.ip.proto,
2658 memcpy(attributes->l3.ipv6.hdr.src_addr,
2659 input->flow.udp6_flow.ip.src_ip,
2660 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2661 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2662 input->flow.udp6_flow.ip.dst_ip,
2663 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2664 attributes->l4.udp.hdr = (struct udp_hdr){
2665 .src_port = input->flow.udp6_flow.src_port,
2666 .dst_port = input->flow.udp6_flow.dst_port,
2668 attributes->items[1] = (struct rte_flow_item){
2669 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2670 .spec = &attributes->l3,
2671 .mask = &attributes->l3,
2673 attributes->items[2] = (struct rte_flow_item){
2674 .type = RTE_FLOW_ITEM_TYPE_UDP,
2675 .spec = &attributes->l4,
2676 .mask = &attributes->l4,
2679 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2680 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2681 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2682 .proto = input->flow.tcp6_flow.ip.proto,
2684 memcpy(attributes->l3.ipv6.hdr.src_addr,
2685 input->flow.tcp6_flow.ip.src_ip,
2686 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2687 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2688 input->flow.tcp6_flow.ip.dst_ip,
2689 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2690 attributes->l4.tcp.hdr = (struct tcp_hdr){
2691 .src_port = input->flow.tcp6_flow.src_port,
2692 .dst_port = input->flow.tcp6_flow.dst_port,
2694 attributes->items[1] = (struct rte_flow_item){
2695 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2696 .spec = &attributes->l3,
2697 .mask = &attributes->l3,
2699 attributes->items[2] = (struct rte_flow_item){
2700 .type = RTE_FLOW_ITEM_TYPE_TCP,
2701 .spec = &attributes->l4,
2702 .mask = &attributes->l4,
2705 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2706 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2707 .hop_limits = input->flow.ipv6_flow.hop_limits,
2708 .proto = input->flow.ipv6_flow.proto,
2710 memcpy(attributes->l3.ipv6.hdr.src_addr,
2711 input->flow.ipv6_flow.src_ip,
2712 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2713 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2714 input->flow.ipv6_flow.dst_ip,
2715 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2716 attributes->items[1] = (struct rte_flow_item){
2717 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2718 .spec = &attributes->l3,
2719 .mask = &attributes->l3,
2723 ERROR("invalid flow type%d",
2724 fdir_filter->input.flow_type);
2731 * Add new flow director filter and store it in list.
2734 * Pointer to Ethernet device.
2735 * @param fdir_filter
2736 * Flow director filter to add.
2739 * 0 on success, errno value on failure.
2742 mlx5_fdir_filter_add(struct rte_eth_dev *dev,
2743 const struct rte_eth_fdir_filter *fdir_filter)
2745 struct priv *priv = dev->data->dev_private;
2746 struct mlx5_fdir attributes = {
2749 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2750 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2754 struct mlx5_flow_parse parser = {
2755 .layer = HASH_RXQ_ETH,
2757 struct rte_flow_error error;
2758 struct rte_flow *flow;
2761 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
2764 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
2765 attributes.actions, &error, &parser);
2768 flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
2769 attributes.items, attributes.actions,
2772 DEBUG("FDIR created %p", (void *)flow);
2779 * Delete specific filter.
2782 * Pointer to Ethernet device.
2783 * @param fdir_filter
2784 * Filter to be deleted.
2787 * 0 on success, errno value on failure.
2790 mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
2791 const struct rte_eth_fdir_filter *fdir_filter)
2793 struct priv *priv = dev->data->dev_private;
2794 struct mlx5_fdir attributes = {
2797 struct mlx5_flow_parse parser = {
2799 .layer = HASH_RXQ_ETH,
2801 struct rte_flow_error error;
2802 struct rte_flow *flow;
2806 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
2809 ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
2810 attributes.actions, &error, &parser);
2814 * Special case for drop action which is only set in the
2815 * specifications when the flow is created. In this situation the
2816 * drop specification is missing.
2819 struct ibv_flow_spec_action_drop *drop;
2821 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
2822 parser.queue[HASH_RXQ_ETH].offset);
2823 *drop = (struct ibv_flow_spec_action_drop){
2824 .type = IBV_FLOW_SPEC_ACTION_DROP,
2825 .size = sizeof(struct ibv_flow_spec_action_drop),
2827 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
2829 TAILQ_FOREACH(flow, &priv->flows, next) {
2830 struct ibv_flow_attr *attr;
2831 struct ibv_spec_header *attr_h;
2833 struct ibv_flow_attr *flow_attr;
2834 struct ibv_spec_header *flow_h;
2836 unsigned int specs_n;
2838 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2839 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2840 /* Compare first the attributes. */
2841 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2843 if (attr->num_of_specs == 0)
2845 spec = (void *)((uintptr_t)attr +
2846 sizeof(struct ibv_flow_attr));
2847 flow_spec = (void *)((uintptr_t)flow_attr +
2848 sizeof(struct ibv_flow_attr));
2849 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2850 for (i = 0; i != specs_n; ++i) {
2853 if (memcmp(spec, flow_spec,
2854 RTE_MIN(attr_h->size, flow_h->size)))
2856 spec = (void *)((uintptr_t)spec + attr_h->size);
2857 flow_spec = (void *)((uintptr_t)flow_spec +
2860 /* At this point, the flow match. */
2863 /* The flow does not match. */
2867 mlx5_flow_list_destroy(dev, &priv->flows, flow);
2869 for (i = 0; i != hash_rxq_init_n; ++i) {
2870 if (parser.queue[i].ibv_attr)
2871 rte_free(parser.queue[i].ibv_attr);
2877 * Update queue for specific filter.
2880 * Pointer to Ethernet device.
2881 * @param fdir_filter
2882 * Filter to be updated.
2885 * 0 on success, errno value on failure.
2888 mlx5_fdir_filter_update(struct rte_eth_dev *dev,
2889 const struct rte_eth_fdir_filter *fdir_filter)
2893 ret = mlx5_fdir_filter_delete(dev, fdir_filter);
2896 ret = mlx5_fdir_filter_add(dev, fdir_filter);
2901 * Flush all filters.
2904 * Pointer to Ethernet device.
2907 mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
2909 struct priv *priv = dev->data->dev_private;
2911 mlx5_flow_list_flush(dev, &priv->flows);
2915 * Get flow director information.
2918 * Pointer to Ethernet device.
2919 * @param[out] fdir_info
2920 * Resulting flow director information.
2923 mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
2925 struct priv *priv = dev->data->dev_private;
2926 struct rte_eth_fdir_masks *mask =
2927 &priv->dev->data->dev_conf.fdir_conf.mask;
2929 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2930 fdir_info->guarant_spc = 0;
2931 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2932 fdir_info->max_flexpayload = 0;
2933 fdir_info->flow_types_mask[0] = 0;
2934 fdir_info->flex_payload_unit = 0;
2935 fdir_info->max_flex_payload_segment_num = 0;
2936 fdir_info->flex_payload_limit = 0;
2937 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2941 * Deal with flow director operations.
2944 * Pointer to Ethernet device.
2946 * Operation to perform.
2948 * Pointer to operation-specific structure.
2951 * 0 on success, errno value on failure.
2954 mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
2957 struct priv *priv = dev->data->dev_private;
2958 enum rte_fdir_mode fdir_mode =
2959 priv->dev->data->dev_conf.fdir_conf.mode;
2962 if (filter_op == RTE_ETH_FILTER_NOP)
2964 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2965 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2966 ERROR("%p: flow director mode %d not supported",
2967 (void *)dev, fdir_mode);
2970 switch (filter_op) {
2971 case RTE_ETH_FILTER_ADD:
2972 ret = mlx5_fdir_filter_add(dev, arg);
2974 case RTE_ETH_FILTER_UPDATE:
2975 ret = mlx5_fdir_filter_update(dev, arg);
2977 case RTE_ETH_FILTER_DELETE:
2978 ret = mlx5_fdir_filter_delete(dev, arg);
2980 case RTE_ETH_FILTER_FLUSH:
2981 mlx5_fdir_filter_flush(dev);
2983 case RTE_ETH_FILTER_INFO:
2984 mlx5_fdir_info_get(dev, arg);
2987 DEBUG("%p: unknown operation %u", (void *)dev,
2996 * Manage filter operations.
2999 * Pointer to Ethernet device structure.
3000 * @param filter_type
3003 * Operation to perform.
3005 * Pointer to operation-specific structure.
3008 * 0 on success, negative errno value on failure.
3011 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3012 enum rte_filter_type filter_type,
3013 enum rte_filter_op filter_op,
3018 switch (filter_type) {
3019 case RTE_ETH_FILTER_GENERIC:
3020 if (filter_op != RTE_ETH_FILTER_GET)
3022 *(const void **)arg = &mlx5_flow_ops;
3024 case RTE_ETH_FILTER_FDIR:
3025 ret = mlx5_fdir_ctrl_func(dev, filter_op, arg);
3028 ERROR("%p: filter type (%d) not supported",
3029 (void *)dev, filter_type);