1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox.
10 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
12 #pragma GCC diagnostic ignored "-Wpedantic"
14 #include <infiniband/verbs.h>
16 #pragma GCC diagnostic error "-Wpedantic"
19 #include <rte_ethdev_driver.h>
21 #include <rte_flow_driver.h>
22 #include <rte_malloc.h>
26 #include "mlx5_defs.h"
28 #include "mlx5_glue.h"
30 /* Define minimal priority for control plane flows. */
31 #define MLX5_CTRL_FLOW_PRIORITY 4
33 /* Internet Protocol versions. */
37 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
38 struct ibv_flow_spec_counter_action {
43 /* Dev ops structure defined in mlx5.c */
44 extern const struct eth_dev_ops mlx5_dev_ops;
45 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
48 mlx5_flow_create_eth(const struct rte_flow_item *item,
49 const void *default_mask,
53 mlx5_flow_create_vlan(const struct rte_flow_item *item,
54 const void *default_mask,
58 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
59 const void *default_mask,
63 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
64 const void *default_mask,
68 mlx5_flow_create_udp(const struct rte_flow_item *item,
69 const void *default_mask,
73 mlx5_flow_create_tcp(const struct rte_flow_item *item,
74 const void *default_mask,
78 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
79 const void *default_mask,
82 struct mlx5_flow_parse;
85 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
89 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
92 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
94 /* Hash RX queue types. */
105 /* Initialization data for hash RX queue. */
106 struct hash_rxq_init {
107 uint64_t hash_fields; /* Fields that participate in the hash. */
108 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
109 unsigned int flow_priority; /* Flow priority to use. */
110 unsigned int ip_version; /* Internet protocol. */
113 /* Initialization data for hash RX queues. */
114 const struct hash_rxq_init hash_rxq_init[] = {
116 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
117 IBV_RX_HASH_DST_IPV4 |
118 IBV_RX_HASH_SRC_PORT_TCP |
119 IBV_RX_HASH_DST_PORT_TCP),
120 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
122 .ip_version = MLX5_IPV4,
125 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
126 IBV_RX_HASH_DST_IPV4 |
127 IBV_RX_HASH_SRC_PORT_UDP |
128 IBV_RX_HASH_DST_PORT_UDP),
129 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
131 .ip_version = MLX5_IPV4,
134 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
135 IBV_RX_HASH_DST_IPV4),
136 .dpdk_rss_hf = (ETH_RSS_IPV4 |
139 .ip_version = MLX5_IPV4,
142 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
143 IBV_RX_HASH_DST_IPV6 |
144 IBV_RX_HASH_SRC_PORT_TCP |
145 IBV_RX_HASH_DST_PORT_TCP),
146 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
148 .ip_version = MLX5_IPV6,
151 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
152 IBV_RX_HASH_DST_IPV6 |
153 IBV_RX_HASH_SRC_PORT_UDP |
154 IBV_RX_HASH_DST_PORT_UDP),
155 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
157 .ip_version = MLX5_IPV6,
160 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
161 IBV_RX_HASH_DST_IPV6),
162 .dpdk_rss_hf = (ETH_RSS_IPV6 |
165 .ip_version = MLX5_IPV6,
174 /* Number of entries in hash_rxq_init[]. */
175 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
177 /** Structure for holding counter stats. */
178 struct mlx5_flow_counter_stats {
179 uint64_t hits; /**< Number of packets matched by the rule. */
180 uint64_t bytes; /**< Number of bytes matched by the rule. */
183 /** Structure for Drop queue. */
184 struct mlx5_hrxq_drop {
185 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
186 struct ibv_qp *qp; /**< Verbs queue pair. */
187 struct ibv_wq *wq; /**< Verbs work queue. */
188 struct ibv_cq *cq; /**< Verbs completion queue. */
191 /* Flows structures. */
193 uint64_t hash_fields; /**< Fields that participate in the hash. */
194 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
195 struct ibv_flow *ibv_flow; /**< Verbs flow. */
196 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
199 /* Drop flows structures. */
200 struct mlx5_flow_drop {
201 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
202 struct ibv_flow *ibv_flow; /**< Verbs flow. */
206 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
207 uint32_t mark:1; /**< Set if the flow is marked. */
208 uint32_t drop:1; /**< Drop queue. */
209 uint16_t queues_n; /**< Number of entries in queue[]. */
210 uint16_t (*queues)[]; /**< Queues indexes to use. */
211 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
212 uint8_t rss_key[40]; /**< copy of the RSS key. */
213 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
214 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
215 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
216 /**< Flow with Rx queue. */
219 /** Static initializer for items. */
221 (const enum rte_flow_item_type []){ \
222 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
225 /** Structure to generate a simple graph of layers supported by the NIC. */
226 struct mlx5_flow_items {
227 /** List of possible actions for these items. */
228 const enum rte_flow_action_type *const actions;
229 /** Bit-masks corresponding to the possibilities for the item. */
232 * Default bit-masks to use when item->mask is not provided. When
233 * \default_mask is also NULL, the full supported bit-mask (\mask) is
236 const void *default_mask;
237 /** Bit-masks size in bytes. */
238 const unsigned int mask_sz;
240 * Conversion function from rte_flow to NIC specific flow.
243 * rte_flow item to convert.
244 * @param default_mask
245 * Default bit-masks to use when item->mask is not provided.
247 * Internal structure to store the conversion.
250 * 0 on success, negative value otherwise.
252 int (*convert)(const struct rte_flow_item *item,
253 const void *default_mask,
255 /** Size in bytes of the destination structure. */
256 const unsigned int dst_sz;
257 /** List of possible following items. */
258 const enum rte_flow_item_type *const items;
261 /** Valid action for this PMD. */
262 static const enum rte_flow_action_type valid_actions[] = {
263 RTE_FLOW_ACTION_TYPE_DROP,
264 RTE_FLOW_ACTION_TYPE_QUEUE,
265 RTE_FLOW_ACTION_TYPE_MARK,
266 RTE_FLOW_ACTION_TYPE_FLAG,
267 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
268 RTE_FLOW_ACTION_TYPE_COUNT,
270 RTE_FLOW_ACTION_TYPE_END,
273 /** Graph of supported items and associated actions. */
274 static const struct mlx5_flow_items mlx5_flow_items[] = {
275 [RTE_FLOW_ITEM_TYPE_END] = {
276 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
277 RTE_FLOW_ITEM_TYPE_VXLAN),
279 [RTE_FLOW_ITEM_TYPE_ETH] = {
280 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
281 RTE_FLOW_ITEM_TYPE_IPV4,
282 RTE_FLOW_ITEM_TYPE_IPV6),
283 .actions = valid_actions,
284 .mask = &(const struct rte_flow_item_eth){
285 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
286 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
289 .default_mask = &rte_flow_item_eth_mask,
290 .mask_sz = sizeof(struct rte_flow_item_eth),
291 .convert = mlx5_flow_create_eth,
292 .dst_sz = sizeof(struct ibv_flow_spec_eth),
294 [RTE_FLOW_ITEM_TYPE_VLAN] = {
295 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
296 RTE_FLOW_ITEM_TYPE_IPV6),
297 .actions = valid_actions,
298 .mask = &(const struct rte_flow_item_vlan){
301 .default_mask = &rte_flow_item_vlan_mask,
302 .mask_sz = sizeof(struct rte_flow_item_vlan),
303 .convert = mlx5_flow_create_vlan,
306 [RTE_FLOW_ITEM_TYPE_IPV4] = {
307 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
308 RTE_FLOW_ITEM_TYPE_TCP),
309 .actions = valid_actions,
310 .mask = &(const struct rte_flow_item_ipv4){
314 .type_of_service = -1,
318 .default_mask = &rte_flow_item_ipv4_mask,
319 .mask_sz = sizeof(struct rte_flow_item_ipv4),
320 .convert = mlx5_flow_create_ipv4,
321 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
323 [RTE_FLOW_ITEM_TYPE_IPV6] = {
324 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
325 RTE_FLOW_ITEM_TYPE_TCP),
326 .actions = valid_actions,
327 .mask = &(const struct rte_flow_item_ipv6){
330 0xff, 0xff, 0xff, 0xff,
331 0xff, 0xff, 0xff, 0xff,
332 0xff, 0xff, 0xff, 0xff,
333 0xff, 0xff, 0xff, 0xff,
336 0xff, 0xff, 0xff, 0xff,
337 0xff, 0xff, 0xff, 0xff,
338 0xff, 0xff, 0xff, 0xff,
339 0xff, 0xff, 0xff, 0xff,
346 .default_mask = &rte_flow_item_ipv6_mask,
347 .mask_sz = sizeof(struct rte_flow_item_ipv6),
348 .convert = mlx5_flow_create_ipv6,
349 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
351 [RTE_FLOW_ITEM_TYPE_UDP] = {
352 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
353 .actions = valid_actions,
354 .mask = &(const struct rte_flow_item_udp){
360 .default_mask = &rte_flow_item_udp_mask,
361 .mask_sz = sizeof(struct rte_flow_item_udp),
362 .convert = mlx5_flow_create_udp,
363 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
365 [RTE_FLOW_ITEM_TYPE_TCP] = {
366 .actions = valid_actions,
367 .mask = &(const struct rte_flow_item_tcp){
373 .default_mask = &rte_flow_item_tcp_mask,
374 .mask_sz = sizeof(struct rte_flow_item_tcp),
375 .convert = mlx5_flow_create_tcp,
376 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
378 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
379 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
380 .actions = valid_actions,
381 .mask = &(const struct rte_flow_item_vxlan){
382 .vni = "\xff\xff\xff",
384 .default_mask = &rte_flow_item_vxlan_mask,
385 .mask_sz = sizeof(struct rte_flow_item_vxlan),
386 .convert = mlx5_flow_create_vxlan,
387 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
391 /** Structure to pass to the conversion function. */
392 struct mlx5_flow_parse {
393 uint32_t inner; /**< Set once VXLAN is encountered. */
395 /**< Whether resources should remain after a validate. */
396 uint32_t drop:1; /**< Target is a drop queue. */
397 uint32_t mark:1; /**< Mark is present in the flow. */
398 uint32_t count:1; /**< Count is present in the flow. */
399 uint32_t mark_id; /**< Mark identifier. */
400 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
401 uint16_t queues_n; /**< Number of entries in queue[]. */
402 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
403 uint8_t rss_key[40]; /**< copy of the RSS key. */
404 enum hash_rxq_type layer; /**< Last pattern layer detected. */
405 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
407 struct ibv_flow_attr *ibv_attr;
408 /**< Pointer to Verbs attributes. */
410 /**< Current position or total size of the attribute. */
411 } queue[RTE_DIM(hash_rxq_init)];
414 static const struct rte_flow_ops mlx5_flow_ops = {
415 .validate = mlx5_flow_validate,
416 .create = mlx5_flow_create,
417 .destroy = mlx5_flow_destroy,
418 .flush = mlx5_flow_flush,
419 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
420 .query = mlx5_flow_query,
424 .isolate = mlx5_flow_isolate,
427 /* Convert FDIR request to Generic flow. */
429 struct rte_flow_attr attr;
430 struct rte_flow_action actions[2];
431 struct rte_flow_item items[4];
432 struct rte_flow_item_eth l2;
433 struct rte_flow_item_eth l2_mask;
435 struct rte_flow_item_ipv4 ipv4;
436 struct rte_flow_item_ipv6 ipv6;
439 struct rte_flow_item_udp udp;
440 struct rte_flow_item_tcp tcp;
442 struct rte_flow_action_queue queue;
445 /* Verbs specification header. */
446 struct ibv_spec_header {
447 enum ibv_flow_spec_type type;
452 * Check support for a given item.
455 * Item specification.
457 * Bit-masks covering supported fields to compare with spec, last and mask in
460 * Bit-Mask size in bytes.
466 mlx5_flow_item_validate(const struct rte_flow_item *item,
467 const uint8_t *mask, unsigned int size)
471 if (!item->spec && (item->mask || item->last))
473 if (item->spec && !item->mask) {
475 const uint8_t *spec = item->spec;
477 for (i = 0; i < size; ++i)
478 if ((spec[i] | mask[i]) != mask[i])
481 if (item->last && !item->mask) {
483 const uint8_t *spec = item->last;
485 for (i = 0; i < size; ++i)
486 if ((spec[i] | mask[i]) != mask[i])
491 const uint8_t *spec = item->spec;
493 for (i = 0; i < size; ++i)
494 if ((spec[i] | mask[i]) != mask[i])
497 if (item->spec && item->last) {
500 const uint8_t *apply = mask;
505 for (i = 0; i < size; ++i) {
506 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
507 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
509 ret = memcmp(spec, last, size);
515 * Copy the RSS configuration from the user ones, of the rss_conf is null,
516 * uses the driver one.
519 * Pointer to private structure.
521 * Internal parser structure.
523 * User RSS configuration to save.
526 * 0 on success, errno value on failure.
529 priv_flow_convert_rss_conf(struct priv *priv __rte_unused,
530 struct mlx5_flow_parse *parser,
531 const struct rte_eth_rss_conf *rss_conf)
534 * This function is also called at the beginning of
535 * priv_flow_convert_actions() to initialize the parser with the
536 * device default RSS configuration.
539 if (rss_conf->rss_hf & MLX5_RSS_HF_MASK)
541 if (rss_conf->rss_key_len != 40)
543 if (rss_conf->rss_key_len && rss_conf->rss_key) {
544 parser->rss_conf.rss_key_len = rss_conf->rss_key_len;
545 memcpy(parser->rss_key, rss_conf->rss_key,
546 rss_conf->rss_key_len);
547 parser->rss_conf.rss_key = parser->rss_key;
549 parser->rss_conf.rss_hf = rss_conf->rss_hf;
555 * Extract attribute to the parser.
558 * Pointer to private structure.
560 * Flow rule attributes.
562 * Perform verbose error reporting if not NULL.
563 * @param[in, out] parser
564 * Internal parser structure.
567 * 0 on success, a negative errno value otherwise and rte_errno is set.
570 priv_flow_convert_attributes(struct priv *priv __rte_unused,
571 const struct rte_flow_attr *attr,
572 struct rte_flow_error *error,
573 struct mlx5_flow_parse *parser __rte_unused)
576 rte_flow_error_set(error, ENOTSUP,
577 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
579 "groups are not supported");
582 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
583 rte_flow_error_set(error, ENOTSUP,
584 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
586 "priorities are not supported");
590 rte_flow_error_set(error, ENOTSUP,
591 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
593 "egress is not supported");
596 if (!attr->ingress) {
597 rte_flow_error_set(error, ENOTSUP,
598 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
600 "only ingress is supported");
607 * Extract actions request to the parser.
610 * Pointer to private structure.
612 * Associated actions (list terminated by the END action).
614 * Perform verbose error reporting if not NULL.
615 * @param[in, out] parser
616 * Internal parser structure.
619 * 0 on success, a negative errno value otherwise and rte_errno is set.
622 priv_flow_convert_actions(struct priv *priv,
623 const struct rte_flow_action actions[],
624 struct rte_flow_error *error,
625 struct mlx5_flow_parse *parser)
628 * Add default RSS configuration necessary for Verbs to create QP even
629 * if no RSS is necessary.
631 priv_flow_convert_rss_conf(priv, parser,
632 (const struct rte_eth_rss_conf *)
634 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
635 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
637 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
639 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
640 const struct rte_flow_action_queue *queue =
641 (const struct rte_flow_action_queue *)
646 if (!queue || (queue->index > (priv->rxqs_n - 1)))
647 goto exit_action_not_supported;
648 for (n = 0; n < parser->queues_n; ++n) {
649 if (parser->queues[n] == queue->index) {
654 if (parser->queues_n > 1 && !found) {
655 rte_flow_error_set(error, ENOTSUP,
656 RTE_FLOW_ERROR_TYPE_ACTION,
658 "queue action not in RSS queues");
662 parser->queues_n = 1;
663 parser->queues[0] = queue->index;
665 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
666 const struct rte_flow_action_rss *rss =
667 (const struct rte_flow_action_rss *)
671 if (!rss || !rss->num) {
672 rte_flow_error_set(error, EINVAL,
673 RTE_FLOW_ERROR_TYPE_ACTION,
678 if (parser->queues_n == 1) {
681 assert(parser->queues_n);
682 for (n = 0; n < rss->num; ++n) {
683 if (parser->queues[0] ==
690 rte_flow_error_set(error, ENOTSUP,
691 RTE_FLOW_ERROR_TYPE_ACTION,
693 "queue action not in RSS"
698 for (n = 0; n < rss->num; ++n) {
699 if (rss->queue[n] >= priv->rxqs_n) {
700 rte_flow_error_set(error, EINVAL,
701 RTE_FLOW_ERROR_TYPE_ACTION,
703 "queue id > number of"
708 for (n = 0; n < rss->num; ++n)
709 parser->queues[n] = rss->queue[n];
710 parser->queues_n = rss->num;
711 if (priv_flow_convert_rss_conf(priv, parser,
713 rte_flow_error_set(error, EINVAL,
714 RTE_FLOW_ERROR_TYPE_ACTION,
716 "wrong RSS configuration");
719 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
720 const struct rte_flow_action_mark *mark =
721 (const struct rte_flow_action_mark *)
725 rte_flow_error_set(error, EINVAL,
726 RTE_FLOW_ERROR_TYPE_ACTION,
728 "mark must be defined");
730 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
731 rte_flow_error_set(error, ENOTSUP,
732 RTE_FLOW_ERROR_TYPE_ACTION,
734 "mark must be between 0"
739 parser->mark_id = mark->id;
740 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
742 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
743 priv->config.flow_counter_en) {
746 goto exit_action_not_supported;
749 if (parser->drop && parser->mark)
751 if (!parser->queues_n && !parser->drop) {
752 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
753 NULL, "no valid action");
757 exit_action_not_supported:
758 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
759 actions, "action not supported");
767 * Pointer to private structure.
769 * Pattern specification (list terminated by the END pattern item).
771 * Perform verbose error reporting if not NULL.
772 * @param[in, out] parser
773 * Internal parser structure.
776 * 0 on success, a negative errno value otherwise and rte_errno is set.
779 priv_flow_convert_items_validate(struct priv *priv __rte_unused,
780 const struct rte_flow_item items[],
781 struct rte_flow_error *error,
782 struct mlx5_flow_parse *parser)
784 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
787 /* Initialise the offsets to start after verbs attribute. */
788 for (i = 0; i != hash_rxq_init_n; ++i)
789 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
790 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
791 const struct mlx5_flow_items *token = NULL;
795 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
799 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
801 if (cur_item->items[i] == items->type) {
802 token = &mlx5_flow_items[items->type];
807 goto exit_item_not_supported;
809 err = mlx5_flow_item_validate(items,
810 (const uint8_t *)cur_item->mask,
813 goto exit_item_not_supported;
814 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
816 rte_flow_error_set(error, ENOTSUP,
817 RTE_FLOW_ERROR_TYPE_ITEM,
819 "cannot recognize multiple"
820 " VXLAN encapsulations");
823 parser->inner = IBV_FLOW_SPEC_INNER;
826 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
828 for (n = 0; n != hash_rxq_init_n; ++n)
829 parser->queue[n].offset += cur_item->dst_sz;
833 parser->queue[HASH_RXQ_ETH].offset +=
834 sizeof(struct ibv_flow_spec_action_drop);
837 for (i = 0; i != hash_rxq_init_n; ++i)
838 parser->queue[i].offset +=
839 sizeof(struct ibv_flow_spec_action_tag);
842 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
844 for (i = 0; i != hash_rxq_init_n; ++i)
845 parser->queue[i].offset += size;
848 exit_item_not_supported:
849 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
850 items, "item not supported");
855 * Allocate memory space to store verbs flow attributes.
858 * Pointer to private structure.
859 * @param[in] priority
862 * Amount of byte to allocate.
864 * Perform verbose error reporting if not NULL.
867 * A verbs flow attribute on success, NULL otherwise.
869 static struct ibv_flow_attr *
870 priv_flow_convert_allocate(struct priv *priv __rte_unused,
871 unsigned int priority,
873 struct rte_flow_error *error)
875 struct ibv_flow_attr *ibv_attr;
877 ibv_attr = rte_calloc(__func__, 1, size, 0);
879 rte_flow_error_set(error, ENOMEM,
880 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
882 "cannot allocate verbs spec attributes.");
885 ibv_attr->priority = priority;
890 * Finalise verbs flow attributes.
893 * Pointer to private structure.
894 * @param[in, out] parser
895 * Internal parser structure.
898 priv_flow_convert_finalise(struct priv *priv __rte_unused,
899 struct mlx5_flow_parse *parser)
901 const unsigned int ipv4 =
902 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
903 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
904 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
905 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
906 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
907 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
910 /* Remove any other flow not matching the pattern. */
911 if (parser->queues_n == 1) {
912 for (i = 0; i != hash_rxq_init_n; ++i) {
913 if (i == HASH_RXQ_ETH)
915 rte_free(parser->queue[i].ibv_attr);
916 parser->queue[i].ibv_attr = NULL;
920 if (parser->layer == HASH_RXQ_ETH) {
924 * This layer becomes useless as the pattern define under
927 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
928 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
930 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
931 for (i = ohmin; i != (ohmax + 1); ++i) {
932 if (!parser->queue[i].ibv_attr)
934 rte_free(parser->queue[i].ibv_attr);
935 parser->queue[i].ibv_attr = NULL;
937 /* Remove impossible flow according to the RSS configuration. */
938 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
939 parser->rss_conf.rss_hf) {
940 /* Remove any other flow. */
941 for (i = hmin; i != (hmax + 1); ++i) {
942 if ((i == parser->layer) ||
943 (!parser->queue[i].ibv_attr))
945 rte_free(parser->queue[i].ibv_attr);
946 parser->queue[i].ibv_attr = NULL;
948 } else if (!parser->queue[ip].ibv_attr) {
949 /* no RSS possible with the current configuration. */
950 parser->queues_n = 1;
955 * Fill missing layers in verbs specifications, or compute the correct
956 * offset to allocate the memory space for the attributes and
959 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
961 struct ibv_flow_spec_ipv4_ext ipv4;
962 struct ibv_flow_spec_ipv6 ipv6;
963 struct ibv_flow_spec_tcp_udp udp_tcp;
968 if (i == parser->layer)
970 if (parser->layer == HASH_RXQ_ETH) {
971 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
972 size = sizeof(struct ibv_flow_spec_ipv4_ext);
973 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
974 .type = IBV_FLOW_SPEC_IPV4_EXT,
978 size = sizeof(struct ibv_flow_spec_ipv6);
979 specs.ipv6 = (struct ibv_flow_spec_ipv6){
980 .type = IBV_FLOW_SPEC_IPV6,
984 if (parser->queue[i].ibv_attr) {
985 dst = (void *)((uintptr_t)
986 parser->queue[i].ibv_attr +
987 parser->queue[i].offset);
988 memcpy(dst, &specs, size);
989 ++parser->queue[i].ibv_attr->num_of_specs;
991 parser->queue[i].offset += size;
993 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
994 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
995 size = sizeof(struct ibv_flow_spec_tcp_udp);
996 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
997 .type = ((i == HASH_RXQ_UDPV4 ||
998 i == HASH_RXQ_UDPV6) ?
1003 if (parser->queue[i].ibv_attr) {
1004 dst = (void *)((uintptr_t)
1005 parser->queue[i].ibv_attr +
1006 parser->queue[i].offset);
1007 memcpy(dst, &specs, size);
1008 ++parser->queue[i].ibv_attr->num_of_specs;
1010 parser->queue[i].offset += size;
1016 * Validate and convert a flow supported by the NIC.
1019 * Pointer to private structure.
1021 * Flow rule attributes.
1022 * @param[in] pattern
1023 * Pattern specification (list terminated by the END pattern item).
1024 * @param[in] actions
1025 * Associated actions (list terminated by the END action).
1027 * Perform verbose error reporting if not NULL.
1028 * @param[in, out] parser
1029 * Internal parser structure.
1032 * 0 on success, a negative errno value otherwise and rte_errno is set.
1035 priv_flow_convert(struct priv *priv,
1036 const struct rte_flow_attr *attr,
1037 const struct rte_flow_item items[],
1038 const struct rte_flow_action actions[],
1039 struct rte_flow_error *error,
1040 struct mlx5_flow_parse *parser)
1042 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1046 /* First step. Validate the attributes, items and actions. */
1047 *parser = (struct mlx5_flow_parse){
1048 .create = parser->create,
1049 .layer = HASH_RXQ_ETH,
1050 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1052 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1055 ret = priv_flow_convert_actions(priv, actions, error, parser);
1058 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1061 priv_flow_convert_finalise(priv, parser);
1064 * Allocate the memory space to store verbs specifications.
1067 unsigned int priority =
1069 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1070 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1072 parser->queue[HASH_RXQ_ETH].ibv_attr =
1073 priv_flow_convert_allocate(priv, priority,
1075 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1077 parser->queue[HASH_RXQ_ETH].offset =
1078 sizeof(struct ibv_flow_attr);
1080 for (i = 0; i != hash_rxq_init_n; ++i) {
1081 unsigned int priority =
1083 hash_rxq_init[i].flow_priority;
1084 unsigned int offset;
1086 if (!(parser->rss_conf.rss_hf &
1087 hash_rxq_init[i].dpdk_rss_hf) &&
1088 (i != HASH_RXQ_ETH))
1090 offset = parser->queue[i].offset;
1091 parser->queue[i].ibv_attr =
1092 priv_flow_convert_allocate(priv, priority,
1094 if (!parser->queue[i].ibv_attr)
1096 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1099 /* Third step. Conversion parse, fill the specifications. */
1101 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1102 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1104 cur_item = &mlx5_flow_items[items->type];
1105 ret = cur_item->convert(items,
1106 (cur_item->default_mask ?
1107 cur_item->default_mask :
1111 rte_flow_error_set(error, ret,
1112 RTE_FLOW_ERROR_TYPE_ITEM,
1113 items, "item not supported");
1118 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1119 if (parser->count && parser->create) {
1120 mlx5_flow_create_count(priv, parser);
1122 goto exit_count_error;
1125 * Last step. Complete missing specification to reach the RSS
1128 if (!parser->drop) {
1129 priv_flow_convert_finalise(priv, parser);
1131 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1133 hash_rxq_init[parser->layer].flow_priority;
1136 /* Only verification is expected, all resources should be released. */
1137 if (!parser->create) {
1138 for (i = 0; i != hash_rxq_init_n; ++i) {
1139 if (parser->queue[i].ibv_attr) {
1140 rte_free(parser->queue[i].ibv_attr);
1141 parser->queue[i].ibv_attr = NULL;
1147 for (i = 0; i != hash_rxq_init_n; ++i) {
1148 if (parser->queue[i].ibv_attr) {
1149 rte_free(parser->queue[i].ibv_attr);
1150 parser->queue[i].ibv_attr = NULL;
1153 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1154 NULL, "cannot allocate verbs spec attributes.");
1157 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1158 NULL, "cannot create counter.");
1163 * Copy the specification created into the flow.
1166 * Internal parser structure.
1168 * Create specification.
1170 * Size in bytes of the specification to copy.
1173 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1179 for (i = 0; i != hash_rxq_init_n; ++i) {
1180 if (!parser->queue[i].ibv_attr)
1182 /* Specification must be the same l3 type or none. */
1183 if (parser->layer == HASH_RXQ_ETH ||
1184 (hash_rxq_init[parser->layer].ip_version ==
1185 hash_rxq_init[i].ip_version) ||
1186 (hash_rxq_init[i].ip_version == 0)) {
1187 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1188 parser->queue[i].offset);
1189 memcpy(dst, src, size);
1190 ++parser->queue[i].ibv_attr->num_of_specs;
1191 parser->queue[i].offset += size;
1197 * Convert Ethernet item to Verbs specification.
1200 * Item specification.
1201 * @param default_mask[in]
1202 * Default bit-masks to use when item->mask is not provided.
1203 * @param data[in, out]
1207 mlx5_flow_create_eth(const struct rte_flow_item *item,
1208 const void *default_mask,
1211 const struct rte_flow_item_eth *spec = item->spec;
1212 const struct rte_flow_item_eth *mask = item->mask;
1213 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1214 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1215 struct ibv_flow_spec_eth eth = {
1216 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1220 /* Don't update layer for the inner pattern. */
1222 parser->layer = HASH_RXQ_ETH;
1227 mask = default_mask;
1228 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1229 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1230 eth.val.ether_type = spec->type;
1231 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1232 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1233 eth.mask.ether_type = mask->type;
1234 /* Remove unwanted bits from values. */
1235 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1236 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1237 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1239 eth.val.ether_type &= eth.mask.ether_type;
1241 mlx5_flow_create_copy(parser, ð, eth_size);
1246 * Convert VLAN item to Verbs specification.
1249 * Item specification.
1250 * @param default_mask[in]
1251 * Default bit-masks to use when item->mask is not provided.
1252 * @param data[in, out]
1256 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1257 const void *default_mask,
1260 const struct rte_flow_item_vlan *spec = item->spec;
1261 const struct rte_flow_item_vlan *mask = item->mask;
1262 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1263 struct ibv_flow_spec_eth *eth;
1264 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1269 mask = default_mask;
1271 for (i = 0; i != hash_rxq_init_n; ++i) {
1272 if (!parser->queue[i].ibv_attr)
1275 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1276 parser->queue[i].offset - eth_size);
1277 eth->val.vlan_tag = spec->tci;
1278 eth->mask.vlan_tag = mask->tci;
1279 eth->val.vlan_tag &= eth->mask.vlan_tag;
1286 * Convert IPv4 item to Verbs specification.
1289 * Item specification.
1290 * @param default_mask[in]
1291 * Default bit-masks to use when item->mask is not provided.
1292 * @param data[in, out]
1296 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1297 const void *default_mask,
1300 const struct rte_flow_item_ipv4 *spec = item->spec;
1301 const struct rte_flow_item_ipv4 *mask = item->mask;
1302 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1303 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1304 struct ibv_flow_spec_ipv4_ext ipv4 = {
1305 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1309 /* Don't update layer for the inner pattern. */
1311 parser->layer = HASH_RXQ_IPV4;
1314 mask = default_mask;
1315 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1316 .src_ip = spec->hdr.src_addr,
1317 .dst_ip = spec->hdr.dst_addr,
1318 .proto = spec->hdr.next_proto_id,
1319 .tos = spec->hdr.type_of_service,
1321 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1322 .src_ip = mask->hdr.src_addr,
1323 .dst_ip = mask->hdr.dst_addr,
1324 .proto = mask->hdr.next_proto_id,
1325 .tos = mask->hdr.type_of_service,
1327 /* Remove unwanted bits from values. */
1328 ipv4.val.src_ip &= ipv4.mask.src_ip;
1329 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1330 ipv4.val.proto &= ipv4.mask.proto;
1331 ipv4.val.tos &= ipv4.mask.tos;
1333 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1338 * Convert IPv6 item to Verbs specification.
1341 * Item specification.
1342 * @param default_mask[in]
1343 * Default bit-masks to use when item->mask is not provided.
1344 * @param data[in, out]
1348 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1349 const void *default_mask,
1352 const struct rte_flow_item_ipv6 *spec = item->spec;
1353 const struct rte_flow_item_ipv6 *mask = item->mask;
1354 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1355 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1356 struct ibv_flow_spec_ipv6 ipv6 = {
1357 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1361 /* Don't update layer for the inner pattern. */
1363 parser->layer = HASH_RXQ_IPV6;
1366 uint32_t vtc_flow_val;
1367 uint32_t vtc_flow_mask;
1370 mask = default_mask;
1371 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1372 RTE_DIM(ipv6.val.src_ip));
1373 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1374 RTE_DIM(ipv6.val.dst_ip));
1375 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1376 RTE_DIM(ipv6.mask.src_ip));
1377 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1378 RTE_DIM(ipv6.mask.dst_ip));
1379 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
1380 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
1381 ipv6.val.flow_label =
1382 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
1384 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
1386 ipv6.val.next_hdr = spec->hdr.proto;
1387 ipv6.val.hop_limit = spec->hdr.hop_limits;
1388 ipv6.mask.flow_label =
1389 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
1391 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
1393 ipv6.mask.next_hdr = mask->hdr.proto;
1394 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1395 /* Remove unwanted bits from values. */
1396 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1397 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1398 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1400 ipv6.val.flow_label &= ipv6.mask.flow_label;
1401 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
1402 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1403 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1405 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1410 * Convert UDP item to Verbs specification.
1413 * Item specification.
1414 * @param default_mask[in]
1415 * Default bit-masks to use when item->mask is not provided.
1416 * @param data[in, out]
1420 mlx5_flow_create_udp(const struct rte_flow_item *item,
1421 const void *default_mask,
1424 const struct rte_flow_item_udp *spec = item->spec;
1425 const struct rte_flow_item_udp *mask = item->mask;
1426 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1427 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1428 struct ibv_flow_spec_tcp_udp udp = {
1429 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1433 /* Don't update layer for the inner pattern. */
1434 if (!parser->inner) {
1435 if (parser->layer == HASH_RXQ_IPV4)
1436 parser->layer = HASH_RXQ_UDPV4;
1438 parser->layer = HASH_RXQ_UDPV6;
1442 mask = default_mask;
1443 udp.val.dst_port = spec->hdr.dst_port;
1444 udp.val.src_port = spec->hdr.src_port;
1445 udp.mask.dst_port = mask->hdr.dst_port;
1446 udp.mask.src_port = mask->hdr.src_port;
1447 /* Remove unwanted bits from values. */
1448 udp.val.src_port &= udp.mask.src_port;
1449 udp.val.dst_port &= udp.mask.dst_port;
1451 mlx5_flow_create_copy(parser, &udp, udp_size);
1456 * Convert TCP item to Verbs specification.
1459 * Item specification.
1460 * @param default_mask[in]
1461 * Default bit-masks to use when item->mask is not provided.
1462 * @param data[in, out]
1466 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1467 const void *default_mask,
1470 const struct rte_flow_item_tcp *spec = item->spec;
1471 const struct rte_flow_item_tcp *mask = item->mask;
1472 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1473 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1474 struct ibv_flow_spec_tcp_udp tcp = {
1475 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1479 /* Don't update layer for the inner pattern. */
1480 if (!parser->inner) {
1481 if (parser->layer == HASH_RXQ_IPV4)
1482 parser->layer = HASH_RXQ_TCPV4;
1484 parser->layer = HASH_RXQ_TCPV6;
1488 mask = default_mask;
1489 tcp.val.dst_port = spec->hdr.dst_port;
1490 tcp.val.src_port = spec->hdr.src_port;
1491 tcp.mask.dst_port = mask->hdr.dst_port;
1492 tcp.mask.src_port = mask->hdr.src_port;
1493 /* Remove unwanted bits from values. */
1494 tcp.val.src_port &= tcp.mask.src_port;
1495 tcp.val.dst_port &= tcp.mask.dst_port;
1497 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1502 * Convert VXLAN item to Verbs specification.
1505 * Item specification.
1506 * @param default_mask[in]
1507 * Default bit-masks to use when item->mask is not provided.
1508 * @param data[in, out]
1512 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1513 const void *default_mask,
1516 const struct rte_flow_item_vxlan *spec = item->spec;
1517 const struct rte_flow_item_vxlan *mask = item->mask;
1518 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1519 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1520 struct ibv_flow_spec_tunnel vxlan = {
1521 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1530 parser->inner = IBV_FLOW_SPEC_INNER;
1533 mask = default_mask;
1534 memcpy(&id.vni[1], spec->vni, 3);
1535 vxlan.val.tunnel_id = id.vlan_id;
1536 memcpy(&id.vni[1], mask->vni, 3);
1537 vxlan.mask.tunnel_id = id.vlan_id;
1538 /* Remove unwanted bits from values. */
1539 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1542 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1543 * layer is defined in the Verbs specification it is interpreted as
1544 * wildcard and all packets will match this rule, if it follows a full
1545 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1546 * before will also match this rule.
1547 * To avoid such situation, VNI 0 is currently refused.
1549 if (!vxlan.val.tunnel_id)
1551 mlx5_flow_create_copy(parser, &vxlan, size);
1556 * Convert mark/flag action to Verbs specification.
1559 * Internal parser structure.
1564 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1566 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1567 struct ibv_flow_spec_action_tag tag = {
1568 .type = IBV_FLOW_SPEC_ACTION_TAG,
1570 .tag_id = mlx5_flow_mark_set(mark_id),
1573 assert(parser->mark);
1574 mlx5_flow_create_copy(parser, &tag, size);
1579 * Convert count action to Verbs specification.
1582 * Pointer to private structure.
1584 * Pointer to MLX5 flow parser structure.
1587 * 0 on success, errno value on failure.
1590 mlx5_flow_create_count(struct priv *priv __rte_unused,
1591 struct mlx5_flow_parse *parser __rte_unused)
1593 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1594 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1595 struct ibv_counter_set_init_attr init_attr = {0};
1596 struct ibv_flow_spec_counter_action counter = {
1597 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1599 .counter_set_handle = 0,
1602 init_attr.counter_set_id = 0;
1603 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
1606 counter.counter_set_handle = parser->cs->handle;
1607 mlx5_flow_create_copy(parser, &counter, size);
1613 * Complete flow rule creation with a drop queue.
1616 * Pointer to private structure.
1618 * Internal parser structure.
1620 * Pointer to the rte_flow.
1622 * Perform verbose error reporting if not NULL.
1625 * 0 on success, errno value on failure.
1628 priv_flow_create_action_queue_drop(struct priv *priv,
1629 struct mlx5_flow_parse *parser,
1630 struct rte_flow *flow,
1631 struct rte_flow_error *error)
1633 struct ibv_flow_spec_action_drop *drop;
1634 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1640 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
1641 parser->queue[HASH_RXQ_ETH].offset);
1642 *drop = (struct ibv_flow_spec_action_drop){
1643 .type = IBV_FLOW_SPEC_ACTION_DROP,
1646 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
1647 parser->queue[HASH_RXQ_ETH].offset += size;
1648 flow->frxq[HASH_RXQ_ETH].ibv_attr =
1649 parser->queue[HASH_RXQ_ETH].ibv_attr;
1651 flow->cs = parser->cs;
1652 if (!priv->dev->data->dev_started)
1654 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
1655 flow->frxq[HASH_RXQ_ETH].ibv_flow =
1656 mlx5_glue->create_flow(priv->flow_drop_queue->qp,
1657 flow->frxq[HASH_RXQ_ETH].ibv_attr);
1658 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1659 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1660 NULL, "flow rule creation failure");
1667 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1668 claim_zero(mlx5_glue->destroy_flow
1669 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
1670 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
1672 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
1673 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
1674 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
1677 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1685 * Create hash Rx queues when RSS is enabled.
1688 * Pointer to private structure.
1690 * Internal parser structure.
1692 * Pointer to the rte_flow.
1694 * Perform verbose error reporting if not NULL.
1697 * 0 on success, a errno value otherwise and rte_errno is set.
1700 priv_flow_create_action_queue_rss(struct priv *priv,
1701 struct mlx5_flow_parse *parser,
1702 struct rte_flow *flow,
1703 struct rte_flow_error *error)
1707 for (i = 0; i != hash_rxq_init_n; ++i) {
1708 uint64_t hash_fields;
1710 if (!parser->queue[i].ibv_attr)
1712 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1713 parser->queue[i].ibv_attr = NULL;
1714 hash_fields = hash_rxq_init[i].hash_fields;
1715 if (!priv->dev->data->dev_started)
1717 flow->frxq[i].hrxq =
1718 mlx5_priv_hrxq_get(priv,
1719 parser->rss_conf.rss_key,
1720 parser->rss_conf.rss_key_len,
1724 if (flow->frxq[i].hrxq)
1726 flow->frxq[i].hrxq =
1727 mlx5_priv_hrxq_new(priv,
1728 parser->rss_conf.rss_key,
1729 parser->rss_conf.rss_key_len,
1733 if (!flow->frxq[i].hrxq) {
1734 rte_flow_error_set(error, ENOMEM,
1735 RTE_FLOW_ERROR_TYPE_HANDLE,
1736 NULL, "cannot create hash rxq");
1744 * Complete flow rule creation.
1747 * Pointer to private structure.
1749 * Internal parser structure.
1751 * Pointer to the rte_flow.
1753 * Perform verbose error reporting if not NULL.
1756 * 0 on success, a errno value otherwise and rte_errno is set.
1759 priv_flow_create_action_queue(struct priv *priv,
1760 struct mlx5_flow_parse *parser,
1761 struct rte_flow *flow,
1762 struct rte_flow_error *error)
1766 unsigned int flows_n = 0;
1770 assert(!parser->drop);
1771 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1775 flow->cs = parser->cs;
1776 if (!priv->dev->data->dev_started)
1778 for (i = 0; i != hash_rxq_init_n; ++i) {
1779 if (!flow->frxq[i].hrxq)
1781 flow->frxq[i].ibv_flow =
1782 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
1783 flow->frxq[i].ibv_attr);
1784 if (!flow->frxq[i].ibv_flow) {
1785 rte_flow_error_set(error, ENOMEM,
1786 RTE_FLOW_ERROR_TYPE_HANDLE,
1787 NULL, "flow rule creation failure");
1792 DEBUG("%p type %d QP %p ibv_flow %p",
1794 (void *)flow->frxq[i].hrxq,
1795 (void *)flow->frxq[i].ibv_flow);
1798 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
1799 NULL, "internal error in flow creation");
1802 for (i = 0; i != parser->queues_n; ++i) {
1803 struct mlx5_rxq_data *q =
1804 (*priv->rxqs)[parser->queues[i]];
1806 q->mark |= parser->mark;
1811 for (i = 0; i != hash_rxq_init_n; ++i) {
1812 if (flow->frxq[i].ibv_flow) {
1813 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1815 claim_zero(mlx5_glue->destroy_flow(ibv_flow));
1817 if (flow->frxq[i].hrxq)
1818 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1819 if (flow->frxq[i].ibv_attr)
1820 rte_free(flow->frxq[i].ibv_attr);
1823 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1834 * Pointer to private structure.
1836 * Pointer to a TAILQ flow list.
1838 * Flow rule attributes.
1839 * @param[in] pattern
1840 * Pattern specification (list terminated by the END pattern item).
1841 * @param[in] actions
1842 * Associated actions (list terminated by the END action).
1844 * Perform verbose error reporting if not NULL.
1847 * A flow on success, NULL otherwise.
1849 static struct rte_flow *
1850 priv_flow_create(struct priv *priv,
1851 struct mlx5_flows *list,
1852 const struct rte_flow_attr *attr,
1853 const struct rte_flow_item items[],
1854 const struct rte_flow_action actions[],
1855 struct rte_flow_error *error)
1857 struct mlx5_flow_parse parser = { .create = 1, };
1858 struct rte_flow *flow = NULL;
1862 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1865 flow = rte_calloc(__func__, 1,
1866 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1869 rte_flow_error_set(error, ENOMEM,
1870 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1872 "cannot allocate flow memory");
1875 /* Copy queues configuration. */
1876 flow->queues = (uint16_t (*)[])(flow + 1);
1877 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1878 flow->queues_n = parser.queues_n;
1879 flow->mark = parser.mark;
1880 /* Copy RSS configuration. */
1881 flow->rss_conf = parser.rss_conf;
1882 flow->rss_conf.rss_key = flow->rss_key;
1883 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1884 /* finalise the flow. */
1886 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1889 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1892 TAILQ_INSERT_TAIL(list, flow, next);
1893 DEBUG("Flow created %p", (void *)flow);
1896 ERROR("flow creation error: %s", error->message);
1897 for (i = 0; i != hash_rxq_init_n; ++i) {
1898 if (parser.queue[i].ibv_attr)
1899 rte_free(parser.queue[i].ibv_attr);
1906 * Validate a flow supported by the NIC.
1908 * @see rte_flow_validate()
1912 mlx5_flow_validate(struct rte_eth_dev *dev,
1913 const struct rte_flow_attr *attr,
1914 const struct rte_flow_item items[],
1915 const struct rte_flow_action actions[],
1916 struct rte_flow_error *error)
1918 struct priv *priv = dev->data->dev_private;
1920 struct mlx5_flow_parse parser = { .create = 0, };
1922 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1929 * @see rte_flow_create()
1933 mlx5_flow_create(struct rte_eth_dev *dev,
1934 const struct rte_flow_attr *attr,
1935 const struct rte_flow_item items[],
1936 const struct rte_flow_action actions[],
1937 struct rte_flow_error *error)
1939 struct priv *priv = dev->data->dev_private;
1940 struct rte_flow *flow;
1942 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1951 * Pointer to private structure.
1953 * Pointer to a TAILQ flow list.
1958 priv_flow_destroy(struct priv *priv,
1959 struct mlx5_flows *list,
1960 struct rte_flow *flow)
1964 if (flow->drop || !flow->mark)
1966 for (i = 0; i != flow->queues_n; ++i) {
1967 struct rte_flow *tmp;
1971 * To remove the mark from the queue, the queue must not be
1972 * present in any other marked flow (RSS or not).
1974 TAILQ_FOREACH(tmp, list, next) {
1976 uint16_t *tqs = NULL;
1981 for (j = 0; j != hash_rxq_init_n; ++j) {
1982 if (!tmp->frxq[j].hrxq)
1984 tqs = tmp->frxq[j].hrxq->ind_table->queues;
1985 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
1989 for (j = 0; (j != tq_n) && !mark; j++)
1990 if (tqs[j] == (*flow->queues)[i])
1993 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
1997 if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
1998 claim_zero(mlx5_glue->destroy_flow
1999 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2000 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
2002 for (i = 0; i != hash_rxq_init_n; ++i) {
2003 struct mlx5_flow *frxq = &flow->frxq[i];
2006 claim_zero(mlx5_glue->destroy_flow
2009 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2011 rte_free(frxq->ibv_attr);
2015 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
2018 TAILQ_REMOVE(list, flow, next);
2019 DEBUG("Flow destroyed %p", (void *)flow);
2024 * Destroy all flows.
2027 * Pointer to private structure.
2029 * Pointer to a TAILQ flow list.
2032 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2034 while (!TAILQ_EMPTY(list)) {
2035 struct rte_flow *flow;
2037 flow = TAILQ_FIRST(list);
2038 priv_flow_destroy(priv, list, flow);
2043 * Create drop queue.
2046 * Pointer to private structure.
2052 priv_flow_create_drop_queue(struct priv *priv)
2054 struct mlx5_hrxq_drop *fdq = NULL;
2058 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2060 WARN("cannot allocate memory for drop queue");
2063 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
2065 WARN("cannot allocate CQ for drop queue");
2068 fdq->wq = mlx5_glue->create_wq
2070 &(struct ibv_wq_init_attr){
2071 .wq_type = IBV_WQT_RQ,
2078 WARN("cannot allocate WQ for drop queue");
2081 fdq->ind_table = mlx5_glue->create_rwq_ind_table
2083 &(struct ibv_rwq_ind_table_init_attr){
2084 .log_ind_tbl_size = 0,
2085 .ind_tbl = &fdq->wq,
2088 if (!fdq->ind_table) {
2089 WARN("cannot allocate indirection table for drop queue");
2092 fdq->qp = mlx5_glue->create_qp_ex
2094 &(struct ibv_qp_init_attr_ex){
2095 .qp_type = IBV_QPT_RAW_PACKET,
2097 IBV_QP_INIT_ATTR_PD |
2098 IBV_QP_INIT_ATTR_IND_TABLE |
2099 IBV_QP_INIT_ATTR_RX_HASH,
2100 .rx_hash_conf = (struct ibv_rx_hash_conf){
2102 IBV_RX_HASH_FUNC_TOEPLITZ,
2103 .rx_hash_key_len = rss_hash_default_key_len,
2104 .rx_hash_key = rss_hash_default_key,
2105 .rx_hash_fields_mask = 0,
2107 .rwq_ind_tbl = fdq->ind_table,
2111 WARN("cannot allocate QP for drop queue");
2114 priv->flow_drop_queue = fdq;
2118 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2120 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2122 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2124 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2127 priv->flow_drop_queue = NULL;
2132 * Delete drop queue.
2135 * Pointer to private structure.
2138 priv_flow_delete_drop_queue(struct priv *priv)
2140 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2145 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2147 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2149 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2151 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2153 priv->flow_drop_queue = NULL;
2160 * Pointer to private structure.
2162 * Pointer to a TAILQ flow list.
2165 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2167 struct rte_flow *flow;
2169 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2171 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2174 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
2176 claim_zero(mlx5_glue->destroy_flow
2177 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2178 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
2179 DEBUG("Flow %p removed", (void *)flow);
2183 /* Verify the flow has not already been cleaned. */
2184 for (i = 0; i != hash_rxq_init_n; ++i) {
2185 if (!flow->frxq[i].ibv_flow)
2188 * Indirection table may be necessary to remove the
2189 * flags in the Rx queues.
2190 * This helps to speed-up the process by avoiding
2193 ind_tbl = flow->frxq[i].hrxq->ind_table;
2196 if (i == hash_rxq_init_n)
2200 for (i = 0; i != ind_tbl->queues_n; ++i)
2201 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2203 for (i = 0; i != hash_rxq_init_n; ++i) {
2204 if (!flow->frxq[i].ibv_flow)
2206 claim_zero(mlx5_glue->destroy_flow
2207 (flow->frxq[i].ibv_flow));
2208 flow->frxq[i].ibv_flow = NULL;
2209 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2210 flow->frxq[i].hrxq = NULL;
2212 DEBUG("Flow %p removed", (void *)flow);
2220 * Pointer to private structure.
2222 * Pointer to a TAILQ flow list.
2225 * 0 on success, a errno value otherwise and rte_errno is set.
2228 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2230 struct rte_flow *flow;
2232 TAILQ_FOREACH(flow, list, next) {
2236 flow->frxq[HASH_RXQ_ETH].ibv_flow =
2237 mlx5_glue->create_flow
2238 (priv->flow_drop_queue->qp,
2239 flow->frxq[HASH_RXQ_ETH].ibv_attr);
2240 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
2241 DEBUG("Flow %p cannot be applied",
2246 DEBUG("Flow %p applied", (void *)flow);
2250 for (i = 0; i != hash_rxq_init_n; ++i) {
2251 if (!flow->frxq[i].ibv_attr)
2253 flow->frxq[i].hrxq =
2254 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2255 flow->rss_conf.rss_key_len,
2256 hash_rxq_init[i].hash_fields,
2259 if (flow->frxq[i].hrxq)
2261 flow->frxq[i].hrxq =
2262 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2263 flow->rss_conf.rss_key_len,
2264 hash_rxq_init[i].hash_fields,
2267 if (!flow->frxq[i].hrxq) {
2268 DEBUG("Flow %p cannot be applied",
2274 flow->frxq[i].ibv_flow =
2275 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
2276 flow->frxq[i].ibv_attr);
2277 if (!flow->frxq[i].ibv_flow) {
2278 DEBUG("Flow %p cannot be applied",
2283 DEBUG("Flow %p applied", (void *)flow);
2287 for (i = 0; i != flow->queues_n; ++i)
2288 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2294 * Verify the flow list is empty
2297 * Pointer to private structure.
2299 * @return the number of flows not released.
2302 priv_flow_verify(struct priv *priv)
2304 struct rte_flow *flow;
2307 TAILQ_FOREACH(flow, &priv->flows, next) {
2308 DEBUG("%p: flow %p still referenced", (void *)priv,
2316 * Enable a control flow configured from the control plane.
2319 * Pointer to Ethernet device.
2321 * An Ethernet flow spec to apply.
2323 * An Ethernet flow mask to apply.
2325 * A VLAN flow spec to apply.
2327 * A VLAN flow mask to apply.
2333 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2334 struct rte_flow_item_eth *eth_spec,
2335 struct rte_flow_item_eth *eth_mask,
2336 struct rte_flow_item_vlan *vlan_spec,
2337 struct rte_flow_item_vlan *vlan_mask)
2339 struct priv *priv = dev->data->dev_private;
2340 const struct rte_flow_attr attr = {
2342 .priority = MLX5_CTRL_FLOW_PRIORITY,
2344 struct rte_flow_item items[] = {
2346 .type = RTE_FLOW_ITEM_TYPE_ETH,
2352 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2353 RTE_FLOW_ITEM_TYPE_END,
2359 .type = RTE_FLOW_ITEM_TYPE_END,
2362 struct rte_flow_action actions[] = {
2364 .type = RTE_FLOW_ACTION_TYPE_RSS,
2367 .type = RTE_FLOW_ACTION_TYPE_END,
2370 struct rte_flow *flow;
2371 struct rte_flow_error error;
2374 struct rte_flow_action_rss rss;
2376 const struct rte_eth_rss_conf *rss_conf;
2378 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2382 if (!priv->reta_idx_n)
2384 for (i = 0; i != priv->reta_idx_n; ++i)
2385 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2386 action_rss.local.rss_conf = &priv->rss_conf;
2387 action_rss.local.num = priv->reta_idx_n;
2388 actions[0].conf = (const void *)&action_rss.rss;
2389 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2397 * Enable a flow control configured from the control plane.
2400 * Pointer to Ethernet device.
2402 * An Ethernet flow spec to apply.
2404 * An Ethernet flow mask to apply.
2410 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2411 struct rte_flow_item_eth *eth_spec,
2412 struct rte_flow_item_eth *eth_mask)
2414 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2420 * @see rte_flow_destroy()
2424 mlx5_flow_destroy(struct rte_eth_dev *dev,
2425 struct rte_flow *flow,
2426 struct rte_flow_error *error __rte_unused)
2428 struct priv *priv = dev->data->dev_private;
2430 priv_flow_destroy(priv, &priv->flows, flow);
2435 * Destroy all flows.
2437 * @see rte_flow_flush()
2441 mlx5_flow_flush(struct rte_eth_dev *dev,
2442 struct rte_flow_error *error __rte_unused)
2444 struct priv *priv = dev->data->dev_private;
2446 priv_flow_flush(priv, &priv->flows);
2450 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2452 * Query flow counter.
2456 * @param counter_value
2457 * returned data from the counter.
2460 * 0 on success, a errno value otherwise and rte_errno is set.
2463 priv_flow_query_count(struct ibv_counter_set *cs,
2464 struct mlx5_flow_counter_stats *counter_stats,
2465 struct rte_flow_query_count *query_count,
2466 struct rte_flow_error *error)
2468 uint64_t counters[2];
2469 struct ibv_query_counter_set_attr query_cs_attr = {
2471 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2473 struct ibv_counter_set_data query_out = {
2475 .outlen = 2 * sizeof(uint64_t),
2477 int res = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
2480 rte_flow_error_set(error, -res,
2481 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2483 "cannot read counter");
2486 query_count->hits_set = 1;
2487 query_count->bytes_set = 1;
2488 query_count->hits = counters[0] - counter_stats->hits;
2489 query_count->bytes = counters[1] - counter_stats->bytes;
2490 if (query_count->reset) {
2491 counter_stats->hits = counters[0];
2492 counter_stats->bytes = counters[1];
2500 * @see rte_flow_query()
2504 mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
2505 struct rte_flow *flow,
2506 enum rte_flow_action_type action __rte_unused,
2508 struct rte_flow_error *error)
2513 res = priv_flow_query_count(flow->cs,
2514 &flow->counter_stats,
2515 (struct rte_flow_query_count *)data,
2518 rte_flow_error_set(error, res,
2519 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2521 "no counter found for flow");
2530 * @see rte_flow_isolate()
2534 mlx5_flow_isolate(struct rte_eth_dev *dev,
2536 struct rte_flow_error *error)
2538 struct priv *priv = dev->data->dev_private;
2540 if (dev->data->dev_started) {
2541 rte_flow_error_set(error, EBUSY,
2542 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2544 "port must be stopped first");
2547 priv->isolated = !!enable;
2549 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2551 priv->dev->dev_ops = &mlx5_dev_ops;
2556 * Convert a flow director filter to a generic flow.
2559 * Private structure.
2560 * @param fdir_filter
2561 * Flow director filter to add.
2563 * Generic flow parameters structure.
2566 * 0 on success, errno value on error.
2569 priv_fdir_filter_convert(struct priv *priv,
2570 const struct rte_eth_fdir_filter *fdir_filter,
2571 struct mlx5_fdir *attributes)
2573 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2575 /* Validate queue number. */
2576 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2577 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2580 attributes->attr.ingress = 1;
2581 attributes->items[0] = (struct rte_flow_item) {
2582 .type = RTE_FLOW_ITEM_TYPE_ETH,
2583 .spec = &attributes->l2,
2584 .mask = &attributes->l2_mask,
2586 switch (fdir_filter->action.behavior) {
2587 case RTE_ETH_FDIR_ACCEPT:
2588 attributes->actions[0] = (struct rte_flow_action){
2589 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2590 .conf = &attributes->queue,
2593 case RTE_ETH_FDIR_REJECT:
2594 attributes->actions[0] = (struct rte_flow_action){
2595 .type = RTE_FLOW_ACTION_TYPE_DROP,
2599 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2602 attributes->queue.index = fdir_filter->action.rx_queue;
2603 switch (fdir_filter->input.flow_type) {
2604 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2605 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2606 .src_addr = input->flow.udp4_flow.ip.src_ip,
2607 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2608 .time_to_live = input->flow.udp4_flow.ip.ttl,
2609 .type_of_service = input->flow.udp4_flow.ip.tos,
2610 .next_proto_id = input->flow.udp4_flow.ip.proto,
2612 attributes->l4.udp.hdr = (struct udp_hdr){
2613 .src_port = input->flow.udp4_flow.src_port,
2614 .dst_port = input->flow.udp4_flow.dst_port,
2616 attributes->items[1] = (struct rte_flow_item){
2617 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2618 .spec = &attributes->l3,
2619 .mask = &attributes->l3,
2621 attributes->items[2] = (struct rte_flow_item){
2622 .type = RTE_FLOW_ITEM_TYPE_UDP,
2623 .spec = &attributes->l4,
2624 .mask = &attributes->l4,
2627 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2628 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2629 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2630 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2631 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2632 .type_of_service = input->flow.tcp4_flow.ip.tos,
2633 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2635 attributes->l4.tcp.hdr = (struct tcp_hdr){
2636 .src_port = input->flow.tcp4_flow.src_port,
2637 .dst_port = input->flow.tcp4_flow.dst_port,
2639 attributes->items[1] = (struct rte_flow_item){
2640 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2641 .spec = &attributes->l3,
2642 .mask = &attributes->l3,
2644 attributes->items[2] = (struct rte_flow_item){
2645 .type = RTE_FLOW_ITEM_TYPE_TCP,
2646 .spec = &attributes->l4,
2647 .mask = &attributes->l4,
2650 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2651 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2652 .src_addr = input->flow.ip4_flow.src_ip,
2653 .dst_addr = input->flow.ip4_flow.dst_ip,
2654 .time_to_live = input->flow.ip4_flow.ttl,
2655 .type_of_service = input->flow.ip4_flow.tos,
2656 .next_proto_id = input->flow.ip4_flow.proto,
2658 attributes->items[1] = (struct rte_flow_item){
2659 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2660 .spec = &attributes->l3,
2661 .mask = &attributes->l3,
2664 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2665 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2666 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2667 .proto = input->flow.udp6_flow.ip.proto,
2669 memcpy(attributes->l3.ipv6.hdr.src_addr,
2670 input->flow.udp6_flow.ip.src_ip,
2671 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2672 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2673 input->flow.udp6_flow.ip.dst_ip,
2674 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2675 attributes->l4.udp.hdr = (struct udp_hdr){
2676 .src_port = input->flow.udp6_flow.src_port,
2677 .dst_port = input->flow.udp6_flow.dst_port,
2679 attributes->items[1] = (struct rte_flow_item){
2680 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2681 .spec = &attributes->l3,
2682 .mask = &attributes->l3,
2684 attributes->items[2] = (struct rte_flow_item){
2685 .type = RTE_FLOW_ITEM_TYPE_UDP,
2686 .spec = &attributes->l4,
2687 .mask = &attributes->l4,
2690 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2691 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2692 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2693 .proto = input->flow.tcp6_flow.ip.proto,
2695 memcpy(attributes->l3.ipv6.hdr.src_addr,
2696 input->flow.tcp6_flow.ip.src_ip,
2697 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2698 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2699 input->flow.tcp6_flow.ip.dst_ip,
2700 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2701 attributes->l4.tcp.hdr = (struct tcp_hdr){
2702 .src_port = input->flow.tcp6_flow.src_port,
2703 .dst_port = input->flow.tcp6_flow.dst_port,
2705 attributes->items[1] = (struct rte_flow_item){
2706 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2707 .spec = &attributes->l3,
2708 .mask = &attributes->l3,
2710 attributes->items[2] = (struct rte_flow_item){
2711 .type = RTE_FLOW_ITEM_TYPE_TCP,
2712 .spec = &attributes->l4,
2713 .mask = &attributes->l4,
2716 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2717 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2718 .hop_limits = input->flow.ipv6_flow.hop_limits,
2719 .proto = input->flow.ipv6_flow.proto,
2721 memcpy(attributes->l3.ipv6.hdr.src_addr,
2722 input->flow.ipv6_flow.src_ip,
2723 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2724 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2725 input->flow.ipv6_flow.dst_ip,
2726 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2727 attributes->items[1] = (struct rte_flow_item){
2728 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2729 .spec = &attributes->l3,
2730 .mask = &attributes->l3,
2734 ERROR("invalid flow type%d",
2735 fdir_filter->input.flow_type);
2742 * Add new flow director filter and store it in list.
2745 * Private structure.
2746 * @param fdir_filter
2747 * Flow director filter to add.
2750 * 0 on success, errno value on failure.
2753 priv_fdir_filter_add(struct priv *priv,
2754 const struct rte_eth_fdir_filter *fdir_filter)
2756 struct mlx5_fdir attributes = {
2759 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2760 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2764 struct mlx5_flow_parse parser = {
2765 .layer = HASH_RXQ_ETH,
2767 struct rte_flow_error error;
2768 struct rte_flow *flow;
2771 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2774 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2775 attributes.actions, &error, &parser);
2778 flow = priv_flow_create(priv,
2785 DEBUG("FDIR created %p", (void *)flow);
2792 * Delete specific filter.
2795 * Private structure.
2796 * @param fdir_filter
2797 * Filter to be deleted.
2800 * 0 on success, errno value on failure.
2803 priv_fdir_filter_delete(struct priv *priv,
2804 const struct rte_eth_fdir_filter *fdir_filter)
2806 struct mlx5_fdir attributes = {
2809 struct mlx5_flow_parse parser = {
2811 .layer = HASH_RXQ_ETH,
2813 struct rte_flow_error error;
2814 struct rte_flow *flow;
2818 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2821 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2822 attributes.actions, &error, &parser);
2826 * Special case for drop action which is only set in the
2827 * specifications when the flow is created. In this situation the
2828 * drop specification is missing.
2831 struct ibv_flow_spec_action_drop *drop;
2833 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
2834 parser.queue[HASH_RXQ_ETH].offset);
2835 *drop = (struct ibv_flow_spec_action_drop){
2836 .type = IBV_FLOW_SPEC_ACTION_DROP,
2837 .size = sizeof(struct ibv_flow_spec_action_drop),
2839 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
2841 TAILQ_FOREACH(flow, &priv->flows, next) {
2842 struct ibv_flow_attr *attr;
2843 struct ibv_spec_header *attr_h;
2845 struct ibv_flow_attr *flow_attr;
2846 struct ibv_spec_header *flow_h;
2848 unsigned int specs_n;
2850 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2851 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2852 /* Compare first the attributes. */
2853 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2855 if (attr->num_of_specs == 0)
2857 spec = (void *)((uintptr_t)attr +
2858 sizeof(struct ibv_flow_attr));
2859 flow_spec = (void *)((uintptr_t)flow_attr +
2860 sizeof(struct ibv_flow_attr));
2861 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2862 for (i = 0; i != specs_n; ++i) {
2865 if (memcmp(spec, flow_spec,
2866 RTE_MIN(attr_h->size, flow_h->size)))
2868 spec = (void *)((uintptr_t)spec + attr_h->size);
2869 flow_spec = (void *)((uintptr_t)flow_spec +
2872 /* At this point, the flow match. */
2875 /* The flow does not match. */
2879 priv_flow_destroy(priv, &priv->flows, flow);
2881 for (i = 0; i != hash_rxq_init_n; ++i) {
2882 if (parser.queue[i].ibv_attr)
2883 rte_free(parser.queue[i].ibv_attr);
2889 * Update queue for specific filter.
2892 * Private structure.
2893 * @param fdir_filter
2894 * Filter to be updated.
2897 * 0 on success, errno value on failure.
2900 priv_fdir_filter_update(struct priv *priv,
2901 const struct rte_eth_fdir_filter *fdir_filter)
2905 ret = priv_fdir_filter_delete(priv, fdir_filter);
2908 ret = priv_fdir_filter_add(priv, fdir_filter);
2913 * Flush all filters.
2916 * Private structure.
2919 priv_fdir_filter_flush(struct priv *priv)
2921 priv_flow_flush(priv, &priv->flows);
2925 * Get flow director information.
2928 * Private structure.
2929 * @param[out] fdir_info
2930 * Resulting flow director information.
2933 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2935 struct rte_eth_fdir_masks *mask =
2936 &priv->dev->data->dev_conf.fdir_conf.mask;
2938 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2939 fdir_info->guarant_spc = 0;
2940 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2941 fdir_info->max_flexpayload = 0;
2942 fdir_info->flow_types_mask[0] = 0;
2943 fdir_info->flex_payload_unit = 0;
2944 fdir_info->max_flex_payload_segment_num = 0;
2945 fdir_info->flex_payload_limit = 0;
2946 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2950 * Deal with flow director operations.
2953 * Pointer to private structure.
2955 * Operation to perform.
2957 * Pointer to operation-specific structure.
2960 * 0 on success, errno value on failure.
2963 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2965 enum rte_fdir_mode fdir_mode =
2966 priv->dev->data->dev_conf.fdir_conf.mode;
2969 if (filter_op == RTE_ETH_FILTER_NOP)
2971 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2972 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2973 ERROR("%p: flow director mode %d not supported",
2974 (void *)priv, fdir_mode);
2977 switch (filter_op) {
2978 case RTE_ETH_FILTER_ADD:
2979 ret = priv_fdir_filter_add(priv, arg);
2981 case RTE_ETH_FILTER_UPDATE:
2982 ret = priv_fdir_filter_update(priv, arg);
2984 case RTE_ETH_FILTER_DELETE:
2985 ret = priv_fdir_filter_delete(priv, arg);
2987 case RTE_ETH_FILTER_FLUSH:
2988 priv_fdir_filter_flush(priv);
2990 case RTE_ETH_FILTER_INFO:
2991 priv_fdir_info_get(priv, arg);
2994 DEBUG("%p: unknown operation %u", (void *)priv,
3003 * Manage filter operations.
3006 * Pointer to Ethernet device structure.
3007 * @param filter_type
3010 * Operation to perform.
3012 * Pointer to operation-specific structure.
3015 * 0 on success, negative errno value on failure.
3018 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3019 enum rte_filter_type filter_type,
3020 enum rte_filter_op filter_op,
3024 struct priv *priv = dev->data->dev_private;
3026 switch (filter_type) {
3027 case RTE_ETH_FILTER_GENERIC:
3028 if (filter_op != RTE_ETH_FILTER_GET)
3030 *(const void **)arg = &mlx5_flow_ops;
3032 case RTE_ETH_FILTER_FDIR:
3033 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3036 ERROR("%p: filter type (%d) not supported",
3037 (void *)dev, filter_type);