1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox.
10 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
12 #pragma GCC diagnostic ignored "-Wpedantic"
14 #include <infiniband/verbs.h>
16 #pragma GCC diagnostic error "-Wpedantic"
19 #include <rte_ethdev_driver.h>
21 #include <rte_flow_driver.h>
22 #include <rte_malloc.h>
26 #include "mlx5_defs.h"
28 #include "mlx5_glue.h"
30 /* Define minimal priority for control plane flows. */
31 #define MLX5_CTRL_FLOW_PRIORITY 4
33 /* Internet Protocol versions. */
37 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
38 struct ibv_flow_spec_counter_action {
43 /* Dev ops structure defined in mlx5.c */
44 extern const struct eth_dev_ops mlx5_dev_ops;
45 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
48 mlx5_flow_create_eth(const struct rte_flow_item *item,
49 const void *default_mask,
53 mlx5_flow_create_vlan(const struct rte_flow_item *item,
54 const void *default_mask,
58 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
59 const void *default_mask,
63 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
64 const void *default_mask,
68 mlx5_flow_create_udp(const struct rte_flow_item *item,
69 const void *default_mask,
73 mlx5_flow_create_tcp(const struct rte_flow_item *item,
74 const void *default_mask,
78 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
79 const void *default_mask,
82 struct mlx5_flow_parse;
85 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
89 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
92 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
94 /* Hash RX queue types. */
105 /* Initialization data for hash RX queue. */
106 struct hash_rxq_init {
107 uint64_t hash_fields; /* Fields that participate in the hash. */
108 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
109 unsigned int flow_priority; /* Flow priority to use. */
110 unsigned int ip_version; /* Internet protocol. */
113 /* Initialization data for hash RX queues. */
114 const struct hash_rxq_init hash_rxq_init[] = {
116 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
117 IBV_RX_HASH_DST_IPV4 |
118 IBV_RX_HASH_SRC_PORT_TCP |
119 IBV_RX_HASH_DST_PORT_TCP),
120 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
122 .ip_version = MLX5_IPV4,
125 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
126 IBV_RX_HASH_DST_IPV4 |
127 IBV_RX_HASH_SRC_PORT_UDP |
128 IBV_RX_HASH_DST_PORT_UDP),
129 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
131 .ip_version = MLX5_IPV4,
134 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
135 IBV_RX_HASH_DST_IPV4),
136 .dpdk_rss_hf = (ETH_RSS_IPV4 |
139 .ip_version = MLX5_IPV4,
142 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
143 IBV_RX_HASH_DST_IPV6 |
144 IBV_RX_HASH_SRC_PORT_TCP |
145 IBV_RX_HASH_DST_PORT_TCP),
146 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
148 .ip_version = MLX5_IPV6,
151 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
152 IBV_RX_HASH_DST_IPV6 |
153 IBV_RX_HASH_SRC_PORT_UDP |
154 IBV_RX_HASH_DST_PORT_UDP),
155 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
157 .ip_version = MLX5_IPV6,
160 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
161 IBV_RX_HASH_DST_IPV6),
162 .dpdk_rss_hf = (ETH_RSS_IPV6 |
165 .ip_version = MLX5_IPV6,
174 /* Number of entries in hash_rxq_init[]. */
175 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
177 /** Structure for holding counter stats. */
178 struct mlx5_flow_counter_stats {
179 uint64_t hits; /**< Number of packets matched by the rule. */
180 uint64_t bytes; /**< Number of bytes matched by the rule. */
183 /** Structure for Drop queue. */
184 struct mlx5_hrxq_drop {
185 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
186 struct ibv_qp *qp; /**< Verbs queue pair. */
187 struct ibv_wq *wq; /**< Verbs work queue. */
188 struct ibv_cq *cq; /**< Verbs completion queue. */
191 /* Flows structures. */
193 uint64_t hash_fields; /**< Fields that participate in the hash. */
194 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
195 struct ibv_flow *ibv_flow; /**< Verbs flow. */
196 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
199 /* Drop flows structures. */
200 struct mlx5_flow_drop {
201 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
202 struct ibv_flow *ibv_flow; /**< Verbs flow. */
206 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
207 uint32_t mark:1; /**< Set if the flow is marked. */
208 uint32_t drop:1; /**< Drop queue. */
209 uint16_t queues_n; /**< Number of entries in queue[]. */
210 uint16_t (*queues)[]; /**< Queues indexes to use. */
211 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
212 uint8_t rss_key[40]; /**< copy of the RSS key. */
213 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
214 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
215 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
216 /**< Flow with Rx queue. */
219 /** Static initializer for items. */
221 (const enum rte_flow_item_type []){ \
222 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
225 /** Structure to generate a simple graph of layers supported by the NIC. */
226 struct mlx5_flow_items {
227 /** List of possible actions for these items. */
228 const enum rte_flow_action_type *const actions;
229 /** Bit-masks corresponding to the possibilities for the item. */
232 * Default bit-masks to use when item->mask is not provided. When
233 * \default_mask is also NULL, the full supported bit-mask (\mask) is
236 const void *default_mask;
237 /** Bit-masks size in bytes. */
238 const unsigned int mask_sz;
240 * Conversion function from rte_flow to NIC specific flow.
243 * rte_flow item to convert.
244 * @param default_mask
245 * Default bit-masks to use when item->mask is not provided.
247 * Internal structure to store the conversion.
250 * 0 on success, negative value otherwise.
252 int (*convert)(const struct rte_flow_item *item,
253 const void *default_mask,
255 /** Size in bytes of the destination structure. */
256 const unsigned int dst_sz;
257 /** List of possible following items. */
258 const enum rte_flow_item_type *const items;
261 /** Valid action for this PMD. */
262 static const enum rte_flow_action_type valid_actions[] = {
263 RTE_FLOW_ACTION_TYPE_DROP,
264 RTE_FLOW_ACTION_TYPE_QUEUE,
265 RTE_FLOW_ACTION_TYPE_MARK,
266 RTE_FLOW_ACTION_TYPE_FLAG,
267 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
268 RTE_FLOW_ACTION_TYPE_COUNT,
270 RTE_FLOW_ACTION_TYPE_END,
273 /** Graph of supported items and associated actions. */
274 static const struct mlx5_flow_items mlx5_flow_items[] = {
275 [RTE_FLOW_ITEM_TYPE_END] = {
276 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
277 RTE_FLOW_ITEM_TYPE_VXLAN),
279 [RTE_FLOW_ITEM_TYPE_ETH] = {
280 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
281 RTE_FLOW_ITEM_TYPE_IPV4,
282 RTE_FLOW_ITEM_TYPE_IPV6),
283 .actions = valid_actions,
284 .mask = &(const struct rte_flow_item_eth){
285 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
286 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
289 .default_mask = &rte_flow_item_eth_mask,
290 .mask_sz = sizeof(struct rte_flow_item_eth),
291 .convert = mlx5_flow_create_eth,
292 .dst_sz = sizeof(struct ibv_flow_spec_eth),
294 [RTE_FLOW_ITEM_TYPE_VLAN] = {
295 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
296 RTE_FLOW_ITEM_TYPE_IPV6),
297 .actions = valid_actions,
298 .mask = &(const struct rte_flow_item_vlan){
301 .default_mask = &rte_flow_item_vlan_mask,
302 .mask_sz = sizeof(struct rte_flow_item_vlan),
303 .convert = mlx5_flow_create_vlan,
306 [RTE_FLOW_ITEM_TYPE_IPV4] = {
307 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
308 RTE_FLOW_ITEM_TYPE_TCP),
309 .actions = valid_actions,
310 .mask = &(const struct rte_flow_item_ipv4){
314 .type_of_service = -1,
318 .default_mask = &rte_flow_item_ipv4_mask,
319 .mask_sz = sizeof(struct rte_flow_item_ipv4),
320 .convert = mlx5_flow_create_ipv4,
321 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
323 [RTE_FLOW_ITEM_TYPE_IPV6] = {
324 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
325 RTE_FLOW_ITEM_TYPE_TCP),
326 .actions = valid_actions,
327 .mask = &(const struct rte_flow_item_ipv6){
330 0xff, 0xff, 0xff, 0xff,
331 0xff, 0xff, 0xff, 0xff,
332 0xff, 0xff, 0xff, 0xff,
333 0xff, 0xff, 0xff, 0xff,
336 0xff, 0xff, 0xff, 0xff,
337 0xff, 0xff, 0xff, 0xff,
338 0xff, 0xff, 0xff, 0xff,
339 0xff, 0xff, 0xff, 0xff,
346 .default_mask = &rte_flow_item_ipv6_mask,
347 .mask_sz = sizeof(struct rte_flow_item_ipv6),
348 .convert = mlx5_flow_create_ipv6,
349 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
351 [RTE_FLOW_ITEM_TYPE_UDP] = {
352 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
353 .actions = valid_actions,
354 .mask = &(const struct rte_flow_item_udp){
360 .default_mask = &rte_flow_item_udp_mask,
361 .mask_sz = sizeof(struct rte_flow_item_udp),
362 .convert = mlx5_flow_create_udp,
363 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
365 [RTE_FLOW_ITEM_TYPE_TCP] = {
366 .actions = valid_actions,
367 .mask = &(const struct rte_flow_item_tcp){
373 .default_mask = &rte_flow_item_tcp_mask,
374 .mask_sz = sizeof(struct rte_flow_item_tcp),
375 .convert = mlx5_flow_create_tcp,
376 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
378 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
379 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
380 .actions = valid_actions,
381 .mask = &(const struct rte_flow_item_vxlan){
382 .vni = "\xff\xff\xff",
384 .default_mask = &rte_flow_item_vxlan_mask,
385 .mask_sz = sizeof(struct rte_flow_item_vxlan),
386 .convert = mlx5_flow_create_vxlan,
387 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
391 /** Structure to pass to the conversion function. */
392 struct mlx5_flow_parse {
393 uint32_t inner; /**< Set once VXLAN is encountered. */
395 /**< Whether resources should remain after a validate. */
396 uint32_t drop:1; /**< Target is a drop queue. */
397 uint32_t mark:1; /**< Mark is present in the flow. */
398 uint32_t count:1; /**< Count is present in the flow. */
399 uint32_t mark_id; /**< Mark identifier. */
400 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
401 uint16_t queues_n; /**< Number of entries in queue[]. */
402 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
403 uint8_t rss_key[40]; /**< copy of the RSS key. */
404 enum hash_rxq_type layer; /**< Last pattern layer detected. */
405 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
407 struct ibv_flow_attr *ibv_attr;
408 /**< Pointer to Verbs attributes. */
410 /**< Current position or total size of the attribute. */
411 } queue[RTE_DIM(hash_rxq_init)];
414 static const struct rte_flow_ops mlx5_flow_ops = {
415 .validate = mlx5_flow_validate,
416 .create = mlx5_flow_create,
417 .destroy = mlx5_flow_destroy,
418 .flush = mlx5_flow_flush,
419 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
420 .query = mlx5_flow_query,
424 .isolate = mlx5_flow_isolate,
427 /* Convert FDIR request to Generic flow. */
429 struct rte_flow_attr attr;
430 struct rte_flow_action actions[2];
431 struct rte_flow_item items[4];
432 struct rte_flow_item_eth l2;
433 struct rte_flow_item_eth l2_mask;
435 struct rte_flow_item_ipv4 ipv4;
436 struct rte_flow_item_ipv6 ipv6;
439 struct rte_flow_item_udp udp;
440 struct rte_flow_item_tcp tcp;
442 struct rte_flow_action_queue queue;
445 /* Verbs specification header. */
446 struct ibv_spec_header {
447 enum ibv_flow_spec_type type;
452 * Check support for a given item.
455 * Item specification.
457 * Bit-masks covering supported fields to compare with spec, last and mask in
460 * Bit-Mask size in bytes.
466 mlx5_flow_item_validate(const struct rte_flow_item *item,
467 const uint8_t *mask, unsigned int size)
471 if (!item->spec && (item->mask || item->last))
473 if (item->spec && !item->mask) {
475 const uint8_t *spec = item->spec;
477 for (i = 0; i < size; ++i)
478 if ((spec[i] | mask[i]) != mask[i])
481 if (item->last && !item->mask) {
483 const uint8_t *spec = item->last;
485 for (i = 0; i < size; ++i)
486 if ((spec[i] | mask[i]) != mask[i])
491 const uint8_t *spec = item->spec;
493 for (i = 0; i < size; ++i)
494 if ((spec[i] | mask[i]) != mask[i])
497 if (item->spec && item->last) {
500 const uint8_t *apply = mask;
505 for (i = 0; i < size; ++i) {
506 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
507 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
509 ret = memcmp(spec, last, size);
515 * Copy the RSS configuration from the user ones, of the rss_conf is null,
516 * uses the driver one.
519 * Pointer to private structure.
521 * Internal parser structure.
523 * User RSS configuration to save.
526 * 0 on success, errno value on failure.
529 priv_flow_convert_rss_conf(struct priv *priv,
530 struct mlx5_flow_parse *parser,
531 const struct rte_eth_rss_conf *rss_conf)
534 * This function is also called at the beginning of
535 * priv_flow_convert_actions() to initialize the parser with the
536 * device default RSS configuration.
540 if (rss_conf->rss_hf & MLX5_RSS_HF_MASK)
542 if (rss_conf->rss_key_len != 40)
544 if (rss_conf->rss_key_len && rss_conf->rss_key) {
545 parser->rss_conf.rss_key_len = rss_conf->rss_key_len;
546 memcpy(parser->rss_key, rss_conf->rss_key,
547 rss_conf->rss_key_len);
548 parser->rss_conf.rss_key = parser->rss_key;
550 parser->rss_conf.rss_hf = rss_conf->rss_hf;
556 * Extract attribute to the parser.
559 * Pointer to private structure.
561 * Flow rule attributes.
563 * Perform verbose error reporting if not NULL.
564 * @param[in, out] parser
565 * Internal parser structure.
568 * 0 on success, a negative errno value otherwise and rte_errno is set.
571 priv_flow_convert_attributes(struct priv *priv,
572 const struct rte_flow_attr *attr,
573 struct rte_flow_error *error,
574 struct mlx5_flow_parse *parser)
579 rte_flow_error_set(error, ENOTSUP,
580 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
582 "groups are not supported");
585 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
586 rte_flow_error_set(error, ENOTSUP,
587 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
589 "priorities are not supported");
593 rte_flow_error_set(error, ENOTSUP,
594 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
596 "egress is not supported");
599 if (!attr->ingress) {
600 rte_flow_error_set(error, ENOTSUP,
601 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
603 "only ingress is supported");
610 * Extract actions request to the parser.
613 * Pointer to private structure.
615 * Associated actions (list terminated by the END action).
617 * Perform verbose error reporting if not NULL.
618 * @param[in, out] parser
619 * Internal parser structure.
622 * 0 on success, a negative errno value otherwise and rte_errno is set.
625 priv_flow_convert_actions(struct priv *priv,
626 const struct rte_flow_action actions[],
627 struct rte_flow_error *error,
628 struct mlx5_flow_parse *parser)
631 * Add default RSS configuration necessary for Verbs to create QP even
632 * if no RSS is necessary.
634 priv_flow_convert_rss_conf(priv, parser,
635 (const struct rte_eth_rss_conf *)
637 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
638 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
640 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
642 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
643 const struct rte_flow_action_queue *queue =
644 (const struct rte_flow_action_queue *)
649 if (!queue || (queue->index > (priv->rxqs_n - 1)))
650 goto exit_action_not_supported;
651 for (n = 0; n < parser->queues_n; ++n) {
652 if (parser->queues[n] == queue->index) {
657 if (parser->queues_n > 1 && !found) {
658 rte_flow_error_set(error, ENOTSUP,
659 RTE_FLOW_ERROR_TYPE_ACTION,
661 "queue action not in RSS queues");
665 parser->queues_n = 1;
666 parser->queues[0] = queue->index;
668 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
669 const struct rte_flow_action_rss *rss =
670 (const struct rte_flow_action_rss *)
674 if (!rss || !rss->num) {
675 rte_flow_error_set(error, EINVAL,
676 RTE_FLOW_ERROR_TYPE_ACTION,
681 if (parser->queues_n == 1) {
684 assert(parser->queues_n);
685 for (n = 0; n < rss->num; ++n) {
686 if (parser->queues[0] ==
693 rte_flow_error_set(error, ENOTSUP,
694 RTE_FLOW_ERROR_TYPE_ACTION,
696 "queue action not in RSS"
701 for (n = 0; n < rss->num; ++n) {
702 if (rss->queue[n] >= priv->rxqs_n) {
703 rte_flow_error_set(error, EINVAL,
704 RTE_FLOW_ERROR_TYPE_ACTION,
706 "queue id > number of"
711 for (n = 0; n < rss->num; ++n)
712 parser->queues[n] = rss->queue[n];
713 parser->queues_n = rss->num;
714 if (priv_flow_convert_rss_conf(priv, parser,
716 rte_flow_error_set(error, EINVAL,
717 RTE_FLOW_ERROR_TYPE_ACTION,
719 "wrong RSS configuration");
722 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
723 const struct rte_flow_action_mark *mark =
724 (const struct rte_flow_action_mark *)
728 rte_flow_error_set(error, EINVAL,
729 RTE_FLOW_ERROR_TYPE_ACTION,
731 "mark must be defined");
733 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
734 rte_flow_error_set(error, ENOTSUP,
735 RTE_FLOW_ERROR_TYPE_ACTION,
737 "mark must be between 0"
742 parser->mark_id = mark->id;
743 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
745 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
746 priv->config.flow_counter_en) {
749 goto exit_action_not_supported;
752 if (parser->drop && parser->mark)
754 if (!parser->queues_n && !parser->drop) {
755 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
756 NULL, "no valid action");
760 exit_action_not_supported:
761 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
762 actions, "action not supported");
770 * Pointer to private structure.
772 * Pattern specification (list terminated by the END pattern item).
774 * Perform verbose error reporting if not NULL.
775 * @param[in, out] parser
776 * Internal parser structure.
779 * 0 on success, a negative errno value otherwise and rte_errno is set.
782 priv_flow_convert_items_validate(struct priv *priv,
783 const struct rte_flow_item items[],
784 struct rte_flow_error *error,
785 struct mlx5_flow_parse *parser)
787 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
791 /* Initialise the offsets to start after verbs attribute. */
792 for (i = 0; i != hash_rxq_init_n; ++i)
793 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
794 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
795 const struct mlx5_flow_items *token = NULL;
799 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
803 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
805 if (cur_item->items[i] == items->type) {
806 token = &mlx5_flow_items[items->type];
811 goto exit_item_not_supported;
813 err = mlx5_flow_item_validate(items,
814 (const uint8_t *)cur_item->mask,
817 goto exit_item_not_supported;
818 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
820 rte_flow_error_set(error, ENOTSUP,
821 RTE_FLOW_ERROR_TYPE_ITEM,
823 "cannot recognize multiple"
824 " VXLAN encapsulations");
827 parser->inner = IBV_FLOW_SPEC_INNER;
830 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
832 for (n = 0; n != hash_rxq_init_n; ++n)
833 parser->queue[n].offset += cur_item->dst_sz;
837 parser->queue[HASH_RXQ_ETH].offset +=
838 sizeof(struct ibv_flow_spec_action_drop);
841 for (i = 0; i != hash_rxq_init_n; ++i)
842 parser->queue[i].offset +=
843 sizeof(struct ibv_flow_spec_action_tag);
846 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
848 for (i = 0; i != hash_rxq_init_n; ++i)
849 parser->queue[i].offset += size;
852 exit_item_not_supported:
853 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
854 items, "item not supported");
859 * Allocate memory space to store verbs flow attributes.
862 * Pointer to private structure.
863 * @param[in] priority
866 * Amount of byte to allocate.
868 * Perform verbose error reporting if not NULL.
871 * A verbs flow attribute on success, NULL otherwise.
873 static struct ibv_flow_attr*
874 priv_flow_convert_allocate(struct priv *priv,
875 unsigned int priority,
877 struct rte_flow_error *error)
879 struct ibv_flow_attr *ibv_attr;
882 ibv_attr = rte_calloc(__func__, 1, size, 0);
884 rte_flow_error_set(error, ENOMEM,
885 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
887 "cannot allocate verbs spec attributes.");
890 ibv_attr->priority = priority;
895 * Finalise verbs flow attributes.
898 * Pointer to private structure.
899 * @param[in, out] parser
900 * Internal parser structure.
903 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
905 const unsigned int ipv4 =
906 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
907 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
908 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
909 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
910 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
911 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
915 /* Remove any other flow not matching the pattern. */
916 if (parser->queues_n == 1) {
917 for (i = 0; i != hash_rxq_init_n; ++i) {
918 if (i == HASH_RXQ_ETH)
920 rte_free(parser->queue[i].ibv_attr);
921 parser->queue[i].ibv_attr = NULL;
925 if (parser->layer == HASH_RXQ_ETH) {
929 * This layer becomes useless as the pattern define under
932 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
933 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
935 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
936 for (i = ohmin; i != (ohmax + 1); ++i) {
937 if (!parser->queue[i].ibv_attr)
939 rte_free(parser->queue[i].ibv_attr);
940 parser->queue[i].ibv_attr = NULL;
942 /* Remove impossible flow according to the RSS configuration. */
943 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
944 parser->rss_conf.rss_hf) {
945 /* Remove any other flow. */
946 for (i = hmin; i != (hmax + 1); ++i) {
947 if ((i == parser->layer) ||
948 (!parser->queue[i].ibv_attr))
950 rte_free(parser->queue[i].ibv_attr);
951 parser->queue[i].ibv_attr = NULL;
953 } else if (!parser->queue[ip].ibv_attr) {
954 /* no RSS possible with the current configuration. */
955 parser->queues_n = 1;
960 * Fill missing layers in verbs specifications, or compute the correct
961 * offset to allocate the memory space for the attributes and
964 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
966 struct ibv_flow_spec_ipv4_ext ipv4;
967 struct ibv_flow_spec_ipv6 ipv6;
968 struct ibv_flow_spec_tcp_udp udp_tcp;
973 if (i == parser->layer)
975 if (parser->layer == HASH_RXQ_ETH) {
976 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
977 size = sizeof(struct ibv_flow_spec_ipv4_ext);
978 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
979 .type = IBV_FLOW_SPEC_IPV4_EXT,
983 size = sizeof(struct ibv_flow_spec_ipv6);
984 specs.ipv6 = (struct ibv_flow_spec_ipv6){
985 .type = IBV_FLOW_SPEC_IPV6,
989 if (parser->queue[i].ibv_attr) {
990 dst = (void *)((uintptr_t)
991 parser->queue[i].ibv_attr +
992 parser->queue[i].offset);
993 memcpy(dst, &specs, size);
994 ++parser->queue[i].ibv_attr->num_of_specs;
996 parser->queue[i].offset += size;
998 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
999 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1000 size = sizeof(struct ibv_flow_spec_tcp_udp);
1001 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1002 .type = ((i == HASH_RXQ_UDPV4 ||
1003 i == HASH_RXQ_UDPV6) ?
1008 if (parser->queue[i].ibv_attr) {
1009 dst = (void *)((uintptr_t)
1010 parser->queue[i].ibv_attr +
1011 parser->queue[i].offset);
1012 memcpy(dst, &specs, size);
1013 ++parser->queue[i].ibv_attr->num_of_specs;
1015 parser->queue[i].offset += size;
1021 * Validate and convert a flow supported by the NIC.
1024 * Pointer to private structure.
1026 * Flow rule attributes.
1027 * @param[in] pattern
1028 * Pattern specification (list terminated by the END pattern item).
1029 * @param[in] actions
1030 * Associated actions (list terminated by the END action).
1032 * Perform verbose error reporting if not NULL.
1033 * @param[in, out] parser
1034 * Internal parser structure.
1037 * 0 on success, a negative errno value otherwise and rte_errno is set.
1040 priv_flow_convert(struct priv *priv,
1041 const struct rte_flow_attr *attr,
1042 const struct rte_flow_item items[],
1043 const struct rte_flow_action actions[],
1044 struct rte_flow_error *error,
1045 struct mlx5_flow_parse *parser)
1047 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1051 /* First step. Validate the attributes, items and actions. */
1052 *parser = (struct mlx5_flow_parse){
1053 .create = parser->create,
1054 .layer = HASH_RXQ_ETH,
1055 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1057 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1060 ret = priv_flow_convert_actions(priv, actions, error, parser);
1063 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1066 priv_flow_convert_finalise(priv, parser);
1069 * Allocate the memory space to store verbs specifications.
1072 unsigned int priority =
1074 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1075 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1077 parser->queue[HASH_RXQ_ETH].ibv_attr =
1078 priv_flow_convert_allocate(priv, priority,
1080 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1082 parser->queue[HASH_RXQ_ETH].offset =
1083 sizeof(struct ibv_flow_attr);
1085 for (i = 0; i != hash_rxq_init_n; ++i) {
1086 unsigned int priority =
1088 hash_rxq_init[i].flow_priority;
1089 unsigned int offset;
1091 if (!(parser->rss_conf.rss_hf &
1092 hash_rxq_init[i].dpdk_rss_hf) &&
1093 (i != HASH_RXQ_ETH))
1095 offset = parser->queue[i].offset;
1096 parser->queue[i].ibv_attr =
1097 priv_flow_convert_allocate(priv, priority,
1099 if (!parser->queue[i].ibv_attr)
1101 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1104 /* Third step. Conversion parse, fill the specifications. */
1106 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1107 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1109 cur_item = &mlx5_flow_items[items->type];
1110 ret = cur_item->convert(items,
1111 (cur_item->default_mask ?
1112 cur_item->default_mask :
1116 rte_flow_error_set(error, ret,
1117 RTE_FLOW_ERROR_TYPE_ITEM,
1118 items, "item not supported");
1123 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1124 if (parser->count && parser->create) {
1125 mlx5_flow_create_count(priv, parser);
1127 goto exit_count_error;
1130 * Last step. Complete missing specification to reach the RSS
1133 if (!parser->drop) {
1134 priv_flow_convert_finalise(priv, parser);
1136 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1138 hash_rxq_init[parser->layer].flow_priority;
1141 /* Only verification is expected, all resources should be released. */
1142 if (!parser->create) {
1143 for (i = 0; i != hash_rxq_init_n; ++i) {
1144 if (parser->queue[i].ibv_attr) {
1145 rte_free(parser->queue[i].ibv_attr);
1146 parser->queue[i].ibv_attr = NULL;
1152 for (i = 0; i != hash_rxq_init_n; ++i) {
1153 if (parser->queue[i].ibv_attr) {
1154 rte_free(parser->queue[i].ibv_attr);
1155 parser->queue[i].ibv_attr = NULL;
1158 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1159 NULL, "cannot allocate verbs spec attributes.");
1162 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1163 NULL, "cannot create counter.");
1168 * Copy the specification created into the flow.
1171 * Internal parser structure.
1173 * Create specification.
1175 * Size in bytes of the specification to copy.
1178 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1184 for (i = 0; i != hash_rxq_init_n; ++i) {
1185 if (!parser->queue[i].ibv_attr)
1187 /* Specification must be the same l3 type or none. */
1188 if (parser->layer == HASH_RXQ_ETH ||
1189 (hash_rxq_init[parser->layer].ip_version ==
1190 hash_rxq_init[i].ip_version) ||
1191 (hash_rxq_init[i].ip_version == 0)) {
1192 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1193 parser->queue[i].offset);
1194 memcpy(dst, src, size);
1195 ++parser->queue[i].ibv_attr->num_of_specs;
1196 parser->queue[i].offset += size;
1202 * Convert Ethernet item to Verbs specification.
1205 * Item specification.
1206 * @param default_mask[in]
1207 * Default bit-masks to use when item->mask is not provided.
1208 * @param data[in, out]
1212 mlx5_flow_create_eth(const struct rte_flow_item *item,
1213 const void *default_mask,
1216 const struct rte_flow_item_eth *spec = item->spec;
1217 const struct rte_flow_item_eth *mask = item->mask;
1218 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1219 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1220 struct ibv_flow_spec_eth eth = {
1221 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1225 /* Don't update layer for the inner pattern. */
1227 parser->layer = HASH_RXQ_ETH;
1232 mask = default_mask;
1233 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1234 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1235 eth.val.ether_type = spec->type;
1236 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1237 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1238 eth.mask.ether_type = mask->type;
1239 /* Remove unwanted bits from values. */
1240 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1241 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1242 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1244 eth.val.ether_type &= eth.mask.ether_type;
1246 mlx5_flow_create_copy(parser, ð, eth_size);
1251 * Convert VLAN item to Verbs specification.
1254 * Item specification.
1255 * @param default_mask[in]
1256 * Default bit-masks to use when item->mask is not provided.
1257 * @param data[in, out]
1261 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1262 const void *default_mask,
1265 const struct rte_flow_item_vlan *spec = item->spec;
1266 const struct rte_flow_item_vlan *mask = item->mask;
1267 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1268 struct ibv_flow_spec_eth *eth;
1269 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1274 mask = default_mask;
1276 for (i = 0; i != hash_rxq_init_n; ++i) {
1277 if (!parser->queue[i].ibv_attr)
1280 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1281 parser->queue[i].offset - eth_size);
1282 eth->val.vlan_tag = spec->tci;
1283 eth->mask.vlan_tag = mask->tci;
1284 eth->val.vlan_tag &= eth->mask.vlan_tag;
1291 * Convert IPv4 item to Verbs specification.
1294 * Item specification.
1295 * @param default_mask[in]
1296 * Default bit-masks to use when item->mask is not provided.
1297 * @param data[in, out]
1301 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1302 const void *default_mask,
1305 const struct rte_flow_item_ipv4 *spec = item->spec;
1306 const struct rte_flow_item_ipv4 *mask = item->mask;
1307 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1308 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1309 struct ibv_flow_spec_ipv4_ext ipv4 = {
1310 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1314 /* Don't update layer for the inner pattern. */
1316 parser->layer = HASH_RXQ_IPV4;
1319 mask = default_mask;
1320 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1321 .src_ip = spec->hdr.src_addr,
1322 .dst_ip = spec->hdr.dst_addr,
1323 .proto = spec->hdr.next_proto_id,
1324 .tos = spec->hdr.type_of_service,
1326 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1327 .src_ip = mask->hdr.src_addr,
1328 .dst_ip = mask->hdr.dst_addr,
1329 .proto = mask->hdr.next_proto_id,
1330 .tos = mask->hdr.type_of_service,
1332 /* Remove unwanted bits from values. */
1333 ipv4.val.src_ip &= ipv4.mask.src_ip;
1334 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1335 ipv4.val.proto &= ipv4.mask.proto;
1336 ipv4.val.tos &= ipv4.mask.tos;
1338 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1343 * Convert IPv6 item to Verbs specification.
1346 * Item specification.
1347 * @param default_mask[in]
1348 * Default bit-masks to use when item->mask is not provided.
1349 * @param data[in, out]
1353 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1354 const void *default_mask,
1357 const struct rte_flow_item_ipv6 *spec = item->spec;
1358 const struct rte_flow_item_ipv6 *mask = item->mask;
1359 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1360 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1361 struct ibv_flow_spec_ipv6 ipv6 = {
1362 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1366 /* Don't update layer for the inner pattern. */
1368 parser->layer = HASH_RXQ_IPV6;
1371 uint32_t vtc_flow_val;
1372 uint32_t vtc_flow_mask;
1375 mask = default_mask;
1376 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1377 RTE_DIM(ipv6.val.src_ip));
1378 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1379 RTE_DIM(ipv6.val.dst_ip));
1380 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1381 RTE_DIM(ipv6.mask.src_ip));
1382 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1383 RTE_DIM(ipv6.mask.dst_ip));
1384 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
1385 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
1386 ipv6.val.flow_label =
1387 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
1389 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
1391 ipv6.val.next_hdr = spec->hdr.proto;
1392 ipv6.val.hop_limit = spec->hdr.hop_limits;
1393 ipv6.mask.flow_label =
1394 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
1396 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
1398 ipv6.mask.next_hdr = mask->hdr.proto;
1399 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1400 /* Remove unwanted bits from values. */
1401 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1402 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1403 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1405 ipv6.val.flow_label &= ipv6.mask.flow_label;
1406 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
1407 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1408 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1410 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1415 * Convert UDP item to Verbs specification.
1418 * Item specification.
1419 * @param default_mask[in]
1420 * Default bit-masks to use when item->mask is not provided.
1421 * @param data[in, out]
1425 mlx5_flow_create_udp(const struct rte_flow_item *item,
1426 const void *default_mask,
1429 const struct rte_flow_item_udp *spec = item->spec;
1430 const struct rte_flow_item_udp *mask = item->mask;
1431 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1432 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1433 struct ibv_flow_spec_tcp_udp udp = {
1434 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1438 /* Don't update layer for the inner pattern. */
1439 if (!parser->inner) {
1440 if (parser->layer == HASH_RXQ_IPV4)
1441 parser->layer = HASH_RXQ_UDPV4;
1443 parser->layer = HASH_RXQ_UDPV6;
1447 mask = default_mask;
1448 udp.val.dst_port = spec->hdr.dst_port;
1449 udp.val.src_port = spec->hdr.src_port;
1450 udp.mask.dst_port = mask->hdr.dst_port;
1451 udp.mask.src_port = mask->hdr.src_port;
1452 /* Remove unwanted bits from values. */
1453 udp.val.src_port &= udp.mask.src_port;
1454 udp.val.dst_port &= udp.mask.dst_port;
1456 mlx5_flow_create_copy(parser, &udp, udp_size);
1461 * Convert TCP item to Verbs specification.
1464 * Item specification.
1465 * @param default_mask[in]
1466 * Default bit-masks to use when item->mask is not provided.
1467 * @param data[in, out]
1471 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1472 const void *default_mask,
1475 const struct rte_flow_item_tcp *spec = item->spec;
1476 const struct rte_flow_item_tcp *mask = item->mask;
1477 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1478 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1479 struct ibv_flow_spec_tcp_udp tcp = {
1480 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1484 /* Don't update layer for the inner pattern. */
1485 if (!parser->inner) {
1486 if (parser->layer == HASH_RXQ_IPV4)
1487 parser->layer = HASH_RXQ_TCPV4;
1489 parser->layer = HASH_RXQ_TCPV6;
1493 mask = default_mask;
1494 tcp.val.dst_port = spec->hdr.dst_port;
1495 tcp.val.src_port = spec->hdr.src_port;
1496 tcp.mask.dst_port = mask->hdr.dst_port;
1497 tcp.mask.src_port = mask->hdr.src_port;
1498 /* Remove unwanted bits from values. */
1499 tcp.val.src_port &= tcp.mask.src_port;
1500 tcp.val.dst_port &= tcp.mask.dst_port;
1502 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1507 * Convert VXLAN item to Verbs specification.
1510 * Item specification.
1511 * @param default_mask[in]
1512 * Default bit-masks to use when item->mask is not provided.
1513 * @param data[in, out]
1517 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1518 const void *default_mask,
1521 const struct rte_flow_item_vxlan *spec = item->spec;
1522 const struct rte_flow_item_vxlan *mask = item->mask;
1523 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1524 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1525 struct ibv_flow_spec_tunnel vxlan = {
1526 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1535 parser->inner = IBV_FLOW_SPEC_INNER;
1538 mask = default_mask;
1539 memcpy(&id.vni[1], spec->vni, 3);
1540 vxlan.val.tunnel_id = id.vlan_id;
1541 memcpy(&id.vni[1], mask->vni, 3);
1542 vxlan.mask.tunnel_id = id.vlan_id;
1543 /* Remove unwanted bits from values. */
1544 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1547 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1548 * layer is defined in the Verbs specification it is interpreted as
1549 * wildcard and all packets will match this rule, if it follows a full
1550 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1551 * before will also match this rule.
1552 * To avoid such situation, VNI 0 is currently refused.
1554 if (!vxlan.val.tunnel_id)
1556 mlx5_flow_create_copy(parser, &vxlan, size);
1561 * Convert mark/flag action to Verbs specification.
1564 * Internal parser structure.
1569 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1571 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1572 struct ibv_flow_spec_action_tag tag = {
1573 .type = IBV_FLOW_SPEC_ACTION_TAG,
1575 .tag_id = mlx5_flow_mark_set(mark_id),
1578 assert(parser->mark);
1579 mlx5_flow_create_copy(parser, &tag, size);
1584 * Convert count action to Verbs specification.
1587 * Pointer to private structure.
1589 * Pointer to MLX5 flow parser structure.
1592 * 0 on success, errno value on failure.
1595 mlx5_flow_create_count(struct priv *priv __rte_unused,
1596 struct mlx5_flow_parse *parser __rte_unused)
1598 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1599 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1600 struct ibv_counter_set_init_attr init_attr = {0};
1601 struct ibv_flow_spec_counter_action counter = {
1602 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1604 .counter_set_handle = 0,
1607 init_attr.counter_set_id = 0;
1608 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
1611 counter.counter_set_handle = parser->cs->handle;
1612 mlx5_flow_create_copy(parser, &counter, size);
1618 * Complete flow rule creation with a drop queue.
1621 * Pointer to private structure.
1623 * Internal parser structure.
1625 * Pointer to the rte_flow.
1627 * Perform verbose error reporting if not NULL.
1630 * 0 on success, errno value on failure.
1633 priv_flow_create_action_queue_drop(struct priv *priv,
1634 struct mlx5_flow_parse *parser,
1635 struct rte_flow *flow,
1636 struct rte_flow_error *error)
1638 struct ibv_flow_spec_action_drop *drop;
1639 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1645 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
1646 parser->queue[HASH_RXQ_ETH].offset);
1647 *drop = (struct ibv_flow_spec_action_drop){
1648 .type = IBV_FLOW_SPEC_ACTION_DROP,
1651 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
1652 parser->queue[HASH_RXQ_ETH].offset += size;
1653 flow->frxq[HASH_RXQ_ETH].ibv_attr =
1654 parser->queue[HASH_RXQ_ETH].ibv_attr;
1656 flow->cs = parser->cs;
1657 if (!priv->dev->data->dev_started)
1659 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
1660 flow->frxq[HASH_RXQ_ETH].ibv_flow =
1661 mlx5_glue->create_flow(priv->flow_drop_queue->qp,
1662 flow->frxq[HASH_RXQ_ETH].ibv_attr);
1663 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1664 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1665 NULL, "flow rule creation failure");
1672 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1673 claim_zero(mlx5_glue->destroy_flow
1674 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
1675 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
1677 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
1678 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
1679 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
1682 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1690 * Create hash Rx queues when RSS is enabled.
1693 * Pointer to private structure.
1695 * Internal parser structure.
1697 * Pointer to the rte_flow.
1699 * Perform verbose error reporting if not NULL.
1702 * 0 on success, a errno value otherwise and rte_errno is set.
1705 priv_flow_create_action_queue_rss(struct priv *priv,
1706 struct mlx5_flow_parse *parser,
1707 struct rte_flow *flow,
1708 struct rte_flow_error *error)
1712 for (i = 0; i != hash_rxq_init_n; ++i) {
1713 uint64_t hash_fields;
1715 if (!parser->queue[i].ibv_attr)
1717 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1718 parser->queue[i].ibv_attr = NULL;
1719 hash_fields = hash_rxq_init[i].hash_fields;
1720 if (!priv->dev->data->dev_started)
1722 flow->frxq[i].hrxq =
1723 mlx5_priv_hrxq_get(priv,
1724 parser->rss_conf.rss_key,
1725 parser->rss_conf.rss_key_len,
1729 if (flow->frxq[i].hrxq)
1731 flow->frxq[i].hrxq =
1732 mlx5_priv_hrxq_new(priv,
1733 parser->rss_conf.rss_key,
1734 parser->rss_conf.rss_key_len,
1738 if (!flow->frxq[i].hrxq) {
1739 rte_flow_error_set(error, ENOMEM,
1740 RTE_FLOW_ERROR_TYPE_HANDLE,
1741 NULL, "cannot create hash rxq");
1749 * Complete flow rule creation.
1752 * Pointer to private structure.
1754 * Internal parser structure.
1756 * Pointer to the rte_flow.
1758 * Perform verbose error reporting if not NULL.
1761 * 0 on success, a errno value otherwise and rte_errno is set.
1764 priv_flow_create_action_queue(struct priv *priv,
1765 struct mlx5_flow_parse *parser,
1766 struct rte_flow *flow,
1767 struct rte_flow_error *error)
1771 unsigned int flows_n = 0;
1775 assert(!parser->drop);
1776 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1780 flow->cs = parser->cs;
1781 if (!priv->dev->data->dev_started)
1783 for (i = 0; i != hash_rxq_init_n; ++i) {
1784 if (!flow->frxq[i].hrxq)
1786 flow->frxq[i].ibv_flow =
1787 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
1788 flow->frxq[i].ibv_attr);
1789 if (!flow->frxq[i].ibv_flow) {
1790 rte_flow_error_set(error, ENOMEM,
1791 RTE_FLOW_ERROR_TYPE_HANDLE,
1792 NULL, "flow rule creation failure");
1797 DEBUG("%p type %d QP %p ibv_flow %p",
1799 (void *)flow->frxq[i].hrxq,
1800 (void *)flow->frxq[i].ibv_flow);
1803 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
1804 NULL, "internal error in flow creation");
1807 for (i = 0; i != parser->queues_n; ++i) {
1808 struct mlx5_rxq_data *q =
1809 (*priv->rxqs)[parser->queues[i]];
1811 q->mark |= parser->mark;
1816 for (i = 0; i != hash_rxq_init_n; ++i) {
1817 if (flow->frxq[i].ibv_flow) {
1818 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1820 claim_zero(mlx5_glue->destroy_flow(ibv_flow));
1822 if (flow->frxq[i].hrxq)
1823 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1824 if (flow->frxq[i].ibv_attr)
1825 rte_free(flow->frxq[i].ibv_attr);
1828 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1839 * Pointer to private structure.
1841 * Pointer to a TAILQ flow list.
1843 * Flow rule attributes.
1844 * @param[in] pattern
1845 * Pattern specification (list terminated by the END pattern item).
1846 * @param[in] actions
1847 * Associated actions (list terminated by the END action).
1849 * Perform verbose error reporting if not NULL.
1852 * A flow on success, NULL otherwise.
1854 static struct rte_flow *
1855 priv_flow_create(struct priv *priv,
1856 struct mlx5_flows *list,
1857 const struct rte_flow_attr *attr,
1858 const struct rte_flow_item items[],
1859 const struct rte_flow_action actions[],
1860 struct rte_flow_error *error)
1862 struct mlx5_flow_parse parser = { .create = 1, };
1863 struct rte_flow *flow = NULL;
1867 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1870 flow = rte_calloc(__func__, 1,
1871 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1874 rte_flow_error_set(error, ENOMEM,
1875 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1877 "cannot allocate flow memory");
1880 /* Copy queues configuration. */
1881 flow->queues = (uint16_t (*)[])(flow + 1);
1882 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1883 flow->queues_n = parser.queues_n;
1884 flow->mark = parser.mark;
1885 /* Copy RSS configuration. */
1886 flow->rss_conf = parser.rss_conf;
1887 flow->rss_conf.rss_key = flow->rss_key;
1888 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1889 /* finalise the flow. */
1891 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1894 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1897 TAILQ_INSERT_TAIL(list, flow, next);
1898 DEBUG("Flow created %p", (void *)flow);
1901 ERROR("flow creation error: %s", error->message);
1902 for (i = 0; i != hash_rxq_init_n; ++i) {
1903 if (parser.queue[i].ibv_attr)
1904 rte_free(parser.queue[i].ibv_attr);
1911 * Validate a flow supported by the NIC.
1913 * @see rte_flow_validate()
1917 mlx5_flow_validate(struct rte_eth_dev *dev,
1918 const struct rte_flow_attr *attr,
1919 const struct rte_flow_item items[],
1920 const struct rte_flow_action actions[],
1921 struct rte_flow_error *error)
1923 struct priv *priv = dev->data->dev_private;
1925 struct mlx5_flow_parse parser = { .create = 0, };
1928 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1936 * @see rte_flow_create()
1940 mlx5_flow_create(struct rte_eth_dev *dev,
1941 const struct rte_flow_attr *attr,
1942 const struct rte_flow_item items[],
1943 const struct rte_flow_action actions[],
1944 struct rte_flow_error *error)
1946 struct priv *priv = dev->data->dev_private;
1947 struct rte_flow *flow;
1950 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1960 * Pointer to private structure.
1962 * Pointer to a TAILQ flow list.
1967 priv_flow_destroy(struct priv *priv,
1968 struct mlx5_flows *list,
1969 struct rte_flow *flow)
1973 if (flow->drop || !flow->mark)
1975 for (i = 0; i != flow->queues_n; ++i) {
1976 struct rte_flow *tmp;
1980 * To remove the mark from the queue, the queue must not be
1981 * present in any other marked flow (RSS or not).
1983 TAILQ_FOREACH(tmp, list, next) {
1985 uint16_t *tqs = NULL;
1990 for (j = 0; j != hash_rxq_init_n; ++j) {
1991 if (!tmp->frxq[j].hrxq)
1993 tqs = tmp->frxq[j].hrxq->ind_table->queues;
1994 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
1998 for (j = 0; (j != tq_n) && !mark; j++)
1999 if (tqs[j] == (*flow->queues)[i])
2002 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2006 if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
2007 claim_zero(mlx5_glue->destroy_flow
2008 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2009 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
2011 for (i = 0; i != hash_rxq_init_n; ++i) {
2012 struct mlx5_flow *frxq = &flow->frxq[i];
2015 claim_zero(mlx5_glue->destroy_flow
2018 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2020 rte_free(frxq->ibv_attr);
2024 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
2027 TAILQ_REMOVE(list, flow, next);
2028 DEBUG("Flow destroyed %p", (void *)flow);
2033 * Destroy all flows.
2036 * Pointer to private structure.
2038 * Pointer to a TAILQ flow list.
2041 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2043 while (!TAILQ_EMPTY(list)) {
2044 struct rte_flow *flow;
2046 flow = TAILQ_FIRST(list);
2047 priv_flow_destroy(priv, list, flow);
2052 * Create drop queue.
2055 * Pointer to private structure.
2061 priv_flow_create_drop_queue(struct priv *priv)
2063 struct mlx5_hrxq_drop *fdq = NULL;
2067 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2069 WARN("cannot allocate memory for drop queue");
2072 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
2074 WARN("cannot allocate CQ for drop queue");
2077 fdq->wq = mlx5_glue->create_wq
2079 &(struct ibv_wq_init_attr){
2080 .wq_type = IBV_WQT_RQ,
2087 WARN("cannot allocate WQ for drop queue");
2090 fdq->ind_table = mlx5_glue->create_rwq_ind_table
2092 &(struct ibv_rwq_ind_table_init_attr){
2093 .log_ind_tbl_size = 0,
2094 .ind_tbl = &fdq->wq,
2097 if (!fdq->ind_table) {
2098 WARN("cannot allocate indirection table for drop queue");
2101 fdq->qp = mlx5_glue->create_qp_ex
2103 &(struct ibv_qp_init_attr_ex){
2104 .qp_type = IBV_QPT_RAW_PACKET,
2106 IBV_QP_INIT_ATTR_PD |
2107 IBV_QP_INIT_ATTR_IND_TABLE |
2108 IBV_QP_INIT_ATTR_RX_HASH,
2109 .rx_hash_conf = (struct ibv_rx_hash_conf){
2111 IBV_RX_HASH_FUNC_TOEPLITZ,
2112 .rx_hash_key_len = rss_hash_default_key_len,
2113 .rx_hash_key = rss_hash_default_key,
2114 .rx_hash_fields_mask = 0,
2116 .rwq_ind_tbl = fdq->ind_table,
2120 WARN("cannot allocate QP for drop queue");
2123 priv->flow_drop_queue = fdq;
2127 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2129 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2131 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2133 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2136 priv->flow_drop_queue = NULL;
2141 * Delete drop queue.
2144 * Pointer to private structure.
2147 priv_flow_delete_drop_queue(struct priv *priv)
2149 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2154 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2156 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2158 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2160 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2162 priv->flow_drop_queue = NULL;
2169 * Pointer to private structure.
2171 * Pointer to a TAILQ flow list.
2174 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2176 struct rte_flow *flow;
2178 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2180 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2183 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
2185 claim_zero(mlx5_glue->destroy_flow
2186 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2187 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
2188 DEBUG("Flow %p removed", (void *)flow);
2192 /* Verify the flow has not already been cleaned. */
2193 for (i = 0; i != hash_rxq_init_n; ++i) {
2194 if (!flow->frxq[i].ibv_flow)
2197 * Indirection table may be necessary to remove the
2198 * flags in the Rx queues.
2199 * This helps to speed-up the process by avoiding
2202 ind_tbl = flow->frxq[i].hrxq->ind_table;
2205 if (i == hash_rxq_init_n)
2209 for (i = 0; i != ind_tbl->queues_n; ++i)
2210 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2212 for (i = 0; i != hash_rxq_init_n; ++i) {
2213 if (!flow->frxq[i].ibv_flow)
2215 claim_zero(mlx5_glue->destroy_flow
2216 (flow->frxq[i].ibv_flow));
2217 flow->frxq[i].ibv_flow = NULL;
2218 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2219 flow->frxq[i].hrxq = NULL;
2221 DEBUG("Flow %p removed", (void *)flow);
2229 * Pointer to private structure.
2231 * Pointer to a TAILQ flow list.
2234 * 0 on success, a errno value otherwise and rte_errno is set.
2237 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2239 struct rte_flow *flow;
2241 TAILQ_FOREACH(flow, list, next) {
2245 flow->frxq[HASH_RXQ_ETH].ibv_flow =
2246 mlx5_glue->create_flow
2247 (priv->flow_drop_queue->qp,
2248 flow->frxq[HASH_RXQ_ETH].ibv_attr);
2249 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
2250 DEBUG("Flow %p cannot be applied",
2255 DEBUG("Flow %p applied", (void *)flow);
2259 for (i = 0; i != hash_rxq_init_n; ++i) {
2260 if (!flow->frxq[i].ibv_attr)
2262 flow->frxq[i].hrxq =
2263 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2264 flow->rss_conf.rss_key_len,
2265 hash_rxq_init[i].hash_fields,
2268 if (flow->frxq[i].hrxq)
2270 flow->frxq[i].hrxq =
2271 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2272 flow->rss_conf.rss_key_len,
2273 hash_rxq_init[i].hash_fields,
2276 if (!flow->frxq[i].hrxq) {
2277 DEBUG("Flow %p cannot be applied",
2283 flow->frxq[i].ibv_flow =
2284 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
2285 flow->frxq[i].ibv_attr);
2286 if (!flow->frxq[i].ibv_flow) {
2287 DEBUG("Flow %p cannot be applied",
2292 DEBUG("Flow %p applied", (void *)flow);
2296 for (i = 0; i != flow->queues_n; ++i)
2297 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2303 * Verify the flow list is empty
2306 * Pointer to private structure.
2308 * @return the number of flows not released.
2311 priv_flow_verify(struct priv *priv)
2313 struct rte_flow *flow;
2316 TAILQ_FOREACH(flow, &priv->flows, next) {
2317 DEBUG("%p: flow %p still referenced", (void *)priv,
2325 * Enable a control flow configured from the control plane.
2328 * Pointer to Ethernet device.
2330 * An Ethernet flow spec to apply.
2332 * An Ethernet flow mask to apply.
2334 * A VLAN flow spec to apply.
2336 * A VLAN flow mask to apply.
2342 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2343 struct rte_flow_item_eth *eth_spec,
2344 struct rte_flow_item_eth *eth_mask,
2345 struct rte_flow_item_vlan *vlan_spec,
2346 struct rte_flow_item_vlan *vlan_mask)
2348 struct priv *priv = dev->data->dev_private;
2349 const struct rte_flow_attr attr = {
2351 .priority = MLX5_CTRL_FLOW_PRIORITY,
2353 struct rte_flow_item items[] = {
2355 .type = RTE_FLOW_ITEM_TYPE_ETH,
2361 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2362 RTE_FLOW_ITEM_TYPE_END,
2368 .type = RTE_FLOW_ITEM_TYPE_END,
2371 struct rte_flow_action actions[] = {
2373 .type = RTE_FLOW_ACTION_TYPE_RSS,
2376 .type = RTE_FLOW_ACTION_TYPE_END,
2379 struct rte_flow *flow;
2380 struct rte_flow_error error;
2383 struct rte_flow_action_rss rss;
2385 const struct rte_eth_rss_conf *rss_conf;
2387 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2391 if (!priv->reta_idx_n)
2393 for (i = 0; i != priv->reta_idx_n; ++i)
2394 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2395 action_rss.local.rss_conf = &priv->rss_conf;
2396 action_rss.local.num = priv->reta_idx_n;
2397 actions[0].conf = (const void *)&action_rss.rss;
2398 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2406 * Enable a flow control configured from the control plane.
2409 * Pointer to Ethernet device.
2411 * An Ethernet flow spec to apply.
2413 * An Ethernet flow mask to apply.
2419 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2420 struct rte_flow_item_eth *eth_spec,
2421 struct rte_flow_item_eth *eth_mask)
2423 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2429 * @see rte_flow_destroy()
2433 mlx5_flow_destroy(struct rte_eth_dev *dev,
2434 struct rte_flow *flow,
2435 struct rte_flow_error *error)
2437 struct priv *priv = dev->data->dev_private;
2441 priv_flow_destroy(priv, &priv->flows, flow);
2447 * Destroy all flows.
2449 * @see rte_flow_flush()
2453 mlx5_flow_flush(struct rte_eth_dev *dev,
2454 struct rte_flow_error *error)
2456 struct priv *priv = dev->data->dev_private;
2460 priv_flow_flush(priv, &priv->flows);
2465 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2467 * Query flow counter.
2471 * @param counter_value
2472 * returned data from the counter.
2475 * 0 on success, a errno value otherwise and rte_errno is set.
2478 priv_flow_query_count(struct ibv_counter_set *cs,
2479 struct mlx5_flow_counter_stats *counter_stats,
2480 struct rte_flow_query_count *query_count,
2481 struct rte_flow_error *error)
2483 uint64_t counters[2];
2484 struct ibv_query_counter_set_attr query_cs_attr = {
2486 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2488 struct ibv_counter_set_data query_out = {
2490 .outlen = 2 * sizeof(uint64_t),
2492 int res = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
2495 rte_flow_error_set(error, -res,
2496 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2498 "cannot read counter");
2501 query_count->hits_set = 1;
2502 query_count->bytes_set = 1;
2503 query_count->hits = counters[0] - counter_stats->hits;
2504 query_count->bytes = counters[1] - counter_stats->bytes;
2505 if (query_count->reset) {
2506 counter_stats->hits = counters[0];
2507 counter_stats->bytes = counters[1];
2515 * @see rte_flow_query()
2519 mlx5_flow_query(struct rte_eth_dev *dev,
2520 struct rte_flow *flow,
2521 enum rte_flow_action_type action __rte_unused,
2523 struct rte_flow_error *error)
2525 struct priv *priv = dev->data->dev_private;
2530 res = priv_flow_query_count(flow->cs,
2531 &flow->counter_stats,
2532 (struct rte_flow_query_count *)data,
2535 rte_flow_error_set(error, res,
2536 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2538 "no counter found for flow");
2548 * @see rte_flow_isolate()
2552 mlx5_flow_isolate(struct rte_eth_dev *dev,
2554 struct rte_flow_error *error)
2556 struct priv *priv = dev->data->dev_private;
2559 if (dev->data->dev_started) {
2560 rte_flow_error_set(error, EBUSY,
2561 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2563 "port must be stopped first");
2567 priv->isolated = !!enable;
2569 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2571 priv->dev->dev_ops = &mlx5_dev_ops;
2577 * Convert a flow director filter to a generic flow.
2580 * Private structure.
2581 * @param fdir_filter
2582 * Flow director filter to add.
2584 * Generic flow parameters structure.
2587 * 0 on success, errno value on error.
2590 priv_fdir_filter_convert(struct priv *priv,
2591 const struct rte_eth_fdir_filter *fdir_filter,
2592 struct mlx5_fdir *attributes)
2594 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2596 /* Validate queue number. */
2597 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2598 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2601 attributes->attr.ingress = 1;
2602 attributes->items[0] = (struct rte_flow_item) {
2603 .type = RTE_FLOW_ITEM_TYPE_ETH,
2604 .spec = &attributes->l2,
2605 .mask = &attributes->l2_mask,
2607 switch (fdir_filter->action.behavior) {
2608 case RTE_ETH_FDIR_ACCEPT:
2609 attributes->actions[0] = (struct rte_flow_action){
2610 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2611 .conf = &attributes->queue,
2614 case RTE_ETH_FDIR_REJECT:
2615 attributes->actions[0] = (struct rte_flow_action){
2616 .type = RTE_FLOW_ACTION_TYPE_DROP,
2620 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2623 attributes->queue.index = fdir_filter->action.rx_queue;
2624 switch (fdir_filter->input.flow_type) {
2625 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2626 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2627 .src_addr = input->flow.udp4_flow.ip.src_ip,
2628 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2629 .time_to_live = input->flow.udp4_flow.ip.ttl,
2630 .type_of_service = input->flow.udp4_flow.ip.tos,
2631 .next_proto_id = input->flow.udp4_flow.ip.proto,
2633 attributes->l4.udp.hdr = (struct udp_hdr){
2634 .src_port = input->flow.udp4_flow.src_port,
2635 .dst_port = input->flow.udp4_flow.dst_port,
2637 attributes->items[1] = (struct rte_flow_item){
2638 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2639 .spec = &attributes->l3,
2640 .mask = &attributes->l3,
2642 attributes->items[2] = (struct rte_flow_item){
2643 .type = RTE_FLOW_ITEM_TYPE_UDP,
2644 .spec = &attributes->l4,
2645 .mask = &attributes->l4,
2648 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2649 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2650 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2651 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2652 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2653 .type_of_service = input->flow.tcp4_flow.ip.tos,
2654 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2656 attributes->l4.tcp.hdr = (struct tcp_hdr){
2657 .src_port = input->flow.tcp4_flow.src_port,
2658 .dst_port = input->flow.tcp4_flow.dst_port,
2660 attributes->items[1] = (struct rte_flow_item){
2661 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2662 .spec = &attributes->l3,
2663 .mask = &attributes->l3,
2665 attributes->items[2] = (struct rte_flow_item){
2666 .type = RTE_FLOW_ITEM_TYPE_TCP,
2667 .spec = &attributes->l4,
2668 .mask = &attributes->l4,
2671 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2672 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2673 .src_addr = input->flow.ip4_flow.src_ip,
2674 .dst_addr = input->flow.ip4_flow.dst_ip,
2675 .time_to_live = input->flow.ip4_flow.ttl,
2676 .type_of_service = input->flow.ip4_flow.tos,
2677 .next_proto_id = input->flow.ip4_flow.proto,
2679 attributes->items[1] = (struct rte_flow_item){
2680 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2681 .spec = &attributes->l3,
2682 .mask = &attributes->l3,
2685 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2686 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2687 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2688 .proto = input->flow.udp6_flow.ip.proto,
2690 memcpy(attributes->l3.ipv6.hdr.src_addr,
2691 input->flow.udp6_flow.ip.src_ip,
2692 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2693 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2694 input->flow.udp6_flow.ip.dst_ip,
2695 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2696 attributes->l4.udp.hdr = (struct udp_hdr){
2697 .src_port = input->flow.udp6_flow.src_port,
2698 .dst_port = input->flow.udp6_flow.dst_port,
2700 attributes->items[1] = (struct rte_flow_item){
2701 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2702 .spec = &attributes->l3,
2703 .mask = &attributes->l3,
2705 attributes->items[2] = (struct rte_flow_item){
2706 .type = RTE_FLOW_ITEM_TYPE_UDP,
2707 .spec = &attributes->l4,
2708 .mask = &attributes->l4,
2711 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2712 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2713 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2714 .proto = input->flow.tcp6_flow.ip.proto,
2716 memcpy(attributes->l3.ipv6.hdr.src_addr,
2717 input->flow.tcp6_flow.ip.src_ip,
2718 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2719 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2720 input->flow.tcp6_flow.ip.dst_ip,
2721 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2722 attributes->l4.tcp.hdr = (struct tcp_hdr){
2723 .src_port = input->flow.tcp6_flow.src_port,
2724 .dst_port = input->flow.tcp6_flow.dst_port,
2726 attributes->items[1] = (struct rte_flow_item){
2727 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2728 .spec = &attributes->l3,
2729 .mask = &attributes->l3,
2731 attributes->items[2] = (struct rte_flow_item){
2732 .type = RTE_FLOW_ITEM_TYPE_TCP,
2733 .spec = &attributes->l4,
2734 .mask = &attributes->l4,
2737 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2738 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2739 .hop_limits = input->flow.ipv6_flow.hop_limits,
2740 .proto = input->flow.ipv6_flow.proto,
2742 memcpy(attributes->l3.ipv6.hdr.src_addr,
2743 input->flow.ipv6_flow.src_ip,
2744 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2745 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2746 input->flow.ipv6_flow.dst_ip,
2747 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2748 attributes->items[1] = (struct rte_flow_item){
2749 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2750 .spec = &attributes->l3,
2751 .mask = &attributes->l3,
2755 ERROR("invalid flow type%d",
2756 fdir_filter->input.flow_type);
2763 * Add new flow director filter and store it in list.
2766 * Private structure.
2767 * @param fdir_filter
2768 * Flow director filter to add.
2771 * 0 on success, errno value on failure.
2774 priv_fdir_filter_add(struct priv *priv,
2775 const struct rte_eth_fdir_filter *fdir_filter)
2777 struct mlx5_fdir attributes = {
2780 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2781 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2785 struct mlx5_flow_parse parser = {
2786 .layer = HASH_RXQ_ETH,
2788 struct rte_flow_error error;
2789 struct rte_flow *flow;
2792 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2795 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2796 attributes.actions, &error, &parser);
2799 flow = priv_flow_create(priv,
2806 DEBUG("FDIR created %p", (void *)flow);
2813 * Delete specific filter.
2816 * Private structure.
2817 * @param fdir_filter
2818 * Filter to be deleted.
2821 * 0 on success, errno value on failure.
2824 priv_fdir_filter_delete(struct priv *priv,
2825 const struct rte_eth_fdir_filter *fdir_filter)
2827 struct mlx5_fdir attributes = {
2830 struct mlx5_flow_parse parser = {
2832 .layer = HASH_RXQ_ETH,
2834 struct rte_flow_error error;
2835 struct rte_flow *flow;
2839 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2842 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2843 attributes.actions, &error, &parser);
2847 * Special case for drop action which is only set in the
2848 * specifications when the flow is created. In this situation the
2849 * drop specification is missing.
2852 struct ibv_flow_spec_action_drop *drop;
2854 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
2855 parser.queue[HASH_RXQ_ETH].offset);
2856 *drop = (struct ibv_flow_spec_action_drop){
2857 .type = IBV_FLOW_SPEC_ACTION_DROP,
2858 .size = sizeof(struct ibv_flow_spec_action_drop),
2860 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
2862 TAILQ_FOREACH(flow, &priv->flows, next) {
2863 struct ibv_flow_attr *attr;
2864 struct ibv_spec_header *attr_h;
2866 struct ibv_flow_attr *flow_attr;
2867 struct ibv_spec_header *flow_h;
2869 unsigned int specs_n;
2871 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2872 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2873 /* Compare first the attributes. */
2874 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2876 if (attr->num_of_specs == 0)
2878 spec = (void *)((uintptr_t)attr +
2879 sizeof(struct ibv_flow_attr));
2880 flow_spec = (void *)((uintptr_t)flow_attr +
2881 sizeof(struct ibv_flow_attr));
2882 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2883 for (i = 0; i != specs_n; ++i) {
2886 if (memcmp(spec, flow_spec,
2887 RTE_MIN(attr_h->size, flow_h->size)))
2889 spec = (void *)((uintptr_t)spec + attr_h->size);
2890 flow_spec = (void *)((uintptr_t)flow_spec +
2893 /* At this point, the flow match. */
2896 /* The flow does not match. */
2900 priv_flow_destroy(priv, &priv->flows, flow);
2902 for (i = 0; i != hash_rxq_init_n; ++i) {
2903 if (parser.queue[i].ibv_attr)
2904 rte_free(parser.queue[i].ibv_attr);
2910 * Update queue for specific filter.
2913 * Private structure.
2914 * @param fdir_filter
2915 * Filter to be updated.
2918 * 0 on success, errno value on failure.
2921 priv_fdir_filter_update(struct priv *priv,
2922 const struct rte_eth_fdir_filter *fdir_filter)
2926 ret = priv_fdir_filter_delete(priv, fdir_filter);
2929 ret = priv_fdir_filter_add(priv, fdir_filter);
2934 * Flush all filters.
2937 * Private structure.
2940 priv_fdir_filter_flush(struct priv *priv)
2942 priv_flow_flush(priv, &priv->flows);
2946 * Get flow director information.
2949 * Private structure.
2950 * @param[out] fdir_info
2951 * Resulting flow director information.
2954 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2956 struct rte_eth_fdir_masks *mask =
2957 &priv->dev->data->dev_conf.fdir_conf.mask;
2959 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2960 fdir_info->guarant_spc = 0;
2961 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2962 fdir_info->max_flexpayload = 0;
2963 fdir_info->flow_types_mask[0] = 0;
2964 fdir_info->flex_payload_unit = 0;
2965 fdir_info->max_flex_payload_segment_num = 0;
2966 fdir_info->flex_payload_limit = 0;
2967 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2971 * Deal with flow director operations.
2974 * Pointer to private structure.
2976 * Operation to perform.
2978 * Pointer to operation-specific structure.
2981 * 0 on success, errno value on failure.
2984 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2986 enum rte_fdir_mode fdir_mode =
2987 priv->dev->data->dev_conf.fdir_conf.mode;
2990 if (filter_op == RTE_ETH_FILTER_NOP)
2992 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2993 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2994 ERROR("%p: flow director mode %d not supported",
2995 (void *)priv, fdir_mode);
2998 switch (filter_op) {
2999 case RTE_ETH_FILTER_ADD:
3000 ret = priv_fdir_filter_add(priv, arg);
3002 case RTE_ETH_FILTER_UPDATE:
3003 ret = priv_fdir_filter_update(priv, arg);
3005 case RTE_ETH_FILTER_DELETE:
3006 ret = priv_fdir_filter_delete(priv, arg);
3008 case RTE_ETH_FILTER_FLUSH:
3009 priv_fdir_filter_flush(priv);
3011 case RTE_ETH_FILTER_INFO:
3012 priv_fdir_info_get(priv, arg);
3015 DEBUG("%p: unknown operation %u", (void *)priv,
3024 * Manage filter operations.
3027 * Pointer to Ethernet device structure.
3028 * @param filter_type
3031 * Operation to perform.
3033 * Pointer to operation-specific structure.
3036 * 0 on success, negative errno value on failure.
3039 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3040 enum rte_filter_type filter_type,
3041 enum rte_filter_op filter_op,
3045 struct priv *priv = dev->data->dev_private;
3047 switch (filter_type) {
3048 case RTE_ETH_FILTER_GENERIC:
3049 if (filter_op != RTE_ETH_FILTER_GET)
3051 *(const void **)arg = &mlx5_flow_ops;
3053 case RTE_ETH_FILTER_FDIR:
3055 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3059 ERROR("%p: filter type (%d) not supported",
3060 (void *)dev, filter_type);