4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
63 struct ibv_counter_set_init_attr {
66 struct ibv_flow_spec_counter_action {
69 struct ibv_counter_set {
74 ibv_destroy_counter_set(struct ibv_counter_set *cs)
81 /* Dev ops structure defined in mlx5.c */
82 extern const struct eth_dev_ops mlx5_dev_ops;
83 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
86 mlx5_flow_create_eth(const struct rte_flow_item *item,
87 const void *default_mask,
91 mlx5_flow_create_vlan(const struct rte_flow_item *item,
92 const void *default_mask,
96 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
97 const void *default_mask,
101 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
102 const void *default_mask,
106 mlx5_flow_create_udp(const struct rte_flow_item *item,
107 const void *default_mask,
111 mlx5_flow_create_tcp(const struct rte_flow_item *item,
112 const void *default_mask,
116 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
117 const void *default_mask,
120 struct mlx5_flow_parse;
123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
132 /* Hash RX queue types. */
143 /* Initialization data for hash RX queue. */
144 struct hash_rxq_init {
145 uint64_t hash_fields; /* Fields that participate in the hash. */
146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
147 unsigned int flow_priority; /* Flow priority to use. */
148 unsigned int ip_version; /* Internet protocol. */
151 /* Initialization data for hash RX queues. */
152 const struct hash_rxq_init hash_rxq_init[] = {
154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
155 IBV_RX_HASH_DST_IPV4 |
156 IBV_RX_HASH_SRC_PORT_TCP |
157 IBV_RX_HASH_DST_PORT_TCP),
158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
160 .ip_version = MLX5_IPV4,
163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
164 IBV_RX_HASH_DST_IPV4 |
165 IBV_RX_HASH_SRC_PORT_UDP |
166 IBV_RX_HASH_DST_PORT_UDP),
167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
169 .ip_version = MLX5_IPV4,
172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
173 IBV_RX_HASH_DST_IPV4),
174 .dpdk_rss_hf = (ETH_RSS_IPV4 |
177 .ip_version = MLX5_IPV4,
180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
181 IBV_RX_HASH_DST_IPV6 |
182 IBV_RX_HASH_SRC_PORT_TCP |
183 IBV_RX_HASH_DST_PORT_TCP),
184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
186 .ip_version = MLX5_IPV6,
189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
190 IBV_RX_HASH_DST_IPV6 |
191 IBV_RX_HASH_SRC_PORT_UDP |
192 IBV_RX_HASH_DST_PORT_UDP),
193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
195 .ip_version = MLX5_IPV6,
198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
199 IBV_RX_HASH_DST_IPV6),
200 .dpdk_rss_hf = (ETH_RSS_IPV6 |
203 .ip_version = MLX5_IPV6,
212 /* Number of entries in hash_rxq_init[]. */
213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
215 /** Structure for holding counter stats. */
216 struct mlx5_flow_counter_stats {
217 uint64_t hits; /**< Number of packets matched by the rule. */
218 uint64_t bytes; /**< Number of bytes matched by the rule. */
221 /** Structure for Drop queue. */
222 struct mlx5_hrxq_drop {
223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
224 struct ibv_qp *qp; /**< Verbs queue pair. */
225 struct ibv_wq *wq; /**< Verbs work queue. */
226 struct ibv_cq *cq; /**< Verbs completion queue. */
229 /* Flows structures. */
231 uint64_t hash_fields; /**< Fields that participate in the hash. */
232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
233 struct ibv_flow *ibv_flow; /**< Verbs flow. */
234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
237 /* Drop flows structures. */
238 struct mlx5_flow_drop {
239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
240 struct ibv_flow *ibv_flow; /**< Verbs flow. */
244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
245 uint32_t mark:1; /**< Set if the flow is marked. */
246 uint32_t drop:1; /**< Drop queue. */
247 uint16_t queues_n; /**< Number of entries in queue[]. */
248 uint16_t (*queues)[]; /**< Queues indexes to use. */
249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
250 uint8_t rss_key[40]; /**< copy of the RSS key. */
251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
255 /**< Flow with Rx queue. */
256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */
260 /** Static initializer for items. */
262 (const enum rte_flow_item_type []){ \
263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
266 /** Structure to generate a simple graph of layers supported by the NIC. */
267 struct mlx5_flow_items {
268 /** List of possible actions for these items. */
269 const enum rte_flow_action_type *const actions;
270 /** Bit-masks corresponding to the possibilities for the item. */
273 * Default bit-masks to use when item->mask is not provided. When
274 * \default_mask is also NULL, the full supported bit-mask (\mask) is
277 const void *default_mask;
278 /** Bit-masks size in bytes. */
279 const unsigned int mask_sz;
281 * Conversion function from rte_flow to NIC specific flow.
284 * rte_flow item to convert.
285 * @param default_mask
286 * Default bit-masks to use when item->mask is not provided.
288 * Internal structure to store the conversion.
291 * 0 on success, negative value otherwise.
293 int (*convert)(const struct rte_flow_item *item,
294 const void *default_mask,
296 /** Size in bytes of the destination structure. */
297 const unsigned int dst_sz;
298 /** List of possible following items. */
299 const enum rte_flow_item_type *const items;
302 /** Valid action for this PMD. */
303 static const enum rte_flow_action_type valid_actions[] = {
304 RTE_FLOW_ACTION_TYPE_DROP,
305 RTE_FLOW_ACTION_TYPE_QUEUE,
306 RTE_FLOW_ACTION_TYPE_MARK,
307 RTE_FLOW_ACTION_TYPE_FLAG,
308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
309 RTE_FLOW_ACTION_TYPE_COUNT,
311 RTE_FLOW_ACTION_TYPE_END,
314 /** Graph of supported items and associated actions. */
315 static const struct mlx5_flow_items mlx5_flow_items[] = {
316 [RTE_FLOW_ITEM_TYPE_END] = {
317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
318 RTE_FLOW_ITEM_TYPE_VXLAN),
320 [RTE_FLOW_ITEM_TYPE_ETH] = {
321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
322 RTE_FLOW_ITEM_TYPE_IPV4,
323 RTE_FLOW_ITEM_TYPE_IPV6),
324 .actions = valid_actions,
325 .mask = &(const struct rte_flow_item_eth){
326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
330 .default_mask = &rte_flow_item_eth_mask,
331 .mask_sz = sizeof(struct rte_flow_item_eth),
332 .convert = mlx5_flow_create_eth,
333 .dst_sz = sizeof(struct ibv_flow_spec_eth),
335 [RTE_FLOW_ITEM_TYPE_VLAN] = {
336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
337 RTE_FLOW_ITEM_TYPE_IPV6),
338 .actions = valid_actions,
339 .mask = &(const struct rte_flow_item_vlan){
342 .default_mask = &rte_flow_item_vlan_mask,
343 .mask_sz = sizeof(struct rte_flow_item_vlan),
344 .convert = mlx5_flow_create_vlan,
347 [RTE_FLOW_ITEM_TYPE_IPV4] = {
348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
349 RTE_FLOW_ITEM_TYPE_TCP),
350 .actions = valid_actions,
351 .mask = &(const struct rte_flow_item_ipv4){
355 .type_of_service = -1,
359 .default_mask = &rte_flow_item_ipv4_mask,
360 .mask_sz = sizeof(struct rte_flow_item_ipv4),
361 .convert = mlx5_flow_create_ipv4,
362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
364 [RTE_FLOW_ITEM_TYPE_IPV6] = {
365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
366 RTE_FLOW_ITEM_TYPE_TCP),
367 .actions = valid_actions,
368 .mask = &(const struct rte_flow_item_ipv6){
371 0xff, 0xff, 0xff, 0xff,
372 0xff, 0xff, 0xff, 0xff,
373 0xff, 0xff, 0xff, 0xff,
374 0xff, 0xff, 0xff, 0xff,
377 0xff, 0xff, 0xff, 0xff,
378 0xff, 0xff, 0xff, 0xff,
379 0xff, 0xff, 0xff, 0xff,
380 0xff, 0xff, 0xff, 0xff,
387 .default_mask = &rte_flow_item_ipv6_mask,
388 .mask_sz = sizeof(struct rte_flow_item_ipv6),
389 .convert = mlx5_flow_create_ipv6,
390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
392 [RTE_FLOW_ITEM_TYPE_UDP] = {
393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
394 .actions = valid_actions,
395 .mask = &(const struct rte_flow_item_udp){
401 .default_mask = &rte_flow_item_udp_mask,
402 .mask_sz = sizeof(struct rte_flow_item_udp),
403 .convert = mlx5_flow_create_udp,
404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
406 [RTE_FLOW_ITEM_TYPE_TCP] = {
407 .actions = valid_actions,
408 .mask = &(const struct rte_flow_item_tcp){
414 .default_mask = &rte_flow_item_tcp_mask,
415 .mask_sz = sizeof(struct rte_flow_item_tcp),
416 .convert = mlx5_flow_create_tcp,
417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
419 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
421 .actions = valid_actions,
422 .mask = &(const struct rte_flow_item_vxlan){
423 .vni = "\xff\xff\xff",
425 .default_mask = &rte_flow_item_vxlan_mask,
426 .mask_sz = sizeof(struct rte_flow_item_vxlan),
427 .convert = mlx5_flow_create_vxlan,
428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
432 /** Structure to pass to the conversion function. */
433 struct mlx5_flow_parse {
434 uint32_t inner; /**< Set once VXLAN is encountered. */
436 /**< Whether resources should remain after a validate. */
437 uint32_t drop:1; /**< Target is a drop queue. */
438 uint32_t mark:1; /**< Mark is present in the flow. */
439 uint32_t count:1; /**< Count is present in the flow. */
440 uint32_t mark_id; /**< Mark identifier. */
441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
442 uint16_t queues_n; /**< Number of entries in queue[]. */
443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
444 uint8_t rss_key[40]; /**< copy of the RSS key. */
445 enum hash_rxq_type layer; /**< Last pattern layer detected. */
446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
449 struct ibv_flow_attr *ibv_attr;
450 /**< Pointer to Verbs attributes. */
452 /**< Current position or total size of the attribute. */
453 } queue[RTE_DIM(hash_rxq_init)];
455 struct ibv_flow_attr *ibv_attr;
456 /**< Pointer to Verbs attributes. */
458 /**< Current position or total size of the attribute. */
463 static const struct rte_flow_ops mlx5_flow_ops = {
464 .validate = mlx5_flow_validate,
465 .create = mlx5_flow_create,
466 .destroy = mlx5_flow_destroy,
467 .flush = mlx5_flow_flush,
468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
469 .query = mlx5_flow_query,
473 .isolate = mlx5_flow_isolate,
476 /* Convert FDIR request to Generic flow. */
478 struct rte_flow_attr attr;
479 struct rte_flow_action actions[2];
480 struct rte_flow_item items[4];
481 struct rte_flow_item_eth l2;
483 struct rte_flow_item_ipv4 ipv4;
484 struct rte_flow_item_ipv6 ipv6;
487 struct rte_flow_item_udp udp;
488 struct rte_flow_item_tcp tcp;
490 struct rte_flow_action_queue queue;
493 /* Verbs specification header. */
494 struct ibv_spec_header {
495 enum ibv_flow_spec_type type;
500 * Check support for a given item.
503 * Item specification.
505 * Bit-masks covering supported fields to compare with spec, last and mask in
508 * Bit-Mask size in bytes.
514 mlx5_flow_item_validate(const struct rte_flow_item *item,
515 const uint8_t *mask, unsigned int size)
519 if (!item->spec && (item->mask || item->last))
521 if (item->spec && !item->mask) {
523 const uint8_t *spec = item->spec;
525 for (i = 0; i < size; ++i)
526 if ((spec[i] | mask[i]) != mask[i])
529 if (item->last && !item->mask) {
531 const uint8_t *spec = item->last;
533 for (i = 0; i < size; ++i)
534 if ((spec[i] | mask[i]) != mask[i])
539 const uint8_t *spec = item->mask;
541 for (i = 0; i < size; ++i)
542 if ((spec[i] | mask[i]) != mask[i])
545 if (item->spec && item->last) {
548 const uint8_t *apply = mask;
553 for (i = 0; i < size; ++i) {
554 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
555 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
557 ret = memcmp(spec, last, size);
563 * Copy the RSS configuration from the user ones.
566 * Pointer to private structure.
568 * Internal parser structure.
570 * User RSS configuration to save.
573 * 0 on success, errno value on failure.
576 priv_flow_convert_rss_conf(struct priv *priv,
577 struct mlx5_flow_parse *parser,
578 const struct rte_eth_rss_conf *rss_conf)
580 const struct rte_eth_rss_conf *rss =
581 rss_conf ? rss_conf : &priv->rss_conf;
583 if (rss->rss_key_len > 40)
585 parser->rss_conf.rss_key_len = rss->rss_key_len;
586 parser->rss_conf.rss_hf = rss->rss_hf;
587 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
588 parser->rss_conf.rss_key = parser->rss_key;
593 * Extract attribute to the parser.
596 * Pointer to private structure.
598 * Flow rule attributes.
600 * Perform verbose error reporting if not NULL.
601 * @param[in, out] parser
602 * Internal parser structure.
605 * 0 on success, a negative errno value otherwise and rte_errno is set.
608 priv_flow_convert_attributes(struct priv *priv,
609 const struct rte_flow_attr *attr,
610 struct rte_flow_error *error,
611 struct mlx5_flow_parse *parser)
616 rte_flow_error_set(error, ENOTSUP,
617 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
619 "groups are not supported");
622 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
623 rte_flow_error_set(error, ENOTSUP,
624 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
626 "priorities are not supported");
630 rte_flow_error_set(error, ENOTSUP,
631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
633 "egress is not supported");
636 if (!attr->ingress) {
637 rte_flow_error_set(error, ENOTSUP,
638 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
640 "only ingress is supported");
647 * Extract actions request to the parser.
650 * Pointer to private structure.
652 * Associated actions (list terminated by the END action).
654 * Perform verbose error reporting if not NULL.
655 * @param[in, out] parser
656 * Internal parser structure.
659 * 0 on success, a negative errno value otherwise and rte_errno is set.
662 priv_flow_convert_actions(struct priv *priv,
663 const struct rte_flow_action actions[],
664 struct rte_flow_error *error,
665 struct mlx5_flow_parse *parser)
668 * Add default RSS configuration necessary for Verbs to create QP even
669 * if no RSS is necessary.
671 priv_flow_convert_rss_conf(priv, parser,
672 (const struct rte_eth_rss_conf *)
674 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
675 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
677 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
679 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
680 const struct rte_flow_action_queue *queue =
681 (const struct rte_flow_action_queue *)
686 if (!queue || (queue->index > (priv->rxqs_n - 1)))
687 goto exit_action_not_supported;
688 for (n = 0; n < parser->queues_n; ++n) {
689 if (parser->queues[n] == queue->index) {
694 if (parser->queues_n > 1 && !found) {
695 rte_flow_error_set(error, ENOTSUP,
696 RTE_FLOW_ERROR_TYPE_ACTION,
698 "queue action not in RSS queues");
702 parser->queues_n = 1;
703 parser->queues[0] = queue->index;
705 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
706 const struct rte_flow_action_rss *rss =
707 (const struct rte_flow_action_rss *)
711 if (!rss || !rss->num) {
712 rte_flow_error_set(error, EINVAL,
713 RTE_FLOW_ERROR_TYPE_ACTION,
718 if (parser->queues_n == 1) {
721 assert(parser->queues_n);
722 for (n = 0; n < rss->num; ++n) {
723 if (parser->queues[0] ==
730 rte_flow_error_set(error, ENOTSUP,
731 RTE_FLOW_ERROR_TYPE_ACTION,
733 "queue action not in RSS"
738 for (n = 0; n < rss->num; ++n) {
739 if (rss->queue[n] >= priv->rxqs_n) {
740 rte_flow_error_set(error, EINVAL,
741 RTE_FLOW_ERROR_TYPE_ACTION,
743 "queue id > number of"
748 for (n = 0; n < rss->num; ++n)
749 parser->queues[n] = rss->queue[n];
750 parser->queues_n = rss->num;
751 if (priv_flow_convert_rss_conf(priv, parser,
753 rte_flow_error_set(error, EINVAL,
754 RTE_FLOW_ERROR_TYPE_ACTION,
756 "wrong RSS configuration");
759 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
760 const struct rte_flow_action_mark *mark =
761 (const struct rte_flow_action_mark *)
765 rte_flow_error_set(error, EINVAL,
766 RTE_FLOW_ERROR_TYPE_ACTION,
768 "mark must be defined");
770 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
771 rte_flow_error_set(error, ENOTSUP,
772 RTE_FLOW_ERROR_TYPE_ACTION,
774 "mark must be between 0"
779 parser->mark_id = mark->id;
780 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
782 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
783 priv->counter_set_supported) {
786 goto exit_action_not_supported;
789 if (parser->drop && parser->mark)
791 if (!parser->queues_n && !parser->drop) {
792 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
793 NULL, "no valid action");
797 exit_action_not_supported:
798 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
799 actions, "action not supported");
807 * Pointer to private structure.
809 * Pattern specification (list terminated by the END pattern item).
811 * Perform verbose error reporting if not NULL.
812 * @param[in, out] parser
813 * Internal parser structure.
816 * 0 on success, a negative errno value otherwise and rte_errno is set.
819 priv_flow_convert_items_validate(struct priv *priv,
820 const struct rte_flow_item items[],
821 struct rte_flow_error *error,
822 struct mlx5_flow_parse *parser)
824 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
828 /* Initialise the offsets to start after verbs attribute. */
830 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
832 for (i = 0; i != hash_rxq_init_n; ++i)
833 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
835 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
836 const struct mlx5_flow_items *token = NULL;
840 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
844 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
846 if (cur_item->items[i] == items->type) {
847 token = &mlx5_flow_items[items->type];
852 goto exit_item_not_supported;
854 err = mlx5_flow_item_validate(items,
855 (const uint8_t *)cur_item->mask,
858 goto exit_item_not_supported;
859 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
861 rte_flow_error_set(error, ENOTSUP,
862 RTE_FLOW_ERROR_TYPE_ITEM,
864 "cannot recognize multiple"
865 " VXLAN encapsulations");
871 parser->drop_q.offset += cur_item->dst_sz;
872 } else if (parser->queues_n == 1) {
873 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
875 for (n = 0; n != hash_rxq_init_n; ++n)
876 parser->queue[n].offset += cur_item->dst_sz;
880 for (i = 0; i != hash_rxq_init_n; ++i)
881 parser->queue[i].offset +=
882 sizeof(struct ibv_flow_spec_action_tag);
885 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
888 parser->drop_q.offset += size;
890 for (i = 0; i != hash_rxq_init_n; ++i)
891 parser->queue[i].offset += size;
895 exit_item_not_supported:
896 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
897 items, "item not supported");
902 * Allocate memory space to store verbs flow attributes.
905 * Pointer to private structure.
906 * @param[in] priority
909 * Amount of byte to allocate.
911 * Perform verbose error reporting if not NULL.
914 * A verbs flow attribute on success, NULL otherwise.
916 static struct ibv_flow_attr*
917 priv_flow_convert_allocate(struct priv *priv,
918 unsigned int priority,
920 struct rte_flow_error *error)
922 struct ibv_flow_attr *ibv_attr;
925 ibv_attr = rte_calloc(__func__, 1, size, 0);
927 rte_flow_error_set(error, ENOMEM,
928 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
930 "cannot allocate verbs spec attributes.");
933 ibv_attr->priority = priority;
938 * Finalise verbs flow attributes.
941 * Pointer to private structure.
942 * @param[in, out] parser
943 * Internal parser structure.
946 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
948 const unsigned int ipv4 =
949 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
950 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
951 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
952 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
953 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
954 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
958 if (parser->layer == HASH_RXQ_ETH) {
962 * This layer becomes useless as the pattern define under
965 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
966 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
968 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
969 for (i = ohmin; i != (ohmax + 1); ++i) {
970 if (!parser->queue[i].ibv_attr)
972 rte_free(parser->queue[i].ibv_attr);
973 parser->queue[i].ibv_attr = NULL;
975 /* Remove impossible flow according to the RSS configuration. */
976 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
977 parser->rss_conf.rss_hf) {
978 /* Remove any other flow. */
979 for (i = hmin; i != (hmax + 1); ++i) {
980 if ((i == parser->layer) ||
981 (!parser->queue[i].ibv_attr))
983 rte_free(parser->queue[i].ibv_attr);
984 parser->queue[i].ibv_attr = NULL;
986 } else if (!parser->queue[ip].ibv_attr) {
987 /* no RSS possible with the current configuration. */
988 parser->queues_n = 1;
993 * Fill missing layers in verbs specifications, or compute the correct
994 * offset to allocate the memory space for the attributes and
997 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
999 struct ibv_flow_spec_ipv4_ext ipv4;
1000 struct ibv_flow_spec_ipv6 ipv6;
1001 struct ibv_flow_spec_tcp_udp udp_tcp;
1006 if (i == parser->layer)
1008 if (parser->layer == HASH_RXQ_ETH) {
1009 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
1010 size = sizeof(struct ibv_flow_spec_ipv4_ext);
1011 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
1012 .type = IBV_FLOW_SPEC_IPV4_EXT |
1017 size = sizeof(struct ibv_flow_spec_ipv6);
1018 specs.ipv6 = (struct ibv_flow_spec_ipv6){
1019 .type = IBV_FLOW_SPEC_IPV6 |
1024 if (parser->queue[i].ibv_attr) {
1025 dst = (void *)((uintptr_t)
1026 parser->queue[i].ibv_attr +
1027 parser->queue[i].offset);
1028 memcpy(dst, &specs, size);
1029 ++parser->queue[i].ibv_attr->num_of_specs;
1031 parser->queue[i].offset += size;
1033 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1034 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1035 size = sizeof(struct ibv_flow_spec_tcp_udp);
1036 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1037 .type = ((i == HASH_RXQ_UDPV4 ||
1038 i == HASH_RXQ_UDPV6) ?
1040 IBV_FLOW_SPEC_TCP) |
1044 if (parser->queue[i].ibv_attr) {
1045 dst = (void *)((uintptr_t)
1046 parser->queue[i].ibv_attr +
1047 parser->queue[i].offset);
1048 memcpy(dst, &specs, size);
1049 ++parser->queue[i].ibv_attr->num_of_specs;
1051 parser->queue[i].offset += size;
1057 * Validate and convert a flow supported by the NIC.
1060 * Pointer to private structure.
1062 * Flow rule attributes.
1063 * @param[in] pattern
1064 * Pattern specification (list terminated by the END pattern item).
1065 * @param[in] actions
1066 * Associated actions (list terminated by the END action).
1068 * Perform verbose error reporting if not NULL.
1069 * @param[in, out] parser
1070 * Internal parser structure.
1073 * 0 on success, a negative errno value otherwise and rte_errno is set.
1076 priv_flow_convert(struct priv *priv,
1077 const struct rte_flow_attr *attr,
1078 const struct rte_flow_item items[],
1079 const struct rte_flow_action actions[],
1080 struct rte_flow_error *error,
1081 struct mlx5_flow_parse *parser)
1083 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1087 /* First step. Validate the attributes, items and actions. */
1088 *parser = (struct mlx5_flow_parse){
1089 .create = parser->create,
1090 .layer = HASH_RXQ_ETH,
1091 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1093 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1096 ret = priv_flow_convert_actions(priv, actions, error, parser);
1099 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1102 priv_flow_convert_finalise(priv, parser);
1105 * Allocate the memory space to store verbs specifications.
1108 parser->drop_q.ibv_attr =
1109 priv_flow_convert_allocate(priv, attr->priority,
1110 parser->drop_q.offset,
1112 if (!parser->drop_q.ibv_attr)
1114 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
1115 } else if (parser->queues_n == 1) {
1116 unsigned int priority =
1118 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1119 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1121 parser->queue[HASH_RXQ_ETH].ibv_attr =
1122 priv_flow_convert_allocate(priv, priority,
1124 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1126 parser->queue[HASH_RXQ_ETH].offset =
1127 sizeof(struct ibv_flow_attr);
1129 for (i = 0; i != hash_rxq_init_n; ++i) {
1130 unsigned int priority =
1132 hash_rxq_init[i].flow_priority;
1133 unsigned int offset;
1135 if (!(parser->rss_conf.rss_hf &
1136 hash_rxq_init[i].dpdk_rss_hf) &&
1137 (i != HASH_RXQ_ETH))
1139 offset = parser->queue[i].offset;
1140 parser->queue[i].ibv_attr =
1141 priv_flow_convert_allocate(priv, priority,
1143 if (!parser->queue[i].ibv_attr)
1145 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1148 /* Third step. Conversion parse, fill the specifications. */
1150 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1151 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1153 cur_item = &mlx5_flow_items[items->type];
1154 ret = cur_item->convert(items,
1155 (cur_item->default_mask ?
1156 cur_item->default_mask :
1160 rte_flow_error_set(error, ENOTSUP,
1161 RTE_FLOW_ERROR_TYPE_ITEM,
1162 items, "item not supported");
1167 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1168 if (parser->count && parser->create) {
1169 mlx5_flow_create_count(priv, parser);
1171 goto exit_count_error;
1174 * Last step. Complete missing specification to reach the RSS
1179 * Drop queue priority needs to be adjusted to
1180 * their most specific layer priority.
1182 parser->drop_q.ibv_attr->priority =
1184 hash_rxq_init[parser->layer].flow_priority;
1185 } else if (parser->queues_n > 1) {
1186 priv_flow_convert_finalise(priv, parser);
1189 * Action queue have their priority overridden with
1190 * Ethernet priority, this priority needs to be adjusted to
1191 * their most specific layer priority.
1193 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1195 hash_rxq_init[parser->layer].flow_priority;
1198 /* Only verification is expected, all resources should be released. */
1199 if (!parser->create) {
1201 rte_free(parser->drop_q.ibv_attr);
1202 parser->drop_q.ibv_attr = NULL;
1204 for (i = 0; i != hash_rxq_init_n; ++i) {
1205 if (parser->queue[i].ibv_attr) {
1206 rte_free(parser->queue[i].ibv_attr);
1207 parser->queue[i].ibv_attr = NULL;
1213 for (i = 0; i != hash_rxq_init_n; ++i) {
1214 if (parser->queue[i].ibv_attr) {
1215 rte_free(parser->queue[i].ibv_attr);
1216 parser->queue[i].ibv_attr = NULL;
1219 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1220 NULL, "cannot allocate verbs spec attributes.");
1223 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1224 NULL, "cannot create counter.");
1229 * Copy the specification created into the flow.
1232 * Internal parser structure.
1234 * Create specification.
1236 * Size in bytes of the specification to copy.
1239 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1246 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1247 parser->drop_q.offset);
1248 memcpy(dst, src, size);
1249 ++parser->drop_q.ibv_attr->num_of_specs;
1250 parser->drop_q.offset += size;
1253 for (i = 0; i != hash_rxq_init_n; ++i) {
1254 if (!parser->queue[i].ibv_attr)
1256 /* Specification must be the same l3 type or none. */
1257 if (parser->layer == HASH_RXQ_ETH ||
1258 (hash_rxq_init[parser->layer].ip_version ==
1259 hash_rxq_init[i].ip_version) ||
1260 (hash_rxq_init[i].ip_version == 0)) {
1261 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1262 parser->queue[i].offset);
1263 memcpy(dst, src, size);
1264 ++parser->queue[i].ibv_attr->num_of_specs;
1265 parser->queue[i].offset += size;
1271 * Convert Ethernet item to Verbs specification.
1274 * Item specification.
1275 * @param default_mask[in]
1276 * Default bit-masks to use when item->mask is not provided.
1277 * @param data[in, out]
1281 mlx5_flow_create_eth(const struct rte_flow_item *item,
1282 const void *default_mask,
1285 const struct rte_flow_item_eth *spec = item->spec;
1286 const struct rte_flow_item_eth *mask = item->mask;
1287 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1288 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1289 struct ibv_flow_spec_eth eth = {
1290 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1294 parser->layer = HASH_RXQ_ETH;
1299 mask = default_mask;
1300 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1301 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1302 eth.val.ether_type = spec->type;
1303 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1304 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1305 eth.mask.ether_type = mask->type;
1306 /* Remove unwanted bits from values. */
1307 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1308 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1309 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1311 eth.val.ether_type &= eth.mask.ether_type;
1313 mlx5_flow_create_copy(parser, ð, eth_size);
1318 * Convert VLAN item to Verbs specification.
1321 * Item specification.
1322 * @param default_mask[in]
1323 * Default bit-masks to use when item->mask is not provided.
1324 * @param data[in, out]
1328 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1329 const void *default_mask,
1332 const struct rte_flow_item_vlan *spec = item->spec;
1333 const struct rte_flow_item_vlan *mask = item->mask;
1334 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1335 struct ibv_flow_spec_eth *eth;
1336 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1341 mask = default_mask;
1344 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1345 parser->drop_q.offset - eth_size);
1346 eth->val.vlan_tag = spec->tci;
1347 eth->mask.vlan_tag = mask->tci;
1348 eth->val.vlan_tag &= eth->mask.vlan_tag;
1351 for (i = 0; i != hash_rxq_init_n; ++i) {
1352 if (!parser->queue[i].ibv_attr)
1355 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1356 parser->queue[i].offset - eth_size);
1357 eth->val.vlan_tag = spec->tci;
1358 eth->mask.vlan_tag = mask->tci;
1359 eth->val.vlan_tag &= eth->mask.vlan_tag;
1366 * Convert IPv4 item to Verbs specification.
1369 * Item specification.
1370 * @param default_mask[in]
1371 * Default bit-masks to use when item->mask is not provided.
1372 * @param data[in, out]
1376 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1377 const void *default_mask,
1380 const struct rte_flow_item_ipv4 *spec = item->spec;
1381 const struct rte_flow_item_ipv4 *mask = item->mask;
1382 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1383 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1384 struct ibv_flow_spec_ipv4_ext ipv4 = {
1385 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1389 parser->layer = HASH_RXQ_IPV4;
1392 mask = default_mask;
1393 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1394 .src_ip = spec->hdr.src_addr,
1395 .dst_ip = spec->hdr.dst_addr,
1396 .proto = spec->hdr.next_proto_id,
1397 .tos = spec->hdr.type_of_service,
1399 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1400 .src_ip = mask->hdr.src_addr,
1401 .dst_ip = mask->hdr.dst_addr,
1402 .proto = mask->hdr.next_proto_id,
1403 .tos = mask->hdr.type_of_service,
1405 /* Remove unwanted bits from values. */
1406 ipv4.val.src_ip &= ipv4.mask.src_ip;
1407 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1408 ipv4.val.proto &= ipv4.mask.proto;
1409 ipv4.val.tos &= ipv4.mask.tos;
1411 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1416 * Convert IPv6 item to Verbs specification.
1419 * Item specification.
1420 * @param default_mask[in]
1421 * Default bit-masks to use when item->mask is not provided.
1422 * @param data[in, out]
1426 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1427 const void *default_mask,
1430 const struct rte_flow_item_ipv6 *spec = item->spec;
1431 const struct rte_flow_item_ipv6 *mask = item->mask;
1432 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1433 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1434 struct ibv_flow_spec_ipv6 ipv6 = {
1435 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1439 parser->layer = HASH_RXQ_IPV6;
1444 mask = default_mask;
1445 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1446 RTE_DIM(ipv6.val.src_ip));
1447 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1448 RTE_DIM(ipv6.val.dst_ip));
1449 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1450 RTE_DIM(ipv6.mask.src_ip));
1451 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1452 RTE_DIM(ipv6.mask.dst_ip));
1453 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1454 ipv6.mask.next_hdr = mask->hdr.proto;
1455 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1456 /* Remove unwanted bits from values. */
1457 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1458 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1459 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1461 ipv6.val.flow_label &= ipv6.mask.flow_label;
1462 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1463 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1465 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1470 * Convert UDP item to Verbs specification.
1473 * Item specification.
1474 * @param default_mask[in]
1475 * Default bit-masks to use when item->mask is not provided.
1476 * @param data[in, out]
1480 mlx5_flow_create_udp(const struct rte_flow_item *item,
1481 const void *default_mask,
1484 const struct rte_flow_item_udp *spec = item->spec;
1485 const struct rte_flow_item_udp *mask = item->mask;
1486 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1487 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1488 struct ibv_flow_spec_tcp_udp udp = {
1489 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1493 if (parser->layer == HASH_RXQ_IPV4)
1494 parser->layer = HASH_RXQ_UDPV4;
1496 parser->layer = HASH_RXQ_UDPV6;
1499 mask = default_mask;
1500 udp.val.dst_port = spec->hdr.dst_port;
1501 udp.val.src_port = spec->hdr.src_port;
1502 udp.mask.dst_port = mask->hdr.dst_port;
1503 udp.mask.src_port = mask->hdr.src_port;
1504 /* Remove unwanted bits from values. */
1505 udp.val.src_port &= udp.mask.src_port;
1506 udp.val.dst_port &= udp.mask.dst_port;
1508 mlx5_flow_create_copy(parser, &udp, udp_size);
1513 * Convert TCP item to Verbs specification.
1516 * Item specification.
1517 * @param default_mask[in]
1518 * Default bit-masks to use when item->mask is not provided.
1519 * @param data[in, out]
1523 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1524 const void *default_mask,
1527 const struct rte_flow_item_tcp *spec = item->spec;
1528 const struct rte_flow_item_tcp *mask = item->mask;
1529 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1530 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1531 struct ibv_flow_spec_tcp_udp tcp = {
1532 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1536 if (parser->layer == HASH_RXQ_IPV4)
1537 parser->layer = HASH_RXQ_TCPV4;
1539 parser->layer = HASH_RXQ_TCPV6;
1542 mask = default_mask;
1543 tcp.val.dst_port = spec->hdr.dst_port;
1544 tcp.val.src_port = spec->hdr.src_port;
1545 tcp.mask.dst_port = mask->hdr.dst_port;
1546 tcp.mask.src_port = mask->hdr.src_port;
1547 /* Remove unwanted bits from values. */
1548 tcp.val.src_port &= tcp.mask.src_port;
1549 tcp.val.dst_port &= tcp.mask.dst_port;
1551 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1556 * Convert VXLAN item to Verbs specification.
1559 * Item specification.
1560 * @param default_mask[in]
1561 * Default bit-masks to use when item->mask is not provided.
1562 * @param data[in, out]
1566 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1567 const void *default_mask,
1570 const struct rte_flow_item_vxlan *spec = item->spec;
1571 const struct rte_flow_item_vxlan *mask = item->mask;
1572 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1573 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1574 struct ibv_flow_spec_tunnel vxlan = {
1575 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1584 parser->inner = IBV_FLOW_SPEC_INNER;
1587 mask = default_mask;
1588 memcpy(&id.vni[1], spec->vni, 3);
1589 vxlan.val.tunnel_id = id.vlan_id;
1590 memcpy(&id.vni[1], mask->vni, 3);
1591 vxlan.mask.tunnel_id = id.vlan_id;
1592 /* Remove unwanted bits from values. */
1593 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1595 mlx5_flow_create_copy(parser, &vxlan, size);
1600 * Convert mark/flag action to Verbs specification.
1603 * Internal parser structure.
1608 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1610 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1611 struct ibv_flow_spec_action_tag tag = {
1612 .type = IBV_FLOW_SPEC_ACTION_TAG,
1614 .tag_id = mlx5_flow_mark_set(mark_id),
1617 assert(parser->mark);
1618 mlx5_flow_create_copy(parser, &tag, size);
1623 * Convert count action to Verbs specification.
1626 * Pointer to private structure.
1628 * Pointer to MLX5 flow parser structure.
1631 * 0 on success, errno value on failure.
1634 mlx5_flow_create_count(struct priv *priv __rte_unused,
1635 struct mlx5_flow_parse *parser __rte_unused)
1637 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1638 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1639 struct ibv_counter_set_init_attr init_attr = {0};
1640 struct ibv_flow_spec_counter_action counter = {
1641 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1643 .counter_set_handle = 0,
1646 init_attr.counter_set_id = 0;
1647 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr);
1650 counter.counter_set_handle = parser->cs->handle;
1651 mlx5_flow_create_copy(parser, &counter, size);
1657 * Complete flow rule creation with a drop queue.
1660 * Pointer to private structure.
1662 * Internal parser structure.
1664 * Pointer to the rte_flow.
1666 * Perform verbose error reporting if not NULL.
1669 * 0 on success, errno value on failure.
1672 priv_flow_create_action_queue_drop(struct priv *priv,
1673 struct mlx5_flow_parse *parser,
1674 struct rte_flow *flow,
1675 struct rte_flow_error *error)
1677 struct ibv_flow_spec_action_drop *drop;
1678 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1684 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1685 parser->drop_q.offset);
1686 *drop = (struct ibv_flow_spec_action_drop){
1687 .type = IBV_FLOW_SPEC_ACTION_DROP,
1690 ++parser->drop_q.ibv_attr->num_of_specs;
1691 parser->drop_q.offset += size;
1692 flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
1693 if (!priv->dev->data->dev_started)
1695 parser->drop_q.ibv_attr = NULL;
1696 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
1697 flow->drxq.ibv_attr);
1699 flow->cs = parser->cs;
1700 if (!flow->drxq.ibv_flow) {
1701 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1702 NULL, "flow rule creation failure");
1709 if (flow->drxq.ibv_flow) {
1710 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1711 flow->drxq.ibv_flow = NULL;
1713 if (flow->drxq.ibv_attr) {
1714 rte_free(flow->drxq.ibv_attr);
1715 flow->drxq.ibv_attr = NULL;
1718 claim_zero(ibv_destroy_counter_set(flow->cs));
1726 * Create hash Rx queues when RSS is enabled.
1729 * Pointer to private structure.
1731 * Internal parser structure.
1733 * Pointer to the rte_flow.
1735 * Perform verbose error reporting if not NULL.
1738 * 0 on success, a errno value otherwise and rte_errno is set.
1741 priv_flow_create_action_queue_rss(struct priv *priv,
1742 struct mlx5_flow_parse *parser,
1743 struct rte_flow *flow,
1744 struct rte_flow_error *error)
1748 for (i = 0; i != hash_rxq_init_n; ++i) {
1749 uint64_t hash_fields;
1751 if (!parser->queue[i].ibv_attr)
1753 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1754 parser->queue[i].ibv_attr = NULL;
1755 hash_fields = hash_rxq_init[i].hash_fields;
1756 if (!priv->dev->data->dev_started)
1758 flow->frxq[i].hrxq =
1759 mlx5_priv_hrxq_get(priv,
1760 parser->rss_conf.rss_key,
1761 parser->rss_conf.rss_key_len,
1764 hash_fields ? parser->queues_n : 1);
1765 if (flow->frxq[i].hrxq)
1767 flow->frxq[i].hrxq =
1768 mlx5_priv_hrxq_new(priv,
1769 parser->rss_conf.rss_key,
1770 parser->rss_conf.rss_key_len,
1773 hash_fields ? parser->queues_n : 1);
1774 if (!flow->frxq[i].hrxq) {
1775 rte_flow_error_set(error, ENOMEM,
1776 RTE_FLOW_ERROR_TYPE_HANDLE,
1777 NULL, "cannot create hash rxq");
1785 * Complete flow rule creation.
1788 * Pointer to private structure.
1790 * Internal parser structure.
1792 * Pointer to the rte_flow.
1794 * Perform verbose error reporting if not NULL.
1797 * 0 on success, a errno value otherwise and rte_errno is set.
1800 priv_flow_create_action_queue(struct priv *priv,
1801 struct mlx5_flow_parse *parser,
1802 struct rte_flow *flow,
1803 struct rte_flow_error *error)
1810 assert(!parser->drop);
1811 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1815 flow->cs = parser->cs;
1816 if (!priv->dev->data->dev_started)
1818 for (i = 0; i != hash_rxq_init_n; ++i) {
1819 if (!flow->frxq[i].hrxq)
1821 flow->frxq[i].ibv_flow =
1822 ibv_create_flow(flow->frxq[i].hrxq->qp,
1823 flow->frxq[i].ibv_attr);
1824 if (!flow->frxq[i].ibv_flow) {
1825 rte_flow_error_set(error, ENOMEM,
1826 RTE_FLOW_ERROR_TYPE_HANDLE,
1827 NULL, "flow rule creation failure");
1831 DEBUG("%p type %d QP %p ibv_flow %p",
1833 (void *)flow->frxq[i].hrxq,
1834 (void *)flow->frxq[i].ibv_flow);
1836 for (i = 0; i != parser->queues_n; ++i) {
1837 struct mlx5_rxq_data *q =
1838 (*priv->rxqs)[parser->queues[i]];
1840 q->mark |= parser->mark;
1845 for (i = 0; i != hash_rxq_init_n; ++i) {
1846 if (flow->frxq[i].ibv_flow) {
1847 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1849 claim_zero(ibv_destroy_flow(ibv_flow));
1851 if (flow->frxq[i].hrxq)
1852 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1853 if (flow->frxq[i].ibv_attr)
1854 rte_free(flow->frxq[i].ibv_attr);
1857 claim_zero(ibv_destroy_counter_set(flow->cs));
1868 * Pointer to private structure.
1870 * Pointer to a TAILQ flow list.
1872 * Flow rule attributes.
1873 * @param[in] pattern
1874 * Pattern specification (list terminated by the END pattern item).
1875 * @param[in] actions
1876 * Associated actions (list terminated by the END action).
1878 * Perform verbose error reporting if not NULL.
1881 * A flow on success, NULL otherwise.
1883 static struct rte_flow *
1884 priv_flow_create(struct priv *priv,
1885 struct mlx5_flows *list,
1886 const struct rte_flow_attr *attr,
1887 const struct rte_flow_item items[],
1888 const struct rte_flow_action actions[],
1889 struct rte_flow_error *error)
1891 struct mlx5_flow_parse parser = { .create = 1, };
1892 struct rte_flow *flow = NULL;
1896 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1899 flow = rte_calloc(__func__, 1,
1900 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1903 rte_flow_error_set(error, ENOMEM,
1904 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1906 "cannot allocate flow memory");
1909 /* Copy queues configuration. */
1910 flow->queues = (uint16_t (*)[])(flow + 1);
1911 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1912 flow->queues_n = parser.queues_n;
1913 /* Copy RSS configuration. */
1914 flow->rss_conf = parser.rss_conf;
1915 flow->rss_conf.rss_key = flow->rss_key;
1916 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1917 /* finalise the flow. */
1919 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1922 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1925 TAILQ_INSERT_TAIL(list, flow, next);
1926 DEBUG("Flow created %p", (void *)flow);
1930 rte_free(parser.drop_q.ibv_attr);
1932 for (i = 0; i != hash_rxq_init_n; ++i) {
1933 if (parser.queue[i].ibv_attr)
1934 rte_free(parser.queue[i].ibv_attr);
1942 * Validate a flow supported by the NIC.
1944 * @see rte_flow_validate()
1948 mlx5_flow_validate(struct rte_eth_dev *dev,
1949 const struct rte_flow_attr *attr,
1950 const struct rte_flow_item items[],
1951 const struct rte_flow_action actions[],
1952 struct rte_flow_error *error)
1954 struct priv *priv = dev->data->dev_private;
1956 struct mlx5_flow_parse parser = { .create = 0, };
1959 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1967 * @see rte_flow_create()
1971 mlx5_flow_create(struct rte_eth_dev *dev,
1972 const struct rte_flow_attr *attr,
1973 const struct rte_flow_item items[],
1974 const struct rte_flow_action actions[],
1975 struct rte_flow_error *error)
1977 struct priv *priv = dev->data->dev_private;
1978 struct rte_flow *flow;
1981 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1991 * Pointer to private structure.
1993 * Pointer to a TAILQ flow list.
1998 priv_flow_destroy(struct priv *priv,
1999 struct mlx5_flows *list,
2000 struct rte_flow *flow)
2005 claim_zero(ibv_destroy_counter_set(flow->cs));
2008 if (flow->drop || !flow->mark)
2010 for (i = 0; i != flow->queues_n; ++i) {
2011 struct rte_flow *tmp;
2015 * To remove the mark from the queue, the queue must not be
2016 * present in any other marked flow (RSS or not).
2018 TAILQ_FOREACH(tmp, list, next) {
2020 uint16_t *tqs = NULL;
2025 for (j = 0; j != hash_rxq_init_n; ++j) {
2026 if (!tmp->frxq[j].hrxq)
2028 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2029 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2033 for (j = 0; (j != tq_n) && !mark; j++)
2034 if (tqs[j] == (*flow->queues)[i])
2037 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2041 if (flow->drxq.ibv_flow)
2042 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2043 rte_free(flow->drxq.ibv_attr);
2045 for (i = 0; i != hash_rxq_init_n; ++i) {
2046 struct mlx5_flow *frxq = &flow->frxq[i];
2049 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
2051 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2053 rte_free(frxq->ibv_attr);
2056 TAILQ_REMOVE(list, flow, next);
2057 DEBUG("Flow destroyed %p", (void *)flow);
2062 * Destroy all flows.
2065 * Pointer to private structure.
2067 * Pointer to a TAILQ flow list.
2070 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2072 while (!TAILQ_EMPTY(list)) {
2073 struct rte_flow *flow;
2075 flow = TAILQ_FIRST(list);
2076 priv_flow_destroy(priv, list, flow);
2081 * Create drop queue.
2084 * Pointer to private structure.
2090 priv_flow_create_drop_queue(struct priv *priv)
2092 struct mlx5_hrxq_drop *fdq = NULL;
2096 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2098 WARN("cannot allocate memory for drop queue");
2101 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
2103 WARN("cannot allocate CQ for drop queue");
2106 fdq->wq = ibv_create_wq(priv->ctx,
2107 &(struct ibv_wq_init_attr){
2108 .wq_type = IBV_WQT_RQ,
2115 WARN("cannot allocate WQ for drop queue");
2118 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
2119 &(struct ibv_rwq_ind_table_init_attr){
2120 .log_ind_tbl_size = 0,
2121 .ind_tbl = &fdq->wq,
2124 if (!fdq->ind_table) {
2125 WARN("cannot allocate indirection table for drop queue");
2128 fdq->qp = ibv_create_qp_ex(priv->ctx,
2129 &(struct ibv_qp_init_attr_ex){
2130 .qp_type = IBV_QPT_RAW_PACKET,
2132 IBV_QP_INIT_ATTR_PD |
2133 IBV_QP_INIT_ATTR_IND_TABLE |
2134 IBV_QP_INIT_ATTR_RX_HASH,
2135 .rx_hash_conf = (struct ibv_rx_hash_conf){
2137 IBV_RX_HASH_FUNC_TOEPLITZ,
2138 .rx_hash_key_len = rss_hash_default_key_len,
2139 .rx_hash_key = rss_hash_default_key,
2140 .rx_hash_fields_mask = 0,
2142 .rwq_ind_tbl = fdq->ind_table,
2146 WARN("cannot allocate QP for drop queue");
2149 priv->flow_drop_queue = fdq;
2153 claim_zero(ibv_destroy_qp(fdq->qp));
2155 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2157 claim_zero(ibv_destroy_wq(fdq->wq));
2159 claim_zero(ibv_destroy_cq(fdq->cq));
2162 priv->flow_drop_queue = NULL;
2167 * Delete drop queue.
2170 * Pointer to private structure.
2173 priv_flow_delete_drop_queue(struct priv *priv)
2175 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2180 claim_zero(ibv_destroy_qp(fdq->qp));
2182 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2184 claim_zero(ibv_destroy_wq(fdq->wq));
2186 claim_zero(ibv_destroy_cq(fdq->cq));
2188 priv->flow_drop_queue = NULL;
2195 * Pointer to private structure.
2197 * Pointer to a TAILQ flow list.
2200 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2202 struct rte_flow *flow;
2204 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2208 if (!flow->drxq.ibv_flow)
2210 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2211 flow->drxq.ibv_flow = NULL;
2216 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2218 for (i = 0; i != hash_rxq_init_n; ++i) {
2219 if (!flow->frxq[i].hrxq)
2221 ind_tbl = flow->frxq[i].hrxq->ind_table;
2224 for (i = 0; i != ind_tbl->queues_n; ++i)
2225 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2227 for (i = 0; i != hash_rxq_init_n; ++i) {
2228 if (!flow->frxq[i].ibv_flow)
2230 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2231 flow->frxq[i].ibv_flow = NULL;
2232 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2233 flow->frxq[i].hrxq = NULL;
2235 DEBUG("Flow %p removed", (void *)flow);
2243 * Pointer to private structure.
2245 * Pointer to a TAILQ flow list.
2248 * 0 on success, a errno value otherwise and rte_errno is set.
2251 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2253 struct rte_flow *flow;
2255 TAILQ_FOREACH(flow, list, next) {
2259 flow->drxq.ibv_flow =
2260 ibv_create_flow(priv->flow_drop_queue->qp,
2261 flow->drxq.ibv_attr);
2262 if (!flow->drxq.ibv_flow) {
2263 DEBUG("Flow %p cannot be applied",
2268 DEBUG("Flow %p applied", (void *)flow);
2272 for (i = 0; i != hash_rxq_init_n; ++i) {
2273 if (!flow->frxq[i].ibv_attr)
2275 flow->frxq[i].hrxq =
2276 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2277 flow->rss_conf.rss_key_len,
2278 hash_rxq_init[i].hash_fields,
2281 if (flow->frxq[i].hrxq)
2283 flow->frxq[i].hrxq =
2284 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2285 flow->rss_conf.rss_key_len,
2286 hash_rxq_init[i].hash_fields,
2289 if (!flow->frxq[i].hrxq) {
2290 DEBUG("Flow %p cannot be applied",
2296 flow->frxq[i].ibv_flow =
2297 ibv_create_flow(flow->frxq[i].hrxq->qp,
2298 flow->frxq[i].ibv_attr);
2299 if (!flow->frxq[i].ibv_flow) {
2300 DEBUG("Flow %p cannot be applied",
2305 DEBUG("Flow %p applied", (void *)flow);
2309 for (i = 0; i != flow->queues_n; ++i)
2310 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2316 * Verify the flow list is empty
2319 * Pointer to private structure.
2321 * @return the number of flows not released.
2324 priv_flow_verify(struct priv *priv)
2326 struct rte_flow *flow;
2329 TAILQ_FOREACH(flow, &priv->flows, next) {
2330 DEBUG("%p: flow %p still referenced", (void *)priv,
2338 * Enable a control flow configured from the control plane.
2341 * Pointer to Ethernet device.
2343 * An Ethernet flow spec to apply.
2345 * An Ethernet flow mask to apply.
2347 * A VLAN flow spec to apply.
2349 * A VLAN flow mask to apply.
2355 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2356 struct rte_flow_item_eth *eth_spec,
2357 struct rte_flow_item_eth *eth_mask,
2358 struct rte_flow_item_vlan *vlan_spec,
2359 struct rte_flow_item_vlan *vlan_mask)
2361 struct priv *priv = dev->data->dev_private;
2362 const struct rte_flow_attr attr = {
2364 .priority = MLX5_CTRL_FLOW_PRIORITY,
2366 struct rte_flow_item items[] = {
2368 .type = RTE_FLOW_ITEM_TYPE_ETH,
2374 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2375 RTE_FLOW_ITEM_TYPE_END,
2381 .type = RTE_FLOW_ITEM_TYPE_END,
2384 struct rte_flow_action actions[] = {
2386 .type = RTE_FLOW_ACTION_TYPE_RSS,
2389 .type = RTE_FLOW_ACTION_TYPE_END,
2392 struct rte_flow *flow;
2393 struct rte_flow_error error;
2396 struct rte_flow_action_rss rss;
2398 const struct rte_eth_rss_conf *rss_conf;
2400 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2404 if (!priv->reta_idx_n)
2406 for (i = 0; i != priv->reta_idx_n; ++i)
2407 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2408 action_rss.local.rss_conf = &priv->rss_conf;
2409 action_rss.local.num = priv->reta_idx_n;
2410 actions[0].conf = (const void *)&action_rss.rss;
2411 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2419 * Enable a flow control configured from the control plane.
2422 * Pointer to Ethernet device.
2424 * An Ethernet flow spec to apply.
2426 * An Ethernet flow mask to apply.
2432 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2433 struct rte_flow_item_eth *eth_spec,
2434 struct rte_flow_item_eth *eth_mask)
2436 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2442 * @see rte_flow_destroy()
2446 mlx5_flow_destroy(struct rte_eth_dev *dev,
2447 struct rte_flow *flow,
2448 struct rte_flow_error *error)
2450 struct priv *priv = dev->data->dev_private;
2454 priv_flow_destroy(priv, &priv->flows, flow);
2460 * Destroy all flows.
2462 * @see rte_flow_flush()
2466 mlx5_flow_flush(struct rte_eth_dev *dev,
2467 struct rte_flow_error *error)
2469 struct priv *priv = dev->data->dev_private;
2473 priv_flow_flush(priv, &priv->flows);
2478 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2480 * Query flow counter.
2484 * @param counter_value
2485 * returned data from the counter.
2488 * 0 on success, a errno value otherwise and rte_errno is set.
2491 priv_flow_query_count(struct ibv_counter_set *cs,
2492 struct mlx5_flow_counter_stats *counter_stats,
2493 struct rte_flow_query_count *query_count,
2494 struct rte_flow_error *error)
2496 uint64_t counters[2];
2497 struct ibv_query_counter_set_attr query_cs_attr = {
2499 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2501 struct ibv_counter_set_data query_out = {
2503 .outlen = 2 * sizeof(uint64_t),
2505 int res = ibv_query_counter_set(&query_cs_attr, &query_out);
2508 rte_flow_error_set(error, -res,
2509 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2511 "cannot read counter");
2514 query_count->hits_set = 1;
2515 query_count->bytes_set = 1;
2516 query_count->hits = counters[0] - counter_stats->hits;
2517 query_count->bytes = counters[1] - counter_stats->bytes;
2518 if (query_count->reset) {
2519 counter_stats->hits = counters[0];
2520 counter_stats->bytes = counters[1];
2528 * @see rte_flow_query()
2532 mlx5_flow_query(struct rte_eth_dev *dev,
2533 struct rte_flow *flow,
2534 enum rte_flow_action_type action __rte_unused,
2536 struct rte_flow_error *error)
2538 struct priv *priv = dev->data->dev_private;
2543 res = priv_flow_query_count(flow->cs,
2544 &flow->counter_stats,
2545 (struct rte_flow_query_count *)data,
2548 rte_flow_error_set(error, res,
2549 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2551 "no counter found for flow");
2561 * @see rte_flow_isolate()
2565 mlx5_flow_isolate(struct rte_eth_dev *dev,
2567 struct rte_flow_error *error)
2569 struct priv *priv = dev->data->dev_private;
2572 if (dev->data->dev_started) {
2573 rte_flow_error_set(error, EBUSY,
2574 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2576 "port must be stopped first");
2580 priv->isolated = !!enable;
2582 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2584 priv->dev->dev_ops = &mlx5_dev_ops;
2590 * Convert a flow director filter to a generic flow.
2593 * Private structure.
2594 * @param fdir_filter
2595 * Flow director filter to add.
2597 * Generic flow parameters structure.
2600 * 0 on success, errno value on error.
2603 priv_fdir_filter_convert(struct priv *priv,
2604 const struct rte_eth_fdir_filter *fdir_filter,
2605 struct mlx5_fdir *attributes)
2607 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2609 /* Validate queue number. */
2610 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2611 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2614 attributes->attr.ingress = 1;
2615 attributes->items[0] = (struct rte_flow_item) {
2616 .type = RTE_FLOW_ITEM_TYPE_ETH,
2617 .spec = &attributes->l2,
2619 switch (fdir_filter->action.behavior) {
2620 case RTE_ETH_FDIR_ACCEPT:
2621 attributes->actions[0] = (struct rte_flow_action){
2622 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2623 .conf = &attributes->queue,
2626 case RTE_ETH_FDIR_REJECT:
2627 attributes->actions[0] = (struct rte_flow_action){
2628 .type = RTE_FLOW_ACTION_TYPE_DROP,
2632 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2635 attributes->queue.index = fdir_filter->action.rx_queue;
2636 switch (fdir_filter->input.flow_type) {
2637 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2638 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2639 .src_addr = input->flow.udp4_flow.ip.src_ip,
2640 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2641 .time_to_live = input->flow.udp4_flow.ip.ttl,
2642 .type_of_service = input->flow.udp4_flow.ip.tos,
2643 .next_proto_id = input->flow.udp4_flow.ip.proto,
2645 attributes->l4.udp.hdr = (struct udp_hdr){
2646 .src_port = input->flow.udp4_flow.src_port,
2647 .dst_port = input->flow.udp4_flow.dst_port,
2649 attributes->items[1] = (struct rte_flow_item){
2650 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2651 .spec = &attributes->l3,
2653 attributes->items[2] = (struct rte_flow_item){
2654 .type = RTE_FLOW_ITEM_TYPE_UDP,
2655 .spec = &attributes->l4,
2658 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2659 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2660 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2661 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2662 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2663 .type_of_service = input->flow.tcp4_flow.ip.tos,
2664 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2666 attributes->l4.tcp.hdr = (struct tcp_hdr){
2667 .src_port = input->flow.tcp4_flow.src_port,
2668 .dst_port = input->flow.tcp4_flow.dst_port,
2670 attributes->items[1] = (struct rte_flow_item){
2671 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2672 .spec = &attributes->l3,
2674 attributes->items[2] = (struct rte_flow_item){
2675 .type = RTE_FLOW_ITEM_TYPE_TCP,
2676 .spec = &attributes->l4,
2679 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2680 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2681 .src_addr = input->flow.ip4_flow.src_ip,
2682 .dst_addr = input->flow.ip4_flow.dst_ip,
2683 .time_to_live = input->flow.ip4_flow.ttl,
2684 .type_of_service = input->flow.ip4_flow.tos,
2685 .next_proto_id = input->flow.ip4_flow.proto,
2687 attributes->items[1] = (struct rte_flow_item){
2688 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2689 .spec = &attributes->l3,
2692 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2693 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2694 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2695 .proto = input->flow.udp6_flow.ip.proto,
2697 memcpy(attributes->l3.ipv6.hdr.src_addr,
2698 input->flow.udp6_flow.ip.src_ip,
2699 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2700 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2701 input->flow.udp6_flow.ip.dst_ip,
2702 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2703 attributes->l4.udp.hdr = (struct udp_hdr){
2704 .src_port = input->flow.udp6_flow.src_port,
2705 .dst_port = input->flow.udp6_flow.dst_port,
2707 attributes->items[1] = (struct rte_flow_item){
2708 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2709 .spec = &attributes->l3,
2711 attributes->items[2] = (struct rte_flow_item){
2712 .type = RTE_FLOW_ITEM_TYPE_UDP,
2713 .spec = &attributes->l4,
2716 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2717 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2718 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2719 .proto = input->flow.tcp6_flow.ip.proto,
2721 memcpy(attributes->l3.ipv6.hdr.src_addr,
2722 input->flow.tcp6_flow.ip.src_ip,
2723 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2724 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2725 input->flow.tcp6_flow.ip.dst_ip,
2726 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2727 attributes->l4.tcp.hdr = (struct tcp_hdr){
2728 .src_port = input->flow.tcp6_flow.src_port,
2729 .dst_port = input->flow.tcp6_flow.dst_port,
2731 attributes->items[1] = (struct rte_flow_item){
2732 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2733 .spec = &attributes->l3,
2735 attributes->items[2] = (struct rte_flow_item){
2736 .type = RTE_FLOW_ITEM_TYPE_UDP,
2737 .spec = &attributes->l4,
2740 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2741 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2742 .hop_limits = input->flow.ipv6_flow.hop_limits,
2743 .proto = input->flow.ipv6_flow.proto,
2745 memcpy(attributes->l3.ipv6.hdr.src_addr,
2746 input->flow.ipv6_flow.src_ip,
2747 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2748 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2749 input->flow.ipv6_flow.dst_ip,
2750 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2751 attributes->items[1] = (struct rte_flow_item){
2752 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2753 .spec = &attributes->l3,
2757 ERROR("invalid flow type%d",
2758 fdir_filter->input.flow_type);
2765 * Add new flow director filter and store it in list.
2768 * Private structure.
2769 * @param fdir_filter
2770 * Flow director filter to add.
2773 * 0 on success, errno value on failure.
2776 priv_fdir_filter_add(struct priv *priv,
2777 const struct rte_eth_fdir_filter *fdir_filter)
2779 struct mlx5_fdir attributes = {
2782 struct mlx5_flow_parse parser = {
2783 .layer = HASH_RXQ_ETH,
2785 struct rte_flow_error error;
2786 struct rte_flow *flow;
2789 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2792 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2793 attributes.actions, &error, &parser);
2796 flow = priv_flow_create(priv,
2803 DEBUG("FDIR created %p", (void *)flow);
2810 * Delete specific filter.
2813 * Private structure.
2814 * @param fdir_filter
2815 * Filter to be deleted.
2818 * 0 on success, errno value on failure.
2821 priv_fdir_filter_delete(struct priv *priv,
2822 const struct rte_eth_fdir_filter *fdir_filter)
2824 struct mlx5_fdir attributes;
2825 struct mlx5_flow_parse parser = {
2827 .layer = HASH_RXQ_ETH,
2829 struct rte_flow_error error;
2830 struct rte_flow *flow;
2834 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2837 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2838 attributes.actions, &error, &parser);
2841 TAILQ_FOREACH(flow, &priv->flows, next) {
2842 struct ibv_flow_attr *attr;
2843 struct ibv_spec_header *attr_h;
2845 struct ibv_flow_attr *flow_attr;
2846 struct ibv_spec_header *flow_h;
2848 unsigned int specs_n;
2851 attr = parser.drop_q.ibv_attr;
2853 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2855 flow_attr = flow->drxq.ibv_attr;
2857 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2858 /* Compare first the attributes. */
2859 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2861 if (attr->num_of_specs == 0)
2863 spec = (void *)((uintptr_t)attr +
2864 sizeof(struct ibv_flow_attr));
2865 flow_spec = (void *)((uintptr_t)flow_attr +
2866 sizeof(struct ibv_flow_attr));
2867 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2868 for (i = 0; i != specs_n; ++i) {
2871 if (memcmp(spec, flow_spec,
2872 RTE_MIN(attr_h->size, flow_h->size)))
2874 spec = (void *)((uintptr_t)attr + attr_h->size);
2875 flow_spec = (void *)((uintptr_t)flow_attr +
2878 /* At this point, the flow match. */
2882 priv_flow_destroy(priv, &priv->flows, flow);
2885 rte_free(parser.drop_q.ibv_attr);
2887 for (i = 0; i != hash_rxq_init_n; ++i) {
2888 if (parser.queue[i].ibv_attr)
2889 rte_free(parser.queue[i].ibv_attr);
2896 * Update queue for specific filter.
2899 * Private structure.
2900 * @param fdir_filter
2901 * Filter to be updated.
2904 * 0 on success, errno value on failure.
2907 priv_fdir_filter_update(struct priv *priv,
2908 const struct rte_eth_fdir_filter *fdir_filter)
2912 ret = priv_fdir_filter_delete(priv, fdir_filter);
2915 ret = priv_fdir_filter_add(priv, fdir_filter);
2920 * Flush all filters.
2923 * Private structure.
2926 priv_fdir_filter_flush(struct priv *priv)
2928 priv_flow_flush(priv, &priv->flows);
2932 * Get flow director information.
2935 * Private structure.
2936 * @param[out] fdir_info
2937 * Resulting flow director information.
2940 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2942 struct rte_eth_fdir_masks *mask =
2943 &priv->dev->data->dev_conf.fdir_conf.mask;
2945 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2946 fdir_info->guarant_spc = 0;
2947 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2948 fdir_info->max_flexpayload = 0;
2949 fdir_info->flow_types_mask[0] = 0;
2950 fdir_info->flex_payload_unit = 0;
2951 fdir_info->max_flex_payload_segment_num = 0;
2952 fdir_info->flex_payload_limit = 0;
2953 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2957 * Deal with flow director operations.
2960 * Pointer to private structure.
2962 * Operation to perform.
2964 * Pointer to operation-specific structure.
2967 * 0 on success, errno value on failure.
2970 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2972 enum rte_fdir_mode fdir_mode =
2973 priv->dev->data->dev_conf.fdir_conf.mode;
2976 if (filter_op == RTE_ETH_FILTER_NOP)
2978 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2979 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2980 ERROR("%p: flow director mode %d not supported",
2981 (void *)priv, fdir_mode);
2984 switch (filter_op) {
2985 case RTE_ETH_FILTER_ADD:
2986 ret = priv_fdir_filter_add(priv, arg);
2988 case RTE_ETH_FILTER_UPDATE:
2989 ret = priv_fdir_filter_update(priv, arg);
2991 case RTE_ETH_FILTER_DELETE:
2992 ret = priv_fdir_filter_delete(priv, arg);
2994 case RTE_ETH_FILTER_FLUSH:
2995 priv_fdir_filter_flush(priv);
2997 case RTE_ETH_FILTER_INFO:
2998 priv_fdir_info_get(priv, arg);
3001 DEBUG("%p: unknown operation %u", (void *)priv,
3010 * Manage filter operations.
3013 * Pointer to Ethernet device structure.
3014 * @param filter_type
3017 * Operation to perform.
3019 * Pointer to operation-specific structure.
3022 * 0 on success, negative errno value on failure.
3025 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3026 enum rte_filter_type filter_type,
3027 enum rte_filter_op filter_op,
3031 struct priv *priv = dev->data->dev_private;
3033 switch (filter_type) {
3034 case RTE_ETH_FILTER_GENERIC:
3035 if (filter_op != RTE_ETH_FILTER_GET)
3037 *(const void **)arg = &mlx5_flow_ops;
3039 case RTE_ETH_FILTER_FDIR:
3041 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3045 ERROR("%p: filter type (%d) not supported",
3046 (void *)dev, filter_type);