4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
63 struct ibv_counter_set_init_attr {
66 struct ibv_flow_spec_counter_action {
69 struct ibv_counter_set {
74 ibv_destroy_counter_set(struct ibv_counter_set *cs)
81 /* Dev ops structure defined in mlx5.c */
82 extern const struct eth_dev_ops mlx5_dev_ops;
83 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
86 mlx5_flow_create_eth(const struct rte_flow_item *item,
87 const void *default_mask,
91 mlx5_flow_create_vlan(const struct rte_flow_item *item,
92 const void *default_mask,
96 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
97 const void *default_mask,
101 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
102 const void *default_mask,
106 mlx5_flow_create_udp(const struct rte_flow_item *item,
107 const void *default_mask,
111 mlx5_flow_create_tcp(const struct rte_flow_item *item,
112 const void *default_mask,
116 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
117 const void *default_mask,
120 struct mlx5_flow_parse;
123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
132 /* Hash RX queue types. */
143 /* Initialization data for hash RX queue. */
144 struct hash_rxq_init {
145 uint64_t hash_fields; /* Fields that participate in the hash. */
146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
147 unsigned int flow_priority; /* Flow priority to use. */
148 unsigned int ip_version; /* Internet protocol. */
151 /* Initialization data for hash RX queues. */
152 const struct hash_rxq_init hash_rxq_init[] = {
154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
155 IBV_RX_HASH_DST_IPV4 |
156 IBV_RX_HASH_SRC_PORT_TCP |
157 IBV_RX_HASH_DST_PORT_TCP),
158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
160 .ip_version = MLX5_IPV4,
163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
164 IBV_RX_HASH_DST_IPV4 |
165 IBV_RX_HASH_SRC_PORT_UDP |
166 IBV_RX_HASH_DST_PORT_UDP),
167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
169 .ip_version = MLX5_IPV4,
172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
173 IBV_RX_HASH_DST_IPV4),
174 .dpdk_rss_hf = (ETH_RSS_IPV4 |
177 .ip_version = MLX5_IPV4,
180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
181 IBV_RX_HASH_DST_IPV6 |
182 IBV_RX_HASH_SRC_PORT_TCP |
183 IBV_RX_HASH_DST_PORT_TCP),
184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
186 .ip_version = MLX5_IPV6,
189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
190 IBV_RX_HASH_DST_IPV6 |
191 IBV_RX_HASH_SRC_PORT_UDP |
192 IBV_RX_HASH_DST_PORT_UDP),
193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
195 .ip_version = MLX5_IPV6,
198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
199 IBV_RX_HASH_DST_IPV6),
200 .dpdk_rss_hf = (ETH_RSS_IPV6 |
203 .ip_version = MLX5_IPV6,
212 /* Number of entries in hash_rxq_init[]. */
213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
215 /** Structure for holding counter stats. */
216 struct mlx5_flow_counter_stats {
217 uint64_t hits; /**< Number of packets matched by the rule. */
218 uint64_t bytes; /**< Number of bytes matched by the rule. */
221 /** Structure for Drop queue. */
222 struct mlx5_hrxq_drop {
223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
224 struct ibv_qp *qp; /**< Verbs queue pair. */
225 struct ibv_wq *wq; /**< Verbs work queue. */
226 struct ibv_cq *cq; /**< Verbs completion queue. */
229 /* Flows structures. */
231 uint64_t hash_fields; /**< Fields that participate in the hash. */
232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
233 struct ibv_flow *ibv_flow; /**< Verbs flow. */
234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
237 /* Drop flows structures. */
238 struct mlx5_flow_drop {
239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
240 struct ibv_flow *ibv_flow; /**< Verbs flow. */
244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
245 uint32_t mark:1; /**< Set if the flow is marked. */
246 uint32_t drop:1; /**< Drop queue. */
247 uint16_t queues_n; /**< Number of entries in queue[]. */
248 uint16_t (*queues)[]; /**< Queues indexes to use. */
249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
250 uint8_t rss_key[40]; /**< copy of the RSS key. */
251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
255 /**< Flow with Rx queue. */
256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */
260 /** Static initializer for items. */
262 (const enum rte_flow_item_type []){ \
263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
266 /** Structure to generate a simple graph of layers supported by the NIC. */
267 struct mlx5_flow_items {
268 /** List of possible actions for these items. */
269 const enum rte_flow_action_type *const actions;
270 /** Bit-masks corresponding to the possibilities for the item. */
273 * Default bit-masks to use when item->mask is not provided. When
274 * \default_mask is also NULL, the full supported bit-mask (\mask) is
277 const void *default_mask;
278 /** Bit-masks size in bytes. */
279 const unsigned int mask_sz;
281 * Conversion function from rte_flow to NIC specific flow.
284 * rte_flow item to convert.
285 * @param default_mask
286 * Default bit-masks to use when item->mask is not provided.
288 * Internal structure to store the conversion.
291 * 0 on success, negative value otherwise.
293 int (*convert)(const struct rte_flow_item *item,
294 const void *default_mask,
296 /** Size in bytes of the destination structure. */
297 const unsigned int dst_sz;
298 /** List of possible following items. */
299 const enum rte_flow_item_type *const items;
302 /** Valid action for this PMD. */
303 static const enum rte_flow_action_type valid_actions[] = {
304 RTE_FLOW_ACTION_TYPE_DROP,
305 RTE_FLOW_ACTION_TYPE_QUEUE,
306 RTE_FLOW_ACTION_TYPE_MARK,
307 RTE_FLOW_ACTION_TYPE_FLAG,
308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
309 RTE_FLOW_ACTION_TYPE_COUNT,
311 RTE_FLOW_ACTION_TYPE_END,
314 /** Graph of supported items and associated actions. */
315 static const struct mlx5_flow_items mlx5_flow_items[] = {
316 [RTE_FLOW_ITEM_TYPE_END] = {
317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
318 RTE_FLOW_ITEM_TYPE_VXLAN),
320 [RTE_FLOW_ITEM_TYPE_ETH] = {
321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
322 RTE_FLOW_ITEM_TYPE_IPV4,
323 RTE_FLOW_ITEM_TYPE_IPV6),
324 .actions = valid_actions,
325 .mask = &(const struct rte_flow_item_eth){
326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
330 .default_mask = &rte_flow_item_eth_mask,
331 .mask_sz = sizeof(struct rte_flow_item_eth),
332 .convert = mlx5_flow_create_eth,
333 .dst_sz = sizeof(struct ibv_flow_spec_eth),
335 [RTE_FLOW_ITEM_TYPE_VLAN] = {
336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
337 RTE_FLOW_ITEM_TYPE_IPV6),
338 .actions = valid_actions,
339 .mask = &(const struct rte_flow_item_vlan){
342 .default_mask = &rte_flow_item_vlan_mask,
343 .mask_sz = sizeof(struct rte_flow_item_vlan),
344 .convert = mlx5_flow_create_vlan,
347 [RTE_FLOW_ITEM_TYPE_IPV4] = {
348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
349 RTE_FLOW_ITEM_TYPE_TCP),
350 .actions = valid_actions,
351 .mask = &(const struct rte_flow_item_ipv4){
355 .type_of_service = -1,
359 .default_mask = &rte_flow_item_ipv4_mask,
360 .mask_sz = sizeof(struct rte_flow_item_ipv4),
361 .convert = mlx5_flow_create_ipv4,
362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
364 [RTE_FLOW_ITEM_TYPE_IPV6] = {
365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
366 RTE_FLOW_ITEM_TYPE_TCP),
367 .actions = valid_actions,
368 .mask = &(const struct rte_flow_item_ipv6){
371 0xff, 0xff, 0xff, 0xff,
372 0xff, 0xff, 0xff, 0xff,
373 0xff, 0xff, 0xff, 0xff,
374 0xff, 0xff, 0xff, 0xff,
377 0xff, 0xff, 0xff, 0xff,
378 0xff, 0xff, 0xff, 0xff,
379 0xff, 0xff, 0xff, 0xff,
380 0xff, 0xff, 0xff, 0xff,
387 .default_mask = &rte_flow_item_ipv6_mask,
388 .mask_sz = sizeof(struct rte_flow_item_ipv6),
389 .convert = mlx5_flow_create_ipv6,
390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
392 [RTE_FLOW_ITEM_TYPE_UDP] = {
393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
394 .actions = valid_actions,
395 .mask = &(const struct rte_flow_item_udp){
401 .default_mask = &rte_flow_item_udp_mask,
402 .mask_sz = sizeof(struct rte_flow_item_udp),
403 .convert = mlx5_flow_create_udp,
404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
406 [RTE_FLOW_ITEM_TYPE_TCP] = {
407 .actions = valid_actions,
408 .mask = &(const struct rte_flow_item_tcp){
414 .default_mask = &rte_flow_item_tcp_mask,
415 .mask_sz = sizeof(struct rte_flow_item_tcp),
416 .convert = mlx5_flow_create_tcp,
417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
419 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
421 .actions = valid_actions,
422 .mask = &(const struct rte_flow_item_vxlan){
423 .vni = "\xff\xff\xff",
425 .default_mask = &rte_flow_item_vxlan_mask,
426 .mask_sz = sizeof(struct rte_flow_item_vxlan),
427 .convert = mlx5_flow_create_vxlan,
428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
432 /** Structure to pass to the conversion function. */
433 struct mlx5_flow_parse {
434 uint32_t inner; /**< Set once VXLAN is encountered. */
436 /**< Whether resources should remain after a validate. */
437 uint32_t drop:1; /**< Target is a drop queue. */
438 uint32_t mark:1; /**< Mark is present in the flow. */
439 uint32_t count:1; /**< Count is present in the flow. */
440 uint32_t mark_id; /**< Mark identifier. */
441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
442 uint16_t queues_n; /**< Number of entries in queue[]. */
443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
444 uint8_t rss_key[40]; /**< copy of the RSS key. */
445 enum hash_rxq_type layer; /**< Last pattern layer detected. */
446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
449 struct ibv_flow_attr *ibv_attr;
450 /**< Pointer to Verbs attributes. */
452 /**< Current position or total size of the attribute. */
453 } queue[RTE_DIM(hash_rxq_init)];
455 struct ibv_flow_attr *ibv_attr;
456 /**< Pointer to Verbs attributes. */
458 /**< Current position or total size of the attribute. */
463 static const struct rte_flow_ops mlx5_flow_ops = {
464 .validate = mlx5_flow_validate,
465 .create = mlx5_flow_create,
466 .destroy = mlx5_flow_destroy,
467 .flush = mlx5_flow_flush,
468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
469 .query = mlx5_flow_query,
473 .isolate = mlx5_flow_isolate,
476 /* Convert FDIR request to Generic flow. */
478 struct rte_flow_attr attr;
479 struct rte_flow_action actions[2];
480 struct rte_flow_item items[4];
481 struct rte_flow_item_eth l2;
483 struct rte_flow_item_ipv4 ipv4;
484 struct rte_flow_item_ipv6 ipv6;
487 struct rte_flow_item_udp udp;
488 struct rte_flow_item_tcp tcp;
490 struct rte_flow_action_queue queue;
493 /* Verbs specification header. */
494 struct ibv_spec_header {
495 enum ibv_flow_spec_type type;
500 * Check support for a given item.
503 * Item specification.
505 * Bit-masks covering supported fields to compare with spec, last and mask in
508 * Bit-Mask size in bytes.
514 mlx5_flow_item_validate(const struct rte_flow_item *item,
515 const uint8_t *mask, unsigned int size)
519 if (!item->spec && (item->mask || item->last))
521 if (item->spec && !item->mask) {
523 const uint8_t *spec = item->spec;
525 for (i = 0; i < size; ++i)
526 if ((spec[i] | mask[i]) != mask[i])
529 if (item->last && !item->mask) {
531 const uint8_t *spec = item->last;
533 for (i = 0; i < size; ++i)
534 if ((spec[i] | mask[i]) != mask[i])
539 const uint8_t *spec = item->mask;
541 for (i = 0; i < size; ++i)
542 if ((spec[i] | mask[i]) != mask[i])
545 if (item->spec && item->last) {
548 const uint8_t *apply = mask;
553 for (i = 0; i < size; ++i) {
554 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
555 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
557 ret = memcmp(spec, last, size);
563 * Copy the RSS configuration from the user ones.
566 * Pointer to private structure.
568 * Internal parser structure.
570 * User RSS configuration to save.
573 * 0 on success, errno value on failure.
576 priv_flow_convert_rss_conf(struct priv *priv,
577 struct mlx5_flow_parse *parser,
578 const struct rte_eth_rss_conf *rss_conf)
580 const struct rte_eth_rss_conf *rss =
581 rss_conf ? rss_conf : &priv->rss_conf;
583 if (rss->rss_key_len > 40)
585 parser->rss_conf.rss_key_len = rss->rss_key_len;
586 parser->rss_conf.rss_hf = rss->rss_hf;
587 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
588 parser->rss_conf.rss_key = parser->rss_key;
593 * Extract attribute to the parser.
596 * Pointer to private structure.
598 * Flow rule attributes.
600 * Perform verbose error reporting if not NULL.
601 * @param[in, out] parser
602 * Internal parser structure.
605 * 0 on success, a negative errno value otherwise and rte_errno is set.
608 priv_flow_convert_attributes(struct priv *priv,
609 const struct rte_flow_attr *attr,
610 struct rte_flow_error *error,
611 struct mlx5_flow_parse *parser)
616 rte_flow_error_set(error, ENOTSUP,
617 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
619 "groups are not supported");
622 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
623 rte_flow_error_set(error, ENOTSUP,
624 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
626 "priorities are not supported");
630 rte_flow_error_set(error, ENOTSUP,
631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
633 "egress is not supported");
636 if (!attr->ingress) {
637 rte_flow_error_set(error, ENOTSUP,
638 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
640 "only ingress is supported");
647 * Extract actions request to the parser.
650 * Pointer to private structure.
652 * Associated actions (list terminated by the END action).
654 * Perform verbose error reporting if not NULL.
655 * @param[in, out] parser
656 * Internal parser structure.
659 * 0 on success, a negative errno value otherwise and rte_errno is set.
662 priv_flow_convert_actions(struct priv *priv,
663 const struct rte_flow_action actions[],
664 struct rte_flow_error *error,
665 struct mlx5_flow_parse *parser)
668 * Add default RSS configuration necessary for Verbs to create QP even
669 * if no RSS is necessary.
671 priv_flow_convert_rss_conf(priv, parser,
672 (const struct rte_eth_rss_conf *)
674 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
675 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
677 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
679 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
680 const struct rte_flow_action_queue *queue =
681 (const struct rte_flow_action_queue *)
686 if (!queue || (queue->index > (priv->rxqs_n - 1)))
687 goto exit_action_not_supported;
688 for (n = 0; n < parser->queues_n; ++n) {
689 if (parser->queues[n] == queue->index) {
694 if (parser->queues_n > 1 && !found) {
695 rte_flow_error_set(error, ENOTSUP,
696 RTE_FLOW_ERROR_TYPE_ACTION,
698 "queue action not in RSS queues");
702 parser->queues_n = 1;
703 parser->queues[0] = queue->index;
705 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
706 const struct rte_flow_action_rss *rss =
707 (const struct rte_flow_action_rss *)
711 if (!rss || !rss->num) {
712 rte_flow_error_set(error, EINVAL,
713 RTE_FLOW_ERROR_TYPE_ACTION,
718 if (parser->queues_n == 1) {
721 assert(parser->queues_n);
722 for (n = 0; n < rss->num; ++n) {
723 if (parser->queues[0] ==
730 rte_flow_error_set(error, ENOTSUP,
731 RTE_FLOW_ERROR_TYPE_ACTION,
733 "queue action not in RSS"
738 for (n = 0; n < rss->num; ++n) {
739 if (rss->queue[n] >= priv->rxqs_n) {
740 rte_flow_error_set(error, EINVAL,
741 RTE_FLOW_ERROR_TYPE_ACTION,
743 "queue id > number of"
748 for (n = 0; n < rss->num; ++n)
749 parser->queues[n] = rss->queue[n];
750 parser->queues_n = rss->num;
751 if (priv_flow_convert_rss_conf(priv, parser,
753 rte_flow_error_set(error, EINVAL,
754 RTE_FLOW_ERROR_TYPE_ACTION,
756 "wrong RSS configuration");
759 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
760 const struct rte_flow_action_mark *mark =
761 (const struct rte_flow_action_mark *)
765 rte_flow_error_set(error, EINVAL,
766 RTE_FLOW_ERROR_TYPE_ACTION,
768 "mark must be defined");
770 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
771 rte_flow_error_set(error, ENOTSUP,
772 RTE_FLOW_ERROR_TYPE_ACTION,
774 "mark must be between 0"
779 parser->mark_id = mark->id;
780 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
782 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
783 priv->counter_set_supported) {
786 goto exit_action_not_supported;
789 if (!parser->queues_n && !parser->drop) {
790 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
791 NULL, "no valid action");
795 exit_action_not_supported:
796 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
797 actions, "action not supported");
805 * Pointer to private structure.
807 * Pattern specification (list terminated by the END pattern item).
809 * Perform verbose error reporting if not NULL.
810 * @param[in, out] parser
811 * Internal parser structure.
814 * 0 on success, a negative errno value otherwise and rte_errno is set.
817 priv_flow_convert_items_validate(struct priv *priv,
818 const struct rte_flow_item items[],
819 struct rte_flow_error *error,
820 struct mlx5_flow_parse *parser)
822 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
826 /* Initialise the offsets to start after verbs attribute. */
828 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
830 for (i = 0; i != hash_rxq_init_n; ++i)
831 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
833 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
834 const struct mlx5_flow_items *token = NULL;
838 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
842 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
844 if (cur_item->items[i] == items->type) {
845 token = &mlx5_flow_items[items->type];
850 goto exit_item_not_supported;
852 err = mlx5_flow_item_validate(items,
853 (const uint8_t *)cur_item->mask,
856 goto exit_item_not_supported;
857 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
859 rte_flow_error_set(error, ENOTSUP,
860 RTE_FLOW_ERROR_TYPE_ITEM,
862 "cannot recognize multiple"
863 " VXLAN encapsulations");
869 parser->drop_q.offset += cur_item->dst_sz;
870 } else if (parser->queues_n == 1) {
871 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
873 for (n = 0; n != hash_rxq_init_n; ++n)
874 parser->queue[n].offset += cur_item->dst_sz;
878 for (i = 0; i != hash_rxq_init_n; ++i)
879 parser->queue[i].offset +=
880 sizeof(struct ibv_flow_spec_action_tag);
883 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
886 parser->drop_q.offset += size;
888 for (i = 0; i != hash_rxq_init_n; ++i)
889 parser->queue[i].offset += size;
893 exit_item_not_supported:
894 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
895 items, "item not supported");
900 * Allocate memory space to store verbs flow attributes.
903 * Pointer to private structure.
904 * @param[in] priority
907 * Amount of byte to allocate.
909 * Perform verbose error reporting if not NULL.
912 * A verbs flow attribute on success, NULL otherwise.
914 static struct ibv_flow_attr*
915 priv_flow_convert_allocate(struct priv *priv,
916 unsigned int priority,
918 struct rte_flow_error *error)
920 struct ibv_flow_attr *ibv_attr;
923 ibv_attr = rte_calloc(__func__, 1, size, 0);
925 rte_flow_error_set(error, ENOMEM,
926 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
928 "cannot allocate verbs spec attributes.");
931 ibv_attr->priority = priority;
936 * Finalise verbs flow attributes.
939 * Pointer to private structure.
940 * @param[in, out] parser
941 * Internal parser structure.
944 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
946 const unsigned int ipv4 =
947 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
948 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
949 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
950 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
951 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
952 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
956 if (parser->layer == HASH_RXQ_ETH) {
960 * This layer becomes useless as the pattern define under
963 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
964 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
966 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
967 for (i = ohmin; i != (ohmax + 1); ++i) {
968 if (!parser->queue[i].ibv_attr)
970 rte_free(parser->queue[i].ibv_attr);
971 parser->queue[i].ibv_attr = NULL;
973 /* Remove impossible flow according to the RSS configuration. */
974 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
975 parser->rss_conf.rss_hf) {
976 /* Remove any other flow. */
977 for (i = hmin; i != (hmax + 1); ++i) {
978 if ((i == parser->layer) ||
979 (!parser->queue[i].ibv_attr))
981 rte_free(parser->queue[i].ibv_attr);
982 parser->queue[i].ibv_attr = NULL;
984 } else if (!parser->queue[ip].ibv_attr) {
985 /* no RSS possible with the current configuration. */
986 parser->queues_n = 1;
991 * Fill missing layers in verbs specifications, or compute the correct
992 * offset to allocate the memory space for the attributes and
995 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
997 struct ibv_flow_spec_ipv4_ext ipv4;
998 struct ibv_flow_spec_ipv6 ipv6;
999 struct ibv_flow_spec_tcp_udp udp_tcp;
1004 if (i == parser->layer)
1006 if (parser->layer == HASH_RXQ_ETH) {
1007 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
1008 size = sizeof(struct ibv_flow_spec_ipv4_ext);
1009 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
1010 .type = IBV_FLOW_SPEC_IPV4_EXT |
1015 size = sizeof(struct ibv_flow_spec_ipv6);
1016 specs.ipv6 = (struct ibv_flow_spec_ipv6){
1017 .type = IBV_FLOW_SPEC_IPV6 |
1022 if (parser->queue[i].ibv_attr) {
1023 dst = (void *)((uintptr_t)
1024 parser->queue[i].ibv_attr +
1025 parser->queue[i].offset);
1026 memcpy(dst, &specs, size);
1027 ++parser->queue[i].ibv_attr->num_of_specs;
1029 parser->queue[i].offset += size;
1031 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1032 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1033 size = sizeof(struct ibv_flow_spec_tcp_udp);
1034 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1035 .type = ((i == HASH_RXQ_UDPV4 ||
1036 i == HASH_RXQ_UDPV6) ?
1038 IBV_FLOW_SPEC_TCP) |
1042 if (parser->queue[i].ibv_attr) {
1043 dst = (void *)((uintptr_t)
1044 parser->queue[i].ibv_attr +
1045 parser->queue[i].offset);
1046 memcpy(dst, &specs, size);
1047 ++parser->queue[i].ibv_attr->num_of_specs;
1049 parser->queue[i].offset += size;
1055 * Validate and convert a flow supported by the NIC.
1058 * Pointer to private structure.
1060 * Flow rule attributes.
1061 * @param[in] pattern
1062 * Pattern specification (list terminated by the END pattern item).
1063 * @param[in] actions
1064 * Associated actions (list terminated by the END action).
1066 * Perform verbose error reporting if not NULL.
1067 * @param[in, out] parser
1068 * Internal parser structure.
1071 * 0 on success, a negative errno value otherwise and rte_errno is set.
1074 priv_flow_convert(struct priv *priv,
1075 const struct rte_flow_attr *attr,
1076 const struct rte_flow_item items[],
1077 const struct rte_flow_action actions[],
1078 struct rte_flow_error *error,
1079 struct mlx5_flow_parse *parser)
1081 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1085 /* First step. Validate the attributes, items and actions. */
1086 *parser = (struct mlx5_flow_parse){
1087 .create = parser->create,
1088 .layer = HASH_RXQ_ETH,
1089 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1091 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1094 ret = priv_flow_convert_actions(priv, actions, error, parser);
1097 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1100 priv_flow_convert_finalise(priv, parser);
1103 * Allocate the memory space to store verbs specifications.
1106 parser->drop_q.ibv_attr =
1107 priv_flow_convert_allocate(priv, attr->priority,
1108 parser->drop_q.offset,
1110 if (!parser->drop_q.ibv_attr)
1112 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
1113 } else if (parser->queues_n == 1) {
1114 unsigned int priority =
1116 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1117 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1119 parser->queue[HASH_RXQ_ETH].ibv_attr =
1120 priv_flow_convert_allocate(priv, priority,
1122 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1124 parser->queue[HASH_RXQ_ETH].offset =
1125 sizeof(struct ibv_flow_attr);
1127 for (i = 0; i != hash_rxq_init_n; ++i) {
1128 unsigned int priority =
1130 hash_rxq_init[i].flow_priority;
1131 unsigned int offset;
1133 if (!(parser->rss_conf.rss_hf &
1134 hash_rxq_init[i].dpdk_rss_hf) &&
1135 (i != HASH_RXQ_ETH))
1137 offset = parser->queue[i].offset;
1138 parser->queue[i].ibv_attr =
1139 priv_flow_convert_allocate(priv, priority,
1141 if (!parser->queue[i].ibv_attr)
1143 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1146 /* Third step. Conversion parse, fill the specifications. */
1148 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1149 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1151 cur_item = &mlx5_flow_items[items->type];
1152 ret = cur_item->convert(items,
1153 (cur_item->default_mask ?
1154 cur_item->default_mask :
1158 rte_flow_error_set(error, ENOTSUP,
1159 RTE_FLOW_ERROR_TYPE_ITEM,
1160 items, "item not supported");
1165 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1166 if (parser->count && parser->create) {
1167 mlx5_flow_create_count(priv, parser);
1169 goto exit_count_error;
1172 * Last step. Complete missing specification to reach the RSS
1175 if (parser->queues_n > 1) {
1176 priv_flow_convert_finalise(priv, parser);
1177 } else if (!parser->drop) {
1179 * Action queue have their priority overridden with
1180 * Ethernet priority, this priority needs to be adjusted to
1181 * their most specific layer priority.
1183 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1185 hash_rxq_init[parser->layer].flow_priority;
1188 /* Only verification is expected, all resources should be released. */
1189 if (!parser->create) {
1191 rte_free(parser->drop_q.ibv_attr);
1192 parser->drop_q.ibv_attr = NULL;
1194 for (i = 0; i != hash_rxq_init_n; ++i) {
1195 if (parser->queue[i].ibv_attr) {
1196 rte_free(parser->queue[i].ibv_attr);
1197 parser->queue[i].ibv_attr = NULL;
1203 for (i = 0; i != hash_rxq_init_n; ++i) {
1204 if (parser->queue[i].ibv_attr) {
1205 rte_free(parser->queue[i].ibv_attr);
1206 parser->queue[i].ibv_attr = NULL;
1209 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1210 NULL, "cannot allocate verbs spec attributes.");
1213 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1214 NULL, "cannot create counter.");
1219 * Copy the specification created into the flow.
1222 * Internal parser structure.
1224 * Create specification.
1226 * Size in bytes of the specification to copy.
1229 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1236 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1237 parser->drop_q.offset);
1238 memcpy(dst, src, size);
1239 ++parser->drop_q.ibv_attr->num_of_specs;
1240 parser->drop_q.offset += size;
1243 for (i = 0; i != hash_rxq_init_n; ++i) {
1244 if (!parser->queue[i].ibv_attr)
1246 /* Specification must be the same l3 type or none. */
1247 if (parser->layer == HASH_RXQ_ETH ||
1248 (hash_rxq_init[parser->layer].ip_version ==
1249 hash_rxq_init[i].ip_version) ||
1250 (hash_rxq_init[i].ip_version == 0)) {
1251 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1252 parser->queue[i].offset);
1253 memcpy(dst, src, size);
1254 ++parser->queue[i].ibv_attr->num_of_specs;
1255 parser->queue[i].offset += size;
1261 * Convert Ethernet item to Verbs specification.
1264 * Item specification.
1265 * @param default_mask[in]
1266 * Default bit-masks to use when item->mask is not provided.
1267 * @param data[in, out]
1271 mlx5_flow_create_eth(const struct rte_flow_item *item,
1272 const void *default_mask,
1275 const struct rte_flow_item_eth *spec = item->spec;
1276 const struct rte_flow_item_eth *mask = item->mask;
1277 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1278 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1279 struct ibv_flow_spec_eth eth = {
1280 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1284 parser->layer = HASH_RXQ_ETH;
1289 mask = default_mask;
1290 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1291 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1292 eth.val.ether_type = spec->type;
1293 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1294 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1295 eth.mask.ether_type = mask->type;
1296 /* Remove unwanted bits from values. */
1297 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1298 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1299 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1301 eth.val.ether_type &= eth.mask.ether_type;
1303 mlx5_flow_create_copy(parser, ð, eth_size);
1308 * Convert VLAN item to Verbs specification.
1311 * Item specification.
1312 * @param default_mask[in]
1313 * Default bit-masks to use when item->mask is not provided.
1314 * @param data[in, out]
1318 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1319 const void *default_mask,
1322 const struct rte_flow_item_vlan *spec = item->spec;
1323 const struct rte_flow_item_vlan *mask = item->mask;
1324 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1325 struct ibv_flow_spec_eth *eth;
1326 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1331 mask = default_mask;
1334 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1335 parser->drop_q.offset - eth_size);
1336 eth->val.vlan_tag = spec->tci;
1337 eth->mask.vlan_tag = mask->tci;
1338 eth->val.vlan_tag &= eth->mask.vlan_tag;
1341 for (i = 0; i != hash_rxq_init_n; ++i) {
1342 if (!parser->queue[i].ibv_attr)
1345 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1346 parser->queue[i].offset - eth_size);
1347 eth->val.vlan_tag = spec->tci;
1348 eth->mask.vlan_tag = mask->tci;
1349 eth->val.vlan_tag &= eth->mask.vlan_tag;
1356 * Convert IPv4 item to Verbs specification.
1359 * Item specification.
1360 * @param default_mask[in]
1361 * Default bit-masks to use when item->mask is not provided.
1362 * @param data[in, out]
1366 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1367 const void *default_mask,
1370 const struct rte_flow_item_ipv4 *spec = item->spec;
1371 const struct rte_flow_item_ipv4 *mask = item->mask;
1372 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1373 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1374 struct ibv_flow_spec_ipv4_ext ipv4 = {
1375 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1379 parser->layer = HASH_RXQ_IPV4;
1382 mask = default_mask;
1383 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1384 .src_ip = spec->hdr.src_addr,
1385 .dst_ip = spec->hdr.dst_addr,
1386 .proto = spec->hdr.next_proto_id,
1387 .tos = spec->hdr.type_of_service,
1389 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1390 .src_ip = mask->hdr.src_addr,
1391 .dst_ip = mask->hdr.dst_addr,
1392 .proto = mask->hdr.next_proto_id,
1393 .tos = mask->hdr.type_of_service,
1395 /* Remove unwanted bits from values. */
1396 ipv4.val.src_ip &= ipv4.mask.src_ip;
1397 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1398 ipv4.val.proto &= ipv4.mask.proto;
1399 ipv4.val.tos &= ipv4.mask.tos;
1401 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1406 * Convert IPv6 item to Verbs specification.
1409 * Item specification.
1410 * @param default_mask[in]
1411 * Default bit-masks to use when item->mask is not provided.
1412 * @param data[in, out]
1416 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1417 const void *default_mask,
1420 const struct rte_flow_item_ipv6 *spec = item->spec;
1421 const struct rte_flow_item_ipv6 *mask = item->mask;
1422 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1423 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1424 struct ibv_flow_spec_ipv6 ipv6 = {
1425 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1429 parser->layer = HASH_RXQ_IPV6;
1434 mask = default_mask;
1435 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1436 RTE_DIM(ipv6.val.src_ip));
1437 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1438 RTE_DIM(ipv6.val.dst_ip));
1439 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1440 RTE_DIM(ipv6.mask.src_ip));
1441 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1442 RTE_DIM(ipv6.mask.dst_ip));
1443 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1444 ipv6.mask.next_hdr = mask->hdr.proto;
1445 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1446 /* Remove unwanted bits from values. */
1447 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1448 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1449 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1451 ipv6.val.flow_label &= ipv6.mask.flow_label;
1452 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1453 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1455 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1460 * Convert UDP item to Verbs specification.
1463 * Item specification.
1464 * @param default_mask[in]
1465 * Default bit-masks to use when item->mask is not provided.
1466 * @param data[in, out]
1470 mlx5_flow_create_udp(const struct rte_flow_item *item,
1471 const void *default_mask,
1474 const struct rte_flow_item_udp *spec = item->spec;
1475 const struct rte_flow_item_udp *mask = item->mask;
1476 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1477 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1478 struct ibv_flow_spec_tcp_udp udp = {
1479 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1483 if (parser->layer == HASH_RXQ_IPV4)
1484 parser->layer = HASH_RXQ_UDPV4;
1486 parser->layer = HASH_RXQ_UDPV6;
1489 mask = default_mask;
1490 udp.val.dst_port = spec->hdr.dst_port;
1491 udp.val.src_port = spec->hdr.src_port;
1492 udp.mask.dst_port = mask->hdr.dst_port;
1493 udp.mask.src_port = mask->hdr.src_port;
1494 /* Remove unwanted bits from values. */
1495 udp.val.src_port &= udp.mask.src_port;
1496 udp.val.dst_port &= udp.mask.dst_port;
1498 mlx5_flow_create_copy(parser, &udp, udp_size);
1503 * Convert TCP item to Verbs specification.
1506 * Item specification.
1507 * @param default_mask[in]
1508 * Default bit-masks to use when item->mask is not provided.
1509 * @param data[in, out]
1513 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1514 const void *default_mask,
1517 const struct rte_flow_item_tcp *spec = item->spec;
1518 const struct rte_flow_item_tcp *mask = item->mask;
1519 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1520 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1521 struct ibv_flow_spec_tcp_udp tcp = {
1522 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1526 if (parser->layer == HASH_RXQ_IPV4)
1527 parser->layer = HASH_RXQ_TCPV4;
1529 parser->layer = HASH_RXQ_TCPV6;
1532 mask = default_mask;
1533 tcp.val.dst_port = spec->hdr.dst_port;
1534 tcp.val.src_port = spec->hdr.src_port;
1535 tcp.mask.dst_port = mask->hdr.dst_port;
1536 tcp.mask.src_port = mask->hdr.src_port;
1537 /* Remove unwanted bits from values. */
1538 tcp.val.src_port &= tcp.mask.src_port;
1539 tcp.val.dst_port &= tcp.mask.dst_port;
1541 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1546 * Convert VXLAN item to Verbs specification.
1549 * Item specification.
1550 * @param default_mask[in]
1551 * Default bit-masks to use when item->mask is not provided.
1552 * @param data[in, out]
1556 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1557 const void *default_mask,
1560 const struct rte_flow_item_vxlan *spec = item->spec;
1561 const struct rte_flow_item_vxlan *mask = item->mask;
1562 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1563 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1564 struct ibv_flow_spec_tunnel vxlan = {
1565 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1574 parser->inner = IBV_FLOW_SPEC_INNER;
1577 mask = default_mask;
1578 memcpy(&id.vni[1], spec->vni, 3);
1579 vxlan.val.tunnel_id = id.vlan_id;
1580 memcpy(&id.vni[1], mask->vni, 3);
1581 vxlan.mask.tunnel_id = id.vlan_id;
1582 /* Remove unwanted bits from values. */
1583 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1585 mlx5_flow_create_copy(parser, &vxlan, size);
1590 * Convert mark/flag action to Verbs specification.
1593 * Internal parser structure.
1598 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1600 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1601 struct ibv_flow_spec_action_tag tag = {
1602 .type = IBV_FLOW_SPEC_ACTION_TAG,
1604 .tag_id = mlx5_flow_mark_set(mark_id),
1607 assert(parser->mark);
1608 mlx5_flow_create_copy(parser, &tag, size);
1613 * Convert count action to Verbs specification.
1616 * Pointer to private structure.
1618 * Pointer to MLX5 flow parser structure.
1621 * 0 on success, errno value on failure.
1624 mlx5_flow_create_count(struct priv *priv __rte_unused,
1625 struct mlx5_flow_parse *parser __rte_unused)
1627 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1628 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1629 struct ibv_counter_set_init_attr init_attr = {0};
1630 struct ibv_flow_spec_counter_action counter = {
1631 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1633 .counter_set_handle = 0,
1636 init_attr.counter_set_id = 0;
1637 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr);
1640 counter.counter_set_handle = parser->cs->handle;
1641 mlx5_flow_create_copy(parser, &counter, size);
1647 * Complete flow rule creation with a drop queue.
1650 * Pointer to private structure.
1652 * Internal parser structure.
1654 * Pointer to the rte_flow.
1656 * Perform verbose error reporting if not NULL.
1659 * 0 on success, errno value on failure.
1662 priv_flow_create_action_queue_drop(struct priv *priv,
1663 struct mlx5_flow_parse *parser,
1664 struct rte_flow *flow,
1665 struct rte_flow_error *error)
1667 struct ibv_flow_spec_action_drop *drop;
1668 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1674 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1675 parser->drop_q.offset);
1676 *drop = (struct ibv_flow_spec_action_drop){
1677 .type = IBV_FLOW_SPEC_ACTION_DROP,
1680 ++parser->drop_q.ibv_attr->num_of_specs;
1681 parser->drop_q.offset += size;
1682 if (!priv->dev->data->dev_started)
1684 flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
1685 parser->drop_q.ibv_attr = NULL;
1686 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
1687 flow->drxq.ibv_attr);
1689 flow->cs = parser->cs;
1690 if (!flow->drxq.ibv_flow) {
1691 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1692 NULL, "flow rule creation failure");
1699 if (flow->drxq.ibv_flow) {
1700 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1701 flow->drxq.ibv_flow = NULL;
1703 if (flow->drxq.ibv_attr) {
1704 rte_free(flow->drxq.ibv_attr);
1705 flow->drxq.ibv_attr = NULL;
1708 claim_zero(ibv_destroy_counter_set(flow->cs));
1716 * Create hash Rx queues when RSS is enabled.
1719 * Pointer to private structure.
1721 * Internal parser structure.
1723 * Pointer to the rte_flow.
1725 * Perform verbose error reporting if not NULL.
1728 * 0 on success, a errno value otherwise and rte_errno is set.
1731 priv_flow_create_action_queue_rss(struct priv *priv,
1732 struct mlx5_flow_parse *parser,
1733 struct rte_flow *flow,
1734 struct rte_flow_error *error)
1738 for (i = 0; i != hash_rxq_init_n; ++i) {
1739 uint64_t hash_fields;
1741 if (!parser->queue[i].ibv_attr)
1743 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1744 parser->queue[i].ibv_attr = NULL;
1745 hash_fields = hash_rxq_init[i].hash_fields;
1746 flow->frxq[i].hrxq =
1747 mlx5_priv_hrxq_get(priv,
1748 parser->rss_conf.rss_key,
1749 parser->rss_conf.rss_key_len,
1752 hash_fields ? parser->queues_n : 1);
1753 if (flow->frxq[i].hrxq)
1755 flow->frxq[i].hrxq =
1756 mlx5_priv_hrxq_new(priv,
1757 parser->rss_conf.rss_key,
1758 parser->rss_conf.rss_key_len,
1761 hash_fields ? parser->queues_n : 1);
1762 if (!flow->frxq[i].hrxq) {
1763 rte_flow_error_set(error, ENOMEM,
1764 RTE_FLOW_ERROR_TYPE_HANDLE,
1765 NULL, "cannot create hash rxq");
1773 * Complete flow rule creation.
1776 * Pointer to private structure.
1778 * Internal parser structure.
1780 * Pointer to the rte_flow.
1782 * Perform verbose error reporting if not NULL.
1785 * 0 on success, a errno value otherwise and rte_errno is set.
1788 priv_flow_create_action_queue(struct priv *priv,
1789 struct mlx5_flow_parse *parser,
1790 struct rte_flow *flow,
1791 struct rte_flow_error *error)
1798 assert(!parser->drop);
1799 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1803 flow->cs = parser->cs;
1804 if (!priv->dev->data->dev_started)
1806 for (i = 0; i != hash_rxq_init_n; ++i) {
1807 if (!flow->frxq[i].hrxq)
1809 flow->frxq[i].ibv_flow =
1810 ibv_create_flow(flow->frxq[i].hrxq->qp,
1811 flow->frxq[i].ibv_attr);
1812 if (!flow->frxq[i].ibv_flow) {
1813 rte_flow_error_set(error, ENOMEM,
1814 RTE_FLOW_ERROR_TYPE_HANDLE,
1815 NULL, "flow rule creation failure");
1819 DEBUG("%p type %d QP %p ibv_flow %p",
1821 (void *)flow->frxq[i].hrxq,
1822 (void *)flow->frxq[i].ibv_flow);
1824 for (i = 0; i != parser->queues_n; ++i) {
1825 struct mlx5_rxq_data *q =
1826 (*priv->rxqs)[parser->queues[i]];
1828 q->mark |= parser->mark;
1833 for (i = 0; i != hash_rxq_init_n; ++i) {
1834 if (flow->frxq[i].ibv_flow) {
1835 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1837 claim_zero(ibv_destroy_flow(ibv_flow));
1839 if (flow->frxq[i].hrxq)
1840 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1841 if (flow->frxq[i].ibv_attr)
1842 rte_free(flow->frxq[i].ibv_attr);
1845 claim_zero(ibv_destroy_counter_set(flow->cs));
1856 * Pointer to private structure.
1858 * Pointer to a TAILQ flow list.
1860 * Flow rule attributes.
1861 * @param[in] pattern
1862 * Pattern specification (list terminated by the END pattern item).
1863 * @param[in] actions
1864 * Associated actions (list terminated by the END action).
1866 * Perform verbose error reporting if not NULL.
1869 * A flow on success, NULL otherwise.
1871 static struct rte_flow *
1872 priv_flow_create(struct priv *priv,
1873 struct mlx5_flows *list,
1874 const struct rte_flow_attr *attr,
1875 const struct rte_flow_item items[],
1876 const struct rte_flow_action actions[],
1877 struct rte_flow_error *error)
1879 struct mlx5_flow_parse parser = { .create = 1, };
1880 struct rte_flow *flow = NULL;
1884 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1887 flow = rte_calloc(__func__, 1,
1888 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1891 rte_flow_error_set(error, ENOMEM,
1892 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1894 "cannot allocate flow memory");
1897 /* Copy queues configuration. */
1898 flow->queues = (uint16_t (*)[])(flow + 1);
1899 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1900 flow->queues_n = parser.queues_n;
1901 /* Copy RSS configuration. */
1902 flow->rss_conf = parser.rss_conf;
1903 flow->rss_conf.rss_key = flow->rss_key;
1904 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1905 /* finalise the flow. */
1907 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1910 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1913 TAILQ_INSERT_TAIL(list, flow, next);
1914 DEBUG("Flow created %p", (void *)flow);
1918 rte_free(parser.drop_q.ibv_attr);
1920 for (i = 0; i != hash_rxq_init_n; ++i) {
1921 if (parser.queue[i].ibv_attr)
1922 rte_free(parser.queue[i].ibv_attr);
1930 * Validate a flow supported by the NIC.
1932 * @see rte_flow_validate()
1936 mlx5_flow_validate(struct rte_eth_dev *dev,
1937 const struct rte_flow_attr *attr,
1938 const struct rte_flow_item items[],
1939 const struct rte_flow_action actions[],
1940 struct rte_flow_error *error)
1942 struct priv *priv = dev->data->dev_private;
1944 struct mlx5_flow_parse parser = { .create = 0, };
1947 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1955 * @see rte_flow_create()
1959 mlx5_flow_create(struct rte_eth_dev *dev,
1960 const struct rte_flow_attr *attr,
1961 const struct rte_flow_item items[],
1962 const struct rte_flow_action actions[],
1963 struct rte_flow_error *error)
1965 struct priv *priv = dev->data->dev_private;
1966 struct rte_flow *flow;
1969 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1979 * Pointer to private structure.
1981 * Pointer to a TAILQ flow list.
1986 priv_flow_destroy(struct priv *priv,
1987 struct mlx5_flows *list,
1988 struct rte_flow *flow)
1993 claim_zero(ibv_destroy_counter_set(flow->cs));
1996 if (flow->drop || !flow->mark)
1998 for (i = 0; i != flow->queues_n; ++i) {
1999 struct rte_flow *tmp;
2003 * To remove the mark from the queue, the queue must not be
2004 * present in any other marked flow (RSS or not).
2006 TAILQ_FOREACH(tmp, list, next) {
2008 uint16_t *tqs = NULL;
2013 for (j = 0; j != hash_rxq_init_n; ++j) {
2014 if (!tmp->frxq[j].hrxq)
2016 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2017 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2021 for (j = 0; (j != tq_n) && !mark; j++)
2022 if (tqs[j] == (*flow->queues)[i])
2025 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2029 if (flow->drxq.ibv_flow)
2030 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2031 rte_free(flow->drxq.ibv_attr);
2033 for (i = 0; i != hash_rxq_init_n; ++i) {
2034 struct mlx5_flow *frxq = &flow->frxq[i];
2037 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
2039 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2041 rte_free(frxq->ibv_attr);
2044 TAILQ_REMOVE(list, flow, next);
2045 DEBUG("Flow destroyed %p", (void *)flow);
2050 * Destroy all flows.
2053 * Pointer to private structure.
2055 * Pointer to a TAILQ flow list.
2058 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2060 while (!TAILQ_EMPTY(list)) {
2061 struct rte_flow *flow;
2063 flow = TAILQ_FIRST(list);
2064 priv_flow_destroy(priv, list, flow);
2069 * Create drop queue.
2072 * Pointer to private structure.
2078 priv_flow_create_drop_queue(struct priv *priv)
2080 struct mlx5_hrxq_drop *fdq = NULL;
2084 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2086 WARN("cannot allocate memory for drop queue");
2089 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
2091 WARN("cannot allocate CQ for drop queue");
2094 fdq->wq = ibv_create_wq(priv->ctx,
2095 &(struct ibv_wq_init_attr){
2096 .wq_type = IBV_WQT_RQ,
2103 WARN("cannot allocate WQ for drop queue");
2106 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
2107 &(struct ibv_rwq_ind_table_init_attr){
2108 .log_ind_tbl_size = 0,
2109 .ind_tbl = &fdq->wq,
2112 if (!fdq->ind_table) {
2113 WARN("cannot allocate indirection table for drop queue");
2116 fdq->qp = ibv_create_qp_ex(priv->ctx,
2117 &(struct ibv_qp_init_attr_ex){
2118 .qp_type = IBV_QPT_RAW_PACKET,
2120 IBV_QP_INIT_ATTR_PD |
2121 IBV_QP_INIT_ATTR_IND_TABLE |
2122 IBV_QP_INIT_ATTR_RX_HASH,
2123 .rx_hash_conf = (struct ibv_rx_hash_conf){
2125 IBV_RX_HASH_FUNC_TOEPLITZ,
2126 .rx_hash_key_len = rss_hash_default_key_len,
2127 .rx_hash_key = rss_hash_default_key,
2128 .rx_hash_fields_mask = 0,
2130 .rwq_ind_tbl = fdq->ind_table,
2134 WARN("cannot allocate QP for drop queue");
2137 priv->flow_drop_queue = fdq;
2141 claim_zero(ibv_destroy_qp(fdq->qp));
2143 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2145 claim_zero(ibv_destroy_wq(fdq->wq));
2147 claim_zero(ibv_destroy_cq(fdq->cq));
2150 priv->flow_drop_queue = NULL;
2155 * Delete drop queue.
2158 * Pointer to private structure.
2161 priv_flow_delete_drop_queue(struct priv *priv)
2163 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2168 claim_zero(ibv_destroy_qp(fdq->qp));
2170 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2172 claim_zero(ibv_destroy_wq(fdq->wq));
2174 claim_zero(ibv_destroy_cq(fdq->cq));
2176 priv->flow_drop_queue = NULL;
2183 * Pointer to private structure.
2185 * Pointer to a TAILQ flow list.
2188 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2190 struct rte_flow *flow;
2192 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2196 if (!flow->drxq.ibv_flow)
2198 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2199 flow->drxq.ibv_flow = NULL;
2204 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2206 for (i = 0; i != hash_rxq_init_n; ++i) {
2207 if (!flow->frxq[i].hrxq)
2209 ind_tbl = flow->frxq[i].hrxq->ind_table;
2212 for (i = 0; i != ind_tbl->queues_n; ++i)
2213 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2215 for (i = 0; i != hash_rxq_init_n; ++i) {
2216 if (!flow->frxq[i].ibv_flow)
2218 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2219 flow->frxq[i].ibv_flow = NULL;
2220 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2221 flow->frxq[i].hrxq = NULL;
2223 DEBUG("Flow %p removed", (void *)flow);
2231 * Pointer to private structure.
2233 * Pointer to a TAILQ flow list.
2236 * 0 on success, a errno value otherwise and rte_errno is set.
2239 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2241 struct rte_flow *flow;
2243 TAILQ_FOREACH(flow, list, next) {
2247 flow->drxq.ibv_flow =
2248 ibv_create_flow(priv->flow_drop_queue->qp,
2249 flow->drxq.ibv_attr);
2250 if (!flow->drxq.ibv_flow) {
2251 DEBUG("Flow %p cannot be applied",
2256 DEBUG("Flow %p applied", (void *)flow);
2260 for (i = 0; i != hash_rxq_init_n; ++i) {
2261 if (!flow->frxq[i].ibv_attr)
2263 flow->frxq[i].hrxq =
2264 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2265 flow->rss_conf.rss_key_len,
2266 hash_rxq_init[i].hash_fields,
2269 if (flow->frxq[i].hrxq)
2271 flow->frxq[i].hrxq =
2272 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2273 flow->rss_conf.rss_key_len,
2274 hash_rxq_init[i].hash_fields,
2277 if (!flow->frxq[i].hrxq) {
2278 DEBUG("Flow %p cannot be applied",
2284 flow->frxq[i].ibv_flow =
2285 ibv_create_flow(flow->frxq[i].hrxq->qp,
2286 flow->frxq[i].ibv_attr);
2287 if (!flow->frxq[i].ibv_flow) {
2288 DEBUG("Flow %p cannot be applied",
2293 DEBUG("Flow %p applied", (void *)flow);
2297 for (i = 0; i != flow->queues_n; ++i)
2298 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2304 * Verify the flow list is empty
2307 * Pointer to private structure.
2309 * @return the number of flows not released.
2312 priv_flow_verify(struct priv *priv)
2314 struct rte_flow *flow;
2317 TAILQ_FOREACH(flow, &priv->flows, next) {
2318 DEBUG("%p: flow %p still referenced", (void *)priv,
2326 * Enable a control flow configured from the control plane.
2329 * Pointer to Ethernet device.
2331 * An Ethernet flow spec to apply.
2333 * An Ethernet flow mask to apply.
2335 * A VLAN flow spec to apply.
2337 * A VLAN flow mask to apply.
2343 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2344 struct rte_flow_item_eth *eth_spec,
2345 struct rte_flow_item_eth *eth_mask,
2346 struct rte_flow_item_vlan *vlan_spec,
2347 struct rte_flow_item_vlan *vlan_mask)
2349 struct priv *priv = dev->data->dev_private;
2350 const struct rte_flow_attr attr = {
2352 .priority = MLX5_CTRL_FLOW_PRIORITY,
2354 struct rte_flow_item items[] = {
2356 .type = RTE_FLOW_ITEM_TYPE_ETH,
2362 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2363 RTE_FLOW_ITEM_TYPE_END,
2369 .type = RTE_FLOW_ITEM_TYPE_END,
2372 struct rte_flow_action actions[] = {
2374 .type = RTE_FLOW_ACTION_TYPE_RSS,
2377 .type = RTE_FLOW_ACTION_TYPE_END,
2380 struct rte_flow *flow;
2381 struct rte_flow_error error;
2384 struct rte_flow_action_rss rss;
2386 const struct rte_eth_rss_conf *rss_conf;
2388 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2392 if (!priv->reta_idx_n)
2394 for (i = 0; i != priv->reta_idx_n; ++i)
2395 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2396 action_rss.local.rss_conf = &priv->rss_conf;
2397 action_rss.local.num = priv->reta_idx_n;
2398 actions[0].conf = (const void *)&action_rss.rss;
2399 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2407 * Enable a flow control configured from the control plane.
2410 * Pointer to Ethernet device.
2412 * An Ethernet flow spec to apply.
2414 * An Ethernet flow mask to apply.
2420 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2421 struct rte_flow_item_eth *eth_spec,
2422 struct rte_flow_item_eth *eth_mask)
2424 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2430 * @see rte_flow_destroy()
2434 mlx5_flow_destroy(struct rte_eth_dev *dev,
2435 struct rte_flow *flow,
2436 struct rte_flow_error *error)
2438 struct priv *priv = dev->data->dev_private;
2442 priv_flow_destroy(priv, &priv->flows, flow);
2448 * Destroy all flows.
2450 * @see rte_flow_flush()
2454 mlx5_flow_flush(struct rte_eth_dev *dev,
2455 struct rte_flow_error *error)
2457 struct priv *priv = dev->data->dev_private;
2461 priv_flow_flush(priv, &priv->flows);
2466 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2468 * Query flow counter.
2472 * @param counter_value
2473 * returned data from the counter.
2476 * 0 on success, a errno value otherwise and rte_errno is set.
2479 priv_flow_query_count(struct ibv_counter_set *cs,
2480 struct mlx5_flow_counter_stats *counter_stats,
2481 struct rte_flow_query_count *query_count,
2482 struct rte_flow_error *error)
2484 uint64_t counters[2];
2485 struct ibv_query_counter_set_attr query_cs_attr = {
2487 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2489 struct ibv_counter_set_data query_out = {
2491 .outlen = 2 * sizeof(uint64_t),
2493 int res = ibv_query_counter_set(&query_cs_attr, &query_out);
2496 rte_flow_error_set(error, -res,
2497 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2499 "cannot read counter");
2502 query_count->hits_set = 1;
2503 query_count->bytes_set = 1;
2504 query_count->hits = counters[0] - counter_stats->hits;
2505 query_count->bytes = counters[1] - counter_stats->bytes;
2506 if (query_count->reset) {
2507 counter_stats->hits = counters[0];
2508 counter_stats->bytes = counters[1];
2516 * @see rte_flow_query()
2520 mlx5_flow_query(struct rte_eth_dev *dev,
2521 struct rte_flow *flow,
2522 enum rte_flow_action_type action __rte_unused,
2524 struct rte_flow_error *error)
2526 struct priv *priv = dev->data->dev_private;
2531 res = priv_flow_query_count(flow->cs,
2532 &flow->counter_stats,
2533 (struct rte_flow_query_count *)data,
2536 rte_flow_error_set(error, res,
2537 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2539 "no counter found for flow");
2549 * @see rte_flow_isolate()
2553 mlx5_flow_isolate(struct rte_eth_dev *dev,
2555 struct rte_flow_error *error)
2557 struct priv *priv = dev->data->dev_private;
2560 if (dev->data->dev_started) {
2561 rte_flow_error_set(error, EBUSY,
2562 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2564 "port must be stopped first");
2568 priv->isolated = !!enable;
2570 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2572 priv->dev->dev_ops = &mlx5_dev_ops;
2578 * Convert a flow director filter to a generic flow.
2581 * Private structure.
2582 * @param fdir_filter
2583 * Flow director filter to add.
2585 * Generic flow parameters structure.
2588 * 0 on success, errno value on error.
2591 priv_fdir_filter_convert(struct priv *priv,
2592 const struct rte_eth_fdir_filter *fdir_filter,
2593 struct mlx5_fdir *attributes)
2595 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2597 /* Validate queue number. */
2598 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2599 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2602 /* Validate the behavior. */
2603 if (fdir_filter->action.behavior != RTE_ETH_FDIR_ACCEPT) {
2604 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2607 attributes->attr.ingress = 1;
2608 attributes->items[0] = (struct rte_flow_item) {
2609 .type = RTE_FLOW_ITEM_TYPE_ETH,
2610 .spec = &attributes->l2,
2612 attributes->actions[0] = (struct rte_flow_action){
2613 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2614 .conf = &attributes->queue,
2616 attributes->queue.index = fdir_filter->action.rx_queue;
2617 switch (fdir_filter->input.flow_type) {
2618 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2619 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2620 .src_addr = input->flow.udp4_flow.ip.src_ip,
2621 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2622 .time_to_live = input->flow.udp4_flow.ip.ttl,
2623 .type_of_service = input->flow.udp4_flow.ip.tos,
2624 .next_proto_id = input->flow.udp4_flow.ip.proto,
2626 attributes->l4.udp.hdr = (struct udp_hdr){
2627 .src_port = input->flow.udp4_flow.src_port,
2628 .dst_port = input->flow.udp4_flow.dst_port,
2630 attributes->items[1] = (struct rte_flow_item){
2631 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2632 .spec = &attributes->l3,
2634 attributes->items[2] = (struct rte_flow_item){
2635 .type = RTE_FLOW_ITEM_TYPE_UDP,
2636 .spec = &attributes->l4,
2639 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2640 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2641 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2642 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2643 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2644 .type_of_service = input->flow.tcp4_flow.ip.tos,
2645 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2647 attributes->l4.tcp.hdr = (struct tcp_hdr){
2648 .src_port = input->flow.tcp4_flow.src_port,
2649 .dst_port = input->flow.tcp4_flow.dst_port,
2651 attributes->items[1] = (struct rte_flow_item){
2652 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2653 .spec = &attributes->l3,
2655 attributes->items[2] = (struct rte_flow_item){
2656 .type = RTE_FLOW_ITEM_TYPE_TCP,
2657 .spec = &attributes->l4,
2660 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2661 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2662 .src_addr = input->flow.ip4_flow.src_ip,
2663 .dst_addr = input->flow.ip4_flow.dst_ip,
2664 .time_to_live = input->flow.ip4_flow.ttl,
2665 .type_of_service = input->flow.ip4_flow.tos,
2666 .next_proto_id = input->flow.ip4_flow.proto,
2668 attributes->items[1] = (struct rte_flow_item){
2669 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2670 .spec = &attributes->l3,
2673 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2674 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2675 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2676 .proto = input->flow.udp6_flow.ip.proto,
2678 memcpy(attributes->l3.ipv6.hdr.src_addr,
2679 input->flow.udp6_flow.ip.src_ip,
2680 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2681 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2682 input->flow.udp6_flow.ip.dst_ip,
2683 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2684 attributes->l4.udp.hdr = (struct udp_hdr){
2685 .src_port = input->flow.udp6_flow.src_port,
2686 .dst_port = input->flow.udp6_flow.dst_port,
2688 attributes->items[1] = (struct rte_flow_item){
2689 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2690 .spec = &attributes->l3,
2692 attributes->items[2] = (struct rte_flow_item){
2693 .type = RTE_FLOW_ITEM_TYPE_UDP,
2694 .spec = &attributes->l4,
2697 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2698 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2699 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2700 .proto = input->flow.tcp6_flow.ip.proto,
2702 memcpy(attributes->l3.ipv6.hdr.src_addr,
2703 input->flow.tcp6_flow.ip.src_ip,
2704 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2705 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2706 input->flow.tcp6_flow.ip.dst_ip,
2707 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2708 attributes->l4.tcp.hdr = (struct tcp_hdr){
2709 .src_port = input->flow.tcp6_flow.src_port,
2710 .dst_port = input->flow.tcp6_flow.dst_port,
2712 attributes->items[1] = (struct rte_flow_item){
2713 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2714 .spec = &attributes->l3,
2716 attributes->items[2] = (struct rte_flow_item){
2717 .type = RTE_FLOW_ITEM_TYPE_UDP,
2718 .spec = &attributes->l4,
2721 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2722 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2723 .hop_limits = input->flow.ipv6_flow.hop_limits,
2724 .proto = input->flow.ipv6_flow.proto,
2726 memcpy(attributes->l3.ipv6.hdr.src_addr,
2727 input->flow.ipv6_flow.src_ip,
2728 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2729 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2730 input->flow.ipv6_flow.dst_ip,
2731 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2732 attributes->items[1] = (struct rte_flow_item){
2733 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2734 .spec = &attributes->l3,
2738 ERROR("invalid flow type%d",
2739 fdir_filter->input.flow_type);
2746 * Add new flow director filter and store it in list.
2749 * Private structure.
2750 * @param fdir_filter
2751 * Flow director filter to add.
2754 * 0 on success, errno value on failure.
2757 priv_fdir_filter_add(struct priv *priv,
2758 const struct rte_eth_fdir_filter *fdir_filter)
2760 struct mlx5_fdir attributes = {
2763 struct mlx5_flow_parse parser = {
2764 .layer = HASH_RXQ_ETH,
2766 struct rte_flow_error error;
2767 struct rte_flow *flow;
2770 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2773 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2774 attributes.actions, &error, &parser);
2777 flow = priv_flow_create(priv,
2784 TAILQ_INSERT_TAIL(&priv->flows, flow, next);
2785 DEBUG("FDIR created %p", (void *)flow);
2792 * Delete specific filter.
2795 * Private structure.
2796 * @param fdir_filter
2797 * Filter to be deleted.
2800 * 0 on success, errno value on failure.
2803 priv_fdir_filter_delete(struct priv *priv,
2804 const struct rte_eth_fdir_filter *fdir_filter)
2806 struct mlx5_fdir attributes;
2807 struct mlx5_flow_parse parser = {
2809 .layer = HASH_RXQ_ETH,
2811 struct rte_flow_error error;
2812 struct rte_flow *flow;
2816 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2819 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2820 attributes.actions, &error, &parser);
2823 TAILQ_FOREACH(flow, &priv->flows, next) {
2824 struct ibv_flow_attr *attr;
2825 struct ibv_spec_header *attr_h;
2827 struct ibv_flow_attr *flow_attr;
2828 struct ibv_spec_header *flow_h;
2830 unsigned int specs_n;
2833 attr = parser.drop_q.ibv_attr;
2835 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2837 flow_attr = flow->drxq.ibv_attr;
2839 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2840 /* Compare first the attributes. */
2841 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2843 if (attr->num_of_specs == 0)
2845 spec = (void *)((uintptr_t)attr +
2846 sizeof(struct ibv_flow_attr));
2847 flow_spec = (void *)((uintptr_t)flow_attr +
2848 sizeof(struct ibv_flow_attr));
2849 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2850 for (i = 0; i != specs_n; ++i) {
2853 if (memcmp(spec, flow_spec,
2854 RTE_MIN(attr_h->size, flow_h->size)))
2856 spec = (void *)((uintptr_t)attr + attr_h->size);
2857 flow_spec = (void *)((uintptr_t)flow_attr +
2860 /* At this point, the flow match. */
2864 priv_flow_destroy(priv, &priv->flows, flow);
2867 rte_free(parser.drop_q.ibv_attr);
2869 for (i = 0; i != hash_rxq_init_n; ++i) {
2870 if (parser.queue[i].ibv_attr)
2871 rte_free(parser.queue[i].ibv_attr);
2878 * Update queue for specific filter.
2881 * Private structure.
2882 * @param fdir_filter
2883 * Filter to be updated.
2886 * 0 on success, errno value on failure.
2889 priv_fdir_filter_update(struct priv *priv,
2890 const struct rte_eth_fdir_filter *fdir_filter)
2894 ret = priv_fdir_filter_delete(priv, fdir_filter);
2897 ret = priv_fdir_filter_add(priv, fdir_filter);
2902 * Flush all filters.
2905 * Private structure.
2908 priv_fdir_filter_flush(struct priv *priv)
2910 priv_flow_flush(priv, &priv->flows);
2914 * Get flow director information.
2917 * Private structure.
2918 * @param[out] fdir_info
2919 * Resulting flow director information.
2922 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2924 struct rte_eth_fdir_masks *mask =
2925 &priv->dev->data->dev_conf.fdir_conf.mask;
2927 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2928 fdir_info->guarant_spc = 0;
2929 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2930 fdir_info->max_flexpayload = 0;
2931 fdir_info->flow_types_mask[0] = 0;
2932 fdir_info->flex_payload_unit = 0;
2933 fdir_info->max_flex_payload_segment_num = 0;
2934 fdir_info->flex_payload_limit = 0;
2935 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2939 * Deal with flow director operations.
2942 * Pointer to private structure.
2944 * Operation to perform.
2946 * Pointer to operation-specific structure.
2949 * 0 on success, errno value on failure.
2952 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2954 enum rte_fdir_mode fdir_mode =
2955 priv->dev->data->dev_conf.fdir_conf.mode;
2958 if (filter_op == RTE_ETH_FILTER_NOP)
2960 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2961 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2962 ERROR("%p: flow director mode %d not supported",
2963 (void *)priv, fdir_mode);
2966 switch (filter_op) {
2967 case RTE_ETH_FILTER_ADD:
2968 ret = priv_fdir_filter_add(priv, arg);
2970 case RTE_ETH_FILTER_UPDATE:
2971 ret = priv_fdir_filter_update(priv, arg);
2973 case RTE_ETH_FILTER_DELETE:
2974 ret = priv_fdir_filter_delete(priv, arg);
2976 case RTE_ETH_FILTER_FLUSH:
2977 priv_fdir_filter_flush(priv);
2979 case RTE_ETH_FILTER_INFO:
2980 priv_fdir_info_get(priv, arg);
2983 DEBUG("%p: unknown operation %u", (void *)priv,
2992 * Manage filter operations.
2995 * Pointer to Ethernet device structure.
2996 * @param filter_type
2999 * Operation to perform.
3001 * Pointer to operation-specific structure.
3004 * 0 on success, negative errno value on failure.
3007 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3008 enum rte_filter_type filter_type,
3009 enum rte_filter_op filter_op,
3013 struct priv *priv = dev->data->dev_private;
3015 switch (filter_type) {
3016 case RTE_ETH_FILTER_GENERIC:
3017 if (filter_op != RTE_ETH_FILTER_GET)
3019 *(const void **)arg = &mlx5_flow_ops;
3021 case RTE_ETH_FILTER_FDIR:
3023 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3027 ERROR("%p: filter type (%d) not supported",
3028 (void *)dev, filter_type);