4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
63 struct ibv_counter_set_init_attr {
66 struct ibv_flow_spec_counter_action {
69 struct ibv_counter_set {
74 ibv_destroy_counter_set(struct ibv_counter_set *cs)
81 /* Dev ops structure defined in mlx5.c */
82 extern const struct eth_dev_ops mlx5_dev_ops;
83 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
86 mlx5_flow_create_eth(const struct rte_flow_item *item,
87 const void *default_mask,
91 mlx5_flow_create_vlan(const struct rte_flow_item *item,
92 const void *default_mask,
96 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
97 const void *default_mask,
101 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
102 const void *default_mask,
106 mlx5_flow_create_udp(const struct rte_flow_item *item,
107 const void *default_mask,
111 mlx5_flow_create_tcp(const struct rte_flow_item *item,
112 const void *default_mask,
116 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
117 const void *default_mask,
120 struct mlx5_flow_parse;
123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
132 /* Hash RX queue types. */
143 /* Initialization data for hash RX queue. */
144 struct hash_rxq_init {
145 uint64_t hash_fields; /* Fields that participate in the hash. */
146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
147 unsigned int flow_priority; /* Flow priority to use. */
148 unsigned int ip_version; /* Internet protocol. */
151 /* Initialization data for hash RX queues. */
152 const struct hash_rxq_init hash_rxq_init[] = {
154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
155 IBV_RX_HASH_DST_IPV4 |
156 IBV_RX_HASH_SRC_PORT_TCP |
157 IBV_RX_HASH_DST_PORT_TCP),
158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
160 .ip_version = MLX5_IPV4,
163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
164 IBV_RX_HASH_DST_IPV4 |
165 IBV_RX_HASH_SRC_PORT_UDP |
166 IBV_RX_HASH_DST_PORT_UDP),
167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
169 .ip_version = MLX5_IPV4,
172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
173 IBV_RX_HASH_DST_IPV4),
174 .dpdk_rss_hf = (ETH_RSS_IPV4 |
177 .ip_version = MLX5_IPV4,
180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
181 IBV_RX_HASH_DST_IPV6 |
182 IBV_RX_HASH_SRC_PORT_TCP |
183 IBV_RX_HASH_DST_PORT_TCP),
184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
186 .ip_version = MLX5_IPV6,
189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
190 IBV_RX_HASH_DST_IPV6 |
191 IBV_RX_HASH_SRC_PORT_UDP |
192 IBV_RX_HASH_DST_PORT_UDP),
193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
195 .ip_version = MLX5_IPV6,
198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
199 IBV_RX_HASH_DST_IPV6),
200 .dpdk_rss_hf = (ETH_RSS_IPV6 |
203 .ip_version = MLX5_IPV6,
212 /* Number of entries in hash_rxq_init[]. */
213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
215 /** Structure for holding counter stats. */
216 struct mlx5_flow_counter_stats {
217 uint64_t hits; /**< Number of packets matched by the rule. */
218 uint64_t bytes; /**< Number of bytes matched by the rule. */
221 /** Structure for Drop queue. */
222 struct mlx5_hrxq_drop {
223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
224 struct ibv_qp *qp; /**< Verbs queue pair. */
225 struct ibv_wq *wq; /**< Verbs work queue. */
226 struct ibv_cq *cq; /**< Verbs completion queue. */
229 /* Flows structures. */
231 uint64_t hash_fields; /**< Fields that participate in the hash. */
232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
233 struct ibv_flow *ibv_flow; /**< Verbs flow. */
234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
237 /* Drop flows structures. */
238 struct mlx5_flow_drop {
239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
240 struct ibv_flow *ibv_flow; /**< Verbs flow. */
244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
245 uint32_t mark:1; /**< Set if the flow is marked. */
246 uint32_t drop:1; /**< Drop queue. */
247 uint16_t queues_n; /**< Number of entries in queue[]. */
248 uint16_t (*queues)[]; /**< Queues indexes to use. */
249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
250 uint8_t rss_key[40]; /**< copy of the RSS key. */
251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
255 /**< Flow with Rx queue. */
256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */
260 /** Static initializer for items. */
262 (const enum rte_flow_item_type []){ \
263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
266 /** Structure to generate a simple graph of layers supported by the NIC. */
267 struct mlx5_flow_items {
268 /** List of possible actions for these items. */
269 const enum rte_flow_action_type *const actions;
270 /** Bit-masks corresponding to the possibilities for the item. */
273 * Default bit-masks to use when item->mask is not provided. When
274 * \default_mask is also NULL, the full supported bit-mask (\mask) is
277 const void *default_mask;
278 /** Bit-masks size in bytes. */
279 const unsigned int mask_sz;
281 * Conversion function from rte_flow to NIC specific flow.
284 * rte_flow item to convert.
285 * @param default_mask
286 * Default bit-masks to use when item->mask is not provided.
288 * Internal structure to store the conversion.
291 * 0 on success, negative value otherwise.
293 int (*convert)(const struct rte_flow_item *item,
294 const void *default_mask,
296 /** Size in bytes of the destination structure. */
297 const unsigned int dst_sz;
298 /** List of possible following items. */
299 const enum rte_flow_item_type *const items;
302 /** Valid action for this PMD. */
303 static const enum rte_flow_action_type valid_actions[] = {
304 RTE_FLOW_ACTION_TYPE_DROP,
305 RTE_FLOW_ACTION_TYPE_QUEUE,
306 RTE_FLOW_ACTION_TYPE_MARK,
307 RTE_FLOW_ACTION_TYPE_FLAG,
308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
309 RTE_FLOW_ACTION_TYPE_COUNT,
311 RTE_FLOW_ACTION_TYPE_END,
314 /** Graph of supported items and associated actions. */
315 static const struct mlx5_flow_items mlx5_flow_items[] = {
316 [RTE_FLOW_ITEM_TYPE_END] = {
317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
318 RTE_FLOW_ITEM_TYPE_VXLAN),
320 [RTE_FLOW_ITEM_TYPE_ETH] = {
321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
322 RTE_FLOW_ITEM_TYPE_IPV4,
323 RTE_FLOW_ITEM_TYPE_IPV6),
324 .actions = valid_actions,
325 .mask = &(const struct rte_flow_item_eth){
326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
330 .default_mask = &rte_flow_item_eth_mask,
331 .mask_sz = sizeof(struct rte_flow_item_eth),
332 .convert = mlx5_flow_create_eth,
333 .dst_sz = sizeof(struct ibv_flow_spec_eth),
335 [RTE_FLOW_ITEM_TYPE_VLAN] = {
336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
337 RTE_FLOW_ITEM_TYPE_IPV6),
338 .actions = valid_actions,
339 .mask = &(const struct rte_flow_item_vlan){
342 .default_mask = &rte_flow_item_vlan_mask,
343 .mask_sz = sizeof(struct rte_flow_item_vlan),
344 .convert = mlx5_flow_create_vlan,
347 [RTE_FLOW_ITEM_TYPE_IPV4] = {
348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
349 RTE_FLOW_ITEM_TYPE_TCP),
350 .actions = valid_actions,
351 .mask = &(const struct rte_flow_item_ipv4){
355 .type_of_service = -1,
359 .default_mask = &rte_flow_item_ipv4_mask,
360 .mask_sz = sizeof(struct rte_flow_item_ipv4),
361 .convert = mlx5_flow_create_ipv4,
362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
364 [RTE_FLOW_ITEM_TYPE_IPV6] = {
365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
366 RTE_FLOW_ITEM_TYPE_TCP),
367 .actions = valid_actions,
368 .mask = &(const struct rte_flow_item_ipv6){
371 0xff, 0xff, 0xff, 0xff,
372 0xff, 0xff, 0xff, 0xff,
373 0xff, 0xff, 0xff, 0xff,
374 0xff, 0xff, 0xff, 0xff,
377 0xff, 0xff, 0xff, 0xff,
378 0xff, 0xff, 0xff, 0xff,
379 0xff, 0xff, 0xff, 0xff,
380 0xff, 0xff, 0xff, 0xff,
387 .default_mask = &rte_flow_item_ipv6_mask,
388 .mask_sz = sizeof(struct rte_flow_item_ipv6),
389 .convert = mlx5_flow_create_ipv6,
390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
392 [RTE_FLOW_ITEM_TYPE_UDP] = {
393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
394 .actions = valid_actions,
395 .mask = &(const struct rte_flow_item_udp){
401 .default_mask = &rte_flow_item_udp_mask,
402 .mask_sz = sizeof(struct rte_flow_item_udp),
403 .convert = mlx5_flow_create_udp,
404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
406 [RTE_FLOW_ITEM_TYPE_TCP] = {
407 .actions = valid_actions,
408 .mask = &(const struct rte_flow_item_tcp){
414 .default_mask = &rte_flow_item_tcp_mask,
415 .mask_sz = sizeof(struct rte_flow_item_tcp),
416 .convert = mlx5_flow_create_tcp,
417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
419 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
421 .actions = valid_actions,
422 .mask = &(const struct rte_flow_item_vxlan){
423 .vni = "\xff\xff\xff",
425 .default_mask = &rte_flow_item_vxlan_mask,
426 .mask_sz = sizeof(struct rte_flow_item_vxlan),
427 .convert = mlx5_flow_create_vxlan,
428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
432 /** Structure to pass to the conversion function. */
433 struct mlx5_flow_parse {
434 uint32_t inner; /**< Set once VXLAN is encountered. */
436 /**< Whether resources should remain after a validate. */
437 uint32_t drop:1; /**< Target is a drop queue. */
438 uint32_t mark:1; /**< Mark is present in the flow. */
439 uint32_t count:1; /**< Count is present in the flow. */
440 uint32_t mark_id; /**< Mark identifier. */
441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
442 uint16_t queues_n; /**< Number of entries in queue[]. */
443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
444 uint8_t rss_key[40]; /**< copy of the RSS key. */
445 enum hash_rxq_type layer; /**< Last pattern layer detected. */
446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
449 struct ibv_flow_attr *ibv_attr;
450 /**< Pointer to Verbs attributes. */
452 /**< Current position or total size of the attribute. */
453 } queue[RTE_DIM(hash_rxq_init)];
455 struct ibv_flow_attr *ibv_attr;
456 /**< Pointer to Verbs attributes. */
458 /**< Current position or total size of the attribute. */
463 static const struct rte_flow_ops mlx5_flow_ops = {
464 .validate = mlx5_flow_validate,
465 .create = mlx5_flow_create,
466 .destroy = mlx5_flow_destroy,
467 .flush = mlx5_flow_flush,
468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
469 .query = mlx5_flow_query,
473 .isolate = mlx5_flow_isolate,
476 /* Convert FDIR request to Generic flow. */
478 struct rte_flow_attr attr;
479 struct rte_flow_action actions[2];
480 struct rte_flow_item items[4];
481 struct rte_flow_item_eth l2;
483 struct rte_flow_item_ipv4 ipv4;
484 struct rte_flow_item_ipv6 ipv6;
487 struct rte_flow_item_udp udp;
488 struct rte_flow_item_tcp tcp;
490 struct rte_flow_action_queue queue;
493 /* Verbs specification header. */
494 struct ibv_spec_header {
495 enum ibv_flow_spec_type type;
500 * Check support for a given item.
503 * Item specification.
505 * Bit-masks covering supported fields to compare with spec, last and mask in
508 * Bit-Mask size in bytes.
514 mlx5_flow_item_validate(const struct rte_flow_item *item,
515 const uint8_t *mask, unsigned int size)
519 if (!item->spec && (item->mask || item->last))
521 if (item->spec && !item->mask) {
523 const uint8_t *spec = item->spec;
525 for (i = 0; i < size; ++i)
526 if ((spec[i] | mask[i]) != mask[i])
529 if (item->last && !item->mask) {
531 const uint8_t *spec = item->last;
533 for (i = 0; i < size; ++i)
534 if ((spec[i] | mask[i]) != mask[i])
539 const uint8_t *spec = item->mask;
541 for (i = 0; i < size; ++i)
542 if ((spec[i] | mask[i]) != mask[i])
545 if (item->spec && item->last) {
548 const uint8_t *apply = mask;
553 for (i = 0; i < size; ++i) {
554 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
555 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
557 ret = memcmp(spec, last, size);
563 * Copy the RSS configuration from the user ones.
566 * Pointer to private structure.
568 * Internal parser structure.
570 * User RSS configuration to save.
573 * 0 on success, errno value on failure.
576 priv_flow_convert_rss_conf(struct priv *priv,
577 struct mlx5_flow_parse *parser,
578 const struct rte_eth_rss_conf *rss_conf)
580 const struct rte_eth_rss_conf *rss =
581 rss_conf ? rss_conf : &priv->rss_conf;
583 if (rss->rss_key_len > 40)
585 parser->rss_conf.rss_key_len = rss->rss_key_len;
586 parser->rss_conf.rss_hf = rss->rss_hf;
587 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
588 parser->rss_conf.rss_key = parser->rss_key;
593 * Extract attribute to the parser.
596 * Pointer to private structure.
598 * Flow rule attributes.
600 * Perform verbose error reporting if not NULL.
601 * @param[in, out] parser
602 * Internal parser structure.
605 * 0 on success, a negative errno value otherwise and rte_errno is set.
608 priv_flow_convert_attributes(struct priv *priv,
609 const struct rte_flow_attr *attr,
610 struct rte_flow_error *error,
611 struct mlx5_flow_parse *parser)
616 rte_flow_error_set(error, ENOTSUP,
617 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
619 "groups are not supported");
622 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
623 rte_flow_error_set(error, ENOTSUP,
624 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
626 "priorities are not supported");
630 rte_flow_error_set(error, ENOTSUP,
631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
633 "egress is not supported");
636 if (!attr->ingress) {
637 rte_flow_error_set(error, ENOTSUP,
638 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
640 "only ingress is supported");
647 * Extract actions request to the parser.
650 * Pointer to private structure.
652 * Associated actions (list terminated by the END action).
654 * Perform verbose error reporting if not NULL.
655 * @param[in, out] parser
656 * Internal parser structure.
659 * 0 on success, a negative errno value otherwise and rte_errno is set.
662 priv_flow_convert_actions(struct priv *priv,
663 const struct rte_flow_action actions[],
664 struct rte_flow_error *error,
665 struct mlx5_flow_parse *parser)
668 * Add default RSS configuration necessary for Verbs to create QP even
669 * if no RSS is necessary.
671 priv_flow_convert_rss_conf(priv, parser,
672 (const struct rte_eth_rss_conf *)
674 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
675 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
677 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
679 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
680 const struct rte_flow_action_queue *queue =
681 (const struct rte_flow_action_queue *)
686 if (!queue || (queue->index > (priv->rxqs_n - 1)))
687 goto exit_action_not_supported;
688 for (n = 0; n < parser->queues_n; ++n) {
689 if (parser->queues[n] == queue->index) {
694 if (parser->queues_n > 1 && !found) {
695 rte_flow_error_set(error, ENOTSUP,
696 RTE_FLOW_ERROR_TYPE_ACTION,
698 "queue action not in RSS queues");
702 parser->queues_n = 1;
703 parser->queues[0] = queue->index;
705 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
706 const struct rte_flow_action_rss *rss =
707 (const struct rte_flow_action_rss *)
711 if (!rss || !rss->num) {
712 rte_flow_error_set(error, EINVAL,
713 RTE_FLOW_ERROR_TYPE_ACTION,
718 if (parser->queues_n == 1) {
721 assert(parser->queues_n);
722 for (n = 0; n < rss->num; ++n) {
723 if (parser->queues[0] ==
730 rte_flow_error_set(error, ENOTSUP,
731 RTE_FLOW_ERROR_TYPE_ACTION,
733 "queue action not in RSS"
738 for (n = 0; n < rss->num; ++n) {
739 if (rss->queue[n] >= priv->rxqs_n) {
740 rte_flow_error_set(error, EINVAL,
741 RTE_FLOW_ERROR_TYPE_ACTION,
743 "queue id > number of"
748 for (n = 0; n < rss->num; ++n)
749 parser->queues[n] = rss->queue[n];
750 parser->queues_n = rss->num;
751 if (priv_flow_convert_rss_conf(priv, parser,
753 rte_flow_error_set(error, EINVAL,
754 RTE_FLOW_ERROR_TYPE_ACTION,
756 "wrong RSS configuration");
759 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
760 const struct rte_flow_action_mark *mark =
761 (const struct rte_flow_action_mark *)
765 rte_flow_error_set(error, EINVAL,
766 RTE_FLOW_ERROR_TYPE_ACTION,
768 "mark must be defined");
770 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
771 rte_flow_error_set(error, ENOTSUP,
772 RTE_FLOW_ERROR_TYPE_ACTION,
774 "mark must be between 0"
779 parser->mark_id = mark->id;
780 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
782 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
783 priv->counter_set_supported) {
786 goto exit_action_not_supported;
789 if (parser->drop && parser->mark)
791 if (!parser->queues_n && !parser->drop) {
792 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
793 NULL, "no valid action");
797 exit_action_not_supported:
798 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
799 actions, "action not supported");
807 * Pointer to private structure.
809 * Pattern specification (list terminated by the END pattern item).
811 * Perform verbose error reporting if not NULL.
812 * @param[in, out] parser
813 * Internal parser structure.
816 * 0 on success, a negative errno value otherwise and rte_errno is set.
819 priv_flow_convert_items_validate(struct priv *priv,
820 const struct rte_flow_item items[],
821 struct rte_flow_error *error,
822 struct mlx5_flow_parse *parser)
824 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
828 /* Initialise the offsets to start after verbs attribute. */
830 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
832 for (i = 0; i != hash_rxq_init_n; ++i)
833 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
835 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
836 const struct mlx5_flow_items *token = NULL;
840 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
844 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
846 if (cur_item->items[i] == items->type) {
847 token = &mlx5_flow_items[items->type];
852 goto exit_item_not_supported;
854 err = mlx5_flow_item_validate(items,
855 (const uint8_t *)cur_item->mask,
858 goto exit_item_not_supported;
859 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
861 rte_flow_error_set(error, ENOTSUP,
862 RTE_FLOW_ERROR_TYPE_ITEM,
864 "cannot recognize multiple"
865 " VXLAN encapsulations");
871 parser->drop_q.offset += cur_item->dst_sz;
872 } else if (parser->queues_n == 1) {
873 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
875 for (n = 0; n != hash_rxq_init_n; ++n)
876 parser->queue[n].offset += cur_item->dst_sz;
880 for (i = 0; i != hash_rxq_init_n; ++i)
881 parser->queue[i].offset +=
882 sizeof(struct ibv_flow_spec_action_tag);
885 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
888 parser->drop_q.offset += size;
890 for (i = 0; i != hash_rxq_init_n; ++i)
891 parser->queue[i].offset += size;
895 exit_item_not_supported:
896 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
897 items, "item not supported");
902 * Allocate memory space to store verbs flow attributes.
905 * Pointer to private structure.
906 * @param[in] priority
909 * Amount of byte to allocate.
911 * Perform verbose error reporting if not NULL.
914 * A verbs flow attribute on success, NULL otherwise.
916 static struct ibv_flow_attr*
917 priv_flow_convert_allocate(struct priv *priv,
918 unsigned int priority,
920 struct rte_flow_error *error)
922 struct ibv_flow_attr *ibv_attr;
925 ibv_attr = rte_calloc(__func__, 1, size, 0);
927 rte_flow_error_set(error, ENOMEM,
928 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
930 "cannot allocate verbs spec attributes.");
933 ibv_attr->priority = priority;
938 * Finalise verbs flow attributes.
941 * Pointer to private structure.
942 * @param[in, out] parser
943 * Internal parser structure.
946 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
948 const unsigned int ipv4 =
949 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
950 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
951 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
952 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
953 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
954 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
958 if (parser->layer == HASH_RXQ_ETH) {
962 * This layer becomes useless as the pattern define under
965 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
966 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
968 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
969 for (i = ohmin; i != (ohmax + 1); ++i) {
970 if (!parser->queue[i].ibv_attr)
972 rte_free(parser->queue[i].ibv_attr);
973 parser->queue[i].ibv_attr = NULL;
975 /* Remove impossible flow according to the RSS configuration. */
976 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
977 parser->rss_conf.rss_hf) {
978 /* Remove any other flow. */
979 for (i = hmin; i != (hmax + 1); ++i) {
980 if ((i == parser->layer) ||
981 (!parser->queue[i].ibv_attr))
983 rte_free(parser->queue[i].ibv_attr);
984 parser->queue[i].ibv_attr = NULL;
986 } else if (!parser->queue[ip].ibv_attr) {
987 /* no RSS possible with the current configuration. */
988 parser->queues_n = 1;
993 * Fill missing layers in verbs specifications, or compute the correct
994 * offset to allocate the memory space for the attributes and
997 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
999 struct ibv_flow_spec_ipv4_ext ipv4;
1000 struct ibv_flow_spec_ipv6 ipv6;
1001 struct ibv_flow_spec_tcp_udp udp_tcp;
1006 if (i == parser->layer)
1008 if (parser->layer == HASH_RXQ_ETH) {
1009 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
1010 size = sizeof(struct ibv_flow_spec_ipv4_ext);
1011 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
1012 .type = IBV_FLOW_SPEC_IPV4_EXT |
1017 size = sizeof(struct ibv_flow_spec_ipv6);
1018 specs.ipv6 = (struct ibv_flow_spec_ipv6){
1019 .type = IBV_FLOW_SPEC_IPV6 |
1024 if (parser->queue[i].ibv_attr) {
1025 dst = (void *)((uintptr_t)
1026 parser->queue[i].ibv_attr +
1027 parser->queue[i].offset);
1028 memcpy(dst, &specs, size);
1029 ++parser->queue[i].ibv_attr->num_of_specs;
1031 parser->queue[i].offset += size;
1033 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1034 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1035 size = sizeof(struct ibv_flow_spec_tcp_udp);
1036 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1037 .type = ((i == HASH_RXQ_UDPV4 ||
1038 i == HASH_RXQ_UDPV6) ?
1040 IBV_FLOW_SPEC_TCP) |
1044 if (parser->queue[i].ibv_attr) {
1045 dst = (void *)((uintptr_t)
1046 parser->queue[i].ibv_attr +
1047 parser->queue[i].offset);
1048 memcpy(dst, &specs, size);
1049 ++parser->queue[i].ibv_attr->num_of_specs;
1051 parser->queue[i].offset += size;
1057 * Validate and convert a flow supported by the NIC.
1060 * Pointer to private structure.
1062 * Flow rule attributes.
1063 * @param[in] pattern
1064 * Pattern specification (list terminated by the END pattern item).
1065 * @param[in] actions
1066 * Associated actions (list terminated by the END action).
1068 * Perform verbose error reporting if not NULL.
1069 * @param[in, out] parser
1070 * Internal parser structure.
1073 * 0 on success, a negative errno value otherwise and rte_errno is set.
1076 priv_flow_convert(struct priv *priv,
1077 const struct rte_flow_attr *attr,
1078 const struct rte_flow_item items[],
1079 const struct rte_flow_action actions[],
1080 struct rte_flow_error *error,
1081 struct mlx5_flow_parse *parser)
1083 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1087 /* First step. Validate the attributes, items and actions. */
1088 *parser = (struct mlx5_flow_parse){
1089 .create = parser->create,
1090 .layer = HASH_RXQ_ETH,
1091 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1093 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1096 ret = priv_flow_convert_actions(priv, actions, error, parser);
1099 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1102 priv_flow_convert_finalise(priv, parser);
1105 * Allocate the memory space to store verbs specifications.
1108 parser->drop_q.ibv_attr =
1109 priv_flow_convert_allocate(priv, attr->priority,
1110 parser->drop_q.offset,
1112 if (!parser->drop_q.ibv_attr)
1114 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
1115 } else if (parser->queues_n == 1) {
1116 unsigned int priority =
1118 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1119 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1121 parser->queue[HASH_RXQ_ETH].ibv_attr =
1122 priv_flow_convert_allocate(priv, priority,
1124 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1126 parser->queue[HASH_RXQ_ETH].offset =
1127 sizeof(struct ibv_flow_attr);
1129 for (i = 0; i != hash_rxq_init_n; ++i) {
1130 unsigned int priority =
1132 hash_rxq_init[i].flow_priority;
1133 unsigned int offset;
1135 if (!(parser->rss_conf.rss_hf &
1136 hash_rxq_init[i].dpdk_rss_hf) &&
1137 (i != HASH_RXQ_ETH))
1139 offset = parser->queue[i].offset;
1140 parser->queue[i].ibv_attr =
1141 priv_flow_convert_allocate(priv, priority,
1143 if (!parser->queue[i].ibv_attr)
1145 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1148 /* Third step. Conversion parse, fill the specifications. */
1150 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1151 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1153 cur_item = &mlx5_flow_items[items->type];
1154 ret = cur_item->convert(items,
1155 (cur_item->default_mask ?
1156 cur_item->default_mask :
1160 rte_flow_error_set(error, ENOTSUP,
1161 RTE_FLOW_ERROR_TYPE_ITEM,
1162 items, "item not supported");
1167 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1168 if (parser->count && parser->create) {
1169 mlx5_flow_create_count(priv, parser);
1171 goto exit_count_error;
1174 * Last step. Complete missing specification to reach the RSS
1177 if (parser->queues_n > 1) {
1178 priv_flow_convert_finalise(priv, parser);
1179 } else if (!parser->drop) {
1181 * Action queue have their priority overridden with
1182 * Ethernet priority, this priority needs to be adjusted to
1183 * their most specific layer priority.
1185 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1187 hash_rxq_init[parser->layer].flow_priority;
1190 /* Only verification is expected, all resources should be released. */
1191 if (!parser->create) {
1193 rte_free(parser->drop_q.ibv_attr);
1194 parser->drop_q.ibv_attr = NULL;
1196 for (i = 0; i != hash_rxq_init_n; ++i) {
1197 if (parser->queue[i].ibv_attr) {
1198 rte_free(parser->queue[i].ibv_attr);
1199 parser->queue[i].ibv_attr = NULL;
1205 for (i = 0; i != hash_rxq_init_n; ++i) {
1206 if (parser->queue[i].ibv_attr) {
1207 rte_free(parser->queue[i].ibv_attr);
1208 parser->queue[i].ibv_attr = NULL;
1211 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1212 NULL, "cannot allocate verbs spec attributes.");
1215 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1216 NULL, "cannot create counter.");
1221 * Copy the specification created into the flow.
1224 * Internal parser structure.
1226 * Create specification.
1228 * Size in bytes of the specification to copy.
1231 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1238 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1239 parser->drop_q.offset);
1240 memcpy(dst, src, size);
1241 ++parser->drop_q.ibv_attr->num_of_specs;
1242 parser->drop_q.offset += size;
1245 for (i = 0; i != hash_rxq_init_n; ++i) {
1246 if (!parser->queue[i].ibv_attr)
1248 /* Specification must be the same l3 type or none. */
1249 if (parser->layer == HASH_RXQ_ETH ||
1250 (hash_rxq_init[parser->layer].ip_version ==
1251 hash_rxq_init[i].ip_version) ||
1252 (hash_rxq_init[i].ip_version == 0)) {
1253 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1254 parser->queue[i].offset);
1255 memcpy(dst, src, size);
1256 ++parser->queue[i].ibv_attr->num_of_specs;
1257 parser->queue[i].offset += size;
1263 * Convert Ethernet item to Verbs specification.
1266 * Item specification.
1267 * @param default_mask[in]
1268 * Default bit-masks to use when item->mask is not provided.
1269 * @param data[in, out]
1273 mlx5_flow_create_eth(const struct rte_flow_item *item,
1274 const void *default_mask,
1277 const struct rte_flow_item_eth *spec = item->spec;
1278 const struct rte_flow_item_eth *mask = item->mask;
1279 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1280 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1281 struct ibv_flow_spec_eth eth = {
1282 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1286 parser->layer = HASH_RXQ_ETH;
1291 mask = default_mask;
1292 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1293 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1294 eth.val.ether_type = spec->type;
1295 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1296 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1297 eth.mask.ether_type = mask->type;
1298 /* Remove unwanted bits from values. */
1299 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1300 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1301 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1303 eth.val.ether_type &= eth.mask.ether_type;
1305 mlx5_flow_create_copy(parser, ð, eth_size);
1310 * Convert VLAN item to Verbs specification.
1313 * Item specification.
1314 * @param default_mask[in]
1315 * Default bit-masks to use when item->mask is not provided.
1316 * @param data[in, out]
1320 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1321 const void *default_mask,
1324 const struct rte_flow_item_vlan *spec = item->spec;
1325 const struct rte_flow_item_vlan *mask = item->mask;
1326 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1327 struct ibv_flow_spec_eth *eth;
1328 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1333 mask = default_mask;
1336 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1337 parser->drop_q.offset - eth_size);
1338 eth->val.vlan_tag = spec->tci;
1339 eth->mask.vlan_tag = mask->tci;
1340 eth->val.vlan_tag &= eth->mask.vlan_tag;
1343 for (i = 0; i != hash_rxq_init_n; ++i) {
1344 if (!parser->queue[i].ibv_attr)
1347 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1348 parser->queue[i].offset - eth_size);
1349 eth->val.vlan_tag = spec->tci;
1350 eth->mask.vlan_tag = mask->tci;
1351 eth->val.vlan_tag &= eth->mask.vlan_tag;
1358 * Convert IPv4 item to Verbs specification.
1361 * Item specification.
1362 * @param default_mask[in]
1363 * Default bit-masks to use when item->mask is not provided.
1364 * @param data[in, out]
1368 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1369 const void *default_mask,
1372 const struct rte_flow_item_ipv4 *spec = item->spec;
1373 const struct rte_flow_item_ipv4 *mask = item->mask;
1374 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1375 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1376 struct ibv_flow_spec_ipv4_ext ipv4 = {
1377 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1381 parser->layer = HASH_RXQ_IPV4;
1384 mask = default_mask;
1385 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1386 .src_ip = spec->hdr.src_addr,
1387 .dst_ip = spec->hdr.dst_addr,
1388 .proto = spec->hdr.next_proto_id,
1389 .tos = spec->hdr.type_of_service,
1391 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1392 .src_ip = mask->hdr.src_addr,
1393 .dst_ip = mask->hdr.dst_addr,
1394 .proto = mask->hdr.next_proto_id,
1395 .tos = mask->hdr.type_of_service,
1397 /* Remove unwanted bits from values. */
1398 ipv4.val.src_ip &= ipv4.mask.src_ip;
1399 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1400 ipv4.val.proto &= ipv4.mask.proto;
1401 ipv4.val.tos &= ipv4.mask.tos;
1403 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1408 * Convert IPv6 item to Verbs specification.
1411 * Item specification.
1412 * @param default_mask[in]
1413 * Default bit-masks to use when item->mask is not provided.
1414 * @param data[in, out]
1418 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1419 const void *default_mask,
1422 const struct rte_flow_item_ipv6 *spec = item->spec;
1423 const struct rte_flow_item_ipv6 *mask = item->mask;
1424 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1425 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1426 struct ibv_flow_spec_ipv6 ipv6 = {
1427 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1431 parser->layer = HASH_RXQ_IPV6;
1436 mask = default_mask;
1437 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1438 RTE_DIM(ipv6.val.src_ip));
1439 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1440 RTE_DIM(ipv6.val.dst_ip));
1441 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1442 RTE_DIM(ipv6.mask.src_ip));
1443 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1444 RTE_DIM(ipv6.mask.dst_ip));
1445 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1446 ipv6.mask.next_hdr = mask->hdr.proto;
1447 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1448 /* Remove unwanted bits from values. */
1449 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1450 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1451 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1453 ipv6.val.flow_label &= ipv6.mask.flow_label;
1454 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1455 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1457 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1462 * Convert UDP item to Verbs specification.
1465 * Item specification.
1466 * @param default_mask[in]
1467 * Default bit-masks to use when item->mask is not provided.
1468 * @param data[in, out]
1472 mlx5_flow_create_udp(const struct rte_flow_item *item,
1473 const void *default_mask,
1476 const struct rte_flow_item_udp *spec = item->spec;
1477 const struct rte_flow_item_udp *mask = item->mask;
1478 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1479 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1480 struct ibv_flow_spec_tcp_udp udp = {
1481 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1485 if (parser->layer == HASH_RXQ_IPV4)
1486 parser->layer = HASH_RXQ_UDPV4;
1488 parser->layer = HASH_RXQ_UDPV6;
1491 mask = default_mask;
1492 udp.val.dst_port = spec->hdr.dst_port;
1493 udp.val.src_port = spec->hdr.src_port;
1494 udp.mask.dst_port = mask->hdr.dst_port;
1495 udp.mask.src_port = mask->hdr.src_port;
1496 /* Remove unwanted bits from values. */
1497 udp.val.src_port &= udp.mask.src_port;
1498 udp.val.dst_port &= udp.mask.dst_port;
1500 mlx5_flow_create_copy(parser, &udp, udp_size);
1505 * Convert TCP item to Verbs specification.
1508 * Item specification.
1509 * @param default_mask[in]
1510 * Default bit-masks to use when item->mask is not provided.
1511 * @param data[in, out]
1515 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1516 const void *default_mask,
1519 const struct rte_flow_item_tcp *spec = item->spec;
1520 const struct rte_flow_item_tcp *mask = item->mask;
1521 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1522 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1523 struct ibv_flow_spec_tcp_udp tcp = {
1524 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1528 if (parser->layer == HASH_RXQ_IPV4)
1529 parser->layer = HASH_RXQ_TCPV4;
1531 parser->layer = HASH_RXQ_TCPV6;
1534 mask = default_mask;
1535 tcp.val.dst_port = spec->hdr.dst_port;
1536 tcp.val.src_port = spec->hdr.src_port;
1537 tcp.mask.dst_port = mask->hdr.dst_port;
1538 tcp.mask.src_port = mask->hdr.src_port;
1539 /* Remove unwanted bits from values. */
1540 tcp.val.src_port &= tcp.mask.src_port;
1541 tcp.val.dst_port &= tcp.mask.dst_port;
1543 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1548 * Convert VXLAN item to Verbs specification.
1551 * Item specification.
1552 * @param default_mask[in]
1553 * Default bit-masks to use when item->mask is not provided.
1554 * @param data[in, out]
1558 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1559 const void *default_mask,
1562 const struct rte_flow_item_vxlan *spec = item->spec;
1563 const struct rte_flow_item_vxlan *mask = item->mask;
1564 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1565 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1566 struct ibv_flow_spec_tunnel vxlan = {
1567 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1576 parser->inner = IBV_FLOW_SPEC_INNER;
1579 mask = default_mask;
1580 memcpy(&id.vni[1], spec->vni, 3);
1581 vxlan.val.tunnel_id = id.vlan_id;
1582 memcpy(&id.vni[1], mask->vni, 3);
1583 vxlan.mask.tunnel_id = id.vlan_id;
1584 /* Remove unwanted bits from values. */
1585 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1587 mlx5_flow_create_copy(parser, &vxlan, size);
1592 * Convert mark/flag action to Verbs specification.
1595 * Internal parser structure.
1600 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1602 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1603 struct ibv_flow_spec_action_tag tag = {
1604 .type = IBV_FLOW_SPEC_ACTION_TAG,
1606 .tag_id = mlx5_flow_mark_set(mark_id),
1609 assert(parser->mark);
1610 mlx5_flow_create_copy(parser, &tag, size);
1615 * Convert count action to Verbs specification.
1618 * Pointer to private structure.
1620 * Pointer to MLX5 flow parser structure.
1623 * 0 on success, errno value on failure.
1626 mlx5_flow_create_count(struct priv *priv __rte_unused,
1627 struct mlx5_flow_parse *parser __rte_unused)
1629 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1630 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1631 struct ibv_counter_set_init_attr init_attr = {0};
1632 struct ibv_flow_spec_counter_action counter = {
1633 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1635 .counter_set_handle = 0,
1638 init_attr.counter_set_id = 0;
1639 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr);
1642 counter.counter_set_handle = parser->cs->handle;
1643 mlx5_flow_create_copy(parser, &counter, size);
1649 * Complete flow rule creation with a drop queue.
1652 * Pointer to private structure.
1654 * Internal parser structure.
1656 * Pointer to the rte_flow.
1658 * Perform verbose error reporting if not NULL.
1661 * 0 on success, errno value on failure.
1664 priv_flow_create_action_queue_drop(struct priv *priv,
1665 struct mlx5_flow_parse *parser,
1666 struct rte_flow *flow,
1667 struct rte_flow_error *error)
1669 struct ibv_flow_spec_action_drop *drop;
1670 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1676 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1677 parser->drop_q.offset);
1678 *drop = (struct ibv_flow_spec_action_drop){
1679 .type = IBV_FLOW_SPEC_ACTION_DROP,
1682 ++parser->drop_q.ibv_attr->num_of_specs;
1683 parser->drop_q.offset += size;
1684 flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
1685 if (!priv->dev->data->dev_started)
1687 parser->drop_q.ibv_attr = NULL;
1688 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
1689 flow->drxq.ibv_attr);
1691 flow->cs = parser->cs;
1692 if (!flow->drxq.ibv_flow) {
1693 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1694 NULL, "flow rule creation failure");
1701 if (flow->drxq.ibv_flow) {
1702 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1703 flow->drxq.ibv_flow = NULL;
1705 if (flow->drxq.ibv_attr) {
1706 rte_free(flow->drxq.ibv_attr);
1707 flow->drxq.ibv_attr = NULL;
1710 claim_zero(ibv_destroy_counter_set(flow->cs));
1718 * Create hash Rx queues when RSS is enabled.
1721 * Pointer to private structure.
1723 * Internal parser structure.
1725 * Pointer to the rte_flow.
1727 * Perform verbose error reporting if not NULL.
1730 * 0 on success, a errno value otherwise and rte_errno is set.
1733 priv_flow_create_action_queue_rss(struct priv *priv,
1734 struct mlx5_flow_parse *parser,
1735 struct rte_flow *flow,
1736 struct rte_flow_error *error)
1740 for (i = 0; i != hash_rxq_init_n; ++i) {
1741 uint64_t hash_fields;
1743 if (!parser->queue[i].ibv_attr)
1745 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1746 parser->queue[i].ibv_attr = NULL;
1747 hash_fields = hash_rxq_init[i].hash_fields;
1748 if (!priv->dev->data->dev_started)
1750 flow->frxq[i].hrxq =
1751 mlx5_priv_hrxq_get(priv,
1752 parser->rss_conf.rss_key,
1753 parser->rss_conf.rss_key_len,
1756 hash_fields ? parser->queues_n : 1);
1757 if (flow->frxq[i].hrxq)
1759 flow->frxq[i].hrxq =
1760 mlx5_priv_hrxq_new(priv,
1761 parser->rss_conf.rss_key,
1762 parser->rss_conf.rss_key_len,
1765 hash_fields ? parser->queues_n : 1);
1766 if (!flow->frxq[i].hrxq) {
1767 rte_flow_error_set(error, ENOMEM,
1768 RTE_FLOW_ERROR_TYPE_HANDLE,
1769 NULL, "cannot create hash rxq");
1777 * Complete flow rule creation.
1780 * Pointer to private structure.
1782 * Internal parser structure.
1784 * Pointer to the rte_flow.
1786 * Perform verbose error reporting if not NULL.
1789 * 0 on success, a errno value otherwise and rte_errno is set.
1792 priv_flow_create_action_queue(struct priv *priv,
1793 struct mlx5_flow_parse *parser,
1794 struct rte_flow *flow,
1795 struct rte_flow_error *error)
1802 assert(!parser->drop);
1803 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1807 flow->cs = parser->cs;
1808 if (!priv->dev->data->dev_started)
1810 for (i = 0; i != hash_rxq_init_n; ++i) {
1811 if (!flow->frxq[i].hrxq)
1813 flow->frxq[i].ibv_flow =
1814 ibv_create_flow(flow->frxq[i].hrxq->qp,
1815 flow->frxq[i].ibv_attr);
1816 if (!flow->frxq[i].ibv_flow) {
1817 rte_flow_error_set(error, ENOMEM,
1818 RTE_FLOW_ERROR_TYPE_HANDLE,
1819 NULL, "flow rule creation failure");
1823 DEBUG("%p type %d QP %p ibv_flow %p",
1825 (void *)flow->frxq[i].hrxq,
1826 (void *)flow->frxq[i].ibv_flow);
1828 for (i = 0; i != parser->queues_n; ++i) {
1829 struct mlx5_rxq_data *q =
1830 (*priv->rxqs)[parser->queues[i]];
1832 q->mark |= parser->mark;
1837 for (i = 0; i != hash_rxq_init_n; ++i) {
1838 if (flow->frxq[i].ibv_flow) {
1839 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1841 claim_zero(ibv_destroy_flow(ibv_flow));
1843 if (flow->frxq[i].hrxq)
1844 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1845 if (flow->frxq[i].ibv_attr)
1846 rte_free(flow->frxq[i].ibv_attr);
1849 claim_zero(ibv_destroy_counter_set(flow->cs));
1860 * Pointer to private structure.
1862 * Pointer to a TAILQ flow list.
1864 * Flow rule attributes.
1865 * @param[in] pattern
1866 * Pattern specification (list terminated by the END pattern item).
1867 * @param[in] actions
1868 * Associated actions (list terminated by the END action).
1870 * Perform verbose error reporting if not NULL.
1873 * A flow on success, NULL otherwise.
1875 static struct rte_flow *
1876 priv_flow_create(struct priv *priv,
1877 struct mlx5_flows *list,
1878 const struct rte_flow_attr *attr,
1879 const struct rte_flow_item items[],
1880 const struct rte_flow_action actions[],
1881 struct rte_flow_error *error)
1883 struct mlx5_flow_parse parser = { .create = 1, };
1884 struct rte_flow *flow = NULL;
1888 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1891 flow = rte_calloc(__func__, 1,
1892 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1895 rte_flow_error_set(error, ENOMEM,
1896 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1898 "cannot allocate flow memory");
1901 /* Copy queues configuration. */
1902 flow->queues = (uint16_t (*)[])(flow + 1);
1903 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1904 flow->queues_n = parser.queues_n;
1905 /* Copy RSS configuration. */
1906 flow->rss_conf = parser.rss_conf;
1907 flow->rss_conf.rss_key = flow->rss_key;
1908 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1909 /* finalise the flow. */
1911 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1914 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1917 TAILQ_INSERT_TAIL(list, flow, next);
1918 DEBUG("Flow created %p", (void *)flow);
1922 rte_free(parser.drop_q.ibv_attr);
1924 for (i = 0; i != hash_rxq_init_n; ++i) {
1925 if (parser.queue[i].ibv_attr)
1926 rte_free(parser.queue[i].ibv_attr);
1934 * Validate a flow supported by the NIC.
1936 * @see rte_flow_validate()
1940 mlx5_flow_validate(struct rte_eth_dev *dev,
1941 const struct rte_flow_attr *attr,
1942 const struct rte_flow_item items[],
1943 const struct rte_flow_action actions[],
1944 struct rte_flow_error *error)
1946 struct priv *priv = dev->data->dev_private;
1948 struct mlx5_flow_parse parser = { .create = 0, };
1951 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1959 * @see rte_flow_create()
1963 mlx5_flow_create(struct rte_eth_dev *dev,
1964 const struct rte_flow_attr *attr,
1965 const struct rte_flow_item items[],
1966 const struct rte_flow_action actions[],
1967 struct rte_flow_error *error)
1969 struct priv *priv = dev->data->dev_private;
1970 struct rte_flow *flow;
1973 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1983 * Pointer to private structure.
1985 * Pointer to a TAILQ flow list.
1990 priv_flow_destroy(struct priv *priv,
1991 struct mlx5_flows *list,
1992 struct rte_flow *flow)
1997 claim_zero(ibv_destroy_counter_set(flow->cs));
2000 if (flow->drop || !flow->mark)
2002 for (i = 0; i != flow->queues_n; ++i) {
2003 struct rte_flow *tmp;
2007 * To remove the mark from the queue, the queue must not be
2008 * present in any other marked flow (RSS or not).
2010 TAILQ_FOREACH(tmp, list, next) {
2012 uint16_t *tqs = NULL;
2017 for (j = 0; j != hash_rxq_init_n; ++j) {
2018 if (!tmp->frxq[j].hrxq)
2020 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2021 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2025 for (j = 0; (j != tq_n) && !mark; j++)
2026 if (tqs[j] == (*flow->queues)[i])
2029 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2033 if (flow->drxq.ibv_flow)
2034 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2035 rte_free(flow->drxq.ibv_attr);
2037 for (i = 0; i != hash_rxq_init_n; ++i) {
2038 struct mlx5_flow *frxq = &flow->frxq[i];
2041 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
2043 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2045 rte_free(frxq->ibv_attr);
2048 TAILQ_REMOVE(list, flow, next);
2049 DEBUG("Flow destroyed %p", (void *)flow);
2054 * Destroy all flows.
2057 * Pointer to private structure.
2059 * Pointer to a TAILQ flow list.
2062 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2064 while (!TAILQ_EMPTY(list)) {
2065 struct rte_flow *flow;
2067 flow = TAILQ_FIRST(list);
2068 priv_flow_destroy(priv, list, flow);
2073 * Create drop queue.
2076 * Pointer to private structure.
2082 priv_flow_create_drop_queue(struct priv *priv)
2084 struct mlx5_hrxq_drop *fdq = NULL;
2088 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2090 WARN("cannot allocate memory for drop queue");
2093 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
2095 WARN("cannot allocate CQ for drop queue");
2098 fdq->wq = ibv_create_wq(priv->ctx,
2099 &(struct ibv_wq_init_attr){
2100 .wq_type = IBV_WQT_RQ,
2107 WARN("cannot allocate WQ for drop queue");
2110 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
2111 &(struct ibv_rwq_ind_table_init_attr){
2112 .log_ind_tbl_size = 0,
2113 .ind_tbl = &fdq->wq,
2116 if (!fdq->ind_table) {
2117 WARN("cannot allocate indirection table for drop queue");
2120 fdq->qp = ibv_create_qp_ex(priv->ctx,
2121 &(struct ibv_qp_init_attr_ex){
2122 .qp_type = IBV_QPT_RAW_PACKET,
2124 IBV_QP_INIT_ATTR_PD |
2125 IBV_QP_INIT_ATTR_IND_TABLE |
2126 IBV_QP_INIT_ATTR_RX_HASH,
2127 .rx_hash_conf = (struct ibv_rx_hash_conf){
2129 IBV_RX_HASH_FUNC_TOEPLITZ,
2130 .rx_hash_key_len = rss_hash_default_key_len,
2131 .rx_hash_key = rss_hash_default_key,
2132 .rx_hash_fields_mask = 0,
2134 .rwq_ind_tbl = fdq->ind_table,
2138 WARN("cannot allocate QP for drop queue");
2141 priv->flow_drop_queue = fdq;
2145 claim_zero(ibv_destroy_qp(fdq->qp));
2147 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2149 claim_zero(ibv_destroy_wq(fdq->wq));
2151 claim_zero(ibv_destroy_cq(fdq->cq));
2154 priv->flow_drop_queue = NULL;
2159 * Delete drop queue.
2162 * Pointer to private structure.
2165 priv_flow_delete_drop_queue(struct priv *priv)
2167 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2172 claim_zero(ibv_destroy_qp(fdq->qp));
2174 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2176 claim_zero(ibv_destroy_wq(fdq->wq));
2178 claim_zero(ibv_destroy_cq(fdq->cq));
2180 priv->flow_drop_queue = NULL;
2187 * Pointer to private structure.
2189 * Pointer to a TAILQ flow list.
2192 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2194 struct rte_flow *flow;
2196 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2200 if (!flow->drxq.ibv_flow)
2202 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2203 flow->drxq.ibv_flow = NULL;
2208 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2210 for (i = 0; i != hash_rxq_init_n; ++i) {
2211 if (!flow->frxq[i].hrxq)
2213 ind_tbl = flow->frxq[i].hrxq->ind_table;
2216 for (i = 0; i != ind_tbl->queues_n; ++i)
2217 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2219 for (i = 0; i != hash_rxq_init_n; ++i) {
2220 if (!flow->frxq[i].ibv_flow)
2222 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2223 flow->frxq[i].ibv_flow = NULL;
2224 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2225 flow->frxq[i].hrxq = NULL;
2227 DEBUG("Flow %p removed", (void *)flow);
2235 * Pointer to private structure.
2237 * Pointer to a TAILQ flow list.
2240 * 0 on success, a errno value otherwise and rte_errno is set.
2243 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2245 struct rte_flow *flow;
2247 TAILQ_FOREACH(flow, list, next) {
2251 flow->drxq.ibv_flow =
2252 ibv_create_flow(priv->flow_drop_queue->qp,
2253 flow->drxq.ibv_attr);
2254 if (!flow->drxq.ibv_flow) {
2255 DEBUG("Flow %p cannot be applied",
2260 DEBUG("Flow %p applied", (void *)flow);
2264 for (i = 0; i != hash_rxq_init_n; ++i) {
2265 if (!flow->frxq[i].ibv_attr)
2267 flow->frxq[i].hrxq =
2268 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2269 flow->rss_conf.rss_key_len,
2270 hash_rxq_init[i].hash_fields,
2273 if (flow->frxq[i].hrxq)
2275 flow->frxq[i].hrxq =
2276 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2277 flow->rss_conf.rss_key_len,
2278 hash_rxq_init[i].hash_fields,
2281 if (!flow->frxq[i].hrxq) {
2282 DEBUG("Flow %p cannot be applied",
2288 flow->frxq[i].ibv_flow =
2289 ibv_create_flow(flow->frxq[i].hrxq->qp,
2290 flow->frxq[i].ibv_attr);
2291 if (!flow->frxq[i].ibv_flow) {
2292 DEBUG("Flow %p cannot be applied",
2297 DEBUG("Flow %p applied", (void *)flow);
2301 for (i = 0; i != flow->queues_n; ++i)
2302 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2308 * Verify the flow list is empty
2311 * Pointer to private structure.
2313 * @return the number of flows not released.
2316 priv_flow_verify(struct priv *priv)
2318 struct rte_flow *flow;
2321 TAILQ_FOREACH(flow, &priv->flows, next) {
2322 DEBUG("%p: flow %p still referenced", (void *)priv,
2330 * Enable a control flow configured from the control plane.
2333 * Pointer to Ethernet device.
2335 * An Ethernet flow spec to apply.
2337 * An Ethernet flow mask to apply.
2339 * A VLAN flow spec to apply.
2341 * A VLAN flow mask to apply.
2347 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2348 struct rte_flow_item_eth *eth_spec,
2349 struct rte_flow_item_eth *eth_mask,
2350 struct rte_flow_item_vlan *vlan_spec,
2351 struct rte_flow_item_vlan *vlan_mask)
2353 struct priv *priv = dev->data->dev_private;
2354 const struct rte_flow_attr attr = {
2356 .priority = MLX5_CTRL_FLOW_PRIORITY,
2358 struct rte_flow_item items[] = {
2360 .type = RTE_FLOW_ITEM_TYPE_ETH,
2366 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2367 RTE_FLOW_ITEM_TYPE_END,
2373 .type = RTE_FLOW_ITEM_TYPE_END,
2376 struct rte_flow_action actions[] = {
2378 .type = RTE_FLOW_ACTION_TYPE_RSS,
2381 .type = RTE_FLOW_ACTION_TYPE_END,
2384 struct rte_flow *flow;
2385 struct rte_flow_error error;
2388 struct rte_flow_action_rss rss;
2390 const struct rte_eth_rss_conf *rss_conf;
2392 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2396 if (!priv->reta_idx_n)
2398 for (i = 0; i != priv->reta_idx_n; ++i)
2399 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2400 action_rss.local.rss_conf = &priv->rss_conf;
2401 action_rss.local.num = priv->reta_idx_n;
2402 actions[0].conf = (const void *)&action_rss.rss;
2403 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2411 * Enable a flow control configured from the control plane.
2414 * Pointer to Ethernet device.
2416 * An Ethernet flow spec to apply.
2418 * An Ethernet flow mask to apply.
2424 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2425 struct rte_flow_item_eth *eth_spec,
2426 struct rte_flow_item_eth *eth_mask)
2428 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2434 * @see rte_flow_destroy()
2438 mlx5_flow_destroy(struct rte_eth_dev *dev,
2439 struct rte_flow *flow,
2440 struct rte_flow_error *error)
2442 struct priv *priv = dev->data->dev_private;
2446 priv_flow_destroy(priv, &priv->flows, flow);
2452 * Destroy all flows.
2454 * @see rte_flow_flush()
2458 mlx5_flow_flush(struct rte_eth_dev *dev,
2459 struct rte_flow_error *error)
2461 struct priv *priv = dev->data->dev_private;
2465 priv_flow_flush(priv, &priv->flows);
2470 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2472 * Query flow counter.
2476 * @param counter_value
2477 * returned data from the counter.
2480 * 0 on success, a errno value otherwise and rte_errno is set.
2483 priv_flow_query_count(struct ibv_counter_set *cs,
2484 struct mlx5_flow_counter_stats *counter_stats,
2485 struct rte_flow_query_count *query_count,
2486 struct rte_flow_error *error)
2488 uint64_t counters[2];
2489 struct ibv_query_counter_set_attr query_cs_attr = {
2491 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2493 struct ibv_counter_set_data query_out = {
2495 .outlen = 2 * sizeof(uint64_t),
2497 int res = ibv_query_counter_set(&query_cs_attr, &query_out);
2500 rte_flow_error_set(error, -res,
2501 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2503 "cannot read counter");
2506 query_count->hits_set = 1;
2507 query_count->bytes_set = 1;
2508 query_count->hits = counters[0] - counter_stats->hits;
2509 query_count->bytes = counters[1] - counter_stats->bytes;
2510 if (query_count->reset) {
2511 counter_stats->hits = counters[0];
2512 counter_stats->bytes = counters[1];
2520 * @see rte_flow_query()
2524 mlx5_flow_query(struct rte_eth_dev *dev,
2525 struct rte_flow *flow,
2526 enum rte_flow_action_type action __rte_unused,
2528 struct rte_flow_error *error)
2530 struct priv *priv = dev->data->dev_private;
2535 res = priv_flow_query_count(flow->cs,
2536 &flow->counter_stats,
2537 (struct rte_flow_query_count *)data,
2540 rte_flow_error_set(error, res,
2541 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2543 "no counter found for flow");
2553 * @see rte_flow_isolate()
2557 mlx5_flow_isolate(struct rte_eth_dev *dev,
2559 struct rte_flow_error *error)
2561 struct priv *priv = dev->data->dev_private;
2564 if (dev->data->dev_started) {
2565 rte_flow_error_set(error, EBUSY,
2566 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2568 "port must be stopped first");
2572 priv->isolated = !!enable;
2574 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2576 priv->dev->dev_ops = &mlx5_dev_ops;
2582 * Convert a flow director filter to a generic flow.
2585 * Private structure.
2586 * @param fdir_filter
2587 * Flow director filter to add.
2589 * Generic flow parameters structure.
2592 * 0 on success, errno value on error.
2595 priv_fdir_filter_convert(struct priv *priv,
2596 const struct rte_eth_fdir_filter *fdir_filter,
2597 struct mlx5_fdir *attributes)
2599 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2601 /* Validate queue number. */
2602 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2603 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2606 attributes->attr.ingress = 1;
2607 attributes->items[0] = (struct rte_flow_item) {
2608 .type = RTE_FLOW_ITEM_TYPE_ETH,
2609 .spec = &attributes->l2,
2611 switch (fdir_filter->action.behavior) {
2612 case RTE_ETH_FDIR_ACCEPT:
2613 attributes->actions[0] = (struct rte_flow_action){
2614 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2615 .conf = &attributes->queue,
2618 case RTE_ETH_FDIR_REJECT:
2619 attributes->actions[0] = (struct rte_flow_action){
2620 .type = RTE_FLOW_ACTION_TYPE_DROP,
2624 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2627 attributes->queue.index = fdir_filter->action.rx_queue;
2628 switch (fdir_filter->input.flow_type) {
2629 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2630 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2631 .src_addr = input->flow.udp4_flow.ip.src_ip,
2632 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2633 .time_to_live = input->flow.udp4_flow.ip.ttl,
2634 .type_of_service = input->flow.udp4_flow.ip.tos,
2635 .next_proto_id = input->flow.udp4_flow.ip.proto,
2637 attributes->l4.udp.hdr = (struct udp_hdr){
2638 .src_port = input->flow.udp4_flow.src_port,
2639 .dst_port = input->flow.udp4_flow.dst_port,
2641 attributes->items[1] = (struct rte_flow_item){
2642 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2643 .spec = &attributes->l3,
2645 attributes->items[2] = (struct rte_flow_item){
2646 .type = RTE_FLOW_ITEM_TYPE_UDP,
2647 .spec = &attributes->l4,
2650 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2651 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2652 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2653 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2654 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2655 .type_of_service = input->flow.tcp4_flow.ip.tos,
2656 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2658 attributes->l4.tcp.hdr = (struct tcp_hdr){
2659 .src_port = input->flow.tcp4_flow.src_port,
2660 .dst_port = input->flow.tcp4_flow.dst_port,
2662 attributes->items[1] = (struct rte_flow_item){
2663 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2664 .spec = &attributes->l3,
2666 attributes->items[2] = (struct rte_flow_item){
2667 .type = RTE_FLOW_ITEM_TYPE_TCP,
2668 .spec = &attributes->l4,
2671 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2672 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2673 .src_addr = input->flow.ip4_flow.src_ip,
2674 .dst_addr = input->flow.ip4_flow.dst_ip,
2675 .time_to_live = input->flow.ip4_flow.ttl,
2676 .type_of_service = input->flow.ip4_flow.tos,
2677 .next_proto_id = input->flow.ip4_flow.proto,
2679 attributes->items[1] = (struct rte_flow_item){
2680 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2681 .spec = &attributes->l3,
2684 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2685 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2686 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2687 .proto = input->flow.udp6_flow.ip.proto,
2689 memcpy(attributes->l3.ipv6.hdr.src_addr,
2690 input->flow.udp6_flow.ip.src_ip,
2691 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2692 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2693 input->flow.udp6_flow.ip.dst_ip,
2694 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2695 attributes->l4.udp.hdr = (struct udp_hdr){
2696 .src_port = input->flow.udp6_flow.src_port,
2697 .dst_port = input->flow.udp6_flow.dst_port,
2699 attributes->items[1] = (struct rte_flow_item){
2700 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2701 .spec = &attributes->l3,
2703 attributes->items[2] = (struct rte_flow_item){
2704 .type = RTE_FLOW_ITEM_TYPE_UDP,
2705 .spec = &attributes->l4,
2708 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2709 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2710 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2711 .proto = input->flow.tcp6_flow.ip.proto,
2713 memcpy(attributes->l3.ipv6.hdr.src_addr,
2714 input->flow.tcp6_flow.ip.src_ip,
2715 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2716 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2717 input->flow.tcp6_flow.ip.dst_ip,
2718 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2719 attributes->l4.tcp.hdr = (struct tcp_hdr){
2720 .src_port = input->flow.tcp6_flow.src_port,
2721 .dst_port = input->flow.tcp6_flow.dst_port,
2723 attributes->items[1] = (struct rte_flow_item){
2724 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2725 .spec = &attributes->l3,
2727 attributes->items[2] = (struct rte_flow_item){
2728 .type = RTE_FLOW_ITEM_TYPE_UDP,
2729 .spec = &attributes->l4,
2732 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2733 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2734 .hop_limits = input->flow.ipv6_flow.hop_limits,
2735 .proto = input->flow.ipv6_flow.proto,
2737 memcpy(attributes->l3.ipv6.hdr.src_addr,
2738 input->flow.ipv6_flow.src_ip,
2739 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2740 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2741 input->flow.ipv6_flow.dst_ip,
2742 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2743 attributes->items[1] = (struct rte_flow_item){
2744 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2745 .spec = &attributes->l3,
2749 ERROR("invalid flow type%d",
2750 fdir_filter->input.flow_type);
2757 * Add new flow director filter and store it in list.
2760 * Private structure.
2761 * @param fdir_filter
2762 * Flow director filter to add.
2765 * 0 on success, errno value on failure.
2768 priv_fdir_filter_add(struct priv *priv,
2769 const struct rte_eth_fdir_filter *fdir_filter)
2771 struct mlx5_fdir attributes = {
2774 struct mlx5_flow_parse parser = {
2775 .layer = HASH_RXQ_ETH,
2777 struct rte_flow_error error;
2778 struct rte_flow *flow;
2781 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2784 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2785 attributes.actions, &error, &parser);
2788 flow = priv_flow_create(priv,
2795 DEBUG("FDIR created %p", (void *)flow);
2802 * Delete specific filter.
2805 * Private structure.
2806 * @param fdir_filter
2807 * Filter to be deleted.
2810 * 0 on success, errno value on failure.
2813 priv_fdir_filter_delete(struct priv *priv,
2814 const struct rte_eth_fdir_filter *fdir_filter)
2816 struct mlx5_fdir attributes;
2817 struct mlx5_flow_parse parser = {
2819 .layer = HASH_RXQ_ETH,
2821 struct rte_flow_error error;
2822 struct rte_flow *flow;
2826 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2829 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2830 attributes.actions, &error, &parser);
2833 TAILQ_FOREACH(flow, &priv->flows, next) {
2834 struct ibv_flow_attr *attr;
2835 struct ibv_spec_header *attr_h;
2837 struct ibv_flow_attr *flow_attr;
2838 struct ibv_spec_header *flow_h;
2840 unsigned int specs_n;
2843 attr = parser.drop_q.ibv_attr;
2845 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2847 flow_attr = flow->drxq.ibv_attr;
2849 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2850 /* Compare first the attributes. */
2851 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2853 if (attr->num_of_specs == 0)
2855 spec = (void *)((uintptr_t)attr +
2856 sizeof(struct ibv_flow_attr));
2857 flow_spec = (void *)((uintptr_t)flow_attr +
2858 sizeof(struct ibv_flow_attr));
2859 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2860 for (i = 0; i != specs_n; ++i) {
2863 if (memcmp(spec, flow_spec,
2864 RTE_MIN(attr_h->size, flow_h->size)))
2866 spec = (void *)((uintptr_t)attr + attr_h->size);
2867 flow_spec = (void *)((uintptr_t)flow_attr +
2870 /* At this point, the flow match. */
2874 priv_flow_destroy(priv, &priv->flows, flow);
2877 rte_free(parser.drop_q.ibv_attr);
2879 for (i = 0; i != hash_rxq_init_n; ++i) {
2880 if (parser.queue[i].ibv_attr)
2881 rte_free(parser.queue[i].ibv_attr);
2888 * Update queue for specific filter.
2891 * Private structure.
2892 * @param fdir_filter
2893 * Filter to be updated.
2896 * 0 on success, errno value on failure.
2899 priv_fdir_filter_update(struct priv *priv,
2900 const struct rte_eth_fdir_filter *fdir_filter)
2904 ret = priv_fdir_filter_delete(priv, fdir_filter);
2907 ret = priv_fdir_filter_add(priv, fdir_filter);
2912 * Flush all filters.
2915 * Private structure.
2918 priv_fdir_filter_flush(struct priv *priv)
2920 priv_flow_flush(priv, &priv->flows);
2924 * Get flow director information.
2927 * Private structure.
2928 * @param[out] fdir_info
2929 * Resulting flow director information.
2932 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2934 struct rte_eth_fdir_masks *mask =
2935 &priv->dev->data->dev_conf.fdir_conf.mask;
2937 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2938 fdir_info->guarant_spc = 0;
2939 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2940 fdir_info->max_flexpayload = 0;
2941 fdir_info->flow_types_mask[0] = 0;
2942 fdir_info->flex_payload_unit = 0;
2943 fdir_info->max_flex_payload_segment_num = 0;
2944 fdir_info->flex_payload_limit = 0;
2945 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2949 * Deal with flow director operations.
2952 * Pointer to private structure.
2954 * Operation to perform.
2956 * Pointer to operation-specific structure.
2959 * 0 on success, errno value on failure.
2962 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2964 enum rte_fdir_mode fdir_mode =
2965 priv->dev->data->dev_conf.fdir_conf.mode;
2968 if (filter_op == RTE_ETH_FILTER_NOP)
2970 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2971 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2972 ERROR("%p: flow director mode %d not supported",
2973 (void *)priv, fdir_mode);
2976 switch (filter_op) {
2977 case RTE_ETH_FILTER_ADD:
2978 ret = priv_fdir_filter_add(priv, arg);
2980 case RTE_ETH_FILTER_UPDATE:
2981 ret = priv_fdir_filter_update(priv, arg);
2983 case RTE_ETH_FILTER_DELETE:
2984 ret = priv_fdir_filter_delete(priv, arg);
2986 case RTE_ETH_FILTER_FLUSH:
2987 priv_fdir_filter_flush(priv);
2989 case RTE_ETH_FILTER_INFO:
2990 priv_fdir_info_get(priv, arg);
2993 DEBUG("%p: unknown operation %u", (void *)priv,
3002 * Manage filter operations.
3005 * Pointer to Ethernet device structure.
3006 * @param filter_type
3009 * Operation to perform.
3011 * Pointer to operation-specific structure.
3014 * 0 on success, negative errno value on failure.
3017 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3018 enum rte_filter_type filter_type,
3019 enum rte_filter_op filter_op,
3023 struct priv *priv = dev->data->dev_private;
3025 switch (filter_type) {
3026 case RTE_ETH_FILTER_GENERIC:
3027 if (filter_op != RTE_ETH_FILTER_GET)
3029 *(const void **)arg = &mlx5_flow_ops;
3031 case RTE_ETH_FILTER_FDIR:
3033 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3037 ERROR("%p: filter type (%d) not supported",
3038 (void *)dev, filter_type);