4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev_driver.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
54 #include "mlx5_defs.h"
56 #include "mlx5_glue.h"
58 /* Define minimal priority for control plane flows. */
59 #define MLX5_CTRL_FLOW_PRIORITY 4
61 /* Internet Protocol versions. */
65 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
66 struct ibv_flow_spec_counter_action {
71 /* Dev ops structure defined in mlx5.c */
72 extern const struct eth_dev_ops mlx5_dev_ops;
73 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
76 mlx5_flow_create_eth(const struct rte_flow_item *item,
77 const void *default_mask,
81 mlx5_flow_create_vlan(const struct rte_flow_item *item,
82 const void *default_mask,
86 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
87 const void *default_mask,
91 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
92 const void *default_mask,
96 mlx5_flow_create_udp(const struct rte_flow_item *item,
97 const void *default_mask,
101 mlx5_flow_create_tcp(const struct rte_flow_item *item,
102 const void *default_mask,
106 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
107 const void *default_mask,
110 struct mlx5_flow_parse;
113 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
117 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
120 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
122 /* Hash RX queue types. */
133 /* Initialization data for hash RX queue. */
134 struct hash_rxq_init {
135 uint64_t hash_fields; /* Fields that participate in the hash. */
136 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
137 unsigned int flow_priority; /* Flow priority to use. */
138 unsigned int ip_version; /* Internet protocol. */
141 /* Initialization data for hash RX queues. */
142 const struct hash_rxq_init hash_rxq_init[] = {
144 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
145 IBV_RX_HASH_DST_IPV4 |
146 IBV_RX_HASH_SRC_PORT_TCP |
147 IBV_RX_HASH_DST_PORT_TCP),
148 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
150 .ip_version = MLX5_IPV4,
153 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
154 IBV_RX_HASH_DST_IPV4 |
155 IBV_RX_HASH_SRC_PORT_UDP |
156 IBV_RX_HASH_DST_PORT_UDP),
157 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
159 .ip_version = MLX5_IPV4,
162 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
163 IBV_RX_HASH_DST_IPV4),
164 .dpdk_rss_hf = (ETH_RSS_IPV4 |
167 .ip_version = MLX5_IPV4,
170 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
171 IBV_RX_HASH_DST_IPV6 |
172 IBV_RX_HASH_SRC_PORT_TCP |
173 IBV_RX_HASH_DST_PORT_TCP),
174 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
176 .ip_version = MLX5_IPV6,
179 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
180 IBV_RX_HASH_DST_IPV6 |
181 IBV_RX_HASH_SRC_PORT_UDP |
182 IBV_RX_HASH_DST_PORT_UDP),
183 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
185 .ip_version = MLX5_IPV6,
188 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
189 IBV_RX_HASH_DST_IPV6),
190 .dpdk_rss_hf = (ETH_RSS_IPV6 |
193 .ip_version = MLX5_IPV6,
202 /* Number of entries in hash_rxq_init[]. */
203 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
205 /** Structure for holding counter stats. */
206 struct mlx5_flow_counter_stats {
207 uint64_t hits; /**< Number of packets matched by the rule. */
208 uint64_t bytes; /**< Number of bytes matched by the rule. */
211 /** Structure for Drop queue. */
212 struct mlx5_hrxq_drop {
213 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
214 struct ibv_qp *qp; /**< Verbs queue pair. */
215 struct ibv_wq *wq; /**< Verbs work queue. */
216 struct ibv_cq *cq; /**< Verbs completion queue. */
219 /* Flows structures. */
221 uint64_t hash_fields; /**< Fields that participate in the hash. */
222 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
223 struct ibv_flow *ibv_flow; /**< Verbs flow. */
224 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
227 /* Drop flows structures. */
228 struct mlx5_flow_drop {
229 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
230 struct ibv_flow *ibv_flow; /**< Verbs flow. */
234 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
235 uint32_t mark:1; /**< Set if the flow is marked. */
236 uint32_t drop:1; /**< Drop queue. */
237 uint16_t queues_n; /**< Number of entries in queue[]. */
238 uint16_t (*queues)[]; /**< Queues indexes to use. */
239 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
240 uint8_t rss_key[40]; /**< copy of the RSS key. */
241 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
242 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
243 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
244 /**< Flow with Rx queue. */
247 /** Static initializer for items. */
249 (const enum rte_flow_item_type []){ \
250 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
253 /** Structure to generate a simple graph of layers supported by the NIC. */
254 struct mlx5_flow_items {
255 /** List of possible actions for these items. */
256 const enum rte_flow_action_type *const actions;
257 /** Bit-masks corresponding to the possibilities for the item. */
260 * Default bit-masks to use when item->mask is not provided. When
261 * \default_mask is also NULL, the full supported bit-mask (\mask) is
264 const void *default_mask;
265 /** Bit-masks size in bytes. */
266 const unsigned int mask_sz;
268 * Conversion function from rte_flow to NIC specific flow.
271 * rte_flow item to convert.
272 * @param default_mask
273 * Default bit-masks to use when item->mask is not provided.
275 * Internal structure to store the conversion.
278 * 0 on success, negative value otherwise.
280 int (*convert)(const struct rte_flow_item *item,
281 const void *default_mask,
283 /** Size in bytes of the destination structure. */
284 const unsigned int dst_sz;
285 /** List of possible following items. */
286 const enum rte_flow_item_type *const items;
289 /** Valid action for this PMD. */
290 static const enum rte_flow_action_type valid_actions[] = {
291 RTE_FLOW_ACTION_TYPE_DROP,
292 RTE_FLOW_ACTION_TYPE_QUEUE,
293 RTE_FLOW_ACTION_TYPE_MARK,
294 RTE_FLOW_ACTION_TYPE_FLAG,
295 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
296 RTE_FLOW_ACTION_TYPE_COUNT,
298 RTE_FLOW_ACTION_TYPE_END,
301 /** Graph of supported items and associated actions. */
302 static const struct mlx5_flow_items mlx5_flow_items[] = {
303 [RTE_FLOW_ITEM_TYPE_END] = {
304 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
305 RTE_FLOW_ITEM_TYPE_VXLAN),
307 [RTE_FLOW_ITEM_TYPE_ETH] = {
308 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
309 RTE_FLOW_ITEM_TYPE_IPV4,
310 RTE_FLOW_ITEM_TYPE_IPV6),
311 .actions = valid_actions,
312 .mask = &(const struct rte_flow_item_eth){
313 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
314 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
317 .default_mask = &rte_flow_item_eth_mask,
318 .mask_sz = sizeof(struct rte_flow_item_eth),
319 .convert = mlx5_flow_create_eth,
320 .dst_sz = sizeof(struct ibv_flow_spec_eth),
322 [RTE_FLOW_ITEM_TYPE_VLAN] = {
323 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
324 RTE_FLOW_ITEM_TYPE_IPV6),
325 .actions = valid_actions,
326 .mask = &(const struct rte_flow_item_vlan){
329 .default_mask = &rte_flow_item_vlan_mask,
330 .mask_sz = sizeof(struct rte_flow_item_vlan),
331 .convert = mlx5_flow_create_vlan,
334 [RTE_FLOW_ITEM_TYPE_IPV4] = {
335 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
336 RTE_FLOW_ITEM_TYPE_TCP),
337 .actions = valid_actions,
338 .mask = &(const struct rte_flow_item_ipv4){
342 .type_of_service = -1,
347 .default_mask = &rte_flow_item_ipv4_mask,
348 .mask_sz = sizeof(struct rte_flow_item_ipv4),
349 .convert = mlx5_flow_create_ipv4,
350 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
352 [RTE_FLOW_ITEM_TYPE_IPV6] = {
353 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
354 RTE_FLOW_ITEM_TYPE_TCP),
355 .actions = valid_actions,
356 .mask = &(const struct rte_flow_item_ipv6){
359 0xff, 0xff, 0xff, 0xff,
360 0xff, 0xff, 0xff, 0xff,
361 0xff, 0xff, 0xff, 0xff,
362 0xff, 0xff, 0xff, 0xff,
365 0xff, 0xff, 0xff, 0xff,
366 0xff, 0xff, 0xff, 0xff,
367 0xff, 0xff, 0xff, 0xff,
368 0xff, 0xff, 0xff, 0xff,
375 .default_mask = &rte_flow_item_ipv6_mask,
376 .mask_sz = sizeof(struct rte_flow_item_ipv6),
377 .convert = mlx5_flow_create_ipv6,
378 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
380 [RTE_FLOW_ITEM_TYPE_UDP] = {
381 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
382 .actions = valid_actions,
383 .mask = &(const struct rte_flow_item_udp){
389 .default_mask = &rte_flow_item_udp_mask,
390 .mask_sz = sizeof(struct rte_flow_item_udp),
391 .convert = mlx5_flow_create_udp,
392 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
394 [RTE_FLOW_ITEM_TYPE_TCP] = {
395 .actions = valid_actions,
396 .mask = &(const struct rte_flow_item_tcp){
402 .default_mask = &rte_flow_item_tcp_mask,
403 .mask_sz = sizeof(struct rte_flow_item_tcp),
404 .convert = mlx5_flow_create_tcp,
405 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
407 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
408 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
409 .actions = valid_actions,
410 .mask = &(const struct rte_flow_item_vxlan){
411 .vni = "\xff\xff\xff",
413 .default_mask = &rte_flow_item_vxlan_mask,
414 .mask_sz = sizeof(struct rte_flow_item_vxlan),
415 .convert = mlx5_flow_create_vxlan,
416 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
420 /** Structure to pass to the conversion function. */
421 struct mlx5_flow_parse {
422 uint32_t inner; /**< Set once VXLAN is encountered. */
423 uint32_t allmulti:1; /**< Set once allmulti dst MAC is encountered. */
425 /**< Whether resources should remain after a validate. */
426 uint32_t drop:1; /**< Target is a drop queue. */
427 uint32_t mark:1; /**< Mark is present in the flow. */
428 uint32_t count:1; /**< Count is present in the flow. */
429 uint32_t mark_id; /**< Mark identifier. */
430 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
431 uint16_t queues_n; /**< Number of entries in queue[]. */
432 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
433 uint8_t rss_key[40]; /**< copy of the RSS key. */
434 enum hash_rxq_type layer; /**< Last pattern layer detected. */
435 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
437 struct ibv_flow_attr *ibv_attr;
438 /**< Pointer to Verbs attributes. */
440 /**< Current position or total size of the attribute. */
441 } queue[RTE_DIM(hash_rxq_init)];
444 static const struct rte_flow_ops mlx5_flow_ops = {
445 .validate = mlx5_flow_validate,
446 .create = mlx5_flow_create,
447 .destroy = mlx5_flow_destroy,
448 .flush = mlx5_flow_flush,
449 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
450 .query = mlx5_flow_query,
454 .isolate = mlx5_flow_isolate,
457 /* Convert FDIR request to Generic flow. */
459 struct rte_flow_attr attr;
460 struct rte_flow_action actions[2];
461 struct rte_flow_item items[4];
462 struct rte_flow_item_eth l2;
463 struct rte_flow_item_eth l2_mask;
465 struct rte_flow_item_ipv4 ipv4;
466 struct rte_flow_item_ipv6 ipv6;
469 struct rte_flow_item_udp udp;
470 struct rte_flow_item_tcp tcp;
472 struct rte_flow_action_queue queue;
475 /* Verbs specification header. */
476 struct ibv_spec_header {
477 enum ibv_flow_spec_type type;
482 * Check support for a given item.
485 * Item specification.
487 * Bit-masks covering supported fields to compare with spec, last and mask in
490 * Bit-Mask size in bytes.
496 mlx5_flow_item_validate(const struct rte_flow_item *item,
497 const uint8_t *mask, unsigned int size)
501 if (!item->spec && (item->mask || item->last))
503 if (item->spec && !item->mask) {
505 const uint8_t *spec = item->spec;
507 for (i = 0; i < size; ++i)
508 if ((spec[i] | mask[i]) != mask[i])
511 if (item->last && !item->mask) {
513 const uint8_t *spec = item->last;
515 for (i = 0; i < size; ++i)
516 if ((spec[i] | mask[i]) != mask[i])
521 const uint8_t *spec = item->spec;
523 for (i = 0; i < size; ++i)
524 if ((spec[i] | mask[i]) != mask[i])
527 if (item->spec && item->last) {
530 const uint8_t *apply = mask;
535 for (i = 0; i < size; ++i) {
536 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
537 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
539 ret = memcmp(spec, last, size);
545 * Copy the RSS configuration from the user ones.
548 * Pointer to private structure.
550 * Internal parser structure.
552 * User RSS configuration to save.
555 * 0 on success, errno value on failure.
558 priv_flow_convert_rss_conf(struct priv *priv,
559 struct mlx5_flow_parse *parser,
560 const struct rte_eth_rss_conf *rss_conf)
562 const struct rte_eth_rss_conf *rss;
565 if (rss_conf->rss_hf & MLX5_RSS_HF_MASK)
569 rss = &priv->rss_conf;
571 if (rss->rss_key_len > 40)
573 parser->rss_conf.rss_key_len = rss->rss_key_len;
574 parser->rss_conf.rss_hf = rss->rss_hf;
575 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
576 parser->rss_conf.rss_key = parser->rss_key;
581 * Extract attribute to the parser.
584 * Pointer to private structure.
586 * Flow rule attributes.
588 * Perform verbose error reporting if not NULL.
589 * @param[in, out] parser
590 * Internal parser structure.
593 * 0 on success, a negative errno value otherwise and rte_errno is set.
596 priv_flow_convert_attributes(struct priv *priv,
597 const struct rte_flow_attr *attr,
598 struct rte_flow_error *error,
599 struct mlx5_flow_parse *parser)
604 rte_flow_error_set(error, ENOTSUP,
605 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
607 "groups are not supported");
610 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
611 rte_flow_error_set(error, ENOTSUP,
612 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
614 "priorities are not supported");
618 rte_flow_error_set(error, ENOTSUP,
619 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
621 "egress is not supported");
624 if (!attr->ingress) {
625 rte_flow_error_set(error, ENOTSUP,
626 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
628 "only ingress is supported");
635 * Extract actions request to the parser.
638 * Pointer to private structure.
640 * Associated actions (list terminated by the END action).
642 * Perform verbose error reporting if not NULL.
643 * @param[in, out] parser
644 * Internal parser structure.
647 * 0 on success, a negative errno value otherwise and rte_errno is set.
650 priv_flow_convert_actions(struct priv *priv,
651 const struct rte_flow_action actions[],
652 struct rte_flow_error *error,
653 struct mlx5_flow_parse *parser)
656 * Add default RSS configuration necessary for Verbs to create QP even
657 * if no RSS is necessary.
659 priv_flow_convert_rss_conf(priv, parser,
660 (const struct rte_eth_rss_conf *)
662 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
663 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
665 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
667 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
668 const struct rte_flow_action_queue *queue =
669 (const struct rte_flow_action_queue *)
674 if (!queue || (queue->index > (priv->rxqs_n - 1)))
675 goto exit_action_not_supported;
676 for (n = 0; n < parser->queues_n; ++n) {
677 if (parser->queues[n] == queue->index) {
682 if (parser->queues_n > 1 && !found) {
683 rte_flow_error_set(error, ENOTSUP,
684 RTE_FLOW_ERROR_TYPE_ACTION,
686 "queue action not in RSS queues");
690 parser->queues_n = 1;
691 parser->queues[0] = queue->index;
693 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
694 const struct rte_flow_action_rss *rss =
695 (const struct rte_flow_action_rss *)
699 if (!rss || !rss->num) {
700 rte_flow_error_set(error, EINVAL,
701 RTE_FLOW_ERROR_TYPE_ACTION,
706 if (parser->queues_n == 1) {
709 assert(parser->queues_n);
710 for (n = 0; n < rss->num; ++n) {
711 if (parser->queues[0] ==
718 rte_flow_error_set(error, ENOTSUP,
719 RTE_FLOW_ERROR_TYPE_ACTION,
721 "queue action not in RSS"
726 for (n = 0; n < rss->num; ++n) {
727 if (rss->queue[n] >= priv->rxqs_n) {
728 rte_flow_error_set(error, EINVAL,
729 RTE_FLOW_ERROR_TYPE_ACTION,
731 "queue id > number of"
736 for (n = 0; n < rss->num; ++n)
737 parser->queues[n] = rss->queue[n];
738 parser->queues_n = rss->num;
739 if (priv_flow_convert_rss_conf(priv, parser,
741 rte_flow_error_set(error, EINVAL,
742 RTE_FLOW_ERROR_TYPE_ACTION,
744 "wrong RSS configuration");
747 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
748 const struct rte_flow_action_mark *mark =
749 (const struct rte_flow_action_mark *)
753 rte_flow_error_set(error, EINVAL,
754 RTE_FLOW_ERROR_TYPE_ACTION,
756 "mark must be defined");
758 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
759 rte_flow_error_set(error, ENOTSUP,
760 RTE_FLOW_ERROR_TYPE_ACTION,
762 "mark must be between 0"
767 parser->mark_id = mark->id;
768 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
770 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
771 priv->config.flow_counter_en) {
774 goto exit_action_not_supported;
777 if (parser->drop && parser->mark)
779 if (!parser->queues_n && !parser->drop) {
780 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
781 NULL, "no valid action");
785 exit_action_not_supported:
786 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
787 actions, "action not supported");
795 * Pointer to private structure.
797 * Pattern specification (list terminated by the END pattern item).
799 * Perform verbose error reporting if not NULL.
800 * @param[in, out] parser
801 * Internal parser structure.
804 * 0 on success, a negative errno value otherwise and rte_errno is set.
807 priv_flow_convert_items_validate(struct priv *priv,
808 const struct rte_flow_item items[],
809 struct rte_flow_error *error,
810 struct mlx5_flow_parse *parser)
812 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
816 /* Initialise the offsets to start after verbs attribute. */
817 for (i = 0; i != hash_rxq_init_n; ++i)
818 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
819 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
820 const struct mlx5_flow_items *token = NULL;
824 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
828 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
830 if (cur_item->items[i] == items->type) {
831 token = &mlx5_flow_items[items->type];
836 goto exit_item_not_supported;
838 err = mlx5_flow_item_validate(items,
839 (const uint8_t *)cur_item->mask,
842 goto exit_item_not_supported;
843 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
845 rte_flow_error_set(error, ENOTSUP,
846 RTE_FLOW_ERROR_TYPE_ITEM,
848 "cannot recognize multiple"
849 " VXLAN encapsulations");
852 parser->inner = IBV_FLOW_SPEC_INNER;
854 if (parser->drop || parser->queues_n == 1) {
855 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
857 for (n = 0; n != hash_rxq_init_n; ++n)
858 parser->queue[n].offset += cur_item->dst_sz;
862 parser->queue[HASH_RXQ_ETH].offset +=
863 sizeof(struct ibv_flow_spec_action_drop);
866 for (i = 0; i != hash_rxq_init_n; ++i)
867 parser->queue[i].offset +=
868 sizeof(struct ibv_flow_spec_action_tag);
871 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
873 for (i = 0; i != hash_rxq_init_n; ++i)
874 parser->queue[i].offset += size;
877 exit_item_not_supported:
878 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
879 items, "item not supported");
884 * Allocate memory space to store verbs flow attributes.
887 * Pointer to private structure.
888 * @param[in] priority
891 * Amount of byte to allocate.
893 * Perform verbose error reporting if not NULL.
896 * A verbs flow attribute on success, NULL otherwise.
898 static struct ibv_flow_attr*
899 priv_flow_convert_allocate(struct priv *priv,
900 unsigned int priority,
902 struct rte_flow_error *error)
904 struct ibv_flow_attr *ibv_attr;
907 ibv_attr = rte_calloc(__func__, 1, size, 0);
909 rte_flow_error_set(error, ENOMEM,
910 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
912 "cannot allocate verbs spec attributes.");
915 ibv_attr->priority = priority;
920 * Finalise verbs flow attributes.
923 * Pointer to private structure.
924 * @param[in, out] parser
925 * Internal parser structure.
928 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
930 const unsigned int ipv4 =
931 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
932 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
933 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
934 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
935 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
936 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
940 if (parser->layer == HASH_RXQ_ETH) {
944 * This layer becomes useless as the pattern define under
947 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
948 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
950 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
951 for (i = ohmin; i != (ohmax + 1); ++i) {
952 if (!parser->queue[i].ibv_attr)
954 rte_free(parser->queue[i].ibv_attr);
955 parser->queue[i].ibv_attr = NULL;
957 /* Remove impossible flow according to the RSS configuration. */
958 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
959 parser->rss_conf.rss_hf) {
960 /* Remove any other flow. */
961 for (i = hmin; i != (hmax + 1); ++i) {
962 if ((i == parser->layer) ||
963 (!parser->queue[i].ibv_attr))
965 rte_free(parser->queue[i].ibv_attr);
966 parser->queue[i].ibv_attr = NULL;
968 } else if (!parser->queue[ip].ibv_attr) {
969 /* no RSS possible with the current configuration. */
970 parser->queues_n = 1;
975 * Fill missing layers in verbs specifications, or compute the correct
976 * offset to allocate the memory space for the attributes and
979 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
981 struct ibv_flow_spec_ipv4_ext ipv4;
982 struct ibv_flow_spec_ipv6 ipv6;
983 struct ibv_flow_spec_tcp_udp udp_tcp;
988 if (i == parser->layer)
990 if (parser->layer == HASH_RXQ_ETH) {
991 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
992 size = sizeof(struct ibv_flow_spec_ipv4_ext);
993 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
994 .type = IBV_FLOW_SPEC_IPV4_EXT,
998 size = sizeof(struct ibv_flow_spec_ipv6);
999 specs.ipv6 = (struct ibv_flow_spec_ipv6){
1000 .type = IBV_FLOW_SPEC_IPV6,
1004 if (parser->queue[i].ibv_attr) {
1005 dst = (void *)((uintptr_t)
1006 parser->queue[i].ibv_attr +
1007 parser->queue[i].offset);
1008 memcpy(dst, &specs, size);
1009 ++parser->queue[i].ibv_attr->num_of_specs;
1011 parser->queue[i].offset += size;
1013 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1014 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1015 size = sizeof(struct ibv_flow_spec_tcp_udp);
1016 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1017 .type = ((i == HASH_RXQ_UDPV4 ||
1018 i == HASH_RXQ_UDPV6) ?
1023 if (parser->queue[i].ibv_attr) {
1024 dst = (void *)((uintptr_t)
1025 parser->queue[i].ibv_attr +
1026 parser->queue[i].offset);
1027 memcpy(dst, &specs, size);
1028 ++parser->queue[i].ibv_attr->num_of_specs;
1030 parser->queue[i].offset += size;
1036 * Validate and convert a flow supported by the NIC.
1039 * Pointer to private structure.
1041 * Flow rule attributes.
1042 * @param[in] pattern
1043 * Pattern specification (list terminated by the END pattern item).
1044 * @param[in] actions
1045 * Associated actions (list terminated by the END action).
1047 * Perform verbose error reporting if not NULL.
1048 * @param[in, out] parser
1049 * Internal parser structure.
1052 * 0 on success, a negative errno value otherwise and rte_errno is set.
1055 priv_flow_convert(struct priv *priv,
1056 const struct rte_flow_attr *attr,
1057 const struct rte_flow_item items[],
1058 const struct rte_flow_action actions[],
1059 struct rte_flow_error *error,
1060 struct mlx5_flow_parse *parser)
1062 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1066 /* First step. Validate the attributes, items and actions. */
1067 *parser = (struct mlx5_flow_parse){
1068 .create = parser->create,
1069 .layer = HASH_RXQ_ETH,
1070 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1072 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1075 ret = priv_flow_convert_actions(priv, actions, error, parser);
1078 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1081 priv_flow_convert_finalise(priv, parser);
1084 * Allocate the memory space to store verbs specifications.
1086 if (parser->drop || parser->queues_n == 1) {
1087 unsigned int priority =
1089 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1090 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1092 parser->queue[HASH_RXQ_ETH].ibv_attr =
1093 priv_flow_convert_allocate(priv, priority,
1095 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1097 parser->queue[HASH_RXQ_ETH].offset =
1098 sizeof(struct ibv_flow_attr);
1100 for (i = 0; i != hash_rxq_init_n; ++i) {
1101 unsigned int priority =
1103 hash_rxq_init[i].flow_priority;
1104 unsigned int offset;
1106 if (!(parser->rss_conf.rss_hf &
1107 hash_rxq_init[i].dpdk_rss_hf) &&
1108 (i != HASH_RXQ_ETH))
1110 offset = parser->queue[i].offset;
1111 parser->queue[i].ibv_attr =
1112 priv_flow_convert_allocate(priv, priority,
1114 if (!parser->queue[i].ibv_attr)
1116 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1119 /* Third step. Conversion parse, fill the specifications. */
1121 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1122 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1124 cur_item = &mlx5_flow_items[items->type];
1125 ret = cur_item->convert(items,
1126 (cur_item->default_mask ?
1127 cur_item->default_mask :
1131 rte_flow_error_set(error, ret,
1132 RTE_FLOW_ERROR_TYPE_ITEM,
1133 items, "item not supported");
1138 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1139 if (parser->count && parser->create) {
1140 mlx5_flow_create_count(priv, parser);
1142 goto exit_count_error;
1145 * Last step. Complete missing specification to reach the RSS
1148 if (parser->queues_n > 1) {
1149 priv_flow_convert_finalise(priv, parser);
1152 * Action queue have their priority overridden with
1153 * Ethernet priority, this priority needs to be adjusted to
1154 * their most specific layer priority.
1156 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1158 hash_rxq_init[parser->layer].flow_priority;
1160 if (parser->allmulti &&
1161 parser->layer == HASH_RXQ_ETH) {
1162 for (i = 0; i != hash_rxq_init_n; ++i) {
1163 if (!parser->queue[i].ibv_attr)
1165 if (parser->queue[i].ibv_attr->num_of_specs != 1)
1167 parser->queue[i].ibv_attr->type =
1168 IBV_FLOW_ATTR_MC_DEFAULT;
1172 /* Only verification is expected, all resources should be released. */
1173 if (!parser->create) {
1174 for (i = 0; i != hash_rxq_init_n; ++i) {
1175 if (parser->queue[i].ibv_attr) {
1176 rte_free(parser->queue[i].ibv_attr);
1177 parser->queue[i].ibv_attr = NULL;
1183 for (i = 0; i != hash_rxq_init_n; ++i) {
1184 if (parser->queue[i].ibv_attr) {
1185 rte_free(parser->queue[i].ibv_attr);
1186 parser->queue[i].ibv_attr = NULL;
1189 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1190 NULL, "cannot allocate verbs spec attributes.");
1193 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1194 NULL, "cannot create counter.");
1199 * Copy the specification created into the flow.
1202 * Internal parser structure.
1204 * Create specification.
1206 * Size in bytes of the specification to copy.
1209 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1215 for (i = 0; i != hash_rxq_init_n; ++i) {
1216 if (!parser->queue[i].ibv_attr)
1218 /* Specification must be the same l3 type or none. */
1219 if (parser->layer == HASH_RXQ_ETH ||
1220 (hash_rxq_init[parser->layer].ip_version ==
1221 hash_rxq_init[i].ip_version) ||
1222 (hash_rxq_init[i].ip_version == 0)) {
1223 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1224 parser->queue[i].offset);
1225 memcpy(dst, src, size);
1226 ++parser->queue[i].ibv_attr->num_of_specs;
1227 parser->queue[i].offset += size;
1233 * Convert Ethernet item to Verbs specification.
1236 * Item specification.
1237 * @param default_mask[in]
1238 * Default bit-masks to use when item->mask is not provided.
1239 * @param data[in, out]
1243 mlx5_flow_create_eth(const struct rte_flow_item *item,
1244 const void *default_mask,
1247 const struct rte_flow_item_eth *spec = item->spec;
1248 const struct rte_flow_item_eth *mask = item->mask;
1249 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1250 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1251 struct ibv_flow_spec_eth eth = {
1252 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1256 /* Don't update layer for the inner pattern. */
1258 parser->layer = HASH_RXQ_ETH;
1263 mask = default_mask;
1264 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1265 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1266 eth.val.ether_type = spec->type;
1267 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1268 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1269 eth.mask.ether_type = mask->type;
1270 /* Remove unwanted bits from values. */
1271 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1272 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1273 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1275 eth.val.ether_type &= eth.mask.ether_type;
1277 mlx5_flow_create_copy(parser, ð, eth_size);
1278 parser->allmulti = eth.val.dst_mac[0] & 1;
1283 * Convert VLAN item to Verbs specification.
1286 * Item specification.
1287 * @param default_mask[in]
1288 * Default bit-masks to use when item->mask is not provided.
1289 * @param data[in, out]
1293 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1294 const void *default_mask,
1297 const struct rte_flow_item_vlan *spec = item->spec;
1298 const struct rte_flow_item_vlan *mask = item->mask;
1299 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1300 struct ibv_flow_spec_eth *eth;
1301 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1306 mask = default_mask;
1308 for (i = 0; i != hash_rxq_init_n; ++i) {
1309 if (!parser->queue[i].ibv_attr)
1312 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1313 parser->queue[i].offset - eth_size);
1314 eth->val.vlan_tag = spec->tci;
1315 eth->mask.vlan_tag = mask->tci;
1316 eth->val.vlan_tag &= eth->mask.vlan_tag;
1323 * Convert IPv4 item to Verbs specification.
1326 * Item specification.
1327 * @param default_mask[in]
1328 * Default bit-masks to use when item->mask is not provided.
1329 * @param data[in, out]
1333 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1334 const void *default_mask,
1337 const struct rte_flow_item_ipv4 *spec = item->spec;
1338 const struct rte_flow_item_ipv4 *mask = item->mask;
1339 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1340 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1341 struct ibv_flow_spec_ipv4_ext ipv4 = {
1342 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1346 /* Don't update layer for the inner pattern. */
1348 parser->layer = HASH_RXQ_IPV4;
1351 mask = default_mask;
1352 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1353 .src_ip = spec->hdr.src_addr,
1354 .dst_ip = spec->hdr.dst_addr,
1355 .proto = spec->hdr.next_proto_id,
1356 .tos = spec->hdr.type_of_service,
1358 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1359 .src_ip = mask->hdr.src_addr,
1360 .dst_ip = mask->hdr.dst_addr,
1361 .proto = mask->hdr.next_proto_id,
1362 .tos = mask->hdr.type_of_service,
1364 /* Remove unwanted bits from values. */
1365 ipv4.val.src_ip &= ipv4.mask.src_ip;
1366 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1367 ipv4.val.proto &= ipv4.mask.proto;
1368 ipv4.val.tos &= ipv4.mask.tos;
1370 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1375 * Convert IPv6 item to Verbs specification.
1378 * Item specification.
1379 * @param default_mask[in]
1380 * Default bit-masks to use when item->mask is not provided.
1381 * @param data[in, out]
1385 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1386 const void *default_mask,
1389 const struct rte_flow_item_ipv6 *spec = item->spec;
1390 const struct rte_flow_item_ipv6 *mask = item->mask;
1391 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1392 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1393 struct ibv_flow_spec_ipv6 ipv6 = {
1394 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1398 /* Don't update layer for the inner pattern. */
1400 parser->layer = HASH_RXQ_IPV6;
1403 uint32_t vtc_flow_val;
1404 uint32_t vtc_flow_mask;
1407 mask = default_mask;
1408 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1409 RTE_DIM(ipv6.val.src_ip));
1410 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1411 RTE_DIM(ipv6.val.dst_ip));
1412 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1413 RTE_DIM(ipv6.mask.src_ip));
1414 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1415 RTE_DIM(ipv6.mask.dst_ip));
1416 vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
1417 vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
1418 ipv6.val.flow_label =
1419 rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
1421 ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
1423 ipv6.val.next_hdr = spec->hdr.proto;
1424 ipv6.val.hop_limit = spec->hdr.hop_limits;
1425 ipv6.mask.flow_label =
1426 rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
1428 ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
1430 ipv6.mask.next_hdr = mask->hdr.proto;
1431 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1432 /* Remove unwanted bits from values. */
1433 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1434 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1435 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1437 ipv6.val.flow_label &= ipv6.mask.flow_label;
1438 ipv6.val.traffic_class &= ipv6.mask.traffic_class;
1439 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1440 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1442 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1447 * Convert UDP item to Verbs specification.
1450 * Item specification.
1451 * @param default_mask[in]
1452 * Default bit-masks to use when item->mask is not provided.
1453 * @param data[in, out]
1457 mlx5_flow_create_udp(const struct rte_flow_item *item,
1458 const void *default_mask,
1461 const struct rte_flow_item_udp *spec = item->spec;
1462 const struct rte_flow_item_udp *mask = item->mask;
1463 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1464 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1465 struct ibv_flow_spec_tcp_udp udp = {
1466 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1470 /* Don't update layer for the inner pattern. */
1471 if (!parser->inner) {
1472 if (parser->layer == HASH_RXQ_IPV4)
1473 parser->layer = HASH_RXQ_UDPV4;
1475 parser->layer = HASH_RXQ_UDPV6;
1479 mask = default_mask;
1480 udp.val.dst_port = spec->hdr.dst_port;
1481 udp.val.src_port = spec->hdr.src_port;
1482 udp.mask.dst_port = mask->hdr.dst_port;
1483 udp.mask.src_port = mask->hdr.src_port;
1484 /* Remove unwanted bits from values. */
1485 udp.val.src_port &= udp.mask.src_port;
1486 udp.val.dst_port &= udp.mask.dst_port;
1488 mlx5_flow_create_copy(parser, &udp, udp_size);
1493 * Convert TCP item to Verbs specification.
1496 * Item specification.
1497 * @param default_mask[in]
1498 * Default bit-masks to use when item->mask is not provided.
1499 * @param data[in, out]
1503 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1504 const void *default_mask,
1507 const struct rte_flow_item_tcp *spec = item->spec;
1508 const struct rte_flow_item_tcp *mask = item->mask;
1509 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1510 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1511 struct ibv_flow_spec_tcp_udp tcp = {
1512 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1516 /* Don't update layer for the inner pattern. */
1517 if (!parser->inner) {
1518 if (parser->layer == HASH_RXQ_IPV4)
1519 parser->layer = HASH_RXQ_TCPV4;
1521 parser->layer = HASH_RXQ_TCPV6;
1525 mask = default_mask;
1526 tcp.val.dst_port = spec->hdr.dst_port;
1527 tcp.val.src_port = spec->hdr.src_port;
1528 tcp.mask.dst_port = mask->hdr.dst_port;
1529 tcp.mask.src_port = mask->hdr.src_port;
1530 /* Remove unwanted bits from values. */
1531 tcp.val.src_port &= tcp.mask.src_port;
1532 tcp.val.dst_port &= tcp.mask.dst_port;
1534 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1539 * Convert VXLAN item to Verbs specification.
1542 * Item specification.
1543 * @param default_mask[in]
1544 * Default bit-masks to use when item->mask is not provided.
1545 * @param data[in, out]
1549 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1550 const void *default_mask,
1553 const struct rte_flow_item_vxlan *spec = item->spec;
1554 const struct rte_flow_item_vxlan *mask = item->mask;
1555 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1556 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1557 struct ibv_flow_spec_tunnel vxlan = {
1558 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1567 parser->inner = IBV_FLOW_SPEC_INNER;
1570 mask = default_mask;
1571 memcpy(&id.vni[1], spec->vni, 3);
1572 vxlan.val.tunnel_id = id.vlan_id;
1573 memcpy(&id.vni[1], mask->vni, 3);
1574 vxlan.mask.tunnel_id = id.vlan_id;
1575 /* Remove unwanted bits from values. */
1576 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1579 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1580 * layer is defined in the Verbs specification it is interpreted as
1581 * wildcard and all packets will match this rule, if it follows a full
1582 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1583 * before will also match this rule.
1584 * To avoid such situation, VNI 0 is currently refused.
1586 if (!vxlan.val.tunnel_id)
1588 mlx5_flow_create_copy(parser, &vxlan, size);
1593 * Convert mark/flag action to Verbs specification.
1596 * Internal parser structure.
1601 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1603 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1604 struct ibv_flow_spec_action_tag tag = {
1605 .type = IBV_FLOW_SPEC_ACTION_TAG,
1607 .tag_id = mlx5_flow_mark_set(mark_id),
1610 assert(parser->mark);
1611 mlx5_flow_create_copy(parser, &tag, size);
1616 * Convert count action to Verbs specification.
1619 * Pointer to private structure.
1621 * Pointer to MLX5 flow parser structure.
1624 * 0 on success, errno value on failure.
1627 mlx5_flow_create_count(struct priv *priv __rte_unused,
1628 struct mlx5_flow_parse *parser __rte_unused)
1630 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1631 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1632 struct ibv_counter_set_init_attr init_attr = {0};
1633 struct ibv_flow_spec_counter_action counter = {
1634 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1636 .counter_set_handle = 0,
1639 init_attr.counter_set_id = 0;
1640 parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
1643 counter.counter_set_handle = parser->cs->handle;
1644 mlx5_flow_create_copy(parser, &counter, size);
1650 * Complete flow rule creation with a drop queue.
1653 * Pointer to private structure.
1655 * Internal parser structure.
1657 * Pointer to the rte_flow.
1659 * Perform verbose error reporting if not NULL.
1662 * 0 on success, errno value on failure.
1665 priv_flow_create_action_queue_drop(struct priv *priv,
1666 struct mlx5_flow_parse *parser,
1667 struct rte_flow *flow,
1668 struct rte_flow_error *error)
1670 struct ibv_flow_spec_action_drop *drop;
1671 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1677 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
1678 parser->queue[HASH_RXQ_ETH].offset);
1679 *drop = (struct ibv_flow_spec_action_drop){
1680 .type = IBV_FLOW_SPEC_ACTION_DROP,
1683 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
1684 parser->queue[HASH_RXQ_ETH].offset += size;
1685 flow->frxq[HASH_RXQ_ETH].ibv_attr =
1686 parser->queue[HASH_RXQ_ETH].ibv_attr;
1688 flow->cs = parser->cs;
1689 if (!priv->dev->data->dev_started)
1691 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
1692 flow->frxq[HASH_RXQ_ETH].ibv_flow =
1693 mlx5_glue->create_flow(priv->flow_drop_queue->qp,
1694 flow->frxq[HASH_RXQ_ETH].ibv_attr);
1695 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1696 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1697 NULL, "flow rule creation failure");
1704 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1705 claim_zero(mlx5_glue->destroy_flow
1706 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
1707 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
1709 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
1710 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
1711 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
1714 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1722 * Create hash Rx queues when RSS is enabled.
1725 * Pointer to private structure.
1727 * Internal parser structure.
1729 * Pointer to the rte_flow.
1731 * Perform verbose error reporting if not NULL.
1734 * 0 on success, a errno value otherwise and rte_errno is set.
1737 priv_flow_create_action_queue_rss(struct priv *priv,
1738 struct mlx5_flow_parse *parser,
1739 struct rte_flow *flow,
1740 struct rte_flow_error *error)
1744 for (i = 0; i != hash_rxq_init_n; ++i) {
1745 uint64_t hash_fields;
1747 if (!parser->queue[i].ibv_attr)
1749 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1750 parser->queue[i].ibv_attr = NULL;
1751 hash_fields = hash_rxq_init[i].hash_fields;
1752 if (!priv->dev->data->dev_started)
1754 flow->frxq[i].hrxq =
1755 mlx5_priv_hrxq_get(priv,
1756 parser->rss_conf.rss_key,
1757 parser->rss_conf.rss_key_len,
1761 if (flow->frxq[i].hrxq)
1763 flow->frxq[i].hrxq =
1764 mlx5_priv_hrxq_new(priv,
1765 parser->rss_conf.rss_key,
1766 parser->rss_conf.rss_key_len,
1770 if (!flow->frxq[i].hrxq) {
1771 rte_flow_error_set(error, ENOMEM,
1772 RTE_FLOW_ERROR_TYPE_HANDLE,
1773 NULL, "cannot create hash rxq");
1781 * Complete flow rule creation.
1784 * Pointer to private structure.
1786 * Internal parser structure.
1788 * Pointer to the rte_flow.
1790 * Perform verbose error reporting if not NULL.
1793 * 0 on success, a errno value otherwise and rte_errno is set.
1796 priv_flow_create_action_queue(struct priv *priv,
1797 struct mlx5_flow_parse *parser,
1798 struct rte_flow *flow,
1799 struct rte_flow_error *error)
1806 assert(!parser->drop);
1807 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1811 flow->cs = parser->cs;
1812 if (!priv->dev->data->dev_started)
1814 for (i = 0; i != hash_rxq_init_n; ++i) {
1815 if (!flow->frxq[i].hrxq)
1817 flow->frxq[i].ibv_flow =
1818 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
1819 flow->frxq[i].ibv_attr);
1820 if (!flow->frxq[i].ibv_flow) {
1821 rte_flow_error_set(error, ENOMEM,
1822 RTE_FLOW_ERROR_TYPE_HANDLE,
1823 NULL, "flow rule creation failure");
1827 DEBUG("%p type %d QP %p ibv_flow %p",
1829 (void *)flow->frxq[i].hrxq,
1830 (void *)flow->frxq[i].ibv_flow);
1832 for (i = 0; i != parser->queues_n; ++i) {
1833 struct mlx5_rxq_data *q =
1834 (*priv->rxqs)[parser->queues[i]];
1836 q->mark |= parser->mark;
1841 for (i = 0; i != hash_rxq_init_n; ++i) {
1842 if (flow->frxq[i].ibv_flow) {
1843 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1845 claim_zero(mlx5_glue->destroy_flow(ibv_flow));
1847 if (flow->frxq[i].hrxq)
1848 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1849 if (flow->frxq[i].ibv_attr)
1850 rte_free(flow->frxq[i].ibv_attr);
1853 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
1864 * Pointer to private structure.
1866 * Pointer to a TAILQ flow list.
1868 * Flow rule attributes.
1869 * @param[in] pattern
1870 * Pattern specification (list terminated by the END pattern item).
1871 * @param[in] actions
1872 * Associated actions (list terminated by the END action).
1874 * Perform verbose error reporting if not NULL.
1877 * A flow on success, NULL otherwise.
1879 static struct rte_flow *
1880 priv_flow_create(struct priv *priv,
1881 struct mlx5_flows *list,
1882 const struct rte_flow_attr *attr,
1883 const struct rte_flow_item items[],
1884 const struct rte_flow_action actions[],
1885 struct rte_flow_error *error)
1887 struct mlx5_flow_parse parser = { .create = 1, };
1888 struct rte_flow *flow = NULL;
1892 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1895 flow = rte_calloc(__func__, 1,
1896 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1899 rte_flow_error_set(error, ENOMEM,
1900 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1902 "cannot allocate flow memory");
1905 /* Copy queues configuration. */
1906 flow->queues = (uint16_t (*)[])(flow + 1);
1907 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1908 flow->queues_n = parser.queues_n;
1909 flow->mark = parser.mark;
1910 /* Copy RSS configuration. */
1911 flow->rss_conf = parser.rss_conf;
1912 flow->rss_conf.rss_key = flow->rss_key;
1913 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1914 /* finalise the flow. */
1916 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1919 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1922 TAILQ_INSERT_TAIL(list, flow, next);
1923 DEBUG("Flow created %p", (void *)flow);
1926 for (i = 0; i != hash_rxq_init_n; ++i) {
1927 if (parser.queue[i].ibv_attr)
1928 rte_free(parser.queue[i].ibv_attr);
1935 * Validate a flow supported by the NIC.
1937 * @see rte_flow_validate()
1941 mlx5_flow_validate(struct rte_eth_dev *dev,
1942 const struct rte_flow_attr *attr,
1943 const struct rte_flow_item items[],
1944 const struct rte_flow_action actions[],
1945 struct rte_flow_error *error)
1947 struct priv *priv = dev->data->dev_private;
1949 struct mlx5_flow_parse parser = { .create = 0, };
1952 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1960 * @see rte_flow_create()
1964 mlx5_flow_create(struct rte_eth_dev *dev,
1965 const struct rte_flow_attr *attr,
1966 const struct rte_flow_item items[],
1967 const struct rte_flow_action actions[],
1968 struct rte_flow_error *error)
1970 struct priv *priv = dev->data->dev_private;
1971 struct rte_flow *flow;
1974 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1984 * Pointer to private structure.
1986 * Pointer to a TAILQ flow list.
1991 priv_flow_destroy(struct priv *priv,
1992 struct mlx5_flows *list,
1993 struct rte_flow *flow)
1997 if (flow->drop || !flow->mark)
1999 for (i = 0; i != flow->queues_n; ++i) {
2000 struct rte_flow *tmp;
2004 * To remove the mark from the queue, the queue must not be
2005 * present in any other marked flow (RSS or not).
2007 TAILQ_FOREACH(tmp, list, next) {
2009 uint16_t *tqs = NULL;
2014 for (j = 0; j != hash_rxq_init_n; ++j) {
2015 if (!tmp->frxq[j].hrxq)
2017 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2018 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2022 for (j = 0; (j != tq_n) && !mark; j++)
2023 if (tqs[j] == (*flow->queues)[i])
2026 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2030 if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
2031 claim_zero(mlx5_glue->destroy_flow
2032 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2033 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
2035 for (i = 0; i != hash_rxq_init_n; ++i) {
2036 struct mlx5_flow *frxq = &flow->frxq[i];
2039 claim_zero(mlx5_glue->destroy_flow
2042 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2044 rte_free(frxq->ibv_attr);
2048 claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
2051 TAILQ_REMOVE(list, flow, next);
2052 DEBUG("Flow destroyed %p", (void *)flow);
2057 * Destroy all flows.
2060 * Pointer to private structure.
2062 * Pointer to a TAILQ flow list.
2065 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2067 while (!TAILQ_EMPTY(list)) {
2068 struct rte_flow *flow;
2070 flow = TAILQ_FIRST(list);
2071 priv_flow_destroy(priv, list, flow);
2076 * Create drop queue.
2079 * Pointer to private structure.
2085 priv_flow_create_drop_queue(struct priv *priv)
2087 struct mlx5_hrxq_drop *fdq = NULL;
2091 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2093 WARN("cannot allocate memory for drop queue");
2096 fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
2098 WARN("cannot allocate CQ for drop queue");
2101 fdq->wq = mlx5_glue->create_wq
2103 &(struct ibv_wq_init_attr){
2104 .wq_type = IBV_WQT_RQ,
2111 WARN("cannot allocate WQ for drop queue");
2114 fdq->ind_table = mlx5_glue->create_rwq_ind_table
2116 &(struct ibv_rwq_ind_table_init_attr){
2117 .log_ind_tbl_size = 0,
2118 .ind_tbl = &fdq->wq,
2121 if (!fdq->ind_table) {
2122 WARN("cannot allocate indirection table for drop queue");
2125 fdq->qp = mlx5_glue->create_qp_ex
2127 &(struct ibv_qp_init_attr_ex){
2128 .qp_type = IBV_QPT_RAW_PACKET,
2130 IBV_QP_INIT_ATTR_PD |
2131 IBV_QP_INIT_ATTR_IND_TABLE |
2132 IBV_QP_INIT_ATTR_RX_HASH,
2133 .rx_hash_conf = (struct ibv_rx_hash_conf){
2135 IBV_RX_HASH_FUNC_TOEPLITZ,
2136 .rx_hash_key_len = rss_hash_default_key_len,
2137 .rx_hash_key = rss_hash_default_key,
2138 .rx_hash_fields_mask = 0,
2140 .rwq_ind_tbl = fdq->ind_table,
2144 WARN("cannot allocate QP for drop queue");
2147 priv->flow_drop_queue = fdq;
2151 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2153 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2155 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2157 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2160 priv->flow_drop_queue = NULL;
2165 * Delete drop queue.
2168 * Pointer to private structure.
2171 priv_flow_delete_drop_queue(struct priv *priv)
2173 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2178 claim_zero(mlx5_glue->destroy_qp(fdq->qp));
2180 claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
2182 claim_zero(mlx5_glue->destroy_wq(fdq->wq));
2184 claim_zero(mlx5_glue->destroy_cq(fdq->cq));
2186 priv->flow_drop_queue = NULL;
2193 * Pointer to private structure.
2195 * Pointer to a TAILQ flow list.
2198 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2200 struct rte_flow *flow;
2202 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2206 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
2208 claim_zero(mlx5_glue->destroy_flow
2209 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2210 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
2215 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2217 for (i = 0; i != hash_rxq_init_n; ++i) {
2218 if (!flow->frxq[i].hrxq)
2220 ind_tbl = flow->frxq[i].hrxq->ind_table;
2223 for (i = 0; i != ind_tbl->queues_n; ++i)
2224 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2226 for (i = 0; i != hash_rxq_init_n; ++i) {
2227 if (!flow->frxq[i].ibv_flow)
2229 claim_zero(mlx5_glue->destroy_flow
2230 (flow->frxq[i].ibv_flow));
2231 flow->frxq[i].ibv_flow = NULL;
2232 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2233 flow->frxq[i].hrxq = NULL;
2235 DEBUG("Flow %p removed", (void *)flow);
2243 * Pointer to private structure.
2245 * Pointer to a TAILQ flow list.
2248 * 0 on success, a errno value otherwise and rte_errno is set.
2251 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2253 struct rte_flow *flow;
2255 TAILQ_FOREACH(flow, list, next) {
2259 flow->frxq[HASH_RXQ_ETH].ibv_flow =
2260 mlx5_glue->create_flow
2261 (priv->flow_drop_queue->qp,
2262 flow->frxq[HASH_RXQ_ETH].ibv_attr);
2263 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
2264 DEBUG("Flow %p cannot be applied",
2269 DEBUG("Flow %p applied", (void *)flow);
2273 for (i = 0; i != hash_rxq_init_n; ++i) {
2274 if (!flow->frxq[i].ibv_attr)
2276 flow->frxq[i].hrxq =
2277 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2278 flow->rss_conf.rss_key_len,
2279 hash_rxq_init[i].hash_fields,
2282 if (flow->frxq[i].hrxq)
2284 flow->frxq[i].hrxq =
2285 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2286 flow->rss_conf.rss_key_len,
2287 hash_rxq_init[i].hash_fields,
2290 if (!flow->frxq[i].hrxq) {
2291 DEBUG("Flow %p cannot be applied",
2297 flow->frxq[i].ibv_flow =
2298 mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
2299 flow->frxq[i].ibv_attr);
2300 if (!flow->frxq[i].ibv_flow) {
2301 DEBUG("Flow %p cannot be applied",
2306 DEBUG("Flow %p applied", (void *)flow);
2310 for (i = 0; i != flow->queues_n; ++i)
2311 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2317 * Verify the flow list is empty
2320 * Pointer to private structure.
2322 * @return the number of flows not released.
2325 priv_flow_verify(struct priv *priv)
2327 struct rte_flow *flow;
2330 TAILQ_FOREACH(flow, &priv->flows, next) {
2331 DEBUG("%p: flow %p still referenced", (void *)priv,
2339 * Enable a control flow configured from the control plane.
2342 * Pointer to Ethernet device.
2344 * An Ethernet flow spec to apply.
2346 * An Ethernet flow mask to apply.
2348 * A VLAN flow spec to apply.
2350 * A VLAN flow mask to apply.
2356 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2357 struct rte_flow_item_eth *eth_spec,
2358 struct rte_flow_item_eth *eth_mask,
2359 struct rte_flow_item_vlan *vlan_spec,
2360 struct rte_flow_item_vlan *vlan_mask)
2362 struct priv *priv = dev->data->dev_private;
2363 const struct rte_flow_attr attr = {
2365 .priority = MLX5_CTRL_FLOW_PRIORITY,
2367 struct rte_flow_item items[] = {
2369 .type = RTE_FLOW_ITEM_TYPE_ETH,
2375 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2376 RTE_FLOW_ITEM_TYPE_END,
2382 .type = RTE_FLOW_ITEM_TYPE_END,
2385 struct rte_flow_action actions[] = {
2387 .type = RTE_FLOW_ACTION_TYPE_RSS,
2390 .type = RTE_FLOW_ACTION_TYPE_END,
2393 struct rte_flow *flow;
2394 struct rte_flow_error error;
2397 struct rte_flow_action_rss rss;
2399 const struct rte_eth_rss_conf *rss_conf;
2401 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2405 if (!priv->reta_idx_n)
2407 for (i = 0; i != priv->reta_idx_n; ++i)
2408 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2409 action_rss.local.rss_conf = &priv->rss_conf;
2410 action_rss.local.num = priv->reta_idx_n;
2411 actions[0].conf = (const void *)&action_rss.rss;
2412 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2420 * Enable a flow control configured from the control plane.
2423 * Pointer to Ethernet device.
2425 * An Ethernet flow spec to apply.
2427 * An Ethernet flow mask to apply.
2433 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2434 struct rte_flow_item_eth *eth_spec,
2435 struct rte_flow_item_eth *eth_mask)
2437 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2443 * @see rte_flow_destroy()
2447 mlx5_flow_destroy(struct rte_eth_dev *dev,
2448 struct rte_flow *flow,
2449 struct rte_flow_error *error)
2451 struct priv *priv = dev->data->dev_private;
2455 priv_flow_destroy(priv, &priv->flows, flow);
2461 * Destroy all flows.
2463 * @see rte_flow_flush()
2467 mlx5_flow_flush(struct rte_eth_dev *dev,
2468 struct rte_flow_error *error)
2470 struct priv *priv = dev->data->dev_private;
2474 priv_flow_flush(priv, &priv->flows);
2479 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2481 * Query flow counter.
2485 * @param counter_value
2486 * returned data from the counter.
2489 * 0 on success, a errno value otherwise and rte_errno is set.
2492 priv_flow_query_count(struct ibv_counter_set *cs,
2493 struct mlx5_flow_counter_stats *counter_stats,
2494 struct rte_flow_query_count *query_count,
2495 struct rte_flow_error *error)
2497 uint64_t counters[2];
2498 struct ibv_query_counter_set_attr query_cs_attr = {
2500 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2502 struct ibv_counter_set_data query_out = {
2504 .outlen = 2 * sizeof(uint64_t),
2506 int res = mlx5_glue->query_counter_set(&query_cs_attr, &query_out);
2509 rte_flow_error_set(error, -res,
2510 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2512 "cannot read counter");
2515 query_count->hits_set = 1;
2516 query_count->bytes_set = 1;
2517 query_count->hits = counters[0] - counter_stats->hits;
2518 query_count->bytes = counters[1] - counter_stats->bytes;
2519 if (query_count->reset) {
2520 counter_stats->hits = counters[0];
2521 counter_stats->bytes = counters[1];
2529 * @see rte_flow_query()
2533 mlx5_flow_query(struct rte_eth_dev *dev,
2534 struct rte_flow *flow,
2535 enum rte_flow_action_type action __rte_unused,
2537 struct rte_flow_error *error)
2539 struct priv *priv = dev->data->dev_private;
2544 res = priv_flow_query_count(flow->cs,
2545 &flow->counter_stats,
2546 (struct rte_flow_query_count *)data,
2549 rte_flow_error_set(error, res,
2550 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2552 "no counter found for flow");
2562 * @see rte_flow_isolate()
2566 mlx5_flow_isolate(struct rte_eth_dev *dev,
2568 struct rte_flow_error *error)
2570 struct priv *priv = dev->data->dev_private;
2573 if (dev->data->dev_started) {
2574 rte_flow_error_set(error, EBUSY,
2575 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2577 "port must be stopped first");
2581 priv->isolated = !!enable;
2583 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2585 priv->dev->dev_ops = &mlx5_dev_ops;
2591 * Convert a flow director filter to a generic flow.
2594 * Private structure.
2595 * @param fdir_filter
2596 * Flow director filter to add.
2598 * Generic flow parameters structure.
2601 * 0 on success, errno value on error.
2604 priv_fdir_filter_convert(struct priv *priv,
2605 const struct rte_eth_fdir_filter *fdir_filter,
2606 struct mlx5_fdir *attributes)
2608 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2610 /* Validate queue number. */
2611 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2612 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2615 attributes->attr.ingress = 1;
2616 attributes->items[0] = (struct rte_flow_item) {
2617 .type = RTE_FLOW_ITEM_TYPE_ETH,
2618 .spec = &attributes->l2,
2619 .mask = &attributes->l2_mask,
2621 switch (fdir_filter->action.behavior) {
2622 case RTE_ETH_FDIR_ACCEPT:
2623 attributes->actions[0] = (struct rte_flow_action){
2624 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2625 .conf = &attributes->queue,
2628 case RTE_ETH_FDIR_REJECT:
2629 attributes->actions[0] = (struct rte_flow_action){
2630 .type = RTE_FLOW_ACTION_TYPE_DROP,
2634 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2637 attributes->queue.index = fdir_filter->action.rx_queue;
2638 switch (fdir_filter->input.flow_type) {
2639 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2640 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2641 .src_addr = input->flow.udp4_flow.ip.src_ip,
2642 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2643 .time_to_live = input->flow.udp4_flow.ip.ttl,
2644 .type_of_service = input->flow.udp4_flow.ip.tos,
2645 .next_proto_id = input->flow.udp4_flow.ip.proto,
2647 attributes->l4.udp.hdr = (struct udp_hdr){
2648 .src_port = input->flow.udp4_flow.src_port,
2649 .dst_port = input->flow.udp4_flow.dst_port,
2651 attributes->items[1] = (struct rte_flow_item){
2652 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2653 .spec = &attributes->l3,
2654 .mask = &attributes->l3,
2656 attributes->items[2] = (struct rte_flow_item){
2657 .type = RTE_FLOW_ITEM_TYPE_UDP,
2658 .spec = &attributes->l4,
2659 .mask = &attributes->l4,
2662 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2663 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2664 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2665 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2666 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2667 .type_of_service = input->flow.tcp4_flow.ip.tos,
2668 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2670 attributes->l4.tcp.hdr = (struct tcp_hdr){
2671 .src_port = input->flow.tcp4_flow.src_port,
2672 .dst_port = input->flow.tcp4_flow.dst_port,
2674 attributes->items[1] = (struct rte_flow_item){
2675 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2676 .spec = &attributes->l3,
2677 .mask = &attributes->l3,
2679 attributes->items[2] = (struct rte_flow_item){
2680 .type = RTE_FLOW_ITEM_TYPE_TCP,
2681 .spec = &attributes->l4,
2682 .mask = &attributes->l4,
2685 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2686 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2687 .src_addr = input->flow.ip4_flow.src_ip,
2688 .dst_addr = input->flow.ip4_flow.dst_ip,
2689 .time_to_live = input->flow.ip4_flow.ttl,
2690 .type_of_service = input->flow.ip4_flow.tos,
2691 .next_proto_id = input->flow.ip4_flow.proto,
2693 attributes->items[1] = (struct rte_flow_item){
2694 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2695 .spec = &attributes->l3,
2696 .mask = &attributes->l3,
2699 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2700 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2701 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2702 .proto = input->flow.udp6_flow.ip.proto,
2704 memcpy(attributes->l3.ipv6.hdr.src_addr,
2705 input->flow.udp6_flow.ip.src_ip,
2706 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2707 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2708 input->flow.udp6_flow.ip.dst_ip,
2709 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2710 attributes->l4.udp.hdr = (struct udp_hdr){
2711 .src_port = input->flow.udp6_flow.src_port,
2712 .dst_port = input->flow.udp6_flow.dst_port,
2714 attributes->items[1] = (struct rte_flow_item){
2715 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2716 .spec = &attributes->l3,
2717 .mask = &attributes->l3,
2719 attributes->items[2] = (struct rte_flow_item){
2720 .type = RTE_FLOW_ITEM_TYPE_UDP,
2721 .spec = &attributes->l4,
2722 .mask = &attributes->l4,
2725 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2726 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2727 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2728 .proto = input->flow.tcp6_flow.ip.proto,
2730 memcpy(attributes->l3.ipv6.hdr.src_addr,
2731 input->flow.tcp6_flow.ip.src_ip,
2732 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2733 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2734 input->flow.tcp6_flow.ip.dst_ip,
2735 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2736 attributes->l4.tcp.hdr = (struct tcp_hdr){
2737 .src_port = input->flow.tcp6_flow.src_port,
2738 .dst_port = input->flow.tcp6_flow.dst_port,
2740 attributes->items[1] = (struct rte_flow_item){
2741 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2742 .spec = &attributes->l3,
2743 .mask = &attributes->l3,
2745 attributes->items[2] = (struct rte_flow_item){
2746 .type = RTE_FLOW_ITEM_TYPE_TCP,
2747 .spec = &attributes->l4,
2748 .mask = &attributes->l4,
2751 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2752 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2753 .hop_limits = input->flow.ipv6_flow.hop_limits,
2754 .proto = input->flow.ipv6_flow.proto,
2756 memcpy(attributes->l3.ipv6.hdr.src_addr,
2757 input->flow.ipv6_flow.src_ip,
2758 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2759 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2760 input->flow.ipv6_flow.dst_ip,
2761 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2762 attributes->items[1] = (struct rte_flow_item){
2763 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2764 .spec = &attributes->l3,
2765 .mask = &attributes->l3,
2769 ERROR("invalid flow type%d",
2770 fdir_filter->input.flow_type);
2777 * Add new flow director filter and store it in list.
2780 * Private structure.
2781 * @param fdir_filter
2782 * Flow director filter to add.
2785 * 0 on success, errno value on failure.
2788 priv_fdir_filter_add(struct priv *priv,
2789 const struct rte_eth_fdir_filter *fdir_filter)
2791 struct mlx5_fdir attributes = {
2794 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2795 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2799 struct mlx5_flow_parse parser = {
2800 .layer = HASH_RXQ_ETH,
2802 struct rte_flow_error error;
2803 struct rte_flow *flow;
2806 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2809 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2810 attributes.actions, &error, &parser);
2813 flow = priv_flow_create(priv,
2820 DEBUG("FDIR created %p", (void *)flow);
2827 * Delete specific filter.
2830 * Private structure.
2831 * @param fdir_filter
2832 * Filter to be deleted.
2835 * 0 on success, errno value on failure.
2838 priv_fdir_filter_delete(struct priv *priv,
2839 const struct rte_eth_fdir_filter *fdir_filter)
2841 struct mlx5_fdir attributes = {
2844 struct mlx5_flow_parse parser = {
2846 .layer = HASH_RXQ_ETH,
2848 struct rte_flow_error error;
2849 struct rte_flow *flow;
2853 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2856 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2857 attributes.actions, &error, &parser);
2861 * Special case for drop action which is only set in the
2862 * specifications when the flow is created. In this situation the
2863 * drop specification is missing.
2866 struct ibv_flow_spec_action_drop *drop;
2868 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
2869 parser.queue[HASH_RXQ_ETH].offset);
2870 *drop = (struct ibv_flow_spec_action_drop){
2871 .type = IBV_FLOW_SPEC_ACTION_DROP,
2872 .size = sizeof(struct ibv_flow_spec_action_drop),
2874 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
2876 TAILQ_FOREACH(flow, &priv->flows, next) {
2877 struct ibv_flow_attr *attr;
2878 struct ibv_spec_header *attr_h;
2880 struct ibv_flow_attr *flow_attr;
2881 struct ibv_spec_header *flow_h;
2883 unsigned int specs_n;
2885 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2886 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2887 /* Compare first the attributes. */
2888 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2890 if (attr->num_of_specs == 0)
2892 spec = (void *)((uintptr_t)attr +
2893 sizeof(struct ibv_flow_attr));
2894 flow_spec = (void *)((uintptr_t)flow_attr +
2895 sizeof(struct ibv_flow_attr));
2896 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2897 for (i = 0; i != specs_n; ++i) {
2900 if (memcmp(spec, flow_spec,
2901 RTE_MIN(attr_h->size, flow_h->size)))
2903 spec = (void *)((uintptr_t)spec + attr_h->size);
2904 flow_spec = (void *)((uintptr_t)flow_spec +
2907 /* At this point, the flow match. */
2910 /* The flow does not match. */
2914 priv_flow_destroy(priv, &priv->flows, flow);
2916 for (i = 0; i != hash_rxq_init_n; ++i) {
2917 if (parser.queue[i].ibv_attr)
2918 rte_free(parser.queue[i].ibv_attr);
2924 * Update queue for specific filter.
2927 * Private structure.
2928 * @param fdir_filter
2929 * Filter to be updated.
2932 * 0 on success, errno value on failure.
2935 priv_fdir_filter_update(struct priv *priv,
2936 const struct rte_eth_fdir_filter *fdir_filter)
2940 ret = priv_fdir_filter_delete(priv, fdir_filter);
2943 ret = priv_fdir_filter_add(priv, fdir_filter);
2948 * Flush all filters.
2951 * Private structure.
2954 priv_fdir_filter_flush(struct priv *priv)
2956 priv_flow_flush(priv, &priv->flows);
2960 * Get flow director information.
2963 * Private structure.
2964 * @param[out] fdir_info
2965 * Resulting flow director information.
2968 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2970 struct rte_eth_fdir_masks *mask =
2971 &priv->dev->data->dev_conf.fdir_conf.mask;
2973 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2974 fdir_info->guarant_spc = 0;
2975 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2976 fdir_info->max_flexpayload = 0;
2977 fdir_info->flow_types_mask[0] = 0;
2978 fdir_info->flex_payload_unit = 0;
2979 fdir_info->max_flex_payload_segment_num = 0;
2980 fdir_info->flex_payload_limit = 0;
2981 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2985 * Deal with flow director operations.
2988 * Pointer to private structure.
2990 * Operation to perform.
2992 * Pointer to operation-specific structure.
2995 * 0 on success, errno value on failure.
2998 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
3000 enum rte_fdir_mode fdir_mode =
3001 priv->dev->data->dev_conf.fdir_conf.mode;
3004 if (filter_op == RTE_ETH_FILTER_NOP)
3006 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
3007 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
3008 ERROR("%p: flow director mode %d not supported",
3009 (void *)priv, fdir_mode);
3012 switch (filter_op) {
3013 case RTE_ETH_FILTER_ADD:
3014 ret = priv_fdir_filter_add(priv, arg);
3016 case RTE_ETH_FILTER_UPDATE:
3017 ret = priv_fdir_filter_update(priv, arg);
3019 case RTE_ETH_FILTER_DELETE:
3020 ret = priv_fdir_filter_delete(priv, arg);
3022 case RTE_ETH_FILTER_FLUSH:
3023 priv_fdir_filter_flush(priv);
3025 case RTE_ETH_FILTER_INFO:
3026 priv_fdir_info_get(priv, arg);
3029 DEBUG("%p: unknown operation %u", (void *)priv,
3038 * Manage filter operations.
3041 * Pointer to Ethernet device structure.
3042 * @param filter_type
3045 * Operation to perform.
3047 * Pointer to operation-specific structure.
3050 * 0 on success, negative errno value on failure.
3053 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3054 enum rte_filter_type filter_type,
3055 enum rte_filter_op filter_op,
3059 struct priv *priv = dev->data->dev_private;
3061 switch (filter_type) {
3062 case RTE_ETH_FILTER_GENERIC:
3063 if (filter_op != RTE_ETH_FILTER_GET)
3065 *(const void **)arg = &mlx5_flow_ops;
3067 case RTE_ETH_FILTER_FDIR:
3069 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3073 ERROR("%p: filter type (%d) not supported",
3074 (void *)dev, filter_type);