4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
63 struct ibv_counter_set_init_attr {
66 struct ibv_flow_spec_counter_action {
69 struct ibv_counter_set {
74 ibv_destroy_counter_set(struct ibv_counter_set *cs)
81 /* Dev ops structure defined in mlx5.c */
82 extern const struct eth_dev_ops mlx5_dev_ops;
83 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
86 mlx5_flow_create_eth(const struct rte_flow_item *item,
87 const void *default_mask,
91 mlx5_flow_create_vlan(const struct rte_flow_item *item,
92 const void *default_mask,
96 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
97 const void *default_mask,
101 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
102 const void *default_mask,
106 mlx5_flow_create_udp(const struct rte_flow_item *item,
107 const void *default_mask,
111 mlx5_flow_create_tcp(const struct rte_flow_item *item,
112 const void *default_mask,
116 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
117 const void *default_mask,
120 struct mlx5_flow_parse;
123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
132 /* Hash RX queue types. */
143 /* Initialization data for hash RX queue. */
144 struct hash_rxq_init {
145 uint64_t hash_fields; /* Fields that participate in the hash. */
146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
147 unsigned int flow_priority; /* Flow priority to use. */
148 unsigned int ip_version; /* Internet protocol. */
151 /* Initialization data for hash RX queues. */
152 const struct hash_rxq_init hash_rxq_init[] = {
154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
155 IBV_RX_HASH_DST_IPV4 |
156 IBV_RX_HASH_SRC_PORT_TCP |
157 IBV_RX_HASH_DST_PORT_TCP),
158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
160 .ip_version = MLX5_IPV4,
163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
164 IBV_RX_HASH_DST_IPV4 |
165 IBV_RX_HASH_SRC_PORT_UDP |
166 IBV_RX_HASH_DST_PORT_UDP),
167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
169 .ip_version = MLX5_IPV4,
172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
173 IBV_RX_HASH_DST_IPV4),
174 .dpdk_rss_hf = (ETH_RSS_IPV4 |
177 .ip_version = MLX5_IPV4,
180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
181 IBV_RX_HASH_DST_IPV6 |
182 IBV_RX_HASH_SRC_PORT_TCP |
183 IBV_RX_HASH_DST_PORT_TCP),
184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
186 .ip_version = MLX5_IPV6,
189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
190 IBV_RX_HASH_DST_IPV6 |
191 IBV_RX_HASH_SRC_PORT_UDP |
192 IBV_RX_HASH_DST_PORT_UDP),
193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
195 .ip_version = MLX5_IPV6,
198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
199 IBV_RX_HASH_DST_IPV6),
200 .dpdk_rss_hf = (ETH_RSS_IPV6 |
203 .ip_version = MLX5_IPV6,
212 /* Number of entries in hash_rxq_init[]. */
213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
215 /** Structure for holding counter stats. */
216 struct mlx5_flow_counter_stats {
217 uint64_t hits; /**< Number of packets matched by the rule. */
218 uint64_t bytes; /**< Number of bytes matched by the rule. */
221 /** Structure for Drop queue. */
222 struct mlx5_hrxq_drop {
223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
224 struct ibv_qp *qp; /**< Verbs queue pair. */
225 struct ibv_wq *wq; /**< Verbs work queue. */
226 struct ibv_cq *cq; /**< Verbs completion queue. */
229 /* Flows structures. */
231 uint64_t hash_fields; /**< Fields that participate in the hash. */
232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
233 struct ibv_flow *ibv_flow; /**< Verbs flow. */
234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
237 /* Drop flows structures. */
238 struct mlx5_flow_drop {
239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
240 struct ibv_flow *ibv_flow; /**< Verbs flow. */
244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
245 uint32_t mark:1; /**< Set if the flow is marked. */
246 uint32_t drop:1; /**< Drop queue. */
247 uint16_t queues_n; /**< Number of entries in queue[]. */
248 uint16_t (*queues)[]; /**< Queues indexes to use. */
249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
250 uint8_t rss_key[40]; /**< copy of the RSS key. */
251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
253 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
254 /**< Flow with Rx queue. */
257 /** Static initializer for items. */
259 (const enum rte_flow_item_type []){ \
260 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
263 /** Structure to generate a simple graph of layers supported by the NIC. */
264 struct mlx5_flow_items {
265 /** List of possible actions for these items. */
266 const enum rte_flow_action_type *const actions;
267 /** Bit-masks corresponding to the possibilities for the item. */
270 * Default bit-masks to use when item->mask is not provided. When
271 * \default_mask is also NULL, the full supported bit-mask (\mask) is
274 const void *default_mask;
275 /** Bit-masks size in bytes. */
276 const unsigned int mask_sz;
278 * Conversion function from rte_flow to NIC specific flow.
281 * rte_flow item to convert.
282 * @param default_mask
283 * Default bit-masks to use when item->mask is not provided.
285 * Internal structure to store the conversion.
288 * 0 on success, negative value otherwise.
290 int (*convert)(const struct rte_flow_item *item,
291 const void *default_mask,
293 /** Size in bytes of the destination structure. */
294 const unsigned int dst_sz;
295 /** List of possible following items. */
296 const enum rte_flow_item_type *const items;
299 /** Valid action for this PMD. */
300 static const enum rte_flow_action_type valid_actions[] = {
301 RTE_FLOW_ACTION_TYPE_DROP,
302 RTE_FLOW_ACTION_TYPE_QUEUE,
303 RTE_FLOW_ACTION_TYPE_MARK,
304 RTE_FLOW_ACTION_TYPE_FLAG,
305 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
306 RTE_FLOW_ACTION_TYPE_COUNT,
308 RTE_FLOW_ACTION_TYPE_END,
311 /** Graph of supported items and associated actions. */
312 static const struct mlx5_flow_items mlx5_flow_items[] = {
313 [RTE_FLOW_ITEM_TYPE_END] = {
314 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
315 RTE_FLOW_ITEM_TYPE_VXLAN),
317 [RTE_FLOW_ITEM_TYPE_ETH] = {
318 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
319 RTE_FLOW_ITEM_TYPE_IPV4,
320 RTE_FLOW_ITEM_TYPE_IPV6),
321 .actions = valid_actions,
322 .mask = &(const struct rte_flow_item_eth){
323 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
324 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
327 .default_mask = &rte_flow_item_eth_mask,
328 .mask_sz = sizeof(struct rte_flow_item_eth),
329 .convert = mlx5_flow_create_eth,
330 .dst_sz = sizeof(struct ibv_flow_spec_eth),
332 [RTE_FLOW_ITEM_TYPE_VLAN] = {
333 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
334 RTE_FLOW_ITEM_TYPE_IPV6),
335 .actions = valid_actions,
336 .mask = &(const struct rte_flow_item_vlan){
339 .default_mask = &rte_flow_item_vlan_mask,
340 .mask_sz = sizeof(struct rte_flow_item_vlan),
341 .convert = mlx5_flow_create_vlan,
344 [RTE_FLOW_ITEM_TYPE_IPV4] = {
345 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
346 RTE_FLOW_ITEM_TYPE_TCP),
347 .actions = valid_actions,
348 .mask = &(const struct rte_flow_item_ipv4){
352 .type_of_service = -1,
356 .default_mask = &rte_flow_item_ipv4_mask,
357 .mask_sz = sizeof(struct rte_flow_item_ipv4),
358 .convert = mlx5_flow_create_ipv4,
359 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
361 [RTE_FLOW_ITEM_TYPE_IPV6] = {
362 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
363 RTE_FLOW_ITEM_TYPE_TCP),
364 .actions = valid_actions,
365 .mask = &(const struct rte_flow_item_ipv6){
368 0xff, 0xff, 0xff, 0xff,
369 0xff, 0xff, 0xff, 0xff,
370 0xff, 0xff, 0xff, 0xff,
371 0xff, 0xff, 0xff, 0xff,
374 0xff, 0xff, 0xff, 0xff,
375 0xff, 0xff, 0xff, 0xff,
376 0xff, 0xff, 0xff, 0xff,
377 0xff, 0xff, 0xff, 0xff,
384 .default_mask = &rte_flow_item_ipv6_mask,
385 .mask_sz = sizeof(struct rte_flow_item_ipv6),
386 .convert = mlx5_flow_create_ipv6,
387 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
389 [RTE_FLOW_ITEM_TYPE_UDP] = {
390 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
391 .actions = valid_actions,
392 .mask = &(const struct rte_flow_item_udp){
398 .default_mask = &rte_flow_item_udp_mask,
399 .mask_sz = sizeof(struct rte_flow_item_udp),
400 .convert = mlx5_flow_create_udp,
401 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
403 [RTE_FLOW_ITEM_TYPE_TCP] = {
404 .actions = valid_actions,
405 .mask = &(const struct rte_flow_item_tcp){
411 .default_mask = &rte_flow_item_tcp_mask,
412 .mask_sz = sizeof(struct rte_flow_item_tcp),
413 .convert = mlx5_flow_create_tcp,
414 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
416 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
417 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
418 .actions = valid_actions,
419 .mask = &(const struct rte_flow_item_vxlan){
420 .vni = "\xff\xff\xff",
422 .default_mask = &rte_flow_item_vxlan_mask,
423 .mask_sz = sizeof(struct rte_flow_item_vxlan),
424 .convert = mlx5_flow_create_vxlan,
425 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
429 /** Structure to pass to the conversion function. */
430 struct mlx5_flow_parse {
431 uint32_t inner; /**< Set once VXLAN is encountered. */
432 uint32_t allmulti:1; /**< Set once allmulti dst MAC is encountered. */
434 /**< Whether resources should remain after a validate. */
435 uint32_t drop:1; /**< Target is a drop queue. */
436 uint32_t mark:1; /**< Mark is present in the flow. */
437 uint32_t count:1; /**< Count is present in the flow. */
438 uint32_t mark_id; /**< Mark identifier. */
439 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
440 uint16_t queues_n; /**< Number of entries in queue[]. */
441 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
442 uint8_t rss_key[40]; /**< copy of the RSS key. */
443 enum hash_rxq_type layer; /**< Last pattern layer detected. */
444 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
446 struct ibv_flow_attr *ibv_attr;
447 /**< Pointer to Verbs attributes. */
449 /**< Current position or total size of the attribute. */
450 } queue[RTE_DIM(hash_rxq_init)];
453 static const struct rte_flow_ops mlx5_flow_ops = {
454 .validate = mlx5_flow_validate,
455 .create = mlx5_flow_create,
456 .destroy = mlx5_flow_destroy,
457 .flush = mlx5_flow_flush,
458 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
459 .query = mlx5_flow_query,
463 .isolate = mlx5_flow_isolate,
466 /* Convert FDIR request to Generic flow. */
468 struct rte_flow_attr attr;
469 struct rte_flow_action actions[2];
470 struct rte_flow_item items[4];
471 struct rte_flow_item_eth l2;
472 struct rte_flow_item_eth l2_mask;
474 struct rte_flow_item_ipv4 ipv4;
475 struct rte_flow_item_ipv6 ipv6;
478 struct rte_flow_item_udp udp;
479 struct rte_flow_item_tcp tcp;
481 struct rte_flow_action_queue queue;
484 /* Verbs specification header. */
485 struct ibv_spec_header {
486 enum ibv_flow_spec_type type;
491 * Check support for a given item.
494 * Item specification.
496 * Bit-masks covering supported fields to compare with spec, last and mask in
499 * Bit-Mask size in bytes.
505 mlx5_flow_item_validate(const struct rte_flow_item *item,
506 const uint8_t *mask, unsigned int size)
510 if (!item->spec && (item->mask || item->last))
512 if (item->spec && !item->mask) {
514 const uint8_t *spec = item->spec;
516 for (i = 0; i < size; ++i)
517 if ((spec[i] | mask[i]) != mask[i])
520 if (item->last && !item->mask) {
522 const uint8_t *spec = item->last;
524 for (i = 0; i < size; ++i)
525 if ((spec[i] | mask[i]) != mask[i])
530 const uint8_t *spec = item->mask;
532 for (i = 0; i < size; ++i)
533 if ((spec[i] | mask[i]) != mask[i])
536 if (item->spec && item->last) {
539 const uint8_t *apply = mask;
544 for (i = 0; i < size; ++i) {
545 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
546 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
548 ret = memcmp(spec, last, size);
554 * Copy the RSS configuration from the user ones.
557 * Pointer to private structure.
559 * Internal parser structure.
561 * User RSS configuration to save.
564 * 0 on success, errno value on failure.
567 priv_flow_convert_rss_conf(struct priv *priv,
568 struct mlx5_flow_parse *parser,
569 const struct rte_eth_rss_conf *rss_conf)
571 const struct rte_eth_rss_conf *rss =
572 rss_conf ? rss_conf : &priv->rss_conf;
574 if (rss->rss_key_len > 40)
576 parser->rss_conf.rss_key_len = rss->rss_key_len;
577 parser->rss_conf.rss_hf = rss->rss_hf;
578 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
579 parser->rss_conf.rss_key = parser->rss_key;
584 * Extract attribute to the parser.
587 * Pointer to private structure.
589 * Flow rule attributes.
591 * Perform verbose error reporting if not NULL.
592 * @param[in, out] parser
593 * Internal parser structure.
596 * 0 on success, a negative errno value otherwise and rte_errno is set.
599 priv_flow_convert_attributes(struct priv *priv,
600 const struct rte_flow_attr *attr,
601 struct rte_flow_error *error,
602 struct mlx5_flow_parse *parser)
607 rte_flow_error_set(error, ENOTSUP,
608 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
610 "groups are not supported");
613 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
614 rte_flow_error_set(error, ENOTSUP,
615 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
617 "priorities are not supported");
621 rte_flow_error_set(error, ENOTSUP,
622 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
624 "egress is not supported");
627 if (!attr->ingress) {
628 rte_flow_error_set(error, ENOTSUP,
629 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
631 "only ingress is supported");
638 * Extract actions request to the parser.
641 * Pointer to private structure.
643 * Associated actions (list terminated by the END action).
645 * Perform verbose error reporting if not NULL.
646 * @param[in, out] parser
647 * Internal parser structure.
650 * 0 on success, a negative errno value otherwise and rte_errno is set.
653 priv_flow_convert_actions(struct priv *priv,
654 const struct rte_flow_action actions[],
655 struct rte_flow_error *error,
656 struct mlx5_flow_parse *parser)
659 * Add default RSS configuration necessary for Verbs to create QP even
660 * if no RSS is necessary.
662 priv_flow_convert_rss_conf(priv, parser,
663 (const struct rte_eth_rss_conf *)
665 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
666 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
668 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
670 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
671 const struct rte_flow_action_queue *queue =
672 (const struct rte_flow_action_queue *)
677 if (!queue || (queue->index > (priv->rxqs_n - 1)))
678 goto exit_action_not_supported;
679 for (n = 0; n < parser->queues_n; ++n) {
680 if (parser->queues[n] == queue->index) {
685 if (parser->queues_n > 1 && !found) {
686 rte_flow_error_set(error, ENOTSUP,
687 RTE_FLOW_ERROR_TYPE_ACTION,
689 "queue action not in RSS queues");
693 parser->queues_n = 1;
694 parser->queues[0] = queue->index;
696 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
697 const struct rte_flow_action_rss *rss =
698 (const struct rte_flow_action_rss *)
702 if (!rss || !rss->num) {
703 rte_flow_error_set(error, EINVAL,
704 RTE_FLOW_ERROR_TYPE_ACTION,
709 if (parser->queues_n == 1) {
712 assert(parser->queues_n);
713 for (n = 0; n < rss->num; ++n) {
714 if (parser->queues[0] ==
721 rte_flow_error_set(error, ENOTSUP,
722 RTE_FLOW_ERROR_TYPE_ACTION,
724 "queue action not in RSS"
729 for (n = 0; n < rss->num; ++n) {
730 if (rss->queue[n] >= priv->rxqs_n) {
731 rte_flow_error_set(error, EINVAL,
732 RTE_FLOW_ERROR_TYPE_ACTION,
734 "queue id > number of"
739 for (n = 0; n < rss->num; ++n)
740 parser->queues[n] = rss->queue[n];
741 parser->queues_n = rss->num;
742 if (priv_flow_convert_rss_conf(priv, parser,
744 rte_flow_error_set(error, EINVAL,
745 RTE_FLOW_ERROR_TYPE_ACTION,
747 "wrong RSS configuration");
750 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
751 const struct rte_flow_action_mark *mark =
752 (const struct rte_flow_action_mark *)
756 rte_flow_error_set(error, EINVAL,
757 RTE_FLOW_ERROR_TYPE_ACTION,
759 "mark must be defined");
761 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
762 rte_flow_error_set(error, ENOTSUP,
763 RTE_FLOW_ERROR_TYPE_ACTION,
765 "mark must be between 0"
770 parser->mark_id = mark->id;
771 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
773 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
774 priv->counter_set_supported) {
777 goto exit_action_not_supported;
780 if (parser->drop && parser->mark)
782 if (!parser->queues_n && !parser->drop) {
783 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
784 NULL, "no valid action");
788 exit_action_not_supported:
789 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
790 actions, "action not supported");
798 * Pointer to private structure.
800 * Pattern specification (list terminated by the END pattern item).
802 * Perform verbose error reporting if not NULL.
803 * @param[in, out] parser
804 * Internal parser structure.
807 * 0 on success, a negative errno value otherwise and rte_errno is set.
810 priv_flow_convert_items_validate(struct priv *priv,
811 const struct rte_flow_item items[],
812 struct rte_flow_error *error,
813 struct mlx5_flow_parse *parser)
815 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
819 /* Initialise the offsets to start after verbs attribute. */
820 for (i = 0; i != hash_rxq_init_n; ++i)
821 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
822 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
823 const struct mlx5_flow_items *token = NULL;
827 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
831 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
833 if (cur_item->items[i] == items->type) {
834 token = &mlx5_flow_items[items->type];
839 goto exit_item_not_supported;
841 err = mlx5_flow_item_validate(items,
842 (const uint8_t *)cur_item->mask,
845 goto exit_item_not_supported;
846 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
848 rte_flow_error_set(error, ENOTSUP,
849 RTE_FLOW_ERROR_TYPE_ITEM,
851 "cannot recognize multiple"
852 " VXLAN encapsulations");
855 parser->inner = IBV_FLOW_SPEC_INNER;
857 if (parser->drop || parser->queues_n == 1) {
858 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
860 for (n = 0; n != hash_rxq_init_n; ++n)
861 parser->queue[n].offset += cur_item->dst_sz;
865 for (i = 0; i != hash_rxq_init_n; ++i)
866 parser->queue[i].offset +=
867 sizeof(struct ibv_flow_spec_action_tag);
870 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
872 for (i = 0; i != hash_rxq_init_n; ++i)
873 parser->queue[i].offset += size;
876 exit_item_not_supported:
877 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
878 items, "item not supported");
883 * Allocate memory space to store verbs flow attributes.
886 * Pointer to private structure.
887 * @param[in] priority
890 * Amount of byte to allocate.
892 * Perform verbose error reporting if not NULL.
895 * A verbs flow attribute on success, NULL otherwise.
897 static struct ibv_flow_attr*
898 priv_flow_convert_allocate(struct priv *priv,
899 unsigned int priority,
901 struct rte_flow_error *error)
903 struct ibv_flow_attr *ibv_attr;
906 ibv_attr = rte_calloc(__func__, 1, size, 0);
908 rte_flow_error_set(error, ENOMEM,
909 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
911 "cannot allocate verbs spec attributes.");
914 ibv_attr->priority = priority;
919 * Finalise verbs flow attributes.
922 * Pointer to private structure.
923 * @param[in, out] parser
924 * Internal parser structure.
927 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
929 const unsigned int ipv4 =
930 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
931 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
932 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
933 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
934 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
935 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
939 if (parser->layer == HASH_RXQ_ETH) {
943 * This layer becomes useless as the pattern define under
946 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
947 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
949 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
950 for (i = ohmin; i != (ohmax + 1); ++i) {
951 if (!parser->queue[i].ibv_attr)
953 rte_free(parser->queue[i].ibv_attr);
954 parser->queue[i].ibv_attr = NULL;
956 /* Remove impossible flow according to the RSS configuration. */
957 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
958 parser->rss_conf.rss_hf) {
959 /* Remove any other flow. */
960 for (i = hmin; i != (hmax + 1); ++i) {
961 if ((i == parser->layer) ||
962 (!parser->queue[i].ibv_attr))
964 rte_free(parser->queue[i].ibv_attr);
965 parser->queue[i].ibv_attr = NULL;
967 } else if (!parser->queue[ip].ibv_attr) {
968 /* no RSS possible with the current configuration. */
969 parser->queues_n = 1;
974 * Fill missing layers in verbs specifications, or compute the correct
975 * offset to allocate the memory space for the attributes and
978 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
980 struct ibv_flow_spec_ipv4_ext ipv4;
981 struct ibv_flow_spec_ipv6 ipv6;
982 struct ibv_flow_spec_tcp_udp udp_tcp;
987 if (i == parser->layer)
989 if (parser->layer == HASH_RXQ_ETH) {
990 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
991 size = sizeof(struct ibv_flow_spec_ipv4_ext);
992 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
993 .type = IBV_FLOW_SPEC_IPV4_EXT,
997 size = sizeof(struct ibv_flow_spec_ipv6);
998 specs.ipv6 = (struct ibv_flow_spec_ipv6){
999 .type = IBV_FLOW_SPEC_IPV6,
1003 if (parser->queue[i].ibv_attr) {
1004 dst = (void *)((uintptr_t)
1005 parser->queue[i].ibv_attr +
1006 parser->queue[i].offset);
1007 memcpy(dst, &specs, size);
1008 ++parser->queue[i].ibv_attr->num_of_specs;
1010 parser->queue[i].offset += size;
1012 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1013 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1014 size = sizeof(struct ibv_flow_spec_tcp_udp);
1015 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1016 .type = ((i == HASH_RXQ_UDPV4 ||
1017 i == HASH_RXQ_UDPV6) ?
1022 if (parser->queue[i].ibv_attr) {
1023 dst = (void *)((uintptr_t)
1024 parser->queue[i].ibv_attr +
1025 parser->queue[i].offset);
1026 memcpy(dst, &specs, size);
1027 ++parser->queue[i].ibv_attr->num_of_specs;
1029 parser->queue[i].offset += size;
1035 * Validate and convert a flow supported by the NIC.
1038 * Pointer to private structure.
1040 * Flow rule attributes.
1041 * @param[in] pattern
1042 * Pattern specification (list terminated by the END pattern item).
1043 * @param[in] actions
1044 * Associated actions (list terminated by the END action).
1046 * Perform verbose error reporting if not NULL.
1047 * @param[in, out] parser
1048 * Internal parser structure.
1051 * 0 on success, a negative errno value otherwise and rte_errno is set.
1054 priv_flow_convert(struct priv *priv,
1055 const struct rte_flow_attr *attr,
1056 const struct rte_flow_item items[],
1057 const struct rte_flow_action actions[],
1058 struct rte_flow_error *error,
1059 struct mlx5_flow_parse *parser)
1061 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1065 /* First step. Validate the attributes, items and actions. */
1066 *parser = (struct mlx5_flow_parse){
1067 .create = parser->create,
1068 .layer = HASH_RXQ_ETH,
1069 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1071 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1074 ret = priv_flow_convert_actions(priv, actions, error, parser);
1077 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1080 priv_flow_convert_finalise(priv, parser);
1083 * Allocate the memory space to store verbs specifications.
1085 if (parser->drop || parser->queues_n == 1) {
1086 unsigned int priority =
1088 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1089 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1091 parser->queue[HASH_RXQ_ETH].ibv_attr =
1092 priv_flow_convert_allocate(priv, priority,
1094 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1096 parser->queue[HASH_RXQ_ETH].offset =
1097 sizeof(struct ibv_flow_attr);
1099 for (i = 0; i != hash_rxq_init_n; ++i) {
1100 unsigned int priority =
1102 hash_rxq_init[i].flow_priority;
1103 unsigned int offset;
1105 if (!(parser->rss_conf.rss_hf &
1106 hash_rxq_init[i].dpdk_rss_hf) &&
1107 (i != HASH_RXQ_ETH))
1109 offset = parser->queue[i].offset;
1110 parser->queue[i].ibv_attr =
1111 priv_flow_convert_allocate(priv, priority,
1113 if (!parser->queue[i].ibv_attr)
1115 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1118 /* Third step. Conversion parse, fill the specifications. */
1120 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1121 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1123 cur_item = &mlx5_flow_items[items->type];
1124 ret = cur_item->convert(items,
1125 (cur_item->default_mask ?
1126 cur_item->default_mask :
1130 rte_flow_error_set(error, ret,
1131 RTE_FLOW_ERROR_TYPE_ITEM,
1132 items, "item not supported");
1137 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1138 if (parser->count && parser->create) {
1139 mlx5_flow_create_count(priv, parser);
1141 goto exit_count_error;
1144 * Last step. Complete missing specification to reach the RSS
1147 if (parser->queues_n > 1) {
1148 priv_flow_convert_finalise(priv, parser);
1151 * Action queue have their priority overridden with
1152 * Ethernet priority, this priority needs to be adjusted to
1153 * their most specific layer priority.
1155 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1157 hash_rxq_init[parser->layer].flow_priority;
1160 /* Only verification is expected, all resources should be released. */
1161 if (!parser->create) {
1162 for (i = 0; i != hash_rxq_init_n; ++i) {
1163 if (parser->queue[i].ibv_attr) {
1164 rte_free(parser->queue[i].ibv_attr);
1165 parser->queue[i].ibv_attr = NULL;
1169 if (parser->allmulti &&
1170 parser->layer == HASH_RXQ_ETH) {
1171 for (i = 0; i != hash_rxq_init_n; ++i) {
1172 if (!parser->queue[i].ibv_attr)
1174 if (parser->queue[i].ibv_attr->num_of_specs != 1)
1176 parser->queue[i].ibv_attr->type =
1177 IBV_FLOW_ATTR_MC_DEFAULT;
1182 for (i = 0; i != hash_rxq_init_n; ++i) {
1183 if (parser->queue[i].ibv_attr) {
1184 rte_free(parser->queue[i].ibv_attr);
1185 parser->queue[i].ibv_attr = NULL;
1188 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1189 NULL, "cannot allocate verbs spec attributes.");
1192 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1193 NULL, "cannot create counter.");
1198 * Copy the specification created into the flow.
1201 * Internal parser structure.
1203 * Create specification.
1205 * Size in bytes of the specification to copy.
1208 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1214 for (i = 0; i != hash_rxq_init_n; ++i) {
1215 if (!parser->queue[i].ibv_attr)
1217 /* Specification must be the same l3 type or none. */
1218 if (parser->layer == HASH_RXQ_ETH ||
1219 (hash_rxq_init[parser->layer].ip_version ==
1220 hash_rxq_init[i].ip_version) ||
1221 (hash_rxq_init[i].ip_version == 0)) {
1222 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1223 parser->queue[i].offset);
1224 memcpy(dst, src, size);
1225 ++parser->queue[i].ibv_attr->num_of_specs;
1226 parser->queue[i].offset += size;
1232 * Convert Ethernet item to Verbs specification.
1235 * Item specification.
1236 * @param default_mask[in]
1237 * Default bit-masks to use when item->mask is not provided.
1238 * @param data[in, out]
1242 mlx5_flow_create_eth(const struct rte_flow_item *item,
1243 const void *default_mask,
1246 const struct rte_flow_item_eth *spec = item->spec;
1247 const struct rte_flow_item_eth *mask = item->mask;
1248 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1249 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1250 struct ibv_flow_spec_eth eth = {
1251 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1255 /* Don't update layer for the inner pattern. */
1257 parser->layer = HASH_RXQ_ETH;
1262 mask = default_mask;
1263 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1264 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1265 eth.val.ether_type = spec->type;
1266 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1267 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1268 eth.mask.ether_type = mask->type;
1269 /* Remove unwanted bits from values. */
1270 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1271 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1272 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1274 eth.val.ether_type &= eth.mask.ether_type;
1276 mlx5_flow_create_copy(parser, ð, eth_size);
1277 parser->allmulti = eth.val.dst_mac[0] & 1;
1282 * Convert VLAN item to Verbs specification.
1285 * Item specification.
1286 * @param default_mask[in]
1287 * Default bit-masks to use when item->mask is not provided.
1288 * @param data[in, out]
1292 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1293 const void *default_mask,
1296 const struct rte_flow_item_vlan *spec = item->spec;
1297 const struct rte_flow_item_vlan *mask = item->mask;
1298 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1299 struct ibv_flow_spec_eth *eth;
1300 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1305 mask = default_mask;
1307 for (i = 0; i != hash_rxq_init_n; ++i) {
1308 if (!parser->queue[i].ibv_attr)
1311 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1312 parser->queue[i].offset - eth_size);
1313 eth->val.vlan_tag = spec->tci;
1314 eth->mask.vlan_tag = mask->tci;
1315 eth->val.vlan_tag &= eth->mask.vlan_tag;
1322 * Convert IPv4 item to Verbs specification.
1325 * Item specification.
1326 * @param default_mask[in]
1327 * Default bit-masks to use when item->mask is not provided.
1328 * @param data[in, out]
1332 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1333 const void *default_mask,
1336 const struct rte_flow_item_ipv4 *spec = item->spec;
1337 const struct rte_flow_item_ipv4 *mask = item->mask;
1338 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1339 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1340 struct ibv_flow_spec_ipv4_ext ipv4 = {
1341 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1345 /* Don't update layer for the inner pattern. */
1347 parser->layer = HASH_RXQ_IPV4;
1350 mask = default_mask;
1351 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1352 .src_ip = spec->hdr.src_addr,
1353 .dst_ip = spec->hdr.dst_addr,
1354 .proto = spec->hdr.next_proto_id,
1355 .tos = spec->hdr.type_of_service,
1357 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1358 .src_ip = mask->hdr.src_addr,
1359 .dst_ip = mask->hdr.dst_addr,
1360 .proto = mask->hdr.next_proto_id,
1361 .tos = mask->hdr.type_of_service,
1363 /* Remove unwanted bits from values. */
1364 ipv4.val.src_ip &= ipv4.mask.src_ip;
1365 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1366 ipv4.val.proto &= ipv4.mask.proto;
1367 ipv4.val.tos &= ipv4.mask.tos;
1369 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1374 * Convert IPv6 item to Verbs specification.
1377 * Item specification.
1378 * @param default_mask[in]
1379 * Default bit-masks to use when item->mask is not provided.
1380 * @param data[in, out]
1384 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1385 const void *default_mask,
1388 const struct rte_flow_item_ipv6 *spec = item->spec;
1389 const struct rte_flow_item_ipv6 *mask = item->mask;
1390 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1391 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1392 struct ibv_flow_spec_ipv6 ipv6 = {
1393 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1397 /* Don't update layer for the inner pattern. */
1399 parser->layer = HASH_RXQ_IPV6;
1404 mask = default_mask;
1405 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1406 RTE_DIM(ipv6.val.src_ip));
1407 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1408 RTE_DIM(ipv6.val.dst_ip));
1409 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1410 RTE_DIM(ipv6.mask.src_ip));
1411 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1412 RTE_DIM(ipv6.mask.dst_ip));
1413 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1414 ipv6.mask.next_hdr = mask->hdr.proto;
1415 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1416 /* Remove unwanted bits from values. */
1417 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1418 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1419 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1421 ipv6.val.flow_label &= ipv6.mask.flow_label;
1422 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1423 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1425 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1430 * Convert UDP item to Verbs specification.
1433 * Item specification.
1434 * @param default_mask[in]
1435 * Default bit-masks to use when item->mask is not provided.
1436 * @param data[in, out]
1440 mlx5_flow_create_udp(const struct rte_flow_item *item,
1441 const void *default_mask,
1444 const struct rte_flow_item_udp *spec = item->spec;
1445 const struct rte_flow_item_udp *mask = item->mask;
1446 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1447 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1448 struct ibv_flow_spec_tcp_udp udp = {
1449 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1453 /* Don't update layer for the inner pattern. */
1454 if (!parser->inner) {
1455 if (parser->layer == HASH_RXQ_IPV4)
1456 parser->layer = HASH_RXQ_UDPV4;
1458 parser->layer = HASH_RXQ_UDPV6;
1462 mask = default_mask;
1463 udp.val.dst_port = spec->hdr.dst_port;
1464 udp.val.src_port = spec->hdr.src_port;
1465 udp.mask.dst_port = mask->hdr.dst_port;
1466 udp.mask.src_port = mask->hdr.src_port;
1467 /* Remove unwanted bits from values. */
1468 udp.val.src_port &= udp.mask.src_port;
1469 udp.val.dst_port &= udp.mask.dst_port;
1471 mlx5_flow_create_copy(parser, &udp, udp_size);
1476 * Convert TCP item to Verbs specification.
1479 * Item specification.
1480 * @param default_mask[in]
1481 * Default bit-masks to use when item->mask is not provided.
1482 * @param data[in, out]
1486 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1487 const void *default_mask,
1490 const struct rte_flow_item_tcp *spec = item->spec;
1491 const struct rte_flow_item_tcp *mask = item->mask;
1492 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1493 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1494 struct ibv_flow_spec_tcp_udp tcp = {
1495 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1499 /* Don't update layer for the inner pattern. */
1500 if (!parser->inner) {
1501 if (parser->layer == HASH_RXQ_IPV4)
1502 parser->layer = HASH_RXQ_TCPV4;
1504 parser->layer = HASH_RXQ_TCPV6;
1508 mask = default_mask;
1509 tcp.val.dst_port = spec->hdr.dst_port;
1510 tcp.val.src_port = spec->hdr.src_port;
1511 tcp.mask.dst_port = mask->hdr.dst_port;
1512 tcp.mask.src_port = mask->hdr.src_port;
1513 /* Remove unwanted bits from values. */
1514 tcp.val.src_port &= tcp.mask.src_port;
1515 tcp.val.dst_port &= tcp.mask.dst_port;
1517 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1522 * Convert VXLAN item to Verbs specification.
1525 * Item specification.
1526 * @param default_mask[in]
1527 * Default bit-masks to use when item->mask is not provided.
1528 * @param data[in, out]
1532 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1533 const void *default_mask,
1536 const struct rte_flow_item_vxlan *spec = item->spec;
1537 const struct rte_flow_item_vxlan *mask = item->mask;
1538 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1539 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1540 struct ibv_flow_spec_tunnel vxlan = {
1541 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1550 parser->inner = IBV_FLOW_SPEC_INNER;
1553 mask = default_mask;
1554 memcpy(&id.vni[1], spec->vni, 3);
1555 vxlan.val.tunnel_id = id.vlan_id;
1556 memcpy(&id.vni[1], mask->vni, 3);
1557 vxlan.mask.tunnel_id = id.vlan_id;
1558 /* Remove unwanted bits from values. */
1559 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1562 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1563 * layer is defined in the Verbs specification it is interpreted as
1564 * wildcard and all packets will match this rule, if it follows a full
1565 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1566 * before will also match this rule.
1567 * To avoid such situation, VNI 0 is currently refused.
1569 if (!vxlan.val.tunnel_id)
1571 mlx5_flow_create_copy(parser, &vxlan, size);
1576 * Convert mark/flag action to Verbs specification.
1579 * Internal parser structure.
1584 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1586 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1587 struct ibv_flow_spec_action_tag tag = {
1588 .type = IBV_FLOW_SPEC_ACTION_TAG,
1590 .tag_id = mlx5_flow_mark_set(mark_id),
1593 assert(parser->mark);
1594 mlx5_flow_create_copy(parser, &tag, size);
1599 * Convert count action to Verbs specification.
1602 * Pointer to private structure.
1604 * Pointer to MLX5 flow parser structure.
1607 * 0 on success, errno value on failure.
1610 mlx5_flow_create_count(struct priv *priv __rte_unused,
1611 struct mlx5_flow_parse *parser __rte_unused)
1613 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1614 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1615 struct ibv_counter_set_init_attr init_attr = {0};
1616 struct ibv_flow_spec_counter_action counter = {
1617 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1619 .counter_set_handle = 0,
1622 init_attr.counter_set_id = 0;
1623 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr);
1626 counter.counter_set_handle = parser->cs->handle;
1627 mlx5_flow_create_copy(parser, &counter, size);
1633 * Complete flow rule creation with a drop queue.
1636 * Pointer to private structure.
1638 * Internal parser structure.
1640 * Pointer to the rte_flow.
1642 * Perform verbose error reporting if not NULL.
1645 * 0 on success, errno value on failure.
1648 priv_flow_create_action_queue_drop(struct priv *priv,
1649 struct mlx5_flow_parse *parser,
1650 struct rte_flow *flow,
1651 struct rte_flow_error *error)
1653 struct ibv_flow_spec_action_drop *drop;
1654 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1660 drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
1661 parser->queue[HASH_RXQ_ETH].offset);
1662 *drop = (struct ibv_flow_spec_action_drop){
1663 .type = IBV_FLOW_SPEC_ACTION_DROP,
1666 ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
1667 parser->queue[HASH_RXQ_ETH].offset += size;
1668 flow->frxq[HASH_RXQ_ETH].ibv_attr =
1669 parser->queue[HASH_RXQ_ETH].ibv_attr;
1671 flow->cs = parser->cs;
1672 if (!priv->dev->data->dev_started)
1674 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
1675 flow->frxq[HASH_RXQ_ETH].ibv_flow =
1676 ibv_create_flow(priv->flow_drop_queue->qp,
1677 flow->frxq[HASH_RXQ_ETH].ibv_attr);
1678 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1679 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1680 NULL, "flow rule creation failure");
1687 if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
1688 claim_zero(ibv_destroy_flow(flow->frxq[HASH_RXQ_ETH].ibv_flow));
1689 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
1691 if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
1692 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
1693 flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
1696 claim_zero(ibv_destroy_counter_set(flow->cs));
1704 * Create hash Rx queues when RSS is enabled.
1707 * Pointer to private structure.
1709 * Internal parser structure.
1711 * Pointer to the rte_flow.
1713 * Perform verbose error reporting if not NULL.
1716 * 0 on success, a errno value otherwise and rte_errno is set.
1719 priv_flow_create_action_queue_rss(struct priv *priv,
1720 struct mlx5_flow_parse *parser,
1721 struct rte_flow *flow,
1722 struct rte_flow_error *error)
1726 for (i = 0; i != hash_rxq_init_n; ++i) {
1727 uint64_t hash_fields;
1729 if (!parser->queue[i].ibv_attr)
1731 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1732 parser->queue[i].ibv_attr = NULL;
1733 hash_fields = hash_rxq_init[i].hash_fields;
1734 if (!priv->dev->data->dev_started)
1736 flow->frxq[i].hrxq =
1737 mlx5_priv_hrxq_get(priv,
1738 parser->rss_conf.rss_key,
1739 parser->rss_conf.rss_key_len,
1743 if (flow->frxq[i].hrxq)
1745 flow->frxq[i].hrxq =
1746 mlx5_priv_hrxq_new(priv,
1747 parser->rss_conf.rss_key,
1748 parser->rss_conf.rss_key_len,
1752 if (!flow->frxq[i].hrxq) {
1753 rte_flow_error_set(error, ENOMEM,
1754 RTE_FLOW_ERROR_TYPE_HANDLE,
1755 NULL, "cannot create hash rxq");
1763 * Complete flow rule creation.
1766 * Pointer to private structure.
1768 * Internal parser structure.
1770 * Pointer to the rte_flow.
1772 * Perform verbose error reporting if not NULL.
1775 * 0 on success, a errno value otherwise and rte_errno is set.
1778 priv_flow_create_action_queue(struct priv *priv,
1779 struct mlx5_flow_parse *parser,
1780 struct rte_flow *flow,
1781 struct rte_flow_error *error)
1788 assert(!parser->drop);
1789 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1793 flow->cs = parser->cs;
1794 if (!priv->dev->data->dev_started)
1796 for (i = 0; i != hash_rxq_init_n; ++i) {
1797 if (!flow->frxq[i].hrxq)
1799 flow->frxq[i].ibv_flow =
1800 ibv_create_flow(flow->frxq[i].hrxq->qp,
1801 flow->frxq[i].ibv_attr);
1802 if (!flow->frxq[i].ibv_flow) {
1803 rte_flow_error_set(error, ENOMEM,
1804 RTE_FLOW_ERROR_TYPE_HANDLE,
1805 NULL, "flow rule creation failure");
1809 DEBUG("%p type %d QP %p ibv_flow %p",
1811 (void *)flow->frxq[i].hrxq,
1812 (void *)flow->frxq[i].ibv_flow);
1814 for (i = 0; i != parser->queues_n; ++i) {
1815 struct mlx5_rxq_data *q =
1816 (*priv->rxqs)[parser->queues[i]];
1818 q->mark |= parser->mark;
1823 for (i = 0; i != hash_rxq_init_n; ++i) {
1824 if (flow->frxq[i].ibv_flow) {
1825 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1827 claim_zero(ibv_destroy_flow(ibv_flow));
1829 if (flow->frxq[i].hrxq)
1830 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1831 if (flow->frxq[i].ibv_attr)
1832 rte_free(flow->frxq[i].ibv_attr);
1835 claim_zero(ibv_destroy_counter_set(flow->cs));
1846 * Pointer to private structure.
1848 * Pointer to a TAILQ flow list.
1850 * Flow rule attributes.
1851 * @param[in] pattern
1852 * Pattern specification (list terminated by the END pattern item).
1853 * @param[in] actions
1854 * Associated actions (list terminated by the END action).
1856 * Perform verbose error reporting if not NULL.
1859 * A flow on success, NULL otherwise.
1861 static struct rte_flow *
1862 priv_flow_create(struct priv *priv,
1863 struct mlx5_flows *list,
1864 const struct rte_flow_attr *attr,
1865 const struct rte_flow_item items[],
1866 const struct rte_flow_action actions[],
1867 struct rte_flow_error *error)
1869 struct mlx5_flow_parse parser = { .create = 1, };
1870 struct rte_flow *flow = NULL;
1874 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1877 flow = rte_calloc(__func__, 1,
1878 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1881 rte_flow_error_set(error, ENOMEM,
1882 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1884 "cannot allocate flow memory");
1887 /* Copy queues configuration. */
1888 flow->queues = (uint16_t (*)[])(flow + 1);
1889 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1890 flow->queues_n = parser.queues_n;
1891 flow->mark = parser.mark;
1892 /* Copy RSS configuration. */
1893 flow->rss_conf = parser.rss_conf;
1894 flow->rss_conf.rss_key = flow->rss_key;
1895 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1896 /* finalise the flow. */
1898 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1901 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1904 TAILQ_INSERT_TAIL(list, flow, next);
1905 DEBUG("Flow created %p", (void *)flow);
1908 for (i = 0; i != hash_rxq_init_n; ++i) {
1909 if (parser.queue[i].ibv_attr)
1910 rte_free(parser.queue[i].ibv_attr);
1917 * Validate a flow supported by the NIC.
1919 * @see rte_flow_validate()
1923 mlx5_flow_validate(struct rte_eth_dev *dev,
1924 const struct rte_flow_attr *attr,
1925 const struct rte_flow_item items[],
1926 const struct rte_flow_action actions[],
1927 struct rte_flow_error *error)
1929 struct priv *priv = dev->data->dev_private;
1931 struct mlx5_flow_parse parser = { .create = 0, };
1934 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1942 * @see rte_flow_create()
1946 mlx5_flow_create(struct rte_eth_dev *dev,
1947 const struct rte_flow_attr *attr,
1948 const struct rte_flow_item items[],
1949 const struct rte_flow_action actions[],
1950 struct rte_flow_error *error)
1952 struct priv *priv = dev->data->dev_private;
1953 struct rte_flow *flow;
1956 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1966 * Pointer to private structure.
1968 * Pointer to a TAILQ flow list.
1973 priv_flow_destroy(struct priv *priv,
1974 struct mlx5_flows *list,
1975 struct rte_flow *flow)
1979 if (flow->drop || !flow->mark)
1981 for (i = 0; i != flow->queues_n; ++i) {
1982 struct rte_flow *tmp;
1986 * To remove the mark from the queue, the queue must not be
1987 * present in any other marked flow (RSS or not).
1989 TAILQ_FOREACH(tmp, list, next) {
1991 uint16_t *tqs = NULL;
1996 for (j = 0; j != hash_rxq_init_n; ++j) {
1997 if (!tmp->frxq[j].hrxq)
1999 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2000 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2004 for (j = 0; (j != tq_n) && !mark; j++)
2005 if (tqs[j] == (*flow->queues)[i])
2008 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2012 if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
2013 claim_zero(ibv_destroy_flow
2014 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2015 rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
2017 for (i = 0; i != hash_rxq_init_n; ++i) {
2018 struct mlx5_flow *frxq = &flow->frxq[i];
2021 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
2023 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2025 rte_free(frxq->ibv_attr);
2029 claim_zero(ibv_destroy_counter_set(flow->cs));
2032 TAILQ_REMOVE(list, flow, next);
2033 DEBUG("Flow destroyed %p", (void *)flow);
2038 * Destroy all flows.
2041 * Pointer to private structure.
2043 * Pointer to a TAILQ flow list.
2046 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2048 while (!TAILQ_EMPTY(list)) {
2049 struct rte_flow *flow;
2051 flow = TAILQ_FIRST(list);
2052 priv_flow_destroy(priv, list, flow);
2057 * Create drop queue.
2060 * Pointer to private structure.
2066 priv_flow_create_drop_queue(struct priv *priv)
2068 struct mlx5_hrxq_drop *fdq = NULL;
2072 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2074 WARN("cannot allocate memory for drop queue");
2077 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
2079 WARN("cannot allocate CQ for drop queue");
2082 fdq->wq = ibv_create_wq(priv->ctx,
2083 &(struct ibv_wq_init_attr){
2084 .wq_type = IBV_WQT_RQ,
2091 WARN("cannot allocate WQ for drop queue");
2094 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
2095 &(struct ibv_rwq_ind_table_init_attr){
2096 .log_ind_tbl_size = 0,
2097 .ind_tbl = &fdq->wq,
2100 if (!fdq->ind_table) {
2101 WARN("cannot allocate indirection table for drop queue");
2104 fdq->qp = ibv_create_qp_ex(priv->ctx,
2105 &(struct ibv_qp_init_attr_ex){
2106 .qp_type = IBV_QPT_RAW_PACKET,
2108 IBV_QP_INIT_ATTR_PD |
2109 IBV_QP_INIT_ATTR_IND_TABLE |
2110 IBV_QP_INIT_ATTR_RX_HASH,
2111 .rx_hash_conf = (struct ibv_rx_hash_conf){
2113 IBV_RX_HASH_FUNC_TOEPLITZ,
2114 .rx_hash_key_len = rss_hash_default_key_len,
2115 .rx_hash_key = rss_hash_default_key,
2116 .rx_hash_fields_mask = 0,
2118 .rwq_ind_tbl = fdq->ind_table,
2122 WARN("cannot allocate QP for drop queue");
2125 priv->flow_drop_queue = fdq;
2129 claim_zero(ibv_destroy_qp(fdq->qp));
2131 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2133 claim_zero(ibv_destroy_wq(fdq->wq));
2135 claim_zero(ibv_destroy_cq(fdq->cq));
2138 priv->flow_drop_queue = NULL;
2143 * Delete drop queue.
2146 * Pointer to private structure.
2149 priv_flow_delete_drop_queue(struct priv *priv)
2151 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2156 claim_zero(ibv_destroy_qp(fdq->qp));
2158 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2160 claim_zero(ibv_destroy_wq(fdq->wq));
2162 claim_zero(ibv_destroy_cq(fdq->cq));
2164 priv->flow_drop_queue = NULL;
2171 * Pointer to private structure.
2173 * Pointer to a TAILQ flow list.
2176 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2178 struct rte_flow *flow;
2180 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2184 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
2186 claim_zero(ibv_destroy_flow
2187 (flow->frxq[HASH_RXQ_ETH].ibv_flow));
2188 flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
2193 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2195 for (i = 0; i != hash_rxq_init_n; ++i) {
2196 if (!flow->frxq[i].hrxq)
2198 ind_tbl = flow->frxq[i].hrxq->ind_table;
2201 for (i = 0; i != ind_tbl->queues_n; ++i)
2202 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2204 for (i = 0; i != hash_rxq_init_n; ++i) {
2205 if (!flow->frxq[i].ibv_flow)
2207 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2208 flow->frxq[i].ibv_flow = NULL;
2209 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2210 flow->frxq[i].hrxq = NULL;
2212 DEBUG("Flow %p removed", (void *)flow);
2220 * Pointer to private structure.
2222 * Pointer to a TAILQ flow list.
2225 * 0 on success, a errno value otherwise and rte_errno is set.
2228 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2230 struct rte_flow *flow;
2232 TAILQ_FOREACH(flow, list, next) {
2236 flow->frxq[HASH_RXQ_ETH].ibv_flow =
2238 (priv->flow_drop_queue->qp,
2239 flow->frxq[HASH_RXQ_ETH].ibv_attr);
2240 if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
2241 DEBUG("Flow %p cannot be applied",
2246 DEBUG("Flow %p applied", (void *)flow);
2250 for (i = 0; i != hash_rxq_init_n; ++i) {
2251 if (!flow->frxq[i].ibv_attr)
2253 flow->frxq[i].hrxq =
2254 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2255 flow->rss_conf.rss_key_len,
2256 hash_rxq_init[i].hash_fields,
2259 if (flow->frxq[i].hrxq)
2261 flow->frxq[i].hrxq =
2262 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2263 flow->rss_conf.rss_key_len,
2264 hash_rxq_init[i].hash_fields,
2267 if (!flow->frxq[i].hrxq) {
2268 DEBUG("Flow %p cannot be applied",
2274 flow->frxq[i].ibv_flow =
2275 ibv_create_flow(flow->frxq[i].hrxq->qp,
2276 flow->frxq[i].ibv_attr);
2277 if (!flow->frxq[i].ibv_flow) {
2278 DEBUG("Flow %p cannot be applied",
2283 DEBUG("Flow %p applied", (void *)flow);
2287 for (i = 0; i != flow->queues_n; ++i)
2288 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2294 * Verify the flow list is empty
2297 * Pointer to private structure.
2299 * @return the number of flows not released.
2302 priv_flow_verify(struct priv *priv)
2304 struct rte_flow *flow;
2307 TAILQ_FOREACH(flow, &priv->flows, next) {
2308 DEBUG("%p: flow %p still referenced", (void *)priv,
2316 * Enable a control flow configured from the control plane.
2319 * Pointer to Ethernet device.
2321 * An Ethernet flow spec to apply.
2323 * An Ethernet flow mask to apply.
2325 * A VLAN flow spec to apply.
2327 * A VLAN flow mask to apply.
2333 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2334 struct rte_flow_item_eth *eth_spec,
2335 struct rte_flow_item_eth *eth_mask,
2336 struct rte_flow_item_vlan *vlan_spec,
2337 struct rte_flow_item_vlan *vlan_mask)
2339 struct priv *priv = dev->data->dev_private;
2340 const struct rte_flow_attr attr = {
2342 .priority = MLX5_CTRL_FLOW_PRIORITY,
2344 struct rte_flow_item items[] = {
2346 .type = RTE_FLOW_ITEM_TYPE_ETH,
2352 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2353 RTE_FLOW_ITEM_TYPE_END,
2359 .type = RTE_FLOW_ITEM_TYPE_END,
2362 struct rte_flow_action actions[] = {
2364 .type = RTE_FLOW_ACTION_TYPE_RSS,
2367 .type = RTE_FLOW_ACTION_TYPE_END,
2370 struct rte_flow *flow;
2371 struct rte_flow_error error;
2374 struct rte_flow_action_rss rss;
2376 const struct rte_eth_rss_conf *rss_conf;
2378 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2382 if (!priv->reta_idx_n)
2384 for (i = 0; i != priv->reta_idx_n; ++i)
2385 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2386 action_rss.local.rss_conf = &priv->rss_conf;
2387 action_rss.local.num = priv->reta_idx_n;
2388 actions[0].conf = (const void *)&action_rss.rss;
2389 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2397 * Enable a flow control configured from the control plane.
2400 * Pointer to Ethernet device.
2402 * An Ethernet flow spec to apply.
2404 * An Ethernet flow mask to apply.
2410 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2411 struct rte_flow_item_eth *eth_spec,
2412 struct rte_flow_item_eth *eth_mask)
2414 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2420 * @see rte_flow_destroy()
2424 mlx5_flow_destroy(struct rte_eth_dev *dev,
2425 struct rte_flow *flow,
2426 struct rte_flow_error *error)
2428 struct priv *priv = dev->data->dev_private;
2432 priv_flow_destroy(priv, &priv->flows, flow);
2438 * Destroy all flows.
2440 * @see rte_flow_flush()
2444 mlx5_flow_flush(struct rte_eth_dev *dev,
2445 struct rte_flow_error *error)
2447 struct priv *priv = dev->data->dev_private;
2451 priv_flow_flush(priv, &priv->flows);
2456 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2458 * Query flow counter.
2462 * @param counter_value
2463 * returned data from the counter.
2466 * 0 on success, a errno value otherwise and rte_errno is set.
2469 priv_flow_query_count(struct ibv_counter_set *cs,
2470 struct mlx5_flow_counter_stats *counter_stats,
2471 struct rte_flow_query_count *query_count,
2472 struct rte_flow_error *error)
2474 uint64_t counters[2];
2475 struct ibv_query_counter_set_attr query_cs_attr = {
2477 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2479 struct ibv_counter_set_data query_out = {
2481 .outlen = 2 * sizeof(uint64_t),
2483 int res = ibv_query_counter_set(&query_cs_attr, &query_out);
2486 rte_flow_error_set(error, -res,
2487 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2489 "cannot read counter");
2492 query_count->hits_set = 1;
2493 query_count->bytes_set = 1;
2494 query_count->hits = counters[0] - counter_stats->hits;
2495 query_count->bytes = counters[1] - counter_stats->bytes;
2496 if (query_count->reset) {
2497 counter_stats->hits = counters[0];
2498 counter_stats->bytes = counters[1];
2506 * @see rte_flow_query()
2510 mlx5_flow_query(struct rte_eth_dev *dev,
2511 struct rte_flow *flow,
2512 enum rte_flow_action_type action __rte_unused,
2514 struct rte_flow_error *error)
2516 struct priv *priv = dev->data->dev_private;
2521 res = priv_flow_query_count(flow->cs,
2522 &flow->counter_stats,
2523 (struct rte_flow_query_count *)data,
2526 rte_flow_error_set(error, res,
2527 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2529 "no counter found for flow");
2539 * @see rte_flow_isolate()
2543 mlx5_flow_isolate(struct rte_eth_dev *dev,
2545 struct rte_flow_error *error)
2547 struct priv *priv = dev->data->dev_private;
2550 if (dev->data->dev_started) {
2551 rte_flow_error_set(error, EBUSY,
2552 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2554 "port must be stopped first");
2558 priv->isolated = !!enable;
2560 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2562 priv->dev->dev_ops = &mlx5_dev_ops;
2568 * Convert a flow director filter to a generic flow.
2571 * Private structure.
2572 * @param fdir_filter
2573 * Flow director filter to add.
2575 * Generic flow parameters structure.
2578 * 0 on success, errno value on error.
2581 priv_fdir_filter_convert(struct priv *priv,
2582 const struct rte_eth_fdir_filter *fdir_filter,
2583 struct mlx5_fdir *attributes)
2585 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2587 /* Validate queue number. */
2588 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2589 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2592 attributes->attr.ingress = 1;
2593 attributes->items[0] = (struct rte_flow_item) {
2594 .type = RTE_FLOW_ITEM_TYPE_ETH,
2595 .spec = &attributes->l2,
2596 .mask = &attributes->l2_mask,
2598 switch (fdir_filter->action.behavior) {
2599 case RTE_ETH_FDIR_ACCEPT:
2600 attributes->actions[0] = (struct rte_flow_action){
2601 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2602 .conf = &attributes->queue,
2605 case RTE_ETH_FDIR_REJECT:
2606 attributes->actions[0] = (struct rte_flow_action){
2607 .type = RTE_FLOW_ACTION_TYPE_DROP,
2611 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2614 attributes->queue.index = fdir_filter->action.rx_queue;
2615 switch (fdir_filter->input.flow_type) {
2616 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2617 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2618 .src_addr = input->flow.udp4_flow.ip.src_ip,
2619 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2620 .time_to_live = input->flow.udp4_flow.ip.ttl,
2621 .type_of_service = input->flow.udp4_flow.ip.tos,
2622 .next_proto_id = input->flow.udp4_flow.ip.proto,
2624 attributes->l4.udp.hdr = (struct udp_hdr){
2625 .src_port = input->flow.udp4_flow.src_port,
2626 .dst_port = input->flow.udp4_flow.dst_port,
2628 attributes->items[1] = (struct rte_flow_item){
2629 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2630 .spec = &attributes->l3,
2632 attributes->items[2] = (struct rte_flow_item){
2633 .type = RTE_FLOW_ITEM_TYPE_UDP,
2634 .spec = &attributes->l4,
2637 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2638 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2639 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2640 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2641 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2642 .type_of_service = input->flow.tcp4_flow.ip.tos,
2643 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2645 attributes->l4.tcp.hdr = (struct tcp_hdr){
2646 .src_port = input->flow.tcp4_flow.src_port,
2647 .dst_port = input->flow.tcp4_flow.dst_port,
2649 attributes->items[1] = (struct rte_flow_item){
2650 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2651 .spec = &attributes->l3,
2653 attributes->items[2] = (struct rte_flow_item){
2654 .type = RTE_FLOW_ITEM_TYPE_TCP,
2655 .spec = &attributes->l4,
2658 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2659 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2660 .src_addr = input->flow.ip4_flow.src_ip,
2661 .dst_addr = input->flow.ip4_flow.dst_ip,
2662 .time_to_live = input->flow.ip4_flow.ttl,
2663 .type_of_service = input->flow.ip4_flow.tos,
2664 .next_proto_id = input->flow.ip4_flow.proto,
2666 attributes->items[1] = (struct rte_flow_item){
2667 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2668 .spec = &attributes->l3,
2671 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2672 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2673 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2674 .proto = input->flow.udp6_flow.ip.proto,
2676 memcpy(attributes->l3.ipv6.hdr.src_addr,
2677 input->flow.udp6_flow.ip.src_ip,
2678 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2679 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2680 input->flow.udp6_flow.ip.dst_ip,
2681 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2682 attributes->l4.udp.hdr = (struct udp_hdr){
2683 .src_port = input->flow.udp6_flow.src_port,
2684 .dst_port = input->flow.udp6_flow.dst_port,
2686 attributes->items[1] = (struct rte_flow_item){
2687 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2688 .spec = &attributes->l3,
2690 attributes->items[2] = (struct rte_flow_item){
2691 .type = RTE_FLOW_ITEM_TYPE_UDP,
2692 .spec = &attributes->l4,
2695 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2696 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2697 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2698 .proto = input->flow.tcp6_flow.ip.proto,
2700 memcpy(attributes->l3.ipv6.hdr.src_addr,
2701 input->flow.tcp6_flow.ip.src_ip,
2702 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2703 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2704 input->flow.tcp6_flow.ip.dst_ip,
2705 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2706 attributes->l4.tcp.hdr = (struct tcp_hdr){
2707 .src_port = input->flow.tcp6_flow.src_port,
2708 .dst_port = input->flow.tcp6_flow.dst_port,
2710 attributes->items[1] = (struct rte_flow_item){
2711 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2712 .spec = &attributes->l3,
2714 attributes->items[2] = (struct rte_flow_item){
2715 .type = RTE_FLOW_ITEM_TYPE_TCP,
2716 .spec = &attributes->l4,
2719 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2720 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2721 .hop_limits = input->flow.ipv6_flow.hop_limits,
2722 .proto = input->flow.ipv6_flow.proto,
2724 memcpy(attributes->l3.ipv6.hdr.src_addr,
2725 input->flow.ipv6_flow.src_ip,
2726 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2727 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2728 input->flow.ipv6_flow.dst_ip,
2729 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2730 attributes->items[1] = (struct rte_flow_item){
2731 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2732 .spec = &attributes->l3,
2736 ERROR("invalid flow type%d",
2737 fdir_filter->input.flow_type);
2744 * Add new flow director filter and store it in list.
2747 * Private structure.
2748 * @param fdir_filter
2749 * Flow director filter to add.
2752 * 0 on success, errno value on failure.
2755 priv_fdir_filter_add(struct priv *priv,
2756 const struct rte_eth_fdir_filter *fdir_filter)
2758 struct mlx5_fdir attributes = {
2761 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2762 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2766 struct mlx5_flow_parse parser = {
2767 .layer = HASH_RXQ_ETH,
2769 struct rte_flow_error error;
2770 struct rte_flow *flow;
2773 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2776 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2777 attributes.actions, &error, &parser);
2780 flow = priv_flow_create(priv,
2787 DEBUG("FDIR created %p", (void *)flow);
2794 * Delete specific filter.
2797 * Private structure.
2798 * @param fdir_filter
2799 * Filter to be deleted.
2802 * 0 on success, errno value on failure.
2805 priv_fdir_filter_delete(struct priv *priv,
2806 const struct rte_eth_fdir_filter *fdir_filter)
2808 struct mlx5_fdir attributes = {
2811 struct mlx5_flow_parse parser = {
2813 .layer = HASH_RXQ_ETH,
2815 struct rte_flow_error error;
2816 struct rte_flow *flow;
2820 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2823 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2824 attributes.actions, &error, &parser);
2828 * Special case for drop action which is only set in the
2829 * specifications when the flow is created. In this situation the
2830 * drop specification is missing.
2833 struct ibv_flow_spec_action_drop *drop;
2835 drop = (void *)((uintptr_t)parser.queue[HASH_RXQ_ETH].ibv_attr +
2836 parser.queue[HASH_RXQ_ETH].offset);
2837 *drop = (struct ibv_flow_spec_action_drop){
2838 .type = IBV_FLOW_SPEC_ACTION_DROP,
2839 .size = sizeof(struct ibv_flow_spec_action_drop),
2841 parser.queue[HASH_RXQ_ETH].ibv_attr->num_of_specs++;
2843 TAILQ_FOREACH(flow, &priv->flows, next) {
2844 struct ibv_flow_attr *attr;
2845 struct ibv_spec_header *attr_h;
2847 struct ibv_flow_attr *flow_attr;
2848 struct ibv_spec_header *flow_h;
2850 unsigned int specs_n;
2852 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2853 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2854 /* Compare first the attributes. */
2855 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2857 if (attr->num_of_specs == 0)
2859 spec = (void *)((uintptr_t)attr +
2860 sizeof(struct ibv_flow_attr));
2861 flow_spec = (void *)((uintptr_t)flow_attr +
2862 sizeof(struct ibv_flow_attr));
2863 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2864 for (i = 0; i != specs_n; ++i) {
2867 if (memcmp(spec, flow_spec,
2868 RTE_MIN(attr_h->size, flow_h->size)))
2870 spec = (void *)((uintptr_t)spec + attr_h->size);
2871 flow_spec = (void *)((uintptr_t)flow_spec +
2874 /* At this point, the flow match. */
2877 /* The flow does not match. */
2881 priv_flow_destroy(priv, &priv->flows, flow);
2883 for (i = 0; i != hash_rxq_init_n; ++i) {
2884 if (parser.queue[i].ibv_attr)
2885 rte_free(parser.queue[i].ibv_attr);
2891 * Update queue for specific filter.
2894 * Private structure.
2895 * @param fdir_filter
2896 * Filter to be updated.
2899 * 0 on success, errno value on failure.
2902 priv_fdir_filter_update(struct priv *priv,
2903 const struct rte_eth_fdir_filter *fdir_filter)
2907 ret = priv_fdir_filter_delete(priv, fdir_filter);
2910 ret = priv_fdir_filter_add(priv, fdir_filter);
2915 * Flush all filters.
2918 * Private structure.
2921 priv_fdir_filter_flush(struct priv *priv)
2923 priv_flow_flush(priv, &priv->flows);
2927 * Get flow director information.
2930 * Private structure.
2931 * @param[out] fdir_info
2932 * Resulting flow director information.
2935 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2937 struct rte_eth_fdir_masks *mask =
2938 &priv->dev->data->dev_conf.fdir_conf.mask;
2940 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2941 fdir_info->guarant_spc = 0;
2942 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2943 fdir_info->max_flexpayload = 0;
2944 fdir_info->flow_types_mask[0] = 0;
2945 fdir_info->flex_payload_unit = 0;
2946 fdir_info->max_flex_payload_segment_num = 0;
2947 fdir_info->flex_payload_limit = 0;
2948 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2952 * Deal with flow director operations.
2955 * Pointer to private structure.
2957 * Operation to perform.
2959 * Pointer to operation-specific structure.
2962 * 0 on success, errno value on failure.
2965 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2967 enum rte_fdir_mode fdir_mode =
2968 priv->dev->data->dev_conf.fdir_conf.mode;
2971 if (filter_op == RTE_ETH_FILTER_NOP)
2973 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2974 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2975 ERROR("%p: flow director mode %d not supported",
2976 (void *)priv, fdir_mode);
2979 switch (filter_op) {
2980 case RTE_ETH_FILTER_ADD:
2981 ret = priv_fdir_filter_add(priv, arg);
2983 case RTE_ETH_FILTER_UPDATE:
2984 ret = priv_fdir_filter_update(priv, arg);
2986 case RTE_ETH_FILTER_DELETE:
2987 ret = priv_fdir_filter_delete(priv, arg);
2989 case RTE_ETH_FILTER_FLUSH:
2990 priv_fdir_filter_flush(priv);
2992 case RTE_ETH_FILTER_INFO:
2993 priv_fdir_info_get(priv, arg);
2996 DEBUG("%p: unknown operation %u", (void *)priv,
3005 * Manage filter operations.
3008 * Pointer to Ethernet device structure.
3009 * @param filter_type
3012 * Operation to perform.
3014 * Pointer to operation-specific structure.
3017 * 0 on success, negative errno value on failure.
3020 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3021 enum rte_filter_type filter_type,
3022 enum rte_filter_op filter_op,
3026 struct priv *priv = dev->data->dev_private;
3028 switch (filter_type) {
3029 case RTE_ETH_FILTER_GENERIC:
3030 if (filter_op != RTE_ETH_FILTER_GET)
3032 *(const void **)arg = &mlx5_flow_ops;
3034 case RTE_ETH_FILTER_FDIR:
3036 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3040 ERROR("%p: filter type (%d) not supported",
3041 (void *)dev, filter_type);