4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
63 struct ibv_counter_set_init_attr {
66 struct ibv_flow_spec_counter_action {
69 struct ibv_counter_set {
74 ibv_destroy_counter_set(struct ibv_counter_set *cs)
81 /* Dev ops structure defined in mlx5.c */
82 extern const struct eth_dev_ops mlx5_dev_ops;
83 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
86 mlx5_flow_create_eth(const struct rte_flow_item *item,
87 const void *default_mask,
91 mlx5_flow_create_vlan(const struct rte_flow_item *item,
92 const void *default_mask,
96 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
97 const void *default_mask,
101 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
102 const void *default_mask,
106 mlx5_flow_create_udp(const struct rte_flow_item *item,
107 const void *default_mask,
111 mlx5_flow_create_tcp(const struct rte_flow_item *item,
112 const void *default_mask,
116 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
117 const void *default_mask,
120 struct mlx5_flow_parse;
123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
132 /* Hash RX queue types. */
143 /* Initialization data for hash RX queue. */
144 struct hash_rxq_init {
145 uint64_t hash_fields; /* Fields that participate in the hash. */
146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
147 unsigned int flow_priority; /* Flow priority to use. */
148 unsigned int ip_version; /* Internet protocol. */
151 /* Initialization data for hash RX queues. */
152 const struct hash_rxq_init hash_rxq_init[] = {
154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
155 IBV_RX_HASH_DST_IPV4 |
156 IBV_RX_HASH_SRC_PORT_TCP |
157 IBV_RX_HASH_DST_PORT_TCP),
158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
160 .ip_version = MLX5_IPV4,
163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
164 IBV_RX_HASH_DST_IPV4 |
165 IBV_RX_HASH_SRC_PORT_UDP |
166 IBV_RX_HASH_DST_PORT_UDP),
167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
169 .ip_version = MLX5_IPV4,
172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
173 IBV_RX_HASH_DST_IPV4),
174 .dpdk_rss_hf = (ETH_RSS_IPV4 |
177 .ip_version = MLX5_IPV4,
180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
181 IBV_RX_HASH_DST_IPV6 |
182 IBV_RX_HASH_SRC_PORT_TCP |
183 IBV_RX_HASH_DST_PORT_TCP),
184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
186 .ip_version = MLX5_IPV6,
189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
190 IBV_RX_HASH_DST_IPV6 |
191 IBV_RX_HASH_SRC_PORT_UDP |
192 IBV_RX_HASH_DST_PORT_UDP),
193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
195 .ip_version = MLX5_IPV6,
198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
199 IBV_RX_HASH_DST_IPV6),
200 .dpdk_rss_hf = (ETH_RSS_IPV6 |
203 .ip_version = MLX5_IPV6,
212 /* Number of entries in hash_rxq_init[]. */
213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
215 /** Structure for holding counter stats. */
216 struct mlx5_flow_counter_stats {
217 uint64_t hits; /**< Number of packets matched by the rule. */
218 uint64_t bytes; /**< Number of bytes matched by the rule. */
221 /** Structure for Drop queue. */
222 struct mlx5_hrxq_drop {
223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
224 struct ibv_qp *qp; /**< Verbs queue pair. */
225 struct ibv_wq *wq; /**< Verbs work queue. */
226 struct ibv_cq *cq; /**< Verbs completion queue. */
229 /* Flows structures. */
231 uint64_t hash_fields; /**< Fields that participate in the hash. */
232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
233 struct ibv_flow *ibv_flow; /**< Verbs flow. */
234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
237 /* Drop flows structures. */
238 struct mlx5_flow_drop {
239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
240 struct ibv_flow *ibv_flow; /**< Verbs flow. */
244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
245 uint32_t mark:1; /**< Set if the flow is marked. */
246 uint32_t drop:1; /**< Drop queue. */
247 uint16_t queues_n; /**< Number of entries in queue[]. */
248 uint16_t (*queues)[]; /**< Queues indexes to use. */
249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
250 uint8_t rss_key[40]; /**< copy of the RSS key. */
251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
255 /**< Flow with Rx queue. */
256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */
260 /** Static initializer for items. */
262 (const enum rte_flow_item_type []){ \
263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
266 /** Structure to generate a simple graph of layers supported by the NIC. */
267 struct mlx5_flow_items {
268 /** List of possible actions for these items. */
269 const enum rte_flow_action_type *const actions;
270 /** Bit-masks corresponding to the possibilities for the item. */
273 * Default bit-masks to use when item->mask is not provided. When
274 * \default_mask is also NULL, the full supported bit-mask (\mask) is
277 const void *default_mask;
278 /** Bit-masks size in bytes. */
279 const unsigned int mask_sz;
281 * Conversion function from rte_flow to NIC specific flow.
284 * rte_flow item to convert.
285 * @param default_mask
286 * Default bit-masks to use when item->mask is not provided.
288 * Internal structure to store the conversion.
291 * 0 on success, negative value otherwise.
293 int (*convert)(const struct rte_flow_item *item,
294 const void *default_mask,
296 /** Size in bytes of the destination structure. */
297 const unsigned int dst_sz;
298 /** List of possible following items. */
299 const enum rte_flow_item_type *const items;
302 /** Valid action for this PMD. */
303 static const enum rte_flow_action_type valid_actions[] = {
304 RTE_FLOW_ACTION_TYPE_DROP,
305 RTE_FLOW_ACTION_TYPE_QUEUE,
306 RTE_FLOW_ACTION_TYPE_MARK,
307 RTE_FLOW_ACTION_TYPE_FLAG,
308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
309 RTE_FLOW_ACTION_TYPE_COUNT,
311 RTE_FLOW_ACTION_TYPE_END,
314 /** Graph of supported items and associated actions. */
315 static const struct mlx5_flow_items mlx5_flow_items[] = {
316 [RTE_FLOW_ITEM_TYPE_END] = {
317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
318 RTE_FLOW_ITEM_TYPE_VXLAN),
320 [RTE_FLOW_ITEM_TYPE_ETH] = {
321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
322 RTE_FLOW_ITEM_TYPE_IPV4,
323 RTE_FLOW_ITEM_TYPE_IPV6),
324 .actions = valid_actions,
325 .mask = &(const struct rte_flow_item_eth){
326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
330 .default_mask = &rte_flow_item_eth_mask,
331 .mask_sz = sizeof(struct rte_flow_item_eth),
332 .convert = mlx5_flow_create_eth,
333 .dst_sz = sizeof(struct ibv_flow_spec_eth),
335 [RTE_FLOW_ITEM_TYPE_VLAN] = {
336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
337 RTE_FLOW_ITEM_TYPE_IPV6),
338 .actions = valid_actions,
339 .mask = &(const struct rte_flow_item_vlan){
342 .default_mask = &rte_flow_item_vlan_mask,
343 .mask_sz = sizeof(struct rte_flow_item_vlan),
344 .convert = mlx5_flow_create_vlan,
347 [RTE_FLOW_ITEM_TYPE_IPV4] = {
348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
349 RTE_FLOW_ITEM_TYPE_TCP),
350 .actions = valid_actions,
351 .mask = &(const struct rte_flow_item_ipv4){
355 .type_of_service = -1,
359 .default_mask = &rte_flow_item_ipv4_mask,
360 .mask_sz = sizeof(struct rte_flow_item_ipv4),
361 .convert = mlx5_flow_create_ipv4,
362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
364 [RTE_FLOW_ITEM_TYPE_IPV6] = {
365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
366 RTE_FLOW_ITEM_TYPE_TCP),
367 .actions = valid_actions,
368 .mask = &(const struct rte_flow_item_ipv6){
371 0xff, 0xff, 0xff, 0xff,
372 0xff, 0xff, 0xff, 0xff,
373 0xff, 0xff, 0xff, 0xff,
374 0xff, 0xff, 0xff, 0xff,
377 0xff, 0xff, 0xff, 0xff,
378 0xff, 0xff, 0xff, 0xff,
379 0xff, 0xff, 0xff, 0xff,
380 0xff, 0xff, 0xff, 0xff,
387 .default_mask = &rte_flow_item_ipv6_mask,
388 .mask_sz = sizeof(struct rte_flow_item_ipv6),
389 .convert = mlx5_flow_create_ipv6,
390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
392 [RTE_FLOW_ITEM_TYPE_UDP] = {
393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
394 .actions = valid_actions,
395 .mask = &(const struct rte_flow_item_udp){
401 .default_mask = &rte_flow_item_udp_mask,
402 .mask_sz = sizeof(struct rte_flow_item_udp),
403 .convert = mlx5_flow_create_udp,
404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
406 [RTE_FLOW_ITEM_TYPE_TCP] = {
407 .actions = valid_actions,
408 .mask = &(const struct rte_flow_item_tcp){
414 .default_mask = &rte_flow_item_tcp_mask,
415 .mask_sz = sizeof(struct rte_flow_item_tcp),
416 .convert = mlx5_flow_create_tcp,
417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
419 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
421 .actions = valid_actions,
422 .mask = &(const struct rte_flow_item_vxlan){
423 .vni = "\xff\xff\xff",
425 .default_mask = &rte_flow_item_vxlan_mask,
426 .mask_sz = sizeof(struct rte_flow_item_vxlan),
427 .convert = mlx5_flow_create_vxlan,
428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
432 /** Structure to pass to the conversion function. */
433 struct mlx5_flow_parse {
434 uint32_t inner; /**< Set once VXLAN is encountered. */
436 /**< Whether resources should remain after a validate. */
437 uint32_t drop:1; /**< Target is a drop queue. */
438 uint32_t mark:1; /**< Mark is present in the flow. */
439 uint32_t count:1; /**< Count is present in the flow. */
440 uint32_t mark_id; /**< Mark identifier. */
441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
442 uint16_t queues_n; /**< Number of entries in queue[]. */
443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
444 uint8_t rss_key[40]; /**< copy of the RSS key. */
445 enum hash_rxq_type layer; /**< Last pattern layer detected. */
446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
449 struct ibv_flow_attr *ibv_attr;
450 /**< Pointer to Verbs attributes. */
452 /**< Current position or total size of the attribute. */
453 } queue[RTE_DIM(hash_rxq_init)];
455 struct ibv_flow_attr *ibv_attr;
456 /**< Pointer to Verbs attributes. */
458 /**< Current position or total size of the attribute. */
463 static const struct rte_flow_ops mlx5_flow_ops = {
464 .validate = mlx5_flow_validate,
465 .create = mlx5_flow_create,
466 .destroy = mlx5_flow_destroy,
467 .flush = mlx5_flow_flush,
468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
469 .query = mlx5_flow_query,
473 .isolate = mlx5_flow_isolate,
476 /* Convert FDIR request to Generic flow. */
478 struct rte_flow_attr attr;
479 struct rte_flow_action actions[2];
480 struct rte_flow_item items[4];
481 struct rte_flow_item_eth l2;
483 struct rte_flow_item_ipv4 ipv4;
484 struct rte_flow_item_ipv6 ipv6;
487 struct rte_flow_item_udp udp;
488 struct rte_flow_item_tcp tcp;
490 struct rte_flow_action_queue queue;
493 /* Verbs specification header. */
494 struct ibv_spec_header {
495 enum ibv_flow_spec_type type;
500 * Check support for a given item.
503 * Item specification.
505 * Bit-masks covering supported fields to compare with spec, last and mask in
508 * Bit-Mask size in bytes.
514 mlx5_flow_item_validate(const struct rte_flow_item *item,
515 const uint8_t *mask, unsigned int size)
519 if (!item->spec && (item->mask || item->last))
521 if (item->spec && !item->mask) {
523 const uint8_t *spec = item->spec;
525 for (i = 0; i < size; ++i)
526 if ((spec[i] | mask[i]) != mask[i])
529 if (item->last && !item->mask) {
531 const uint8_t *spec = item->last;
533 for (i = 0; i < size; ++i)
534 if ((spec[i] | mask[i]) != mask[i])
539 const uint8_t *spec = item->mask;
541 for (i = 0; i < size; ++i)
542 if ((spec[i] | mask[i]) != mask[i])
545 if (item->spec && item->last) {
548 const uint8_t *apply = mask;
553 for (i = 0; i < size; ++i) {
554 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
555 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
557 ret = memcmp(spec, last, size);
563 * Copy the RSS configuration from the user ones.
566 * Pointer to private structure.
568 * Internal parser structure.
570 * User RSS configuration to save.
573 * 0 on success, errno value on failure.
576 priv_flow_convert_rss_conf(struct priv *priv,
577 struct mlx5_flow_parse *parser,
578 const struct rte_eth_rss_conf *rss_conf)
580 const struct rte_eth_rss_conf *rss =
581 rss_conf ? rss_conf : &priv->rss_conf;
583 if (rss->rss_key_len > 40)
585 parser->rss_conf.rss_key_len = rss->rss_key_len;
586 parser->rss_conf.rss_hf = rss->rss_hf;
587 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
588 parser->rss_conf.rss_key = parser->rss_key;
593 * Extract attribute to the parser.
596 * Pointer to private structure.
598 * Flow rule attributes.
600 * Perform verbose error reporting if not NULL.
601 * @param[in, out] parser
602 * Internal parser structure.
605 * 0 on success, a negative errno value otherwise and rte_errno is set.
608 priv_flow_convert_attributes(struct priv *priv,
609 const struct rte_flow_attr *attr,
610 struct rte_flow_error *error,
611 struct mlx5_flow_parse *parser)
616 rte_flow_error_set(error, ENOTSUP,
617 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
619 "groups are not supported");
622 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
623 rte_flow_error_set(error, ENOTSUP,
624 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
626 "priorities are not supported");
630 rte_flow_error_set(error, ENOTSUP,
631 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
633 "egress is not supported");
636 if (!attr->ingress) {
637 rte_flow_error_set(error, ENOTSUP,
638 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
640 "only ingress is supported");
647 * Extract actions request to the parser.
650 * Pointer to private structure.
652 * Associated actions (list terminated by the END action).
654 * Perform verbose error reporting if not NULL.
655 * @param[in, out] parser
656 * Internal parser structure.
659 * 0 on success, a negative errno value otherwise and rte_errno is set.
662 priv_flow_convert_actions(struct priv *priv,
663 const struct rte_flow_action actions[],
664 struct rte_flow_error *error,
665 struct mlx5_flow_parse *parser)
668 * Add default RSS configuration necessary for Verbs to create QP even
669 * if no RSS is necessary.
671 priv_flow_convert_rss_conf(priv, parser,
672 (const struct rte_eth_rss_conf *)
674 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
675 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
677 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
679 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
680 const struct rte_flow_action_queue *queue =
681 (const struct rte_flow_action_queue *)
686 if (!queue || (queue->index > (priv->rxqs_n - 1)))
687 goto exit_action_not_supported;
688 for (n = 0; n < parser->queues_n; ++n) {
689 if (parser->queues[n] == queue->index) {
694 if (parser->queues_n > 1 && !found) {
695 rte_flow_error_set(error, ENOTSUP,
696 RTE_FLOW_ERROR_TYPE_ACTION,
698 "queue action not in RSS queues");
702 parser->queues_n = 1;
703 parser->queues[0] = queue->index;
705 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
706 const struct rte_flow_action_rss *rss =
707 (const struct rte_flow_action_rss *)
711 if (!rss || !rss->num) {
712 rte_flow_error_set(error, EINVAL,
713 RTE_FLOW_ERROR_TYPE_ACTION,
718 if (parser->queues_n == 1) {
721 assert(parser->queues_n);
722 for (n = 0; n < rss->num; ++n) {
723 if (parser->queues[0] ==
730 rte_flow_error_set(error, ENOTSUP,
731 RTE_FLOW_ERROR_TYPE_ACTION,
733 "queue action not in RSS"
738 for (n = 0; n < rss->num; ++n) {
739 if (rss->queue[n] >= priv->rxqs_n) {
740 rte_flow_error_set(error, EINVAL,
741 RTE_FLOW_ERROR_TYPE_ACTION,
743 "queue id > number of"
748 for (n = 0; n < rss->num; ++n)
749 parser->queues[n] = rss->queue[n];
750 parser->queues_n = rss->num;
751 if (priv_flow_convert_rss_conf(priv, parser,
753 rte_flow_error_set(error, EINVAL,
754 RTE_FLOW_ERROR_TYPE_ACTION,
756 "wrong RSS configuration");
759 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
760 const struct rte_flow_action_mark *mark =
761 (const struct rte_flow_action_mark *)
765 rte_flow_error_set(error, EINVAL,
766 RTE_FLOW_ERROR_TYPE_ACTION,
768 "mark must be defined");
770 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
771 rte_flow_error_set(error, ENOTSUP,
772 RTE_FLOW_ERROR_TYPE_ACTION,
774 "mark must be between 0"
779 parser->mark_id = mark->id;
780 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
782 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
783 priv->counter_set_supported) {
786 goto exit_action_not_supported;
789 if (!parser->queues_n && !parser->drop) {
790 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
791 NULL, "no valid action");
795 exit_action_not_supported:
796 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
797 actions, "action not supported");
805 * Pointer to private structure.
807 * Pattern specification (list terminated by the END pattern item).
809 * Perform verbose error reporting if not NULL.
810 * @param[in, out] parser
811 * Internal parser structure.
814 * 0 on success, a negative errno value otherwise and rte_errno is set.
817 priv_flow_convert_items_validate(struct priv *priv,
818 const struct rte_flow_item items[],
819 struct rte_flow_error *error,
820 struct mlx5_flow_parse *parser)
822 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
826 /* Initialise the offsets to start after verbs attribute. */
828 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
830 for (i = 0; i != hash_rxq_init_n; ++i)
831 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
833 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
834 const struct mlx5_flow_items *token = NULL;
838 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
842 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
844 if (cur_item->items[i] == items->type) {
845 token = &mlx5_flow_items[items->type];
850 goto exit_item_not_supported;
852 err = mlx5_flow_item_validate(items,
853 (const uint8_t *)cur_item->mask,
856 goto exit_item_not_supported;
857 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
859 rte_flow_error_set(error, ENOTSUP,
860 RTE_FLOW_ERROR_TYPE_ITEM,
862 "cannot recognize multiple"
863 " VXLAN encapsulations");
869 parser->drop_q.offset += cur_item->dst_sz;
870 } else if (parser->queues_n == 1) {
871 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
873 for (n = 0; n != hash_rxq_init_n; ++n)
874 parser->queue[n].offset += cur_item->dst_sz;
878 for (i = 0; i != hash_rxq_init_n; ++i)
879 parser->queue[i].offset +=
880 sizeof(struct ibv_flow_spec_action_tag);
883 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
886 parser->drop_q.offset += size;
888 for (i = 0; i != hash_rxq_init_n; ++i)
889 parser->queue[i].offset += size;
893 exit_item_not_supported:
894 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
895 items, "item not supported");
900 * Allocate memory space to store verbs flow attributes.
903 * Pointer to private structure.
904 * @param[in] priority
907 * Amount of byte to allocate.
909 * Perform verbose error reporting if not NULL.
912 * A verbs flow attribute on success, NULL otherwise.
914 static struct ibv_flow_attr*
915 priv_flow_convert_allocate(struct priv *priv,
916 unsigned int priority,
918 struct rte_flow_error *error)
920 struct ibv_flow_attr *ibv_attr;
923 ibv_attr = rte_calloc(__func__, 1, size, 0);
925 rte_flow_error_set(error, ENOMEM,
926 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
928 "cannot allocate verbs spec attributes.");
931 ibv_attr->priority = priority;
936 * Finalise verbs flow attributes.
939 * Pointer to private structure.
940 * @param[in, out] parser
941 * Internal parser structure.
944 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
946 const unsigned int ipv4 =
947 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
948 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
949 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
950 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
951 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
952 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
956 if (parser->layer == HASH_RXQ_ETH) {
960 * This layer becomes useless as the pattern define under
963 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
964 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
966 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
967 for (i = ohmin; i != (ohmax + 1); ++i) {
968 if (!parser->queue[i].ibv_attr)
970 rte_free(parser->queue[i].ibv_attr);
971 parser->queue[i].ibv_attr = NULL;
973 /* Remove impossible flow according to the RSS configuration. */
974 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
975 parser->rss_conf.rss_hf) {
976 /* Remove any other flow. */
977 for (i = hmin; i != (hmax + 1); ++i) {
978 if ((i == parser->layer) ||
979 (!parser->queue[i].ibv_attr))
981 rte_free(parser->queue[i].ibv_attr);
982 parser->queue[i].ibv_attr = NULL;
984 } else if (!parser->queue[ip].ibv_attr) {
985 /* no RSS possible with the current configuration. */
986 parser->queues_n = 1;
991 * Fill missing layers in verbs specifications, or compute the correct
992 * offset to allocate the memory space for the attributes and
995 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
997 struct ibv_flow_spec_ipv4_ext ipv4;
998 struct ibv_flow_spec_ipv6 ipv6;
999 struct ibv_flow_spec_tcp_udp udp_tcp;
1004 if (i == parser->layer)
1006 if (parser->layer == HASH_RXQ_ETH) {
1007 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
1008 size = sizeof(struct ibv_flow_spec_ipv4_ext);
1009 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
1010 .type = IBV_FLOW_SPEC_IPV4_EXT |
1015 size = sizeof(struct ibv_flow_spec_ipv6);
1016 specs.ipv6 = (struct ibv_flow_spec_ipv6){
1017 .type = IBV_FLOW_SPEC_IPV6 |
1022 if (parser->queue[i].ibv_attr) {
1023 dst = (void *)((uintptr_t)
1024 parser->queue[i].ibv_attr +
1025 parser->queue[i].offset);
1026 memcpy(dst, &specs, size);
1027 ++parser->queue[i].ibv_attr->num_of_specs;
1029 parser->queue[i].offset += size;
1031 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1032 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1033 size = sizeof(struct ibv_flow_spec_tcp_udp);
1034 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1035 .type = ((i == HASH_RXQ_UDPV4 ||
1036 i == HASH_RXQ_UDPV6) ?
1038 IBV_FLOW_SPEC_TCP) |
1042 if (parser->queue[i].ibv_attr) {
1043 dst = (void *)((uintptr_t)
1044 parser->queue[i].ibv_attr +
1045 parser->queue[i].offset);
1046 memcpy(dst, &specs, size);
1047 ++parser->queue[i].ibv_attr->num_of_specs;
1049 parser->queue[i].offset += size;
1055 * Validate and convert a flow supported by the NIC.
1058 * Pointer to private structure.
1060 * Flow rule attributes.
1061 * @param[in] pattern
1062 * Pattern specification (list terminated by the END pattern item).
1063 * @param[in] actions
1064 * Associated actions (list terminated by the END action).
1066 * Perform verbose error reporting if not NULL.
1067 * @param[in, out] parser
1068 * Internal parser structure.
1071 * 0 on success, a negative errno value otherwise and rte_errno is set.
1074 priv_flow_convert(struct priv *priv,
1075 const struct rte_flow_attr *attr,
1076 const struct rte_flow_item items[],
1077 const struct rte_flow_action actions[],
1078 struct rte_flow_error *error,
1079 struct mlx5_flow_parse *parser)
1081 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1085 /* First step. Validate the attributes, items and actions. */
1086 *parser = (struct mlx5_flow_parse){
1087 .create = parser->create,
1088 .layer = HASH_RXQ_ETH,
1089 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1091 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1094 ret = priv_flow_convert_actions(priv, actions, error, parser);
1097 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1100 priv_flow_convert_finalise(priv, parser);
1103 * Allocate the memory space to store verbs specifications.
1106 parser->drop_q.ibv_attr =
1107 priv_flow_convert_allocate(priv, attr->priority,
1108 parser->drop_q.offset,
1110 if (!parser->drop_q.ibv_attr)
1112 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
1113 } else if (parser->queues_n == 1) {
1114 unsigned int priority =
1116 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1117 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1119 parser->queue[HASH_RXQ_ETH].ibv_attr =
1120 priv_flow_convert_allocate(priv, priority,
1122 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1124 parser->queue[HASH_RXQ_ETH].offset =
1125 sizeof(struct ibv_flow_attr);
1127 for (i = 0; i != hash_rxq_init_n; ++i) {
1128 unsigned int priority =
1130 hash_rxq_init[i].flow_priority;
1131 unsigned int offset;
1133 if (!(parser->rss_conf.rss_hf &
1134 hash_rxq_init[i].dpdk_rss_hf) &&
1135 (i != HASH_RXQ_ETH))
1137 offset = parser->queue[i].offset;
1138 parser->queue[i].ibv_attr =
1139 priv_flow_convert_allocate(priv, priority,
1141 if (!parser->queue[i].ibv_attr)
1143 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1146 /* Third step. Conversion parse, fill the specifications. */
1148 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1149 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1151 cur_item = &mlx5_flow_items[items->type];
1152 ret = cur_item->convert(items,
1153 (cur_item->default_mask ?
1154 cur_item->default_mask :
1158 rte_flow_error_set(error, ENOTSUP,
1159 RTE_FLOW_ERROR_TYPE_ITEM,
1160 items, "item not supported");
1165 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1166 if (parser->count && parser->create) {
1167 mlx5_flow_create_count(priv, parser);
1169 goto exit_count_error;
1172 * Last step. Complete missing specification to reach the RSS
1175 if (parser->queues_n > 1)
1176 priv_flow_convert_finalise(priv, parser);
1178 /* Only verification is expected, all resources should be released. */
1179 if (!parser->create) {
1181 rte_free(parser->drop_q.ibv_attr);
1182 parser->drop_q.ibv_attr = NULL;
1184 for (i = 0; i != hash_rxq_init_n; ++i) {
1185 if (parser->queue[i].ibv_attr) {
1186 rte_free(parser->queue[i].ibv_attr);
1187 parser->queue[i].ibv_attr = NULL;
1193 for (i = 0; i != hash_rxq_init_n; ++i) {
1194 if (parser->queue[i].ibv_attr) {
1195 rte_free(parser->queue[i].ibv_attr);
1196 parser->queue[i].ibv_attr = NULL;
1199 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1200 NULL, "cannot allocate verbs spec attributes.");
1203 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1204 NULL, "cannot create counter.");
1209 * Copy the specification created into the flow.
1212 * Internal parser structure.
1214 * Create specification.
1216 * Size in bytes of the specification to copy.
1219 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1226 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1227 parser->drop_q.offset);
1228 memcpy(dst, src, size);
1229 ++parser->drop_q.ibv_attr->num_of_specs;
1230 parser->drop_q.offset += size;
1233 for (i = 0; i != hash_rxq_init_n; ++i) {
1234 if (!parser->queue[i].ibv_attr)
1236 /* Specification must be the same l3 type or none. */
1237 if (parser->layer == HASH_RXQ_ETH ||
1238 (hash_rxq_init[parser->layer].ip_version ==
1239 hash_rxq_init[i].ip_version) ||
1240 (hash_rxq_init[i].ip_version == 0)) {
1241 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1242 parser->queue[i].offset);
1243 memcpy(dst, src, size);
1244 ++parser->queue[i].ibv_attr->num_of_specs;
1245 parser->queue[i].offset += size;
1251 * Convert Ethernet item to Verbs specification.
1254 * Item specification.
1255 * @param default_mask[in]
1256 * Default bit-masks to use when item->mask is not provided.
1257 * @param data[in, out]
1261 mlx5_flow_create_eth(const struct rte_flow_item *item,
1262 const void *default_mask,
1265 const struct rte_flow_item_eth *spec = item->spec;
1266 const struct rte_flow_item_eth *mask = item->mask;
1267 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1268 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1269 struct ibv_flow_spec_eth eth = {
1270 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1274 parser->layer = HASH_RXQ_ETH;
1279 mask = default_mask;
1280 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1281 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1282 eth.val.ether_type = spec->type;
1283 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1284 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1285 eth.mask.ether_type = mask->type;
1286 /* Remove unwanted bits from values. */
1287 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1288 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1289 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1291 eth.val.ether_type &= eth.mask.ether_type;
1293 mlx5_flow_create_copy(parser, ð, eth_size);
1298 * Convert VLAN item to Verbs specification.
1301 * Item specification.
1302 * @param default_mask[in]
1303 * Default bit-masks to use when item->mask is not provided.
1304 * @param data[in, out]
1308 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1309 const void *default_mask,
1312 const struct rte_flow_item_vlan *spec = item->spec;
1313 const struct rte_flow_item_vlan *mask = item->mask;
1314 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1315 struct ibv_flow_spec_eth *eth;
1316 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1321 mask = default_mask;
1324 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1325 parser->drop_q.offset - eth_size);
1326 eth->val.vlan_tag = spec->tci;
1327 eth->mask.vlan_tag = mask->tci;
1328 eth->val.vlan_tag &= eth->mask.vlan_tag;
1331 for (i = 0; i != hash_rxq_init_n; ++i) {
1332 if (!parser->queue[i].ibv_attr)
1335 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1336 parser->queue[i].offset - eth_size);
1337 eth->val.vlan_tag = spec->tci;
1338 eth->mask.vlan_tag = mask->tci;
1339 eth->val.vlan_tag &= eth->mask.vlan_tag;
1346 * Convert IPv4 item to Verbs specification.
1349 * Item specification.
1350 * @param default_mask[in]
1351 * Default bit-masks to use when item->mask is not provided.
1352 * @param data[in, out]
1356 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1357 const void *default_mask,
1360 const struct rte_flow_item_ipv4 *spec = item->spec;
1361 const struct rte_flow_item_ipv4 *mask = item->mask;
1362 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1363 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1364 struct ibv_flow_spec_ipv4_ext ipv4 = {
1365 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1369 parser->layer = HASH_RXQ_IPV4;
1372 mask = default_mask;
1373 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1374 .src_ip = spec->hdr.src_addr,
1375 .dst_ip = spec->hdr.dst_addr,
1376 .proto = spec->hdr.next_proto_id,
1377 .tos = spec->hdr.type_of_service,
1379 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1380 .src_ip = mask->hdr.src_addr,
1381 .dst_ip = mask->hdr.dst_addr,
1382 .proto = mask->hdr.next_proto_id,
1383 .tos = mask->hdr.type_of_service,
1385 /* Remove unwanted bits from values. */
1386 ipv4.val.src_ip &= ipv4.mask.src_ip;
1387 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1388 ipv4.val.proto &= ipv4.mask.proto;
1389 ipv4.val.tos &= ipv4.mask.tos;
1391 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1396 * Convert IPv6 item to Verbs specification.
1399 * Item specification.
1400 * @param default_mask[in]
1401 * Default bit-masks to use when item->mask is not provided.
1402 * @param data[in, out]
1406 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1407 const void *default_mask,
1410 const struct rte_flow_item_ipv6 *spec = item->spec;
1411 const struct rte_flow_item_ipv6 *mask = item->mask;
1412 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1413 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1414 struct ibv_flow_spec_ipv6 ipv6 = {
1415 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1419 parser->layer = HASH_RXQ_IPV6;
1424 mask = default_mask;
1425 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1426 RTE_DIM(ipv6.val.src_ip));
1427 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1428 RTE_DIM(ipv6.val.dst_ip));
1429 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1430 RTE_DIM(ipv6.mask.src_ip));
1431 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1432 RTE_DIM(ipv6.mask.dst_ip));
1433 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1434 ipv6.mask.next_hdr = mask->hdr.proto;
1435 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1436 /* Remove unwanted bits from values. */
1437 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1438 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1439 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1441 ipv6.val.flow_label &= ipv6.mask.flow_label;
1442 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1443 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1445 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1450 * Convert UDP item to Verbs specification.
1453 * Item specification.
1454 * @param default_mask[in]
1455 * Default bit-masks to use when item->mask is not provided.
1456 * @param data[in, out]
1460 mlx5_flow_create_udp(const struct rte_flow_item *item,
1461 const void *default_mask,
1464 const struct rte_flow_item_udp *spec = item->spec;
1465 const struct rte_flow_item_udp *mask = item->mask;
1466 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1467 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1468 struct ibv_flow_spec_tcp_udp udp = {
1469 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1473 if (parser->layer == HASH_RXQ_IPV4)
1474 parser->layer = HASH_RXQ_UDPV4;
1476 parser->layer = HASH_RXQ_UDPV6;
1479 mask = default_mask;
1480 udp.val.dst_port = spec->hdr.dst_port;
1481 udp.val.src_port = spec->hdr.src_port;
1482 udp.mask.dst_port = mask->hdr.dst_port;
1483 udp.mask.src_port = mask->hdr.src_port;
1484 /* Remove unwanted bits from values. */
1485 udp.val.src_port &= udp.mask.src_port;
1486 udp.val.dst_port &= udp.mask.dst_port;
1488 mlx5_flow_create_copy(parser, &udp, udp_size);
1493 * Convert TCP item to Verbs specification.
1496 * Item specification.
1497 * @param default_mask[in]
1498 * Default bit-masks to use when item->mask is not provided.
1499 * @param data[in, out]
1503 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1504 const void *default_mask,
1507 const struct rte_flow_item_tcp *spec = item->spec;
1508 const struct rte_flow_item_tcp *mask = item->mask;
1509 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1510 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1511 struct ibv_flow_spec_tcp_udp tcp = {
1512 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1516 if (parser->layer == HASH_RXQ_IPV4)
1517 parser->layer = HASH_RXQ_TCPV4;
1519 parser->layer = HASH_RXQ_TCPV6;
1522 mask = default_mask;
1523 tcp.val.dst_port = spec->hdr.dst_port;
1524 tcp.val.src_port = spec->hdr.src_port;
1525 tcp.mask.dst_port = mask->hdr.dst_port;
1526 tcp.mask.src_port = mask->hdr.src_port;
1527 /* Remove unwanted bits from values. */
1528 tcp.val.src_port &= tcp.mask.src_port;
1529 tcp.val.dst_port &= tcp.mask.dst_port;
1531 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1536 * Convert VXLAN item to Verbs specification.
1539 * Item specification.
1540 * @param default_mask[in]
1541 * Default bit-masks to use when item->mask is not provided.
1542 * @param data[in, out]
1546 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1547 const void *default_mask,
1550 const struct rte_flow_item_vxlan *spec = item->spec;
1551 const struct rte_flow_item_vxlan *mask = item->mask;
1552 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1553 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1554 struct ibv_flow_spec_tunnel vxlan = {
1555 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1564 parser->inner = IBV_FLOW_SPEC_INNER;
1567 mask = default_mask;
1568 memcpy(&id.vni[1], spec->vni, 3);
1569 vxlan.val.tunnel_id = id.vlan_id;
1570 memcpy(&id.vni[1], mask->vni, 3);
1571 vxlan.mask.tunnel_id = id.vlan_id;
1572 /* Remove unwanted bits from values. */
1573 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1575 mlx5_flow_create_copy(parser, &vxlan, size);
1580 * Convert mark/flag action to Verbs specification.
1583 * Internal parser structure.
1588 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1590 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1591 struct ibv_flow_spec_action_tag tag = {
1592 .type = IBV_FLOW_SPEC_ACTION_TAG,
1594 .tag_id = mlx5_flow_mark_set(mark_id),
1597 assert(parser->mark);
1598 mlx5_flow_create_copy(parser, &tag, size);
1603 * Convert count action to Verbs specification.
1606 * Pointer to private structure.
1608 * Pointer to MLX5 flow parser structure.
1611 * 0 on success, errno value on failure.
1614 mlx5_flow_create_count(struct priv *priv __rte_unused,
1615 struct mlx5_flow_parse *parser __rte_unused)
1617 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1618 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1619 struct ibv_counter_set_init_attr init_attr = {0};
1620 struct ibv_flow_spec_counter_action counter = {
1621 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1623 .counter_set_handle = 0,
1626 init_attr.counter_set_id = 0;
1627 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr);
1630 counter.counter_set_handle = parser->cs->handle;
1631 mlx5_flow_create_copy(parser, &counter, size);
1637 * Complete flow rule creation with a drop queue.
1640 * Pointer to private structure.
1642 * Internal parser structure.
1644 * Pointer to the rte_flow.
1646 * Perform verbose error reporting if not NULL.
1649 * 0 on success, errno value on failure.
1652 priv_flow_create_action_queue_drop(struct priv *priv,
1653 struct mlx5_flow_parse *parser,
1654 struct rte_flow *flow,
1655 struct rte_flow_error *error)
1657 struct ibv_flow_spec_action_drop *drop;
1658 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1664 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1665 parser->drop_q.offset);
1666 *drop = (struct ibv_flow_spec_action_drop){
1667 .type = IBV_FLOW_SPEC_ACTION_DROP,
1670 ++parser->drop_q.ibv_attr->num_of_specs;
1671 parser->drop_q.offset += size;
1672 if (!priv->dev->data->dev_started)
1674 flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
1675 parser->drop_q.ibv_attr = NULL;
1676 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
1677 flow->drxq.ibv_attr);
1679 flow->cs = parser->cs;
1680 if (!flow->drxq.ibv_flow) {
1681 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1682 NULL, "flow rule creation failure");
1689 if (flow->drxq.ibv_flow) {
1690 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1691 flow->drxq.ibv_flow = NULL;
1693 if (flow->drxq.ibv_attr) {
1694 rte_free(flow->drxq.ibv_attr);
1695 flow->drxq.ibv_attr = NULL;
1698 claim_zero(ibv_destroy_counter_set(flow->cs));
1706 * Create hash Rx queues when RSS is enabled.
1709 * Pointer to private structure.
1711 * Internal parser structure.
1713 * Pointer to the rte_flow.
1715 * Perform verbose error reporting if not NULL.
1718 * 0 on success, a errno value otherwise and rte_errno is set.
1721 priv_flow_create_action_queue_rss(struct priv *priv,
1722 struct mlx5_flow_parse *parser,
1723 struct rte_flow *flow,
1724 struct rte_flow_error *error)
1728 for (i = 0; i != hash_rxq_init_n; ++i) {
1729 uint64_t hash_fields;
1731 if (!parser->queue[i].ibv_attr)
1733 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1734 parser->queue[i].ibv_attr = NULL;
1735 hash_fields = hash_rxq_init[i].hash_fields;
1736 flow->frxq[i].hrxq =
1737 mlx5_priv_hrxq_get(priv,
1738 parser->rss_conf.rss_key,
1739 parser->rss_conf.rss_key_len,
1742 hash_fields ? parser->queues_n : 1);
1743 if (flow->frxq[i].hrxq)
1745 flow->frxq[i].hrxq =
1746 mlx5_priv_hrxq_new(priv,
1747 parser->rss_conf.rss_key,
1748 parser->rss_conf.rss_key_len,
1751 hash_fields ? parser->queues_n : 1);
1752 if (!flow->frxq[i].hrxq) {
1753 rte_flow_error_set(error, ENOMEM,
1754 RTE_FLOW_ERROR_TYPE_HANDLE,
1755 NULL, "cannot create hash rxq");
1763 * Complete flow rule creation.
1766 * Pointer to private structure.
1768 * Internal parser structure.
1770 * Pointer to the rte_flow.
1772 * Perform verbose error reporting if not NULL.
1775 * 0 on success, a errno value otherwise and rte_errno is set.
1778 priv_flow_create_action_queue(struct priv *priv,
1779 struct mlx5_flow_parse *parser,
1780 struct rte_flow *flow,
1781 struct rte_flow_error *error)
1788 assert(!parser->drop);
1789 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1793 flow->cs = parser->cs;
1794 if (!priv->dev->data->dev_started)
1796 for (i = 0; i != hash_rxq_init_n; ++i) {
1797 if (!flow->frxq[i].hrxq)
1799 flow->frxq[i].ibv_flow =
1800 ibv_create_flow(flow->frxq[i].hrxq->qp,
1801 flow->frxq[i].ibv_attr);
1802 if (!flow->frxq[i].ibv_flow) {
1803 rte_flow_error_set(error, ENOMEM,
1804 RTE_FLOW_ERROR_TYPE_HANDLE,
1805 NULL, "flow rule creation failure");
1809 DEBUG("%p type %d QP %p ibv_flow %p",
1811 (void *)flow->frxq[i].hrxq,
1812 (void *)flow->frxq[i].ibv_flow);
1814 for (i = 0; i != parser->queues_n; ++i) {
1815 struct mlx5_rxq_data *q =
1816 (*priv->rxqs)[parser->queues[i]];
1818 q->mark |= parser->mark;
1823 for (i = 0; i != hash_rxq_init_n; ++i) {
1824 if (flow->frxq[i].ibv_flow) {
1825 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1827 claim_zero(ibv_destroy_flow(ibv_flow));
1829 if (flow->frxq[i].hrxq)
1830 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1831 if (flow->frxq[i].ibv_attr)
1832 rte_free(flow->frxq[i].ibv_attr);
1835 claim_zero(ibv_destroy_counter_set(flow->cs));
1846 * Pointer to private structure.
1848 * Pointer to a TAILQ flow list.
1850 * Flow rule attributes.
1851 * @param[in] pattern
1852 * Pattern specification (list terminated by the END pattern item).
1853 * @param[in] actions
1854 * Associated actions (list terminated by the END action).
1856 * Perform verbose error reporting if not NULL.
1859 * A flow on success, NULL otherwise.
1861 static struct rte_flow *
1862 priv_flow_create(struct priv *priv,
1863 struct mlx5_flows *list,
1864 const struct rte_flow_attr *attr,
1865 const struct rte_flow_item items[],
1866 const struct rte_flow_action actions[],
1867 struct rte_flow_error *error)
1869 struct mlx5_flow_parse parser = { .create = 1, };
1870 struct rte_flow *flow = NULL;
1874 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1877 flow = rte_calloc(__func__, 1,
1878 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1881 rte_flow_error_set(error, ENOMEM,
1882 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1884 "cannot allocate flow memory");
1887 /* Copy queues configuration. */
1888 flow->queues = (uint16_t (*)[])(flow + 1);
1889 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1890 flow->queues_n = parser.queues_n;
1891 /* Copy RSS configuration. */
1892 flow->rss_conf = parser.rss_conf;
1893 flow->rss_conf.rss_key = flow->rss_key;
1894 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1895 /* finalise the flow. */
1897 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1900 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1903 TAILQ_INSERT_TAIL(list, flow, next);
1904 DEBUG("Flow created %p", (void *)flow);
1908 rte_free(parser.drop_q.ibv_attr);
1910 for (i = 0; i != hash_rxq_init_n; ++i) {
1911 if (parser.queue[i].ibv_attr)
1912 rte_free(parser.queue[i].ibv_attr);
1920 * Validate a flow supported by the NIC.
1922 * @see rte_flow_validate()
1926 mlx5_flow_validate(struct rte_eth_dev *dev,
1927 const struct rte_flow_attr *attr,
1928 const struct rte_flow_item items[],
1929 const struct rte_flow_action actions[],
1930 struct rte_flow_error *error)
1932 struct priv *priv = dev->data->dev_private;
1934 struct mlx5_flow_parse parser = { .create = 0, };
1937 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1945 * @see rte_flow_create()
1949 mlx5_flow_create(struct rte_eth_dev *dev,
1950 const struct rte_flow_attr *attr,
1951 const struct rte_flow_item items[],
1952 const struct rte_flow_action actions[],
1953 struct rte_flow_error *error)
1955 struct priv *priv = dev->data->dev_private;
1956 struct rte_flow *flow;
1959 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1969 * Pointer to private structure.
1971 * Pointer to a TAILQ flow list.
1976 priv_flow_destroy(struct priv *priv,
1977 struct mlx5_flows *list,
1978 struct rte_flow *flow)
1983 claim_zero(ibv_destroy_counter_set(flow->cs));
1986 if (flow->drop || !flow->mark)
1988 for (i = 0; i != flow->queues_n; ++i) {
1989 struct rte_flow *tmp;
1993 * To remove the mark from the queue, the queue must not be
1994 * present in any other marked flow (RSS or not).
1996 TAILQ_FOREACH(tmp, list, next) {
1998 uint16_t *tqs = NULL;
2003 for (j = 0; j != hash_rxq_init_n; ++j) {
2004 if (!tmp->frxq[j].hrxq)
2006 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2007 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2011 for (j = 0; (j != tq_n) && !mark; j++)
2012 if (tqs[j] == (*flow->queues)[i])
2015 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2019 if (flow->drxq.ibv_flow)
2020 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2021 rte_free(flow->drxq.ibv_attr);
2023 for (i = 0; i != hash_rxq_init_n; ++i) {
2024 struct mlx5_flow *frxq = &flow->frxq[i];
2027 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
2029 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2031 rte_free(frxq->ibv_attr);
2034 TAILQ_REMOVE(list, flow, next);
2035 DEBUG("Flow destroyed %p", (void *)flow);
2040 * Destroy all flows.
2043 * Pointer to private structure.
2045 * Pointer to a TAILQ flow list.
2048 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2050 while (!TAILQ_EMPTY(list)) {
2051 struct rte_flow *flow;
2053 flow = TAILQ_FIRST(list);
2054 priv_flow_destroy(priv, list, flow);
2059 * Create drop queue.
2062 * Pointer to private structure.
2068 priv_flow_create_drop_queue(struct priv *priv)
2070 struct mlx5_hrxq_drop *fdq = NULL;
2074 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2076 WARN("cannot allocate memory for drop queue");
2079 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
2081 WARN("cannot allocate CQ for drop queue");
2084 fdq->wq = ibv_create_wq(priv->ctx,
2085 &(struct ibv_wq_init_attr){
2086 .wq_type = IBV_WQT_RQ,
2093 WARN("cannot allocate WQ for drop queue");
2096 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
2097 &(struct ibv_rwq_ind_table_init_attr){
2098 .log_ind_tbl_size = 0,
2099 .ind_tbl = &fdq->wq,
2102 if (!fdq->ind_table) {
2103 WARN("cannot allocate indirection table for drop queue");
2106 fdq->qp = ibv_create_qp_ex(priv->ctx,
2107 &(struct ibv_qp_init_attr_ex){
2108 .qp_type = IBV_QPT_RAW_PACKET,
2110 IBV_QP_INIT_ATTR_PD |
2111 IBV_QP_INIT_ATTR_IND_TABLE |
2112 IBV_QP_INIT_ATTR_RX_HASH,
2113 .rx_hash_conf = (struct ibv_rx_hash_conf){
2115 IBV_RX_HASH_FUNC_TOEPLITZ,
2116 .rx_hash_key_len = rss_hash_default_key_len,
2117 .rx_hash_key = rss_hash_default_key,
2118 .rx_hash_fields_mask = 0,
2120 .rwq_ind_tbl = fdq->ind_table,
2124 WARN("cannot allocate QP for drop queue");
2127 priv->flow_drop_queue = fdq;
2131 claim_zero(ibv_destroy_qp(fdq->qp));
2133 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2135 claim_zero(ibv_destroy_wq(fdq->wq));
2137 claim_zero(ibv_destroy_cq(fdq->cq));
2140 priv->flow_drop_queue = NULL;
2145 * Delete drop queue.
2148 * Pointer to private structure.
2151 priv_flow_delete_drop_queue(struct priv *priv)
2153 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2158 claim_zero(ibv_destroy_qp(fdq->qp));
2160 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2162 claim_zero(ibv_destroy_wq(fdq->wq));
2164 claim_zero(ibv_destroy_cq(fdq->cq));
2166 priv->flow_drop_queue = NULL;
2173 * Pointer to private structure.
2175 * Pointer to a TAILQ flow list.
2178 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2180 struct rte_flow *flow;
2182 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2186 if (!flow->drxq.ibv_flow)
2188 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2189 flow->drxq.ibv_flow = NULL;
2194 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2196 for (i = 0; i != hash_rxq_init_n; ++i) {
2197 if (!flow->frxq[i].hrxq)
2199 ind_tbl = flow->frxq[i].hrxq->ind_table;
2202 for (i = 0; i != ind_tbl->queues_n; ++i)
2203 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2205 for (i = 0; i != hash_rxq_init_n; ++i) {
2206 if (!flow->frxq[i].ibv_flow)
2208 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2209 flow->frxq[i].ibv_flow = NULL;
2210 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2211 flow->frxq[i].hrxq = NULL;
2213 DEBUG("Flow %p removed", (void *)flow);
2221 * Pointer to private structure.
2223 * Pointer to a TAILQ flow list.
2226 * 0 on success, a errno value otherwise and rte_errno is set.
2229 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2231 struct rte_flow *flow;
2233 TAILQ_FOREACH(flow, list, next) {
2237 flow->drxq.ibv_flow =
2238 ibv_create_flow(priv->flow_drop_queue->qp,
2239 flow->drxq.ibv_attr);
2240 if (!flow->drxq.ibv_flow) {
2241 DEBUG("Flow %p cannot be applied",
2246 DEBUG("Flow %p applied", (void *)flow);
2250 for (i = 0; i != hash_rxq_init_n; ++i) {
2251 if (!flow->frxq[i].ibv_attr)
2253 flow->frxq[i].hrxq =
2254 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2255 flow->rss_conf.rss_key_len,
2256 hash_rxq_init[i].hash_fields,
2259 if (flow->frxq[i].hrxq)
2261 flow->frxq[i].hrxq =
2262 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2263 flow->rss_conf.rss_key_len,
2264 hash_rxq_init[i].hash_fields,
2267 if (!flow->frxq[i].hrxq) {
2268 DEBUG("Flow %p cannot be applied",
2274 flow->frxq[i].ibv_flow =
2275 ibv_create_flow(flow->frxq[i].hrxq->qp,
2276 flow->frxq[i].ibv_attr);
2277 if (!flow->frxq[i].ibv_flow) {
2278 DEBUG("Flow %p cannot be applied",
2283 DEBUG("Flow %p applied", (void *)flow);
2287 for (i = 0; i != flow->queues_n; ++i)
2288 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2294 * Verify the flow list is empty
2297 * Pointer to private structure.
2299 * @return the number of flows not released.
2302 priv_flow_verify(struct priv *priv)
2304 struct rte_flow *flow;
2307 TAILQ_FOREACH(flow, &priv->flows, next) {
2308 DEBUG("%p: flow %p still referenced", (void *)priv,
2316 * Enable a control flow configured from the control plane.
2319 * Pointer to Ethernet device.
2321 * An Ethernet flow spec to apply.
2323 * An Ethernet flow mask to apply.
2325 * A VLAN flow spec to apply.
2327 * A VLAN flow mask to apply.
2333 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2334 struct rte_flow_item_eth *eth_spec,
2335 struct rte_flow_item_eth *eth_mask,
2336 struct rte_flow_item_vlan *vlan_spec,
2337 struct rte_flow_item_vlan *vlan_mask)
2339 struct priv *priv = dev->data->dev_private;
2340 const struct rte_flow_attr attr = {
2342 .priority = MLX5_CTRL_FLOW_PRIORITY,
2344 struct rte_flow_item items[] = {
2346 .type = RTE_FLOW_ITEM_TYPE_ETH,
2352 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2353 RTE_FLOW_ITEM_TYPE_END,
2359 .type = RTE_FLOW_ITEM_TYPE_END,
2362 struct rte_flow_action actions[] = {
2364 .type = RTE_FLOW_ACTION_TYPE_RSS,
2367 .type = RTE_FLOW_ACTION_TYPE_END,
2370 struct rte_flow *flow;
2371 struct rte_flow_error error;
2374 struct rte_flow_action_rss rss;
2376 const struct rte_eth_rss_conf *rss_conf;
2378 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2382 if (!priv->reta_idx_n)
2384 for (i = 0; i != priv->reta_idx_n; ++i)
2385 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2386 action_rss.local.rss_conf = &priv->rss_conf;
2387 action_rss.local.num = priv->reta_idx_n;
2388 actions[0].conf = (const void *)&action_rss.rss;
2389 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2397 * Enable a flow control configured from the control plane.
2400 * Pointer to Ethernet device.
2402 * An Ethernet flow spec to apply.
2404 * An Ethernet flow mask to apply.
2410 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2411 struct rte_flow_item_eth *eth_spec,
2412 struct rte_flow_item_eth *eth_mask)
2414 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2420 * @see rte_flow_destroy()
2424 mlx5_flow_destroy(struct rte_eth_dev *dev,
2425 struct rte_flow *flow,
2426 struct rte_flow_error *error)
2428 struct priv *priv = dev->data->dev_private;
2432 priv_flow_destroy(priv, &priv->flows, flow);
2438 * Destroy all flows.
2440 * @see rte_flow_flush()
2444 mlx5_flow_flush(struct rte_eth_dev *dev,
2445 struct rte_flow_error *error)
2447 struct priv *priv = dev->data->dev_private;
2451 priv_flow_flush(priv, &priv->flows);
2456 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2458 * Query flow counter.
2462 * @param counter_value
2463 * returned data from the counter.
2466 * 0 on success, a errno value otherwise and rte_errno is set.
2469 priv_flow_query_count(struct ibv_counter_set *cs,
2470 struct mlx5_flow_counter_stats *counter_stats,
2471 struct rte_flow_query_count *query_count,
2472 struct rte_flow_error *error)
2474 uint64_t counters[2];
2475 struct ibv_query_counter_set_attr query_cs_attr = {
2477 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2479 struct ibv_counter_set_data query_out = {
2481 .outlen = 2 * sizeof(uint64_t),
2483 int res = ibv_query_counter_set(&query_cs_attr, &query_out);
2486 rte_flow_error_set(error, -res,
2487 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2489 "cannot read counter");
2492 query_count->hits_set = 1;
2493 query_count->bytes_set = 1;
2494 query_count->hits = counters[0] - counter_stats->hits;
2495 query_count->bytes = counters[1] - counter_stats->bytes;
2496 if (query_count->reset) {
2497 counter_stats->hits = counters[0];
2498 counter_stats->bytes = counters[1];
2506 * @see rte_flow_query()
2510 mlx5_flow_query(struct rte_eth_dev *dev,
2511 struct rte_flow *flow,
2512 enum rte_flow_action_type action __rte_unused,
2514 struct rte_flow_error *error)
2516 struct priv *priv = dev->data->dev_private;
2521 res = priv_flow_query_count(flow->cs,
2522 &flow->counter_stats,
2523 (struct rte_flow_query_count *)data,
2526 rte_flow_error_set(error, res,
2527 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2529 "no counter found for flow");
2539 * @see rte_flow_isolate()
2543 mlx5_flow_isolate(struct rte_eth_dev *dev,
2545 struct rte_flow_error *error)
2547 struct priv *priv = dev->data->dev_private;
2550 if (dev->data->dev_started) {
2551 rte_flow_error_set(error, EBUSY,
2552 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2554 "port must be stopped first");
2558 priv->isolated = !!enable;
2560 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2562 priv->dev->dev_ops = &mlx5_dev_ops;
2568 * Convert a flow director filter to a generic flow.
2571 * Private structure.
2572 * @param fdir_filter
2573 * Flow director filter to add.
2575 * Generic flow parameters structure.
2578 * 0 on success, errno value on error.
2581 priv_fdir_filter_convert(struct priv *priv,
2582 const struct rte_eth_fdir_filter *fdir_filter,
2583 struct mlx5_fdir *attributes)
2585 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2587 /* Validate queue number. */
2588 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2589 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2592 /* Validate the behavior. */
2593 if (fdir_filter->action.behavior != RTE_ETH_FDIR_ACCEPT) {
2594 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2597 attributes->attr.ingress = 1;
2598 attributes->items[0] = (struct rte_flow_item) {
2599 .type = RTE_FLOW_ITEM_TYPE_ETH,
2600 .spec = &attributes->l2,
2602 attributes->actions[0] = (struct rte_flow_action){
2603 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2604 .conf = &attributes->queue,
2606 attributes->queue.index = fdir_filter->action.rx_queue;
2607 switch (fdir_filter->input.flow_type) {
2608 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2609 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2610 .src_addr = input->flow.udp4_flow.ip.src_ip,
2611 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2612 .time_to_live = input->flow.udp4_flow.ip.ttl,
2613 .type_of_service = input->flow.udp4_flow.ip.tos,
2614 .next_proto_id = input->flow.udp4_flow.ip.proto,
2616 attributes->l4.udp.hdr = (struct udp_hdr){
2617 .src_port = input->flow.udp4_flow.src_port,
2618 .dst_port = input->flow.udp4_flow.dst_port,
2620 attributes->items[1] = (struct rte_flow_item){
2621 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2622 .spec = &attributes->l3,
2624 attributes->items[2] = (struct rte_flow_item){
2625 .type = RTE_FLOW_ITEM_TYPE_UDP,
2626 .spec = &attributes->l4,
2629 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2630 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2631 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2632 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2633 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2634 .type_of_service = input->flow.tcp4_flow.ip.tos,
2635 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2637 attributes->l4.tcp.hdr = (struct tcp_hdr){
2638 .src_port = input->flow.tcp4_flow.src_port,
2639 .dst_port = input->flow.tcp4_flow.dst_port,
2641 attributes->items[1] = (struct rte_flow_item){
2642 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2643 .spec = &attributes->l3,
2645 attributes->items[2] = (struct rte_flow_item){
2646 .type = RTE_FLOW_ITEM_TYPE_TCP,
2647 .spec = &attributes->l4,
2650 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2651 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2652 .src_addr = input->flow.ip4_flow.src_ip,
2653 .dst_addr = input->flow.ip4_flow.dst_ip,
2654 .time_to_live = input->flow.ip4_flow.ttl,
2655 .type_of_service = input->flow.ip4_flow.tos,
2656 .next_proto_id = input->flow.ip4_flow.proto,
2658 attributes->items[1] = (struct rte_flow_item){
2659 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2660 .spec = &attributes->l3,
2663 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2664 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2665 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2666 .proto = input->flow.udp6_flow.ip.proto,
2668 memcpy(attributes->l3.ipv6.hdr.src_addr,
2669 input->flow.udp6_flow.ip.src_ip,
2670 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2671 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2672 input->flow.udp6_flow.ip.dst_ip,
2673 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2674 attributes->l4.udp.hdr = (struct udp_hdr){
2675 .src_port = input->flow.udp6_flow.src_port,
2676 .dst_port = input->flow.udp6_flow.dst_port,
2678 attributes->items[1] = (struct rte_flow_item){
2679 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2680 .spec = &attributes->l3,
2682 attributes->items[2] = (struct rte_flow_item){
2683 .type = RTE_FLOW_ITEM_TYPE_UDP,
2684 .spec = &attributes->l4,
2687 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2688 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2689 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2690 .proto = input->flow.tcp6_flow.ip.proto,
2692 memcpy(attributes->l3.ipv6.hdr.src_addr,
2693 input->flow.tcp6_flow.ip.src_ip,
2694 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2695 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2696 input->flow.tcp6_flow.ip.dst_ip,
2697 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2698 attributes->l4.tcp.hdr = (struct tcp_hdr){
2699 .src_port = input->flow.tcp6_flow.src_port,
2700 .dst_port = input->flow.tcp6_flow.dst_port,
2702 attributes->items[1] = (struct rte_flow_item){
2703 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2704 .spec = &attributes->l3,
2706 attributes->items[2] = (struct rte_flow_item){
2707 .type = RTE_FLOW_ITEM_TYPE_UDP,
2708 .spec = &attributes->l4,
2711 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2712 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2713 .hop_limits = input->flow.ipv6_flow.hop_limits,
2714 .proto = input->flow.ipv6_flow.proto,
2716 memcpy(attributes->l3.ipv6.hdr.src_addr,
2717 input->flow.ipv6_flow.src_ip,
2718 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2719 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2720 input->flow.ipv6_flow.dst_ip,
2721 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2722 attributes->items[1] = (struct rte_flow_item){
2723 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2724 .spec = &attributes->l3,
2728 ERROR("invalid flow type%d",
2729 fdir_filter->input.flow_type);
2736 * Add new flow director filter and store it in list.
2739 * Private structure.
2740 * @param fdir_filter
2741 * Flow director filter to add.
2744 * 0 on success, errno value on failure.
2747 priv_fdir_filter_add(struct priv *priv,
2748 const struct rte_eth_fdir_filter *fdir_filter)
2750 struct mlx5_fdir attributes = {
2753 struct mlx5_flow_parse parser = {
2754 .layer = HASH_RXQ_ETH,
2756 struct rte_flow_error error;
2757 struct rte_flow *flow;
2760 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2763 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2764 attributes.actions, &error, &parser);
2767 flow = priv_flow_create(priv,
2774 TAILQ_INSERT_TAIL(&priv->flows, flow, next);
2775 DEBUG("FDIR created %p", (void *)flow);
2782 * Delete specific filter.
2785 * Private structure.
2786 * @param fdir_filter
2787 * Filter to be deleted.
2790 * 0 on success, errno value on failure.
2793 priv_fdir_filter_delete(struct priv *priv,
2794 const struct rte_eth_fdir_filter *fdir_filter)
2796 struct mlx5_fdir attributes;
2797 struct mlx5_flow_parse parser = {
2799 .layer = HASH_RXQ_ETH,
2801 struct rte_flow_error error;
2802 struct rte_flow *flow;
2806 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2809 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2810 attributes.actions, &error, &parser);
2813 TAILQ_FOREACH(flow, &priv->flows, next) {
2814 struct ibv_flow_attr *attr;
2815 struct ibv_spec_header *attr_h;
2817 struct ibv_flow_attr *flow_attr;
2818 struct ibv_spec_header *flow_h;
2820 unsigned int specs_n;
2823 attr = parser.drop_q.ibv_attr;
2825 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2827 flow_attr = flow->drxq.ibv_attr;
2829 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2830 /* Compare first the attributes. */
2831 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2833 if (attr->num_of_specs == 0)
2835 spec = (void *)((uintptr_t)attr +
2836 sizeof(struct ibv_flow_attr));
2837 flow_spec = (void *)((uintptr_t)flow_attr +
2838 sizeof(struct ibv_flow_attr));
2839 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2840 for (i = 0; i != specs_n; ++i) {
2843 if (memcmp(spec, flow_spec,
2844 RTE_MIN(attr_h->size, flow_h->size)))
2846 spec = (void *)((uintptr_t)attr + attr_h->size);
2847 flow_spec = (void *)((uintptr_t)flow_attr +
2850 /* At this point, the flow match. */
2854 priv_flow_destroy(priv, &priv->flows, flow);
2857 rte_free(parser.drop_q.ibv_attr);
2859 for (i = 0; i != hash_rxq_init_n; ++i) {
2860 if (parser.queue[i].ibv_attr)
2861 rte_free(parser.queue[i].ibv_attr);
2868 * Update queue for specific filter.
2871 * Private structure.
2872 * @param fdir_filter
2873 * Filter to be updated.
2876 * 0 on success, errno value on failure.
2879 priv_fdir_filter_update(struct priv *priv,
2880 const struct rte_eth_fdir_filter *fdir_filter)
2884 ret = priv_fdir_filter_delete(priv, fdir_filter);
2887 ret = priv_fdir_filter_add(priv, fdir_filter);
2892 * Flush all filters.
2895 * Private structure.
2898 priv_fdir_filter_flush(struct priv *priv)
2900 priv_flow_flush(priv, &priv->flows);
2904 * Get flow director information.
2907 * Private structure.
2908 * @param[out] fdir_info
2909 * Resulting flow director information.
2912 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2914 struct rte_eth_fdir_masks *mask =
2915 &priv->dev->data->dev_conf.fdir_conf.mask;
2917 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2918 fdir_info->guarant_spc = 0;
2919 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2920 fdir_info->max_flexpayload = 0;
2921 fdir_info->flow_types_mask[0] = 0;
2922 fdir_info->flex_payload_unit = 0;
2923 fdir_info->max_flex_payload_segment_num = 0;
2924 fdir_info->flex_payload_limit = 0;
2925 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2929 * Deal with flow director operations.
2932 * Pointer to private structure.
2934 * Operation to perform.
2936 * Pointer to operation-specific structure.
2939 * 0 on success, errno value on failure.
2942 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2944 enum rte_fdir_mode fdir_mode =
2945 priv->dev->data->dev_conf.fdir_conf.mode;
2948 if (filter_op == RTE_ETH_FILTER_NOP)
2950 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2951 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2952 ERROR("%p: flow director mode %d not supported",
2953 (void *)priv, fdir_mode);
2956 switch (filter_op) {
2957 case RTE_ETH_FILTER_ADD:
2958 ret = priv_fdir_filter_add(priv, arg);
2960 case RTE_ETH_FILTER_UPDATE:
2961 ret = priv_fdir_filter_update(priv, arg);
2963 case RTE_ETH_FILTER_DELETE:
2964 ret = priv_fdir_filter_delete(priv, arg);
2966 case RTE_ETH_FILTER_FLUSH:
2967 priv_fdir_filter_flush(priv);
2969 case RTE_ETH_FILTER_INFO:
2970 priv_fdir_info_get(priv, arg);
2973 DEBUG("%p: unknown operation %u", (void *)priv,
2982 * Manage filter operations.
2985 * Pointer to Ethernet device structure.
2986 * @param filter_type
2989 * Operation to perform.
2991 * Pointer to operation-specific structure.
2994 * 0 on success, negative errno value on failure.
2997 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
2998 enum rte_filter_type filter_type,
2999 enum rte_filter_op filter_op,
3003 struct priv *priv = dev->data->dev_private;
3005 switch (filter_type) {
3006 case RTE_ETH_FILTER_GENERIC:
3007 if (filter_op != RTE_ETH_FILTER_GET)
3009 *(const void **)arg = &mlx5_flow_ops;
3011 case RTE_ETH_FILTER_FDIR:
3013 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3017 ERROR("%p: filter type (%d) not supported",
3018 (void *)dev, filter_type);