4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
62 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
63 struct ibv_counter_set_init_attr {
66 struct ibv_flow_spec_counter_action {
69 struct ibv_counter_set {
74 ibv_destroy_counter_set(struct ibv_counter_set *cs)
81 /* Dev ops structure defined in mlx5.c */
82 extern const struct eth_dev_ops mlx5_dev_ops;
83 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
86 mlx5_flow_create_eth(const struct rte_flow_item *item,
87 const void *default_mask,
91 mlx5_flow_create_vlan(const struct rte_flow_item *item,
92 const void *default_mask,
96 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
97 const void *default_mask,
101 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
102 const void *default_mask,
106 mlx5_flow_create_udp(const struct rte_flow_item *item,
107 const void *default_mask,
111 mlx5_flow_create_tcp(const struct rte_flow_item *item,
112 const void *default_mask,
116 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
117 const void *default_mask,
120 struct mlx5_flow_parse;
123 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
127 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
130 mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
132 /* Hash RX queue types. */
143 /* Initialization data for hash RX queue. */
144 struct hash_rxq_init {
145 uint64_t hash_fields; /* Fields that participate in the hash. */
146 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
147 unsigned int flow_priority; /* Flow priority to use. */
148 unsigned int ip_version; /* Internet protocol. */
151 /* Initialization data for hash RX queues. */
152 const struct hash_rxq_init hash_rxq_init[] = {
154 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
155 IBV_RX_HASH_DST_IPV4 |
156 IBV_RX_HASH_SRC_PORT_TCP |
157 IBV_RX_HASH_DST_PORT_TCP),
158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
160 .ip_version = MLX5_IPV4,
163 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
164 IBV_RX_HASH_DST_IPV4 |
165 IBV_RX_HASH_SRC_PORT_UDP |
166 IBV_RX_HASH_DST_PORT_UDP),
167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
169 .ip_version = MLX5_IPV4,
172 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
173 IBV_RX_HASH_DST_IPV4),
174 .dpdk_rss_hf = (ETH_RSS_IPV4 |
177 .ip_version = MLX5_IPV4,
180 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
181 IBV_RX_HASH_DST_IPV6 |
182 IBV_RX_HASH_SRC_PORT_TCP |
183 IBV_RX_HASH_DST_PORT_TCP),
184 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
186 .ip_version = MLX5_IPV6,
189 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
190 IBV_RX_HASH_DST_IPV6 |
191 IBV_RX_HASH_SRC_PORT_UDP |
192 IBV_RX_HASH_DST_PORT_UDP),
193 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
195 .ip_version = MLX5_IPV6,
198 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
199 IBV_RX_HASH_DST_IPV6),
200 .dpdk_rss_hf = (ETH_RSS_IPV6 |
203 .ip_version = MLX5_IPV6,
212 /* Number of entries in hash_rxq_init[]. */
213 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
215 /** Structure for holding counter stats. */
216 struct mlx5_flow_counter_stats {
217 uint64_t hits; /**< Number of packets matched by the rule. */
218 uint64_t bytes; /**< Number of bytes matched by the rule. */
221 /** Structure for Drop queue. */
222 struct mlx5_hrxq_drop {
223 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
224 struct ibv_qp *qp; /**< Verbs queue pair. */
225 struct ibv_wq *wq; /**< Verbs work queue. */
226 struct ibv_cq *cq; /**< Verbs completion queue. */
229 /* Flows structures. */
231 uint64_t hash_fields; /**< Fields that participate in the hash. */
232 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
233 struct ibv_flow *ibv_flow; /**< Verbs flow. */
234 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
237 /* Drop flows structures. */
238 struct mlx5_flow_drop {
239 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
240 struct ibv_flow *ibv_flow; /**< Verbs flow. */
244 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
245 uint32_t mark:1; /**< Set if the flow is marked. */
246 uint32_t drop:1; /**< Drop queue. */
247 uint16_t queues_n; /**< Number of entries in queue[]. */
248 uint16_t (*queues)[]; /**< Queues indexes to use. */
249 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
250 uint8_t rss_key[40]; /**< copy of the RSS key. */
251 struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
252 struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
254 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
255 /**< Flow with Rx queue. */
256 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */
260 /** Static initializer for items. */
262 (const enum rte_flow_item_type []){ \
263 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
266 /** Structure to generate a simple graph of layers supported by the NIC. */
267 struct mlx5_flow_items {
268 /** List of possible actions for these items. */
269 const enum rte_flow_action_type *const actions;
270 /** Bit-masks corresponding to the possibilities for the item. */
273 * Default bit-masks to use when item->mask is not provided. When
274 * \default_mask is also NULL, the full supported bit-mask (\mask) is
277 const void *default_mask;
278 /** Bit-masks size in bytes. */
279 const unsigned int mask_sz;
281 * Conversion function from rte_flow to NIC specific flow.
284 * rte_flow item to convert.
285 * @param default_mask
286 * Default bit-masks to use when item->mask is not provided.
288 * Internal structure to store the conversion.
291 * 0 on success, negative value otherwise.
293 int (*convert)(const struct rte_flow_item *item,
294 const void *default_mask,
296 /** Size in bytes of the destination structure. */
297 const unsigned int dst_sz;
298 /** List of possible following items. */
299 const enum rte_flow_item_type *const items;
302 /** Valid action for this PMD. */
303 static const enum rte_flow_action_type valid_actions[] = {
304 RTE_FLOW_ACTION_TYPE_DROP,
305 RTE_FLOW_ACTION_TYPE_QUEUE,
306 RTE_FLOW_ACTION_TYPE_MARK,
307 RTE_FLOW_ACTION_TYPE_FLAG,
308 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
309 RTE_FLOW_ACTION_TYPE_COUNT,
311 RTE_FLOW_ACTION_TYPE_END,
314 /** Graph of supported items and associated actions. */
315 static const struct mlx5_flow_items mlx5_flow_items[] = {
316 [RTE_FLOW_ITEM_TYPE_END] = {
317 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
318 RTE_FLOW_ITEM_TYPE_VXLAN),
320 [RTE_FLOW_ITEM_TYPE_ETH] = {
321 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
322 RTE_FLOW_ITEM_TYPE_IPV4,
323 RTE_FLOW_ITEM_TYPE_IPV6),
324 .actions = valid_actions,
325 .mask = &(const struct rte_flow_item_eth){
326 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
327 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
330 .default_mask = &rte_flow_item_eth_mask,
331 .mask_sz = sizeof(struct rte_flow_item_eth),
332 .convert = mlx5_flow_create_eth,
333 .dst_sz = sizeof(struct ibv_flow_spec_eth),
335 [RTE_FLOW_ITEM_TYPE_VLAN] = {
336 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
337 RTE_FLOW_ITEM_TYPE_IPV6),
338 .actions = valid_actions,
339 .mask = &(const struct rte_flow_item_vlan){
342 .default_mask = &rte_flow_item_vlan_mask,
343 .mask_sz = sizeof(struct rte_flow_item_vlan),
344 .convert = mlx5_flow_create_vlan,
347 [RTE_FLOW_ITEM_TYPE_IPV4] = {
348 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
349 RTE_FLOW_ITEM_TYPE_TCP),
350 .actions = valid_actions,
351 .mask = &(const struct rte_flow_item_ipv4){
355 .type_of_service = -1,
359 .default_mask = &rte_flow_item_ipv4_mask,
360 .mask_sz = sizeof(struct rte_flow_item_ipv4),
361 .convert = mlx5_flow_create_ipv4,
362 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
364 [RTE_FLOW_ITEM_TYPE_IPV6] = {
365 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
366 RTE_FLOW_ITEM_TYPE_TCP),
367 .actions = valid_actions,
368 .mask = &(const struct rte_flow_item_ipv6){
371 0xff, 0xff, 0xff, 0xff,
372 0xff, 0xff, 0xff, 0xff,
373 0xff, 0xff, 0xff, 0xff,
374 0xff, 0xff, 0xff, 0xff,
377 0xff, 0xff, 0xff, 0xff,
378 0xff, 0xff, 0xff, 0xff,
379 0xff, 0xff, 0xff, 0xff,
380 0xff, 0xff, 0xff, 0xff,
387 .default_mask = &rte_flow_item_ipv6_mask,
388 .mask_sz = sizeof(struct rte_flow_item_ipv6),
389 .convert = mlx5_flow_create_ipv6,
390 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
392 [RTE_FLOW_ITEM_TYPE_UDP] = {
393 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
394 .actions = valid_actions,
395 .mask = &(const struct rte_flow_item_udp){
401 .default_mask = &rte_flow_item_udp_mask,
402 .mask_sz = sizeof(struct rte_flow_item_udp),
403 .convert = mlx5_flow_create_udp,
404 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
406 [RTE_FLOW_ITEM_TYPE_TCP] = {
407 .actions = valid_actions,
408 .mask = &(const struct rte_flow_item_tcp){
414 .default_mask = &rte_flow_item_tcp_mask,
415 .mask_sz = sizeof(struct rte_flow_item_tcp),
416 .convert = mlx5_flow_create_tcp,
417 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
419 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
420 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
421 .actions = valid_actions,
422 .mask = &(const struct rte_flow_item_vxlan){
423 .vni = "\xff\xff\xff",
425 .default_mask = &rte_flow_item_vxlan_mask,
426 .mask_sz = sizeof(struct rte_flow_item_vxlan),
427 .convert = mlx5_flow_create_vxlan,
428 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
432 /** Structure to pass to the conversion function. */
433 struct mlx5_flow_parse {
434 uint32_t inner; /**< Set once VXLAN is encountered. */
436 /**< Whether resources should remain after a validate. */
437 uint32_t drop:1; /**< Target is a drop queue. */
438 uint32_t mark:1; /**< Mark is present in the flow. */
439 uint32_t count:1; /**< Count is present in the flow. */
440 uint32_t mark_id; /**< Mark identifier. */
441 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
442 uint16_t queues_n; /**< Number of entries in queue[]. */
443 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
444 uint8_t rss_key[40]; /**< copy of the RSS key. */
445 enum hash_rxq_type layer; /**< Last pattern layer detected. */
446 struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
449 struct ibv_flow_attr *ibv_attr;
450 /**< Pointer to Verbs attributes. */
452 /**< Current position or total size of the attribute. */
453 } queue[RTE_DIM(hash_rxq_init)];
455 struct ibv_flow_attr *ibv_attr;
456 /**< Pointer to Verbs attributes. */
458 /**< Current position or total size of the attribute. */
463 static const struct rte_flow_ops mlx5_flow_ops = {
464 .validate = mlx5_flow_validate,
465 .create = mlx5_flow_create,
466 .destroy = mlx5_flow_destroy,
467 .flush = mlx5_flow_flush,
468 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
469 .query = mlx5_flow_query,
473 .isolate = mlx5_flow_isolate,
476 /* Convert FDIR request to Generic flow. */
478 struct rte_flow_attr attr;
479 struct rte_flow_action actions[2];
480 struct rte_flow_item items[4];
481 struct rte_flow_item_eth l2;
482 struct rte_flow_item_eth l2_mask;
484 struct rte_flow_item_ipv4 ipv4;
485 struct rte_flow_item_ipv6 ipv6;
488 struct rte_flow_item_udp udp;
489 struct rte_flow_item_tcp tcp;
491 struct rte_flow_action_queue queue;
494 /* Verbs specification header. */
495 struct ibv_spec_header {
496 enum ibv_flow_spec_type type;
501 * Check support for a given item.
504 * Item specification.
506 * Bit-masks covering supported fields to compare with spec, last and mask in
509 * Bit-Mask size in bytes.
515 mlx5_flow_item_validate(const struct rte_flow_item *item,
516 const uint8_t *mask, unsigned int size)
520 if (!item->spec && (item->mask || item->last))
522 if (item->spec && !item->mask) {
524 const uint8_t *spec = item->spec;
526 for (i = 0; i < size; ++i)
527 if ((spec[i] | mask[i]) != mask[i])
530 if (item->last && !item->mask) {
532 const uint8_t *spec = item->last;
534 for (i = 0; i < size; ++i)
535 if ((spec[i] | mask[i]) != mask[i])
540 const uint8_t *spec = item->mask;
542 for (i = 0; i < size; ++i)
543 if ((spec[i] | mask[i]) != mask[i])
546 if (item->spec && item->last) {
549 const uint8_t *apply = mask;
554 for (i = 0; i < size; ++i) {
555 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
556 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
558 ret = memcmp(spec, last, size);
564 * Copy the RSS configuration from the user ones.
567 * Pointer to private structure.
569 * Internal parser structure.
571 * User RSS configuration to save.
574 * 0 on success, errno value on failure.
577 priv_flow_convert_rss_conf(struct priv *priv,
578 struct mlx5_flow_parse *parser,
579 const struct rte_eth_rss_conf *rss_conf)
581 const struct rte_eth_rss_conf *rss =
582 rss_conf ? rss_conf : &priv->rss_conf;
584 if (rss->rss_key_len > 40)
586 parser->rss_conf.rss_key_len = rss->rss_key_len;
587 parser->rss_conf.rss_hf = rss->rss_hf;
588 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
589 parser->rss_conf.rss_key = parser->rss_key;
594 * Extract attribute to the parser.
597 * Pointer to private structure.
599 * Flow rule attributes.
601 * Perform verbose error reporting if not NULL.
602 * @param[in, out] parser
603 * Internal parser structure.
606 * 0 on success, a negative errno value otherwise and rte_errno is set.
609 priv_flow_convert_attributes(struct priv *priv,
610 const struct rte_flow_attr *attr,
611 struct rte_flow_error *error,
612 struct mlx5_flow_parse *parser)
617 rte_flow_error_set(error, ENOTSUP,
618 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
620 "groups are not supported");
623 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
624 rte_flow_error_set(error, ENOTSUP,
625 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
627 "priorities are not supported");
631 rte_flow_error_set(error, ENOTSUP,
632 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
634 "egress is not supported");
637 if (!attr->ingress) {
638 rte_flow_error_set(error, ENOTSUP,
639 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
641 "only ingress is supported");
648 * Extract actions request to the parser.
651 * Pointer to private structure.
653 * Associated actions (list terminated by the END action).
655 * Perform verbose error reporting if not NULL.
656 * @param[in, out] parser
657 * Internal parser structure.
660 * 0 on success, a negative errno value otherwise and rte_errno is set.
663 priv_flow_convert_actions(struct priv *priv,
664 const struct rte_flow_action actions[],
665 struct rte_flow_error *error,
666 struct mlx5_flow_parse *parser)
669 * Add default RSS configuration necessary for Verbs to create QP even
670 * if no RSS is necessary.
672 priv_flow_convert_rss_conf(priv, parser,
673 (const struct rte_eth_rss_conf *)
675 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
676 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
678 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
680 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
681 const struct rte_flow_action_queue *queue =
682 (const struct rte_flow_action_queue *)
687 if (!queue || (queue->index > (priv->rxqs_n - 1)))
688 goto exit_action_not_supported;
689 for (n = 0; n < parser->queues_n; ++n) {
690 if (parser->queues[n] == queue->index) {
695 if (parser->queues_n > 1 && !found) {
696 rte_flow_error_set(error, ENOTSUP,
697 RTE_FLOW_ERROR_TYPE_ACTION,
699 "queue action not in RSS queues");
703 parser->queues_n = 1;
704 parser->queues[0] = queue->index;
706 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
707 const struct rte_flow_action_rss *rss =
708 (const struct rte_flow_action_rss *)
712 if (!rss || !rss->num) {
713 rte_flow_error_set(error, EINVAL,
714 RTE_FLOW_ERROR_TYPE_ACTION,
719 if (parser->queues_n == 1) {
722 assert(parser->queues_n);
723 for (n = 0; n < rss->num; ++n) {
724 if (parser->queues[0] ==
731 rte_flow_error_set(error, ENOTSUP,
732 RTE_FLOW_ERROR_TYPE_ACTION,
734 "queue action not in RSS"
739 for (n = 0; n < rss->num; ++n) {
740 if (rss->queue[n] >= priv->rxqs_n) {
741 rte_flow_error_set(error, EINVAL,
742 RTE_FLOW_ERROR_TYPE_ACTION,
744 "queue id > number of"
749 for (n = 0; n < rss->num; ++n)
750 parser->queues[n] = rss->queue[n];
751 parser->queues_n = rss->num;
752 if (priv_flow_convert_rss_conf(priv, parser,
754 rte_flow_error_set(error, EINVAL,
755 RTE_FLOW_ERROR_TYPE_ACTION,
757 "wrong RSS configuration");
760 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
761 const struct rte_flow_action_mark *mark =
762 (const struct rte_flow_action_mark *)
766 rte_flow_error_set(error, EINVAL,
767 RTE_FLOW_ERROR_TYPE_ACTION,
769 "mark must be defined");
771 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
772 rte_flow_error_set(error, ENOTSUP,
773 RTE_FLOW_ERROR_TYPE_ACTION,
775 "mark must be between 0"
780 parser->mark_id = mark->id;
781 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
783 } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
784 priv->counter_set_supported) {
787 goto exit_action_not_supported;
790 if (parser->drop && parser->mark)
792 if (!parser->queues_n && !parser->drop) {
793 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
794 NULL, "no valid action");
798 exit_action_not_supported:
799 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
800 actions, "action not supported");
808 * Pointer to private structure.
810 * Pattern specification (list terminated by the END pattern item).
812 * Perform verbose error reporting if not NULL.
813 * @param[in, out] parser
814 * Internal parser structure.
817 * 0 on success, a negative errno value otherwise and rte_errno is set.
820 priv_flow_convert_items_validate(struct priv *priv,
821 const struct rte_flow_item items[],
822 struct rte_flow_error *error,
823 struct mlx5_flow_parse *parser)
825 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
829 /* Initialise the offsets to start after verbs attribute. */
831 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
833 for (i = 0; i != hash_rxq_init_n; ++i)
834 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
836 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
837 const struct mlx5_flow_items *token = NULL;
841 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
845 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
847 if (cur_item->items[i] == items->type) {
848 token = &mlx5_flow_items[items->type];
853 goto exit_item_not_supported;
855 err = mlx5_flow_item_validate(items,
856 (const uint8_t *)cur_item->mask,
859 goto exit_item_not_supported;
860 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
862 rte_flow_error_set(error, ENOTSUP,
863 RTE_FLOW_ERROR_TYPE_ITEM,
865 "cannot recognize multiple"
866 " VXLAN encapsulations");
869 parser->inner = IBV_FLOW_SPEC_INNER;
872 parser->drop_q.offset += cur_item->dst_sz;
873 } else if (parser->queues_n == 1) {
874 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
876 for (n = 0; n != hash_rxq_init_n; ++n)
877 parser->queue[n].offset += cur_item->dst_sz;
881 for (i = 0; i != hash_rxq_init_n; ++i)
882 parser->queue[i].offset +=
883 sizeof(struct ibv_flow_spec_action_tag);
886 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
889 parser->drop_q.offset += size;
891 for (i = 0; i != hash_rxq_init_n; ++i)
892 parser->queue[i].offset += size;
896 exit_item_not_supported:
897 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
898 items, "item not supported");
903 * Allocate memory space to store verbs flow attributes.
906 * Pointer to private structure.
907 * @param[in] priority
910 * Amount of byte to allocate.
912 * Perform verbose error reporting if not NULL.
915 * A verbs flow attribute on success, NULL otherwise.
917 static struct ibv_flow_attr*
918 priv_flow_convert_allocate(struct priv *priv,
919 unsigned int priority,
921 struct rte_flow_error *error)
923 struct ibv_flow_attr *ibv_attr;
926 ibv_attr = rte_calloc(__func__, 1, size, 0);
928 rte_flow_error_set(error, ENOMEM,
929 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
931 "cannot allocate verbs spec attributes.");
934 ibv_attr->priority = priority;
939 * Finalise verbs flow attributes.
942 * Pointer to private structure.
943 * @param[in, out] parser
944 * Internal parser structure.
947 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
949 const unsigned int ipv4 =
950 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
951 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
952 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
953 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
954 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
955 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
959 if (parser->layer == HASH_RXQ_ETH) {
963 * This layer becomes useless as the pattern define under
966 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
967 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
969 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
970 for (i = ohmin; i != (ohmax + 1); ++i) {
971 if (!parser->queue[i].ibv_attr)
973 rte_free(parser->queue[i].ibv_attr);
974 parser->queue[i].ibv_attr = NULL;
976 /* Remove impossible flow according to the RSS configuration. */
977 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
978 parser->rss_conf.rss_hf) {
979 /* Remove any other flow. */
980 for (i = hmin; i != (hmax + 1); ++i) {
981 if ((i == parser->layer) ||
982 (!parser->queue[i].ibv_attr))
984 rte_free(parser->queue[i].ibv_attr);
985 parser->queue[i].ibv_attr = NULL;
987 } else if (!parser->queue[ip].ibv_attr) {
988 /* no RSS possible with the current configuration. */
989 parser->queues_n = 1;
994 * Fill missing layers in verbs specifications, or compute the correct
995 * offset to allocate the memory space for the attributes and
998 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
1000 struct ibv_flow_spec_ipv4_ext ipv4;
1001 struct ibv_flow_spec_ipv6 ipv6;
1002 struct ibv_flow_spec_tcp_udp udp_tcp;
1007 if (i == parser->layer)
1009 if (parser->layer == HASH_RXQ_ETH) {
1010 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
1011 size = sizeof(struct ibv_flow_spec_ipv4_ext);
1012 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
1013 .type = IBV_FLOW_SPEC_IPV4_EXT,
1017 size = sizeof(struct ibv_flow_spec_ipv6);
1018 specs.ipv6 = (struct ibv_flow_spec_ipv6){
1019 .type = IBV_FLOW_SPEC_IPV6,
1023 if (parser->queue[i].ibv_attr) {
1024 dst = (void *)((uintptr_t)
1025 parser->queue[i].ibv_attr +
1026 parser->queue[i].offset);
1027 memcpy(dst, &specs, size);
1028 ++parser->queue[i].ibv_attr->num_of_specs;
1030 parser->queue[i].offset += size;
1032 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
1033 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
1034 size = sizeof(struct ibv_flow_spec_tcp_udp);
1035 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
1036 .type = ((i == HASH_RXQ_UDPV4 ||
1037 i == HASH_RXQ_UDPV6) ?
1042 if (parser->queue[i].ibv_attr) {
1043 dst = (void *)((uintptr_t)
1044 parser->queue[i].ibv_attr +
1045 parser->queue[i].offset);
1046 memcpy(dst, &specs, size);
1047 ++parser->queue[i].ibv_attr->num_of_specs;
1049 parser->queue[i].offset += size;
1055 * Validate and convert a flow supported by the NIC.
1058 * Pointer to private structure.
1060 * Flow rule attributes.
1061 * @param[in] pattern
1062 * Pattern specification (list terminated by the END pattern item).
1063 * @param[in] actions
1064 * Associated actions (list terminated by the END action).
1066 * Perform verbose error reporting if not NULL.
1067 * @param[in, out] parser
1068 * Internal parser structure.
1071 * 0 on success, a negative errno value otherwise and rte_errno is set.
1074 priv_flow_convert(struct priv *priv,
1075 const struct rte_flow_attr *attr,
1076 const struct rte_flow_item items[],
1077 const struct rte_flow_action actions[],
1078 struct rte_flow_error *error,
1079 struct mlx5_flow_parse *parser)
1081 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1085 /* First step. Validate the attributes, items and actions. */
1086 *parser = (struct mlx5_flow_parse){
1087 .create = parser->create,
1088 .layer = HASH_RXQ_ETH,
1089 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1091 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1094 ret = priv_flow_convert_actions(priv, actions, error, parser);
1097 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1100 priv_flow_convert_finalise(priv, parser);
1103 * Allocate the memory space to store verbs specifications.
1106 parser->drop_q.ibv_attr =
1107 priv_flow_convert_allocate(priv, attr->priority,
1108 parser->drop_q.offset,
1110 if (!parser->drop_q.ibv_attr)
1112 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
1113 } else if (parser->queues_n == 1) {
1114 unsigned int priority =
1116 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1117 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1119 parser->queue[HASH_RXQ_ETH].ibv_attr =
1120 priv_flow_convert_allocate(priv, priority,
1122 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1124 parser->queue[HASH_RXQ_ETH].offset =
1125 sizeof(struct ibv_flow_attr);
1127 for (i = 0; i != hash_rxq_init_n; ++i) {
1128 unsigned int priority =
1130 hash_rxq_init[i].flow_priority;
1131 unsigned int offset;
1133 if (!(parser->rss_conf.rss_hf &
1134 hash_rxq_init[i].dpdk_rss_hf) &&
1135 (i != HASH_RXQ_ETH))
1137 offset = parser->queue[i].offset;
1138 parser->queue[i].ibv_attr =
1139 priv_flow_convert_allocate(priv, priority,
1141 if (!parser->queue[i].ibv_attr)
1143 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1146 /* Third step. Conversion parse, fill the specifications. */
1148 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1149 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1151 cur_item = &mlx5_flow_items[items->type];
1152 ret = cur_item->convert(items,
1153 (cur_item->default_mask ?
1154 cur_item->default_mask :
1158 rte_flow_error_set(error, ret,
1159 RTE_FLOW_ERROR_TYPE_ITEM,
1160 items, "item not supported");
1165 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1166 if (parser->count && parser->create) {
1167 mlx5_flow_create_count(priv, parser);
1169 goto exit_count_error;
1172 * Last step. Complete missing specification to reach the RSS
1177 * Drop queue priority needs to be adjusted to
1178 * their most specific layer priority.
1180 parser->drop_q.ibv_attr->priority =
1182 hash_rxq_init[parser->layer].flow_priority;
1183 } else if (parser->queues_n > 1) {
1184 priv_flow_convert_finalise(priv, parser);
1187 * Action queue have their priority overridden with
1188 * Ethernet priority, this priority needs to be adjusted to
1189 * their most specific layer priority.
1191 parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
1193 hash_rxq_init[parser->layer].flow_priority;
1196 /* Only verification is expected, all resources should be released. */
1197 if (!parser->create) {
1199 rte_free(parser->drop_q.ibv_attr);
1200 parser->drop_q.ibv_attr = NULL;
1202 for (i = 0; i != hash_rxq_init_n; ++i) {
1203 if (parser->queue[i].ibv_attr) {
1204 rte_free(parser->queue[i].ibv_attr);
1205 parser->queue[i].ibv_attr = NULL;
1211 for (i = 0; i != hash_rxq_init_n; ++i) {
1212 if (parser->queue[i].ibv_attr) {
1213 rte_free(parser->queue[i].ibv_attr);
1214 parser->queue[i].ibv_attr = NULL;
1217 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1218 NULL, "cannot allocate verbs spec attributes.");
1221 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1222 NULL, "cannot create counter.");
1227 * Copy the specification created into the flow.
1230 * Internal parser structure.
1232 * Create specification.
1234 * Size in bytes of the specification to copy.
1237 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1244 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1245 parser->drop_q.offset);
1246 memcpy(dst, src, size);
1247 ++parser->drop_q.ibv_attr->num_of_specs;
1248 parser->drop_q.offset += size;
1251 for (i = 0; i != hash_rxq_init_n; ++i) {
1252 if (!parser->queue[i].ibv_attr)
1254 /* Specification must be the same l3 type or none. */
1255 if (parser->layer == HASH_RXQ_ETH ||
1256 (hash_rxq_init[parser->layer].ip_version ==
1257 hash_rxq_init[i].ip_version) ||
1258 (hash_rxq_init[i].ip_version == 0)) {
1259 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1260 parser->queue[i].offset);
1261 memcpy(dst, src, size);
1262 ++parser->queue[i].ibv_attr->num_of_specs;
1263 parser->queue[i].offset += size;
1269 * Convert Ethernet item to Verbs specification.
1272 * Item specification.
1273 * @param default_mask[in]
1274 * Default bit-masks to use when item->mask is not provided.
1275 * @param data[in, out]
1279 mlx5_flow_create_eth(const struct rte_flow_item *item,
1280 const void *default_mask,
1283 const struct rte_flow_item_eth *spec = item->spec;
1284 const struct rte_flow_item_eth *mask = item->mask;
1285 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1286 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1287 struct ibv_flow_spec_eth eth = {
1288 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1292 /* Don't update layer for the inner pattern. */
1294 parser->layer = HASH_RXQ_ETH;
1299 mask = default_mask;
1300 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1301 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1302 eth.val.ether_type = spec->type;
1303 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1304 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1305 eth.mask.ether_type = mask->type;
1306 /* Remove unwanted bits from values. */
1307 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1308 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1309 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1311 eth.val.ether_type &= eth.mask.ether_type;
1313 mlx5_flow_create_copy(parser, ð, eth_size);
1318 * Convert VLAN item to Verbs specification.
1321 * Item specification.
1322 * @param default_mask[in]
1323 * Default bit-masks to use when item->mask is not provided.
1324 * @param data[in, out]
1328 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1329 const void *default_mask,
1332 const struct rte_flow_item_vlan *spec = item->spec;
1333 const struct rte_flow_item_vlan *mask = item->mask;
1334 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1335 struct ibv_flow_spec_eth *eth;
1336 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1341 mask = default_mask;
1344 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1345 parser->drop_q.offset - eth_size);
1346 eth->val.vlan_tag = spec->tci;
1347 eth->mask.vlan_tag = mask->tci;
1348 eth->val.vlan_tag &= eth->mask.vlan_tag;
1351 for (i = 0; i != hash_rxq_init_n; ++i) {
1352 if (!parser->queue[i].ibv_attr)
1355 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1356 parser->queue[i].offset - eth_size);
1357 eth->val.vlan_tag = spec->tci;
1358 eth->mask.vlan_tag = mask->tci;
1359 eth->val.vlan_tag &= eth->mask.vlan_tag;
1366 * Convert IPv4 item to Verbs specification.
1369 * Item specification.
1370 * @param default_mask[in]
1371 * Default bit-masks to use when item->mask is not provided.
1372 * @param data[in, out]
1376 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1377 const void *default_mask,
1380 const struct rte_flow_item_ipv4 *spec = item->spec;
1381 const struct rte_flow_item_ipv4 *mask = item->mask;
1382 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1383 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1384 struct ibv_flow_spec_ipv4_ext ipv4 = {
1385 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1389 /* Don't update layer for the inner pattern. */
1391 parser->layer = HASH_RXQ_IPV4;
1394 mask = default_mask;
1395 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1396 .src_ip = spec->hdr.src_addr,
1397 .dst_ip = spec->hdr.dst_addr,
1398 .proto = spec->hdr.next_proto_id,
1399 .tos = spec->hdr.type_of_service,
1401 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1402 .src_ip = mask->hdr.src_addr,
1403 .dst_ip = mask->hdr.dst_addr,
1404 .proto = mask->hdr.next_proto_id,
1405 .tos = mask->hdr.type_of_service,
1407 /* Remove unwanted bits from values. */
1408 ipv4.val.src_ip &= ipv4.mask.src_ip;
1409 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1410 ipv4.val.proto &= ipv4.mask.proto;
1411 ipv4.val.tos &= ipv4.mask.tos;
1413 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1418 * Convert IPv6 item to Verbs specification.
1421 * Item specification.
1422 * @param default_mask[in]
1423 * Default bit-masks to use when item->mask is not provided.
1424 * @param data[in, out]
1428 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1429 const void *default_mask,
1432 const struct rte_flow_item_ipv6 *spec = item->spec;
1433 const struct rte_flow_item_ipv6 *mask = item->mask;
1434 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1435 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1436 struct ibv_flow_spec_ipv6 ipv6 = {
1437 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1441 /* Don't update layer for the inner pattern. */
1443 parser->layer = HASH_RXQ_IPV6;
1448 mask = default_mask;
1449 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1450 RTE_DIM(ipv6.val.src_ip));
1451 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1452 RTE_DIM(ipv6.val.dst_ip));
1453 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1454 RTE_DIM(ipv6.mask.src_ip));
1455 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1456 RTE_DIM(ipv6.mask.dst_ip));
1457 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1458 ipv6.mask.next_hdr = mask->hdr.proto;
1459 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1460 /* Remove unwanted bits from values. */
1461 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1462 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1463 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1465 ipv6.val.flow_label &= ipv6.mask.flow_label;
1466 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1467 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1469 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1474 * Convert UDP item to Verbs specification.
1477 * Item specification.
1478 * @param default_mask[in]
1479 * Default bit-masks to use when item->mask is not provided.
1480 * @param data[in, out]
1484 mlx5_flow_create_udp(const struct rte_flow_item *item,
1485 const void *default_mask,
1488 const struct rte_flow_item_udp *spec = item->spec;
1489 const struct rte_flow_item_udp *mask = item->mask;
1490 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1491 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1492 struct ibv_flow_spec_tcp_udp udp = {
1493 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1497 /* Don't update layer for the inner pattern. */
1498 if (!parser->inner) {
1499 if (parser->layer == HASH_RXQ_IPV4)
1500 parser->layer = HASH_RXQ_UDPV4;
1502 parser->layer = HASH_RXQ_UDPV6;
1506 mask = default_mask;
1507 udp.val.dst_port = spec->hdr.dst_port;
1508 udp.val.src_port = spec->hdr.src_port;
1509 udp.mask.dst_port = mask->hdr.dst_port;
1510 udp.mask.src_port = mask->hdr.src_port;
1511 /* Remove unwanted bits from values. */
1512 udp.val.src_port &= udp.mask.src_port;
1513 udp.val.dst_port &= udp.mask.dst_port;
1515 mlx5_flow_create_copy(parser, &udp, udp_size);
1520 * Convert TCP item to Verbs specification.
1523 * Item specification.
1524 * @param default_mask[in]
1525 * Default bit-masks to use when item->mask is not provided.
1526 * @param data[in, out]
1530 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1531 const void *default_mask,
1534 const struct rte_flow_item_tcp *spec = item->spec;
1535 const struct rte_flow_item_tcp *mask = item->mask;
1536 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1537 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1538 struct ibv_flow_spec_tcp_udp tcp = {
1539 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1543 /* Don't update layer for the inner pattern. */
1544 if (!parser->inner) {
1545 if (parser->layer == HASH_RXQ_IPV4)
1546 parser->layer = HASH_RXQ_TCPV4;
1548 parser->layer = HASH_RXQ_TCPV6;
1552 mask = default_mask;
1553 tcp.val.dst_port = spec->hdr.dst_port;
1554 tcp.val.src_port = spec->hdr.src_port;
1555 tcp.mask.dst_port = mask->hdr.dst_port;
1556 tcp.mask.src_port = mask->hdr.src_port;
1557 /* Remove unwanted bits from values. */
1558 tcp.val.src_port &= tcp.mask.src_port;
1559 tcp.val.dst_port &= tcp.mask.dst_port;
1561 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1566 * Convert VXLAN item to Verbs specification.
1569 * Item specification.
1570 * @param default_mask[in]
1571 * Default bit-masks to use when item->mask is not provided.
1572 * @param data[in, out]
1576 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1577 const void *default_mask,
1580 const struct rte_flow_item_vxlan *spec = item->spec;
1581 const struct rte_flow_item_vxlan *mask = item->mask;
1582 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1583 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1584 struct ibv_flow_spec_tunnel vxlan = {
1585 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1594 parser->inner = IBV_FLOW_SPEC_INNER;
1597 mask = default_mask;
1598 memcpy(&id.vni[1], spec->vni, 3);
1599 vxlan.val.tunnel_id = id.vlan_id;
1600 memcpy(&id.vni[1], mask->vni, 3);
1601 vxlan.mask.tunnel_id = id.vlan_id;
1602 /* Remove unwanted bits from values. */
1603 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1606 * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
1607 * layer is defined in the Verbs specification it is interpreted as
1608 * wildcard and all packets will match this rule, if it follows a full
1609 * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
1610 * before will also match this rule.
1611 * To avoid such situation, VNI 0 is currently refused.
1613 if (!vxlan.val.tunnel_id)
1615 mlx5_flow_create_copy(parser, &vxlan, size);
1620 * Convert mark/flag action to Verbs specification.
1623 * Internal parser structure.
1628 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1630 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1631 struct ibv_flow_spec_action_tag tag = {
1632 .type = IBV_FLOW_SPEC_ACTION_TAG,
1634 .tag_id = mlx5_flow_mark_set(mark_id),
1637 assert(parser->mark);
1638 mlx5_flow_create_copy(parser, &tag, size);
1643 * Convert count action to Verbs specification.
1646 * Pointer to private structure.
1648 * Pointer to MLX5 flow parser structure.
1651 * 0 on success, errno value on failure.
1654 mlx5_flow_create_count(struct priv *priv __rte_unused,
1655 struct mlx5_flow_parse *parser __rte_unused)
1657 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
1658 unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
1659 struct ibv_counter_set_init_attr init_attr = {0};
1660 struct ibv_flow_spec_counter_action counter = {
1661 .type = IBV_FLOW_SPEC_ACTION_COUNT,
1663 .counter_set_handle = 0,
1666 init_attr.counter_set_id = 0;
1667 parser->cs = ibv_create_counter_set(priv->ctx, &init_attr);
1670 counter.counter_set_handle = parser->cs->handle;
1671 mlx5_flow_create_copy(parser, &counter, size);
1677 * Complete flow rule creation with a drop queue.
1680 * Pointer to private structure.
1682 * Internal parser structure.
1684 * Pointer to the rte_flow.
1686 * Perform verbose error reporting if not NULL.
1689 * 0 on success, errno value on failure.
1692 priv_flow_create_action_queue_drop(struct priv *priv,
1693 struct mlx5_flow_parse *parser,
1694 struct rte_flow *flow,
1695 struct rte_flow_error *error)
1697 struct ibv_flow_spec_action_drop *drop;
1698 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1704 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1705 parser->drop_q.offset);
1706 *drop = (struct ibv_flow_spec_action_drop){
1707 .type = IBV_FLOW_SPEC_ACTION_DROP,
1710 ++parser->drop_q.ibv_attr->num_of_specs;
1711 parser->drop_q.offset += size;
1712 flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
1714 flow->cs = parser->cs;
1715 if (!priv->dev->data->dev_started)
1717 parser->drop_q.ibv_attr = NULL;
1718 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
1719 flow->drxq.ibv_attr);
1720 if (!flow->drxq.ibv_flow) {
1721 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1722 NULL, "flow rule creation failure");
1729 if (flow->drxq.ibv_flow) {
1730 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1731 flow->drxq.ibv_flow = NULL;
1733 if (flow->drxq.ibv_attr) {
1734 rte_free(flow->drxq.ibv_attr);
1735 flow->drxq.ibv_attr = NULL;
1738 claim_zero(ibv_destroy_counter_set(flow->cs));
1746 * Create hash Rx queues when RSS is enabled.
1749 * Pointer to private structure.
1751 * Internal parser structure.
1753 * Pointer to the rte_flow.
1755 * Perform verbose error reporting if not NULL.
1758 * 0 on success, a errno value otherwise and rte_errno is set.
1761 priv_flow_create_action_queue_rss(struct priv *priv,
1762 struct mlx5_flow_parse *parser,
1763 struct rte_flow *flow,
1764 struct rte_flow_error *error)
1768 for (i = 0; i != hash_rxq_init_n; ++i) {
1769 uint64_t hash_fields;
1771 if (!parser->queue[i].ibv_attr)
1773 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1774 parser->queue[i].ibv_attr = NULL;
1775 hash_fields = hash_rxq_init[i].hash_fields;
1776 if (!priv->dev->data->dev_started)
1778 flow->frxq[i].hrxq =
1779 mlx5_priv_hrxq_get(priv,
1780 parser->rss_conf.rss_key,
1781 parser->rss_conf.rss_key_len,
1785 if (flow->frxq[i].hrxq)
1787 flow->frxq[i].hrxq =
1788 mlx5_priv_hrxq_new(priv,
1789 parser->rss_conf.rss_key,
1790 parser->rss_conf.rss_key_len,
1794 if (!flow->frxq[i].hrxq) {
1795 rte_flow_error_set(error, ENOMEM,
1796 RTE_FLOW_ERROR_TYPE_HANDLE,
1797 NULL, "cannot create hash rxq");
1805 * Complete flow rule creation.
1808 * Pointer to private structure.
1810 * Internal parser structure.
1812 * Pointer to the rte_flow.
1814 * Perform verbose error reporting if not NULL.
1817 * 0 on success, a errno value otherwise and rte_errno is set.
1820 priv_flow_create_action_queue(struct priv *priv,
1821 struct mlx5_flow_parse *parser,
1822 struct rte_flow *flow,
1823 struct rte_flow_error *error)
1830 assert(!parser->drop);
1831 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1835 flow->cs = parser->cs;
1836 if (!priv->dev->data->dev_started)
1838 for (i = 0; i != hash_rxq_init_n; ++i) {
1839 if (!flow->frxq[i].hrxq)
1841 flow->frxq[i].ibv_flow =
1842 ibv_create_flow(flow->frxq[i].hrxq->qp,
1843 flow->frxq[i].ibv_attr);
1844 if (!flow->frxq[i].ibv_flow) {
1845 rte_flow_error_set(error, ENOMEM,
1846 RTE_FLOW_ERROR_TYPE_HANDLE,
1847 NULL, "flow rule creation failure");
1851 DEBUG("%p type %d QP %p ibv_flow %p",
1853 (void *)flow->frxq[i].hrxq,
1854 (void *)flow->frxq[i].ibv_flow);
1856 for (i = 0; i != parser->queues_n; ++i) {
1857 struct mlx5_rxq_data *q =
1858 (*priv->rxqs)[parser->queues[i]];
1860 q->mark |= parser->mark;
1865 for (i = 0; i != hash_rxq_init_n; ++i) {
1866 if (flow->frxq[i].ibv_flow) {
1867 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1869 claim_zero(ibv_destroy_flow(ibv_flow));
1871 if (flow->frxq[i].hrxq)
1872 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1873 if (flow->frxq[i].ibv_attr)
1874 rte_free(flow->frxq[i].ibv_attr);
1877 claim_zero(ibv_destroy_counter_set(flow->cs));
1888 * Pointer to private structure.
1890 * Pointer to a TAILQ flow list.
1892 * Flow rule attributes.
1893 * @param[in] pattern
1894 * Pattern specification (list terminated by the END pattern item).
1895 * @param[in] actions
1896 * Associated actions (list terminated by the END action).
1898 * Perform verbose error reporting if not NULL.
1901 * A flow on success, NULL otherwise.
1903 static struct rte_flow *
1904 priv_flow_create(struct priv *priv,
1905 struct mlx5_flows *list,
1906 const struct rte_flow_attr *attr,
1907 const struct rte_flow_item items[],
1908 const struct rte_flow_action actions[],
1909 struct rte_flow_error *error)
1911 struct mlx5_flow_parse parser = { .create = 1, };
1912 struct rte_flow *flow = NULL;
1916 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1919 flow = rte_calloc(__func__, 1,
1920 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1923 rte_flow_error_set(error, ENOMEM,
1924 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1926 "cannot allocate flow memory");
1929 /* Copy queues configuration. */
1930 flow->queues = (uint16_t (*)[])(flow + 1);
1931 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1932 flow->queues_n = parser.queues_n;
1933 flow->mark = parser.mark;
1934 /* Copy RSS configuration. */
1935 flow->rss_conf = parser.rss_conf;
1936 flow->rss_conf.rss_key = flow->rss_key;
1937 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1938 /* finalise the flow. */
1940 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1943 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1946 TAILQ_INSERT_TAIL(list, flow, next);
1947 DEBUG("Flow created %p", (void *)flow);
1951 rte_free(parser.drop_q.ibv_attr);
1953 for (i = 0; i != hash_rxq_init_n; ++i) {
1954 if (parser.queue[i].ibv_attr)
1955 rte_free(parser.queue[i].ibv_attr);
1963 * Validate a flow supported by the NIC.
1965 * @see rte_flow_validate()
1969 mlx5_flow_validate(struct rte_eth_dev *dev,
1970 const struct rte_flow_attr *attr,
1971 const struct rte_flow_item items[],
1972 const struct rte_flow_action actions[],
1973 struct rte_flow_error *error)
1975 struct priv *priv = dev->data->dev_private;
1977 struct mlx5_flow_parse parser = { .create = 0, };
1980 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1988 * @see rte_flow_create()
1992 mlx5_flow_create(struct rte_eth_dev *dev,
1993 const struct rte_flow_attr *attr,
1994 const struct rte_flow_item items[],
1995 const struct rte_flow_action actions[],
1996 struct rte_flow_error *error)
1998 struct priv *priv = dev->data->dev_private;
1999 struct rte_flow *flow;
2002 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
2012 * Pointer to private structure.
2014 * Pointer to a TAILQ flow list.
2019 priv_flow_destroy(struct priv *priv,
2020 struct mlx5_flows *list,
2021 struct rte_flow *flow)
2025 if (flow->drop || !flow->mark)
2027 for (i = 0; i != flow->queues_n; ++i) {
2028 struct rte_flow *tmp;
2032 * To remove the mark from the queue, the queue must not be
2033 * present in any other marked flow (RSS or not).
2035 TAILQ_FOREACH(tmp, list, next) {
2037 uint16_t *tqs = NULL;
2042 for (j = 0; j != hash_rxq_init_n; ++j) {
2043 if (!tmp->frxq[j].hrxq)
2045 tqs = tmp->frxq[j].hrxq->ind_table->queues;
2046 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
2050 for (j = 0; (j != tq_n) && !mark; j++)
2051 if (tqs[j] == (*flow->queues)[i])
2054 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
2058 if (flow->drxq.ibv_flow)
2059 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2060 rte_free(flow->drxq.ibv_attr);
2062 for (i = 0; i != hash_rxq_init_n; ++i) {
2063 struct mlx5_flow *frxq = &flow->frxq[i];
2066 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
2068 mlx5_priv_hrxq_release(priv, frxq->hrxq);
2070 rte_free(frxq->ibv_attr);
2074 claim_zero(ibv_destroy_counter_set(flow->cs));
2077 TAILQ_REMOVE(list, flow, next);
2078 DEBUG("Flow destroyed %p", (void *)flow);
2083 * Destroy all flows.
2086 * Pointer to private structure.
2088 * Pointer to a TAILQ flow list.
2091 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
2093 while (!TAILQ_EMPTY(list)) {
2094 struct rte_flow *flow;
2096 flow = TAILQ_FIRST(list);
2097 priv_flow_destroy(priv, list, flow);
2102 * Create drop queue.
2105 * Pointer to private structure.
2111 priv_flow_create_drop_queue(struct priv *priv)
2113 struct mlx5_hrxq_drop *fdq = NULL;
2117 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
2119 WARN("cannot allocate memory for drop queue");
2122 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
2124 WARN("cannot allocate CQ for drop queue");
2127 fdq->wq = ibv_create_wq(priv->ctx,
2128 &(struct ibv_wq_init_attr){
2129 .wq_type = IBV_WQT_RQ,
2136 WARN("cannot allocate WQ for drop queue");
2139 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
2140 &(struct ibv_rwq_ind_table_init_attr){
2141 .log_ind_tbl_size = 0,
2142 .ind_tbl = &fdq->wq,
2145 if (!fdq->ind_table) {
2146 WARN("cannot allocate indirection table for drop queue");
2149 fdq->qp = ibv_create_qp_ex(priv->ctx,
2150 &(struct ibv_qp_init_attr_ex){
2151 .qp_type = IBV_QPT_RAW_PACKET,
2153 IBV_QP_INIT_ATTR_PD |
2154 IBV_QP_INIT_ATTR_IND_TABLE |
2155 IBV_QP_INIT_ATTR_RX_HASH,
2156 .rx_hash_conf = (struct ibv_rx_hash_conf){
2158 IBV_RX_HASH_FUNC_TOEPLITZ,
2159 .rx_hash_key_len = rss_hash_default_key_len,
2160 .rx_hash_key = rss_hash_default_key,
2161 .rx_hash_fields_mask = 0,
2163 .rwq_ind_tbl = fdq->ind_table,
2167 WARN("cannot allocate QP for drop queue");
2170 priv->flow_drop_queue = fdq;
2174 claim_zero(ibv_destroy_qp(fdq->qp));
2176 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2178 claim_zero(ibv_destroy_wq(fdq->wq));
2180 claim_zero(ibv_destroy_cq(fdq->cq));
2183 priv->flow_drop_queue = NULL;
2188 * Delete drop queue.
2191 * Pointer to private structure.
2194 priv_flow_delete_drop_queue(struct priv *priv)
2196 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2201 claim_zero(ibv_destroy_qp(fdq->qp));
2203 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2205 claim_zero(ibv_destroy_wq(fdq->wq));
2207 claim_zero(ibv_destroy_cq(fdq->cq));
2209 priv->flow_drop_queue = NULL;
2216 * Pointer to private structure.
2218 * Pointer to a TAILQ flow list.
2221 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2223 struct rte_flow *flow;
2225 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2229 if (!flow->drxq.ibv_flow)
2231 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2232 flow->drxq.ibv_flow = NULL;
2237 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2239 for (i = 0; i != hash_rxq_init_n; ++i) {
2240 if (!flow->frxq[i].hrxq)
2242 ind_tbl = flow->frxq[i].hrxq->ind_table;
2245 for (i = 0; i != ind_tbl->queues_n; ++i)
2246 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2248 for (i = 0; i != hash_rxq_init_n; ++i) {
2249 if (!flow->frxq[i].ibv_flow)
2251 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2252 flow->frxq[i].ibv_flow = NULL;
2253 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2254 flow->frxq[i].hrxq = NULL;
2256 DEBUG("Flow %p removed", (void *)flow);
2264 * Pointer to private structure.
2266 * Pointer to a TAILQ flow list.
2269 * 0 on success, a errno value otherwise and rte_errno is set.
2272 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2274 struct rte_flow *flow;
2276 TAILQ_FOREACH(flow, list, next) {
2280 flow->drxq.ibv_flow =
2281 ibv_create_flow(priv->flow_drop_queue->qp,
2282 flow->drxq.ibv_attr);
2283 if (!flow->drxq.ibv_flow) {
2284 DEBUG("Flow %p cannot be applied",
2289 DEBUG("Flow %p applied", (void *)flow);
2293 for (i = 0; i != hash_rxq_init_n; ++i) {
2294 if (!flow->frxq[i].ibv_attr)
2296 flow->frxq[i].hrxq =
2297 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2298 flow->rss_conf.rss_key_len,
2299 hash_rxq_init[i].hash_fields,
2302 if (flow->frxq[i].hrxq)
2304 flow->frxq[i].hrxq =
2305 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2306 flow->rss_conf.rss_key_len,
2307 hash_rxq_init[i].hash_fields,
2310 if (!flow->frxq[i].hrxq) {
2311 DEBUG("Flow %p cannot be applied",
2317 flow->frxq[i].ibv_flow =
2318 ibv_create_flow(flow->frxq[i].hrxq->qp,
2319 flow->frxq[i].ibv_attr);
2320 if (!flow->frxq[i].ibv_flow) {
2321 DEBUG("Flow %p cannot be applied",
2326 DEBUG("Flow %p applied", (void *)flow);
2330 for (i = 0; i != flow->queues_n; ++i)
2331 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2337 * Verify the flow list is empty
2340 * Pointer to private structure.
2342 * @return the number of flows not released.
2345 priv_flow_verify(struct priv *priv)
2347 struct rte_flow *flow;
2350 TAILQ_FOREACH(flow, &priv->flows, next) {
2351 DEBUG("%p: flow %p still referenced", (void *)priv,
2359 * Enable a control flow configured from the control plane.
2362 * Pointer to Ethernet device.
2364 * An Ethernet flow spec to apply.
2366 * An Ethernet flow mask to apply.
2368 * A VLAN flow spec to apply.
2370 * A VLAN flow mask to apply.
2376 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2377 struct rte_flow_item_eth *eth_spec,
2378 struct rte_flow_item_eth *eth_mask,
2379 struct rte_flow_item_vlan *vlan_spec,
2380 struct rte_flow_item_vlan *vlan_mask)
2382 struct priv *priv = dev->data->dev_private;
2383 const struct rte_flow_attr attr = {
2385 .priority = MLX5_CTRL_FLOW_PRIORITY,
2387 struct rte_flow_item items[] = {
2389 .type = RTE_FLOW_ITEM_TYPE_ETH,
2395 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2396 RTE_FLOW_ITEM_TYPE_END,
2402 .type = RTE_FLOW_ITEM_TYPE_END,
2405 struct rte_flow_action actions[] = {
2407 .type = RTE_FLOW_ACTION_TYPE_RSS,
2410 .type = RTE_FLOW_ACTION_TYPE_END,
2413 struct rte_flow *flow;
2414 struct rte_flow_error error;
2417 struct rte_flow_action_rss rss;
2419 const struct rte_eth_rss_conf *rss_conf;
2421 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2425 if (!priv->reta_idx_n)
2427 for (i = 0; i != priv->reta_idx_n; ++i)
2428 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2429 action_rss.local.rss_conf = &priv->rss_conf;
2430 action_rss.local.num = priv->reta_idx_n;
2431 actions[0].conf = (const void *)&action_rss.rss;
2432 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2440 * Enable a flow control configured from the control plane.
2443 * Pointer to Ethernet device.
2445 * An Ethernet flow spec to apply.
2447 * An Ethernet flow mask to apply.
2453 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2454 struct rte_flow_item_eth *eth_spec,
2455 struct rte_flow_item_eth *eth_mask)
2457 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2463 * @see rte_flow_destroy()
2467 mlx5_flow_destroy(struct rte_eth_dev *dev,
2468 struct rte_flow *flow,
2469 struct rte_flow_error *error)
2471 struct priv *priv = dev->data->dev_private;
2475 priv_flow_destroy(priv, &priv->flows, flow);
2481 * Destroy all flows.
2483 * @see rte_flow_flush()
2487 mlx5_flow_flush(struct rte_eth_dev *dev,
2488 struct rte_flow_error *error)
2490 struct priv *priv = dev->data->dev_private;
2494 priv_flow_flush(priv, &priv->flows);
2499 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
2501 * Query flow counter.
2505 * @param counter_value
2506 * returned data from the counter.
2509 * 0 on success, a errno value otherwise and rte_errno is set.
2512 priv_flow_query_count(struct ibv_counter_set *cs,
2513 struct mlx5_flow_counter_stats *counter_stats,
2514 struct rte_flow_query_count *query_count,
2515 struct rte_flow_error *error)
2517 uint64_t counters[2];
2518 struct ibv_query_counter_set_attr query_cs_attr = {
2520 .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
2522 struct ibv_counter_set_data query_out = {
2524 .outlen = 2 * sizeof(uint64_t),
2526 int res = ibv_query_counter_set(&query_cs_attr, &query_out);
2529 rte_flow_error_set(error, -res,
2530 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2532 "cannot read counter");
2535 query_count->hits_set = 1;
2536 query_count->bytes_set = 1;
2537 query_count->hits = counters[0] - counter_stats->hits;
2538 query_count->bytes = counters[1] - counter_stats->bytes;
2539 if (query_count->reset) {
2540 counter_stats->hits = counters[0];
2541 counter_stats->bytes = counters[1];
2549 * @see rte_flow_query()
2553 mlx5_flow_query(struct rte_eth_dev *dev,
2554 struct rte_flow *flow,
2555 enum rte_flow_action_type action __rte_unused,
2557 struct rte_flow_error *error)
2559 struct priv *priv = dev->data->dev_private;
2564 res = priv_flow_query_count(flow->cs,
2565 &flow->counter_stats,
2566 (struct rte_flow_query_count *)data,
2569 rte_flow_error_set(error, res,
2570 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2572 "no counter found for flow");
2582 * @see rte_flow_isolate()
2586 mlx5_flow_isolate(struct rte_eth_dev *dev,
2588 struct rte_flow_error *error)
2590 struct priv *priv = dev->data->dev_private;
2593 if (dev->data->dev_started) {
2594 rte_flow_error_set(error, EBUSY,
2595 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2597 "port must be stopped first");
2601 priv->isolated = !!enable;
2603 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2605 priv->dev->dev_ops = &mlx5_dev_ops;
2611 * Convert a flow director filter to a generic flow.
2614 * Private structure.
2615 * @param fdir_filter
2616 * Flow director filter to add.
2618 * Generic flow parameters structure.
2621 * 0 on success, errno value on error.
2624 priv_fdir_filter_convert(struct priv *priv,
2625 const struct rte_eth_fdir_filter *fdir_filter,
2626 struct mlx5_fdir *attributes)
2628 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2630 /* Validate queue number. */
2631 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2632 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2635 attributes->attr.ingress = 1;
2636 attributes->items[0] = (struct rte_flow_item) {
2637 .type = RTE_FLOW_ITEM_TYPE_ETH,
2638 .spec = &attributes->l2,
2639 .mask = &attributes->l2_mask,
2641 switch (fdir_filter->action.behavior) {
2642 case RTE_ETH_FDIR_ACCEPT:
2643 attributes->actions[0] = (struct rte_flow_action){
2644 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2645 .conf = &attributes->queue,
2648 case RTE_ETH_FDIR_REJECT:
2649 attributes->actions[0] = (struct rte_flow_action){
2650 .type = RTE_FLOW_ACTION_TYPE_DROP,
2654 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2657 attributes->queue.index = fdir_filter->action.rx_queue;
2658 switch (fdir_filter->input.flow_type) {
2659 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2660 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2661 .src_addr = input->flow.udp4_flow.ip.src_ip,
2662 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2663 .time_to_live = input->flow.udp4_flow.ip.ttl,
2664 .type_of_service = input->flow.udp4_flow.ip.tos,
2665 .next_proto_id = input->flow.udp4_flow.ip.proto,
2667 attributes->l4.udp.hdr = (struct udp_hdr){
2668 .src_port = input->flow.udp4_flow.src_port,
2669 .dst_port = input->flow.udp4_flow.dst_port,
2671 attributes->items[1] = (struct rte_flow_item){
2672 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2673 .spec = &attributes->l3,
2675 attributes->items[2] = (struct rte_flow_item){
2676 .type = RTE_FLOW_ITEM_TYPE_UDP,
2677 .spec = &attributes->l4,
2680 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2681 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2682 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2683 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2684 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2685 .type_of_service = input->flow.tcp4_flow.ip.tos,
2686 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2688 attributes->l4.tcp.hdr = (struct tcp_hdr){
2689 .src_port = input->flow.tcp4_flow.src_port,
2690 .dst_port = input->flow.tcp4_flow.dst_port,
2692 attributes->items[1] = (struct rte_flow_item){
2693 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2694 .spec = &attributes->l3,
2696 attributes->items[2] = (struct rte_flow_item){
2697 .type = RTE_FLOW_ITEM_TYPE_TCP,
2698 .spec = &attributes->l4,
2701 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2702 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2703 .src_addr = input->flow.ip4_flow.src_ip,
2704 .dst_addr = input->flow.ip4_flow.dst_ip,
2705 .time_to_live = input->flow.ip4_flow.ttl,
2706 .type_of_service = input->flow.ip4_flow.tos,
2707 .next_proto_id = input->flow.ip4_flow.proto,
2709 attributes->items[1] = (struct rte_flow_item){
2710 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2711 .spec = &attributes->l3,
2714 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2715 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2716 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2717 .proto = input->flow.udp6_flow.ip.proto,
2719 memcpy(attributes->l3.ipv6.hdr.src_addr,
2720 input->flow.udp6_flow.ip.src_ip,
2721 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2722 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2723 input->flow.udp6_flow.ip.dst_ip,
2724 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2725 attributes->l4.udp.hdr = (struct udp_hdr){
2726 .src_port = input->flow.udp6_flow.src_port,
2727 .dst_port = input->flow.udp6_flow.dst_port,
2729 attributes->items[1] = (struct rte_flow_item){
2730 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2731 .spec = &attributes->l3,
2733 attributes->items[2] = (struct rte_flow_item){
2734 .type = RTE_FLOW_ITEM_TYPE_UDP,
2735 .spec = &attributes->l4,
2738 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2739 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2740 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2741 .proto = input->flow.tcp6_flow.ip.proto,
2743 memcpy(attributes->l3.ipv6.hdr.src_addr,
2744 input->flow.tcp6_flow.ip.src_ip,
2745 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2746 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2747 input->flow.tcp6_flow.ip.dst_ip,
2748 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2749 attributes->l4.tcp.hdr = (struct tcp_hdr){
2750 .src_port = input->flow.tcp6_flow.src_port,
2751 .dst_port = input->flow.tcp6_flow.dst_port,
2753 attributes->items[1] = (struct rte_flow_item){
2754 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2755 .spec = &attributes->l3,
2757 attributes->items[2] = (struct rte_flow_item){
2758 .type = RTE_FLOW_ITEM_TYPE_TCP,
2759 .spec = &attributes->l4,
2762 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2763 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2764 .hop_limits = input->flow.ipv6_flow.hop_limits,
2765 .proto = input->flow.ipv6_flow.proto,
2767 memcpy(attributes->l3.ipv6.hdr.src_addr,
2768 input->flow.ipv6_flow.src_ip,
2769 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2770 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2771 input->flow.ipv6_flow.dst_ip,
2772 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2773 attributes->items[1] = (struct rte_flow_item){
2774 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2775 .spec = &attributes->l3,
2779 ERROR("invalid flow type%d",
2780 fdir_filter->input.flow_type);
2787 * Add new flow director filter and store it in list.
2790 * Private structure.
2791 * @param fdir_filter
2792 * Flow director filter to add.
2795 * 0 on success, errno value on failure.
2798 priv_fdir_filter_add(struct priv *priv,
2799 const struct rte_eth_fdir_filter *fdir_filter)
2801 struct mlx5_fdir attributes = {
2804 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2805 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
2809 struct mlx5_flow_parse parser = {
2810 .layer = HASH_RXQ_ETH,
2812 struct rte_flow_error error;
2813 struct rte_flow *flow;
2816 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2819 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2820 attributes.actions, &error, &parser);
2823 flow = priv_flow_create(priv,
2830 DEBUG("FDIR created %p", (void *)flow);
2837 * Delete specific filter.
2840 * Private structure.
2841 * @param fdir_filter
2842 * Filter to be deleted.
2845 * 0 on success, errno value on failure.
2848 priv_fdir_filter_delete(struct priv *priv,
2849 const struct rte_eth_fdir_filter *fdir_filter)
2851 struct mlx5_fdir attributes = {
2854 struct mlx5_flow_parse parser = {
2856 .layer = HASH_RXQ_ETH,
2858 struct rte_flow_error error;
2859 struct rte_flow *flow;
2863 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2866 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2867 attributes.actions, &error, &parser);
2871 * Special case for drop action which is only set in the
2872 * specifications when the flow is created. In this situation the
2873 * drop specification is missing.
2876 struct ibv_flow_spec_action_drop *drop;
2878 drop = (void *)((uintptr_t)parser.drop_q.ibv_attr +
2879 parser.drop_q.offset);
2880 *drop = (struct ibv_flow_spec_action_drop){
2881 .type = IBV_FLOW_SPEC_ACTION_DROP,
2882 .size = sizeof(struct ibv_flow_spec_action_drop),
2884 parser.drop_q.ibv_attr->num_of_specs++;
2886 TAILQ_FOREACH(flow, &priv->flows, next) {
2887 struct ibv_flow_attr *attr;
2888 struct ibv_spec_header *attr_h;
2890 struct ibv_flow_attr *flow_attr;
2891 struct ibv_spec_header *flow_h;
2893 unsigned int specs_n;
2896 attr = parser.drop_q.ibv_attr;
2898 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2900 flow_attr = flow->drxq.ibv_attr;
2902 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2903 /* Compare first the attributes. */
2904 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2906 if (attr->num_of_specs == 0)
2908 spec = (void *)((uintptr_t)attr +
2909 sizeof(struct ibv_flow_attr));
2910 flow_spec = (void *)((uintptr_t)flow_attr +
2911 sizeof(struct ibv_flow_attr));
2912 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2913 for (i = 0; i != specs_n; ++i) {
2916 if (memcmp(spec, flow_spec,
2917 RTE_MIN(attr_h->size, flow_h->size)))
2919 spec = (void *)((uintptr_t)spec + attr_h->size);
2920 flow_spec = (void *)((uintptr_t)flow_spec +
2923 /* At this point, the flow match. */
2926 /* The flow does not match. */
2930 priv_flow_destroy(priv, &priv->flows, flow);
2933 rte_free(parser.drop_q.ibv_attr);
2935 for (i = 0; i != hash_rxq_init_n; ++i) {
2936 if (parser.queue[i].ibv_attr)
2937 rte_free(parser.queue[i].ibv_attr);
2944 * Update queue for specific filter.
2947 * Private structure.
2948 * @param fdir_filter
2949 * Filter to be updated.
2952 * 0 on success, errno value on failure.
2955 priv_fdir_filter_update(struct priv *priv,
2956 const struct rte_eth_fdir_filter *fdir_filter)
2960 ret = priv_fdir_filter_delete(priv, fdir_filter);
2963 ret = priv_fdir_filter_add(priv, fdir_filter);
2968 * Flush all filters.
2971 * Private structure.
2974 priv_fdir_filter_flush(struct priv *priv)
2976 priv_flow_flush(priv, &priv->flows);
2980 * Get flow director information.
2983 * Private structure.
2984 * @param[out] fdir_info
2985 * Resulting flow director information.
2988 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2990 struct rte_eth_fdir_masks *mask =
2991 &priv->dev->data->dev_conf.fdir_conf.mask;
2993 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2994 fdir_info->guarant_spc = 0;
2995 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2996 fdir_info->max_flexpayload = 0;
2997 fdir_info->flow_types_mask[0] = 0;
2998 fdir_info->flex_payload_unit = 0;
2999 fdir_info->max_flex_payload_segment_num = 0;
3000 fdir_info->flex_payload_limit = 0;
3001 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
3005 * Deal with flow director operations.
3008 * Pointer to private structure.
3010 * Operation to perform.
3012 * Pointer to operation-specific structure.
3015 * 0 on success, errno value on failure.
3018 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
3020 enum rte_fdir_mode fdir_mode =
3021 priv->dev->data->dev_conf.fdir_conf.mode;
3024 if (filter_op == RTE_ETH_FILTER_NOP)
3026 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
3027 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
3028 ERROR("%p: flow director mode %d not supported",
3029 (void *)priv, fdir_mode);
3032 switch (filter_op) {
3033 case RTE_ETH_FILTER_ADD:
3034 ret = priv_fdir_filter_add(priv, arg);
3036 case RTE_ETH_FILTER_UPDATE:
3037 ret = priv_fdir_filter_update(priv, arg);
3039 case RTE_ETH_FILTER_DELETE:
3040 ret = priv_fdir_filter_delete(priv, arg);
3042 case RTE_ETH_FILTER_FLUSH:
3043 priv_fdir_filter_flush(priv);
3045 case RTE_ETH_FILTER_INFO:
3046 priv_fdir_info_get(priv, arg);
3049 DEBUG("%p: unknown operation %u", (void *)priv,
3058 * Manage filter operations.
3061 * Pointer to Ethernet device structure.
3062 * @param filter_type
3065 * Operation to perform.
3067 * Pointer to operation-specific structure.
3070 * 0 on success, negative errno value on failure.
3073 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
3074 enum rte_filter_type filter_type,
3075 enum rte_filter_op filter_op,
3079 struct priv *priv = dev->data->dev_private;
3081 switch (filter_type) {
3082 case RTE_ETH_FILTER_GENERIC:
3083 if (filter_op != RTE_ETH_FILTER_GET)
3085 *(const void **)arg = &mlx5_flow_ops;
3087 case RTE_ETH_FILTER_FDIR:
3089 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
3093 ERROR("%p: filter type (%d) not supported",
3094 (void *)dev, filter_type);