4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
62 /* Dev ops structure defined in mlx5.c */
63 extern const struct eth_dev_ops mlx5_dev_ops;
64 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
67 mlx5_flow_create_eth(const struct rte_flow_item *item,
68 const void *default_mask,
72 mlx5_flow_create_vlan(const struct rte_flow_item *item,
73 const void *default_mask,
77 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
78 const void *default_mask,
82 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
83 const void *default_mask,
87 mlx5_flow_create_udp(const struct rte_flow_item *item,
88 const void *default_mask,
92 mlx5_flow_create_tcp(const struct rte_flow_item *item,
93 const void *default_mask,
97 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
98 const void *default_mask,
101 struct mlx5_flow_parse;
104 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
108 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
110 /* Hash RX queue types. */
121 /* Initialization data for hash RX queue. */
122 struct hash_rxq_init {
123 uint64_t hash_fields; /* Fields that participate in the hash. */
124 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
125 unsigned int flow_priority; /* Flow priority to use. */
126 unsigned int ip_version; /* Internet protocol. */
129 /* Initialization data for hash RX queues. */
130 const struct hash_rxq_init hash_rxq_init[] = {
132 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
133 IBV_RX_HASH_DST_IPV4 |
134 IBV_RX_HASH_SRC_PORT_TCP |
135 IBV_RX_HASH_DST_PORT_TCP),
136 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
138 .ip_version = MLX5_IPV4,
141 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
142 IBV_RX_HASH_DST_IPV4 |
143 IBV_RX_HASH_SRC_PORT_UDP |
144 IBV_RX_HASH_DST_PORT_UDP),
145 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
147 .ip_version = MLX5_IPV4,
150 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
151 IBV_RX_HASH_DST_IPV4),
152 .dpdk_rss_hf = (ETH_RSS_IPV4 |
155 .ip_version = MLX5_IPV4,
158 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
159 IBV_RX_HASH_DST_IPV6 |
160 IBV_RX_HASH_SRC_PORT_TCP |
161 IBV_RX_HASH_DST_PORT_TCP),
162 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
164 .ip_version = MLX5_IPV6,
167 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
168 IBV_RX_HASH_DST_IPV6 |
169 IBV_RX_HASH_SRC_PORT_UDP |
170 IBV_RX_HASH_DST_PORT_UDP),
171 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
173 .ip_version = MLX5_IPV6,
176 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
177 IBV_RX_HASH_DST_IPV6),
178 .dpdk_rss_hf = (ETH_RSS_IPV6 |
181 .ip_version = MLX5_IPV6,
190 /* Number of entries in hash_rxq_init[]. */
191 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
193 /** Structure for Drop queue. */
194 struct mlx5_hrxq_drop {
195 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
196 struct ibv_qp *qp; /**< Verbs queue pair. */
197 struct ibv_wq *wq; /**< Verbs work queue. */
198 struct ibv_cq *cq; /**< Verbs completion queue. */
201 /* Flows structures. */
203 uint64_t hash_fields; /**< Fields that participate in the hash. */
204 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
205 struct ibv_flow *ibv_flow; /**< Verbs flow. */
206 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
209 /* Drop flows structures. */
210 struct mlx5_flow_drop {
211 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
212 struct ibv_flow *ibv_flow; /**< Verbs flow. */
216 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
217 uint32_t mark:1; /**< Set if the flow is marked. */
218 uint32_t drop:1; /**< Drop queue. */
219 uint16_t queues_n; /**< Number of entries in queue[]. */
220 uint16_t (*queues)[]; /**< Queues indexes to use. */
221 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
222 uint8_t rss_key[40]; /**< copy of the RSS key. */
224 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
225 /**< Flow with Rx queue. */
226 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */
230 /** Static initializer for items. */
232 (const enum rte_flow_item_type []){ \
233 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
236 /** Structure to generate a simple graph of layers supported by the NIC. */
237 struct mlx5_flow_items {
238 /** List of possible actions for these items. */
239 const enum rte_flow_action_type *const actions;
240 /** Bit-masks corresponding to the possibilities for the item. */
243 * Default bit-masks to use when item->mask is not provided. When
244 * \default_mask is also NULL, the full supported bit-mask (\mask) is
247 const void *default_mask;
248 /** Bit-masks size in bytes. */
249 const unsigned int mask_sz;
251 * Conversion function from rte_flow to NIC specific flow.
254 * rte_flow item to convert.
255 * @param default_mask
256 * Default bit-masks to use when item->mask is not provided.
258 * Internal structure to store the conversion.
261 * 0 on success, negative value otherwise.
263 int (*convert)(const struct rte_flow_item *item,
264 const void *default_mask,
266 /** Size in bytes of the destination structure. */
267 const unsigned int dst_sz;
268 /** List of possible following items. */
269 const enum rte_flow_item_type *const items;
272 /** Valid action for this PMD. */
273 static const enum rte_flow_action_type valid_actions[] = {
274 RTE_FLOW_ACTION_TYPE_DROP,
275 RTE_FLOW_ACTION_TYPE_QUEUE,
276 RTE_FLOW_ACTION_TYPE_MARK,
277 RTE_FLOW_ACTION_TYPE_FLAG,
278 RTE_FLOW_ACTION_TYPE_END,
281 /** Graph of supported items and associated actions. */
282 static const struct mlx5_flow_items mlx5_flow_items[] = {
283 [RTE_FLOW_ITEM_TYPE_END] = {
284 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
285 RTE_FLOW_ITEM_TYPE_VXLAN),
287 [RTE_FLOW_ITEM_TYPE_ETH] = {
288 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
289 RTE_FLOW_ITEM_TYPE_IPV4,
290 RTE_FLOW_ITEM_TYPE_IPV6),
291 .actions = valid_actions,
292 .mask = &(const struct rte_flow_item_eth){
293 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
294 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
297 .default_mask = &rte_flow_item_eth_mask,
298 .mask_sz = sizeof(struct rte_flow_item_eth),
299 .convert = mlx5_flow_create_eth,
300 .dst_sz = sizeof(struct ibv_flow_spec_eth),
302 [RTE_FLOW_ITEM_TYPE_VLAN] = {
303 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
304 RTE_FLOW_ITEM_TYPE_IPV6),
305 .actions = valid_actions,
306 .mask = &(const struct rte_flow_item_vlan){
309 .default_mask = &rte_flow_item_vlan_mask,
310 .mask_sz = sizeof(struct rte_flow_item_vlan),
311 .convert = mlx5_flow_create_vlan,
314 [RTE_FLOW_ITEM_TYPE_IPV4] = {
315 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
316 RTE_FLOW_ITEM_TYPE_TCP),
317 .actions = valid_actions,
318 .mask = &(const struct rte_flow_item_ipv4){
322 .type_of_service = -1,
326 .default_mask = &rte_flow_item_ipv4_mask,
327 .mask_sz = sizeof(struct rte_flow_item_ipv4),
328 .convert = mlx5_flow_create_ipv4,
329 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
331 [RTE_FLOW_ITEM_TYPE_IPV6] = {
332 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
333 RTE_FLOW_ITEM_TYPE_TCP),
334 .actions = valid_actions,
335 .mask = &(const struct rte_flow_item_ipv6){
338 0xff, 0xff, 0xff, 0xff,
339 0xff, 0xff, 0xff, 0xff,
340 0xff, 0xff, 0xff, 0xff,
341 0xff, 0xff, 0xff, 0xff,
344 0xff, 0xff, 0xff, 0xff,
345 0xff, 0xff, 0xff, 0xff,
346 0xff, 0xff, 0xff, 0xff,
347 0xff, 0xff, 0xff, 0xff,
354 .default_mask = &rte_flow_item_ipv6_mask,
355 .mask_sz = sizeof(struct rte_flow_item_ipv6),
356 .convert = mlx5_flow_create_ipv6,
357 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
359 [RTE_FLOW_ITEM_TYPE_UDP] = {
360 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
361 .actions = valid_actions,
362 .mask = &(const struct rte_flow_item_udp){
368 .default_mask = &rte_flow_item_udp_mask,
369 .mask_sz = sizeof(struct rte_flow_item_udp),
370 .convert = mlx5_flow_create_udp,
371 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
373 [RTE_FLOW_ITEM_TYPE_TCP] = {
374 .actions = valid_actions,
375 .mask = &(const struct rte_flow_item_tcp){
381 .default_mask = &rte_flow_item_tcp_mask,
382 .mask_sz = sizeof(struct rte_flow_item_tcp),
383 .convert = mlx5_flow_create_tcp,
384 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
386 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
387 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
388 .actions = valid_actions,
389 .mask = &(const struct rte_flow_item_vxlan){
390 .vni = "\xff\xff\xff",
392 .default_mask = &rte_flow_item_vxlan_mask,
393 .mask_sz = sizeof(struct rte_flow_item_vxlan),
394 .convert = mlx5_flow_create_vxlan,
395 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
399 /** Structure to pass to the conversion function. */
400 struct mlx5_flow_parse {
401 uint32_t inner; /**< Set once VXLAN is encountered. */
403 /**< Whether resources should remain after a validate. */
404 uint32_t drop:1; /**< Target is a drop queue. */
405 uint32_t mark:1; /**< Mark is present in the flow. */
406 uint32_t mark_id; /**< Mark identifier. */
407 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
408 uint16_t queues_n; /**< Number of entries in queue[]. */
409 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
410 uint8_t rss_key[40]; /**< copy of the RSS key. */
411 enum hash_rxq_type layer; /**< Last pattern layer detected. */
414 struct ibv_flow_attr *ibv_attr;
415 /**< Pointer to Verbs attributes. */
417 /**< Current position or total size of the attribute. */
418 } queue[RTE_DIM(hash_rxq_init)];
420 struct ibv_flow_attr *ibv_attr;
421 /**< Pointer to Verbs attributes. */
423 /**< Current position or total size of the attribute. */
428 static const struct rte_flow_ops mlx5_flow_ops = {
429 .validate = mlx5_flow_validate,
430 .create = mlx5_flow_create,
431 .destroy = mlx5_flow_destroy,
432 .flush = mlx5_flow_flush,
434 .isolate = mlx5_flow_isolate,
437 /* Convert FDIR request to Generic flow. */
439 struct rte_flow_attr attr;
440 struct rte_flow_action actions[2];
441 struct rte_flow_item items[4];
442 struct rte_flow_item_eth l2;
444 struct rte_flow_item_ipv4 ipv4;
445 struct rte_flow_item_ipv6 ipv6;
448 struct rte_flow_item_udp udp;
449 struct rte_flow_item_tcp tcp;
451 struct rte_flow_action_queue queue;
454 /* Verbs specification header. */
455 struct ibv_spec_header {
456 enum ibv_flow_spec_type type;
461 * Check support for a given item.
464 * Item specification.
466 * Bit-masks covering supported fields to compare with spec, last and mask in
469 * Bit-Mask size in bytes.
475 mlx5_flow_item_validate(const struct rte_flow_item *item,
476 const uint8_t *mask, unsigned int size)
480 if (!item->spec && (item->mask || item->last))
482 if (item->spec && !item->mask) {
484 const uint8_t *spec = item->spec;
486 for (i = 0; i < size; ++i)
487 if ((spec[i] | mask[i]) != mask[i])
490 if (item->last && !item->mask) {
492 const uint8_t *spec = item->last;
494 for (i = 0; i < size; ++i)
495 if ((spec[i] | mask[i]) != mask[i])
500 const uint8_t *spec = item->mask;
502 for (i = 0; i < size; ++i)
503 if ((spec[i] | mask[i]) != mask[i])
506 if (item->spec && item->last) {
509 const uint8_t *apply = mask;
514 for (i = 0; i < size; ++i) {
515 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
516 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
518 ret = memcmp(spec, last, size);
524 * Copy the RSS configuration from the user ones.
527 * Pointer to private structure.
529 * Internal parser structure.
531 * User RSS configuration to save.
534 * 0 on success, errno value on failure.
537 priv_flow_convert_rss_conf(struct priv *priv,
538 struct mlx5_flow_parse *parser,
539 const struct rte_eth_rss_conf *rss_conf)
541 const struct rte_eth_rss_conf *rss =
542 rss_conf ? rss_conf : &priv->rss_conf;
544 if (rss->rss_key_len > 40)
546 parser->rss_conf.rss_key_len = rss->rss_key_len;
547 parser->rss_conf.rss_hf = rss->rss_hf;
548 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
549 parser->rss_conf.rss_key = parser->rss_key;
554 * Extract attribute to the parser.
557 * Pointer to private structure.
559 * Flow rule attributes.
561 * Perform verbose error reporting if not NULL.
562 * @param[in, out] parser
563 * Internal parser structure.
566 * 0 on success, a negative errno value otherwise and rte_errno is set.
569 priv_flow_convert_attributes(struct priv *priv,
570 const struct rte_flow_attr *attr,
571 struct rte_flow_error *error,
572 struct mlx5_flow_parse *parser)
577 rte_flow_error_set(error, ENOTSUP,
578 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
580 "groups are not supported");
583 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
584 rte_flow_error_set(error, ENOTSUP,
585 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
587 "priorities are not supported");
591 rte_flow_error_set(error, ENOTSUP,
592 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
594 "egress is not supported");
597 if (!attr->ingress) {
598 rte_flow_error_set(error, ENOTSUP,
599 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
601 "only ingress is supported");
608 * Extract actions request to the parser.
611 * Pointer to private structure.
613 * Associated actions (list terminated by the END action).
615 * Perform verbose error reporting if not NULL.
616 * @param[in, out] parser
617 * Internal parser structure.
620 * 0 on success, a negative errno value otherwise and rte_errno is set.
623 priv_flow_convert_actions(struct priv *priv,
624 const struct rte_flow_action actions[],
625 struct rte_flow_error *error,
626 struct mlx5_flow_parse *parser)
629 * Add default RSS configuration necessary for Verbs to create QP even
630 * if no RSS is necessary.
632 priv_flow_convert_rss_conf(priv, parser,
633 (const struct rte_eth_rss_conf *)
635 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
636 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
638 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
640 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
641 const struct rte_flow_action_queue *queue =
642 (const struct rte_flow_action_queue *)
647 if (!queue || (queue->index > (priv->rxqs_n - 1)))
648 goto exit_action_not_supported;
649 for (n = 0; n < parser->queues_n; ++n) {
650 if (parser->queues[n] == queue->index) {
655 if (parser->queues_n > 1 && !found) {
656 rte_flow_error_set(error, ENOTSUP,
657 RTE_FLOW_ERROR_TYPE_ACTION,
659 "queue action not in RSS queues");
663 parser->queues_n = 1;
664 parser->queues[0] = queue->index;
666 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
667 const struct rte_flow_action_rss *rss =
668 (const struct rte_flow_action_rss *)
672 if (!rss || !rss->num) {
673 rte_flow_error_set(error, EINVAL,
674 RTE_FLOW_ERROR_TYPE_ACTION,
679 if (parser->queues_n == 1) {
682 assert(parser->queues_n);
683 for (n = 0; n < rss->num; ++n) {
684 if (parser->queues[0] ==
691 rte_flow_error_set(error, ENOTSUP,
692 RTE_FLOW_ERROR_TYPE_ACTION,
694 "queue action not in RSS"
699 for (n = 0; n < rss->num; ++n) {
700 if (rss->queue[n] >= priv->rxqs_n) {
701 rte_flow_error_set(error, EINVAL,
702 RTE_FLOW_ERROR_TYPE_ACTION,
704 "queue id > number of"
709 for (n = 0; n < rss->num; ++n)
710 parser->queues[n] = rss->queue[n];
711 parser->queues_n = rss->num;
712 if (priv_flow_convert_rss_conf(priv, parser,
714 rte_flow_error_set(error, EINVAL,
715 RTE_FLOW_ERROR_TYPE_ACTION,
717 "wrong RSS configuration");
720 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
721 const struct rte_flow_action_mark *mark =
722 (const struct rte_flow_action_mark *)
726 rte_flow_error_set(error, EINVAL,
727 RTE_FLOW_ERROR_TYPE_ACTION,
729 "mark must be defined");
731 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
732 rte_flow_error_set(error, ENOTSUP,
733 RTE_FLOW_ERROR_TYPE_ACTION,
735 "mark must be between 0"
740 parser->mark_id = mark->id;
741 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
744 goto exit_action_not_supported;
747 if (!parser->queues_n && !parser->drop) {
748 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
749 NULL, "no valid action");
753 exit_action_not_supported:
754 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
755 actions, "action not supported");
763 * Pointer to private structure.
765 * Pattern specification (list terminated by the END pattern item).
767 * Perform verbose error reporting if not NULL.
768 * @param[in, out] parser
769 * Internal parser structure.
772 * 0 on success, a negative errno value otherwise and rte_errno is set.
775 priv_flow_convert_items_validate(struct priv *priv,
776 const struct rte_flow_item items[],
777 struct rte_flow_error *error,
778 struct mlx5_flow_parse *parser)
780 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
784 /* Initialise the offsets to start after verbs attribute. */
786 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
788 for (i = 0; i != hash_rxq_init_n; ++i)
789 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
791 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
792 const struct mlx5_flow_items *token = NULL;
796 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
800 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
802 if (cur_item->items[i] == items->type) {
803 token = &mlx5_flow_items[items->type];
808 goto exit_item_not_supported;
810 err = mlx5_flow_item_validate(items,
811 (const uint8_t *)cur_item->mask,
814 goto exit_item_not_supported;
815 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
817 rte_flow_error_set(error, ENOTSUP,
818 RTE_FLOW_ERROR_TYPE_ITEM,
820 "cannot recognize multiple"
821 " VXLAN encapsulations");
827 parser->drop_q.offset += cur_item->dst_sz;
828 } else if (parser->queues_n == 1) {
829 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
831 for (n = 0; n != hash_rxq_init_n; ++n)
832 parser->queue[n].offset += cur_item->dst_sz;
836 for (i = 0; i != hash_rxq_init_n; ++i)
837 parser->queue[i].offset +=
838 sizeof(struct ibv_flow_spec_action_tag);
841 exit_item_not_supported:
842 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
843 items, "item not supported");
848 * Allocate memory space to store verbs flow attributes.
851 * Pointer to private structure.
852 * @param[in] priority
855 * Amount of byte to allocate.
857 * Perform verbose error reporting if not NULL.
860 * A verbs flow attribute on success, NULL otherwise.
862 static struct ibv_flow_attr*
863 priv_flow_convert_allocate(struct priv *priv,
864 unsigned int priority,
866 struct rte_flow_error *error)
868 struct ibv_flow_attr *ibv_attr;
871 ibv_attr = rte_calloc(__func__, 1, size, 0);
873 rte_flow_error_set(error, ENOMEM,
874 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
876 "cannot allocate verbs spec attributes.");
879 ibv_attr->priority = priority;
884 * Finalise verbs flow attributes.
887 * Pointer to private structure.
888 * @param[in, out] parser
889 * Internal parser structure.
892 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
894 const unsigned int ipv4 =
895 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
896 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
897 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
898 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
899 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
900 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
904 if (parser->layer == HASH_RXQ_ETH) {
908 * This layer becomes useless as the pattern define under
911 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
912 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
914 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
915 for (i = ohmin; i != (ohmax + 1); ++i) {
916 if (!parser->queue[i].ibv_attr)
918 rte_free(parser->queue[i].ibv_attr);
919 parser->queue[i].ibv_attr = NULL;
921 /* Remove impossible flow according to the RSS configuration. */
922 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
923 parser->rss_conf.rss_hf) {
924 /* Remove any other flow. */
925 for (i = hmin; i != (hmax + 1); ++i) {
926 if ((i == parser->layer) ||
927 (!parser->queue[i].ibv_attr))
929 rte_free(parser->queue[i].ibv_attr);
930 parser->queue[i].ibv_attr = NULL;
932 } else if (!parser->queue[ip].ibv_attr) {
933 /* no RSS possible with the current configuration. */
934 parser->queues_n = 1;
939 * Fill missing layers in verbs specifications, or compute the correct
940 * offset to allocate the memory space for the attributes and
943 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
945 struct ibv_flow_spec_ipv4_ext ipv4;
946 struct ibv_flow_spec_ipv6 ipv6;
947 struct ibv_flow_spec_tcp_udp udp_tcp;
952 if (i == parser->layer)
954 if (parser->layer == HASH_RXQ_ETH) {
955 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
956 size = sizeof(struct ibv_flow_spec_ipv4_ext);
957 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
958 .type = IBV_FLOW_SPEC_IPV4_EXT |
963 size = sizeof(struct ibv_flow_spec_ipv6);
964 specs.ipv6 = (struct ibv_flow_spec_ipv6){
965 .type = IBV_FLOW_SPEC_IPV6 |
970 if (parser->queue[i].ibv_attr) {
971 dst = (void *)((uintptr_t)
972 parser->queue[i].ibv_attr +
973 parser->queue[i].offset);
974 memcpy(dst, &specs, size);
975 ++parser->queue[i].ibv_attr->num_of_specs;
977 parser->queue[i].offset += size;
979 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
980 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
981 size = sizeof(struct ibv_flow_spec_tcp_udp);
982 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
983 .type = ((i == HASH_RXQ_UDPV4 ||
984 i == HASH_RXQ_UDPV6) ?
990 if (parser->queue[i].ibv_attr) {
991 dst = (void *)((uintptr_t)
992 parser->queue[i].ibv_attr +
993 parser->queue[i].offset);
994 memcpy(dst, &specs, size);
995 ++parser->queue[i].ibv_attr->num_of_specs;
997 parser->queue[i].offset += size;
1003 * Validate and convert a flow supported by the NIC.
1006 * Pointer to private structure.
1008 * Flow rule attributes.
1009 * @param[in] pattern
1010 * Pattern specification (list terminated by the END pattern item).
1011 * @param[in] actions
1012 * Associated actions (list terminated by the END action).
1014 * Perform verbose error reporting if not NULL.
1015 * @param[in, out] parser
1016 * Internal parser structure.
1019 * 0 on success, a negative errno value otherwise and rte_errno is set.
1022 priv_flow_convert(struct priv *priv,
1023 const struct rte_flow_attr *attr,
1024 const struct rte_flow_item items[],
1025 const struct rte_flow_action actions[],
1026 struct rte_flow_error *error,
1027 struct mlx5_flow_parse *parser)
1029 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1033 /* First step. Validate the attributes, items and actions. */
1034 *parser = (struct mlx5_flow_parse){
1035 .create = parser->create,
1036 .layer = HASH_RXQ_ETH,
1037 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1039 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1042 ret = priv_flow_convert_actions(priv, actions, error, parser);
1045 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1048 priv_flow_convert_finalise(priv, parser);
1051 * Allocate the memory space to store verbs specifications.
1054 parser->drop_q.ibv_attr =
1055 priv_flow_convert_allocate(priv, attr->priority,
1056 parser->drop_q.offset,
1058 if (!parser->drop_q.ibv_attr)
1060 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
1061 } else if (parser->queues_n == 1) {
1062 unsigned int priority =
1064 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1065 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1067 parser->queue[HASH_RXQ_ETH].ibv_attr =
1068 priv_flow_convert_allocate(priv, priority,
1070 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1072 parser->queue[HASH_RXQ_ETH].offset =
1073 sizeof(struct ibv_flow_attr);
1075 for (i = 0; i != hash_rxq_init_n; ++i) {
1076 unsigned int priority =
1078 hash_rxq_init[i].flow_priority;
1079 unsigned int offset;
1081 if (!(parser->rss_conf.rss_hf &
1082 hash_rxq_init[i].dpdk_rss_hf) &&
1083 (i != HASH_RXQ_ETH))
1085 offset = parser->queue[i].offset;
1086 parser->queue[i].ibv_attr =
1087 priv_flow_convert_allocate(priv, priority,
1089 if (!parser->queue[i].ibv_attr)
1091 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1094 /* Third step. Conversion parse, fill the specifications. */
1096 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1097 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1099 cur_item = &mlx5_flow_items[items->type];
1100 ret = cur_item->convert(items,
1101 (cur_item->default_mask ?
1102 cur_item->default_mask :
1106 rte_flow_error_set(error, ENOTSUP,
1107 RTE_FLOW_ERROR_TYPE_ITEM,
1108 items, "item not supported");
1113 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1115 * Last step. Complete missing specification to reach the RSS
1118 if (parser->queues_n > 1)
1119 priv_flow_convert_finalise(priv, parser);
1121 /* Only verification is expected, all resources should be released. */
1122 if (!parser->create) {
1124 rte_free(parser->drop_q.ibv_attr);
1125 parser->drop_q.ibv_attr = NULL;
1127 for (i = 0; i != hash_rxq_init_n; ++i) {
1128 if (parser->queue[i].ibv_attr) {
1129 rte_free(parser->queue[i].ibv_attr);
1130 parser->queue[i].ibv_attr = NULL;
1136 for (i = 0; i != hash_rxq_init_n; ++i) {
1137 if (parser->queue[i].ibv_attr) {
1138 rte_free(parser->queue[i].ibv_attr);
1139 parser->queue[i].ibv_attr = NULL;
1142 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1143 NULL, "cannot allocate verbs spec attributes.");
1148 * Copy the specification created into the flow.
1151 * Internal parser structure.
1153 * Create specification.
1155 * Size in bytes of the specification to copy.
1158 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1165 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1166 parser->drop_q.offset);
1167 memcpy(dst, src, size);
1168 ++parser->drop_q.ibv_attr->num_of_specs;
1169 parser->drop_q.offset += size;
1172 for (i = 0; i != hash_rxq_init_n; ++i) {
1173 if (!parser->queue[i].ibv_attr)
1175 /* Specification must be the same l3 type or none. */
1176 if (parser->layer == HASH_RXQ_ETH ||
1177 (hash_rxq_init[parser->layer].ip_version ==
1178 hash_rxq_init[i].ip_version) ||
1179 (hash_rxq_init[i].ip_version == 0)) {
1180 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1181 parser->queue[i].offset);
1182 memcpy(dst, src, size);
1183 ++parser->queue[i].ibv_attr->num_of_specs;
1184 parser->queue[i].offset += size;
1190 * Convert Ethernet item to Verbs specification.
1193 * Item specification.
1194 * @param default_mask[in]
1195 * Default bit-masks to use when item->mask is not provided.
1196 * @param data[in, out]
1200 mlx5_flow_create_eth(const struct rte_flow_item *item,
1201 const void *default_mask,
1204 const struct rte_flow_item_eth *spec = item->spec;
1205 const struct rte_flow_item_eth *mask = item->mask;
1206 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1207 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1208 struct ibv_flow_spec_eth eth = {
1209 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1213 parser->layer = HASH_RXQ_ETH;
1218 mask = default_mask;
1219 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1220 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1221 eth.val.ether_type = spec->type;
1222 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1223 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1224 eth.mask.ether_type = mask->type;
1225 /* Remove unwanted bits from values. */
1226 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1227 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1228 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1230 eth.val.ether_type &= eth.mask.ether_type;
1232 mlx5_flow_create_copy(parser, ð, eth_size);
1237 * Convert VLAN item to Verbs specification.
1240 * Item specification.
1241 * @param default_mask[in]
1242 * Default bit-masks to use when item->mask is not provided.
1243 * @param data[in, out]
1247 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1248 const void *default_mask,
1251 const struct rte_flow_item_vlan *spec = item->spec;
1252 const struct rte_flow_item_vlan *mask = item->mask;
1253 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1254 struct ibv_flow_spec_eth *eth;
1255 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1260 mask = default_mask;
1263 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1264 parser->drop_q.offset - eth_size);
1265 eth->val.vlan_tag = spec->tci;
1266 eth->mask.vlan_tag = mask->tci;
1267 eth->val.vlan_tag &= eth->mask.vlan_tag;
1270 for (i = 0; i != hash_rxq_init_n; ++i) {
1271 if (!parser->queue[i].ibv_attr)
1274 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1275 parser->queue[i].offset - eth_size);
1276 eth->val.vlan_tag = spec->tci;
1277 eth->mask.vlan_tag = mask->tci;
1278 eth->val.vlan_tag &= eth->mask.vlan_tag;
1285 * Convert IPv4 item to Verbs specification.
1288 * Item specification.
1289 * @param default_mask[in]
1290 * Default bit-masks to use when item->mask is not provided.
1291 * @param data[in, out]
1295 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1296 const void *default_mask,
1299 const struct rte_flow_item_ipv4 *spec = item->spec;
1300 const struct rte_flow_item_ipv4 *mask = item->mask;
1301 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1302 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1303 struct ibv_flow_spec_ipv4_ext ipv4 = {
1304 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1308 parser->layer = HASH_RXQ_IPV4;
1311 mask = default_mask;
1312 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1313 .src_ip = spec->hdr.src_addr,
1314 .dst_ip = spec->hdr.dst_addr,
1315 .proto = spec->hdr.next_proto_id,
1316 .tos = spec->hdr.type_of_service,
1318 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1319 .src_ip = mask->hdr.src_addr,
1320 .dst_ip = mask->hdr.dst_addr,
1321 .proto = mask->hdr.next_proto_id,
1322 .tos = mask->hdr.type_of_service,
1324 /* Remove unwanted bits from values. */
1325 ipv4.val.src_ip &= ipv4.mask.src_ip;
1326 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1327 ipv4.val.proto &= ipv4.mask.proto;
1328 ipv4.val.tos &= ipv4.mask.tos;
1330 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1335 * Convert IPv6 item to Verbs specification.
1338 * Item specification.
1339 * @param default_mask[in]
1340 * Default bit-masks to use when item->mask is not provided.
1341 * @param data[in, out]
1345 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1346 const void *default_mask,
1349 const struct rte_flow_item_ipv6 *spec = item->spec;
1350 const struct rte_flow_item_ipv6 *mask = item->mask;
1351 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1352 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1353 struct ibv_flow_spec_ipv6 ipv6 = {
1354 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1358 parser->layer = HASH_RXQ_IPV6;
1363 mask = default_mask;
1364 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1365 RTE_DIM(ipv6.val.src_ip));
1366 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1367 RTE_DIM(ipv6.val.dst_ip));
1368 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1369 RTE_DIM(ipv6.mask.src_ip));
1370 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1371 RTE_DIM(ipv6.mask.dst_ip));
1372 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1373 ipv6.mask.next_hdr = mask->hdr.proto;
1374 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1375 /* Remove unwanted bits from values. */
1376 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1377 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1378 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1380 ipv6.val.flow_label &= ipv6.mask.flow_label;
1381 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1382 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1384 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1389 * Convert UDP item to Verbs specification.
1392 * Item specification.
1393 * @param default_mask[in]
1394 * Default bit-masks to use when item->mask is not provided.
1395 * @param data[in, out]
1399 mlx5_flow_create_udp(const struct rte_flow_item *item,
1400 const void *default_mask,
1403 const struct rte_flow_item_udp *spec = item->spec;
1404 const struct rte_flow_item_udp *mask = item->mask;
1405 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1406 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1407 struct ibv_flow_spec_tcp_udp udp = {
1408 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1412 if (parser->layer == HASH_RXQ_IPV4)
1413 parser->layer = HASH_RXQ_UDPV4;
1415 parser->layer = HASH_RXQ_UDPV6;
1418 mask = default_mask;
1419 udp.val.dst_port = spec->hdr.dst_port;
1420 udp.val.src_port = spec->hdr.src_port;
1421 udp.mask.dst_port = mask->hdr.dst_port;
1422 udp.mask.src_port = mask->hdr.src_port;
1423 /* Remove unwanted bits from values. */
1424 udp.val.src_port &= udp.mask.src_port;
1425 udp.val.dst_port &= udp.mask.dst_port;
1427 mlx5_flow_create_copy(parser, &udp, udp_size);
1432 * Convert TCP item to Verbs specification.
1435 * Item specification.
1436 * @param default_mask[in]
1437 * Default bit-masks to use when item->mask is not provided.
1438 * @param data[in, out]
1442 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1443 const void *default_mask,
1446 const struct rte_flow_item_tcp *spec = item->spec;
1447 const struct rte_flow_item_tcp *mask = item->mask;
1448 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1449 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1450 struct ibv_flow_spec_tcp_udp tcp = {
1451 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1455 if (parser->layer == HASH_RXQ_IPV4)
1456 parser->layer = HASH_RXQ_TCPV4;
1458 parser->layer = HASH_RXQ_TCPV6;
1461 mask = default_mask;
1462 tcp.val.dst_port = spec->hdr.dst_port;
1463 tcp.val.src_port = spec->hdr.src_port;
1464 tcp.mask.dst_port = mask->hdr.dst_port;
1465 tcp.mask.src_port = mask->hdr.src_port;
1466 /* Remove unwanted bits from values. */
1467 tcp.val.src_port &= tcp.mask.src_port;
1468 tcp.val.dst_port &= tcp.mask.dst_port;
1470 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1475 * Convert VXLAN item to Verbs specification.
1478 * Item specification.
1479 * @param default_mask[in]
1480 * Default bit-masks to use when item->mask is not provided.
1481 * @param data[in, out]
1485 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1486 const void *default_mask,
1489 const struct rte_flow_item_vxlan *spec = item->spec;
1490 const struct rte_flow_item_vxlan *mask = item->mask;
1491 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1492 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1493 struct ibv_flow_spec_tunnel vxlan = {
1494 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1503 parser->inner = IBV_FLOW_SPEC_INNER;
1506 mask = default_mask;
1507 memcpy(&id.vni[1], spec->vni, 3);
1508 vxlan.val.tunnel_id = id.vlan_id;
1509 memcpy(&id.vni[1], mask->vni, 3);
1510 vxlan.mask.tunnel_id = id.vlan_id;
1511 /* Remove unwanted bits from values. */
1512 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1514 mlx5_flow_create_copy(parser, &vxlan, size);
1519 * Convert mark/flag action to Verbs specification.
1522 * Internal parser structure.
1527 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1529 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1530 struct ibv_flow_spec_action_tag tag = {
1531 .type = IBV_FLOW_SPEC_ACTION_TAG,
1533 .tag_id = mlx5_flow_mark_set(mark_id),
1536 assert(parser->mark);
1537 mlx5_flow_create_copy(parser, &tag, size);
1542 * Complete flow rule creation with a drop queue.
1545 * Pointer to private structure.
1547 * Internal parser structure.
1549 * Pointer to the rte_flow.
1551 * Perform verbose error reporting if not NULL.
1554 * 0 on success, errno value on failure.
1557 priv_flow_create_action_queue_drop(struct priv *priv,
1558 struct mlx5_flow_parse *parser,
1559 struct rte_flow *flow,
1560 struct rte_flow_error *error)
1562 struct ibv_flow_spec_action_drop *drop;
1563 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1569 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1570 parser->drop_q.offset);
1571 *drop = (struct ibv_flow_spec_action_drop){
1572 .type = IBV_FLOW_SPEC_ACTION_DROP,
1575 ++parser->drop_q.ibv_attr->num_of_specs;
1576 parser->drop_q.offset += size;
1577 if (!priv->dev->data->dev_started)
1579 flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
1580 parser->drop_q.ibv_attr = NULL;
1581 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
1582 flow->drxq.ibv_attr);
1583 if (!flow->drxq.ibv_flow) {
1584 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1585 NULL, "flow rule creation failure");
1592 if (flow->drxq.ibv_flow) {
1593 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1594 flow->drxq.ibv_flow = NULL;
1596 if (flow->drxq.ibv_attr) {
1597 rte_free(flow->drxq.ibv_attr);
1598 flow->drxq.ibv_attr = NULL;
1604 * Create hash Rx queues when RSS is enabled.
1607 * Pointer to private structure.
1609 * Internal parser structure.
1611 * Pointer to the rte_flow.
1613 * Perform verbose error reporting if not NULL.
1616 * 0 on success, a errno value otherwise and rte_errno is set.
1619 priv_flow_create_action_queue_rss(struct priv *priv,
1620 struct mlx5_flow_parse *parser,
1621 struct rte_flow *flow,
1622 struct rte_flow_error *error)
1626 for (i = 0; i != hash_rxq_init_n; ++i) {
1627 uint64_t hash_fields;
1629 if (!parser->queue[i].ibv_attr)
1631 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1632 parser->queue[i].ibv_attr = NULL;
1633 hash_fields = hash_rxq_init[i].hash_fields;
1634 flow->frxq[i].hrxq =
1635 mlx5_priv_hrxq_get(priv,
1636 parser->rss_conf.rss_key,
1637 parser->rss_conf.rss_key_len,
1640 hash_fields ? parser->queues_n : 1);
1641 if (flow->frxq[i].hrxq)
1643 flow->frxq[i].hrxq =
1644 mlx5_priv_hrxq_new(priv,
1645 parser->rss_conf.rss_key,
1646 parser->rss_conf.rss_key_len,
1649 hash_fields ? parser->queues_n : 1);
1650 if (!flow->frxq[i].hrxq) {
1651 rte_flow_error_set(error, ENOMEM,
1652 RTE_FLOW_ERROR_TYPE_HANDLE,
1653 NULL, "cannot create hash rxq");
1661 * Complete flow rule creation.
1664 * Pointer to private structure.
1666 * Internal parser structure.
1668 * Pointer to the rte_flow.
1670 * Perform verbose error reporting if not NULL.
1673 * 0 on success, a errno value otherwise and rte_errno is set.
1676 priv_flow_create_action_queue(struct priv *priv,
1677 struct mlx5_flow_parse *parser,
1678 struct rte_flow *flow,
1679 struct rte_flow_error *error)
1686 assert(!parser->drop);
1687 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1690 if (!priv->dev->data->dev_started)
1692 for (i = 0; i != hash_rxq_init_n; ++i) {
1693 if (!flow->frxq[i].hrxq)
1695 flow->frxq[i].ibv_flow =
1696 ibv_create_flow(flow->frxq[i].hrxq->qp,
1697 flow->frxq[i].ibv_attr);
1698 if (!flow->frxq[i].ibv_flow) {
1699 rte_flow_error_set(error, ENOMEM,
1700 RTE_FLOW_ERROR_TYPE_HANDLE,
1701 NULL, "flow rule creation failure");
1705 DEBUG("%p type %d QP %p ibv_flow %p",
1707 (void *)flow->frxq[i].hrxq,
1708 (void *)flow->frxq[i].ibv_flow);
1710 for (i = 0; i != parser->queues_n; ++i) {
1711 struct mlx5_rxq_data *q =
1712 (*priv->rxqs)[parser->queues[i]];
1714 q->mark |= parser->mark;
1719 for (i = 0; i != hash_rxq_init_n; ++i) {
1720 if (flow->frxq[i].ibv_flow) {
1721 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1723 claim_zero(ibv_destroy_flow(ibv_flow));
1725 if (flow->frxq[i].hrxq)
1726 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1727 if (flow->frxq[i].ibv_attr)
1728 rte_free(flow->frxq[i].ibv_attr);
1737 * Pointer to private structure.
1739 * Pointer to a TAILQ flow list.
1741 * Flow rule attributes.
1742 * @param[in] pattern
1743 * Pattern specification (list terminated by the END pattern item).
1744 * @param[in] actions
1745 * Associated actions (list terminated by the END action).
1747 * Perform verbose error reporting if not NULL.
1750 * A flow on success, NULL otherwise.
1752 static struct rte_flow *
1753 priv_flow_create(struct priv *priv,
1754 struct mlx5_flows *list,
1755 const struct rte_flow_attr *attr,
1756 const struct rte_flow_item items[],
1757 const struct rte_flow_action actions[],
1758 struct rte_flow_error *error)
1760 struct mlx5_flow_parse parser = { .create = 1, };
1761 struct rte_flow *flow = NULL;
1765 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1768 flow = rte_calloc(__func__, 1,
1769 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1772 rte_flow_error_set(error, ENOMEM,
1773 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1775 "cannot allocate flow memory");
1778 /* Copy queues configuration. */
1779 flow->queues = (uint16_t (*)[])(flow + 1);
1780 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1781 flow->queues_n = parser.queues_n;
1782 /* Copy RSS configuration. */
1783 flow->rss_conf = parser.rss_conf;
1784 flow->rss_conf.rss_key = flow->rss_key;
1785 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1786 /* finalise the flow. */
1788 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1791 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1794 TAILQ_INSERT_TAIL(list, flow, next);
1795 DEBUG("Flow created %p", (void *)flow);
1799 rte_free(parser.drop_q.ibv_attr);
1801 for (i = 0; i != hash_rxq_init_n; ++i) {
1802 if (parser.queue[i].ibv_attr)
1803 rte_free(parser.queue[i].ibv_attr);
1811 * Validate a flow supported by the NIC.
1813 * @see rte_flow_validate()
1817 mlx5_flow_validate(struct rte_eth_dev *dev,
1818 const struct rte_flow_attr *attr,
1819 const struct rte_flow_item items[],
1820 const struct rte_flow_action actions[],
1821 struct rte_flow_error *error)
1823 struct priv *priv = dev->data->dev_private;
1825 struct mlx5_flow_parse parser = { .create = 0, };
1828 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1836 * @see rte_flow_create()
1840 mlx5_flow_create(struct rte_eth_dev *dev,
1841 const struct rte_flow_attr *attr,
1842 const struct rte_flow_item items[],
1843 const struct rte_flow_action actions[],
1844 struct rte_flow_error *error)
1846 struct priv *priv = dev->data->dev_private;
1847 struct rte_flow *flow;
1850 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1860 * Pointer to private structure.
1862 * Pointer to a TAILQ flow list.
1867 priv_flow_destroy(struct priv *priv,
1868 struct mlx5_flows *list,
1869 struct rte_flow *flow)
1873 if (flow->drop || !flow->mark)
1875 for (i = 0; i != flow->queues_n; ++i) {
1876 struct rte_flow *tmp;
1880 * To remove the mark from the queue, the queue must not be
1881 * present in any other marked flow (RSS or not).
1883 TAILQ_FOREACH(tmp, list, next) {
1885 uint16_t *tqs = NULL;
1890 for (j = 0; j != hash_rxq_init_n; ++j) {
1891 if (!tmp->frxq[j].hrxq)
1893 tqs = tmp->frxq[j].hrxq->ind_table->queues;
1894 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
1898 for (j = 0; (j != tq_n) && !mark; j++)
1899 if (tqs[j] == (*flow->queues)[i])
1902 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
1906 if (flow->drxq.ibv_flow)
1907 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1908 rte_free(flow->drxq.ibv_attr);
1910 for (i = 0; i != hash_rxq_init_n; ++i) {
1911 struct mlx5_flow *frxq = &flow->frxq[i];
1914 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
1916 mlx5_priv_hrxq_release(priv, frxq->hrxq);
1918 rte_free(frxq->ibv_attr);
1921 TAILQ_REMOVE(list, flow, next);
1922 DEBUG("Flow destroyed %p", (void *)flow);
1927 * Destroy all flows.
1930 * Pointer to private structure.
1932 * Pointer to a TAILQ flow list.
1935 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
1937 while (!TAILQ_EMPTY(list)) {
1938 struct rte_flow *flow;
1940 flow = TAILQ_FIRST(list);
1941 priv_flow_destroy(priv, list, flow);
1946 * Create drop queue.
1949 * Pointer to private structure.
1955 priv_flow_create_drop_queue(struct priv *priv)
1957 struct mlx5_hrxq_drop *fdq = NULL;
1961 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
1963 WARN("cannot allocate memory for drop queue");
1966 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
1968 WARN("cannot allocate CQ for drop queue");
1971 fdq->wq = ibv_create_wq(priv->ctx,
1972 &(struct ibv_wq_init_attr){
1973 .wq_type = IBV_WQT_RQ,
1980 WARN("cannot allocate WQ for drop queue");
1983 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
1984 &(struct ibv_rwq_ind_table_init_attr){
1985 .log_ind_tbl_size = 0,
1986 .ind_tbl = &fdq->wq,
1989 if (!fdq->ind_table) {
1990 WARN("cannot allocate indirection table for drop queue");
1993 fdq->qp = ibv_create_qp_ex(priv->ctx,
1994 &(struct ibv_qp_init_attr_ex){
1995 .qp_type = IBV_QPT_RAW_PACKET,
1997 IBV_QP_INIT_ATTR_PD |
1998 IBV_QP_INIT_ATTR_IND_TABLE |
1999 IBV_QP_INIT_ATTR_RX_HASH,
2000 .rx_hash_conf = (struct ibv_rx_hash_conf){
2002 IBV_RX_HASH_FUNC_TOEPLITZ,
2003 .rx_hash_key_len = rss_hash_default_key_len,
2004 .rx_hash_key = rss_hash_default_key,
2005 .rx_hash_fields_mask = 0,
2007 .rwq_ind_tbl = fdq->ind_table,
2011 WARN("cannot allocate QP for drop queue");
2014 priv->flow_drop_queue = fdq;
2018 claim_zero(ibv_destroy_qp(fdq->qp));
2020 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2022 claim_zero(ibv_destroy_wq(fdq->wq));
2024 claim_zero(ibv_destroy_cq(fdq->cq));
2027 priv->flow_drop_queue = NULL;
2032 * Delete drop queue.
2035 * Pointer to private structure.
2038 priv_flow_delete_drop_queue(struct priv *priv)
2040 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2045 claim_zero(ibv_destroy_qp(fdq->qp));
2047 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2049 claim_zero(ibv_destroy_wq(fdq->wq));
2051 claim_zero(ibv_destroy_cq(fdq->cq));
2053 priv->flow_drop_queue = NULL;
2060 * Pointer to private structure.
2062 * Pointer to a TAILQ flow list.
2065 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2067 struct rte_flow *flow;
2069 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2073 if (!flow->drxq.ibv_flow)
2075 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2076 flow->drxq.ibv_flow = NULL;
2081 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2083 for (i = 0; i != hash_rxq_init_n; ++i) {
2084 if (!flow->frxq[i].hrxq)
2086 ind_tbl = flow->frxq[i].hrxq->ind_table;
2089 for (i = 0; i != ind_tbl->queues_n; ++i)
2090 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2092 for (i = 0; i != hash_rxq_init_n; ++i) {
2093 if (!flow->frxq[i].ibv_flow)
2095 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2096 flow->frxq[i].ibv_flow = NULL;
2097 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2098 flow->frxq[i].hrxq = NULL;
2100 DEBUG("Flow %p removed", (void *)flow);
2108 * Pointer to private structure.
2110 * Pointer to a TAILQ flow list.
2113 * 0 on success, a errno value otherwise and rte_errno is set.
2116 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2118 struct rte_flow *flow;
2120 TAILQ_FOREACH(flow, list, next) {
2124 flow->drxq.ibv_flow =
2125 ibv_create_flow(priv->flow_drop_queue->qp,
2126 flow->drxq.ibv_attr);
2127 if (!flow->drxq.ibv_flow) {
2128 DEBUG("Flow %p cannot be applied",
2133 DEBUG("Flow %p applied", (void *)flow);
2137 for (i = 0; i != hash_rxq_init_n; ++i) {
2138 if (!flow->frxq[i].ibv_attr)
2140 flow->frxq[i].hrxq =
2141 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2142 flow->rss_conf.rss_key_len,
2143 hash_rxq_init[i].hash_fields,
2146 if (flow->frxq[i].hrxq)
2148 flow->frxq[i].hrxq =
2149 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2150 flow->rss_conf.rss_key_len,
2151 hash_rxq_init[i].hash_fields,
2154 if (!flow->frxq[i].hrxq) {
2155 DEBUG("Flow %p cannot be applied",
2161 flow->frxq[i].ibv_flow =
2162 ibv_create_flow(flow->frxq[i].hrxq->qp,
2163 flow->frxq[i].ibv_attr);
2164 if (!flow->frxq[i].ibv_flow) {
2165 DEBUG("Flow %p cannot be applied",
2170 DEBUG("Flow %p applied", (void *)flow);
2174 for (i = 0; i != flow->queues_n; ++i)
2175 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2181 * Verify the flow list is empty
2184 * Pointer to private structure.
2186 * @return the number of flows not released.
2189 priv_flow_verify(struct priv *priv)
2191 struct rte_flow *flow;
2194 TAILQ_FOREACH(flow, &priv->flows, next) {
2195 DEBUG("%p: flow %p still referenced", (void *)priv,
2203 * Enable a control flow configured from the control plane.
2206 * Pointer to Ethernet device.
2208 * An Ethernet flow spec to apply.
2210 * An Ethernet flow mask to apply.
2212 * A VLAN flow spec to apply.
2214 * A VLAN flow mask to apply.
2220 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2221 struct rte_flow_item_eth *eth_spec,
2222 struct rte_flow_item_eth *eth_mask,
2223 struct rte_flow_item_vlan *vlan_spec,
2224 struct rte_flow_item_vlan *vlan_mask)
2226 struct priv *priv = dev->data->dev_private;
2227 const struct rte_flow_attr attr = {
2229 .priority = MLX5_CTRL_FLOW_PRIORITY,
2231 struct rte_flow_item items[] = {
2233 .type = RTE_FLOW_ITEM_TYPE_ETH,
2239 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2240 RTE_FLOW_ITEM_TYPE_END,
2246 .type = RTE_FLOW_ITEM_TYPE_END,
2249 struct rte_flow_action actions[] = {
2251 .type = RTE_FLOW_ACTION_TYPE_RSS,
2254 .type = RTE_FLOW_ACTION_TYPE_END,
2257 struct rte_flow *flow;
2258 struct rte_flow_error error;
2261 struct rte_flow_action_rss rss;
2263 const struct rte_eth_rss_conf *rss_conf;
2265 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2269 if (!priv->reta_idx_n)
2271 for (i = 0; i != priv->reta_idx_n; ++i)
2272 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2273 action_rss.local.rss_conf = &priv->rss_conf;
2274 action_rss.local.num = priv->reta_idx_n;
2275 actions[0].conf = (const void *)&action_rss.rss;
2276 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2284 * Enable a flow control configured from the control plane.
2287 * Pointer to Ethernet device.
2289 * An Ethernet flow spec to apply.
2291 * An Ethernet flow mask to apply.
2297 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2298 struct rte_flow_item_eth *eth_spec,
2299 struct rte_flow_item_eth *eth_mask)
2301 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2307 * @see rte_flow_destroy()
2311 mlx5_flow_destroy(struct rte_eth_dev *dev,
2312 struct rte_flow *flow,
2313 struct rte_flow_error *error)
2315 struct priv *priv = dev->data->dev_private;
2319 priv_flow_destroy(priv, &priv->flows, flow);
2325 * Destroy all flows.
2327 * @see rte_flow_flush()
2331 mlx5_flow_flush(struct rte_eth_dev *dev,
2332 struct rte_flow_error *error)
2334 struct priv *priv = dev->data->dev_private;
2338 priv_flow_flush(priv, &priv->flows);
2346 * @see rte_flow_isolate()
2350 mlx5_flow_isolate(struct rte_eth_dev *dev,
2352 struct rte_flow_error *error)
2354 struct priv *priv = dev->data->dev_private;
2357 if (dev->data->dev_started) {
2358 rte_flow_error_set(error, EBUSY,
2359 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2361 "port must be stopped first");
2365 priv->isolated = !!enable;
2367 priv->dev->dev_ops = &mlx5_dev_ops_isolate;
2369 priv->dev->dev_ops = &mlx5_dev_ops;
2375 * Convert a flow director filter to a generic flow.
2378 * Private structure.
2379 * @param fdir_filter
2380 * Flow director filter to add.
2382 * Generic flow parameters structure.
2385 * 0 on success, errno value on error.
2388 priv_fdir_filter_convert(struct priv *priv,
2389 const struct rte_eth_fdir_filter *fdir_filter,
2390 struct mlx5_fdir *attributes)
2392 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2394 /* Validate queue number. */
2395 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2396 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2399 /* Validate the behavior. */
2400 if (fdir_filter->action.behavior != RTE_ETH_FDIR_ACCEPT) {
2401 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2404 attributes->attr.ingress = 1;
2405 attributes->items[0] = (struct rte_flow_item) {
2406 .type = RTE_FLOW_ITEM_TYPE_ETH,
2407 .spec = &attributes->l2,
2409 attributes->actions[0] = (struct rte_flow_action){
2410 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2411 .conf = &attributes->queue,
2413 attributes->queue.index = fdir_filter->action.rx_queue;
2414 switch (fdir_filter->input.flow_type) {
2415 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2416 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2417 .src_addr = input->flow.udp4_flow.ip.src_ip,
2418 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2419 .time_to_live = input->flow.udp4_flow.ip.ttl,
2420 .type_of_service = input->flow.udp4_flow.ip.tos,
2421 .next_proto_id = input->flow.udp4_flow.ip.proto,
2423 attributes->l4.udp.hdr = (struct udp_hdr){
2424 .src_port = input->flow.udp4_flow.src_port,
2425 .dst_port = input->flow.udp4_flow.dst_port,
2427 attributes->items[1] = (struct rte_flow_item){
2428 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2429 .spec = &attributes->l3,
2431 attributes->items[2] = (struct rte_flow_item){
2432 .type = RTE_FLOW_ITEM_TYPE_UDP,
2433 .spec = &attributes->l4,
2436 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2437 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2438 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2439 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2440 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2441 .type_of_service = input->flow.tcp4_flow.ip.tos,
2442 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2444 attributes->l4.tcp.hdr = (struct tcp_hdr){
2445 .src_port = input->flow.tcp4_flow.src_port,
2446 .dst_port = input->flow.tcp4_flow.dst_port,
2448 attributes->items[1] = (struct rte_flow_item){
2449 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2450 .spec = &attributes->l3,
2452 attributes->items[2] = (struct rte_flow_item){
2453 .type = RTE_FLOW_ITEM_TYPE_TCP,
2454 .spec = &attributes->l4,
2457 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2458 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2459 .src_addr = input->flow.ip4_flow.src_ip,
2460 .dst_addr = input->flow.ip4_flow.dst_ip,
2461 .time_to_live = input->flow.ip4_flow.ttl,
2462 .type_of_service = input->flow.ip4_flow.tos,
2463 .next_proto_id = input->flow.ip4_flow.proto,
2465 attributes->items[1] = (struct rte_flow_item){
2466 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2467 .spec = &attributes->l3,
2470 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2471 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2472 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2473 .proto = input->flow.udp6_flow.ip.proto,
2475 memcpy(attributes->l3.ipv6.hdr.src_addr,
2476 input->flow.udp6_flow.ip.src_ip,
2477 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2478 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2479 input->flow.udp6_flow.ip.dst_ip,
2480 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2481 attributes->l4.udp.hdr = (struct udp_hdr){
2482 .src_port = input->flow.udp6_flow.src_port,
2483 .dst_port = input->flow.udp6_flow.dst_port,
2485 attributes->items[1] = (struct rte_flow_item){
2486 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2487 .spec = &attributes->l3,
2489 attributes->items[2] = (struct rte_flow_item){
2490 .type = RTE_FLOW_ITEM_TYPE_UDP,
2491 .spec = &attributes->l4,
2494 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2495 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2496 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2497 .proto = input->flow.tcp6_flow.ip.proto,
2499 memcpy(attributes->l3.ipv6.hdr.src_addr,
2500 input->flow.tcp6_flow.ip.src_ip,
2501 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2502 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2503 input->flow.tcp6_flow.ip.dst_ip,
2504 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2505 attributes->l4.tcp.hdr = (struct tcp_hdr){
2506 .src_port = input->flow.tcp6_flow.src_port,
2507 .dst_port = input->flow.tcp6_flow.dst_port,
2509 attributes->items[1] = (struct rte_flow_item){
2510 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2511 .spec = &attributes->l3,
2513 attributes->items[2] = (struct rte_flow_item){
2514 .type = RTE_FLOW_ITEM_TYPE_UDP,
2515 .spec = &attributes->l4,
2518 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2519 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2520 .hop_limits = input->flow.ipv6_flow.hop_limits,
2521 .proto = input->flow.ipv6_flow.proto,
2523 memcpy(attributes->l3.ipv6.hdr.src_addr,
2524 input->flow.ipv6_flow.src_ip,
2525 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2526 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2527 input->flow.ipv6_flow.dst_ip,
2528 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2529 attributes->items[1] = (struct rte_flow_item){
2530 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2531 .spec = &attributes->l3,
2535 ERROR("invalid flow type%d",
2536 fdir_filter->input.flow_type);
2543 * Add new flow director filter and store it in list.
2546 * Private structure.
2547 * @param fdir_filter
2548 * Flow director filter to add.
2551 * 0 on success, errno value on failure.
2554 priv_fdir_filter_add(struct priv *priv,
2555 const struct rte_eth_fdir_filter *fdir_filter)
2557 struct mlx5_fdir attributes = {
2560 struct mlx5_flow_parse parser = {
2561 .layer = HASH_RXQ_ETH,
2563 struct rte_flow_error error;
2564 struct rte_flow *flow;
2567 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2570 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2571 attributes.actions, &error, &parser);
2574 flow = priv_flow_create(priv,
2581 TAILQ_INSERT_TAIL(&priv->flows, flow, next);
2582 DEBUG("FDIR created %p", (void *)flow);
2589 * Delete specific filter.
2592 * Private structure.
2593 * @param fdir_filter
2594 * Filter to be deleted.
2597 * 0 on success, errno value on failure.
2600 priv_fdir_filter_delete(struct priv *priv,
2601 const struct rte_eth_fdir_filter *fdir_filter)
2603 struct mlx5_fdir attributes;
2604 struct mlx5_flow_parse parser = {
2606 .layer = HASH_RXQ_ETH,
2608 struct rte_flow_error error;
2609 struct rte_flow *flow;
2613 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2616 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2617 attributes.actions, &error, &parser);
2620 TAILQ_FOREACH(flow, &priv->flows, next) {
2621 struct ibv_flow_attr *attr;
2622 struct ibv_spec_header *attr_h;
2624 struct ibv_flow_attr *flow_attr;
2625 struct ibv_spec_header *flow_h;
2627 unsigned int specs_n;
2630 attr = parser.drop_q.ibv_attr;
2632 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2634 flow_attr = flow->drxq.ibv_attr;
2636 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2637 /* Compare first the attributes. */
2638 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2640 if (attr->num_of_specs == 0)
2642 spec = (void *)((uintptr_t)attr +
2643 sizeof(struct ibv_flow_attr));
2644 flow_spec = (void *)((uintptr_t)flow_attr +
2645 sizeof(struct ibv_flow_attr));
2646 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2647 for (i = 0; i != specs_n; ++i) {
2650 if (memcmp(spec, flow_spec,
2651 RTE_MIN(attr_h->size, flow_h->size)))
2653 spec = (void *)((uintptr_t)attr + attr_h->size);
2654 flow_spec = (void *)((uintptr_t)flow_attr +
2657 /* At this point, the flow match. */
2661 priv_flow_destroy(priv, &priv->flows, flow);
2664 rte_free(parser.drop_q.ibv_attr);
2666 for (i = 0; i != hash_rxq_init_n; ++i) {
2667 if (parser.queue[i].ibv_attr)
2668 rte_free(parser.queue[i].ibv_attr);
2675 * Update queue for specific filter.
2678 * Private structure.
2679 * @param fdir_filter
2680 * Filter to be updated.
2683 * 0 on success, errno value on failure.
2686 priv_fdir_filter_update(struct priv *priv,
2687 const struct rte_eth_fdir_filter *fdir_filter)
2691 ret = priv_fdir_filter_delete(priv, fdir_filter);
2694 ret = priv_fdir_filter_add(priv, fdir_filter);
2699 * Flush all filters.
2702 * Private structure.
2705 priv_fdir_filter_flush(struct priv *priv)
2707 priv_flow_flush(priv, &priv->flows);
2711 * Get flow director information.
2714 * Private structure.
2715 * @param[out] fdir_info
2716 * Resulting flow director information.
2719 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2721 struct rte_eth_fdir_masks *mask =
2722 &priv->dev->data->dev_conf.fdir_conf.mask;
2724 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2725 fdir_info->guarant_spc = 0;
2726 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2727 fdir_info->max_flexpayload = 0;
2728 fdir_info->flow_types_mask[0] = 0;
2729 fdir_info->flex_payload_unit = 0;
2730 fdir_info->max_flex_payload_segment_num = 0;
2731 fdir_info->flex_payload_limit = 0;
2732 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2736 * Deal with flow director operations.
2739 * Pointer to private structure.
2741 * Operation to perform.
2743 * Pointer to operation-specific structure.
2746 * 0 on success, errno value on failure.
2749 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2751 enum rte_fdir_mode fdir_mode =
2752 priv->dev->data->dev_conf.fdir_conf.mode;
2755 if (filter_op == RTE_ETH_FILTER_NOP)
2757 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2758 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2759 ERROR("%p: flow director mode %d not supported",
2760 (void *)priv, fdir_mode);
2763 switch (filter_op) {
2764 case RTE_ETH_FILTER_ADD:
2765 ret = priv_fdir_filter_add(priv, arg);
2767 case RTE_ETH_FILTER_UPDATE:
2768 ret = priv_fdir_filter_update(priv, arg);
2770 case RTE_ETH_FILTER_DELETE:
2771 ret = priv_fdir_filter_delete(priv, arg);
2773 case RTE_ETH_FILTER_FLUSH:
2774 priv_fdir_filter_flush(priv);
2776 case RTE_ETH_FILTER_INFO:
2777 priv_fdir_info_get(priv, arg);
2780 DEBUG("%p: unknown operation %u", (void *)priv,
2789 * Manage filter operations.
2792 * Pointer to Ethernet device structure.
2793 * @param filter_type
2796 * Operation to perform.
2798 * Pointer to operation-specific structure.
2801 * 0 on success, negative errno value on failure.
2804 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
2805 enum rte_filter_type filter_type,
2806 enum rte_filter_op filter_op,
2810 struct priv *priv = dev->data->dev_private;
2812 switch (filter_type) {
2813 case RTE_ETH_FILTER_GENERIC:
2814 if (filter_op != RTE_ETH_FILTER_GET)
2816 *(const void **)arg = &mlx5_flow_ops;
2818 case RTE_ETH_FILTER_FDIR:
2820 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
2824 ERROR("%p: filter type (%d) not supported",
2825 (void *)dev, filter_type);