4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 /* Define minimal priority for control plane flows. */
56 #define MLX5_CTRL_FLOW_PRIORITY 4
58 /* Internet Protocol versions. */
63 mlx5_flow_create_eth(const struct rte_flow_item *item,
64 const void *default_mask,
68 mlx5_flow_create_vlan(const struct rte_flow_item *item,
69 const void *default_mask,
73 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
74 const void *default_mask,
78 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
79 const void *default_mask,
83 mlx5_flow_create_udp(const struct rte_flow_item *item,
84 const void *default_mask,
88 mlx5_flow_create_tcp(const struct rte_flow_item *item,
89 const void *default_mask,
93 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
94 const void *default_mask,
97 struct mlx5_flow_parse;
100 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
104 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
106 /* Hash RX queue types. */
117 /* Initialization data for hash RX queue. */
118 struct hash_rxq_init {
119 uint64_t hash_fields; /* Fields that participate in the hash. */
120 uint64_t dpdk_rss_hf; /* Matching DPDK RSS hash fields. */
121 unsigned int flow_priority; /* Flow priority to use. */
122 unsigned int ip_version; /* Internet protocol. */
125 /* Initialization data for hash RX queues. */
126 const struct hash_rxq_init hash_rxq_init[] = {
128 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
129 IBV_RX_HASH_DST_IPV4 |
130 IBV_RX_HASH_SRC_PORT_TCP |
131 IBV_RX_HASH_DST_PORT_TCP),
132 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
134 .ip_version = MLX5_IPV4,
137 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
138 IBV_RX_HASH_DST_IPV4 |
139 IBV_RX_HASH_SRC_PORT_UDP |
140 IBV_RX_HASH_DST_PORT_UDP),
141 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
143 .ip_version = MLX5_IPV4,
146 .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
147 IBV_RX_HASH_DST_IPV4),
148 .dpdk_rss_hf = (ETH_RSS_IPV4 |
151 .ip_version = MLX5_IPV4,
154 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
155 IBV_RX_HASH_DST_IPV6 |
156 IBV_RX_HASH_SRC_PORT_TCP |
157 IBV_RX_HASH_DST_PORT_TCP),
158 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
160 .ip_version = MLX5_IPV6,
163 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
164 IBV_RX_HASH_DST_IPV6 |
165 IBV_RX_HASH_SRC_PORT_UDP |
166 IBV_RX_HASH_DST_PORT_UDP),
167 .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
169 .ip_version = MLX5_IPV6,
172 .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
173 IBV_RX_HASH_DST_IPV6),
174 .dpdk_rss_hf = (ETH_RSS_IPV6 |
177 .ip_version = MLX5_IPV6,
186 /* Number of entries in hash_rxq_init[]. */
187 const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
189 /** Structure for Drop queue. */
190 struct mlx5_hrxq_drop {
191 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
192 struct ibv_qp *qp; /**< Verbs queue pair. */
193 struct ibv_wq *wq; /**< Verbs work queue. */
194 struct ibv_cq *cq; /**< Verbs completion queue. */
197 /* Flows structures. */
199 uint64_t hash_fields; /**< Fields that participate in the hash. */
200 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
201 struct ibv_flow *ibv_flow; /**< Verbs flow. */
202 struct mlx5_hrxq *hrxq; /**< Hash Rx queues. */
205 /* Drop flows structures. */
206 struct mlx5_flow_drop {
207 struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
208 struct ibv_flow *ibv_flow; /**< Verbs flow. */
212 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
213 uint32_t mark:1; /**< Set if the flow is marked. */
214 uint32_t drop:1; /**< Drop queue. */
215 uint16_t queues_n; /**< Number of entries in queue[]. */
216 uint16_t (*queues)[]; /**< Queues indexes to use. */
217 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
218 uint8_t rss_key[40]; /**< copy of the RSS key. */
220 struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
221 /**< Flow with Rx queue. */
222 struct mlx5_flow_drop drxq; /**< Flow with drop Rx queue. */
226 /** Static initializer for items. */
228 (const enum rte_flow_item_type []){ \
229 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
232 /** Structure to generate a simple graph of layers supported by the NIC. */
233 struct mlx5_flow_items {
234 /** List of possible actions for these items. */
235 const enum rte_flow_action_type *const actions;
236 /** Bit-masks corresponding to the possibilities for the item. */
239 * Default bit-masks to use when item->mask is not provided. When
240 * \default_mask is also NULL, the full supported bit-mask (\mask) is
243 const void *default_mask;
244 /** Bit-masks size in bytes. */
245 const unsigned int mask_sz;
247 * Conversion function from rte_flow to NIC specific flow.
250 * rte_flow item to convert.
251 * @param default_mask
252 * Default bit-masks to use when item->mask is not provided.
254 * Internal structure to store the conversion.
257 * 0 on success, negative value otherwise.
259 int (*convert)(const struct rte_flow_item *item,
260 const void *default_mask,
262 /** Size in bytes of the destination structure. */
263 const unsigned int dst_sz;
264 /** List of possible following items. */
265 const enum rte_flow_item_type *const items;
268 /** Valid action for this PMD. */
269 static const enum rte_flow_action_type valid_actions[] = {
270 RTE_FLOW_ACTION_TYPE_DROP,
271 RTE_FLOW_ACTION_TYPE_QUEUE,
272 RTE_FLOW_ACTION_TYPE_MARK,
273 RTE_FLOW_ACTION_TYPE_FLAG,
274 RTE_FLOW_ACTION_TYPE_END,
277 /** Graph of supported items and associated actions. */
278 static const struct mlx5_flow_items mlx5_flow_items[] = {
279 [RTE_FLOW_ITEM_TYPE_END] = {
280 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
281 RTE_FLOW_ITEM_TYPE_VXLAN),
283 [RTE_FLOW_ITEM_TYPE_ETH] = {
284 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
285 RTE_FLOW_ITEM_TYPE_IPV4,
286 RTE_FLOW_ITEM_TYPE_IPV6),
287 .actions = valid_actions,
288 .mask = &(const struct rte_flow_item_eth){
289 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
290 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
293 .default_mask = &rte_flow_item_eth_mask,
294 .mask_sz = sizeof(struct rte_flow_item_eth),
295 .convert = mlx5_flow_create_eth,
296 .dst_sz = sizeof(struct ibv_flow_spec_eth),
298 [RTE_FLOW_ITEM_TYPE_VLAN] = {
299 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
300 RTE_FLOW_ITEM_TYPE_IPV6),
301 .actions = valid_actions,
302 .mask = &(const struct rte_flow_item_vlan){
305 .default_mask = &rte_flow_item_vlan_mask,
306 .mask_sz = sizeof(struct rte_flow_item_vlan),
307 .convert = mlx5_flow_create_vlan,
310 [RTE_FLOW_ITEM_TYPE_IPV4] = {
311 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
312 RTE_FLOW_ITEM_TYPE_TCP),
313 .actions = valid_actions,
314 .mask = &(const struct rte_flow_item_ipv4){
318 .type_of_service = -1,
322 .default_mask = &rte_flow_item_ipv4_mask,
323 .mask_sz = sizeof(struct rte_flow_item_ipv4),
324 .convert = mlx5_flow_create_ipv4,
325 .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
327 [RTE_FLOW_ITEM_TYPE_IPV6] = {
328 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
329 RTE_FLOW_ITEM_TYPE_TCP),
330 .actions = valid_actions,
331 .mask = &(const struct rte_flow_item_ipv6){
334 0xff, 0xff, 0xff, 0xff,
335 0xff, 0xff, 0xff, 0xff,
336 0xff, 0xff, 0xff, 0xff,
337 0xff, 0xff, 0xff, 0xff,
340 0xff, 0xff, 0xff, 0xff,
341 0xff, 0xff, 0xff, 0xff,
342 0xff, 0xff, 0xff, 0xff,
343 0xff, 0xff, 0xff, 0xff,
350 .default_mask = &rte_flow_item_ipv6_mask,
351 .mask_sz = sizeof(struct rte_flow_item_ipv6),
352 .convert = mlx5_flow_create_ipv6,
353 .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
355 [RTE_FLOW_ITEM_TYPE_UDP] = {
356 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
357 .actions = valid_actions,
358 .mask = &(const struct rte_flow_item_udp){
364 .default_mask = &rte_flow_item_udp_mask,
365 .mask_sz = sizeof(struct rte_flow_item_udp),
366 .convert = mlx5_flow_create_udp,
367 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
369 [RTE_FLOW_ITEM_TYPE_TCP] = {
370 .actions = valid_actions,
371 .mask = &(const struct rte_flow_item_tcp){
377 .default_mask = &rte_flow_item_tcp_mask,
378 .mask_sz = sizeof(struct rte_flow_item_tcp),
379 .convert = mlx5_flow_create_tcp,
380 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
382 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
383 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
384 .actions = valid_actions,
385 .mask = &(const struct rte_flow_item_vxlan){
386 .vni = "\xff\xff\xff",
388 .default_mask = &rte_flow_item_vxlan_mask,
389 .mask_sz = sizeof(struct rte_flow_item_vxlan),
390 .convert = mlx5_flow_create_vxlan,
391 .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
395 /** Structure to pass to the conversion function. */
396 struct mlx5_flow_parse {
397 uint32_t inner; /**< Set once VXLAN is encountered. */
399 /**< Whether resources should remain after a validate. */
400 uint32_t drop:1; /**< Target is a drop queue. */
401 uint32_t mark:1; /**< Mark is present in the flow. */
402 uint32_t mark_id; /**< Mark identifier. */
403 uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
404 uint16_t queues_n; /**< Number of entries in queue[]. */
405 struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
406 uint8_t rss_key[40]; /**< copy of the RSS key. */
407 enum hash_rxq_type layer; /**< Last pattern layer detected. */
410 struct ibv_flow_attr *ibv_attr;
411 /**< Pointer to Verbs attributes. */
413 /**< Current position or total size of the attribute. */
414 } queue[RTE_DIM(hash_rxq_init)];
416 struct ibv_flow_attr *ibv_attr;
417 /**< Pointer to Verbs attributes. */
419 /**< Current position or total size of the attribute. */
424 static const struct rte_flow_ops mlx5_flow_ops = {
425 .validate = mlx5_flow_validate,
426 .create = mlx5_flow_create,
427 .destroy = mlx5_flow_destroy,
428 .flush = mlx5_flow_flush,
430 .isolate = mlx5_flow_isolate,
433 /* Convert FDIR request to Generic flow. */
435 struct rte_flow_attr attr;
436 struct rte_flow_action actions[2];
437 struct rte_flow_item items[4];
438 struct rte_flow_item_eth l2;
440 struct rte_flow_item_ipv4 ipv4;
441 struct rte_flow_item_ipv6 ipv6;
444 struct rte_flow_item_udp udp;
445 struct rte_flow_item_tcp tcp;
447 struct rte_flow_action_queue queue;
450 /* Verbs specification header. */
451 struct ibv_spec_header {
452 enum ibv_flow_spec_type type;
457 * Check support for a given item.
460 * Item specification.
462 * Bit-masks covering supported fields to compare with spec, last and mask in
465 * Bit-Mask size in bytes.
471 mlx5_flow_item_validate(const struct rte_flow_item *item,
472 const uint8_t *mask, unsigned int size)
476 if (!item->spec && (item->mask || item->last))
478 if (item->spec && !item->mask) {
480 const uint8_t *spec = item->spec;
482 for (i = 0; i < size; ++i)
483 if ((spec[i] | mask[i]) != mask[i])
486 if (item->last && !item->mask) {
488 const uint8_t *spec = item->last;
490 for (i = 0; i < size; ++i)
491 if ((spec[i] | mask[i]) != mask[i])
496 const uint8_t *spec = item->mask;
498 for (i = 0; i < size; ++i)
499 if ((spec[i] | mask[i]) != mask[i])
502 if (item->spec && item->last) {
505 const uint8_t *apply = mask;
510 for (i = 0; i < size; ++i) {
511 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
512 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
514 ret = memcmp(spec, last, size);
520 * Copy the RSS configuration from the user ones.
523 * Pointer to private structure.
525 * Internal parser structure.
527 * User RSS configuration to save.
530 * 0 on success, errno value on failure.
533 priv_flow_convert_rss_conf(struct priv *priv,
534 struct mlx5_flow_parse *parser,
535 const struct rte_eth_rss_conf *rss_conf)
537 const struct rte_eth_rss_conf *rss =
538 rss_conf ? rss_conf : &priv->rss_conf;
540 if (rss->rss_key_len > 40)
542 parser->rss_conf.rss_key_len = rss->rss_key_len;
543 parser->rss_conf.rss_hf = rss->rss_hf;
544 memcpy(parser->rss_key, rss->rss_key, rss->rss_key_len);
545 parser->rss_conf.rss_key = parser->rss_key;
550 * Extract attribute to the parser.
553 * Pointer to private structure.
555 * Flow rule attributes.
557 * Perform verbose error reporting if not NULL.
558 * @param[in, out] parser
559 * Internal parser structure.
562 * 0 on success, a negative errno value otherwise and rte_errno is set.
565 priv_flow_convert_attributes(struct priv *priv,
566 const struct rte_flow_attr *attr,
567 struct rte_flow_error *error,
568 struct mlx5_flow_parse *parser)
573 rte_flow_error_set(error, ENOTSUP,
574 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
576 "groups are not supported");
579 if (attr->priority && attr->priority != MLX5_CTRL_FLOW_PRIORITY) {
580 rte_flow_error_set(error, ENOTSUP,
581 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
583 "priorities are not supported");
587 rte_flow_error_set(error, ENOTSUP,
588 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
590 "egress is not supported");
593 if (!attr->ingress) {
594 rte_flow_error_set(error, ENOTSUP,
595 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
597 "only ingress is supported");
604 * Extract actions request to the parser.
607 * Pointer to private structure.
609 * Associated actions (list terminated by the END action).
611 * Perform verbose error reporting if not NULL.
612 * @param[in, out] parser
613 * Internal parser structure.
616 * 0 on success, a negative errno value otherwise and rte_errno is set.
619 priv_flow_convert_actions(struct priv *priv,
620 const struct rte_flow_action actions[],
621 struct rte_flow_error *error,
622 struct mlx5_flow_parse *parser)
625 * Add default RSS configuration necessary for Verbs to create QP even
626 * if no RSS is necessary.
628 priv_flow_convert_rss_conf(priv, parser,
629 (const struct rte_eth_rss_conf *)
631 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
632 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
634 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
636 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
637 const struct rte_flow_action_queue *queue =
638 (const struct rte_flow_action_queue *)
643 if (!queue || (queue->index > (priv->rxqs_n - 1)))
644 goto exit_action_not_supported;
645 for (n = 0; n < parser->queues_n; ++n) {
646 if (parser->queues[n] == queue->index) {
651 if (parser->queues_n > 1 && !found) {
652 rte_flow_error_set(error, ENOTSUP,
653 RTE_FLOW_ERROR_TYPE_ACTION,
655 "queue action not in RSS queues");
659 parser->queues_n = 1;
660 parser->queues[0] = queue->index;
662 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
663 const struct rte_flow_action_rss *rss =
664 (const struct rte_flow_action_rss *)
668 if (!rss || !rss->num) {
669 rte_flow_error_set(error, EINVAL,
670 RTE_FLOW_ERROR_TYPE_ACTION,
675 if (parser->queues_n == 1) {
678 assert(parser->queues_n);
679 for (n = 0; n < rss->num; ++n) {
680 if (parser->queues[0] ==
687 rte_flow_error_set(error, ENOTSUP,
688 RTE_FLOW_ERROR_TYPE_ACTION,
690 "queue action not in RSS"
695 for (n = 0; n < rss->num; ++n) {
696 if (rss->queue[n] >= priv->rxqs_n) {
697 rte_flow_error_set(error, EINVAL,
698 RTE_FLOW_ERROR_TYPE_ACTION,
700 "queue id > number of"
705 for (n = 0; n < rss->num; ++n)
706 parser->queues[n] = rss->queue[n];
707 parser->queues_n = rss->num;
708 if (priv_flow_convert_rss_conf(priv, parser,
710 rte_flow_error_set(error, EINVAL,
711 RTE_FLOW_ERROR_TYPE_ACTION,
713 "wrong RSS configuration");
716 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
717 const struct rte_flow_action_mark *mark =
718 (const struct rte_flow_action_mark *)
722 rte_flow_error_set(error, EINVAL,
723 RTE_FLOW_ERROR_TYPE_ACTION,
725 "mark must be defined");
727 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
728 rte_flow_error_set(error, ENOTSUP,
729 RTE_FLOW_ERROR_TYPE_ACTION,
731 "mark must be between 0"
736 parser->mark_id = mark->id;
737 } else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
740 goto exit_action_not_supported;
743 if (!parser->queues_n && !parser->drop) {
744 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
745 NULL, "no valid action");
749 exit_action_not_supported:
750 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
751 actions, "action not supported");
759 * Pointer to private structure.
761 * Pattern specification (list terminated by the END pattern item).
763 * Perform verbose error reporting if not NULL.
764 * @param[in, out] parser
765 * Internal parser structure.
768 * 0 on success, a negative errno value otherwise and rte_errno is set.
771 priv_flow_convert_items_validate(struct priv *priv,
772 const struct rte_flow_item items[],
773 struct rte_flow_error *error,
774 struct mlx5_flow_parse *parser)
776 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
780 /* Initialise the offsets to start after verbs attribute. */
782 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
784 for (i = 0; i != hash_rxq_init_n; ++i)
785 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
787 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
788 const struct mlx5_flow_items *token = NULL;
792 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
796 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
798 if (cur_item->items[i] == items->type) {
799 token = &mlx5_flow_items[items->type];
804 goto exit_item_not_supported;
806 err = mlx5_flow_item_validate(items,
807 (const uint8_t *)cur_item->mask,
810 goto exit_item_not_supported;
811 if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
813 rte_flow_error_set(error, ENOTSUP,
814 RTE_FLOW_ERROR_TYPE_ITEM,
816 "cannot recognize multiple"
817 " VXLAN encapsulations");
823 parser->drop_q.offset += cur_item->dst_sz;
824 } else if (parser->queues_n == 1) {
825 parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
827 for (n = 0; n != hash_rxq_init_n; ++n)
828 parser->queue[n].offset += cur_item->dst_sz;
832 for (i = 0; i != hash_rxq_init_n; ++i)
833 parser->queue[i].offset +=
834 sizeof(struct ibv_flow_spec_action_tag);
837 exit_item_not_supported:
838 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
839 items, "item not supported");
844 * Allocate memory space to store verbs flow attributes.
847 * Pointer to private structure.
848 * @param[in] priority
851 * Amount of byte to allocate.
853 * Perform verbose error reporting if not NULL.
856 * A verbs flow attribute on success, NULL otherwise.
858 static struct ibv_flow_attr*
859 priv_flow_convert_allocate(struct priv *priv,
860 unsigned int priority,
862 struct rte_flow_error *error)
864 struct ibv_flow_attr *ibv_attr;
867 ibv_attr = rte_calloc(__func__, 1, size, 0);
869 rte_flow_error_set(error, ENOMEM,
870 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
872 "cannot allocate verbs spec attributes.");
875 ibv_attr->priority = priority;
880 * Finalise verbs flow attributes.
883 * Pointer to private structure.
884 * @param[in, out] parser
885 * Internal parser structure.
888 priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser)
890 const unsigned int ipv4 =
891 hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
892 const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
893 const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
894 const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
895 const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
896 const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
900 if (parser->layer == HASH_RXQ_ETH) {
904 * This layer becomes useless as the pattern define under
907 rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
908 parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
910 /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
911 for (i = ohmin; i != (ohmax + 1); ++i) {
912 if (!parser->queue[i].ibv_attr)
914 rte_free(parser->queue[i].ibv_attr);
915 parser->queue[i].ibv_attr = NULL;
917 /* Remove impossible flow according to the RSS configuration. */
918 if (hash_rxq_init[parser->layer].dpdk_rss_hf &
919 parser->rss_conf.rss_hf) {
920 /* Remove any other flow. */
921 for (i = hmin; i != (hmax + 1); ++i) {
922 if ((i == parser->layer) ||
923 (!parser->queue[i].ibv_attr))
925 rte_free(parser->queue[i].ibv_attr);
926 parser->queue[i].ibv_attr = NULL;
928 } else if (!parser->queue[ip].ibv_attr) {
929 /* no RSS possible with the current configuration. */
930 parser->queues_n = 1;
935 * Fill missing layers in verbs specifications, or compute the correct
936 * offset to allocate the memory space for the attributes and
939 for (i = 0; i != hash_rxq_init_n - 1; ++i) {
941 struct ibv_flow_spec_ipv4_ext ipv4;
942 struct ibv_flow_spec_ipv6 ipv6;
943 struct ibv_flow_spec_tcp_udp udp_tcp;
948 if (i == parser->layer)
950 if (parser->layer == HASH_RXQ_ETH) {
951 if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
952 size = sizeof(struct ibv_flow_spec_ipv4_ext);
953 specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
954 .type = IBV_FLOW_SPEC_IPV4_EXT |
959 size = sizeof(struct ibv_flow_spec_ipv6);
960 specs.ipv6 = (struct ibv_flow_spec_ipv6){
961 .type = IBV_FLOW_SPEC_IPV6 |
966 if (parser->queue[i].ibv_attr) {
967 dst = (void *)((uintptr_t)
968 parser->queue[i].ibv_attr +
969 parser->queue[i].offset);
970 memcpy(dst, &specs, size);
971 ++parser->queue[i].ibv_attr->num_of_specs;
973 parser->queue[i].offset += size;
975 if ((i == HASH_RXQ_UDPV4) || (i == HASH_RXQ_TCPV4) ||
976 (i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
977 size = sizeof(struct ibv_flow_spec_tcp_udp);
978 specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
979 .type = ((i == HASH_RXQ_UDPV4 ||
980 i == HASH_RXQ_UDPV6) ?
986 if (parser->queue[i].ibv_attr) {
987 dst = (void *)((uintptr_t)
988 parser->queue[i].ibv_attr +
989 parser->queue[i].offset);
990 memcpy(dst, &specs, size);
991 ++parser->queue[i].ibv_attr->num_of_specs;
993 parser->queue[i].offset += size;
999 * Validate and convert a flow supported by the NIC.
1002 * Pointer to private structure.
1004 * Flow rule attributes.
1005 * @param[in] pattern
1006 * Pattern specification (list terminated by the END pattern item).
1007 * @param[in] actions
1008 * Associated actions (list terminated by the END action).
1010 * Perform verbose error reporting if not NULL.
1011 * @param[in, out] parser
1012 * Internal parser structure.
1015 * 0 on success, a negative errno value otherwise and rte_errno is set.
1018 priv_flow_convert(struct priv *priv,
1019 const struct rte_flow_attr *attr,
1020 const struct rte_flow_item items[],
1021 const struct rte_flow_action actions[],
1022 struct rte_flow_error *error,
1023 struct mlx5_flow_parse *parser)
1025 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
1029 /* First step. Validate the attributes, items and actions. */
1030 *parser = (struct mlx5_flow_parse){
1031 .create = parser->create,
1032 .layer = HASH_RXQ_ETH,
1033 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1035 ret = priv_flow_convert_attributes(priv, attr, error, parser);
1038 ret = priv_flow_convert_actions(priv, actions, error, parser);
1041 ret = priv_flow_convert_items_validate(priv, items, error, parser);
1044 priv_flow_convert_finalise(priv, parser);
1047 * Allocate the memory space to store verbs specifications.
1050 parser->drop_q.ibv_attr =
1051 priv_flow_convert_allocate(priv, attr->priority,
1052 parser->drop_q.offset,
1054 if (!parser->drop_q.ibv_attr)
1056 parser->drop_q.offset = sizeof(struct ibv_flow_attr);
1057 } else if (parser->queues_n == 1) {
1058 unsigned int priority =
1060 hash_rxq_init[HASH_RXQ_ETH].flow_priority;
1061 unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
1063 parser->queue[HASH_RXQ_ETH].ibv_attr =
1064 priv_flow_convert_allocate(priv, priority,
1066 if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
1068 parser->queue[HASH_RXQ_ETH].offset =
1069 sizeof(struct ibv_flow_attr);
1071 for (i = 0; i != hash_rxq_init_n; ++i) {
1072 unsigned int priority =
1074 hash_rxq_init[i].flow_priority;
1075 unsigned int offset;
1077 if (!(parser->rss_conf.rss_hf &
1078 hash_rxq_init[i].dpdk_rss_hf) &&
1079 (i != HASH_RXQ_ETH))
1081 offset = parser->queue[i].offset;
1082 parser->queue[i].ibv_attr =
1083 priv_flow_convert_allocate(priv, priority,
1085 if (!parser->queue[i].ibv_attr)
1087 parser->queue[i].offset = sizeof(struct ibv_flow_attr);
1090 /* Third step. Conversion parse, fill the specifications. */
1092 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1093 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1095 cur_item = &mlx5_flow_items[items->type];
1096 ret = cur_item->convert(items,
1097 (cur_item->default_mask ?
1098 cur_item->default_mask :
1102 rte_flow_error_set(error, ENOTSUP,
1103 RTE_FLOW_ERROR_TYPE_ITEM,
1104 items, "item not supported");
1109 mlx5_flow_create_flag_mark(parser, parser->mark_id);
1111 * Last step. Complete missing specification to reach the RSS
1114 if (parser->queues_n > 1)
1115 priv_flow_convert_finalise(priv, parser);
1117 /* Only verification is expected, all resources should be released. */
1118 if (!parser->create) {
1120 rte_free(parser->drop_q.ibv_attr);
1121 parser->drop_q.ibv_attr = NULL;
1123 for (i = 0; i != hash_rxq_init_n; ++i) {
1124 if (parser->queue[i].ibv_attr) {
1125 rte_free(parser->queue[i].ibv_attr);
1126 parser->queue[i].ibv_attr = NULL;
1132 for (i = 0; i != hash_rxq_init_n; ++i) {
1133 if (parser->queue[i].ibv_attr) {
1134 rte_free(parser->queue[i].ibv_attr);
1135 parser->queue[i].ibv_attr = NULL;
1138 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1139 NULL, "cannot allocate verbs spec attributes.");
1144 * Copy the specification created into the flow.
1147 * Internal parser structure.
1149 * Create specification.
1151 * Size in bytes of the specification to copy.
1154 mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src,
1161 dst = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1162 parser->drop_q.offset);
1163 memcpy(dst, src, size);
1164 ++parser->drop_q.ibv_attr->num_of_specs;
1165 parser->drop_q.offset += size;
1168 for (i = 0; i != hash_rxq_init_n; ++i) {
1169 if (!parser->queue[i].ibv_attr)
1171 /* Specification must be the same l3 type or none. */
1172 if (parser->layer == HASH_RXQ_ETH ||
1173 (hash_rxq_init[parser->layer].ip_version ==
1174 hash_rxq_init[i].ip_version) ||
1175 (hash_rxq_init[i].ip_version == 0)) {
1176 dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1177 parser->queue[i].offset);
1178 memcpy(dst, src, size);
1179 ++parser->queue[i].ibv_attr->num_of_specs;
1180 parser->queue[i].offset += size;
1186 * Convert Ethernet item to Verbs specification.
1189 * Item specification.
1190 * @param default_mask[in]
1191 * Default bit-masks to use when item->mask is not provided.
1192 * @param data[in, out]
1196 mlx5_flow_create_eth(const struct rte_flow_item *item,
1197 const void *default_mask,
1200 const struct rte_flow_item_eth *spec = item->spec;
1201 const struct rte_flow_item_eth *mask = item->mask;
1202 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1203 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1204 struct ibv_flow_spec_eth eth = {
1205 .type = parser->inner | IBV_FLOW_SPEC_ETH,
1209 parser->layer = HASH_RXQ_ETH;
1214 mask = default_mask;
1215 memcpy(ð.val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
1216 memcpy(ð.val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
1217 eth.val.ether_type = spec->type;
1218 memcpy(ð.mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
1219 memcpy(ð.mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
1220 eth.mask.ether_type = mask->type;
1221 /* Remove unwanted bits from values. */
1222 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
1223 eth.val.dst_mac[i] &= eth.mask.dst_mac[i];
1224 eth.val.src_mac[i] &= eth.mask.src_mac[i];
1226 eth.val.ether_type &= eth.mask.ether_type;
1228 mlx5_flow_create_copy(parser, ð, eth_size);
1233 * Convert VLAN item to Verbs specification.
1236 * Item specification.
1237 * @param default_mask[in]
1238 * Default bit-masks to use when item->mask is not provided.
1239 * @param data[in, out]
1243 mlx5_flow_create_vlan(const struct rte_flow_item *item,
1244 const void *default_mask,
1247 const struct rte_flow_item_vlan *spec = item->spec;
1248 const struct rte_flow_item_vlan *mask = item->mask;
1249 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1250 struct ibv_flow_spec_eth *eth;
1251 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
1256 mask = default_mask;
1259 eth = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1260 parser->drop_q.offset - eth_size);
1261 eth->val.vlan_tag = spec->tci;
1262 eth->mask.vlan_tag = mask->tci;
1263 eth->val.vlan_tag &= eth->mask.vlan_tag;
1266 for (i = 0; i != hash_rxq_init_n; ++i) {
1267 if (!parser->queue[i].ibv_attr)
1270 eth = (void *)((uintptr_t)parser->queue[i].ibv_attr +
1271 parser->queue[i].offset - eth_size);
1272 eth->val.vlan_tag = spec->tci;
1273 eth->mask.vlan_tag = mask->tci;
1274 eth->val.vlan_tag &= eth->mask.vlan_tag;
1281 * Convert IPv4 item to Verbs specification.
1284 * Item specification.
1285 * @param default_mask[in]
1286 * Default bit-masks to use when item->mask is not provided.
1287 * @param data[in, out]
1291 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
1292 const void *default_mask,
1295 const struct rte_flow_item_ipv4 *spec = item->spec;
1296 const struct rte_flow_item_ipv4 *mask = item->mask;
1297 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1298 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
1299 struct ibv_flow_spec_ipv4_ext ipv4 = {
1300 .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
1304 parser->layer = HASH_RXQ_IPV4;
1307 mask = default_mask;
1308 ipv4.val = (struct ibv_flow_ipv4_ext_filter){
1309 .src_ip = spec->hdr.src_addr,
1310 .dst_ip = spec->hdr.dst_addr,
1311 .proto = spec->hdr.next_proto_id,
1312 .tos = spec->hdr.type_of_service,
1314 ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
1315 .src_ip = mask->hdr.src_addr,
1316 .dst_ip = mask->hdr.dst_addr,
1317 .proto = mask->hdr.next_proto_id,
1318 .tos = mask->hdr.type_of_service,
1320 /* Remove unwanted bits from values. */
1321 ipv4.val.src_ip &= ipv4.mask.src_ip;
1322 ipv4.val.dst_ip &= ipv4.mask.dst_ip;
1323 ipv4.val.proto &= ipv4.mask.proto;
1324 ipv4.val.tos &= ipv4.mask.tos;
1326 mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
1331 * Convert IPv6 item to Verbs specification.
1334 * Item specification.
1335 * @param default_mask[in]
1336 * Default bit-masks to use when item->mask is not provided.
1337 * @param data[in, out]
1341 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
1342 const void *default_mask,
1345 const struct rte_flow_item_ipv6 *spec = item->spec;
1346 const struct rte_flow_item_ipv6 *mask = item->mask;
1347 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1348 unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
1349 struct ibv_flow_spec_ipv6 ipv6 = {
1350 .type = parser->inner | IBV_FLOW_SPEC_IPV6,
1354 parser->layer = HASH_RXQ_IPV6;
1359 mask = default_mask;
1360 memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
1361 RTE_DIM(ipv6.val.src_ip));
1362 memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
1363 RTE_DIM(ipv6.val.dst_ip));
1364 memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
1365 RTE_DIM(ipv6.mask.src_ip));
1366 memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
1367 RTE_DIM(ipv6.mask.dst_ip));
1368 ipv6.mask.flow_label = mask->hdr.vtc_flow;
1369 ipv6.mask.next_hdr = mask->hdr.proto;
1370 ipv6.mask.hop_limit = mask->hdr.hop_limits;
1371 /* Remove unwanted bits from values. */
1372 for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
1373 ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
1374 ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
1376 ipv6.val.flow_label &= ipv6.mask.flow_label;
1377 ipv6.val.next_hdr &= ipv6.mask.next_hdr;
1378 ipv6.val.hop_limit &= ipv6.mask.hop_limit;
1380 mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
1385 * Convert UDP item to Verbs specification.
1388 * Item specification.
1389 * @param default_mask[in]
1390 * Default bit-masks to use when item->mask is not provided.
1391 * @param data[in, out]
1395 mlx5_flow_create_udp(const struct rte_flow_item *item,
1396 const void *default_mask,
1399 const struct rte_flow_item_udp *spec = item->spec;
1400 const struct rte_flow_item_udp *mask = item->mask;
1401 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1402 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1403 struct ibv_flow_spec_tcp_udp udp = {
1404 .type = parser->inner | IBV_FLOW_SPEC_UDP,
1408 if (parser->layer == HASH_RXQ_IPV4)
1409 parser->layer = HASH_RXQ_UDPV4;
1411 parser->layer = HASH_RXQ_UDPV6;
1414 mask = default_mask;
1415 udp.val.dst_port = spec->hdr.dst_port;
1416 udp.val.src_port = spec->hdr.src_port;
1417 udp.mask.dst_port = mask->hdr.dst_port;
1418 udp.mask.src_port = mask->hdr.src_port;
1419 /* Remove unwanted bits from values. */
1420 udp.val.src_port &= udp.mask.src_port;
1421 udp.val.dst_port &= udp.mask.dst_port;
1423 mlx5_flow_create_copy(parser, &udp, udp_size);
1428 * Convert TCP item to Verbs specification.
1431 * Item specification.
1432 * @param default_mask[in]
1433 * Default bit-masks to use when item->mask is not provided.
1434 * @param data[in, out]
1438 mlx5_flow_create_tcp(const struct rte_flow_item *item,
1439 const void *default_mask,
1442 const struct rte_flow_item_tcp *spec = item->spec;
1443 const struct rte_flow_item_tcp *mask = item->mask;
1444 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1445 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
1446 struct ibv_flow_spec_tcp_udp tcp = {
1447 .type = parser->inner | IBV_FLOW_SPEC_TCP,
1451 if (parser->layer == HASH_RXQ_IPV4)
1452 parser->layer = HASH_RXQ_TCPV4;
1454 parser->layer = HASH_RXQ_TCPV6;
1457 mask = default_mask;
1458 tcp.val.dst_port = spec->hdr.dst_port;
1459 tcp.val.src_port = spec->hdr.src_port;
1460 tcp.mask.dst_port = mask->hdr.dst_port;
1461 tcp.mask.src_port = mask->hdr.src_port;
1462 /* Remove unwanted bits from values. */
1463 tcp.val.src_port &= tcp.mask.src_port;
1464 tcp.val.dst_port &= tcp.mask.dst_port;
1466 mlx5_flow_create_copy(parser, &tcp, tcp_size);
1471 * Convert VXLAN item to Verbs specification.
1474 * Item specification.
1475 * @param default_mask[in]
1476 * Default bit-masks to use when item->mask is not provided.
1477 * @param data[in, out]
1481 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
1482 const void *default_mask,
1485 const struct rte_flow_item_vxlan *spec = item->spec;
1486 const struct rte_flow_item_vxlan *mask = item->mask;
1487 struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data;
1488 unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
1489 struct ibv_flow_spec_tunnel vxlan = {
1490 .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
1499 parser->inner = IBV_FLOW_SPEC_INNER;
1502 mask = default_mask;
1503 memcpy(&id.vni[1], spec->vni, 3);
1504 vxlan.val.tunnel_id = id.vlan_id;
1505 memcpy(&id.vni[1], mask->vni, 3);
1506 vxlan.mask.tunnel_id = id.vlan_id;
1507 /* Remove unwanted bits from values. */
1508 vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
1510 mlx5_flow_create_copy(parser, &vxlan, size);
1515 * Convert mark/flag action to Verbs specification.
1518 * Internal parser structure.
1523 mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
1525 unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
1526 struct ibv_flow_spec_action_tag tag = {
1527 .type = IBV_FLOW_SPEC_ACTION_TAG,
1529 .tag_id = mlx5_flow_mark_set(mark_id),
1532 assert(parser->mark);
1533 mlx5_flow_create_copy(parser, &tag, size);
1538 * Complete flow rule creation with a drop queue.
1541 * Pointer to private structure.
1543 * Internal parser structure.
1545 * Pointer to the rte_flow.
1547 * Perform verbose error reporting if not NULL.
1550 * 0 on success, errno value on failure.
1553 priv_flow_create_action_queue_drop(struct priv *priv,
1554 struct mlx5_flow_parse *parser,
1555 struct rte_flow *flow,
1556 struct rte_flow_error *error)
1558 struct ibv_flow_spec_action_drop *drop;
1559 unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
1565 drop = (void *)((uintptr_t)parser->drop_q.ibv_attr +
1566 parser->drop_q.offset);
1567 *drop = (struct ibv_flow_spec_action_drop){
1568 .type = IBV_FLOW_SPEC_ACTION_DROP,
1571 ++parser->drop_q.ibv_attr->num_of_specs;
1572 parser->drop_q.offset += size;
1573 if (!priv->dev->data->dev_started)
1575 flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
1576 parser->drop_q.ibv_attr = NULL;
1577 flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
1578 flow->drxq.ibv_attr);
1579 if (!flow->drxq.ibv_flow) {
1580 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1581 NULL, "flow rule creation failure");
1588 if (flow->drxq.ibv_flow) {
1589 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1590 flow->drxq.ibv_flow = NULL;
1592 if (flow->drxq.ibv_attr) {
1593 rte_free(flow->drxq.ibv_attr);
1594 flow->drxq.ibv_attr = NULL;
1600 * Create hash Rx queues when RSS is enabled.
1603 * Pointer to private structure.
1605 * Internal parser structure.
1607 * Pointer to the rte_flow.
1609 * Perform verbose error reporting if not NULL.
1612 * 0 on success, a errno value otherwise and rte_errno is set.
1615 priv_flow_create_action_queue_rss(struct priv *priv,
1616 struct mlx5_flow_parse *parser,
1617 struct rte_flow *flow,
1618 struct rte_flow_error *error)
1622 for (i = 0; i != hash_rxq_init_n; ++i) {
1623 uint64_t hash_fields;
1625 if (!parser->queue[i].ibv_attr)
1627 flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
1628 parser->queue[i].ibv_attr = NULL;
1629 hash_fields = hash_rxq_init[i].hash_fields;
1630 flow->frxq[i].hrxq =
1631 mlx5_priv_hrxq_get(priv,
1632 parser->rss_conf.rss_key,
1633 parser->rss_conf.rss_key_len,
1636 hash_fields ? parser->queues_n : 1);
1637 if (flow->frxq[i].hrxq)
1639 flow->frxq[i].hrxq =
1640 mlx5_priv_hrxq_new(priv,
1641 parser->rss_conf.rss_key,
1642 parser->rss_conf.rss_key_len,
1645 hash_fields ? parser->queues_n : 1);
1646 if (!flow->frxq[i].hrxq) {
1647 rte_flow_error_set(error, ENOMEM,
1648 RTE_FLOW_ERROR_TYPE_HANDLE,
1649 NULL, "cannot create hash rxq");
1657 * Complete flow rule creation.
1660 * Pointer to private structure.
1662 * Internal parser structure.
1664 * Pointer to the rte_flow.
1666 * Perform verbose error reporting if not NULL.
1669 * 0 on success, a errno value otherwise and rte_errno is set.
1672 priv_flow_create_action_queue(struct priv *priv,
1673 struct mlx5_flow_parse *parser,
1674 struct rte_flow *flow,
1675 struct rte_flow_error *error)
1682 assert(!parser->drop);
1683 err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
1686 if (!priv->dev->data->dev_started)
1688 for (i = 0; i != hash_rxq_init_n; ++i) {
1689 if (!flow->frxq[i].hrxq)
1691 flow->frxq[i].ibv_flow =
1692 ibv_create_flow(flow->frxq[i].hrxq->qp,
1693 flow->frxq[i].ibv_attr);
1694 if (!flow->frxq[i].ibv_flow) {
1695 rte_flow_error_set(error, ENOMEM,
1696 RTE_FLOW_ERROR_TYPE_HANDLE,
1697 NULL, "flow rule creation failure");
1701 DEBUG("%p type %d QP %p ibv_flow %p",
1703 (void *)flow->frxq[i].hrxq,
1704 (void *)flow->frxq[i].ibv_flow);
1706 for (i = 0; i != parser->queues_n; ++i) {
1707 struct mlx5_rxq_data *q =
1708 (*priv->rxqs)[parser->queues[i]];
1710 q->mark |= parser->mark;
1715 for (i = 0; i != hash_rxq_init_n; ++i) {
1716 if (flow->frxq[i].ibv_flow) {
1717 struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
1719 claim_zero(ibv_destroy_flow(ibv_flow));
1721 if (flow->frxq[i].hrxq)
1722 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
1723 if (flow->frxq[i].ibv_attr)
1724 rte_free(flow->frxq[i].ibv_attr);
1733 * Pointer to private structure.
1735 * Pointer to a TAILQ flow list.
1737 * Flow rule attributes.
1738 * @param[in] pattern
1739 * Pattern specification (list terminated by the END pattern item).
1740 * @param[in] actions
1741 * Associated actions (list terminated by the END action).
1743 * Perform verbose error reporting if not NULL.
1746 * A flow on success, NULL otherwise.
1748 static struct rte_flow *
1749 priv_flow_create(struct priv *priv,
1750 struct mlx5_flows *list,
1751 const struct rte_flow_attr *attr,
1752 const struct rte_flow_item items[],
1753 const struct rte_flow_action actions[],
1754 struct rte_flow_error *error)
1756 struct mlx5_flow_parse parser = { .create = 1, };
1757 struct rte_flow *flow = NULL;
1761 err = priv_flow_convert(priv, attr, items, actions, error, &parser);
1764 flow = rte_calloc(__func__, 1,
1765 sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
1768 rte_flow_error_set(error, ENOMEM,
1769 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1771 "cannot allocate flow memory");
1774 /* Copy queues configuration. */
1775 flow->queues = (uint16_t (*)[])(flow + 1);
1776 memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
1777 flow->queues_n = parser.queues_n;
1778 /* Copy RSS configuration. */
1779 flow->rss_conf = parser.rss_conf;
1780 flow->rss_conf.rss_key = flow->rss_key;
1781 memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
1782 /* finalise the flow. */
1784 err = priv_flow_create_action_queue_drop(priv, &parser, flow,
1787 err = priv_flow_create_action_queue(priv, &parser, flow, error);
1790 TAILQ_INSERT_TAIL(list, flow, next);
1791 DEBUG("Flow created %p", (void *)flow);
1795 rte_free(parser.drop_q.ibv_attr);
1797 for (i = 0; i != hash_rxq_init_n; ++i) {
1798 if (parser.queue[i].ibv_attr)
1799 rte_free(parser.queue[i].ibv_attr);
1807 * Validate a flow supported by the NIC.
1809 * @see rte_flow_validate()
1813 mlx5_flow_validate(struct rte_eth_dev *dev,
1814 const struct rte_flow_attr *attr,
1815 const struct rte_flow_item items[],
1816 const struct rte_flow_action actions[],
1817 struct rte_flow_error *error)
1819 struct priv *priv = dev->data->dev_private;
1821 struct mlx5_flow_parse parser = { .create = 0, };
1824 ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
1832 * @see rte_flow_create()
1836 mlx5_flow_create(struct rte_eth_dev *dev,
1837 const struct rte_flow_attr *attr,
1838 const struct rte_flow_item items[],
1839 const struct rte_flow_action actions[],
1840 struct rte_flow_error *error)
1842 struct priv *priv = dev->data->dev_private;
1843 struct rte_flow *flow;
1846 flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
1856 * Pointer to private structure.
1858 * Pointer to a TAILQ flow list.
1863 priv_flow_destroy(struct priv *priv,
1864 struct mlx5_flows *list,
1865 struct rte_flow *flow)
1869 if (flow->drop || !flow->mark)
1871 for (i = 0; i != flow->queues_n; ++i) {
1872 struct rte_flow *tmp;
1876 * To remove the mark from the queue, the queue must not be
1877 * present in any other marked flow (RSS or not).
1879 TAILQ_FOREACH(tmp, list, next) {
1881 uint16_t *tqs = NULL;
1886 for (j = 0; j != hash_rxq_init_n; ++j) {
1887 if (!tmp->frxq[j].hrxq)
1889 tqs = tmp->frxq[j].hrxq->ind_table->queues;
1890 tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
1894 for (j = 0; (j != tq_n) && !mark; j++)
1895 if (tqs[j] == (*flow->queues)[i])
1898 (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
1902 if (flow->drxq.ibv_flow)
1903 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
1904 rte_free(flow->drxq.ibv_attr);
1906 for (i = 0; i != hash_rxq_init_n; ++i) {
1907 struct mlx5_flow *frxq = &flow->frxq[i];
1910 claim_zero(ibv_destroy_flow(frxq->ibv_flow));
1912 mlx5_priv_hrxq_release(priv, frxq->hrxq);
1914 rte_free(frxq->ibv_attr);
1917 TAILQ_REMOVE(list, flow, next);
1918 DEBUG("Flow destroyed %p", (void *)flow);
1923 * Destroy all flows.
1926 * Pointer to private structure.
1928 * Pointer to a TAILQ flow list.
1931 priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
1933 while (!TAILQ_EMPTY(list)) {
1934 struct rte_flow *flow;
1936 flow = TAILQ_FIRST(list);
1937 priv_flow_destroy(priv, list, flow);
1942 * Create drop queue.
1945 * Pointer to private structure.
1951 priv_flow_create_drop_queue(struct priv *priv)
1953 struct mlx5_hrxq_drop *fdq = NULL;
1957 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
1959 WARN("cannot allocate memory for drop queue");
1962 fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
1964 WARN("cannot allocate CQ for drop queue");
1967 fdq->wq = ibv_create_wq(priv->ctx,
1968 &(struct ibv_wq_init_attr){
1969 .wq_type = IBV_WQT_RQ,
1976 WARN("cannot allocate WQ for drop queue");
1979 fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
1980 &(struct ibv_rwq_ind_table_init_attr){
1981 .log_ind_tbl_size = 0,
1982 .ind_tbl = &fdq->wq,
1985 if (!fdq->ind_table) {
1986 WARN("cannot allocate indirection table for drop queue");
1989 fdq->qp = ibv_create_qp_ex(priv->ctx,
1990 &(struct ibv_qp_init_attr_ex){
1991 .qp_type = IBV_QPT_RAW_PACKET,
1993 IBV_QP_INIT_ATTR_PD |
1994 IBV_QP_INIT_ATTR_IND_TABLE |
1995 IBV_QP_INIT_ATTR_RX_HASH,
1996 .rx_hash_conf = (struct ibv_rx_hash_conf){
1998 IBV_RX_HASH_FUNC_TOEPLITZ,
1999 .rx_hash_key_len = rss_hash_default_key_len,
2000 .rx_hash_key = rss_hash_default_key,
2001 .rx_hash_fields_mask = 0,
2003 .rwq_ind_tbl = fdq->ind_table,
2007 WARN("cannot allocate QP for drop queue");
2010 priv->flow_drop_queue = fdq;
2014 claim_zero(ibv_destroy_qp(fdq->qp));
2016 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2018 claim_zero(ibv_destroy_wq(fdq->wq));
2020 claim_zero(ibv_destroy_cq(fdq->cq));
2023 priv->flow_drop_queue = NULL;
2028 * Delete drop queue.
2031 * Pointer to private structure.
2034 priv_flow_delete_drop_queue(struct priv *priv)
2036 struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
2041 claim_zero(ibv_destroy_qp(fdq->qp));
2043 claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
2045 claim_zero(ibv_destroy_wq(fdq->wq));
2047 claim_zero(ibv_destroy_cq(fdq->cq));
2049 priv->flow_drop_queue = NULL;
2056 * Pointer to private structure.
2058 * Pointer to a TAILQ flow list.
2061 priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
2063 struct rte_flow *flow;
2065 TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
2069 if (!flow->drxq.ibv_flow)
2071 claim_zero(ibv_destroy_flow(flow->drxq.ibv_flow));
2072 flow->drxq.ibv_flow = NULL;
2077 struct mlx5_ind_table_ibv *ind_tbl = NULL;
2079 for (i = 0; i != hash_rxq_init_n; ++i) {
2080 if (!flow->frxq[i].hrxq)
2082 ind_tbl = flow->frxq[i].hrxq->ind_table;
2085 for (i = 0; i != ind_tbl->queues_n; ++i)
2086 (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
2088 for (i = 0; i != hash_rxq_init_n; ++i) {
2089 if (!flow->frxq[i].ibv_flow)
2091 claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow));
2092 flow->frxq[i].ibv_flow = NULL;
2093 mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
2094 flow->frxq[i].hrxq = NULL;
2096 DEBUG("Flow %p removed", (void *)flow);
2104 * Pointer to private structure.
2106 * Pointer to a TAILQ flow list.
2109 * 0 on success, a errno value otherwise and rte_errno is set.
2112 priv_flow_start(struct priv *priv, struct mlx5_flows *list)
2114 struct rte_flow *flow;
2116 TAILQ_FOREACH(flow, list, next) {
2120 flow->drxq.ibv_flow =
2121 ibv_create_flow(priv->flow_drop_queue->qp,
2122 flow->drxq.ibv_attr);
2123 if (!flow->drxq.ibv_flow) {
2124 DEBUG("Flow %p cannot be applied",
2129 DEBUG("Flow %p applied", (void *)flow);
2133 for (i = 0; i != hash_rxq_init_n; ++i) {
2134 if (!flow->frxq[i].ibv_attr)
2136 flow->frxq[i].hrxq =
2137 mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
2138 flow->rss_conf.rss_key_len,
2139 hash_rxq_init[i].hash_fields,
2142 if (flow->frxq[i].hrxq)
2144 flow->frxq[i].hrxq =
2145 mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
2146 flow->rss_conf.rss_key_len,
2147 hash_rxq_init[i].hash_fields,
2150 if (!flow->frxq[i].hrxq) {
2151 DEBUG("Flow %p cannot be applied",
2157 flow->frxq[i].ibv_flow =
2158 ibv_create_flow(flow->frxq[i].hrxq->qp,
2159 flow->frxq[i].ibv_attr);
2160 if (!flow->frxq[i].ibv_flow) {
2161 DEBUG("Flow %p cannot be applied",
2166 DEBUG("Flow %p applied", (void *)flow);
2170 for (i = 0; i != flow->queues_n; ++i)
2171 (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
2177 * Verify the flow list is empty
2180 * Pointer to private structure.
2182 * @return the number of flows not released.
2185 priv_flow_verify(struct priv *priv)
2187 struct rte_flow *flow;
2190 TAILQ_FOREACH(flow, &priv->flows, next) {
2191 DEBUG("%p: flow %p still referenced", (void *)priv,
2199 * Enable a control flow configured from the control plane.
2202 * Pointer to Ethernet device.
2204 * An Ethernet flow spec to apply.
2206 * An Ethernet flow mask to apply.
2208 * A VLAN flow spec to apply.
2210 * A VLAN flow mask to apply.
2216 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
2217 struct rte_flow_item_eth *eth_spec,
2218 struct rte_flow_item_eth *eth_mask,
2219 struct rte_flow_item_vlan *vlan_spec,
2220 struct rte_flow_item_vlan *vlan_mask)
2222 struct priv *priv = dev->data->dev_private;
2223 const struct rte_flow_attr attr = {
2225 .priority = MLX5_CTRL_FLOW_PRIORITY,
2227 struct rte_flow_item items[] = {
2229 .type = RTE_FLOW_ITEM_TYPE_ETH,
2235 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
2236 RTE_FLOW_ITEM_TYPE_END,
2242 .type = RTE_FLOW_ITEM_TYPE_END,
2245 struct rte_flow_action actions[] = {
2247 .type = RTE_FLOW_ACTION_TYPE_RSS,
2250 .type = RTE_FLOW_ACTION_TYPE_END,
2253 struct rte_flow *flow;
2254 struct rte_flow_error error;
2257 struct rte_flow_action_rss rss;
2259 const struct rte_eth_rss_conf *rss_conf;
2261 uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
2265 if (!priv->reta_idx_n)
2267 for (i = 0; i != priv->reta_idx_n; ++i)
2268 action_rss.local.queue[i] = (*priv->reta_idx)[i];
2269 action_rss.local.rss_conf = &priv->rss_conf;
2270 action_rss.local.num = priv->reta_idx_n;
2271 actions[0].conf = (const void *)&action_rss.rss;
2272 flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
2280 * Enable a flow control configured from the control plane.
2283 * Pointer to Ethernet device.
2285 * An Ethernet flow spec to apply.
2287 * An Ethernet flow mask to apply.
2293 mlx5_ctrl_flow(struct rte_eth_dev *dev,
2294 struct rte_flow_item_eth *eth_spec,
2295 struct rte_flow_item_eth *eth_mask)
2297 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
2303 * @see rte_flow_destroy()
2307 mlx5_flow_destroy(struct rte_eth_dev *dev,
2308 struct rte_flow *flow,
2309 struct rte_flow_error *error)
2311 struct priv *priv = dev->data->dev_private;
2315 priv_flow_destroy(priv, &priv->flows, flow);
2321 * Destroy all flows.
2323 * @see rte_flow_flush()
2327 mlx5_flow_flush(struct rte_eth_dev *dev,
2328 struct rte_flow_error *error)
2330 struct priv *priv = dev->data->dev_private;
2334 priv_flow_flush(priv, &priv->flows);
2342 * @see rte_flow_isolate()
2346 mlx5_flow_isolate(struct rte_eth_dev *dev,
2348 struct rte_flow_error *error)
2350 struct priv *priv = dev->data->dev_private;
2353 if (dev->data->dev_started) {
2354 rte_flow_error_set(error, EBUSY,
2355 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2357 "port must be stopped first");
2361 priv->isolated = !!enable;
2367 * Convert a flow director filter to a generic flow.
2370 * Private structure.
2371 * @param fdir_filter
2372 * Flow director filter to add.
2374 * Generic flow parameters structure.
2377 * 0 on success, errno value on error.
2380 priv_fdir_filter_convert(struct priv *priv,
2381 const struct rte_eth_fdir_filter *fdir_filter,
2382 struct mlx5_fdir *attributes)
2384 const struct rte_eth_fdir_input *input = &fdir_filter->input;
2386 /* Validate queue number. */
2387 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
2388 ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
2391 /* Validate the behavior. */
2392 if (fdir_filter->action.behavior != RTE_ETH_FDIR_ACCEPT) {
2393 ERROR("invalid behavior %d", fdir_filter->action.behavior);
2396 attributes->attr.ingress = 1;
2397 attributes->items[0] = (struct rte_flow_item) {
2398 .type = RTE_FLOW_ITEM_TYPE_ETH,
2399 .spec = &attributes->l2,
2401 attributes->actions[0] = (struct rte_flow_action){
2402 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
2403 .conf = &attributes->queue,
2405 attributes->queue.index = fdir_filter->action.rx_queue;
2406 switch (fdir_filter->input.flow_type) {
2407 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2408 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2409 .src_addr = input->flow.udp4_flow.ip.src_ip,
2410 .dst_addr = input->flow.udp4_flow.ip.dst_ip,
2411 .time_to_live = input->flow.udp4_flow.ip.ttl,
2412 .type_of_service = input->flow.udp4_flow.ip.tos,
2413 .next_proto_id = input->flow.udp4_flow.ip.proto,
2415 attributes->l4.udp.hdr = (struct udp_hdr){
2416 .src_port = input->flow.udp4_flow.src_port,
2417 .dst_port = input->flow.udp4_flow.dst_port,
2419 attributes->items[1] = (struct rte_flow_item){
2420 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2421 .spec = &attributes->l3,
2423 attributes->items[2] = (struct rte_flow_item){
2424 .type = RTE_FLOW_ITEM_TYPE_UDP,
2425 .spec = &attributes->l4,
2428 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2429 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2430 .src_addr = input->flow.tcp4_flow.ip.src_ip,
2431 .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
2432 .time_to_live = input->flow.tcp4_flow.ip.ttl,
2433 .type_of_service = input->flow.tcp4_flow.ip.tos,
2434 .next_proto_id = input->flow.tcp4_flow.ip.proto,
2436 attributes->l4.tcp.hdr = (struct tcp_hdr){
2437 .src_port = input->flow.tcp4_flow.src_port,
2438 .dst_port = input->flow.tcp4_flow.dst_port,
2440 attributes->items[1] = (struct rte_flow_item){
2441 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2442 .spec = &attributes->l3,
2444 attributes->items[2] = (struct rte_flow_item){
2445 .type = RTE_FLOW_ITEM_TYPE_TCP,
2446 .spec = &attributes->l4,
2449 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2450 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
2451 .src_addr = input->flow.ip4_flow.src_ip,
2452 .dst_addr = input->flow.ip4_flow.dst_ip,
2453 .time_to_live = input->flow.ip4_flow.ttl,
2454 .type_of_service = input->flow.ip4_flow.tos,
2455 .next_proto_id = input->flow.ip4_flow.proto,
2457 attributes->items[1] = (struct rte_flow_item){
2458 .type = RTE_FLOW_ITEM_TYPE_IPV4,
2459 .spec = &attributes->l3,
2462 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2463 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2464 .hop_limits = input->flow.udp6_flow.ip.hop_limits,
2465 .proto = input->flow.udp6_flow.ip.proto,
2467 memcpy(attributes->l3.ipv6.hdr.src_addr,
2468 input->flow.udp6_flow.ip.src_ip,
2469 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2470 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2471 input->flow.udp6_flow.ip.dst_ip,
2472 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2473 attributes->l4.udp.hdr = (struct udp_hdr){
2474 .src_port = input->flow.udp6_flow.src_port,
2475 .dst_port = input->flow.udp6_flow.dst_port,
2477 attributes->items[1] = (struct rte_flow_item){
2478 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2479 .spec = &attributes->l3,
2481 attributes->items[2] = (struct rte_flow_item){
2482 .type = RTE_FLOW_ITEM_TYPE_UDP,
2483 .spec = &attributes->l4,
2486 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2487 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2488 .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
2489 .proto = input->flow.tcp6_flow.ip.proto,
2491 memcpy(attributes->l3.ipv6.hdr.src_addr,
2492 input->flow.tcp6_flow.ip.src_ip,
2493 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2494 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2495 input->flow.tcp6_flow.ip.dst_ip,
2496 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2497 attributes->l4.tcp.hdr = (struct tcp_hdr){
2498 .src_port = input->flow.tcp6_flow.src_port,
2499 .dst_port = input->flow.tcp6_flow.dst_port,
2501 attributes->items[1] = (struct rte_flow_item){
2502 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2503 .spec = &attributes->l3,
2505 attributes->items[2] = (struct rte_flow_item){
2506 .type = RTE_FLOW_ITEM_TYPE_UDP,
2507 .spec = &attributes->l4,
2510 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2511 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
2512 .hop_limits = input->flow.ipv6_flow.hop_limits,
2513 .proto = input->flow.ipv6_flow.proto,
2515 memcpy(attributes->l3.ipv6.hdr.src_addr,
2516 input->flow.ipv6_flow.src_ip,
2517 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2518 memcpy(attributes->l3.ipv6.hdr.dst_addr,
2519 input->flow.ipv6_flow.dst_ip,
2520 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
2521 attributes->items[1] = (struct rte_flow_item){
2522 .type = RTE_FLOW_ITEM_TYPE_IPV6,
2523 .spec = &attributes->l3,
2527 ERROR("invalid flow type%d",
2528 fdir_filter->input.flow_type);
2535 * Add new flow director filter and store it in list.
2538 * Private structure.
2539 * @param fdir_filter
2540 * Flow director filter to add.
2543 * 0 on success, errno value on failure.
2546 priv_fdir_filter_add(struct priv *priv,
2547 const struct rte_eth_fdir_filter *fdir_filter)
2549 struct mlx5_fdir attributes = {
2552 struct mlx5_flow_parse parser = {
2553 .layer = HASH_RXQ_ETH,
2555 struct rte_flow_error error;
2556 struct rte_flow *flow;
2559 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2562 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2563 attributes.actions, &error, &parser);
2566 flow = priv_flow_create(priv,
2573 TAILQ_INSERT_TAIL(&priv->flows, flow, next);
2574 DEBUG("FDIR created %p", (void *)flow);
2581 * Delete specific filter.
2584 * Private structure.
2585 * @param fdir_filter
2586 * Filter to be deleted.
2589 * 0 on success, errno value on failure.
2592 priv_fdir_filter_delete(struct priv *priv,
2593 const struct rte_eth_fdir_filter *fdir_filter)
2595 struct mlx5_fdir attributes;
2596 struct mlx5_flow_parse parser = {
2598 .layer = HASH_RXQ_ETH,
2600 struct rte_flow_error error;
2601 struct rte_flow *flow;
2605 ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
2608 ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
2609 attributes.actions, &error, &parser);
2612 TAILQ_FOREACH(flow, &priv->flows, next) {
2613 struct ibv_flow_attr *attr;
2614 struct ibv_spec_header *attr_h;
2616 struct ibv_flow_attr *flow_attr;
2617 struct ibv_spec_header *flow_h;
2619 unsigned int specs_n;
2622 attr = parser.drop_q.ibv_attr;
2624 attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
2626 flow_attr = flow->drxq.ibv_attr;
2628 flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
2629 /* Compare first the attributes. */
2630 if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
2632 if (attr->num_of_specs == 0)
2634 spec = (void *)((uintptr_t)attr +
2635 sizeof(struct ibv_flow_attr));
2636 flow_spec = (void *)((uintptr_t)flow_attr +
2637 sizeof(struct ibv_flow_attr));
2638 specs_n = RTE_MIN(attr->num_of_specs, flow_attr->num_of_specs);
2639 for (i = 0; i != specs_n; ++i) {
2642 if (memcmp(spec, flow_spec,
2643 RTE_MIN(attr_h->size, flow_h->size)))
2645 spec = (void *)((uintptr_t)attr + attr_h->size);
2646 flow_spec = (void *)((uintptr_t)flow_attr +
2649 /* At this point, the flow match. */
2653 priv_flow_destroy(priv, &priv->flows, flow);
2656 rte_free(parser.drop_q.ibv_attr);
2658 for (i = 0; i != hash_rxq_init_n; ++i) {
2659 if (parser.queue[i].ibv_attr)
2660 rte_free(parser.queue[i].ibv_attr);
2667 * Update queue for specific filter.
2670 * Private structure.
2671 * @param fdir_filter
2672 * Filter to be updated.
2675 * 0 on success, errno value on failure.
2678 priv_fdir_filter_update(struct priv *priv,
2679 const struct rte_eth_fdir_filter *fdir_filter)
2683 ret = priv_fdir_filter_delete(priv, fdir_filter);
2686 ret = priv_fdir_filter_add(priv, fdir_filter);
2691 * Flush all filters.
2694 * Private structure.
2697 priv_fdir_filter_flush(struct priv *priv)
2699 priv_flow_flush(priv, &priv->flows);
2703 * Get flow director information.
2706 * Private structure.
2707 * @param[out] fdir_info
2708 * Resulting flow director information.
2711 priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
2713 struct rte_eth_fdir_masks *mask =
2714 &priv->dev->data->dev_conf.fdir_conf.mask;
2716 fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
2717 fdir_info->guarant_spc = 0;
2718 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
2719 fdir_info->max_flexpayload = 0;
2720 fdir_info->flow_types_mask[0] = 0;
2721 fdir_info->flex_payload_unit = 0;
2722 fdir_info->max_flex_payload_segment_num = 0;
2723 fdir_info->flex_payload_limit = 0;
2724 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
2728 * Deal with flow director operations.
2731 * Pointer to private structure.
2733 * Operation to perform.
2735 * Pointer to operation-specific structure.
2738 * 0 on success, errno value on failure.
2741 priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
2743 enum rte_fdir_mode fdir_mode =
2744 priv->dev->data->dev_conf.fdir_conf.mode;
2747 if (filter_op == RTE_ETH_FILTER_NOP)
2749 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
2750 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2751 ERROR("%p: flow director mode %d not supported",
2752 (void *)priv, fdir_mode);
2755 switch (filter_op) {
2756 case RTE_ETH_FILTER_ADD:
2757 ret = priv_fdir_filter_add(priv, arg);
2759 case RTE_ETH_FILTER_UPDATE:
2760 ret = priv_fdir_filter_update(priv, arg);
2762 case RTE_ETH_FILTER_DELETE:
2763 ret = priv_fdir_filter_delete(priv, arg);
2765 case RTE_ETH_FILTER_FLUSH:
2766 priv_fdir_filter_flush(priv);
2768 case RTE_ETH_FILTER_INFO:
2769 priv_fdir_info_get(priv, arg);
2772 DEBUG("%p: unknown operation %u", (void *)priv,
2781 * Manage filter operations.
2784 * Pointer to Ethernet device structure.
2785 * @param filter_type
2788 * Operation to perform.
2790 * Pointer to operation-specific structure.
2793 * 0 on success, negative errno value on failure.
2796 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
2797 enum rte_filter_type filter_type,
2798 enum rte_filter_op filter_op,
2802 struct priv *priv = dev->data->dev_private;
2804 switch (filter_type) {
2805 case RTE_ETH_FILTER_GENERIC:
2806 if (filter_op != RTE_ETH_FILTER_GET)
2808 *(const void **)arg = &mlx5_flow_ops;
2810 case RTE_ETH_FILTER_FDIR:
2812 ret = priv_fdir_ctrl_func(priv, filter_op, arg);
2816 ERROR("%p: filter type (%d) not supported",
2817 (void *)dev, filter_type);