4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
56 mlx5_flow_create_eth(const struct rte_flow_item *item,
57 const void *default_mask,
61 mlx5_flow_create_vlan(const struct rte_flow_item *item,
62 const void *default_mask,
66 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
67 const void *default_mask,
71 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
72 const void *default_mask,
76 mlx5_flow_create_udp(const struct rte_flow_item *item,
77 const void *default_mask,
81 mlx5_flow_create_tcp(const struct rte_flow_item *item,
82 const void *default_mask,
86 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
87 const void *default_mask,
91 LIST_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
92 struct ibv_exp_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
93 struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
94 struct ibv_qp *qp; /**< Verbs queue pair. */
95 struct ibv_exp_flow *ibv_flow; /**< Verbs flow. */
96 struct ibv_exp_wq *wq; /**< Verbs work queue. */
97 struct ibv_cq *cq; /**< Verbs completion queue. */
98 struct rxq *rxq; /**< Pointer to the queue, NULL if drop queue. */
99 uint32_t mark:1; /**< Set if the flow is marked. */
102 /** Static initializer for items. */
104 (const enum rte_flow_item_type []){ \
105 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
108 /** Structure to generate a simple graph of layers supported by the NIC. */
109 struct mlx5_flow_items {
110 /** List of possible actions for these items. */
111 const enum rte_flow_action_type *const actions;
112 /** Bit-masks corresponding to the possibilities for the item. */
115 * Default bit-masks to use when item->mask is not provided. When
116 * \default_mask is also NULL, the full supported bit-mask (\mask) is
119 const void *default_mask;
120 /** Bit-masks size in bytes. */
121 const unsigned int mask_sz;
123 * Conversion function from rte_flow to NIC specific flow.
126 * rte_flow item to convert.
127 * @param default_mask
128 * Default bit-masks to use when item->mask is not provided.
130 * Internal structure to store the conversion.
133 * 0 on success, negative value otherwise.
135 int (*convert)(const struct rte_flow_item *item,
136 const void *default_mask,
138 /** Size in bytes of the destination structure. */
139 const unsigned int dst_sz;
140 /** List of possible following items. */
141 const enum rte_flow_item_type *const items;
144 /** Valid action for this PMD. */
145 static const enum rte_flow_action_type valid_actions[] = {
146 RTE_FLOW_ACTION_TYPE_DROP,
147 RTE_FLOW_ACTION_TYPE_QUEUE,
148 RTE_FLOW_ACTION_TYPE_MARK,
149 RTE_FLOW_ACTION_TYPE_END,
152 /** Graph of supported items and associated actions. */
153 static const struct mlx5_flow_items mlx5_flow_items[] = {
154 [RTE_FLOW_ITEM_TYPE_END] = {
155 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
156 RTE_FLOW_ITEM_TYPE_VXLAN),
158 [RTE_FLOW_ITEM_TYPE_ETH] = {
159 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
160 RTE_FLOW_ITEM_TYPE_IPV4,
161 RTE_FLOW_ITEM_TYPE_IPV6),
162 .actions = valid_actions,
163 .mask = &(const struct rte_flow_item_eth){
164 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
165 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
168 .default_mask = &rte_flow_item_eth_mask,
169 .mask_sz = sizeof(struct rte_flow_item_eth),
170 .convert = mlx5_flow_create_eth,
171 .dst_sz = sizeof(struct ibv_exp_flow_spec_eth),
173 [RTE_FLOW_ITEM_TYPE_VLAN] = {
174 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
175 RTE_FLOW_ITEM_TYPE_IPV6),
176 .actions = valid_actions,
177 .mask = &(const struct rte_flow_item_vlan){
180 .default_mask = &rte_flow_item_vlan_mask,
181 .mask_sz = sizeof(struct rte_flow_item_vlan),
182 .convert = mlx5_flow_create_vlan,
185 [RTE_FLOW_ITEM_TYPE_IPV4] = {
186 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
187 RTE_FLOW_ITEM_TYPE_TCP),
188 .actions = valid_actions,
189 .mask = &(const struct rte_flow_item_ipv4){
193 .type_of_service = -1,
197 .default_mask = &rte_flow_item_ipv4_mask,
198 .mask_sz = sizeof(struct rte_flow_item_ipv4),
199 .convert = mlx5_flow_create_ipv4,
200 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv4_ext),
202 [RTE_FLOW_ITEM_TYPE_IPV6] = {
203 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
204 RTE_FLOW_ITEM_TYPE_TCP),
205 .actions = valid_actions,
206 .mask = &(const struct rte_flow_item_ipv6){
209 0xff, 0xff, 0xff, 0xff,
210 0xff, 0xff, 0xff, 0xff,
211 0xff, 0xff, 0xff, 0xff,
212 0xff, 0xff, 0xff, 0xff,
215 0xff, 0xff, 0xff, 0xff,
216 0xff, 0xff, 0xff, 0xff,
217 0xff, 0xff, 0xff, 0xff,
218 0xff, 0xff, 0xff, 0xff,
222 .default_mask = &rte_flow_item_ipv6_mask,
223 .mask_sz = sizeof(struct rte_flow_item_ipv6),
224 .convert = mlx5_flow_create_ipv6,
225 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv6),
227 [RTE_FLOW_ITEM_TYPE_UDP] = {
228 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
229 .actions = valid_actions,
230 .mask = &(const struct rte_flow_item_udp){
236 .default_mask = &rte_flow_item_udp_mask,
237 .mask_sz = sizeof(struct rte_flow_item_udp),
238 .convert = mlx5_flow_create_udp,
239 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
241 [RTE_FLOW_ITEM_TYPE_TCP] = {
242 .actions = valid_actions,
243 .mask = &(const struct rte_flow_item_tcp){
249 .default_mask = &rte_flow_item_tcp_mask,
250 .mask_sz = sizeof(struct rte_flow_item_tcp),
251 .convert = mlx5_flow_create_tcp,
252 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
254 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
255 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
256 .actions = valid_actions,
257 .mask = &(const struct rte_flow_item_vxlan){
258 .vni = "\xff\xff\xff",
260 .default_mask = &rte_flow_item_vxlan_mask,
261 .mask_sz = sizeof(struct rte_flow_item_vxlan),
262 .convert = mlx5_flow_create_vxlan,
263 .dst_sz = sizeof(struct ibv_exp_flow_spec_tunnel),
267 /** Structure to pass to the conversion function. */
269 struct ibv_exp_flow_attr *ibv_attr; /**< Verbs attribute. */
270 unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
271 uint32_t inner; /**< Set once VXLAN is encountered. */
274 struct mlx5_flow_action {
275 uint32_t queue:1; /**< Target is a receive queue. */
276 uint32_t drop:1; /**< Target is a drop queue. */
277 uint32_t mark:1; /**< Mark is present in the flow. */
278 uint32_t queue_id; /**< Identifier of the queue. */
279 uint32_t mark_id; /**< Mark identifier. */
283 * Check support for a given item.
286 * Item specification.
288 * Bit-masks covering supported fields to compare with spec, last and mask in
291 * Bit-Mask size in bytes.
297 mlx5_flow_item_validate(const struct rte_flow_item *item,
298 const uint8_t *mask, unsigned int size)
302 if (!item->spec && (item->mask || item->last))
304 if (item->spec && !item->mask) {
306 const uint8_t *spec = item->spec;
308 for (i = 0; i < size; ++i)
309 if ((spec[i] | mask[i]) != mask[i])
312 if (item->last && !item->mask) {
314 const uint8_t *spec = item->last;
316 for (i = 0; i < size; ++i)
317 if ((spec[i] | mask[i]) != mask[i])
322 const uint8_t *spec = item->mask;
324 for (i = 0; i < size; ++i)
325 if ((spec[i] | mask[i]) != mask[i])
328 if (item->spec && item->last) {
331 const uint8_t *apply = mask;
336 for (i = 0; i < size; ++i) {
337 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
338 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
340 ret = memcmp(spec, last, size);
346 * Validate a flow supported by the NIC.
349 * Pointer to private structure.
351 * Flow rule attributes.
353 * Pattern specification (list terminated by the END pattern item).
355 * Associated actions (list terminated by the END action).
357 * Perform verbose error reporting if not NULL.
358 * @param[in, out] flow
359 * Flow structure to update.
362 * 0 on success, a negative errno value otherwise and rte_errno is set.
365 priv_flow_validate(struct priv *priv,
366 const struct rte_flow_attr *attr,
367 const struct rte_flow_item items[],
368 const struct rte_flow_action actions[],
369 struct rte_flow_error *error,
370 struct mlx5_flow *flow)
372 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
373 struct mlx5_flow_action action = {
381 rte_flow_error_set(error, ENOTSUP,
382 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
384 "groups are not supported");
387 if (attr->priority) {
388 rte_flow_error_set(error, ENOTSUP,
389 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
391 "priorities are not supported");
395 rte_flow_error_set(error, ENOTSUP,
396 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
398 "egress is not supported");
401 if (!attr->ingress) {
402 rte_flow_error_set(error, ENOTSUP,
403 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
405 "only ingress is supported");
408 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
409 const struct mlx5_flow_items *token = NULL;
413 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
417 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
419 if (cur_item->items[i] == items->type) {
420 token = &mlx5_flow_items[items->type];
425 goto exit_item_not_supported;
427 err = mlx5_flow_item_validate(items,
428 (const uint8_t *)cur_item->mask,
431 goto exit_item_not_supported;
432 if (flow->ibv_attr && cur_item->convert) {
433 err = cur_item->convert(items,
434 (cur_item->default_mask ?
435 cur_item->default_mask :
439 goto exit_item_not_supported;
441 flow->offset += cur_item->dst_sz;
443 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
444 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
446 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
448 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
449 const struct rte_flow_action_queue *queue =
450 (const struct rte_flow_action_queue *)
453 if (!queue || (queue->index > (priv->rxqs_n - 1)))
454 goto exit_action_not_supported;
456 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
457 const struct rte_flow_action_mark *mark =
458 (const struct rte_flow_action_mark *)
462 rte_flow_error_set(error, EINVAL,
463 RTE_FLOW_ERROR_TYPE_ACTION,
465 "mark must be defined");
467 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
468 rte_flow_error_set(error, ENOTSUP,
469 RTE_FLOW_ERROR_TYPE_ACTION,
471 "mark must be between 0"
477 goto exit_action_not_supported;
480 if (action.mark && !flow->ibv_attr && !action.drop)
481 flow->offset += sizeof(struct ibv_exp_flow_spec_action_tag);
482 if (!action.queue && !action.drop) {
483 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
484 NULL, "no valid action");
488 exit_item_not_supported:
489 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
490 items, "item not supported");
492 exit_action_not_supported:
493 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
494 actions, "action not supported");
499 * Validate a flow supported by the NIC.
501 * @see rte_flow_validate()
505 mlx5_flow_validate(struct rte_eth_dev *dev,
506 const struct rte_flow_attr *attr,
507 const struct rte_flow_item items[],
508 const struct rte_flow_action actions[],
509 struct rte_flow_error *error)
511 struct priv *priv = dev->data->dev_private;
513 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr) };
516 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
522 * Convert Ethernet item to Verbs specification.
525 * Item specification.
526 * @param default_mask[in]
527 * Default bit-masks to use when item->mask is not provided.
528 * @param data[in, out]
532 mlx5_flow_create_eth(const struct rte_flow_item *item,
533 const void *default_mask,
536 const struct rte_flow_item_eth *spec = item->spec;
537 const struct rte_flow_item_eth *mask = item->mask;
538 struct mlx5_flow *flow = (struct mlx5_flow *)data;
539 struct ibv_exp_flow_spec_eth *eth;
540 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
543 ++flow->ibv_attr->num_of_specs;
544 flow->ibv_attr->priority = 2;
545 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
546 *eth = (struct ibv_exp_flow_spec_eth) {
547 .type = flow->inner | IBV_EXP_FLOW_SPEC_ETH,
554 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
555 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
556 eth->val.ether_type = spec->type;
557 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
558 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
559 eth->mask.ether_type = mask->type;
560 /* Remove unwanted bits from values. */
561 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
562 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
563 eth->val.src_mac[i] &= eth->mask.src_mac[i];
565 eth->val.ether_type &= eth->mask.ether_type;
570 * Convert VLAN item to Verbs specification.
573 * Item specification.
574 * @param default_mask[in]
575 * Default bit-masks to use when item->mask is not provided.
576 * @param data[in, out]
580 mlx5_flow_create_vlan(const struct rte_flow_item *item,
581 const void *default_mask,
584 const struct rte_flow_item_vlan *spec = item->spec;
585 const struct rte_flow_item_vlan *mask = item->mask;
586 struct mlx5_flow *flow = (struct mlx5_flow *)data;
587 struct ibv_exp_flow_spec_eth *eth;
588 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
590 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
595 eth->val.vlan_tag = spec->tci;
596 eth->mask.vlan_tag = mask->tci;
597 eth->val.vlan_tag &= eth->mask.vlan_tag;
602 * Convert IPv4 item to Verbs specification.
605 * Item specification.
606 * @param default_mask[in]
607 * Default bit-masks to use when item->mask is not provided.
608 * @param data[in, out]
612 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
613 const void *default_mask,
616 const struct rte_flow_item_ipv4 *spec = item->spec;
617 const struct rte_flow_item_ipv4 *mask = item->mask;
618 struct mlx5_flow *flow = (struct mlx5_flow *)data;
619 struct ibv_exp_flow_spec_ipv4_ext *ipv4;
620 unsigned int ipv4_size = sizeof(struct ibv_exp_flow_spec_ipv4_ext);
622 ++flow->ibv_attr->num_of_specs;
623 flow->ibv_attr->priority = 1;
624 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
625 *ipv4 = (struct ibv_exp_flow_spec_ipv4_ext) {
626 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV4_EXT,
633 ipv4->val = (struct ibv_exp_flow_ipv4_ext_filter){
634 .src_ip = spec->hdr.src_addr,
635 .dst_ip = spec->hdr.dst_addr,
636 .proto = spec->hdr.next_proto_id,
637 .tos = spec->hdr.type_of_service,
639 ipv4->mask = (struct ibv_exp_flow_ipv4_ext_filter){
640 .src_ip = mask->hdr.src_addr,
641 .dst_ip = mask->hdr.dst_addr,
642 .proto = mask->hdr.next_proto_id,
643 .tos = mask->hdr.type_of_service,
645 /* Remove unwanted bits from values. */
646 ipv4->val.src_ip &= ipv4->mask.src_ip;
647 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
648 ipv4->val.proto &= ipv4->mask.proto;
649 ipv4->val.tos &= ipv4->mask.tos;
654 * Convert IPv6 item to Verbs specification.
657 * Item specification.
658 * @param default_mask[in]
659 * Default bit-masks to use when item->mask is not provided.
660 * @param data[in, out]
664 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
665 const void *default_mask,
668 const struct rte_flow_item_ipv6 *spec = item->spec;
669 const struct rte_flow_item_ipv6 *mask = item->mask;
670 struct mlx5_flow *flow = (struct mlx5_flow *)data;
671 struct ibv_exp_flow_spec_ipv6 *ipv6;
672 unsigned int ipv6_size = sizeof(struct ibv_exp_flow_spec_ipv6);
675 ++flow->ibv_attr->num_of_specs;
676 flow->ibv_attr->priority = 1;
677 ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
678 *ipv6 = (struct ibv_exp_flow_spec_ipv6) {
679 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV6,
686 memcpy(ipv6->val.src_ip, spec->hdr.src_addr,
687 RTE_DIM(ipv6->val.src_ip));
688 memcpy(ipv6->val.dst_ip, spec->hdr.dst_addr,
689 RTE_DIM(ipv6->val.dst_ip));
690 memcpy(ipv6->mask.src_ip, mask->hdr.src_addr,
691 RTE_DIM(ipv6->mask.src_ip));
692 memcpy(ipv6->mask.dst_ip, mask->hdr.dst_addr,
693 RTE_DIM(ipv6->mask.dst_ip));
694 /* Remove unwanted bits from values. */
695 for (i = 0; i < RTE_DIM(ipv6->val.src_ip); ++i) {
696 ipv6->val.src_ip[i] &= ipv6->mask.src_ip[i];
697 ipv6->val.dst_ip[i] &= ipv6->mask.dst_ip[i];
703 * Convert UDP item to Verbs specification.
706 * Item specification.
707 * @param default_mask[in]
708 * Default bit-masks to use when item->mask is not provided.
709 * @param data[in, out]
713 mlx5_flow_create_udp(const struct rte_flow_item *item,
714 const void *default_mask,
717 const struct rte_flow_item_udp *spec = item->spec;
718 const struct rte_flow_item_udp *mask = item->mask;
719 struct mlx5_flow *flow = (struct mlx5_flow *)data;
720 struct ibv_exp_flow_spec_tcp_udp *udp;
721 unsigned int udp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
723 ++flow->ibv_attr->num_of_specs;
724 flow->ibv_attr->priority = 0;
725 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
726 *udp = (struct ibv_exp_flow_spec_tcp_udp) {
727 .type = flow->inner | IBV_EXP_FLOW_SPEC_UDP,
734 udp->val.dst_port = spec->hdr.dst_port;
735 udp->val.src_port = spec->hdr.src_port;
736 udp->mask.dst_port = mask->hdr.dst_port;
737 udp->mask.src_port = mask->hdr.src_port;
738 /* Remove unwanted bits from values. */
739 udp->val.src_port &= udp->mask.src_port;
740 udp->val.dst_port &= udp->mask.dst_port;
745 * Convert TCP item to Verbs specification.
748 * Item specification.
749 * @param default_mask[in]
750 * Default bit-masks to use when item->mask is not provided.
751 * @param data[in, out]
755 mlx5_flow_create_tcp(const struct rte_flow_item *item,
756 const void *default_mask,
759 const struct rte_flow_item_tcp *spec = item->spec;
760 const struct rte_flow_item_tcp *mask = item->mask;
761 struct mlx5_flow *flow = (struct mlx5_flow *)data;
762 struct ibv_exp_flow_spec_tcp_udp *tcp;
763 unsigned int tcp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
765 ++flow->ibv_attr->num_of_specs;
766 flow->ibv_attr->priority = 0;
767 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
768 *tcp = (struct ibv_exp_flow_spec_tcp_udp) {
769 .type = flow->inner | IBV_EXP_FLOW_SPEC_TCP,
776 tcp->val.dst_port = spec->hdr.dst_port;
777 tcp->val.src_port = spec->hdr.src_port;
778 tcp->mask.dst_port = mask->hdr.dst_port;
779 tcp->mask.src_port = mask->hdr.src_port;
780 /* Remove unwanted bits from values. */
781 tcp->val.src_port &= tcp->mask.src_port;
782 tcp->val.dst_port &= tcp->mask.dst_port;
787 * Convert VXLAN item to Verbs specification.
790 * Item specification.
791 * @param default_mask[in]
792 * Default bit-masks to use when item->mask is not provided.
793 * @param data[in, out]
797 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
798 const void *default_mask,
801 const struct rte_flow_item_vxlan *spec = item->spec;
802 const struct rte_flow_item_vxlan *mask = item->mask;
803 struct mlx5_flow *flow = (struct mlx5_flow *)data;
804 struct ibv_exp_flow_spec_tunnel *vxlan;
805 unsigned int size = sizeof(struct ibv_exp_flow_spec_tunnel);
811 ++flow->ibv_attr->num_of_specs;
812 flow->ibv_attr->priority = 0;
814 vxlan = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
815 *vxlan = (struct ibv_exp_flow_spec_tunnel) {
816 .type = flow->inner | IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL,
819 flow->inner = IBV_EXP_FLOW_SPEC_INNER;
824 memcpy(&id.vni[1], spec->vni, 3);
825 vxlan->val.tunnel_id = id.vlan_id;
826 memcpy(&id.vni[1], mask->vni, 3);
827 vxlan->mask.tunnel_id = id.vlan_id;
828 /* Remove unwanted bits from values. */
829 vxlan->val.tunnel_id &= vxlan->mask.tunnel_id;
834 * Convert mark/flag action to Verbs specification.
837 * Pointer to MLX5 flow structure.
842 mlx5_flow_create_flag_mark(struct mlx5_flow *flow, uint32_t mark_id)
844 struct ibv_exp_flow_spec_action_tag *tag;
845 unsigned int size = sizeof(struct ibv_exp_flow_spec_action_tag);
847 tag = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
848 *tag = (struct ibv_exp_flow_spec_action_tag){
849 .type = IBV_EXP_FLOW_SPEC_ACTION_TAG,
851 .tag_id = mlx5_flow_mark_set(mark_id),
853 ++flow->ibv_attr->num_of_specs;
858 * Complete flow rule creation.
861 * Pointer to private structure.
863 * Verbs flow attributes.
865 * Target action structure.
867 * Perform verbose error reporting if not NULL.
870 * A flow if the rule could be created.
872 static struct rte_flow *
873 priv_flow_create_action_queue(struct priv *priv,
874 struct ibv_exp_flow_attr *ibv_attr,
875 struct mlx5_flow_action *action,
876 struct rte_flow_error *error)
878 struct rxq_ctrl *rxq;
879 struct rte_flow *rte_flow;
883 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
885 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
886 NULL, "cannot allocate flow memory");
891 ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
892 &(struct ibv_exp_cq_init_attr){
896 rte_flow_error_set(error, ENOMEM,
897 RTE_FLOW_ERROR_TYPE_HANDLE,
898 NULL, "cannot allocate CQ");
901 rte_flow->wq = ibv_exp_create_wq(priv->ctx,
902 &(struct ibv_exp_wq_init_attr){
903 .wq_type = IBV_EXP_WQT_RQ,
910 rte_flow_error_set(error, ENOMEM,
911 RTE_FLOW_ERROR_TYPE_HANDLE,
912 NULL, "cannot allocate WQ");
916 rxq = container_of((*priv->rxqs)[action->queue_id],
917 struct rxq_ctrl, rxq);
918 rte_flow->rxq = &rxq->rxq;
919 rxq->rxq.mark |= action->mark;
920 rte_flow->wq = rxq->wq;
922 rte_flow->mark = action->mark;
923 rte_flow->ibv_attr = ibv_attr;
924 rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
926 &(struct ibv_exp_rwq_ind_table_init_attr){
928 .log_ind_tbl_size = 0,
929 .ind_tbl = &rte_flow->wq,
932 if (!rte_flow->ind_table) {
933 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
934 NULL, "cannot allocate indirection table");
937 rte_flow->qp = ibv_exp_create_qp(
939 &(struct ibv_exp_qp_init_attr){
940 .qp_type = IBV_QPT_RAW_PACKET,
942 IBV_EXP_QP_INIT_ATTR_PD |
943 IBV_EXP_QP_INIT_ATTR_PORT |
944 IBV_EXP_QP_INIT_ATTR_RX_HASH,
946 .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
948 IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
949 .rx_hash_key_len = rss_hash_default_key_len,
950 .rx_hash_key = rss_hash_default_key,
951 .rx_hash_fields_mask = 0,
952 .rwq_ind_tbl = rte_flow->ind_table,
954 .port_num = priv->port,
957 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
958 NULL, "cannot allocate QP");
963 rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
965 if (!rte_flow->ibv_flow) {
966 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
967 NULL, "flow rule creation failure");
974 ibv_destroy_qp(rte_flow->qp);
975 if (rte_flow->ind_table)
976 ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table);
977 if (!rte_flow->rxq && rte_flow->wq)
978 ibv_exp_destroy_wq(rte_flow->wq);
979 if (!rte_flow->rxq && rte_flow->cq)
980 ibv_destroy_cq(rte_flow->cq);
989 * Pointer to private structure.
991 * Flow rule attributes.
993 * Pattern specification (list terminated by the END pattern item).
995 * Associated actions (list terminated by the END action).
997 * Perform verbose error reporting if not NULL.
1000 * A flow on success, NULL otherwise.
1002 static struct rte_flow *
1003 priv_flow_create(struct priv *priv,
1004 const struct rte_flow_attr *attr,
1005 const struct rte_flow_item items[],
1006 const struct rte_flow_action actions[],
1007 struct rte_flow_error *error)
1009 struct rte_flow *rte_flow;
1010 struct mlx5_flow_action action;
1011 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr), };
1014 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
1017 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
1018 flow.offset = sizeof(struct ibv_exp_flow_attr);
1019 if (!flow.ibv_attr) {
1020 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1021 NULL, "cannot allocate ibv_attr memory");
1024 *flow.ibv_attr = (struct ibv_exp_flow_attr){
1025 .type = IBV_EXP_FLOW_ATTR_NORMAL,
1026 .size = sizeof(struct ibv_exp_flow_attr),
1027 .priority = attr->priority,
1034 claim_zero(priv_flow_validate(priv, attr, items, actions,
1036 action = (struct mlx5_flow_action){
1040 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1042 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1043 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1045 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1048 ((const struct rte_flow_action_queue *)
1049 actions->conf)->index;
1050 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1053 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
1054 const struct rte_flow_action_mark *mark =
1055 (const struct rte_flow_action_mark *)
1059 action.mark_id = mark->id;
1060 action.mark = !action.drop;
1062 rte_flow_error_set(error, ENOTSUP,
1063 RTE_FLOW_ERROR_TYPE_ACTION,
1064 actions, "unsupported action");
1069 mlx5_flow_create_flag_mark(&flow, action.mark_id);
1070 flow.offset += sizeof(struct ibv_exp_flow_spec_action_tag);
1072 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1078 rte_free(flow.ibv_attr);
1085 * @see rte_flow_create()
1089 mlx5_flow_create(struct rte_eth_dev *dev,
1090 const struct rte_flow_attr *attr,
1091 const struct rte_flow_item items[],
1092 const struct rte_flow_action actions[],
1093 struct rte_flow_error *error)
1095 struct priv *priv = dev->data->dev_private;
1096 struct rte_flow *flow;
1099 flow = priv_flow_create(priv, attr, items, actions, error);
1101 LIST_INSERT_HEAD(&priv->flows, flow, next);
1102 DEBUG("Flow created %p", (void *)flow);
1112 * Pointer to private structure.
1117 priv_flow_destroy(struct priv *priv,
1118 struct rte_flow *flow)
1121 LIST_REMOVE(flow, next);
1123 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1125 claim_zero(ibv_destroy_qp(flow->qp));
1126 if (flow->ind_table)
1127 claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table));
1128 if (!flow->rxq && flow->wq)
1129 claim_zero(ibv_exp_destroy_wq(flow->wq));
1130 if (!flow->rxq && flow->cq)
1131 claim_zero(ibv_destroy_cq(flow->cq));
1133 struct rte_flow *tmp;
1134 uint32_t mark_n = 0;
1136 for (tmp = LIST_FIRST(&priv->flows);
1138 tmp = LIST_NEXT(tmp, next)) {
1139 if ((flow->rxq == tmp->rxq) && tmp->mark)
1142 flow->rxq->mark = !!mark_n;
1144 rte_free(flow->ibv_attr);
1145 DEBUG("Flow destroyed %p", (void *)flow);
1152 * @see rte_flow_destroy()
1156 mlx5_flow_destroy(struct rte_eth_dev *dev,
1157 struct rte_flow *flow,
1158 struct rte_flow_error *error)
1160 struct priv *priv = dev->data->dev_private;
1164 priv_flow_destroy(priv, flow);
1170 * Destroy all flows.
1173 * Pointer to private structure.
1176 priv_flow_flush(struct priv *priv)
1178 while (!LIST_EMPTY(&priv->flows)) {
1179 struct rte_flow *flow;
1181 flow = LIST_FIRST(&priv->flows);
1182 priv_flow_destroy(priv, flow);
1187 * Destroy all flows.
1189 * @see rte_flow_flush()
1193 mlx5_flow_flush(struct rte_eth_dev *dev,
1194 struct rte_flow_error *error)
1196 struct priv *priv = dev->data->dev_private;
1200 priv_flow_flush(priv);
1208 * Called by dev_stop() to remove all flows.
1211 * Pointer to private structure.
1214 priv_flow_stop(struct priv *priv)
1216 struct rte_flow *flow;
1218 for (flow = LIST_FIRST(&priv->flows);
1220 flow = LIST_NEXT(flow, next)) {
1221 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1222 flow->ibv_flow = NULL;
1224 flow->rxq->mark = 0;
1225 DEBUG("Flow %p removed", (void *)flow);
1233 * Pointer to private structure.
1236 * 0 on success, a errno value otherwise and rte_errno is set.
1239 priv_flow_start(struct priv *priv)
1241 struct rte_flow *flow;
1243 for (flow = LIST_FIRST(&priv->flows);
1245 flow = LIST_NEXT(flow, next)) {
1246 flow->ibv_flow = ibv_exp_create_flow(flow->qp,
1248 if (!flow->ibv_flow) {
1249 DEBUG("Flow %p cannot be applied", (void *)flow);
1253 DEBUG("Flow %p applied", (void *)flow);
1255 flow->rxq->mark |= flow->mark;